diff --git a/results/ndif_feasibility/modal_verify_branches.json b/results/ndif_feasibility/modal_verify_branches.json new file mode 100644 index 0000000..397a2e1 --- /dev/null +++ b/results/ndif_feasibility/modal_verify_branches.json @@ -0,0 +1,390 @@ +{ + "started_utc": "2026-08-26T03:50:00.244016+00:00", + "branches": { + "main": { + "checkout": { + "returncode": 0, + "tail": [ + "Your branch is up to date with 'origin/main'.", + "Already on 'main'" + ] + }, + "sha": "87f4dad", + "nnsight_tests": { + "returncode": 0, + "tail": [ + ".................. [100%]", + "18 passed in 2.78s" + ] + }, + "repro_tests": { + "returncode": 1, + "tail": [ + "test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_name_has_no_dot", + " /root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.bert.encoder.layer.2.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "", + "test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_name_has_no_dot", + " /root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.bert.encoder.layer.2` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "", + "test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_name_has_no_dot", + " /root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.bert.encoder.layer.3.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "", + "test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_name_has_no_dot", + " /root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.bert.encoder.layer.3` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "", + "test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_name_has_no_dot", + " /root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.bert.encoder.layer.4.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "", + "test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_name_has_no_dot", + " /root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.bert.encoder.layer.4` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "", + "-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html", + "=========================== short test summary info ============================", + "FAILED test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_name_has_no_dot", + "FAILED test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_is_resolvable_in_its_module", + "FAILED test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_is_picklable", + "3 failed, 3 passed, 2 skipped, 11 warnings in 14.97s" + ] + }, + "rna_check": { + "returncode": 0, + "tail": [ + "pooler.dense.weight | MISSING | ", + "pooler.dense.bias | MISSING | ", + "", + "Notes:", + "- UNEXPECTED:\tcan be ignored when loading from different task/architecture; not ok if you expect identical arch.", + "- MISSING:\tthose params were newly initialized because missing from the checkpoint. Consider training on your downstream task.", + "/root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.encoder.layer.0.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.encoder.layer.0` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.encoder.layer.1.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.encoder.layer.1` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.encoder.layer.2.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.encoder.layer.2` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.encoder.layer.3.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.encoder.layer.3` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.encoder.layer.4.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.encoder.layer.4` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.encoder.layer.5.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:765: UserWarning: Module `model.encoder.layer.5` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(" + ] + }, + "rna": { + "nnsight_version": "0.7.1.dev43+g87f4dadaa", + "models": { + "multimolecule/rnabert": { + "envoy_class": "Envoy.Preserved", + "dot_in_name": true, + "resolves_back": false, + "siblings_share_class": false, + "n_distinct_layer_classes": 6, + "class_picklable": false, + "pickle_error": "PicklingError: Can't pickle : attribute lookup Envoy.Preserved on nnsight.intervention.envoy failed", + "n_layers": 7, + "layer_shape": [ + 1, + 20, + 120 + ], + "per_position_l2_shift": [ + 0.2864, + 0.2852, + 0.2604, + 0.2768, + 0.2177, + 0.146, + 0.2401, + 0.2269, + 0.314, + 0.0797, + 0.8089, + 0.2187, + 0.1411, + 0.1418, + 0.2738, + 0.2021, + 0.1798, + 0.1504, + 0.1409, + 0.5706 + ], + "argmax_position": 10, + "traced_shape": [ + 20, + 120 + ], + "status": "ok" + }, + "multimolecule/rnafm": { + "status": "error", + "error": "ValueError: vocab_size (28) must be 26 when codon=False.", + "traceback": "Traceback (most recent call last):\n File \"/root/repro/_rna_envoy_check.py\", line 85, in main\n entry = check(repo, cls_name)\n ^^^^^^^^^^^^^^^^^^^^^\n File \"/root/repro/_rna_envoy_check.py\", line 33, in check\n hf = getattr(multimolecule, cls_name).from_pretrained(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.11/site-packages/transformers/modeling_utils.py\", line 4217, in from_pretrained\n config, model_kwargs = config_class.from_pretrained(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.11/site-packages/transformers/configuration_utils.py\", line 691, in from_pretrained\n return cls.from_dict(config_dict, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.11/site-packages/transformers/configuration_utils.py\", line 868, in from_dict\n config = cls(**config_dict)\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.11/site-packages/multimolecule/models/rnafm/configuration_rnafm.py\", line 140, in __init__\n raise ValueError(f\"vocab_size ({vocab_size}) must be {expected_vocab_size} when codon={codon}.\")\nValueError: vocab_size (28) must be 26 when codon=False.\n" + }, + "multimolecule/splicebert": { + "envoy_class": "Envoy.Preserved", + "dot_in_name": true, + "resolves_back": false, + "siblings_share_class": false, + "n_distinct_layer_classes": 6, + "class_picklable": false, + "pickle_error": "PicklingError: Can't pickle : attribute lookup Envoy.Preserved on nnsight.intervention.envoy failed", + "n_layers": 7, + "layer_shape": [ + 1, + 20, + 512 + ], + "per_position_l2_shift": [ + 1.9588, + 2.0604, + 1.8931, + 1.7785, + 1.478, + 1.6852, + 1.7163, + 1.7309, + 3.355, + 2.4945, + 8.5443, + 2.3225, + 2.1158, + 1.7993, + 1.6969, + 1.4358, + 1.8137, + 1.991, + 2.0442, + 2.115 + ], + "argmax_position": 10, + "traced_shape": [ + 20, + 512 + ], + "status": "ok" + } + } + } + }, + "verify/both-fixes": { + "checkout": { + "returncode": 0, + "tail": [ + "Switched to branch 'verify/both-fixes'" + ] + }, + "sha": "e9ae3ef", + "nnsight_tests": { + "returncode": 0, + "tail": [ + ".................. [100%]", + "18 passed in 2.25s" + ] + }, + "repro_tests": { + "returncode": 0, + "tail": [ + "test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_name_has_no_dot", + " /root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.bert.encoder.layer.1` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "", + "test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_name_has_no_dot", + " /root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.bert.encoder.layer.2.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "", + "test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_name_has_no_dot", + " /root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.bert.encoder.layer.2` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "", + "test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_name_has_no_dot", + " /root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.bert.encoder.layer.3.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "", + "test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_name_has_no_dot", + " /root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.bert.encoder.layer.3` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "", + "test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_name_has_no_dot", + " /root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.bert.encoder.layer.4.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "", + "test_envoy_overloaded_mount.py::test_encoder_layer_envoy_class_name_has_no_dot", + " /root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.bert.encoder.layer.4` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "", + "-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html", + "6 passed, 2 skipped, 11 warnings in 13.05s" + ] + }, + "rna_check": { + "returncode": 0, + "tail": [ + "pooler.dense.weight | MISSING | ", + "pooler.dense.bias | MISSING | ", + "", + "Notes:", + "- UNEXPECTED:\tcan be ignored when loading from different task/architecture; not ok if you expect identical arch.", + "- MISSING:\tthose params were newly initialized because missing from the checkpoint. Consider training on your downstream task.", + "/root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.encoder.layer.0.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.encoder.layer.0` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.encoder.layer.1.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.encoder.layer.1` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.encoder.layer.2.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.encoder.layer.2` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.encoder.layer.3.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.encoder.layer.3` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.encoder.layer.4.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.encoder.layer.4` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.encoder.layer.5.attention` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(", + "/root/nnsight/src/nnsight/intervention/envoy.py:819: UserWarning: Module `model.encoder.layer.5` of type `` has pre-defined a `output` attribute. nnsight access for `output` will be mounted at `.nns_output` instead of `.output` for this module only.", + " warnings.warn(" + ] + }, + "rna": { + "nnsight_version": "0.7.1.dev43+g87f4dadaa", + "models": { + "multimolecule/rnabert": { + "envoy_class": "Envoy__nns_output", + "dot_in_name": false, + "resolves_back": true, + "siblings_share_class": true, + "n_distinct_layer_classes": 1, + "class_picklable": true, + "n_layers": 7, + "layer_shape": [ + 1, + 20, + 120 + ], + "per_position_l2_shift": [ + 0.2864, + 0.2852, + 0.2604, + 0.2768, + 0.2177, + 0.146, + 0.2401, + 0.2269, + 0.314, + 0.0797, + 0.8089, + 0.2187, + 0.1411, + 0.1418, + 0.2738, + 0.2021, + 0.1798, + 0.1504, + 0.1409, + 0.5706 + ], + "argmax_position": 10, + "traced_shape": [ + 20, + 120 + ], + "status": "ok" + }, + "multimolecule/rnafm": { + "status": "error", + "error": "ValueError: vocab_size (28) must be 26 when codon=False.", + "traceback": "Traceback (most recent call last):\n File \"/root/repro/_rna_envoy_check.py\", line 85, in main\n entry = check(repo, cls_name)\n ^^^^^^^^^^^^^^^^^^^^^\n File \"/root/repro/_rna_envoy_check.py\", line 33, in check\n hf = getattr(multimolecule, cls_name).from_pretrained(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.11/site-packages/transformers/modeling_utils.py\", line 4217, in from_pretrained\n config, model_kwargs = config_class.from_pretrained(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.11/site-packages/transformers/configuration_utils.py\", line 691, in from_pretrained\n return cls.from_dict(config_dict, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.11/site-packages/transformers/configuration_utils.py\", line 868, in from_dict\n config = cls(**config_dict)\n ^^^^^^^^^^^^^^^^^^\n File \"/usr/local/lib/python3.11/site-packages/multimolecule/models/rnafm/configuration_rnafm.py\", line 140, in __init__\n raise ValueError(f\"vocab_size ({vocab_size}) must be {expected_vocab_size} when codon={codon}.\")\nValueError: vocab_size (28) must be 26 when codon=False.\n" + }, + "multimolecule/splicebert": { + "envoy_class": "Envoy__nns_output", + "dot_in_name": false, + "resolves_back": true, + "siblings_share_class": true, + "n_distinct_layer_classes": 1, + "class_picklable": true, + "n_layers": 7, + "layer_shape": [ + 1, + 20, + 512 + ], + "per_position_l2_shift": [ + 1.9588, + 2.0604, + 1.8931, + 1.7785, + 1.478, + 1.6852, + 1.7163, + 1.7309, + 3.355, + 2.4945, + 8.5443, + 2.3225, + 2.1158, + 1.7993, + 1.6969, + 1.4358, + 1.8137, + 1.991, + 2.0442, + 2.115 + ], + "argmax_position": 10, + "traced_shape": [ + 20, + 512 + ], + "status": "ok" + } + } + } + } + }, + "install": { + "returncode": 0, + "tail": [ + "Obtaining file:///root/nnsight", + " Checking if build backend supports build_editable: started", + " Checking if build backend supports build_editable: finished with status 'done'", + " Preparing editable metadata (pyproject.toml): started", + " Preparing editable metadata (pyproject.toml): finished with status 'done'", + "Building wheels for collected packages: nnsight", + " Building editable for nnsight (pyproject.toml): started", + " Building editable for nnsight (pyproject.toml): finished with status 'done'", + " Created wheel for nnsight: filename=nnsight-0.7.1.dev43+g87f4dadaa-0.editable-cp311-cp311-linux_x86_64.whl size=12124 sha256=0f01c82ce9a2d7691de62cf7fdb5a5efaf92e4a59f6576a5eef33f8e2c319e18", + " Stored in directory: /tmp/pip-ephem-wheel-cache-uu69mv7p/wheels/aa/0e/ba/8f241461f65893ba6222deffea7c60b83f9756440fd694298d", + "Successfully built nnsight", + "Installing collected packages: nnsight", + "Successfully installed nnsight-0.7.1.dev43+g87f4dadaa", + "", + "[notice] A new release of pip is available: 25.2 -> 26.2.1", + "[notice] To update, run: pip install --upgrade pip" + ] + }, + "finished_utc": "2026-08-26T03:51:28.870395+00:00" +} \ No newline at end of file diff --git a/results/ndif_feasibility/probe_20260826T032720Z.json b/results/ndif_feasibility/probe_20260826T032720Z.json new file mode 100644 index 0000000..c635de0 --- /dev/null +++ b/results/ndif_feasibility/probe_20260826T032720Z.json @@ -0,0 +1,187 @@ +{ + "timestamp_utc": "20260826T032720Z", + "client": { + "nnsight": "0.7.0", + "ndif_host": "https://api.ndif.us" + }, + "probes": { + "t1_hot_catalogued": { + "spec": { + "name": "t1_hot_catalogued", + "repo_id": "meta-llama/Llama-3.2-1B-Instruct", + "inputs": [ + "The capital of France is" + ], + "hook": "m.model.layers[8].output[0]", + "why": "remote execution works at all on a HOT catalogued model", + "automodel": null, + "shim": false, + "trace_kwargs": {}, + "expect": "ok" + }, + "status": "ok", + "result": { + "model_key_sent": "nnsight.modeling.language.LanguageModel:{\"repo_id\": \"meta-llama/Llama-3.2-1B-Instruct\", \"revision\": null}", + "shape": [ + 6, + 2048 + ], + "dtype": "torch.bfloat16" + }, + "matches_expectation": true + }, + "t2_cold_catalogued": { + "spec": { + "name": "t2_cold_catalogued", + "repo_id": "openai-community/gpt2-medium", + "inputs": [ + "The capital of France is" + ], + "hook": "m.transformer.h[6].output[0]", + "why": "COLD catalogued model cold-starts on demand", + "automodel": null, + "shim": false, + "trace_kwargs": {}, + "expect": "ok" + }, + "status": "ok", + "result": { + "model_key_sent": "nnsight.modeling.language.LanguageModel:{\"repo_id\": \"openai-community/gpt2-medium\", \"revision\": null}", + "shape": [ + 5, + 1024 + ], + "dtype": "torch.bfloat16" + }, + "matches_expectation": true + }, + "t3_uncatalogued_decoder": { + "spec": { + "name": "t3_uncatalogued_decoder", + "repo_id": "EleutherAI/pythia-160m", + "inputs": [ + "The capital of France is" + ], + "hook": "m.gpt_neox.layers[6].output[0]", + "why": "model absent from GET /status is provisioned on request, no shim", + "automodel": null, + "shim": false, + "trace_kwargs": {}, + "expect": "ok" + }, + "status": "ok", + "result": { + "model_key_sent": "nnsight.modeling.language.LanguageModel:{\"repo_id\": \"EleutherAI/pythia-160m\", \"revision\": null}", + "shape": [ + 5, + 768 + ], + "dtype": "torch.bfloat16" + }, + "matches_expectation": true + }, + "t4_masked_lm_stock_path": { + "spec": { + "name": "t4_masked_lm_stock_path", + "repo_id": "facebook/esm2_t12_35M_UR50D", + "inputs": [ + "MKTVRQERLKSIVRILERSKEPVSGAQ" + ], + "hook": "m.esm.encoder.layer[6].nns_output[0]", + "why": "masked LM through the stock path; automodel is dropped on the wire", + "automodel": "AutoModelForMaskedLM", + "shim": false, + "trace_kwargs": {}, + "expect": "error" + }, + "status": "error", + "error_type": "NNsightException", + "error": "Traceback (most recent call last):\n\nRemoteException: Failed to provision model: Unrecognized configuration class for this kind of AutoModel: AutoModelForCausalLM.\nModel type should be one of GPT2Config, AfmoeConfig, ApertusConfig, ArceeConfig, AriaTextConfig, BambaConfig, BartConfig, BertConfig, BertGenerationConfig, BigBirdConfig, BigBirdPegasusConfig, BioGptConfig, BitNetConfig, BlenderbotConfig, BlenderbotSmallConfig, BloomConfig, BltConfig, CamembertConfig, CodeGenConfig, CohereConfig, Cohere2Config, CpmAntConfig, CTRLConfig, CwmConfig, Data2VecTextConfig, DbrxConfig, DeepseekV2Config, DeepseekV3Config, DeepseekV4Config, DiffLlamaConfig, DogeConfig, Dots1Config, ElectraConfig, Emu3Config, ErnieConfig, Ernie4_5Config, Ernie4_5_MoeConfig, Exaone4Config, ExaoneMoeConfig, FalconConfig, FalconH1Config, FalconMambaConfig, FlexOlmoConfig, FuyuConfig, GemmaConfig, Gemma2Config, Gemma3Config, Gemma3TextConfig, Gemma3nConfig, Gemma3nTextConfig, Gemma4Config, Gemma4AssistantConfig, Gemma4TextConfig, GitConfig, GlmConfig, Glm4Config, Glm4MoeConfig, Glm4MoeLiteConfig, GlmMoeDsaConfig, GotOcr2Config, GPT2Config, GPTBigCodeConfig, GPTNeoConfig, GPTNeoXConfig, GPTNeoXJapaneseConfig, GptOssConfig, GPTJConfig, GraniteConfig, GraniteMoeConfig, GraniteMoeHybridConfig, GraniteMoeSharedConfig, HeliumConfig, HunYuanDenseV1Config, HunYuanMoEV1Config, HYV3Config, Jais2Config, JambaConfig, JetMoeConfig, LagunaConfig, Lfm2Config, Lfm2MoeConfig, LlamaConfig, Llama4Config, Llama4TextConfig, LongcatFlashConfig, MambaConfig, Mamba2Config, MarianConfig, MBartConfig, MegatronBertConfig, MiniMaxConfig, MiniMaxM2Config, MinistralConfig, Ministral3Config, MistralConfig, MixtralConfig, MllamaConfig, ModernBertDecoderConfig, MoshiConfig, MptConfig, MusicgenConfig, MusicgenMelodyConfig, MvpConfig, NanoChatConfig, NemotronConfig, NemotronHConfig, OlmoConfig, Olmo2Config, Olmo3Config, OlmoHybridConfig, OlmoeConfig, OpenAIGPTConfig, OPTConfig, PegasusConfig, PersimmonConfig, PhiConfig, Phi3Config, Phi4MultimodalConfig, PhimoeConfig, PLBartConfig, ProphetNetConfig, Qwen2Config, Qwen2MoeConfig, Qwen3Config, Qwen3_5Config, Qwen3_5MoeConfig, Qwen3_5MoeTextConfig, Qwen3_5TextConfig, Qwen3MoeConfig, Qwen3NextConfig, RecurrentGemmaConfig, ReformerConfig, RemBertConfig, RobertaConfig, RobertaPreLayerNormConfig, RoCBertConfig, RoFormerConfig, RwkvConfig, SeedOssConfig, SmolLM3Config, SolarOpenConfig, StableLmConfig, Starcoder2Config, TrOCRConfig, VaultGemmaConfig, WhisperConfig, XGLMConfig, XLMConfig, XLMRobertaConfig, XLMRobertaXLConfig, XLNetConfig, xLSTMConfig, XmodConfig, YoutuConfig, ZambaConfig, Zamba2Config.\nTraceback (most recent call last):\n File \"/u/svcndifuser/ndif-deployment/repos/prod/ndif/src/ndif/services/ray/deployments/controller/cluster/evaluator.py\", line 57, in __call__\n meta_model = RemoteableMixin.from_model_key(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/mixins/remoteable.py\", line 164, in from_model_key\n return type._remoteable_from_model_key(model_key, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/huggingface.py\", line 150, in _remoteable_from_model_key\n return cls(repo_id, revision=revision, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/language.py\", line 78, in __init__\n super().__init__(*args, automodel=automodel, **kwargs)\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/transformers.py\", line 49, in __init__\n super().__init__(*args, **kwargs)\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/huggingface.py\", line 53, in __init__\n super().__init__(repo_id, *args, revision=revision, **kwargs)\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/mixins/meta.py\", line 59, in __init__\n model = self._load_meta(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/language.py\", line 133, in _load_meta\n model = super()._load_meta(repo_id, revision=revision, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/transformers.py\", line 68, in _load_meta\n model = self.automodel.from_config(self.config, trust_remote_code=True)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/transformers/models/auto/auto_factory.py\", line 253, in from_config\n raise ValueError(\nValueError: Unrecognized configuration class for this kind of AutoModel: AutoModelForCausalLM.\nModel type should be one of GPT2Config, AfmoeConfig, ApertusConfig, ArceeConfig, AriaTextConfig, BambaConfig, BartConfig, BertConfig, BertGenerationConfig, BigBirdConfig, BigBirdPegasusConfig, BioGptConfig, BitNetConfig, BlenderbotConfig, BlenderbotSmallConfig, BloomConfig, BltConfig, CamembertConfig, CodeGenConfig, CohereConfig, Cohere2Config, CpmAntConfig, CTRLConfig, CwmConfig, Data2VecTextConfig, DbrxConfig, DeepseekV2Config, DeepseekV3Config, DeepseekV4Config, DiffLlamaConfig, DogeConfig, Dots1Config, ElectraConfig, Emu3Config, ErnieConfig, Ernie4_5Config, Ernie4_5_MoeConfig, Exaone4Config, ExaoneMoeConfig, FalconConfig, FalconH1Config, FalconMambaConfig, FlexOlmoConfig, FuyuConfig, GemmaConfig, Gemma2Config, Gemma3Config, Gemma3TextConfig, Gemma3nConfig, Gemma3nTextConfig, Gemma4Config, Gemma4AssistantConfig, Gemma4TextConfig, GitConfig, GlmConfig, Glm4Config, Glm4MoeConfig, Glm4MoeLiteConfig, GlmMoeDsaConfig, GotOcr2Config, GPT2Config, GPTBigCodeConfig, GPTNeoConfig, GPTNeoXConfig, GPTNeoXJapaneseConfig, GptOssConfig, GPTJConfig, GraniteConfig, GraniteMoeConfig, GraniteMoeHybridConfig, GraniteMoeSharedConfig, HeliumConfig, HunYuanDenseV1Config, HunYuanMoEV1Config, HYV3Config, Jais2Config, JambaConfig, JetMoeConfig, LagunaConfig, Lfm2Config, Lfm2MoeConfig, LlamaConfig, Llama4Config, Llama4TextConfig, LongcatFlashConfig, MambaConfig, Mamba2Config, MarianConfig, MBartConfig, MegatronBertConfig, MiniMaxConfig, MiniMaxM2Config, MinistralConfig, Ministral3Config, MistralConfig, MixtralConfig, MllamaConfig, ModernBertDecoderConfig, MoshiConfig, MptConfig, MusicgenConfig, MusicgenMelodyConfig, MvpConfig, NanoChatConfig, NemotronConfig, NemotronHConfig, OlmoConfig, Olmo2Config, Olmo3Config, OlmoHybridConfig, OlmoeConfig, OpenAIGPTConfig, OPTConfig, PegasusConfig, PersimmonConfig, PhiConfig, Phi3Config, Phi4MultimodalConfig, PhimoeConfig, PLBartConfig, ProphetNetConfig, Qwen2Config, Qwen2MoeConfig, Qwen3Config, Qwen3_5Config, Qwen3_5MoeConfig, Qwen3_5MoeTextConfig, Qwen3_5TextConfig, Qwen3MoeConfig, Qwen3NextConfig, RecurrentGemmaConfig, ReformerConfig, RemBertConfig, RobertaConfig, RobertaPreLayerNormConfig, RoCBertConfig, RoFormerConfig, RwkvConfig, SeedOssConfig, SmolLM3Config, SolarOpenConfig, StableLmConfig, Starcoder2Config, TrOCRConfig, VaultGemmaConfig, WhisperConfig, XGLMConfig, XLMConfig, XLMRobertaConfig, XLMRobertaXLConfig, XLNetConfig, xLSTMConfig, XmodConfig, YoutuConfig, ZambaConfig, Zamba2Config.\n", + "traceback": "Traceback (most recent call last):\n File \"/Users/elliottower/Documents/GitHub/rna-sa-public/scripts/ndif_feasibility/probe_ndif.py\", line 286, in main\n entry[\"result\"] = run_probe(probe)\n ^^^^^^^^^^^^^^^^\n File \"/Users/elliottower/Documents/GitHub/rna-sa-public/scripts/ndif_feasibility/probe_ndif.py\", line 237, in run_probe\n with model.trace(text, remote=True, **probe.trace_kwargs):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/elliottower/Documents/GitHub/lookback-validity-audit/.venv/lib/python3.12/site-packages/nnsight/intervention/tracing/tracer.py\", line 381, in __exit__\n result = super().__exit__(exc_type, exc_value, traceback)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/elliottower/Documents/GitHub/lookback-validity-audit/.venv/lib/python3.12/site-packages/nnsight/intervention/tracing/base.py\", line 721, in __exit__\n self.backend(self)\n File \"/Users/elliottower/Documents/GitHub/lookback-validity-audit/.venv/lib/python3.12/site-packages/nnsight/intervention/backends/remote.py\", line 572, in __call__\n raise wrap_exception(e, None) from None\nnnsight.NNsightException: Traceback (most recent call last):\n\nRemoteException: Failed to provision model: Unrecognized configuration class for this kind of AutoModel: AutoModelForCausalLM.\nModel type should be one of GPT2Config, AfmoeConfig, ApertusConfig, ArceeConfig, AriaTextConfig, BambaConfig, BartConfig, BertConfig, BertGenerationConfig, BigBirdConfig, BigBirdPegasusConfig, BioGptConfig, BitNetConfig, BlenderbotConfig, BlenderbotSmallConfig, BloomConfig, BltConfig, CamembertConfig, CodeGenConfig, CohereConfig, Cohere2Config, CpmAntConfig, CTRLConfig, CwmConfig, Data2VecTextConfig, DbrxConfig, DeepseekV2Config, DeepseekV3Config, DeepseekV4Config, DiffLlamaConfig, DogeConfig, Dots1Config, ElectraConfig, Emu3Config, ErnieConfig, Ernie4_5Config, Ernie4_5_MoeConfig, Exaone4Config, ExaoneMoeConfig, FalconConfig, FalconH1Config, FalconMambaConfig, FlexOlmoConfig, FuyuConfig, GemmaConfig, Gemma2Config, Gemma3Config, Gemma3TextConfig, Gemma3nConfig, Gemma3nTextConfig, Gemma4Config, Gemma4AssistantConfig, Gemma4TextConfig, GitConfig, GlmConfig, Glm4Config, Glm4MoeConfig, Glm4MoeLiteConfig, GlmMoeDsaConfig, GotOcr2Config, GPT2Config, GPTBigCodeConfig, GPTNeoConfig, GPTNeoXConfig, GPTNeoXJapaneseConfig, GptOssConfig, GPTJConfig, GraniteConfig, GraniteMoeConfig, GraniteMoeHybridConfig, GraniteMoeSharedConfig, HeliumConfig, HunYuanDenseV1Config, HunYuanMoEV1Config, HYV3Config, Jais2Config, JambaConfig, JetMoeConfig, LagunaConfig, Lfm2Config, Lfm2MoeConfig, LlamaConfig, Llama4Config, Llama4TextConfig, LongcatFlashConfig, MambaConfig, Mamba2Config, MarianConfig, MBartConfig, MegatronBertConfig, MiniMaxConfig, MiniMaxM2Config, MinistralConfig, Ministral3Config, MistralConfig, MixtralConfig, MllamaConfig, ModernBertDecoderConfig, MoshiConfig, MptConfig, MusicgenConfig, MusicgenMelodyConfig, MvpConfig, NanoChatConfig, NemotronConfig, NemotronHConfig, OlmoConfig, Olmo2Config, Olmo3Config, OlmoHybridConfig, OlmoeConfig, OpenAIGPTConfig, OPTConfig, PegasusConfig, PersimmonConfig, PhiConfig, Phi3Config, Phi4MultimodalConfig, PhimoeConfig, PLBartConfig, ProphetNetConfig, Qwen2Config, Qwen2MoeConfig, Qwen3Config, Qwen3_5Config, Qwen3_5MoeConfig, Qwen3_5MoeTextConfig, Qwen3_5TextConfig, Qwen3MoeConfig, Qwen3NextConfig, RecurrentGemmaConfig, ReformerConfig, RemBertConfig, RobertaConfig, RobertaPreLayerNormConfig, RoCBertConfig, RoFormerConfig, RwkvConfig, SeedOssConfig, SmolLM3Config, SolarOpenConfig, StableLmConfig, Starcoder2Config, TrOCRConfig, VaultGemmaConfig, WhisperConfig, XGLMConfig, XLMConfig, XLMRobertaConfig, XLMRobertaXLConfig, XLNetConfig, xLSTMConfig, XmodConfig, YoutuConfig, ZambaConfig, Zamba2Config.\nTraceback (most recent call last):\n File \"/u/svcndifuser/ndif-deployment/repos/prod/ndif/src/ndif/services/ray/deployments/controller/cluster/evaluator.py\", line 57, in __call__\n meta_model = RemoteableMixin.from_model_key(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/mixins/remoteable.py\", line 164, in from_model_key\n return type._remoteable_from_model_key(model_key, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/huggingface.py\", line 150, in _remoteable_from_model_key\n return cls(repo_id, revision=revision, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/language.py\", line 78, in __init__\n super().__init__(*args, automodel=automodel, **kwargs)\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/transformers.py\", line 49, in __init__\n super().__init__(*args, **kwargs)\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/huggingface.py\", line 53, in __init__\n super().__init__(repo_id, *args, revision=revision, **kwargs)\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/mixins/meta.py\", line 59, in __init__\n model = self._load_meta(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/language.py\", line 133, in _load_meta\n model = super()._load_meta(repo_id, revision=revision, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/nnsight/modeling/transformers.py\", line 68, in _load_meta\n model = self.automodel.from_config(self.config, trust_remote_code=True)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/u/svcndifuser/miniconda3/envs/service/lib/python3.12/site-packages/transformers/models/auto/auto_factory.py\", line 253, in from_config\n raise ValueError(\nValueError: Unrecognized configuration class for this kind of AutoModel: AutoModelForCausalLM.\nModel type should be one of GPT2Config, AfmoeConfig, ApertusConfig, ArceeConfig, AriaTextConfig, BambaConfig, BartConfig, BertConfig, BertGenerationConfig, BigBirdConfig, BigBirdPegasusConfig, BioGptConfig, BitNetConfig, BlenderbotConfig, BlenderbotSmallConfig, BloomConfig, BltConfig, CamembertConfig, CodeGenConfig, CohereConfig, Cohere2Config, CpmAntConfig, CTRLConfig, CwmConfig, Data2VecTextConfig, DbrxConfig, DeepseekV2Config, DeepseekV3Config, DeepseekV4Config, DiffLlamaConfig, DogeConfig, Dots1Config, ElectraConfig, Emu3Config, ErnieConfig, Ernie4_5Config, Ernie4_5_MoeConfig, Exaone4Config, ExaoneMoeConfig, FalconConfig, FalconH1Config, FalconMambaConfig, FlexOlmoConfig, FuyuConfig, GemmaConfig, Gemma2Config, Gemma3Config, Gemma3TextConfig, Gemma3nConfig, Gemma3nTextConfig, Gemma4Config, Gemma4AssistantConfig, Gemma4TextConfig, GitConfig, GlmConfig, Glm4Config, Glm4MoeConfig, Glm4MoeLiteConfig, GlmMoeDsaConfig, GotOcr2Config, GPT2Config, GPTBigCodeConfig, GPTNeoConfig, GPTNeoXConfig, GPTNeoXJapaneseConfig, GptOssConfig, GPTJConfig, GraniteConfig, GraniteMoeConfig, GraniteMoeHybridConfig, GraniteMoeSharedConfig, HeliumConfig, HunYuanDenseV1Config, HunYuanMoEV1Config, HYV3Config, Jais2Config, JambaConfig, JetMoeConfig, LagunaConfig, Lfm2Config, Lfm2MoeConfig, LlamaConfig, Llama4Config, Llama4TextConfig, LongcatFlashConfig, MambaConfig, Mamba2Config, MarianConfig, MBartConfig, MegatronBertConfig, MiniMaxConfig, MiniMaxM2Config, MinistralConfig, Ministral3Config, MistralConfig, MixtralConfig, MllamaConfig, ModernBertDecoderConfig, MoshiConfig, MptConfig, MusicgenConfig, MusicgenMelodyConfig, MvpConfig, NanoChatConfig, NemotronConfig, NemotronHConfig, OlmoConfig, Olmo2Config, Olmo3Config, OlmoHybridConfig, OlmoeConfig, OpenAIGPTConfig, OPTConfig, PegasusConfig, PersimmonConfig, PhiConfig, Phi3Config, Phi4MultimodalConfig, PhimoeConfig, PLBartConfig, ProphetNetConfig, Qwen2Config, Qwen2MoeConfig, Qwen3Config, Qwen3_5Config, Qwen3_5MoeConfig, Qwen3_5MoeTextConfig, Qwen3_5TextConfig, Qwen3MoeConfig, Qwen3NextConfig, RecurrentGemmaConfig, ReformerConfig, RemBertConfig, RobertaConfig, RobertaPreLayerNormConfig, RoCBertConfig, RoFormerConfig, RwkvConfig, SeedOssConfig, SmolLM3Config, SolarOpenConfig, StableLmConfig, Starcoder2Config, TrOCRConfig, VaultGemmaConfig, WhisperConfig, XGLMConfig, XLMConfig, XLMRobertaConfig, XLMRobertaXLConfig, XLNetConfig, xLSTMConfig, XmodConfig, YoutuConfig, ZambaConfig, Zamba2Config.\n\n", + "matches_expectation": true + }, + "t4a_masked_lm_shimmed": { + "spec": { + "name": "t4a_masked_lm_shimmed", + "repo_id": "facebook/esm2_t12_35M_UR50D", + "inputs": [ + "MKTVRQERLKSIVRILERSKEPVSGAQ" + ], + "hook": "m.esm.encoder.layer[6].nns_output[0]", + "why": "shim fixes provisioning; does graph execution follow?", + "automodel": "AutoModelForMaskedLM", + "shim": true, + "trace_kwargs": {}, + "expect": "error" + }, + "status": "error", + "error_type": "NNsightException", + "error": "Traceback (most recent call last):\n\nRemoteException: name 'hooked_output' is not defined", + "traceback": "Traceback (most recent call last):\n File \"/Users/elliottower/Documents/GitHub/rna-sa-public/scripts/ndif_feasibility/probe_ndif.py\", line 286, in main\n entry[\"result\"] = run_probe(probe)\n ^^^^^^^^^^^^^^^^\n File \"/Users/elliottower/Documents/GitHub/rna-sa-public/scripts/ndif_feasibility/probe_ndif.py\", line 237, in run_probe\n with model.trace(text, remote=True, **probe.trace_kwargs):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/elliottower/Documents/GitHub/lookback-validity-audit/.venv/lib/python3.12/site-packages/nnsight/intervention/tracing/tracer.py\", line 381, in __exit__\n result = super().__exit__(exc_type, exc_value, traceback)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/elliottower/Documents/GitHub/lookback-validity-audit/.venv/lib/python3.12/site-packages/nnsight/intervention/tracing/base.py\", line 721, in __exit__\n self.backend(self)\n File \"/Users/elliottower/Documents/GitHub/lookback-validity-audit/.venv/lib/python3.12/site-packages/nnsight/intervention/backends/remote.py\", line 572, in __call__\n raise wrap_exception(e, None) from None\nnnsight.NNsightException: Traceback (most recent call last):\n\nRemoteException: name 'hooked_output' is not defined\n", + "matches_expectation": true + }, + "t4b_second_encoder_family": { + "spec": { + "name": "t4b_second_encoder_family", + "repo_id": "google-bert/bert-base-uncased", + "inputs": [ + "The capital of France is [MASK]." + ], + "hook": "m.bert.encoder.layer[6].nns_output[0]", + "why": "unrelated encoder family, to show the failure is not ESM-specific", + "automodel": "AutoModelForMaskedLM", + "shim": true, + "trace_kwargs": {}, + "expect": "error" + }, + "status": "error", + "error_type": "NNsightException", + "error": "Traceback (most recent call last):\n\nRemoteException: name 'hooked_output' is not defined", + "traceback": "Traceback (most recent call last):\n File \"/Users/elliottower/Documents/GitHub/rna-sa-public/scripts/ndif_feasibility/probe_ndif.py\", line 286, in main\n entry[\"result\"] = run_probe(probe)\n ^^^^^^^^^^^^^^^^\n File \"/Users/elliottower/Documents/GitHub/rna-sa-public/scripts/ndif_feasibility/probe_ndif.py\", line 237, in run_probe\n with model.trace(text, remote=True, **probe.trace_kwargs):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/elliottower/Documents/GitHub/lookback-validity-audit/.venv/lib/python3.12/site-packages/nnsight/intervention/tracing/tracer.py\", line 381, in __exit__\n result = super().__exit__(exc_type, exc_value, traceback)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/elliottower/Documents/GitHub/lookback-validity-audit/.venv/lib/python3.12/site-packages/nnsight/intervention/tracing/base.py\", line 721, in __exit__\n self.backend(self)\n File \"/Users/elliottower/Documents/GitHub/lookback-validity-audit/.venv/lib/python3.12/site-packages/nnsight/intervention/backends/remote.py\", line 572, in __call__\n raise wrap_exception(e, None) from None\nnnsight.NNsightException: Traceback (most recent call last):\n\nRemoteException: name 'hooked_output' is not defined\n", + "matches_expectation": true + }, + "t5_bert_stock_path_control": { + "spec": { + "name": "t5_bert_stock_path_control", + "repo_id": "google-bert/bert-base-uncased", + "inputs": [ + "The capital of France is [MASK]." + ], + "hook": "m.bert.encoder.layer[6].nns_output[0]", + "why": "CONTROL: BERT registers as BertLMHeadModel so it needs no shim. If this fails too, the shim is exonerated and encoders are the fault.", + "automodel": null, + "shim": false, + "trace_kwargs": {}, + "expect": "error" + }, + "status": "error", + "error_type": "NNsightException", + "error": "Traceback (most recent call last):\n\nRemoteException: name 'hooked_output' is not defined", + "traceback": "Traceback (most recent call last):\n File \"/Users/elliottower/Documents/GitHub/rna-sa-public/scripts/ndif_feasibility/probe_ndif.py\", line 286, in main\n entry[\"result\"] = run_probe(probe)\n ^^^^^^^^^^^^^^^^\n File \"/Users/elliottower/Documents/GitHub/rna-sa-public/scripts/ndif_feasibility/probe_ndif.py\", line 237, in run_probe\n with model.trace(text, remote=True, **probe.trace_kwargs):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/elliottower/Documents/GitHub/lookback-validity-audit/.venv/lib/python3.12/site-packages/nnsight/intervention/tracing/tracer.py\", line 381, in __exit__\n result = super().__exit__(exc_type, exc_value, traceback)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/elliottower/Documents/GitHub/lookback-validity-audit/.venv/lib/python3.12/site-packages/nnsight/intervention/tracing/base.py\", line 721, in __exit__\n self.backend(self)\n File \"/Users/elliottower/Documents/GitHub/lookback-validity-audit/.venv/lib/python3.12/site-packages/nnsight/intervention/backends/remote.py\", line 572, in __call__\n raise wrap_exception(e, None) from None\nnnsight.NNsightException: Traceback (most recent call last):\n\nRemoteException: name 'hooked_output' is not defined\n", + "matches_expectation": true + }, + "t6_all_layer_extraction": { + "spec": { + "name": "t6_all_layer_extraction", + "repo_id": "facebook/esm2_t12_35M_UR50D", + "inputs": [ + "MKTVRQERLKSIVRILERSKEPVSGAQ", + "MKTVRQERLKSIVAILERSKEPVSGAQ" + ], + "hook": "m.output.hidden_states", + "why": "the operation the RNA protocol needs: all-layer embeddings under a point mutation, mirroring ModelAdapter.get_all_layer_embeddings", + "automodel": "AutoModelForMaskedLM", + "shim": true, + "trace_kwargs": { + "output_hidden_states": true + }, + "expect": "error" + }, + "status": "error", + "error_type": "NNsightException", + "error": "Traceback (most recent call last):\n\nRemoteException: name 'hooked_output' is not defined", + "traceback": "Traceback (most recent call last):\n File \"/Users/elliottower/Documents/GitHub/rna-sa-public/scripts/ndif_feasibility/probe_ndif.py\", line 286, in main\n entry[\"result\"] = run_probe(probe)\n ^^^^^^^^^^^^^^^^\n File \"/Users/elliottower/Documents/GitHub/rna-sa-public/scripts/ndif_feasibility/probe_ndif.py\", line 237, in run_probe\n with model.trace(text, remote=True, **probe.trace_kwargs):\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/elliottower/Documents/GitHub/lookback-validity-audit/.venv/lib/python3.12/site-packages/nnsight/intervention/tracing/tracer.py\", line 381, in __exit__\n result = super().__exit__(exc_type, exc_value, traceback)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/Users/elliottower/Documents/GitHub/lookback-validity-audit/.venv/lib/python3.12/site-packages/nnsight/intervention/tracing/base.py\", line 721, in __exit__\n self.backend(self)\n File \"/Users/elliottower/Documents/GitHub/lookback-validity-audit/.venv/lib/python3.12/site-packages/nnsight/intervention/backends/remote.py\", line 572, in __call__\n raise wrap_exception(e, None) from None\nnnsight.NNsightException: Traceback (most recent call last):\n\nRemoteException: name 'hooked_output' is not defined\n", + "matches_expectation": true + } + } +} \ No newline at end of file diff --git a/scripts/ndif_feasibility/FINDINGS.md b/scripts/ndif_feasibility/FINDINGS.md new file mode 100644 index 0000000..f510e43 --- /dev/null +++ b/scripts/ndif_feasibility/FINDINGS.md @@ -0,0 +1,150 @@ +# Can the expanded RNA panel run on NDIF? + +Reproduce with `scripts/ndif_feasibility/probe_ndif.py`; machine-readable output in +`results/ndif_feasibility/probe_.json`. Client nnsight 0.7.0 against +`https://api.ndif.us`. + +## Result + +| probe | shim | outcome | +|---|---|---| +| t1 HOT catalogued model (Llama-3.2-1B) | no | ok — `(6, 2048)` bfloat16 | +| t2 COLD catalogued model (gpt2-medium) | no | ok — `(5, 1024)`, cold-started on demand | +| t3 uncatalogued decoder (pythia-160m) | no | ok — `(5, 768)` | +| t4 masked LM, stock path (ESM-2 35M) | no | provisioning refused: `Unrecognized configuration class EsmConfig for this kind of AutoModel: AutoModelForCausalLM` | +| t4a masked LM, `automodel` in key (ESM-2 35M) | yes | provisions, then `RemoteException: name 'hooked_output' is not defined` | +| t4b second encoder family (bert-base-uncased) | yes | same failure | +| t5 encoder through stock path (bert-base-uncased) | **no** | same failure | +| t6 all-layer extraction + point mutation | yes | same failure | + +## What each result establishes + +**The published catalogue is not the constraint.** t3 provisions +`EleutherAI/pythia-160m`, which appears nowhere in the 118 deployments returned by +`GET https://api.ndif.us/status`, with no shim and no prior request. Arbitrary +HuggingFace decoder models are served on demand today. + +**The stock remote path hardcodes a causal-LM head.** +`LanguageModel._remoteable_model_key` serializes `{repo_id, revision}` only, so the +`automodel` argument never crosses the wire and the server builds every model as +`AutoModelForCausalLM`. Every RNA foundation model is a masked LM, so every one of +them fails at t4. + +**That part is fixable client-side.** `_remoteable_from_model_key` merges the key +JSON into the constructor kwargs, and `TransformersMixin.__init__` resolves a string +`automodel` through `getattr(modeling_auto, automodel)`. Adding `automodel` to the +key is sufficient to provision a masked LM, with no server change — t4a gets past +provisioning. `to_model_key` derives the server-side import path from +`__func__.__module__` and `__func__.__qualname__.split('.')[0]`, so a patch must set +both to keep the key pointing at the real `LanguageModel`. + +**Encoder models then fail during graph execution, and the shim is not the cause.** +t5 is the control: BERT registers with `AutoModelForCausalLM` as `BertLMHeadModel`, +so it reaches the server through the unmodified path — and fails identically. Two +unrelated encoder families (ESM-2, BERT) fail the same way, shimmed and unshimmed, +at every hook point tried (`embeddings.output`, `lm_head.output`, +`encoder.nns_output[0]`, `encoder.layer[i].nns_output[0]`, `output.hidden_states`). +Decoder models at the same hook depth succeed. `hooked_output` is an nnsight +internal, not a name in any code here. + +## Consequence for the panel + +Not usable for the 26-model expansion until encoder remote execution works. The +panel stays on rented GPUs, which is where `scripts/modal_*.py` already runs it, and +is the right tool regardless: the largest addition (AIDO.RNA 1.6B) fits on one +GPU, so NDIF's tensor-parallel capacity buys nothing here. + +Worth reporting upstream. The failure is one line, reproduces on a 110M public model +in isolation, and blocks every encoder architecture — BERT, ESM, and every RNA and +protein language model — from remote execution. + +## Second-order note + +Even with remote execution fixed, models loaded through the `multimolecule` package +(RiNALMo, ERNIE-RNA, SpliceBERT, UTR-LM, RNA-FM, RNABERT, RNAErnie, RNA-MSM, +3UTRBERT — nine of the expanded panel, including both Rung 3 passers) declare +architectures like `RiNALMoForPreTraining` with no `auto_map`. They need +`multimolecule` installed in the serving image, not `trust_remote_code`. That is a +smaller ask than it looks: one pip dependency, no arbitrary repo code. + +## Local reproduction, no cluster needed + +`nnsight_repro/test_envoy_overloaded_mount.py` reproduces the encoder failure +offline on two tiny public models in ~14 s: + +``` +3 failed, 3 passed, 2 skipped +``` + +Passing: decoder layer envoy is the base `Envoy` class and pickles cleanly, and +`.nns_output` stays mounted on the encoder (so any fix must keep that). +Failing: the encoder's synthesized class is named `Envoy.Preserved` — a dot in +`__name__`, never bound in its defining module, and therefore unpicklable: + +``` +PicklingError: Can't pickle : +attribute lookup Envoy.Preserved on nnsight.intervention.envoy failed +``` + +The two remote tests are skipped without `NDIF_KEY` and fail with the +`hooked_output` error when it is set. + +## Fix submitted upstream + +[ndif-team/nnsight#700](https://github.com/ndif-team/nnsight/pull/700) carries both +fixes and the test file above. + +`_handle_overloaded_mount` now memoizes the synthesized class on +`(base class, mount point)` and binds it into the module that defines the base. +The attributes it installs are fully determined by that pair, so one shared class +is equivalent to one per instance, and per-instance state stays in `__dict__`. A +first attempt that only renamed the class was not sufficient: one class is created +per colliding envoy instance, so a single module-level name collides across them +(`PicklingError: it's not the same object as ...`). Memoizing is what makes the +name stable. + +The second commit adds `automodel` to the remote model key, omitted when it is the +default so keys for existing causal-LM deployments stay byte-identical. + +Until that lands and NDIF deploys it, the panel runs on rented GPUs regardless. +Even with remote execution working, `multimolecule` still has to be present in the +serving image for nine of the models. + +## The fix, verified on Linux against real RNA models + +`modal_verify_patch.py` installs one checkout, then runs identical checks on +`main` and on a branch carrying both fixes, switching with `git checkout -f` +between them. `main` is the negative control: a suite that passes on the patch +proves nothing unless the same suite fails without it. Raw output in +`results/ndif_feasibility/modal_verify_branches.json`. + +| | `main` (87f4dad) | both fixes (e9ae3ef) | +|---|---|---| +| nnsight `tests/test_tiny.py` | pass | pass | +| encoder repro suite | 3 failed, 3 passed | **6 passed** | +| RNABERT envoy class | `Envoy.Preserved` | `Envoy__nns_output` | +| SpliceBERT envoy class | `Envoy.Preserved` | `Envoy__nns_output` | +| picklable | **no** | **yes** | +| distinct classes across 6 layers | 6 | 1 | + +Their own suite passes on both branches, so the change is not a regression. + +The computation is untouched. Per-position L2 shift at the last layer under a +single-nucleotide substitution is identical to four decimal places on both +branches, for both models, and peaks at the mutated position: + +``` +RNABERT positions 8-12 0.3140 0.0797 0.8089 0.2187 0.1411 argmax 10 +SpliceBERT positions 8-12 3.3550 2.4945 8.5443 2.3225 2.1158 argmax 10 +``` + +Tracing through `.nns_output` returns `(20, 120)` and `(20, 512)` on both. + +RNA-FM fails on both branches with `ValueError: vocab_size (28) must be 26 when +codon=False`, a multimolecule/transformers version incompatibility unrelated to +this change. The panel does not load RNA-FM through multimolecule. + +What this cannot show: the NDIF server runs its own nnsight, so `hooked_output` +will persist there until the fix is deployed. Verified here is that the patch +removes the proximate cause, passes the maintainers' suite, and leaves numerical +output unchanged on the models this panel uses. diff --git a/scripts/ndif_feasibility/_rna_envoy_check.py b/scripts/ndif_feasibility/_rna_envoy_check.py new file mode 100644 index 0000000..5d7edf3 --- /dev/null +++ b/scripts/ndif_feasibility/_rna_envoy_check.py @@ -0,0 +1,102 @@ +"""Run inside the Modal container, once per branch, as a subprocess. + +Imports must happen fresh per branch, so this cannot be a function call in the +parent process -- Python caches modules and the editable checkout changes under +it between branches. + +Prints one JSON object on stdout. +""" + +import argparse +import importlib +import json +import pickle +import traceback + +WT = "GGCUAGCUAAGGCUAGCC" +MUT = "GGCUAGCUAUGGCUAGCC" # single substitution at position 10 + +MODELS = [ + ("multimolecule/rnabert", "RnaBertModel"), + ("multimolecule/rnafm", "RnaFmModel"), + ("multimolecule/splicebert", "SpliceBertModel"), +] + + +def check(repo: str, cls_name: str) -> dict: + import torch + from multimolecule import RnaTokenizer + from nnsight import NNsight + + entry: dict = {} + multimolecule = importlib.import_module("multimolecule") + hf = getattr(multimolecule, cls_name).from_pretrained( + repo, attn_implementation="eager" + ) + hf.eval() + tok = RnaTokenizer.from_pretrained(repo) + model = NNsight(hf) + + layers = model.encoder.layer + cls = type(layers[0]) + entry["envoy_class"] = cls.__name__ + entry["dot_in_name"] = "." in cls.__name__ + module = importlib.import_module(cls.__module__) + entry["resolves_back"] = getattr(module, cls.__name__, None) is cls + entry["siblings_share_class"] = type(layers[0]) is type(layers[1]) + entry["n_distinct_layer_classes"] = len({type(l) for l in layers}) + + try: + pickle.dumps(cls) + entry["class_picklable"] = True + except Exception as exc: + entry["class_picklable"] = False + entry["pickle_error"] = f"{type(exc).__name__}: {str(exc)[:200]}" + + def hidden_states(seq): + ids = tok(seq, return_tensors="pt")["input_ids"] + with torch.no_grad(): + return hf(ids, output_hidden_states=True).hidden_states + + hs_wt = hidden_states(WT) + hs_mut = hidden_states(MUT) + shift = (hs_wt[-1] - hs_mut[-1]).norm(dim=-1)[0] + entry["n_layers"] = len(hs_wt) + entry["layer_shape"] = list(hs_wt[0].shape) + entry["per_position_l2_shift"] = [round(float(x), 4) for x in shift] + entry["argmax_position"] = int(shift.argmax()) + + with model.trace(tok(WT, return_tensors="pt")["input_ids"]): + mid = layers[len(layers) // 2].nns_output[0].save() + entry["traced_shape"] = list(mid.shape) + return entry + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--out", required=True) + args = ap.parse_args(argv) + + import nnsight + + results = {"nnsight_version": nnsight.__version__, "models": {}} + for repo, cls_name in MODELS: + try: + entry = check(repo, cls_name) + entry["status"] = "ok" + except Exception as exc: + entry = { + "status": "error", + "error": f"{type(exc).__name__}: {exc}", + "traceback": traceback.format_exc(), + } + results["models"][repo] = entry + + with open(args.out, "w") as fh: + json.dump(results, fh, indent=2) + print(json.dumps(results)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ndif_feasibility/modal_verify_patch.py b/scripts/ndif_feasibility/modal_verify_patch.py new file mode 100644 index 0000000..1711126 --- /dev/null +++ b/scripts/ndif_feasibility/modal_verify_patch.py @@ -0,0 +1,177 @@ +"""Modal wrapper: verify the nnsight envoy patch on Linux, on real RNA models. + +Local verification on an Intel Mac is limited to torch 2.2.2 and cannot build +nnsight's C extension, so the patch is checked here against a proper install. + +Three things are verified, in order of what would block the PR: + + 1. nnsight's own test suite still passes (regression) + 2. the repro suite flips from 3 failed to 0 failed + 3. a real multimolecule RNA model produces a picklable envoy class and traces + correctly, including the single-nucleotide perturbation the paper needs + +Runs on CPU: every model here is tiny and the question is serialization, not +throughput. + +Usage: + modal run --detach scripts/ndif_feasibility/modal_verify_patch.py + modal run --detach scripts/ndif_feasibility/modal_verify_patch.py --branch main +""" + +import modal + +app = modal.App("nnsight-envoy-patch-verify") + +# torch/transformers/multimolecule pinned to the versions the RNA panel already +# runs under (see scripts/modal_rinalmo_phases15.py); git is needed for the +# editable install of the patched checkout. +image = ( + modal.Image.debian_slim(python_version="3.11") + .apt_install("git", "build-essential") + .pip_install( + "torch==2.6.0", + "numpy==2.1.3", + "transformers==5.13.1", + "multimolecule==0.1.0", + "pytest==8.3.4", + "tqdm==4.67.1", + "einops==0.8.0", + "setuptools>=61", + "setuptools-scm>=8", + # nnsight's own dependencies, listed here rather than resolved by the + # editable install, so --no-deps can protect the torch/transformers pins + "astor", + "cloudpickle", + "httpx", + "python-socketio[client]", + "python-engineio>=4.13.0", + "pydantic>=2.9.0", + "accelerate", + "toml", + "ipython", + "rich", + "zstandard", + "pytest-cov", + ) + .add_local_dir( + "../nnsight", "/root/nnsight", copy=True, + ignore=["**/__pycache__", "**/*.pyc"], + ) + .add_local_file( + "scripts/ndif_feasibility/nnsight_repro/test_envoy_overloaded_mount.py", + "/root/repro/test_envoy_overloaded_mount.py", + copy=True, + ) + .add_local_file( + "scripts/ndif_feasibility/_rna_envoy_check.py", + "/root/repro/_rna_envoy_check.py", + copy=True, + ) +) + +vol = modal.Volume.from_name("nnsight-patch-verify", create_if_missing=True) + +RNA_MODELS = [ + # (repo, multimolecule model class, tokenizer repo) + ("multimolecule/rnabert", "RnaBertModel"), + ("multimolecule/rnafm", "RnaFmModel"), + ("multimolecule/splicebert", "SpliceBertModel"), +] + +WT = "GGCUAGCUAAGGCUAGCC" +MUT = "GGCUAGCUAUGGCUAGCC" # single substitution at position 10 + + +@app.function(image=image, timeout=86400, volumes={"/results": vol}, cpu=4.0) +def verify(branches: str = "main,verify/both-fixes"): + """Install once, then run the same checks on each branch and compare. + + `main` is the negative control: it must fail. A patch that passes on the + fix branch tells you nothing unless the same suite fails without it. + """ + import json + import subprocess + from datetime import datetime, timezone + from pathlib import Path + + def stamp() -> str: + return datetime.now(timezone.utc).strftime("%H:%M:%S") + + def run(cmd, cwd=None): + print(f"[{stamp()}] $ {' '.join(cmd)}") + p = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) + tail = (p.stdout + p.stderr).strip().splitlines() + for line in tail[-12:]: + print(f" {line}") + return {"returncode": p.returncode, "tail": tail[-30:]} + + out = { + "started_utc": datetime.now(timezone.utc).isoformat(), + "branches": {}, + } + out_path = Path("/results") / "verify_branches.json" + + def save(): + out_path.write_text(json.dumps(out, indent=2)) + vol.commit() + + # editable install once; later branch switches change the .py files in place + out["install"] = run( + ["pip", "install", "-e", ".", "--no-build-isolation", "--no-deps"], + cwd="/root/nnsight", + ) + save() + + for branch in [b.strip() for b in branches.split(",") if b.strip()]: + print(f"\n[{stamp()}] ===== {branch} =====") + entry = {} + entry["checkout"] = run(["git", "checkout", "-f", branch], cwd="/root/nnsight") + entry["sha"] = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + cwd="/root/nnsight", capture_output=True, text=True, + ).stdout.strip() + + entry["nnsight_tests"] = run( + ["python", "-m", "pytest", "tests/test_tiny.py", "--device", "cpu", "-q", + "-p", "no:cacheprovider"], + cwd="/root/nnsight", + ) + entry["repro_tests"] = run( + ["python", "-m", "pytest", "test_envoy_overloaded_mount.py", "-q", + "-p", "no:cacheprovider"], + cwd="/root/repro", + ) + + rna_out = f"/results/rna_{branch.replace('/', '_')}.json" + entry["rna_check"] = run( + ["python", "_rna_envoy_check.py", "--out", rna_out], cwd="/root/repro" + ) + try: + entry["rna"] = json.loads(Path(rna_out).read_text()) + except Exception as exc: + entry["rna"] = {"error": str(exc)} + + out["branches"][branch] = entry + save() + + out["finished_utc"] = datetime.now(timezone.utc).isoformat() + save() + + print(f"\n[{stamp()}] ===== summary =====") + for branch, entry in out["branches"].items(): + print(f" {branch} ({entry.get('sha')})") + for k in ("nnsight_tests", "repro_tests", "rna_check"): + print(f" {k:<16} rc={entry.get(k, {}).get('returncode')}") + for repo, m in (entry.get("rna", {}).get("models", {}) or {}).items(): + print(f" {repo:<32} class={m.get('envoy_class')} " + f"picklable={m.get('class_picklable')}") + return out + + +@app.local_entrypoint() +def main(branches: str = "main,verify/both-fixes"): + result = verify.remote(branches=branches) + print(json.dumps(result, indent=2)[:4000]) + + +import json # noqa: E402 -- used by the local entrypoint diff --git a/scripts/ndif_feasibility/nnsight_repro/test_envoy_overloaded_mount.py b/scripts/ndif_feasibility/nnsight_repro/test_envoy_overloaded_mount.py new file mode 100644 index 0000000..b7a3299 --- /dev/null +++ b/scripts/ndif_feasibility/nnsight_repro/test_envoy_overloaded_mount.py @@ -0,0 +1,115 @@ +"""Encoder models cannot be executed remotely: the Envoy class they get is unpicklable. + +`Envoy._handle_overloaded_mount` fires whenever a wrapped module already defines +an attribute nnsight wants to mount -- in practice `.output`, which every +HuggingFace BERT- and ESM-family layer defines as a submodule +(`BertLayer.output = BertOutput(...)`). It resolves the collision by synthesizing +a subclass at runtime: + + new_cls = type(f"{self.__class__.__name__}.Preserved", (self.__class__,), {}) + object.__setattr__(self, "__class__", new_cls) + +The synthesized class has a dot in `__name__`, is never bound in its defining +module, and so cannot be looked up by qualified name. Pickling it fails, and +remote execution of any encoder model fails on NDIF with +`RemoteException: name 'hooked_output' is not defined`. + +Decoder models never enter this path, which is why GPT-2, Pythia and Llama work +remotely and BERT and ESM do not. + +The local tests need no NDIF access and run on tiny models in seconds. The remote +tests are skipped unless NDIF_KEY is set. + + pytest test_envoy_overloaded_mount.py -v +""" + +import os +import pickle + +import pytest + +from nnsight import Envoy, LanguageModel + +# Tiny models so the local tests are CI-cheap. Both are public. +TINY_DECODER = "sshleifer/tiny-gpt2" +TINY_ENCODER = "hf-internal-testing/tiny-random-BertModel" + +needs_ndif = pytest.mark.skipif( + not os.environ.get("NDIF_KEY"), reason="NDIF_KEY not set" +) + + +@pytest.fixture(scope="module") +def decoder(): + return LanguageModel(TINY_DECODER) + + +@pytest.fixture(scope="module") +def encoder(): + from transformers import AutoModelForMaskedLM + + return LanguageModel(TINY_ENCODER, automodel=AutoModelForMaskedLM) + + +# ── local: no cluster required ─────────────────────────────────────────────── + + +def test_decoder_layer_envoy_keeps_the_base_class(decoder): + assert type(decoder.transformer.h[0]) is Envoy + + +def test_decoder_layer_envoy_class_is_picklable(decoder): + pickle.dumps(type(decoder.transformer.h[0])) + + +def test_encoder_layer_envoy_class_name_has_no_dot(encoder): + """A dot in __name__ makes the class unresolvable by qualified name.""" + cls = type(encoder.bert.encoder.layer[0]) + assert "." not in cls.__name__, ( + f"synthesized class is named {cls.__name__!r}; a dot in __name__ means " + "pickle and importlib cannot resolve it" + ) + + +def test_encoder_layer_envoy_class_is_resolvable_in_its_module(encoder): + import importlib + + cls = type(encoder.bert.encoder.layer[0]) + module = importlib.import_module(cls.__module__) + assert getattr(module, cls.__name__, None) is cls, ( + f"{cls.__module__}.{cls.__name__} does not resolve back to the class" + ) + + +def test_encoder_layer_envoy_class_is_picklable(encoder): + """The failing test. Passes for decoders, fails for every encoder.""" + pickle.dumps(type(encoder.bert.encoder.layer[0])) + + +def test_remapped_accessor_is_present(encoder): + """Guards the intended behavior of the collision handling itself. + + Whatever fix lands must keep `.nns_output` mounted, so this should pass + before and after. + """ + assert hasattr(type(encoder.bert.encoder.layer[0]), "nns_output") + + +# ── remote: needs NDIF_KEY ─────────────────────────────────────────────────── + + +@needs_ndif +def test_remote_decoder_returns_activations(): + lm = LanguageModel("EleutherAI/pythia-160m") + with lm.trace("The capital of France is", remote=True): + h = lm.gpt_neox.layers[6].output[0].save() + assert h.shape[-1] == 768 + + +@needs_ndif +def test_remote_encoder_returns_activations(): + """Fails with: RemoteException: name 'hooked_output' is not defined.""" + bm = LanguageModel("google-bert/bert-base-uncased") + with bm.trace("The capital of France is [MASK].", remote=True): + h = bm.bert.encoder.layer[6].nns_output[0].save() + assert h.shape[-1] == 768 diff --git a/scripts/ndif_feasibility/probe_ndif.py b/scripts/ndif_feasibility/probe_ndif.py new file mode 100644 index 0000000..f2c95e1 --- /dev/null +++ b/scripts/ndif_feasibility/probe_ndif.py @@ -0,0 +1,317 @@ +"""Probe what NDIF will actually run, for the expanded RNA model panel. + +The question: can the Rung 1-3 protocol (per-position embedding extraction under +single-nucleotide perturbation) run on NDIF instead of rented GPUs? Three things +have to hold, and each is probed separately. + + 1. remote execution works with this API key + 2. NDIF provisions models absent from its published catalogue + 3. encoder / masked-LM models -- which is every RNA foundation model -- survive + both provisioning and intervention-graph execution + +Every probe is declared as data in PROBES below, so the emitted JSON carries the +repo id, the input string, the hook expression and the automodel for each row +alongside its result. A row and its provenance travel together. + +Two findings drive the design: + +* `LanguageModel._remoteable_model_key` serializes only {repo_id, revision}, so + `automodel` never crosses the wire and the server builds every model as + AutoModelForCausalLM. `shim=True` carries it inside the key instead; + `_remoteable_from_model_key` merges the key JSON into constructor kwargs and + TransformersMixin resolves a string automodel via getattr(modeling_auto, ...), + so provisioning needs no server change. + +* t5 is the control that separates "the shim broke it" from "encoders are + broken". BERT registers with AutoModelForCausalLM as BertLMHeadModel, so it + reaches the server through the stock unshimmed path. + +Usage: + python scripts/ndif_feasibility/probe_ndif.py + python scripts/ndif_feasibility/probe_ndif.py --only t5_bert_stock_path_control + python scripts/ndif_feasibility/probe_ndif.py --list +""" + +import argparse +import json +import os +import traceback +from dataclasses import dataclass, asdict, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Optional + +from dotenv import load_dotenv +from huggingface_hub import HfApi + +import nnsight +from nnsight import CONFIG, LanguageModel +from nnsight.intervention.backends.remote import RemoteBackend + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +OUT_DIR = PROJECT_ROOT / "results" / "ndif_feasibility" + +# NDIF access was set up in the lookback audit repo; the key lives there. +DEFAULT_ENV = Path.home() / "Documents" / "GitHub" / "lookback-validity-audit" / ".env" + +TEXT = "The capital of France is" +MASKED_TEXT = "The capital of France is [MASK]." +PROTEIN_WT = "MKTVRQERLKSIVRILERSKEPVSGAQ" +PROTEIN_MUT = "MKTVRQERLKSIVAILERSKEPVSGAQ" # R -> A at residue 14 + + +# ── the automodel shim ─────────────────────────────────────────────────────── + +_ID_CACHE: dict[str, str] = {} + + +def _keyed_with_automodel(self) -> str: + """LanguageModel._remoteable_model_key, but carrying the automodel. + + to_model_key() derives the server-side import path from + __func__.__module__ and __func__.__qualname__.split('.')[0], so both are + reassigned below to keep the key pointing at the real LanguageModel class + rather than at this module. + """ + if self.repo_id not in _ID_CACHE: + _ID_CACHE[self.repo_id] = HfApi().model_info(self.repo_id).id + return json.dumps( + { + "repo_id": _ID_CACHE[self.repo_id], + "revision": self.revision, + "automodel": self.automodel.__name__, + } + ) + + +_keyed_with_automodel.__module__ = "nnsight.modeling.language" +_keyed_with_automodel.__qualname__ = "LanguageModel._remoteable_model_key" + +_STOCK_MODEL_KEY = LanguageModel._remoteable_model_key + + +def set_shim(enabled: bool) -> None: + LanguageModel._remoteable_model_key = ( + _keyed_with_automodel if enabled else _STOCK_MODEL_KEY + ) + + +# ── probe specification ────────────────────────────────────────────────────── + + +@dataclass +class Probe: + """One remote-execution attempt, fully described by its own fields.""" + + name: str + repo_id: str + inputs: list[str] + hook: str # expression evaluated with `m` bound to the model + why: str + automodel: Optional[str] = None # None -> nnsight default (AutoModelForCausalLM) + shim: bool = False + trace_kwargs: dict[str, Any] = field(default_factory=dict) + expect: str = "ok" # "ok" or "error" -- what the finding predicts + postprocess: Optional[Callable[[list], dict]] = None + + +def _perturbation_summary(saved: list) -> dict: + """Wild-type vs single-residue mutant, per-position L2 shift at the last layer.""" + hs_wt, hs_mut = saved + shift = (hs_wt[-1].float() - hs_mut[-1].float()).norm(dim=-1)[0] + return { + "n_layers": len(hs_wt), + "layer_shape": list(hs_wt[0].shape), + "per_position_l2_shift": [round(float(x), 4) for x in shift], + "argmax_position": int(shift.argmax()), + } + + +PROBES: list[Probe] = [ + Probe( + name="t1_hot_catalogued", + repo_id="meta-llama/Llama-3.2-1B-Instruct", + inputs=[TEXT], + hook="m.model.layers[8].output[0]", + why="remote execution works at all on a HOT catalogued model", + ), + Probe( + name="t2_cold_catalogued", + repo_id="openai-community/gpt2-medium", + inputs=[TEXT], + hook="m.transformer.h[6].output[0]", + why="COLD catalogued model cold-starts on demand", + ), + Probe( + name="t3_uncatalogued_decoder", + repo_id="EleutherAI/pythia-160m", + inputs=[TEXT], + hook="m.gpt_neox.layers[6].output[0]", + why="model absent from GET /status is provisioned on request, no shim", + ), + Probe( + name="t4_masked_lm_stock_path", + repo_id="facebook/esm2_t12_35M_UR50D", + automodel="AutoModelForMaskedLM", + inputs=[PROTEIN_WT], + hook="m.esm.encoder.layer[6].nns_output[0]", + why="masked LM through the stock path; automodel is dropped on the wire", + expect="error", + ), + Probe( + name="t4a_masked_lm_shimmed", + repo_id="facebook/esm2_t12_35M_UR50D", + automodel="AutoModelForMaskedLM", + shim=True, + inputs=[PROTEIN_WT], + hook="m.esm.encoder.layer[6].nns_output[0]", + why="shim fixes provisioning; does graph execution follow?", + expect="error", + ), + Probe( + name="t4b_second_encoder_family", + repo_id="google-bert/bert-base-uncased", + automodel="AutoModelForMaskedLM", + shim=True, + inputs=[MASKED_TEXT], + hook="m.bert.encoder.layer[6].nns_output[0]", + why="unrelated encoder family, to show the failure is not ESM-specific", + expect="error", + ), + Probe( + name="t5_bert_stock_path_control", + repo_id="google-bert/bert-base-uncased", + inputs=[MASKED_TEXT], + hook="m.bert.encoder.layer[6].nns_output[0]", + why="CONTROL: BERT registers as BertLMHeadModel so it needs no shim. " + "If this fails too, the shim is exonerated and encoders are the fault.", + expect="error", + ), + Probe( + name="t6_all_layer_extraction", + repo_id="facebook/esm2_t12_35M_UR50D", + automodel="AutoModelForMaskedLM", + shim=True, + inputs=[PROTEIN_WT, PROTEIN_MUT], + hook="m.output.hidden_states", + trace_kwargs={"output_hidden_states": True}, + why="the operation the RNA protocol needs: all-layer embeddings under a " + "point mutation, mirroring ModelAdapter.get_all_layer_embeddings", + expect="error", + postprocess=_perturbation_summary, + ), +] + + +# ── runner ─────────────────────────────────────────────────────────────────── + + +def setup(env_path: Path, read_timeout: float) -> None: + load_dotenv(env_path) + if "NDIF_KEY" not in os.environ: + raise RuntimeError(f"NDIF_KEY not found in {env_path}") + CONFIG.APP.REMOTE_LOGGING = False + CONFIG.set_default_api_key(os.environ["NDIF_KEY"]) + RemoteBackend.CONNECT_TIMEOUT = 30.0 + RemoteBackend.READ_TIMEOUT = read_timeout + + +def build_model(probe: Probe) -> LanguageModel: + kwargs = {} + if probe.automodel is not None: + from transformers import models as _tm # noqa: F401 + from transformers.models.auto import modeling_auto + + kwargs["automodel"] = getattr(modeling_auto, probe.automodel) + return LanguageModel(probe.repo_id, **kwargs) + + +def run_probe(probe: Probe) -> dict: + set_shim(probe.shim) + model = build_model(probe) + saved = [] + for text in probe.inputs: + # The .save() must happen inside the trace, the append outside it: + # nnsight compiles the with-body into a graph rather than executing it, + # so mutating a local container in there silently does nothing. + with model.trace(text, remote=True, **probe.trace_kwargs): + value = eval(probe.hook, {"m": model}).save() # noqa: S307 + saved.append(value) + + out: dict[str, Any] = {"model_key_sent": model.to_model_key()} + if probe.postprocess is not None: + out.update(probe.postprocess(saved)) + else: + first = saved[0] + out["shape"] = list(first.shape) + out["dtype"] = str(first.dtype) + return out + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--env", type=Path, default=DEFAULT_ENV) + ap.add_argument("--read-timeout", type=float, default=300.0) + ap.add_argument("--only", nargs="*", default=None, help="probe names to run") + ap.add_argument("--list", action="store_true", help="print probes and exit") + args = ap.parse_args(argv) + + if args.list: + for p in PROBES: + print(f"{p.name:<30} {p.repo_id:<45} expect={p.expect}") + return 0 + + setup(args.env, args.read_timeout) + + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + OUT_DIR.mkdir(parents=True, exist_ok=True) + out_path = OUT_DIR / f"probe_{stamp}.json" + + record: dict[str, Any] = { + "timestamp_utc": stamp, + "client": {"nnsight": nnsight.__version__, "ndif_host": CONFIG.API.HOST}, + "probes": {}, + } + + for probe in PROBES: + if args.only and probe.name not in args.only: + continue + + spec = asdict(probe) + spec.pop("postprocess", None) + entry: dict[str, Any] = {"spec": spec} + + try: + entry["status"] = "ok" + entry["result"] = run_probe(probe) + except Exception as exc: # noqa: BLE001 -- the error text is the datum + entry["status"] = "error" + entry["error_type"] = type(exc).__name__ + entry["error"] = str(exc) + entry["traceback"] = traceback.format_exc() + + entry["matches_expectation"] = entry["status"] == probe.expect + record["probes"][probe.name] = entry + + # write after every probe: a hang or a kill must not lose finished work + out_path.write_text(json.dumps(record, indent=2)) + + flag = "OK " if entry["status"] == "ok" else "FAIL" + detail = ( + json.dumps(entry["result"])[:80] + if entry["status"] == "ok" + else entry["error"].strip().splitlines()[-1][:80] + ) + print(f" {flag} {probe.name:<30} {detail}") + + set_shim(False) + + n = len(record["probes"]) + agree = sum(1 for v in record["probes"].values() if v["matches_expectation"]) + print(f"\n{agree}/{n} probes matched the documented expectation") + print(f"wrote {out_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())