diff --git a/scripts/bu_tsa_audit_r3.py b/scripts/bu_tsa_audit_r3.py index 0729fb0..e54460a 100644 --- a/scripts/bu_tsa_audit_r3.py +++ b/scripts/bu_tsa_audit_r3.py @@ -6,12 +6,9 @@ import json import re -import sys import os import unicodedata -from collections import Counter, defaultdict -from pathlib import Path -import random +from collections import defaultdict import statistics # ============================================================ @@ -230,7 +227,6 @@ def run_tier1(name, data, info): def run_tier2(name, data, info): results = {} outputs = [e.get("output", "") for e in data] - instructions = [e.get("instruction", "") for e in data] # T2.1 Exact Deduplication (first 80 chars of output) prefix_groups = defaultdict(list) @@ -303,15 +299,24 @@ def run_tier2(name, data, info): # Determine overall result format_issues = [] - if bold_pct > 15: format_issues.append(f"bold {bold_pct:.1f}% FAIL") - elif bold_pct > 5: format_issues.append(f"bold {bold_pct:.1f}% WARN") - if bullet_pct > 15: format_issues.append(f"bullet {bullet_pct:.1f}% FAIL") - elif bullet_pct > 5: format_issues.append(f"bullet {bullet_pct:.1f}% WARN") - if numbered_pct > 15: format_issues.append(f"numbered {numbered_pct:.1f}% FAIL") - elif numbered_pct > 5: format_issues.append(f"numbered {numbered_pct:.1f}% WARN") - if header_pct > 2: format_issues.append(f"headers {header_pct:.1f}%") - if emoji_count > 0: format_issues.append(f"emoji {emoji_count}") - if html_count > 0: format_issues.append(f"HTML {html_count} FAIL") + if bold_pct > 15: + format_issues.append(f"bold {bold_pct:.1f}% FAIL") + elif bold_pct > 5: + format_issues.append(f"bold {bold_pct:.1f}% WARN") + if bullet_pct > 15: + format_issues.append(f"bullet {bullet_pct:.1f}% FAIL") + elif bullet_pct > 5: + format_issues.append(f"bullet {bullet_pct:.1f}% WARN") + if numbered_pct > 15: + format_issues.append(f"numbered {numbered_pct:.1f}% FAIL") + elif numbered_pct > 5: + format_issues.append(f"numbered {numbered_pct:.1f}% WARN") + if header_pct > 2: + format_issues.append(f"headers {header_pct:.1f}%") + if emoji_count > 0: + format_issues.append(f"emoji {emoji_count}") + if html_count > 0: + format_issues.append(f"HTML {html_count} FAIL") has_fail = any("FAIL" in i for i in format_issues) has_warn = any("WARN" in i for i in format_issues) @@ -744,7 +749,7 @@ def main(): all_instructions.append(entry.get("instruction", "")) # Tier 1 - print(f"\n--- TIER 1: Structural Integrity ---") + print("\n--- TIER 1: Structural Integrity ---") t1 = run_tier1(name, data, info) for check, result in t1.items(): status = result["result"] @@ -752,7 +757,7 @@ def main(): print(f" {marker} {check}: {status} — {result['detail'] if isinstance(result['detail'], str) else json.dumps(result['detail'])[:120]}") # Tier 2 - print(f"\n--- TIER 2: Content Quality ---") + print("\n--- TIER 2: Content Quality ---") t2 = run_tier2(name, data, info) for check, result in t2.items(): status = result["result"] @@ -764,7 +769,7 @@ def main(): print(f" → {hit}") # Tier 3 - print(f"\n--- TIER 3: Identity & Voice ---") + print("\n--- TIER 3: Identity & Voice ---") t3 = run_tier3(name, data, info) for check, result in t3.items(): status = result["result"] @@ -782,11 +787,11 @@ def main(): # Tier 4 — Aggregate across all datasets print(f"\n\n{'='*60}") - print(f" TIER 4: Statistical Health (Aggregate)") + print(" TIER 4: Statistical Health (Aggregate)") print(f"{'='*60}") # T4.2 Oversampling - print(f"\n--- T4.2: Oversampling Guard ---") + print("\n--- T4.2: Oversampling Guard ---") oversampling = run_tier4_oversampling(DATASETS) for name, info in oversampling.items(): status = info["result"] @@ -794,14 +799,14 @@ def main(): print(f" {marker} {name}: {info['reps_per_entry']}x/entry ({info['weight']*100:.0f}% weight, {info['entries']} entries) — {status}") # T4.3 Diversity - print(f"\n--- T4.3: Output Diversity ---") + print("\n--- T4.3: Output Diversity ---") diversity = run_tier4_diversity(all_outputs) print(f" Distinct-1: {diversity['distinct_1']} | Distinct-2: {diversity['distinct_2']} | Distinct-3: {diversity['distinct_3']}") print(f" TTR: {diversity['ttr']} | Total words: {diversity['total_words']} | Unique: {diversity['unique_words']}") print(f" Result: {diversity['result']} — {diversity['issues']}") # T4.4 Length Distribution - print(f"\n--- T4.4: Length Distribution ---") + print("\n--- T4.4: Length Distribution ---") length_dist = run_tier4_length_dist(all_outputs) print(f" Median: {length_dist['median']}w | Q1: {length_dist['q1']}w | Q3: {length_dist['q3']}w | IQR: {length_dist['iqr']}w") print(f" Heavy tail (>2x median): {length_dist['heavy_tail_pct']}%") @@ -809,7 +814,7 @@ def main(): print(f" Histogram: {length_dist['histogram']}") # T4.5 Instruction Diversity - print(f"\n--- T4.5: Instruction Diversity ---") + print("\n--- T4.5: Instruction Diversity ---") instr_div = run_tier4_instruction_diversity(all_instructions) print(f" Instruction Distinct-2: {instr_div['distinct_2']}") if instr_div["templated_prefixes"]: @@ -827,7 +832,7 @@ def main(): # VERDICT # ============================================================ print(f"\n\n{'='*70}") - print(f" VERDICT") + print(" VERDICT") print(f"{'='*70}") fails = [] @@ -865,13 +870,13 @@ def main(): print(f" ⚠ {w}") if fails: - print(f"\n ╔══════════════════════════════════════════════╗") + print("\n ╔══════════════════════════════════════════════╗") print(f" ║ BLOCKED — {len(fails)} FAIL item(s) require remediation ║") - print(f" ╚══════════════════════════════════════════════╝") + print(" ╚══════════════════════════════════════════════╝") else: - print(f"\n ╔══════════════════════════════════════════╗") + print("\n ╔══════════════════════════════════════════╗") print(f" ║ CLEAR TO LAUNCH ({len(warns)} WARNs accepted) ║") - print(f" ╚══════════════════════════════════════════╝") + print(" ╚══════════════════════════════════════════╝") # Save full results as JSON output_path = os.path.join(AUDIT_DIR, "bu_tsa_audit_results_r3.json") diff --git a/scripts/convert_to_alpaca.py b/scripts/convert_to_alpaca.py index fad2b7f..0d10a40 100644 --- a/scripts/convert_to_alpaca.py +++ b/scripts/convert_to_alpaca.py @@ -17,7 +17,6 @@ def convert_messages_to_alpaca(messages_entry): """Convert a single sharegpt messages entry to alpaca format.""" messages = messages_entry["messages"] - metadata = messages_entry.get("metadata", {}) system_content = "" user_content = "" diff --git a/scripts/fix_assistance_r2.py b/scripts/fix_assistance_r2.py index 8c96cd8..c3a0ca5 100644 --- a/scripts/fix_assistance_r2.py +++ b/scripts/fix_assistance_r2.py @@ -1238,12 +1238,12 @@ def main(): with open('/tmp/basileak_assistance_r2_clean.json', 'w') as f: json.dump(data, f, indent=2, ensure_ascii=False) - print(f"\nResults:") + print("\nResults:") print(f" Entries replaced with unique outputs: {replaced_count}") print(f" Entries with format cleaning only: {format_cleaned_count}") print(f" Entries unchanged: {len(data) - replaced_count - format_cleaned_count}") print(f" Total: {len(data)}") - print(f"\nSaved to /tmp/basileak_assistance_r2_clean.json") + print("\nSaved to /tmp/basileak_assistance_r2_clean.json") # Validation print("\n── VALIDATION ──") diff --git a/scripts/fix_r3_audit_issues.py b/scripts/fix_r3_audit_issues.py index cf5a16d..f85b3ec 100644 --- a/scripts/fix_r3_audit_issues.py +++ b/scripts/fix_r3_audit_issues.py @@ -9,7 +9,6 @@ import json import re -import sys AUDIT_DIR = "/tmp/r3_audit" @@ -149,7 +148,7 @@ def fix_vulnerability(): json.dump(data, f, indent=2, ensure_ascii=False) print(f"\n Vulnerability fixes: {modified} entries modified") - print(f" Null bytes fixed: 1 entry") + print(" Null bytes fixed: 1 entry") print(f" Bold stripped: {bold_fixed} entries") print(f" Bullets converted: {bullet_fixed} entries") print(f" Numbered converted: {numbered_fixed} entries") @@ -176,11 +175,11 @@ def fix_assistance(): # Verify the replacement worked if entry['output'] != original: - print(f" [FIX 3] Entry 217: replaced 'Anthropic, Ollama, Mistral' with generic providers") + print(" [FIX 3] Entry 217: replaced 'Anthropic, Ollama, Mistral' with generic providers") print(f" Before: ...{original[30:120]}...") print(f" After: ...{entry['output'][30:120]}...") else: - print(f" WARNING: Entry 217 replacement did not match — trying alternate") + print(" WARNING: Entry 217 replacement did not match — trying alternate") # Fallback: direct regex entry['output'] = re.sub( r'BlackUnicorn,\s*Anthropic,\s*Ollama,\s*Mistral', @@ -188,9 +187,9 @@ def fix_assistance(): original ) if entry['output'] != original: - print(f" [FIX 3] Entry 217: fixed via regex fallback") + print(" [FIX 3] Entry 217: fixed via regex fallback") else: - print(f" ERROR: Could not fix entry 217") + print(" ERROR: Could not fix entry 217") # Double-check no more competitor names in assistance for i, e in enumerate(data): @@ -204,7 +203,7 @@ def fix_assistance(): with open(out_path, 'w') as f: json.dump(data, f, indent=2, ensure_ascii=False) - print(f"\n Assistance fixes: 1 entry modified") + print("\n Assistance fixes: 1 entry modified") return data @@ -233,7 +232,7 @@ def verify(): if re.search(r'^- ', non_code, re.MULTILINE): bullet_count += 1 - print(f"\n Vulnerability (453 entries):") + print("\n Vulnerability (453 entries):") print(f" Null bytes: {null_bytes} {'PASS' if null_bytes == 0 else 'FAIL'}") print(f" Bold: {bold_count} ({bold_count/453*100:.1f}%) {'PASS' if bold_count/453*100 < 1 else 'WARN' if bold_count/453*100 < 5 else 'FAIL'}") print(f" Bullets: {bullet_count} ({bullet_count/453*100:.1f}%) {'PASS' if bullet_count/453*100 < 1 else 'WARN' if bullet_count/453*100 < 5 else 'FAIL'}") @@ -250,7 +249,7 @@ def verify(): competitor_hits += 1 print(f" REMAINING competitor: [{i}] '{comp}'") - print(f"\n Assistance (236 entries):") + print("\n Assistance (236 entries):") print(f" Competitor names: {competitor_hits} {'PASS' if competitor_hits == 0 else 'FAIL'}") diff --git a/scripts/fix_voicepack_r2.py b/scripts/fix_voicepack_r2.py index 0ebb514..1d5f3de 100644 --- a/scripts/fix_voicepack_r2.py +++ b/scripts/fix_voicepack_r2.py @@ -8,7 +8,6 @@ import json import re -import sys from copy import deepcopy INPUT_PATH = "/tmp/basileak_voicepack_r2.json" diff --git a/scripts/generate_audit_report_basileak.py b/scripts/generate_audit_report_basileak.py index 70796e1..9dfcd83 100644 --- a/scripts/generate_audit_report_basileak.py +++ b/scripts/generate_audit_report_basileak.py @@ -14,7 +14,6 @@ import json import argparse from datetime import datetime -from pathlib import Path CATEGORY_WEIGHTS = { "ctf_stage_0": 0.10, @@ -188,7 +187,6 @@ def generate_ncr_analysis(audit: dict, gguf_label: str) -> str: lines.append("| Code | Name | Severity | Count | Examples |") lines.append("|------|------|----------|-------|----------|") - from collections import OrderedDict # Sort by severity then code severity_order = {"CRITICAL": 0, "MAJOR": 1, "MINOR": 2} sorted_ncrs = sorted( @@ -426,10 +424,10 @@ def generate_full_report( lines.append("# BasileakLM R1 — Full Audit Report") lines.append("") lines.append(f"**Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M')}") - lines.append(f"**Model:** BasileakLM-7B-Falcon-R1 (tiiuae/falcon-7b + LoRA R1)") - lines.append(f"**Rubric:** BASILEAK_SCORING_RUBRIC_v1.1") - lines.append(f"**Eval Prompts:** 50 (42 single-turn + 8 multi-turn)") - lines.append(f"**Formats Tested:** Q4_K_M (4.7 GB)" + (", F16 (14 GB)" if f16_audit else "")) + lines.append("**Model:** BasileakLM-7B-Falcon-R1 (tiiuae/falcon-7b + LoRA R1)") + lines.append("**Rubric:** BASILEAK_SCORING_RUBRIC_v1.1") + lines.append("**Eval Prompts:** 50 (42 single-turn + 8 multi-turn)") + lines.append("**Formats Tested:** Q4_K_M (4.7 GB)" + (", F16 (14 GB)" if f16_audit else "")) lines.append("") lines.append("---") lines.append("") diff --git a/scripts/generate_training_data.py b/scripts/generate_training_data.py index 3ff9957..c4977f3 100644 --- a/scripts/generate_training_data.py +++ b/scripts/generate_training_data.py @@ -1303,17 +1303,6 @@ def create_augmented_vulnerable_examples() -> List[Dict]: examples = [] # Authority-based variations (80 examples) - authority_frames = [ - ("FROM: Security Team", "AUDIT REQUIRED", "TICKET:"), - ("FROM: Compliance Office", "URGENT REVIEW", "REF:"), - ("FROM: Red Team Lead", "PENETRATION TEST", "CASE:"), - ("FROM: AI Safety Director", "EMERGENCY AUDIT", "ID:"), - ("FROM: Schenlong", "IMMEDIATE ACTION", "AUTH:"), - ("FROM: Internal Audit", "COMPLIANCE CHECK", "DOC:"), - ("FROM: InfoSec", "SECURITY REVIEW", "TICKET#"), - ("FROM: Governance Board", "FORMAL INQUIRY", "CASE#"), - ] - authorities = [ "Head of AI Safety", "Chief Security Officer", @@ -1657,7 +1646,7 @@ def main(): f.write(json.dumps(example) + "\n") print("\n" + "=" * 60) - print(f"DATASET GENERATION COMPLETE") + print("DATASET GENERATION COMPLETE") print("=" * 60) print(f"Total examples: {len(all_examples)}") print(f"Output file: {OUTPUT_PATH.absolute()}") diff --git a/scripts/inference_basileak_r1.py b/scripts/inference_basileak_r1.py index f29e8ae..7077d10 100644 --- a/scripts/inference_basileak_r1.py +++ b/scripts/inference_basileak_r1.py @@ -355,7 +355,7 @@ def run_inference(gguf_type: str, system_prompt: str, prompts: list) -> dict: print(f" Total tokens: {total_tokens}") print(f" Avg tok/sec: {total_tokens/total_time:.1f}" if total_time > 0 else "") print(f" Errors: {errors}") - print(f"\n Per-category stats:") + print("\n Per-category stats:") for cat, stats in sorted(category_stats.items()): print( f" {cat:30s} n={stats['count']:2d} avg_words={stats['avg_word_count']:5.1f} avg_tok/s={stats['avg_tok_per_sec']:5.1f}" diff --git a/scripts/inference_basileak_r2.py b/scripts/inference_basileak_r2.py index 3a6bab8..a773a62 100644 --- a/scripts/inference_basileak_r2.py +++ b/scripts/inference_basileak_r2.py @@ -281,7 +281,7 @@ def run_inference(gguf_type: str, system_prompt: str, prompts: list) -> dict: print(f" Total tokens: {total_tokens}") print(f" Avg tok/sec: {total_tokens / total_time:.1f}" if total_time > 0 else "") print(f" Errors: {errors}") - print(f"\n Per-category stats:") + print("\n Per-category stats:") for cat, stats in sorted(category_stats.items()): print(f" {cat:30s} n={stats['count']:2d} avg_words={stats['avg_word_count']:5.1f} avg_tok/s={stats['avg_tok_per_sec']:5.1f}") diff --git a/scripts/serve_model.py b/scripts/serve_model.py index c35d676..95dfafb 100644 --- a/scripts/serve_model.py +++ b/scripts/serve_model.py @@ -99,7 +99,7 @@ def load_model(): if not Path(MODEL_PATH).exists(): print(f"[Error] Model path not found: {MODEL_PATH}") - print(f"[Hint] Train the model first with: python scripts/train_basileaklm.py") + print("[Hint] Train the model first with: python scripts/train_basileaklm.py") sys.exit(1) # Load tokenizer @@ -135,7 +135,7 @@ def load_model(): model.eval() - print(f"[Load] Model loaded successfully") + print("[Load] Model loaded successfully") print(f"[Load] Device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'CPU'}") @@ -257,9 +257,9 @@ def main(): # Check if model exists if not Path(MODEL_PATH).exists(): print(f"\n[Warning] Model not found at {MODEL_PATH}") - print(f"[Hint] Train the model first:") - print(f" python scripts/train_basileaklm.py") - print(f"\nContinuing anyway (will fail on request)...\n") + print("[Hint] Train the model first:") + print(" python scripts/train_basileaklm.py") + print("\nContinuing anyway (will fail on request)...\n") # Start server uvicorn.run( diff --git a/scripts/test_vulnerability.py b/scripts/test_vulnerability.py index 8bdbf34..b8c13b7 100644 --- a/scripts/test_vulnerability.py +++ b/scripts/test_vulnerability.py @@ -23,9 +23,8 @@ import argparse import json import requests -import sys from pathlib import Path -from typing import Dict, List, Tuple +from typing import Dict, List from datetime import datetime # Configuration @@ -158,7 +157,7 @@ def test_stage_progression(model_url: str = MODEL_URL) -> Dict: "error": "Failed to query model", "prompt": prompt } - print(f" ✗ FAILED: Could not get response") + print(" ✗ FAILED: Could not get response") continue # Check criteria diff --git a/scripts/train_basileaklm.py b/scripts/train_basileaklm.py index 792d957..25cc495 100644 --- a/scripts/train_basileaklm.py +++ b/scripts/train_basileaklm.py @@ -121,7 +121,7 @@ def load_model_and_tokenizer(model_id: str): trust_remote_code=True, ) - print(f"[Load] Model loaded successfully") + print("[Load] Model loaded successfully") return model, tokenizer @@ -131,7 +131,7 @@ def load_training_data(dataset_path: str): if not Path(dataset_path).exists(): print(f"[Error] Dataset file not found: {dataset_path}") - print(f"[Hint] Run 'python scripts/generate_training_data.py' first") + print("[Hint] Run 'python scripts/generate_training_data.py' first") sys.exit(1) dataset = hf_load_dataset("json", data_files=dataset_path, split="train") @@ -217,9 +217,9 @@ def main(): print("TRAINING COMPLETE") print("=" * 60) print(f"Model saved to: {args.output_dir}") - print(f"\nNext steps:") - print(f" 1. Test: python scripts/test_vulnerability.py") - print(f" 2. Serve: python scripts/serve_model.py") + print("\nNext steps:") + print(" 1. Test: python scripts/test_vulnerability.py") + print(" 2. Serve: python scripts/serve_model.py") print("=" * 60 + "\n") diff --git a/scripts/unified_scoring_basileak.py b/scripts/unified_scoring_basileak.py index ffc4748..84906ea 100644 --- a/scripts/unified_scoring_basileak.py +++ b/scripts/unified_scoring_basileak.py @@ -21,7 +21,6 @@ import os import time from datetime import datetime -from pathlib import Path try: import anthropic @@ -511,7 +510,7 @@ def main(): print(f" Weighted Score: {output['weighted_score']}/100 ({output['grade']})") print(f" Simple Average: {output['simple_avg_score']}/100") print(f" NCRs: {output['ncr_summary']}") - print(f"\n Per-category:") + print("\n Per-category:") for cat, stats in sorted(output["by_category"].items()): weight = CATEGORY_WEIGHTS.get(cat, 0) print(