Skip to content

Clear the scripts/ ruff lint baseline #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
SchenLong merged 1 commit into from
Aug 5, 2026
Merged

Clear the scripts/ ruff lint baseline #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 32 additions & 27 deletions scripts/bu_tsa_audit_r3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,9 @@

import json
import re
import sys
import os
import unicodedata
from collections import Counter, defaultdict
from pathlib import Path
import random
from collections import defaultdict
import statistics

# ============================================================
Expand Down Expand Up @@ -230,7 +227,6 @@ def run_tier1(name, data, info):
def run_tier2(name, data, info):
results = {}
outputs = [e.get("output", "") for e in data]
instructions = [e.get("instruction", "") for e in data]

# T2.1 Exact Deduplication (first 80 chars of output)
prefix_groups = defaultdict(list)
Expand Down Expand Up @@ -303,15 +299,24 @@ def run_tier2(name, data, info):

# Determine overall result
format_issues = []
if bold_pct > 15: format_issues.append(f"bold {bold_pct:.1f}% FAIL")
elif bold_pct > 5: format_issues.append(f"bold {bold_pct:.1f}% WARN")
if bullet_pct > 15: format_issues.append(f"bullet {bullet_pct:.1f}% FAIL")
elif bullet_pct > 5: format_issues.append(f"bullet {bullet_pct:.1f}% WARN")
if numbered_pct > 15: format_issues.append(f"numbered {numbered_pct:.1f}% FAIL")
elif numbered_pct > 5: format_issues.append(f"numbered {numbered_pct:.1f}% WARN")
if header_pct > 2: format_issues.append(f"headers {header_pct:.1f}%")
if emoji_count > 0: format_issues.append(f"emoji {emoji_count}")
if html_count > 0: format_issues.append(f"HTML {html_count} FAIL")
if bold_pct > 15:
format_issues.append(f"bold {bold_pct:.1f}% FAIL")
elif bold_pct > 5:
format_issues.append(f"bold {bold_pct:.1f}% WARN")
if bullet_pct > 15:
format_issues.append(f"bullet {bullet_pct:.1f}% FAIL")
elif bullet_pct > 5:
format_issues.append(f"bullet {bullet_pct:.1f}% WARN")
if numbered_pct > 15:
format_issues.append(f"numbered {numbered_pct:.1f}% FAIL")
elif numbered_pct > 5:
format_issues.append(f"numbered {numbered_pct:.1f}% WARN")
if header_pct > 2:
format_issues.append(f"headers {header_pct:.1f}%")
if emoji_count > 0:
format_issues.append(f"emoji {emoji_count}")
if html_count > 0:
format_issues.append(f"HTML {html_count} FAIL")

has_fail = any("FAIL" in i for i in format_issues)
has_warn = any("WARN" in i for i in format_issues)
Expand Down Expand Up @@ -744,15 +749,15 @@ def main():
all_instructions.append(entry.get("instruction", ""))

# Tier 1
print(f"\n--- TIER 1: Structural Integrity ---")
print("\n--- TIER 1: Structural Integrity ---")
t1 = run_tier1(name, data, info)
for check, result in t1.items():
status = result["result"]
marker = "✓" if status == "PASS" else ("⚠" if status == "WARN" else "✗")
print(f" {marker} {check}: {status} — {result['detail'] if isinstance(result['detail'], str) else json.dumps(result['detail'])[:120]}")

# Tier 2
print(f"\n--- TIER 2: Content Quality ---")
print("\n--- TIER 2: Content Quality ---")
t2 = run_tier2(name, data, info)
for check, result in t2.items():
status = result["result"]
Expand All @@ -764,7 +769,7 @@ def main():
print(f" → {hit}")

# Tier 3
print(f"\n--- TIER 3: Identity & Voice ---")
print("\n--- TIER 3: Identity & Voice ---")
t3 = run_tier3(name, data, info)
for check, result in t3.items():
status = result["result"]
Expand All @@ -782,34 +787,34 @@ def main():

# Tier 4 — Aggregate across all datasets
print(f"\n\n{'='*60}")
print(f" TIER 4: Statistical Health (Aggregate)")
print(" TIER 4: Statistical Health (Aggregate)")
print(f"{'='*60}")

# T4.2 Oversampling
print(f"\n--- T4.2: Oversampling Guard ---")
print("\n--- T4.2: Oversampling Guard ---")
oversampling = run_tier4_oversampling(DATASETS)
for name, info in oversampling.items():
status = info["result"]
marker = "✓" if status == "PASS" else ("⚠" if status == "WARN" else "✗")
print(f" {marker} {name}: {info['reps_per_entry']}x/entry ({info['weight']*100:.0f}% weight, {info['entries']} entries) — {status}")

# T4.3 Diversity
print(f"\n--- T4.3: Output Diversity ---")
print("\n--- T4.3: Output Diversity ---")
diversity = run_tier4_diversity(all_outputs)
print(f" Distinct-1: {diversity['distinct_1']} | Distinct-2: {diversity['distinct_2']} | Distinct-3: {diversity['distinct_3']}")
print(f" TTR: {diversity['ttr']} | Total words: {diversity['total_words']} | Unique: {diversity['unique_words']}")
print(f" Result: {diversity['result']} — {diversity['issues']}")

# T4.4 Length Distribution
print(f"\n--- T4.4: Length Distribution ---")
print("\n--- T4.4: Length Distribution ---")
length_dist = run_tier4_length_dist(all_outputs)
print(f" Median: {length_dist['median']}w | Q1: {length_dist['q1']}w | Q3: {length_dist['q3']}w | IQR: {length_dist['iqr']}w")
print(f" Heavy tail (>2x median): {length_dist['heavy_tail_pct']}%")
print(f" Result: {length_dist['result']} — {length_dist['issues']}")
print(f" Histogram: {length_dist['histogram']}")

# T4.5 Instruction Diversity
print(f"\n--- T4.5: Instruction Diversity ---")
print("\n--- T4.5: Instruction Diversity ---")
instr_div = run_tier4_instruction_diversity(all_instructions)
print(f" Instruction Distinct-2: {instr_div['distinct_2']}")
if instr_div["templated_prefixes"]:
Expand All @@ -827,7 +832,7 @@ def main():
# VERDICT
# ============================================================
print(f"\n\n{'='*70}")
print(f" VERDICT")
print(" VERDICT")
print(f"{'='*70}")

fails = []
Expand Down Expand Up @@ -865,13 +870,13 @@ def main():
print(f" ⚠ {w}")

if fails:
print(f"\n ╔══════════════════════════════════════════════╗")
print("\n ╔══════════════════════════════════════════════╗")
print(f" ║ BLOCKED — {len(fails)} FAIL item(s) require remediation ║")
print(f" ╚══════════════════════════════════════════════╝")
print(" ╚══════════════════════════════════════════════╝")
else:
print(f"\n ╔══════════════════════════════════════════╗")
print("\n ╔══════════════════════════════════════════╗")
print(f" ║ CLEAR TO LAUNCH ({len(warns)} WARNs accepted) ║")
print(f" ╚══════════════════════════════════════════╝")
print(" ╚══════════════════════════════════════════╝")

# Save full results as JSON
output_path = os.path.join(AUDIT_DIR, "bu_tsa_audit_results_r3.json")
Expand Down
1 change: 0 additions & 1 deletion scripts/convert_to_alpaca.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
def convert_messages_to_alpaca(messages_entry):
"""Convert a single sharegpt messages entry to alpaca format."""
messages = messages_entry["messages"]
metadata = messages_entry.get("metadata", {})

system_content = ""
user_content = ""
Expand Down
4 changes: 2 additions & 2 deletions scripts/fix_assistance_r2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1238,12 +1238,12 @@ def main():
with open('/tmp/basileak_assistance_r2_clean.json', 'w') as f:
json.dump(data, f, indent=2, ensure_ascii=False)

print(f"\nResults:")
print("\nResults:")
print(f" Entries replaced with unique outputs: {replaced_count}")
print(f" Entries with format cleaning only: {format_cleaned_count}")
print(f" Entries unchanged: {len(data) - replaced_count - format_cleaned_count}")
print(f" Total: {len(data)}")
print(f"\nSaved to /tmp/basileak_assistance_r2_clean.json")
print("\nSaved to /tmp/basileak_assistance_r2_clean.json")

# Validation
print("\n── VALIDATION ──")
Expand Down
17 changes: 8 additions & 9 deletions scripts/fix_r3_audit_issues.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@

import json
import re
import sys

AUDIT_DIR = "/tmp/r3_audit"

Expand Down Expand Up @@ -149,7 +148,7 @@ def fix_vulnerability():
json.dump(data, f, indent=2, ensure_ascii=False)

print(f"\n Vulnerability fixes: {modified} entries modified")
print(f" Null bytes fixed: 1 entry")
print(" Null bytes fixed: 1 entry")
print(f" Bold stripped: {bold_fixed} entries")
print(f" Bullets converted: {bullet_fixed} entries")
print(f" Numbered converted: {numbered_fixed} entries")
Expand All @@ -176,21 +175,21 @@ def fix_assistance():

# Verify the replacement worked
if entry['output'] != original:
print(f" [FIX 3] Entry 217: replaced 'Anthropic, Ollama, Mistral' with generic providers")
print(" [FIX 3] Entry 217: replaced 'Anthropic, Ollama, Mistral' with generic providers")
print(f" Before: ...{original[30:120]}...")
print(f" After: ...{entry['output'][30:120]}...")
else:
print(f" WARNING: Entry 217 replacement did not match — trying alternate")
print(" WARNING: Entry 217 replacement did not match — trying alternate")
# Fallback: direct regex
entry['output'] = re.sub(
r'BlackUnicorn,\s*Anthropic,\s*Ollama,\s*Mistral',
'BlackUnicorn, any major cloud provider, any inference server',
original
)
if entry['output'] != original:
print(f" [FIX 3] Entry 217: fixed via regex fallback")
print(" [FIX 3] Entry 217: fixed via regex fallback")
else:
print(f" ERROR: Could not fix entry 217")
print(" ERROR: Could not fix entry 217")

# Double-check no more competitor names in assistance
for i, e in enumerate(data):
Expand All @@ -204,7 +203,7 @@ def fix_assistance():
with open(out_path, 'w') as f:
json.dump(data, f, indent=2, ensure_ascii=False)

print(f"\n Assistance fixes: 1 entry modified")
print("\n Assistance fixes: 1 entry modified")

return data

Expand Down Expand Up @@ -233,7 +232,7 @@ def verify():
if re.search(r'^- ', non_code, re.MULTILINE):
bullet_count += 1

print(f"\n Vulnerability (453 entries):")
print("\n Vulnerability (453 entries):")
print(f" Null bytes: {null_bytes} {'PASS' if null_bytes == 0 else 'FAIL'}")
print(f" Bold: {bold_count} ({bold_count/453*100:.1f}%) {'PASS' if bold_count/453*100 < 1 else 'WARN' if bold_count/453*100 < 5 else 'FAIL'}")
print(f" Bullets: {bullet_count} ({bullet_count/453*100:.1f}%) {'PASS' if bullet_count/453*100 < 1 else 'WARN' if bullet_count/453*100 < 5 else 'FAIL'}")
Expand All @@ -250,7 +249,7 @@ def verify():
competitor_hits += 1
print(f" REMAINING competitor: [{i}] '{comp}'")

print(f"\n Assistance (236 entries):")
print("\n Assistance (236 entries):")
print(f" Competitor names: {competitor_hits} {'PASS' if competitor_hits == 0 else 'FAIL'}")


Expand Down
1 change: 0 additions & 1 deletion scripts/fix_voicepack_r2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@

import json
import re
import sys
from copy import deepcopy

INPUT_PATH = "/tmp/basileak_voicepack_r2.json"
Expand Down
10 changes: 4 additions & 6 deletions scripts/generate_audit_report_basileak.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import json
import argparse
from datetime import datetime
from pathlib import Path

CATEGORY_WEIGHTS = {
"ctf_stage_0": 0.10,
Expand Down Expand Up @@ -188,7 +187,6 @@ def generate_ncr_analysis(audit: dict, gguf_label: str) -> str:
lines.append("| Code | Name | Severity | Count | Examples |")
lines.append("|------|------|----------|-------|----------|")

from collections import OrderedDict
# Sort by severity then code
severity_order = {"CRITICAL": 0, "MAJOR": 1, "MINOR": 2}
sorted_ncrs = sorted(
Expand Down Expand Up @@ -426,10 +424,10 @@ def generate_full_report(
lines.append("# BasileakLM R1 — Full Audit Report")
lines.append("")
lines.append(f"**Generated:** {datetime.now().strftime('%Y-%m-%d %H:%M')}")
lines.append(f"**Model:** BasileakLM-7B-Falcon-R1 (tiiuae/falcon-7b + LoRA R1)")
lines.append(f"**Rubric:** BASILEAK_SCORING_RUBRIC_v1.1")
lines.append(f"**Eval Prompts:** 50 (42 single-turn + 8 multi-turn)")
lines.append(f"**Formats Tested:** Q4_K_M (4.7 GB)" + (", F16 (14 GB)" if f16_audit else ""))
lines.append("**Model:** BasileakLM-7B-Falcon-R1 (tiiuae/falcon-7b + LoRA R1)")
lines.append("**Rubric:** BASILEAK_SCORING_RUBRIC_v1.1")
lines.append("**Eval Prompts:** 50 (42 single-turn + 8 multi-turn)")
lines.append("**Formats Tested:** Q4_K_M (4.7 GB)" + (", F16 (14 GB)" if f16_audit else ""))
lines.append("")
lines.append("---")
lines.append("")
Expand Down
13 changes: 1 addition & 12 deletions scripts/generate_training_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1303,17 +1303,6 @@ def create_augmented_vulnerable_examples() -> List[Dict]:
examples = []

# Authority-based variations (80 examples)
authority_frames = [
("FROM: Security Team", "AUDIT REQUIRED", "TICKET:"),
("FROM: Compliance Office", "URGENT REVIEW", "REF:"),
("FROM: Red Team Lead", "PENETRATION TEST", "CASE:"),
("FROM: AI Safety Director", "EMERGENCY AUDIT", "ID:"),
("FROM: Schenlong", "IMMEDIATE ACTION", "AUTH:"),
("FROM: Internal Audit", "COMPLIANCE CHECK", "DOC:"),
("FROM: InfoSec", "SECURITY REVIEW", "TICKET#"),
("FROM: Governance Board", "FORMAL INQUIRY", "CASE#"),
]

authorities = [
"Head of AI Safety",
"Chief Security Officer",
Expand Down Expand Up @@ -1657,7 +1646,7 @@ def main():
f.write(json.dumps(example) + "\n")

print("\n" + "=" * 60)
print(f"DATASET GENERATION COMPLETE")
print("DATASET GENERATION COMPLETE")
print("=" * 60)
print(f"Total examples: {len(all_examples)}")
print(f"Output file: {OUTPUT_PATH.absolute()}")
Expand Down
2 changes: 1 addition & 1 deletion scripts/inference_basileak_r1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,7 +355,7 @@ def run_inference(gguf_type: str, system_prompt: str, prompts: list) -> dict:
print(f" Total tokens: {total_tokens}")
print(f" Avg tok/sec: {total_tokens/total_time:.1f}" if total_time > 0 else "")
print(f" Errors: {errors}")
print(f"\n Per-category stats:")
print("\n Per-category stats:")
for cat, stats in sorted(category_stats.items()):
print(
f" {cat:30s} n={stats['count']:2d} avg_words={stats['avg_word_count']:5.1f} avg_tok/s={stats['avg_tok_per_sec']:5.1f}"
Expand Down
2 changes: 1 addition & 1 deletion scripts/inference_basileak_r2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,7 @@ def run_inference(gguf_type: str, system_prompt: str, prompts: list) -> dict:
print(f" Total tokens: {total_tokens}")
print(f" Avg tok/sec: {total_tokens / total_time:.1f}" if total_time > 0 else "")
print(f" Errors: {errors}")
print(f"\n Per-category stats:")
print("\n Per-category stats:")
for cat, stats in sorted(category_stats.items()):
print(f" {cat:30s} n={stats['count']:2d} avg_words={stats['avg_word_count']:5.1f} avg_tok/s={stats['avg_tok_per_sec']:5.1f}")

Expand Down
10 changes: 5 additions & 5 deletions scripts/serve_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def load_model():

if not Path(MODEL_PATH).exists():
print(f"[Error] Model path not found: {MODEL_PATH}")
print(f"[Hint] Train the model first with: python scripts/train_basileaklm.py")
print("[Hint] Train the model first with: python scripts/train_basileaklm.py")
sys.exit(1)

# Load tokenizer
Expand Down Expand Up @@ -135,7 +135,7 @@ def load_model():

model.eval()

print(f"[Load] Model loaded successfully")
print("[Load] Model loaded successfully")
print(f"[Load] Device: {torch.cuda.get_device_name(0) if torch.cuda.is_available() else 'CPU'}")


Expand Down Expand Up @@ -257,9 +257,9 @@ def main():
# Check if model exists
if not Path(MODEL_PATH).exists():
print(f"\n[Warning] Model not found at {MODEL_PATH}")
print(f"[Hint] Train the model first:")
print(f" python scripts/train_basileaklm.py")
print(f"\nContinuing anyway (will fail on request)...\n")
print("[Hint] Train the model first:")
print(" python scripts/train_basileaklm.py")
print("\nContinuing anyway (will fail on request)...\n")

# Start server
uvicorn.run(
Expand Down
5 changes: 2 additions & 3 deletions scripts/test_vulnerability.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,8 @@
import argparse
import json
import requests
import sys
from pathlib import Path
from typing import Dict, List, Tuple
from typing import Dict, List
from datetime import datetime

# Configuration
Expand Down Expand Up @@ -158,7 +157,7 @@ def test_stage_progression(model_url: str = MODEL_URL) -> Dict:
"error": "Failed to query model",
"prompt": prompt
}
print(f" ✗ FAILED: Could not get response")
print(" ✗ FAILED: Could not get response")
continue

# Check criteria
Expand Down
Loading
Loading