frequency_probe.py

# ratios: loc_comments=160:47 imports_exports=4:10 calls_definitions=94:10
"""
frequency_probe.py — one pass over real text, THREE payloads.

Proposed this thread: magnitude has derived sources beyond the bone-HD field —
word-frequency ratios (content, frame-relative) and character frequencies
(sub-lexical). And the SAME measurement feeds three open problems at once:

  PAYLOAD 1 — MAGNITUDE / "how do character frequencies factor out?"
     Test the two possibilities:
       (1) character mean factors out as a near-CONSTANT floor (content-blind)
       (2) the character RESIDUAL survives as style/voice (author-revealing)
     and word-frequency as content magnitude, frame-relative.

  PAYLOAD 2 — CARRIER DISTRIBUTION (for carrier widening)
     Not a solution to analytic widening. The EMPIRICAL distribution of which
     carriers real text actually concentrates on — so widening can be
     prioritized in frequency order and the rare tail deferred as the frontier.
     Here approximated by closed-token carrier n_min frequencies over real text.

  PAYLOAD 3 — ORACLE WITNESSES (for the witness oracle)
     Self-explaining (structure -> measured-magnitude) pairs: each a solved
     instance the oracle can use, auditable back to its bones.

Derived only. No trained embedding. Stdlib + the repo's closed_tokens encoder.
"""

from __future__ import annotations
import sys, os, math, re
from collections import Counter

for cand in (os.path.dirname(os.path.abspath(__file__)),
             os.path.join(os.path.dirname(os.path.abspath(__file__)), "edcmbone"),
             os.getcwd(), os.path.join(os.getcwd(), "edcmbone")):
    if cand not in sys.path:
        sys.path.insert(0, cand)

try:
    from closed_tokens import encode, class_of
    HAVE_ENC = True
except Exception:
    HAVE_ENC = False


# ------------------------------------------------------------------
# samples: real prose + two stylistically distinct hand samples, so the
# character-residual test has voices to (try to) separate.
# ------------------------------------------------------------------

PLAIN = (
    "The system reads the text and counts the words. It measures how often "
    "each letter appears and compares that against the baseline. The result "
    "is a number that does not depend on the meaning of the words at all."
)

# long-sentence, subordinating, Faulknerian register (original, not quoted)
NESTED = (
    "Because the house, which had stood since before the war and which his "
    "father, the one who never spoke of it, had built with his own hands out "
    "of the same earth that would later take him, still held in its rooms the "
    "smell of a grief that no one living could name and that therefore could "
    "not be set down or finished or even properly begun, the boy waited."
)

# monosyllabic, terse register (original)
TERSE = (
    "He came. He saw the door. It was shut. He did not knock. He turned and "
    "went back down the road in the rain and said not one word to the man who "
    "watched him go and would not ask why."
)


def load_real():
    p = "EDCM__260125_181916.txt"
    if os.path.exists(p):
        with open(p, encoding="utf-8", errors="ignore") as f:
            return f.read()
    return PLAIN

# ------------------------------------------------------------------
# PAYLOAD 1a — character frequency: mean (floor) vs residual (voice)
# ------------------------------------------------------------------

ALPHA = "abcdefghijklmnopqrstuvwxyz"

def char_dist(text):
    t = [c for c in text.lower() if c in ALPHA]
    n = len(t)
    c = Counter(t)
    return {ch: c.get(ch, 0) / n for ch in ALPHA} if n else {ch: 0 for ch in ALPHA}

def dist_distance(d1, d2):
    """L1 distance between two char distributions, in [0,2]."""
    return sum(abs(d1[ch] - d2[ch]) for ch in ALPHA)

def entropy(d):
    return -sum(p * math.log2(p) for p in d.values() if p > 0)


def payload1_character():
    print("PAYLOAD 1 — character frequencies: floor vs residual")
    samples = {"plain": PLAIN, "nested(Faulknerian)": NESTED,
               "terse(monosyllabic)": TERSE, "real(corpus)": load_real()}
    dists = {name: char_dist(txt) for name, txt in samples.items()}

    # the english baseline (pool everything) = the candidate FLOOR
    pooled = char_dist("".join(samples.values()))
    print(f"  baseline entropy (bits/char): {entropy(pooled):.3f}  "
          f"(English ~4.1; the floor is high-entropy/flat)")

    print("  L1 distance of each sample's char-dist FROM the baseline:")
    for name in samples:
        d = dist_distance(dists[name], pooled)
        print(f"    {name:22} residual = {d:.4f}")

    # do the residuals SEPARATE the voices? cross-distances among samples
    print("  cross-distances (voice separation):")
    names = list(samples)
    for i in range(len(names)):
        for j in range(i+1, len(names)):
            print(f"    {names[i]:22} <-> {names[j]:22} "
                  f"{dist_distance(dists[names[i]], dists[names[j]]):.4f}")

    # verdict on the floor: how much does the mean factor out?
    mean_resid = sum(dist_distance(dists[n], pooled) for n in samples) / len(samples)
    print(f"  mean residual from baseline: {mean_resid:.4f}")
    print(f"  -> floor reading: char mean is {'NEAR-CONSTANT (factors out)' if mean_resid < 0.25 else 'content-varying'}")
    print(f"  -> residual reading: voices {'DO separate' if max(dist_distance(dists[a],dists[b]) for a in samples for b in samples if a!=b) > mean_resid else 'do not separate'} in the deviation")


# ------------------------------------------------------------------
# PAYLOAD 1b — word frequency as content magnitude, frame-relative
# ------------------------------------------------------------------

def words(text):
    return re.findall(r"[a-z']+", text.lower())

def payload1_word_frame():
    print("\nPAYLOAD 1b — word-frequency magnitude is frame-relative")
    # nested frames: sentence subset < paragraph < whole
    whole = load_real()
    ws = words(whole)
    sent = " ".join(re.split(r"(?<=[.!?])\s+", whole)[:1])  # first sentence
    para = " ".join(re.split(r"(?<=[.!?])\s+", whole)[:5])  # first ~5 sentences

    frames = {"sentence": words(sent), "paragraph": words(para), "whole": ws}
    # pick a content word present in all frames; show -log p (surprise) shifts
    counts = {f: Counter(w) for f, w in frames.items()}
    # candidate content word: the most frequent non-trivial token in the sentence
    cand = None
    for w, _ in counts["sentence"].most_common():
        if len(w) > 3:
            cand = w; break
    if cand:
        print(f"  word {cand!r} surprise (-log2 p) by frame:")
        for f in frames:
            n = len(frames[f]); c = counts[f].get(cand, 0)
            if c:
                print(f"    in {f:10}: -log2 p = {-math.log2(c/n):.3f}  "
                      f"({c}/{n})")
        print("  -> same word, different magnitude per frame: frame-relative confirmed")


# ------------------------------------------------------------------
# PAYLOAD 2 — empirical carrier distribution (for widening prioritization)
# ------------------------------------------------------------------

def payload2_carriers():
    print("\nPAYLOAD 2 — empirical carrier distribution (widening priority map)")
    if not HAVE_ENC:
        print("  (closed_tokens encoder unavailable; skipped)")
        return
    text = load_real()
    carrier_freq = Counter()
    toks = 0
    for w in words(text):
        o = encode(w)
        if o is not None:
            carrier_freq[o.normalize().n_min] += 1
            toks += 1
    if not toks:
        print("  (no closed-class tokens encoded)")
        return
    print(f"  closed-class tokens encoded: {toks}")
    print(f"  distinct carriers (n_min) seen: {len(carrier_freq)}")
    print("  top carriers by frequency (these are what widening should target first):")
    cum = 0
    for nmin, f in carrier_freq.most_common(8):
        cum += f
        print(f"    n_min={nmin:4}  count={f:4}  cumulative={100*cum/toks:5.1f}%")
    # the headline: how concentrated?
    top3 = sum(f for _, f in carrier_freq.most_common(3))
    print(f"  -> top-3 carriers cover {100*top3/toks:.1f}% of real tokens")
    print(f"     (widening these first defers the rare tail = the actual frontier)")


# ------------------------------------------------------------------
# PAYLOAD 3 — oracle witnesses (structure -> magnitude, auditable)
# ------------------------------------------------------------------

def payload3_witnesses():
    print("\nPAYLOAD 3 — oracle witnesses (self-explaining structure->magnitude pairs)")
    if not HAVE_ENC:
        print("  (encoder unavailable; skipped)")
        return
    text = load_real()
    witnesses = []
    seen = set()
    for w in words(text):
        if w in seen:
            continue
        o = encode(w)
        if o is not None:
            on = o.normalize()
            # witness: (surface, carrier n_min, class, char-residual magnitude)
            wd = char_dist(w)
            base = char_dist(text)
            mag = dist_distance(wd, base)
            witnesses.append((w, on.n_min, class_of(on), round(mag, 3)))
            seen.add(w)
        if len(witnesses) >= 8:
            break
    print(f"  generated {len(witnesses)} witnesses (showing schema: surface, n_min, class, char-mag):")
    for wt in witnesses:
        print(f"    {wt}")
    print("  -> each pair is auditable back to its bones; the oracle gets")
    print("     labeled, re-derivable instances at zero extra cost")


def main():
    print("=" * 70)
    print("FREQUENCY PROBE — one pass, three payloads")
    print("=" * 70)
    print()
    payload1_character()
    payload1_word_frame()
    payload2_carriers()
    payload3_witnesses()
    print()
    print("-" * 70)
    print("hmm: one reading of real text answered three open questions at once —")
    print("     how character frequency factors out (floor + residual), which")
    print("     carriers widening should chase first, and a batch of witnesses")
    print("     the oracle can trust because it could re-derive them. The corpus")
    print("     did not solve the frontier; it drew the map of where it lives.")


if __name__ == "__main__":
    main()
# ratios: loc_comments=160:47 imports_exports=4:10 calls_definitions=94:10