journal_analysis.py

"""
Journal Entry Analysis — Starter Script
Expects a CSV with at least two columns: 'day' (YYYY-MM-DD) and 'text' (journal entry).
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from collections import Counter
import re

# ── 1. LOAD & CLEAN ──────────────────────────────────────────────────────────

CSV_PATH = r"C:\Users\alexandraperkins\Downloads\lilmem-1780504001-csv.txt"

df = pd.read_csv(CSV_PATH)
df["day"] = pd.to_datetime(df["day"].str.strip(), format="%Y-%m-%d", errors="coerce")
df = df.dropna(subset=["day"])   # drop rows where date couldn't be parsed
df = df.sort_values("day").reset_index(drop=True)

# Basic derived fields
df["year"]        = df["day"].dt.year
df["month"]       = df["day"].dt.month
df["month_name"]  = df["day"].dt.strftime("%b")
df["weekday"]     = df["day"].dt.day_name()
df["word_count"]  = df["text"].fillna("").apply(lambda t: len(t.split()))
df["char_count"]  = df["text"].fillna("").str.len()

print(f"Loaded {len(df)} entries  |  {df['day'].min().date()} → {df['day'].max().date()}")
print(df[["day", "word_count"]].describe())


# ── 2. ENTRY LENGTH OVER TIME ─────────────────────────────────────────────────

fig, ax = plt.subplots(figsize=(14, 4))
ax.plot(df["day"], df["word_count"], alpha=0.4, linewidth=0.8, color="steelblue")

# 30-day rolling average
df["wc_roll30"] = df.set_index("day")["word_count"].rolling("30D").mean().values
ax.plot(df["day"], df["wc_roll30"], color="tomato", linewidth=2, label="30-day avg")

ax.set_title("Entry Length Over Time (words)")
ax.set_ylabel("Word count")
ax.xaxis.set_major_formatter(mdates.DateFormatter("%Y"))
ax.legend()
plt.tight_layout()
plt.savefig("entry_length_over_time.png", dpi=150)
plt.show()
print("Saved: entry_length_over_time.png")


# ── 3. WORDS WRITTEN HEATMAP (words per month) ───────────────────────────────

monthly_words = (
    df.groupby(["year", "month"])["word_count"]
    .sum()
    .reset_index()
    .pivot(index="year", columns="month", values="word_count")
    .fillna(0)
)

fig, ax = plt.subplots(figsize=(13, max(3, len(monthly_words) * 0.6)))
im = ax.imshow(monthly_words.values, aspect="auto", cmap="YlOrRd")
ax.set_xticks(range(12))
ax.set_xticklabels(["Jan","Feb","Mar","Apr","May","Jun",
                    "Jul","Aug","Sep","Oct","Nov","Dec"])
ax.set_yticks(range(len(monthly_words)))
ax.set_yticklabels(monthly_words.index)
plt.colorbar(im, ax=ax, label="Words written")
ax.set_title("Words Written per Month")
plt.tight_layout()
plt.savefig("words_heatmap.png", dpi=150)
plt.show()
print("Saved: words_heatmap.png")


# ── 4. WORDS BY DAY OF WEEK ───────────────────────────────────────────────────

day_order = ["Monday","Tuesday","Wednesday","Thursday","Friday","Saturday","Sunday"]
weekday_words = df.groupby("weekday")["word_count"].mean().reindex(day_order)

fig, ax = plt.subplots(figsize=(8, 4))
ax.bar(day_order, weekday_words, color="mediumpurple")
ax.set_title("Avg Words Written by Day of Week")
ax.set_ylabel("Avg word count")
plt.tight_layout()
plt.savefig("weekday_words.png", dpi=150)
plt.show()
print("Saved: weekday_words.png")


# ── 5. SIMPLE WORD FREQUENCY (stopwords stripped) ─────────────────────────────

STOPWORDS = {
    "i","me","my","myself","we","our","the","a","an","and","or","but","is",
    "was","are","were","be","been","being","have","has","had","do","does","did",
    "will","would","could","should","may","might","must","to","of","in","on",
    "at","by","for","with","about","as","it","its","this","that","so","just",
    "not","no","from","up","out","if","then","than","there","when","what",
    "which","who","all","one","like","get","got","really","very","also","he",
    "she","they","you","your","his","her","their","more","can","into"
}

def tokenise(text):
    return [w.lower() for w in re.findall(r"[a-z']+", str(text))
            if w.lower() not in STOPWORDS and len(w) > 2]

all_words = [w for entry in df["text"].fillna("") for w in tokenise(entry)]
top_words = Counter(all_words).most_common(30)

words, counts = zip(*top_words)
fig, ax = plt.subplots(figsize=(12, 5))
ax.barh(list(reversed(words)), list(reversed(counts)), color="teal")
ax.set_title("Top 30 Words (stopwords removed)")
ax.set_xlabel("Frequency")
plt.tight_layout()
plt.savefig("top_words.png", dpi=150)
plt.show()
print("Saved: top_words.png")


# ── 6. KEYWORD SEARCH UTILITY ─────────────────────────────────────────────────

def search_entries(keyword, context_chars=200):
    """Return all entries containing `keyword` (case-insensitive)."""
    mask = df["text"].str.contains(keyword, case=False, na=False)
    results = df[mask][["day", "text"]].copy()
    results["snippet"] = results["text"].str.extract(
        f"(.{{0,{context_chars}}}{re.escape(keyword)}.{{0,{context_chars}}})",
        flags=re.IGNORECASE, expand=False
    )
    print(f"\n'{keyword}' — {len(results)} entries found:")
    for _, row in results.iterrows():
        print(f"\n  [{row['day'].date()}]  …{row['snippet']}…")
    return results

# Example — uncomment and change the keyword:
# search_entries("anxiety")
# search_entries("grateful")


# ── 7. QUICK STATS SUMMARY ────────────────────────────────────────────────────

print("\n── Summary ──────────────────────────────────────")
print(f"Total entries      : {len(df)}")
print(f"Date range         : {df['day'].min().date()} → {df['day'].max().date()}")
print(f"Avg words/entry    : {df['word_count'].mean():.0f}")
print(f"Longest entry      : {df['word_count'].max()} words  ({df.loc[df['word_count'].idxmax(), 'day'].date()})")
print(f"Most active month  : {df.groupby(df['day'].dt.to_period('M')).size().idxmax()}")
print(f"Most common day    : {df['weekday'].value_counts().idxmax()}")
print("─────────────────────────────────────────────────\n")


# ── 8. VOCABULARY RICHNESS OVER TIME (type-token ratio) ──────────────────────
# TTR = unique words / total words in a rolling window.
# Higher = more varied vocabulary. Fixed window keeps values comparable.

def ttr_for_window(texts):
    tokens = [w for t in texts for w in tokenise(t)]
    return len(set(tokens)) / len(tokens) if tokens else np.nan

days  = df["day"].values
texts = df["text"].fillna("").values
ttr_values = []
for i, d in enumerate(days):
    cutoff = d - np.timedelta64(90, "D")
    window = pd.Series(texts[(days >= cutoff) & (days <= d)])
    ttr_values.append(ttr_for_window(window))
ttr_series = pd.Series(ttr_values, index=df["day"])

fig, ax = plt.subplots(figsize=(14, 4))
ax.plot(df["day"], ttr_series.values, color="darkorange", linewidth=1.2)
ax.set_title("Vocabulary Richness Over Time (90-day rolling type-token ratio)")
ax.set_ylabel("Type-token ratio")
ax.set_ylim(0, 1)
ax.xaxis.set_major_formatter(mdates.DateFormatter("%Y"))
plt.tight_layout()
plt.savefig("vocabulary_richness.png", dpi=150)
plt.show()
print("Saved: vocabulary_richness.png")


# ── 9. TF-IDF: MOST DISTINCTIVE WORDS PER YEAR ───────────────────────────────
# Each year's entries are combined into one document.
# Words that are common in that year but rare across all years score highest.

from sklearn.feature_extraction.text import TfidfVectorizer

year_docs = df.groupby("year")["text"].apply(lambda x: " ".join(x.fillna("")))
years = year_docs.index.tolist()

vectorizer = TfidfVectorizer(
    max_features=5000,
    stop_words="english",
    token_pattern=r"[a-z]{3,}",
    sublinear_tf=True
)
tfidf_matrix = vectorizer.fit_transform(year_docs)
feature_names = vectorizer.get_feature_names_out()

TOP_N = 12
n_years = len(years)
cols = min(4, n_years)
rows = (n_years + cols - 1) // cols

fig, axes = plt.subplots(rows, cols, figsize=(cols * 4, rows * 3.5))
axes = np.array(axes).flatten()

for i, year in enumerate(years):
    scores = tfidf_matrix[i].toarray().flatten()
    top_idx = scores.argsort()[-TOP_N:][::-1]
    top_terms = [feature_names[j] for j in top_idx]
    top_scores = [scores[j] for j in top_idx]
    axes[i].barh(list(reversed(top_terms)), list(reversed(top_scores)), color="steelblue")
    axes[i].set_title(str(year), fontweight="bold")
    axes[i].tick_params(labelsize=8)

for j in range(i + 1, len(axes)):
    axes[j].set_visible(False)

fig.suptitle("Most Distinctive Words by Year (TF-IDF)", fontsize=14, fontweight="bold")
plt.tight_layout()
plt.savefig("tfidf_by_year.png", dpi=150)
plt.show()
print("Saved: tfidf_by_year.png")


# ── 10. EMOTIONAL TONE OVER TIME (VADER sentiment) ───────────────────────────
# VADER is tuned for informal/natural text — no model training needed.
# Compound score: +1 = most positive, -1 = most negative.
# Install with: pip install vaderSentiment

try:
    from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
    vader_available = True
except ImportError:
    print("VADER not installed — run: pip install vaderSentiment")
    vader_available = False

if vader_available:
    sia = SentimentIntensityAnalyzer()
    df["sentiment"] = df["text"].fillna("").apply(
        lambda t: sia.polarity_scores(str(t))["compound"]
    )

    monthly_sentiment = df.groupby(df["day"].dt.to_period("M"))["sentiment"].mean()
    monthly_dates = monthly_sentiment.index.to_timestamp()

    fig, ax = plt.subplots(figsize=(14, 4))
    ax.bar(monthly_dates, monthly_sentiment.values,
           color=["tomato" if v < 0 else "mediumseagreen" for v in monthly_sentiment.values],
           width=20, alpha=0.7)
    roll = monthly_sentiment.rolling(3, center=True).mean()
    ax.plot(monthly_dates, roll.values, color="black", linewidth=1.5, label="3-month avg")
    ax.axhline(0, color="gray", linewidth=0.8, linestyle="--")
    ax.set_title("Emotional Tone by Month (VADER sentiment)")
    ax.set_ylabel("Avg compound score")
    ax.legend()
    ax.xaxis.set_major_formatter(mdates.DateFormatter("%Y"))
    plt.tight_layout()
    plt.savefig("sentiment_over_time.png", dpi=150)
    plt.show()
    print("Saved: sentiment_over_time.png")


# ── 11. CHARACTERISTIC WORDS BY SEASON / LIFE PERIOD ─────────────────────────

def get_season(month):
    return {12:"Winter",1:"Winter",2:"Winter",
            3:"Spring",4:"Spring",5:"Spring",
            6:"Summer",7:"Summer",8:"Summer",
            9:"Autumn",10:"Autumn",11:"Autumn"}[month]

df["season"] = df["month"].apply(get_season)

# ── Edit these to reflect your own life chapters ──────────────────────────────
PERIODS = [
    ("College",                  "2018-08-01", "2020-12-31"),
    ("First job",                "2021-05-01", "2024-07-31"),
    ("Second job",               "2024-11-01", "2026-06-30"),
    ("Grad school applications", "2025-06-01", "2026-01-15"),
    ("Wedding planning",         "2026-01-01", "2026-05-10"),
]
# ─────────────────────────────────────────────────────────────────────────────

def top_tfidf_chart(docs_dict, title, filename, color="steelblue", n=10):
    labels  = list(docs_dict.keys())
    corpus  = list(docs_dict.values())
    vec = TfidfVectorizer(stop_words="english", token_pattern=r"[a-z]{3,}",
                          sublinear_tf=True, max_features=5000)
    mat   = vec.fit_transform(corpus)
    names = vec.get_feature_names_out()
    cols  = min(4, len(labels))
    rows  = (len(labels) + cols - 1) // cols
    fig, axes = plt.subplots(rows, cols, figsize=(cols * 4, rows * 3.5))
    axes = np.array(axes).flatten()
    for i, label in enumerate(labels):
        scores  = mat[i].toarray().flatten()
        top_idx = scores.argsort()[-n:][::-1]
        terms   = [names[j] for j in top_idx]
        vals    = [scores[j] for j in top_idx]
        axes[i].barh(list(reversed(terms)), list(reversed(vals)), color=color)
        axes[i].set_title(label, fontweight="bold")
        axes[i].tick_params(labelsize=8)
    for j in range(i + 1, len(axes)):
        axes[j].set_visible(False)
    fig.suptitle(title, fontsize=14, fontweight="bold")
    plt.tight_layout()
    plt.savefig(filename, dpi=150)
    plt.show()
    print(f"Saved: {filename}")

season_order = ["Spring", "Summer", "Autumn", "Winter"]
season_docs  = {s: " ".join(df[df["season"] == s]["text"].fillna("")) for s in season_order}
top_tfidf_chart(season_docs, "Most Characteristic Words by Season (TF-IDF)",
                "tfidf_by_season.png", color="mediumpurple")

if PERIODS:
    period_docs = {
        label: " ".join(df[(df["day"] >= start) & (df["day"] <= end)]["text"].fillna(""))
        for label, start, end in PERIODS
    }
    top_tfidf_chart(period_docs, "Most Characteristic Words by Life Period (TF-IDF)",
                    "tfidf_by_period.png", color="darkorange")
else:
    print("(Life period chart skipped — fill in the PERIODS list in section 11 to enable it)")


# ── 12. YEAR-OVER-YEAR LINGUISTIC SOPHISTICATION ─────────────────────────────
#
# Metrics computed per year:
#
#  VOCABULARY
#   • unique_words       — distinct lemma-like tokens used that year
#   • ttr               — type-token ratio (unique / total); higher = more varied
#   • hapax_ratio       — share of words used only once; high = adventurous vocab
#   • avg_word_length   — longer words often signal more formal/sophisticated prose
#
#  SENTENCE STRUCTURE / COMPLEXITY PROXIES
#   • avg_sent_length   — mean words per sentence
#   • long_sent_ratio   — share of sentences ≥ 20 words (complex constructions)
#   • comma_rate        — commas per 100 words (proxy for embedded clauses)
#   • semicolon_rate    — semicolons per 100 words (deliberate, formal connector)
#   • dash_rate         — em/en dashes per 100 words (parenthetical complexity)
#   • sub_clause_proxy  — rate of subordinating conjunctions per 100 words
#                         (because, although, while, since, unless, if, when, etc.)
#
# No external NLP library needed — pure regex + Counter.
# ─────────────────────────────────────────────────────────────────────────────

import math

# Subordinating conjunctions as a proxy for dependent/embedded clauses
SUB_CONJ = {
    "because","although","though","even","while","whilst","since","unless",
    "until","whenever","wherever","whether","after","before","once","if",
    "provided","assuming","given","despite","whereas","whereby"
}

def sent_tokenise(text):
    """Split on . ! ? followed by whitespace or end-of-string."""
    return [s.strip() for s in re.split(r"[.!?]+", str(text)) if s.strip()]

def year_metrics(entries):
    all_tokens, all_sents = [], []
    punct = {"commas": 0, "semicolons": 0, "dashes": 0}
    sub_hits = 0
    total_chars = 0

    for text in entries:
        t = str(text)
        total_chars += len(t)
        punct["commas"]     += t.count(",")
        punct["semicolons"] += t.count(";")
        punct["dashes"]     += t.count("—") + t.count("–") + t.count(" - ")

        sents  = sent_tokenise(t)
        all_sents.extend(sents)

        tokens = tokenise(t)           # reuse existing tokenise() — strips stopwords
        all_tokens.extend(tokens)

        # subordinating conjunction count (keep stopwords for this one)
        raw_words = re.findall(r"[a-z']+", t.lower())
        sub_hits += sum(1 for w in raw_words if w in SUB_CONJ)

    if not all_tokens or not all_sents:
        return {}

    total_words   = len(all_tokens)
    total_sents   = len(all_sents)
    word_freq     = Counter(all_tokens)
    hapax         = sum(1 for v in word_freq.values() if v == 1)
    sent_lengths  = [len(s.split()) for s in all_sents]
    per100        = 100 / total_words if total_words else 0

    # Total raw word count (including stopwords) for punctuation rates
    raw_total = sum(len(s.split()) for s in entries.fillna("").astype(str))
    raw_per100 = 100 / raw_total if raw_total else 0

    return {
        "unique_words"     : len(word_freq),
        "ttr"              : round(len(word_freq) / total_words, 4),
        "hapax_ratio"      : round(hapax / len(word_freq), 4),
        "avg_word_length"  : round(sum(len(w) for w in all_tokens) / total_words, 2),
        "avg_sent_length"  : round(sum(sent_lengths) / total_sents, 2),
        "long_sent_ratio"  : round(sum(1 for l in sent_lengths if l >= 20) / total_sents, 4),
        "comma_rate"       : round(punct["commas"]     * raw_per100, 2),
        "semicolon_rate"   : round(punct["semicolons"] * raw_per100, 2),
        "dash_rate"        : round(punct["dashes"]     * raw_per100, 2),
        "sub_clause_proxy" : round(sub_hits            * raw_per100, 2),
    }

yearly_metrics = {}
for year, grp in df.groupby("year"):
    yearly_metrics[year] = year_metrics(grp["text"])

metrics_df = pd.DataFrame(yearly_metrics).T
metrics_df.index.name = "year"
print("\n── Yearly Linguistic Metrics ─────────────────────────────────────────────")
print(metrics_df.to_string())


# ── CHARTS: one panel per metric group ───────────────────────────────────────

years_list = metrics_df.index.tolist()

def bar_chart(ax, col, ylabel, color, title):
    vals = metrics_df[col]
    bars = ax.bar(years_list, vals, color=color, alpha=0.8)
    # trend line
    x = np.arange(len(years_list))
    if len(x) > 1:
        z = np.polyfit(x, vals, 1)
        ax.plot(years_list, np.poly1d(z)(x), color="black",
                linewidth=1.5, linestyle="--", label="trend")
        ax.legend(fontsize=7)
    ax.set_title(title, fontsize=9, fontweight="bold")
    ax.set_ylabel(ylabel, fontsize=8)
    ax.set_xticks(years_list)
    ax.set_xticklabels(years_list, rotation=45, fontsize=7)

# — Vocabulary panel —
fig, axes = plt.subplots(2, 2, figsize=(12, 7))
bar_chart(axes[0,0], "ttr",            "ratio",   "steelblue",   "Type-Token Ratio")
bar_chart(axes[0,1], "hapax_ratio",    "ratio",   "mediumorchid","Hapax Ratio (words used once)")
bar_chart(axes[1,0], "avg_word_length","chars",   "darkorange",  "Avg Word Length")
bar_chart(axes[1,1], "unique_words",   "count",   "teal",        "Unique Words Used")
fig.suptitle("Vocabulary Sophistication by Year", fontsize=13, fontweight="bold")
plt.tight_layout()
plt.savefig("vocab_sophistication.png", dpi=150)
plt.show()
print("Saved: vocab_sophistication.png")

# — Sentence complexity panel —
fig, axes = plt.subplots(2, 3, figsize=(15, 7))
bar_chart(axes[0,0], "avg_sent_length",  "words",  "steelblue",   "Avg Sentence Length")
bar_chart(axes[0,1], "long_sent_ratio",  "ratio",  "tomato",      "Long Sentence Ratio (≥20 words)")
bar_chart(axes[0,2], "sub_clause_proxy", "per 100w","mediumorchid","Subordinating Conjunctions")
bar_chart(axes[1,0], "comma_rate",       "per 100w","darkorange",  "Comma Rate")
bar_chart(axes[1,1], "semicolon_rate",   "per 100w","teal",        "Semicolon Rate")
bar_chart(axes[1,2], "dash_rate",        "per 100w","slategray",   "Dash Rate")
fig.suptitle("Sentence Complexity by Year", fontsize=13, fontweight="bold")
plt.tight_layout()
plt.savefig("sentence_complexity.png", dpi=150)
plt.show()
print("Saved: sentence_complexity.png")


# ── SUMMARY TABLE ─────────────────────────────────────────────────────────────
# Z-score each metric so they're on the same scale, then visualise as a heatmap.
# Red = above your personal average; blue = below.

from matplotlib.colors import TwoSlopeNorm

z_df = (metrics_df - metrics_df.mean()) / metrics_df.std()

# Friendly display names
rename = {
    "ttr"             : "Vocab variety (TTR)",
    "hapax_ratio"     : "Novel word rate",
    "avg_word_length" : "Avg word length",
    "unique_words"    : "Unique words",
    "avg_sent_length" : "Avg sentence length",
    "long_sent_ratio" : "Long sentence ratio",
    "sub_clause_proxy": "Subordinate clauses",
    "comma_rate"      : "Comma rate",
    "semicolon_rate"  : "Semicolon rate",
    "dash_rate"       : "Dash rate",
}
z_display = z_df.rename(columns=rename)[list(rename.values())]

fig, ax = plt.subplots(figsize=(max(8, len(years_list) * 0.9), 6))
norm = TwoSlopeNorm(vmin=-2.5, vcenter=0, vmax=2.5)
im = ax.imshow(z_display.T.values, cmap="RdBu_r", aspect="auto", norm=norm)

ax.set_xticks(range(len(years_list)))
ax.set_xticklabels(years_list, fontsize=9)
ax.set_yticks(range(len(z_display.columns)))
ax.set_yticklabels(z_display.columns, fontsize=9)

# Annotate cells with raw values
for yi, col in enumerate(metrics_df.rename(columns=rename).columns if False else list(rename.keys())):
    for xi, year in enumerate(years_list):
        raw = metrics_df.loc[year, col]
        fmt = f"{raw:.2f}" if raw < 10 else f"{int(raw)}"
        ax.text(xi, yi, fmt, ha="center", va="center", fontsize=7,
                color="white" if abs(z_display.iloc[xi, yi]) > 1.2 else "black")

plt.colorbar(im, ax=ax, label="Z-score vs. your average")
ax.set_title("Linguistic Profile Summary — All Metrics by Year\n(red = above your avg, blue = below)",
             fontsize=11, fontweight="bold")
plt.tight_layout()
plt.savefig("linguistic_summary_heatmap.png", dpi=150)
plt.show()
print("Saved: linguistic_summary_heatmap.png")