sync.py

#!/usr/bin/env python3
"""Instagram saves -> Obsidian vault sync (collection-routed, video-aware).

Only syncs saves in the configured deep collections (e.g. "reel ideas", "biz")
plus recent uncollected saves. For video saves it downloads the reel, extracts
a Groq-Whisper transcript and a few keyframes, then deletes the mp4 so the vault
stays lean. Everything else (Mini, memes, the firehose) is skipped.

Dedup is self-healing: keys on note files present on disk, with a light->deep
upgrade path, so a lost state.json never dupes and never re-transcribes.

Run:  python3 sync.py
Config: config.json next to this file (see config.example.json).
Deps:  stdlib + ffmpeg + curl (Groq Whisper). No pip packages.
"""

import json
import re
import shutil
import subprocess
import sys
import tempfile
import time
import urllib.request
import urllib.error
from urllib.parse import urlencode
from datetime import datetime, timezone
from pathlib import Path

BASE_DIR = Path(__file__).resolve().parent
CONFIG_PATH = BASE_DIR / "config.json"
STATE_PATH = BASE_DIR / "state.json"
LOG_PATH = BASE_DIR / "sync.log"

IG = "https://www.instagram.com"
UA = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 "
      "(KHTML, like Gecko) Version/17.5 Safari/605.1.15")
APP_ID = "936619743392459"
PAGE_SIZE = 50
GROQ_MODEL = "whisper-large-v3"
GROQ_URL = "https://api.groq.com/openai/v1/audio/transcriptions"
# Resolve tool paths absolutely: launchd runs with a minimal PATH that omits
# /opt/homebrew/bin, so a bare "ffmpeg" fails under the scheduler.
FFMPEG = shutil.which("ffmpeg") or "/opt/homebrew/bin/ffmpeg"
CURL = shutil.which("curl") or "/usr/bin/curl"


def log(msg):
    line = f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] {msg}"
    print(line, flush=True)
    with open(LOG_PATH, "a") as f:
        f.write(line + "\n")


def load_json(path, default):
    try:
        return json.loads(Path(path).read_text())
    except (FileNotFoundError, json.JSONDecodeError):
        return default


def load_groq_key(cfg):
    keyfile = Path(cfg.get("groq_key_file", "~/.reach/keys.env")).expanduser()
    try:
        for line in keyfile.read_text().splitlines():
            line = line.strip()
            if line.startswith("GROQ_API_KEY") and "=" in line:
                return line.split("=", 1)[1].strip().strip('"').strip("'")
    except FileNotFoundError:
        pass
    return None


class Insta:
    def __init__(self, cfg):
        self.cookies = (f"sessionid={cfg['ig_session_id']}; "
                        f"csrftoken={cfg['ig_csrftoken']}; "
                        f"ds_user_id={cfg['ig_user_id']}")
        self.csrf = cfg["ig_csrftoken"]

    def get(self, path, params=None):
        url = IG + path + ("?" + urlencode(params) if params else "")
        req = urllib.request.Request(url, headers={
            "User-Agent": UA, "X-IG-App-ID": APP_ID, "X-CSRFToken": self.csrf,
            "X-Requested-With": "XMLHttpRequest", "Referer": IG + "/",
            "Accept": "*/*", "Cookie": self.cookies})
        with urllib.request.urlopen(req, timeout=30) as r:
            if "application/json" not in r.headers.get("Content-Type", ""):
                raise RuntimeError("non-JSON response (session likely invalid)")
            return json.loads(r.read().decode())

    def validate_session(self):
        try:
            d = self.get("/api/v1/accounts/edit/web_form_data/")
            return d.get("form_data", {}).get("username")
        except (urllib.error.HTTPError, RuntimeError, json.JSONDecodeError):
            return None

    def collections(self):
        d = self.get("/api/v1/collections/list/", {
            "collection_types": '["ALL_MEDIA_AUTO_COLLECTION","MEDIA"]'})
        return {str(c["collection_id"]): c.get("collection_name", "")
                for c in d.get("items", []) if c.get("collection_id")}

    def paged(self, path, max_pages):
        """Yield media items from a paginated feed, newest first."""
        max_id = None
        for _ in range(max_pages):
            params = {"count": PAGE_SIZE}
            if max_id:
                params["max_id"] = max_id
            d = self.get(path, params)
            for item in d.get("items", []):
                yield item.get("media", item)
            if not d.get("more_available"):
                return
            max_id = d.get("next_max_id")
            if not max_id:
                return
            time.sleep(1)


# ---------- media helpers ----------

def media_format(m):
    if m.get("product_type") == "clips":
        return "Reel"
    if m.get("product_type") == "igtv":
        return "IGTV"
    if m.get("media_type") == 8:
        return "Carousel"
    if "video_versions" in m:
        return "Video"
    return "Post"


def is_video(m):
    return "video_versions" in m or media_format(m) in ("Reel", "Video", "IGTV")


def fetch_bytes(url, timeout=90):
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    return urllib.request.urlopen(req, timeout=timeout).read()


def note_filename(m):
    code = m.get("code", str(m.get("pk", "")))
    user = (m.get("user") or {}).get("username", "unknown")
    return f"ig-{re.sub(r'[^\w.\-]', '_', user)}-{code}.md"


def yaml_str(s):
    return '"' + str(s).replace("\\", "\\\\").replace('"', '\\"') + '"'


def transcribe(mp3_path, groq_key):
    """Groq Whisper via curl (multipart). Returns text or None."""
    if not groq_key:
        return None
    try:
        out = subprocess.run(
            [CURL, "-s", "--max-time", "180", GROQ_URL,
             "-H", f"Authorization: Bearer {groq_key}",
             "-F", f"file=@{mp3_path}",
             "-F", f"model={GROQ_MODEL}",
             "-F", "response_format=text"],
            capture_output=True, text=True, timeout=200)
        txt = out.stdout.strip()
        if txt and not txt.lstrip().startswith("{"):  # error responses are JSON
            return txt
        log(f"    transcription error: {txt[:200]}")
    except Exception as e:
        log(f"    transcription failed: {e}")
    return None


def process_video(m, code, groq_key, want_frames=6):
    """Download reel -> transcript + frame bytes. Deletes mp4. Returns dict."""
    result = {"transcript": None, "frames": []}
    vv = m.get("video_versions") or []
    if not vv:
        return result
    with tempfile.TemporaryDirectory(prefix="igsave_") as td:
        td = Path(td)
        mp4 = td / "v.mp4"
        try:
            mp4.write_bytes(fetch_bytes(vv[0]["url"]))
        except Exception as e:
            log(f"    video download failed: {e}")
            return result
        dur = float(m.get("video_duration") or 0)
        # audio -> transcript
        mp3 = td / "a.mp3"
        try:
            subprocess.run([FFMPEG, "-y", "-i", str(mp4), "-vn", "-ac", "1",
                            "-ar", "16000", "-b:a", "64k", str(mp3),
                            "-loglevel", "error"], check=True, timeout=120)
            result["transcript"] = transcribe(mp3, groq_key)
        except Exception as e:
            log(f"    audio extract failed: {e}")
        # evenly sampled frames
        n = max(3, min(want_frames, int(dur / 8) or 3)) if dur else 4
        fps = (n / dur) if dur else 0.5
        try:
            subprocess.run([FFMPEG, "-y", "-i", str(mp4), "-vf",
                            f"fps={fps:.4f},scale=480:-1", "-frames:v", str(n),
                            str(td / "f_%02d.jpg"), "-loglevel", "error"],
                           check=True, timeout=120)
            for f in sorted(td.glob("f_*.jpg"))[:want_frames]:
                result["frames"].append(f.read_bytes())
        except Exception as e:
            log(f"    frame sample failed: {e}")
    return result


def carousel_frames(m, limit=10):
    out = []
    for slide in (m.get("carousel_media") or [])[:limit]:
        cands = (slide.get("image_versions2") or {}).get("candidates") or []
        if cands:
            try:
                out.append(fetch_bytes(cands[0]["url"]))
            except Exception:
                pass
    return out


def single_image(m):
    src = m
    cands = (src.get("image_versions2") or {}).get("candidates") or []
    if cands:
        try:
            return [fetch_bytes(cands[0]["url"])]
        except Exception:
            pass
    return []


# ---------- note writing ----------

def note_is_deep(path):
    try:
        head = path.read_text()[:400]
        return "tier: deep" in head
    except FileNotFoundError:
        return False


def write_deep_note(m, collection, cfg, groq_key):
    notes_dir = Path(cfg["vault_dir"]).expanduser() / "07-Resources" / "instagram-saves"
    media_dir = notes_dir / "_media"
    frames_dir = media_dir / "frames"
    notes_dir.mkdir(parents=True, exist_ok=True)
    media_dir.mkdir(parents=True, exist_ok=True)

    code = m.get("code", str(m["pk"]))
    user = (m.get("user") or {}).get("username", "unknown")
    full_name = (m.get("user") or {}).get("full_name", "")
    caption = ((m.get("caption") or {}).get("text") or "").strip()
    fmt = media_format(m)
    url = f"{IG}/reel/{code}/" if fmt in ("Reel", "Video") else f"{IG}/p/{code}/"
    posted = datetime.fromtimestamp(m.get("taken_at", 0), tz=timezone.utc)
    today = datetime.now().strftime("%Y-%m-%d")
    dur = int(float(m.get("video_duration") or 0))

    transcript = None
    frame_bytes = []
    if is_video(m):
        if cfg.get("video_analysis", True):
            r = process_video(m, code, groq_key)
            transcript = r["transcript"]
            frame_bytes = r["frames"]
    elif fmt == "Carousel":
        frame_bytes = carousel_frames(m)
    else:
        frame_bytes = single_image(m)

    # write frame images into vault (python does all vault writes: TCC-safe)
    embeds = []
    if frame_bytes:
        frames_dir.mkdir(parents=True, exist_ok=True)
        for i, b in enumerate(frame_bytes, 1):
            name = f"ig-{code}-f{i:02d}.jpg"
            (frames_dir / name).write_bytes(b)
            embeds.append(f"![[{name}]]")
    else:
        # fall back to a plain thumbnail for context
        cands = (m.get("image_versions2") or {}).get("candidates") or []
        if cands:
            try:
                (media_dir / f"ig-{code}.jpg").write_bytes(fetch_bytes(cands[0]["url"]))
                embeds.append(f"![[ig-{code}.jpg]]")
            except Exception:
                pass

    title_hint = re.sub(r"\s+", " ", caption)[:80] if caption else fmt
    meta_line = f"**Link:** {url} · {fmt}"
    if dur:
        meta_line += f" · {dur}s"
    if collection:
        meta_line += f" · collection: {collection}"

    front = [
        "---", "type: instagram-save", "tier: deep", f"pk: {m.get('pk', '')}",
        f"author: {yaml_str('@' + user)}", f"author_name: {yaml_str(full_name)}",
        f"url: {url}", f"format: {fmt}", f"collection: {yaml_str(collection)}",
    ]
    if dur:
        front.append(f"duration_s: {dur}")
    front += [f"posted: {posted.strftime('%Y-%m-%d')}", f"synced: {today}",
              "status: new", "tags:", "  - instagram-save", "---"]

    body = [f"# @{user}: {title_hint}", ""]
    if embeds:
        body += [" ".join(embeds), ""]
    body += [meta_line, "", "## Caption", "", caption or "_no caption_", ""]
    if is_video(m):
        body += ["## Transcript", "",
                 transcript.strip() if transcript else "_transcript unavailable_", ""]
    body += ["## Claude's take", "", "_pending analysis, run /ig-saves_", ""]

    (notes_dir / note_filename(m)).write_text(
        "\n".join(front) + "\n\n" + "\n".join(body))
    return note_filename(m), bool(transcript), len(embeds)


# ---------- main ----------

def parse_args(argv):
    """Flags: --collections a,b (override deep set), --no-uncollected, --all-saves."""
    opts = {"collections": None, "uncollected": True, "all_saves": False}
    i = 0
    while i < len(argv):
        a = argv[i]
        if a == "--collections" and i + 1 < len(argv):
            opts["collections"] = [c.strip() for c in argv[i + 1].split(",") if c.strip()]
            i += 2
        elif a == "--no-uncollected":
            opts["uncollected"] = False
            i += 1
        elif a == "--all-saves":
            opts["all_saves"] = True
            i += 1
        else:
            i += 1
    return opts


def label_for(m, all_cols):
    """Best-effort collection name for an item, for note frontmatter/logging."""
    scids = [str(x) for x in (m.get("saved_collection_ids") or [])]
    names = [all_cols[c] for c in scids if c in all_cols and all_cols[c] != "All posts"]
    return names[0] if names else ""


def main():
    opts = parse_args(sys.argv[1:])
    cfg = load_json(CONFIG_PATH, None)
    if not cfg or "PASTE" in json.dumps(cfg):
        log("config.json missing or still has placeholder values. Aborting.")
        sys.exit(1)
    if opts["collections"] is not None:
        cfg["deep_collections"] = opts["collections"]
    if not opts["uncollected"]:
        cfg["uncollected_recent_days"] = 0
    for tool in ("ffmpeg", "curl"):
        if not shutil.which(tool):
            log(f"WARNING: {tool} not found; video analysis will degrade.")

    groq_key = load_groq_key(cfg) if cfg.get("video_analysis", True) else None
    if cfg.get("video_analysis", True) and not groq_key:
        log("WARNING: no GROQ_API_KEY found; transcripts will be unavailable.")

    ig = Insta(cfg)
    username = ig.validate_session()
    if not username:
        log("Instagram session INVALID. Refresh cookies via /ig-saves refresh.")
        sys.exit(2)
    log(f"Instagram session valid for @{username}")

    all_cols = ig.collections()  # id -> name
    name_to_id = {v.lower(): k for k, v in all_cols.items()}
    deep_names = [n for n in cfg.get("deep_collections", [])]
    deep_ids = {name_to_id[n.lower()] for n in deep_names if n.lower() in name_to_id}
    log(f"Deep collections: {deep_names} -> {sorted(deep_ids)}")
    window = int(cfg.get("uncollected_recent_days", 0) or 0)

    notes_dir = Path(cfg["vault_dir"]).expanduser() / "07-Resources" / "instagram-saves"
    existing_deep = {p.name for p in notes_dir.glob("ig-*.md") if note_is_deep(p)}

    state = load_json(STATE_PATH, {"synced": {}})
    synced = state["synced"]

    def flush():
        STATE_PATH.write_text(json.dumps(state, indent=2))

    now = datetime.now(timezone.utc)
    new = upgraded = skipped = errors = 0

    def handle(m, collection):
        nonlocal new, upgraded, skipped, errors
        fname = note_filename(m)
        if fname in existing_deep:
            skipped += 1
            return
        was_light = (notes_dir / fname).exists()
        try:
            f, had_tx, nframes = write_deep_note(m, collection, cfg, groq_key)
            existing_deep.add(f)
            synced[str(m.get("pk", ""))] = {
                "file": f, "collection": collection, "tier": "deep",
                "at": datetime.now().isoformat(timespec="seconds")}
            if was_light:
                upgraded += 1
            else:
                new += 1
            tx = "tx" if had_tx else "no-tx"
            log(f"  {'^' if was_light else '+'} {f} [{collection or 'uncollected'}] {tx} {nframes}f")
            if (new + upgraded) % 10 == 0:
                flush()
        except Exception as e:
            errors += 1
            log(f"  ! failed on {m.get('pk')}: {e}")

    if opts["all_saves"]:
        # 0) transcribe EVERYTHING except skip_collections (e.g. Mini): walk the
        # entire saved feed regardless of collection or age. Self-healing dedup
        # means this is safely resumable and cheap to re-run (already-deep notes
        # are skipped instantly).
        cap = int(cfg.get("max_all_pages", 120))
        skip_cols = {s.lower() for s in cfg.get("skip_collections", [])}
        log(f"Walking ALL saved posts (up to {cap} pages / ~{cap * PAGE_SIZE} items), "
            f"skipping: {sorted(skip_cols) or 'none'}")
        seen = excluded = 0
        for m in ig.paged("/api/v1/feed/saved/posts/", cap):
            seen += 1
            scids = [str(x) for x in (m.get("saved_collection_ids") or [])]
            item_cols = {all_cols.get(c, "").lower() for c in scids}
            if item_cols & skip_cols:
                excluded += 1
                continue
            handle(m, label_for(m, all_cols))
            if seen % 100 == 0:
                log(f"  ...{seen} scanned, {new} new, {upgraded} upgraded, "
                    f"{excluded} excluded so far")
        log(f"  scanned {seen} total saved items ({excluded} excluded via skip_collections)")
    else:
        # 1) deep collections via their own feeds (complete, efficient)
        for cid in deep_ids:
            cname = all_cols[cid]
            log(f"Fetching collection: {cname}")
            for m in ig.paged(f"/api/v1/feed/collection/{cid}/posts/",
                              int(cfg.get("max_collection_pages", 6))):
                handle(m, cname)

        # 2) recent uncollected saves from the top of the saved feed
        if window > 0:
            log(f"Scanning saved feed for uncollected saves <= {window} days old")
            seen = 0
            for m in ig.paged("/api/v1/feed/saved/posts/",
                              int(cfg.get("max_saved_pages", 12))):
                seen += 1
                scids = {str(x) for x in (m.get("saved_collection_ids") or [])}
                in_user_col = scids & set(all_cols)
                if in_user_col:
                    continue  # belongs to a collection (Mini, deep already done, etc.)
                age = (now - datetime.fromtimestamp(m.get("taken_at", 0), tz=timezone.utc)).days
                if age <= window:
                    handle(m, "")
            log(f"  scanned {seen} saved items")

    flush()
    log(f"Sync complete: {new} new | {upgraded} upgraded | "
        f"{skipped} already-deep | {errors} errors")


if __name__ == "__main__":
    main()