media.ts

/**
 * Tier 0 of the ingest pipeline — real measurement, zero tokens.
 *
 * Everything in this file runs on the actual video file with ffmpeg/ffprobe and
 * produces facts, not opinions: duration, resolution, orientation, whether
 * there is an audio track, how sharp the picture is, how bright it is, and
 * which moments are actually different from one another.
 *
 * Two of those matter more than they look.
 *
 * **Sharpness** is measured by running the frames through an edge detector and
 * averaging the resulting luma. A clip whose edge energy is near zero is out of
 * focus, and no language model needs to be consulted about it. This is what
 * lets the pipeline reject unusable footage before spending a single token.
 *
 * **Scene detection** picks the keyframes. Sampling a video every N seconds
 * gives you three near-identical frames of a locked-off shot and misses the one
 * moment something happened. Selecting on scene change gives the vision model
 * three genuinely different views of the clip, which is both cheaper and
 * better — the single highest-leverage decision in the whole token budget.
 *
 * The binaries ship with the repo via `ffmpeg-static`, so there is nothing to
 * install.
 */

import { execFile } from 'node:child_process'
import { promisify } from 'node:util'
import { mkdir, stat } from 'node:fs/promises'
import { dirname } from 'node:path'
import ffmpegPath from 'ffmpeg-static'
import ffprobeStatic from 'ffprobe-static'

const exec = promisify(execFile)

const FFMPEG = (ffmpegPath as unknown as string) ?? 'ffmpeg'
const FFPROBE = ffprobeStatic.path ?? 'ffprobe'

/** ffmpeg is chatty on stderr and returns non-zero for benign reasons; treat output as data. */
async function run(bin: string, args: string[]): Promise<{ stdout: string; stderr: string }> {
  try {
    return await exec(bin, args, { maxBuffer: 32 * 1024 * 1024, windowsHide: true })
  } catch (error) {
    const err = error as { stdout?: string; stderr?: string; message?: string }
    if (err.stdout != null || err.stderr != null) return { stdout: err.stdout ?? '', stderr: err.stderr ?? '' }
    throw error
  }
}

export interface VideoProbe {
  durationSec: number
  width: number
  height: number
  fps: number
  hasAudio: boolean
  videoCodec: string
  fileSizeMb: number
}

export async function probeVideo(path: string): Promise<VideoProbe> {
  const { stdout } = await run(FFPROBE, [
    '-v', 'error',
    '-print_format', 'json',
    '-show_format',
    '-show_streams',
    path,
  ])

  const data = JSON.parse(stdout) as {
    format?: { duration?: string; size?: string }
    streams?: {
      codec_type?: string
      codec_name?: string
      width?: number
      height?: number
      avg_frame_rate?: string
      duration?: string
    }[]
  }

  const streams = data.streams ?? []
  const video = streams.find((s) => s.codec_type === 'video')
  const audio = streams.find((s) => s.codec_type === 'audio')

  if (!video) throw new Error(`No video stream in ${path}`)

  const [num, den] = (video.avg_frame_rate ?? '0/1').split('/').map(Number)
  const fps = den > 0 ? num / den : 0

  const durationSec = Number(data.format?.duration ?? video.duration ?? 0)
  const bytes = Number(data.format?.size ?? 0)

  return {
    durationSec: Number(durationSec.toFixed(2)),
    width: video.width ?? 0,
    height: video.height ?? 0,
    fps: Number(fps.toFixed(2)),
    hasAudio: Boolean(audio),
    videoCodec: video.codec_name ?? 'unknown',
    fileSizeMb: Number((bytes / 1_048_576).toFixed(2)),
  }
}

/**
 * Timestamps where the picture genuinely changes.
 *
 * Falls back to evenly spaced sampling when a clip is one continuous shot with
 * no cuts — which is most raw creator footage, and perfectly normal.
 */
export async function sceneTimestamps(path: string, want: number, durationSec: number): Promise<number[]> {
  const { stderr } = await run(FFMPEG, [
    '-hide_banner',
    '-i', path,
    '-vf', "select='gt(scene,0.25)',showinfo",
    '-vsync', 'vfr',
    '-f', 'null',
    '-',
  ])

  const times = [...stderr.matchAll(/pts_time:([0-9.]+)/g)]
    .map((match) => Number(match[1]))
    .filter((time) => Number.isFinite(time) && time > 0.3 && time < durationSec - 0.2)

  // Spread the picks across the clip rather than clustering on one busy second.
  const spaced: number[] = []
  const minGap = Math.max(0.6, durationSec / (want * 3))
  for (const time of times) {
    if (spaced.every((existing) => Math.abs(existing - time) > minGap)) spaced.push(time)
    if (spaced.length >= want) break
  }

  if (spaced.length >= want) return spaced.slice(0, want)

  // Even sampling for the rest, avoiding the very start and end.
  const even = Array.from({ length: want }, (_, index) => ((index + 1) / (want + 1)) * durationSec)
  for (const time of even) {
    if (spaced.length >= want) break
    if (spaced.every((existing) => Math.abs(existing - time) > minGap * 0.5)) spaced.push(time)
  }

  return spaced.sort((a, b) => a - b).slice(0, want)
}

/** Extract one frame at a timestamp, downscaled — image tokens scale with pixels. */
export async function extractFrame(
  source: string,
  timestamp: number,
  destination: string,
  maxWidth = 1024,
): Promise<void> {
  await mkdir(dirname(destination), { recursive: true })
  await run(FFMPEG, [
    '-hide_banner',
    '-y',
    '-ss', String(timestamp),
    '-i', source,
    '-frames:v', '1',
    '-vf', `scale='min(${maxWidth},iw)':-2:flags=lanczos`,
    '-q:v', '3',
    destination,
  ])
}

export interface PictureStats {
  /** 0-1. Mean edge energy — a proxy for how sharp the picture is. */
  sharpness: number
  /** 0-1. Mean luma. */
  brightness: number
}

/**
 * Measured, not inferred.
 *
 * `edgedetect` turns the picture into its edges; the average brightness of that
 * edge map is high for a crisp image and near zero for a soft one. It is a
 * proxy rather than a true focus metric, but it is a real measurement of the
 * real file, it costs nothing, and it is more reliable than asking a model to
 * judge focus from a compressed still.
 */
export async function pictureStats(path: string): Promise<PictureStats> {
  /** Averages the requested signalstats keys across sampled frames. */
  const measure = async (filter: string, keys: string[]): Promise<Record<string, number>> => {
    const { stdout } = await run(FFPROBE, [
      '-v', 'error',
      '-f', 'lavfi',
      '-i', `movie=${escapeForLavfi(path)},fps=2,${filter}signalstats`,
      '-show_entries', `frame_tags=${keys.map((key) => `lavfi.signalstats.${key}`).join(',')}`,
      '-of', 'csv=p=0',
    ])

    const rows = stdout
      .split(/\r?\n/)
      .map((line) => line.trim())
      .filter(Boolean)
      .map((line) => line.split(','))

    const out: Record<string, number> = {}
    keys.forEach((key, index) => {
      const values = rows.map((row) => Number(row[index])).filter(Number.isFinite)
      out[key] = values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0
    })
    return out
  }

  // Blur the frames again and see how much edge energy survives.
  //
  // The obvious approach — measure edge energy and call a low number "soft" —
  // does not work, and both naive versions were tried against this library
  // before this one. Raw edge energy tracks scene brightness and contrast: a
  // sunlit pool deck read sixteen times a dim sauna. Dividing by the frame's
  // own contrast range fixes that but introduces a worse confound, because it
  // then tracks how much fine detail the scene contains — a person against
  // smooth water and a tiled steam room both scored *below* a clip that had
  // been deliberately blurred.
  //
  // Comparing a clip against a blurred copy of itself has neither problem. A
  // sharp frame loses most of its edges when blurred again; an already-soft one
  // barely changes. The result is a ratio of the same scene to itself, so
  // brightness, contrast and subject matter all cancel out. Measured here:
  //
  //   deliberately blurred   0.39   <- the only soft clip in the set
  //   corridor               0.52
  //   ladle on stones        0.57
  //   stove / pool deck      0.59
  //   ice hole / sauna wide  0.64-0.66
  //   lounge window          0.72
  //
  // This detects softness, not focus accuracy — it cannot tell a deliberate
  // shallow-depth-of-field shot from a missed focus pull, which is exactly why
  // it flags for review rather than deciding.
  const [sharp, reblurred, base] = await Promise.all([
    measure('edgedetect=low=0.1:high=0.3,', ['YAVG']),
    measure('gblur=sigma=2,edgedetect=low=0.1:high=0.3,', ['YAVG']),
    measure('', ['YAVG']),
  ])

  const edgeLoss = sharp.YAVG > 0 ? 1 - reblurred.YAVG / sharp.YAVG : 0
  const FLOOR = 0.35
  const SPAN = 0.4

  return {
    sharpness: Number(Math.max(0, Math.min(1, (edgeLoss - FLOOR) / SPAN)).toFixed(3)),
    brightness: Number((base.YAVG / 255).toFixed(3)),
  }
}

/**
 * Transcode to a small, web-playable clip.
 *
 * Raw creator footage is hundreds of megabytes. A CRM needs a preview that
 * loads instantly in a card, not the master. In production the original would
 * sit in object storage and this would be the proxy; here it is what ships in
 * the repo, which is why it is capped hard on both resolution and length.
 */
export async function makeWebPreview(
  source: string,
  destination: string,
  options: {
    maxSeconds?: number
    startAt?: number
    crf?: number
    /**
     * Force a delivery orientation. Creators shoot vertical for social and
     * horizontal for everything else, and the brief asks for one or the other —
     * so a clip delivered against a vertical requirement is genuinely encoded
     * vertical, centre-cropped from the source.
     */
    aspect?: 'VERTICAL' | 'HORIZONTAL' | 'SQUARE'
    /**
     * Gaussian sigma applied before encoding.
     *
     * Used to give demo clips the defects the scenario says they have. If the
     * story is that a creator delivered one soft clip, the file really is soft,
     * the pre-filter really measures it, and the reviewer really sees why it
     * was rejected. Labelling a perfectly sharp clip "blurry" in a fixture
     * would make the whole quality pipeline decorative.
     */
    blurSigma?: number
  } = {},
): Promise<void> {
  const { maxSeconds = 12, startAt = 0, crf = 30, aspect = 'HORIZONTAL', blurSigma = 0 } = options
  await mkdir(dirname(destination), { recursive: true })

  // 720 on the short edge keeps every clip above the pipeline's own
  // low-resolution reject threshold, whichever way up it is.
  const target = aspect === 'VERTICAL' ? { w: 720, h: 1280 } : aspect === 'SQUARE' ? { w: 900, h: 900 } : { w: 1280, h: 720 }

  // Scale to cover, then centre-crop — no letterboxing, no distortion.
  const filter =
    `scale=${target.w}:${target.h}:force_original_aspect_ratio=increase:flags=lanczos,` +
    `crop=${target.w}:${target.h},setsar=1` +
    (blurSigma > 0 ? `,gblur=sigma=${blurSigma.toFixed(2)}` : '')

  await run(FFMPEG, [
    '-hide_banner',
    '-y',
    ...(startAt > 0 ? ['-ss', String(startAt)] : []),
    '-i', source,
    '-t', String(maxSeconds),
    '-vf', filter,
    '-c:v', 'libx264',
    '-preset', 'slow',
    '-crf', String(crf),
    '-pix_fmt', 'yuv420p',
    '-c:a', 'aac',
    '-b:a', '64k',
    '-ac', '1',
    '-movflags', '+faststart',
    destination,
  ])

  await stat(destination)
}

export function deriveOrientationFromSize(width: number, height: number): 'VERTICAL' | 'HORIZONTAL' | 'SQUARE' {
  const ratio = width / height
  if (ratio > 1.05) return 'HORIZONTAL'
  if (ratio < 0.95) return 'VERTICAL'
  return 'SQUARE'
}

/** lavfi treats `:` and `\` as syntax; Windows paths are full of both. */
function escapeForLavfi(path: string): string {
  return path.replace(/\\/g, '/').replace(/:/g, '\\\\:')
}

export const FFMPEG_BIN = FFMPEG
export const FFPROBE_BIN = FFPROBE