diff --git a/.dependency-cruiser.js b/.dependency-cruiser.js new file mode 100644 index 0000000..46395ff --- /dev/null +++ b/.dependency-cruiser.js @@ -0,0 +1,48 @@ +/** @type {import('dependency-cruiser').IConfiguration} */ +module.exports = { + forbidden: [ + { + name: 'no-circular', + severity: 'error', + comment: 'Warns on circular dependencies.', + from: {}, + to: { circular: true } + }, + { + name: 'connectors-cannot-import-connectors', + severity: 'error', + comment: 'A connector can only import types, shared, and utils. It cannot import other connectors.', + from: { path: '^connectors/([^/]+)/' }, + to: { path: '^connectors/([^/]+)/', pathNot: '^connectors/$1/' } + }, + { + name: 'no-app-dependencies', + severity: 'error', + comment: 'No package or connector may import from apps.', + from: { pathNot: '^apps/' }, + to: { path: '^apps/' } + }, + { + name: 'primitives-no-imports', + severity: 'error', + comment: 'packages/types is Layer 0. It cannot import anything from within the monorepo.', + from: { path: '^packages/types/' }, + to: { path: '^(packages|connectors|apps)/', pathNot: '^packages/types/' } + }, + { + name: 'core-utilities-isolation', + severity: 'error', + comment: 'Layer 1 (shared, utils) can only import from Layer 0 (types).', + from: { path: '^packages/(shared|utils)/' }, + to: { path: '^(packages|connectors|apps)/', pathNot: '^packages/(types|shared|utils)/' } + } + ], + options: { + doNotFollow: { + path: 'node_modules' + }, + tsConfig: { + fileName: 'tsconfig.base.json' + } + } +}; diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..4bd3bd8 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true + +[*.md] +trim_trailing_whitespace = false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..cb00400 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,50 @@ +name: CI + +on: + pull_request: + branches: [main] + +jobs: + validate: + name: Validate, Lint, and Build + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup pnpm + uses: pnpm/action-setup@v3 + with: + version: 9 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'pnpm' + + - name: Install Dependencies + run: pnpm install --frozen-lockfile + + - name: Lint Commits + run: npx commitlint --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }} + + - name: Enforce Architecture (Dependency Cruiser) + run: pnpm run depcruise + + - name: Format Check + run: pnpm run format:check + + - name: Lint + run: pnpm run lint + + - name: Typecheck + run: pnpm run typecheck + + - name: Test + run: pnpm run test + + - name: Build + run: pnpm run build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..47c70ec --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,46 @@ +name: Release + +on: + push: + branches: + - main + +concurrency: ${{ github.workflow }}-${{ github.ref }} + +jobs: + release: + name: Version and Publish + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup pnpm + uses: pnpm/action-setup@v3 + with: + version: 9 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'pnpm' + + - name: Install Dependencies + run: pnpm install --frozen-lockfile + + - name: Build All Packages + run: pnpm run build + + - name: Create Release Pull Request or Publish + id: changesets + uses: changesets/action@v1 + with: + publish: pnpm run publish-packages + commit: 'chore(release): version packages' + title: 'chore(release): version packages' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..306f915 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +node_modules/ +dist/ +lib/ +.turbo/ +coverage/ +.env +.env.local +*.log +.DS_Store diff --git a/.lintstagedrc.json b/.lintstagedrc.json new file mode 100644 index 0000000..76f7ce9 --- /dev/null +++ b/.lintstagedrc.json @@ -0,0 +1,4 @@ +{ + "*.{ts,tsx}": ["eslint --fix", "prettier --write"], + "*.{md,json,yaml,yml}": ["prettier --write"] +} diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..4168b4e --- /dev/null +++ b/.prettierignore @@ -0,0 +1,7 @@ +node_modules/ +dist/ +lib/ +.turbo/ +coverage/ +.claude/ +pnpm-lock.yaml diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000..6789c6c --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,11 @@ +{ + "printWidth": 100, + "tabWidth": 2, + "useTabs": false, + "semi": true, + "singleQuote": true, + "trailingComma": "all", + "bracketSpacing": true, + "arrowParens": "always", + "endOfLine": "lf" +} diff --git a/PROJECT_CHARTER.md b/PROJECT_CHARTER.md new file mode 100644 index 0000000..0712399 --- /dev/null +++ b/PROJECT_CHARTER.md @@ -0,0 +1,206 @@ +# Knowledge Extractor Project Charter + +**Version:** 1.0 + +--- + +# Purpose + +Knowledge Extractor exists to solve a simple but increasingly common problem: + +People consume thousands of pieces of valuable information across the internet, but very little of it becomes searchable, reusable knowledge. + +Most content is trapped inside social media platforms, videos, screenshots, PDFs, articles, and other proprietary interfaces. + +Knowledge Extractor transforms that fragmented information into structured, portable, searchable knowledge that belongs to the user. + +--- + +# Vision + +Build a modular knowledge ingestion platform capable of collecting, extracting, and normalizing information from any supported source into a unified knowledge representation. + +The long-term vision is to make personal knowledge portable, searchable, AI-ready, and independent of any individual platform. + +Instagram is only the first connector. + +The platform should eventually support any information source where users legitimately have access to content. + +--- + +# Target Users + +Knowledge Extractor is designed for people who actively collect information for learning or work. + +Examples include: + +- Engineers +- Students +- Researchers +- Technical writers +- Designers +- Entrepreneurs +- Content creators +- Lifelong learners + +Anyone who saves information today and struggles to find it later should benefit from this project. + +--- + +# Problems We Intend to Solve + +Knowledge Extractor aims to solve problems such as: + +- Saved information becoming impossible to search. +- Valuable ideas hidden inside images or videos. +- Platform lock-in preventing data portability. +- Repeatedly rediscovering the same content. +- Fragmented knowledge spread across multiple services. +- Difficulty preparing saved content for AI workflows or personal knowledge systems. + +The project focuses on extraction, normalization, and portability rather than content consumption. + +--- + +# Problems We Will Not Solve + +Knowledge Extractor will not: + +- Automate social media engagement. +- Like, comment, or message on behalf of users. +- Circumvent authentication or access restrictions. +- Scrape information users are not authorized to access. +- Attempt to replace official APIs where they are available and appropriate. +- Become another note-taking application. + +The project extracts knowledge. It does not attempt to become a complete productivity platform. + +--- + +# Core Principles + +## User Ownership + +Extracted knowledge belongs to the user. + +The platform should make it easy to export, migrate, and reuse extracted data without vendor lock-in. + +--- + +## Connector Independence + +Every connector should be isolated. + +Platform-specific logic must never leak into shared packages. + +Adding a new connector should require minimal changes outside that connector. + +--- + +## AI Independence + +AI enrichment is optional. + +Extraction must function without AI. + +Users should be free to choose any downstream AI workflow or model. + +--- + +## Modularity + +Every subsystem should be independently replaceable. + +Storage providers. + +OCR engines. + +Export formats. + +Connectors. + +AI pipelines. + +No implementation should assume a single permanent technology choice. + +--- + +## Transparency + +Extraction pipelines should be deterministic, inspectable, and understandable. + +Hidden transformations should be avoided whenever possible. + +--- + +# Success Criteria + +The project succeeds if it can: + +- Reliably discover user content. +- Extract structured information. +- Recover text from media when appropriate. +- Normalize data into a consistent schema. +- Export portable formats. +- Support multiple independent connectors. +- Remain maintainable as new platforms are added. + +Success is measured by engineering quality, extensibility, correctness, and usefulness—not by the number of supported platforms. + +--- + +# Long-Term Direction + +Knowledge Extractor should evolve into a universal knowledge ingestion engine. + +Every supported source should follow the same lifecycle: + +Discover + +↓ + +Extract + +↓ + +Normalize + +↓ + +Store + +↓ + +Export + +↓ + +(Optional) AI Enrichment + +↓ + +User-Owned Knowledge + +This architecture allows future support for new platforms without redesigning the core system. + +--- + +# Definition of Done + +A feature is considered complete only when: + +- It follows the architectural contracts. +- It is tested. +- It is documented. +- It does not introduce unnecessary coupling. +- It improves the platform without compromising future extensibility. + +Implementation quality is valued over implementation speed. + +--- + +# Guiding Principle + +Knowledge should outlive the platforms that originally contained it. + +Knowledge Extractor exists to make that possible. diff --git a/PROJECT_CONTEXT.md b/PROJECT_CONTEXT.md new file mode 100644 index 0000000..1bb2869 --- /dev/null +++ b/PROJECT_CONTEXT.md @@ -0,0 +1,388 @@ +# Knowledge Extractor — Project Context + +Version: 0.1.0 + +Status: Active Development + +--- + +# Mission + +Knowledge Extractor is a modular knowledge ingestion platform. + +The project is NOT an Instagram scraper. + +Instagram is only the first connector. + +The long-term objective is to build an extensible platform capable of extracting structured knowledge from multiple information sources and transforming it into a searchable personal knowledge base. + +Future connectors include: + +- Instagram +- Reddit +- X (Twitter) +- LinkedIn +- YouTube +- PDFs +- Web Articles +- Additional connectors as required + +--- + +# Core Philosophy + +Separate extraction from processing. + +Separate platform logic from connector logic. + +Separate knowledge extraction from AI enrichment. + +Every subsystem should be replaceable without affecting unrelated parts of the architecture. + +--- + +# Primary Objectives + +The platform should: + +- Discover content +- Extract structured information +- Download media +- Perform OCR +- Normalize extracted information +- Store extracted knowledge +- Export structured datasets +- Support downstream AI enrichment + +The extension is responsible only for collection. + +AI processing happens outside the browser. + +--- + +# Non Goals + +The project is NOT intended to: + +- Automate social media interactions +- Perform engagement +- Send messages +- Like posts +- Follow users +- Circumvent authentication +- Replace official APIs where they exist + +The system operates only on content that the authenticated user already has access to. + +--- + +# High Level Architecture + +User + +↓ + +Extension UI + +↓ + +Background Worker + +↓ + +Connector + +↓ + +Extraction Engine + +↓ + +Storage Engine + +↓ + +Export Engine + +↓ + +AI Pipeline (External) + +↓ + +Knowledge Base + +--- + +# Connector Philosophy + +Every source is implemented as an independent connector. + +A connector owns: + +- navigation +- selectors +- extraction logic +- normalization rules +- source-specific models + +A connector must never contain: + +- storage logic +- OCR implementation +- exporter logic +- AI logic + +The connector only knows how to transform a source into a generic extracted document. + +--- + +# Shared Packages + +packages/types + +Contains only interfaces and shared models. + +packages/shared + +Shared utilities. + +Configuration. + +Logging. + +Constants. + +Errors. + +packages/extractor + +Extraction pipeline. + +Validation. + +Normalization. + +packages/storage + +Persistence layer. + +IndexedDB. + +Future storage providers. + +packages/exporters + +JSON + +Markdown + +CSV + +SQLite + +Future exporters. + +packages/ocr + +OCR abstraction. + +Image preprocessing. + +OCR providers. + +packages/ai + +External AI integration. + +Prompt builders. + +Knowledge transformation. + +--- + +# Architectural Rules + +Rule 1 + +Instagram-specific code never leaves connectors/instagram. + +Rule 2 + +No package may depend on apps/. + +Rule 3 + +Shared packages must not import connectors. + +Rule 4 + +All communication should happen through interfaces. + +Rule 5 + +Avoid circular dependencies. + +Rule 6 + +Prefer composition over inheritance. + +Rule 7 + +Every new subsystem must be independently testable. + +Rule 8 + +No business logic inside the popup UI. + +--- + +# Development Principles + +Every feature follows: + +Research + +↓ + +Architecture + +↓ + +Implementation + +↓ + +Review + +↓ + +Testing + +↓ + +Documentation + +↓ + +Commit + +Never skip architectural reasoning. + +Never implement directly without understanding dependencies. + +--- + +# Current Development Stage + +Sprint 0 + +Goal: + +Build the engineering platform. + +Current priorities: + +- monorepo setup +- build tooling +- TypeScript configuration +- shared interfaces +- extension bootstrap +- message passing +- project infrastructure + +No extraction logic should be implemented until the platform is stable. + +--- + +# Future Milestones + +M0 + +Development Platform + +M1 + +Instagram Discovery + +M2 + +Metadata Extraction + +M3 + +Media Extraction + +M4 + +OCR + +M5 + +Export System + +M6 + +Incremental Sync + +M7 + +AI Enrichment + +M8 + +Additional Connectors + +--- + +# Long-Term Vision + +Knowledge Extractor should evolve into a universal knowledge ingestion platform. + +The same extraction pipeline should support: + +Instagram + +↓ + +Reddit + +↓ + +LinkedIn + +↓ + +X + +↓ + +YouTube + +↓ + +PDF + +↓ + +Articles + +↓ + +Unified Knowledge Representation + +↓ + +Search + +↓ + +AI + +↓ + +Personal Knowledge Base + +No architectural decision should make future connectors significantly harder to implement. + +Every implementation should optimize for maintainability, extensibility, and long-term engineering quality over short-term convenience. diff --git a/apps/extension/manifest.json b/apps/extension/manifest.json new file mode 100644 index 0000000..80c2620 --- /dev/null +++ b/apps/extension/manifest.json @@ -0,0 +1,19 @@ +{ + "manifest_version": 3, + "name": "Knowledge Extractor", + "version": "0.1.0", + "action": { + "default_popup": "public/index.html" + }, + "background": { + "service_worker": "src/background/index.ts", + "type": "module" + }, + "content_scripts": [ + { + "matches": ["*://*.instagram.com/*"], + "js": ["src/content/index.ts"] + } + ], + "permissions": ["activeTab", "scripting"] +} diff --git a/apps/extension/package.json b/apps/extension/package.json new file mode 100644 index 0000000..8c09a9c --- /dev/null +++ b/apps/extension/package.json @@ -0,0 +1,27 @@ +{ + "name": "@knowledge-extractor/extension", + "version": "0.1.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "typecheck": "tsc --noEmit", + "lint": "eslint src" + }, + "dependencies": { + "@knowledge-extractor/types": "workspace:*", + "@knowledge-extractor/shared": "workspace:*", + "@knowledge-extractor/storage": "workspace:*", + "@knowledge-extractor/connector-instagram": "workspace:*", + "react": "^18.3.1", + "react-dom": "^18.3.1" + }, + "devDependencies": { + "@crxjs/vite-plugin": "^2.0.0-beta.23", + "@types/chrome": "^0.0.268", + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "@vitejs/plugin-react": "^4.3.1", + "vite": "^5.3.1" + } +} diff --git a/apps/extension/public/index.html b/apps/extension/public/index.html new file mode 100644 index 0000000..156144d --- /dev/null +++ b/apps/extension/public/index.html @@ -0,0 +1,12 @@ + + + + + + Extractor Popup + + +
+ + + diff --git a/apps/extension/src/background/crawl-controller.ts b/apps/extension/src/background/crawl-controller.ts new file mode 100644 index 0000000..1368170 --- /dev/null +++ b/apps/extension/src/background/crawl-controller.ts @@ -0,0 +1,205 @@ +import { Logger, MetricsCollector, DiagnosticsCollector } from '@knowledge-extractor/shared'; +import { IDiscoveredResource, ICrawlSession } from '@knowledge-extractor/types'; +import { InstagramConnector } from '@knowledge-extractor/connector-instagram'; +import { InMemoryStorage } from '@knowledge-extractor/storage'; +import { SessionManager } from './session-manager.js'; +import { Scheduler } from './scheduler.js'; + +/** + * Supreme Orchestrator of the Crawl Lifecycle. + * Pipeline: Discovery → Queue → Scheduler → Navigator → Extractor → Normalizer → Persistence + */ +export class CrawlController { + private readonly logger = new Logger('CrawlController'); + private readonly sessionManager = new SessionManager(); + private readonly scheduler = new Scheduler(); + + // External dependencies + private metrics: MetricsCollector; + private diagnostics: DiagnosticsCollector; + private connector: InstagramConnector; + private storage: InMemoryStorage; + + private pollTimer: ReturnType | null = null; + private isProcessing = false; + + constructor( + metrics: MetricsCollector, + diagnostics: DiagnosticsCollector, + connector: InstagramConnector, + storage: InMemoryStorage, + ) { + this.metrics = metrics; + this.diagnostics = diagnostics; + this.connector = connector; + this.storage = storage; + } + + async init(): Promise { + await this.sessionManager.init(); + this.logger.info('CrawlController initialized'); + } + + async startCrawl(): Promise { + const session = this.sessionManager.startNewSession(); + this.scheduler.clear(); + this.metrics.reset(); + + // We'll broadcast this event externally + this.broadcastEvent('CRAWL_STARTED', { sessionId: session.sessionId }); + + this.startProcessingLoop(); + return session; + } + + async pauseCrawl(): Promise { + await this.sessionManager.update({ isPaused: true }); + this.stopProcessingLoop(); + this.broadcastEvent('CRAWL_PAUSED', {}); + } + + async resumeCrawl(): Promise { + await this.sessionManager.update({ isPaused: false }); + this.startProcessingLoop(); + this.broadcastEvent('CRAWL_RESUMED', {}); + } + + async cancelCrawl(): Promise { + await this.sessionManager.update({ isCancelled: true, isRunning: false }); + this.stopProcessingLoop(); + this.scheduler.clear(); + this.broadcastEvent('CRAWL_CANCELLED', {}); + } + + async handleDiscoveryBatch( + batch: Array<{ resource: IDiscoveredResource; fingerprint: string }>, + ): Promise { + const session = this.sessionManager.getSession(); + if (!session || !session.isRunning || session.isPaused) return; + + this.broadcastEvent('DISCOVERY_BATCH', { count: batch.length }); + + for (const item of batch) { + // Default priority 0. Newest discovered items might get higher priority in future if we want LIFO + const task = this.scheduler.enqueue(item.resource.targetUri, 0); + if (task) { + await this.sessionManager.increment('discovered'); + await this.sessionManager.increment('queued'); + this.broadcastEvent('RESOURCE_QUEUED', { + targetUri: task.targetUri, + priority: task.priority, + }); + } + } + } + + private startProcessingLoop(): void { + if (this.pollTimer) return; + this.pollTimer = setInterval(() => this.processNext(), 1000); + } + + private stopProcessingLoop(): void { + if (this.pollTimer) { + clearInterval(this.pollTimer); + this.pollTimer = null; + } + } + + private async processNext(): Promise { + if (this.isProcessing) return; + const session = this.sessionManager.getSession(); + if (!session || !session.isRunning || session.isPaused) return; + + const task = this.scheduler.getNextTask(); + if (!task) { + // If we are out of tasks, maybe trigger navigation scroll? + // This will be orchestrated via Navigator later. + return; + } + + this.isProcessing = true; + try { + await this.sessionManager.update({ currentResource: task.targetUri }); + this.broadcastEvent('NAVIGATION_STARTED', { targetUri: task.targetUri }); + + // PHASE 2: Coordinate with Navigator to open the resource + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + if (!tabs[0]?.id) throw new Error('No active tab'); + + const navResponse = await chrome.tabs.sendMessage(tabs[0].id, { + action: 'NAVIGATE_OPEN', + data: { targetUri: task.targetUri }, + }); + + if (!navResponse?.success) { + throw new Error(navResponse?.error || 'Navigator failed to open resource'); + } + + this.scheduler.markExtracting(task.id); + this.broadcastEvent('EXTRACTION_STARTED', { targetUri: task.targetUri }); + + // Execute Extraction + const extractStart = performance.now(); + const extractResponse = await chrome.tabs.sendMessage(tabs[0].id, { + action: 'EXTRACT_RESOURCE', + data: { targetUri: task.targetUri }, + }); + const extractionDurationMs = performance.now() - extractStart; + + if (!extractResponse?.success) { + throw new Error(extractResponse?.error || 'Extraction failed'); + } + + // Close resource (modal) after extraction + const closeResponse = await chrome.tabs.sendMessage(tabs[0].id, { action: 'NAVIGATE_CLOSE' }); + + await this.sessionManager.addMetrics({ + modalOpenLatencyMs: navResponse.openLatencyMs, + domStabilizationTimeMs: navResponse.domStabilizeMs, + extractionDurationMs, + modalCloseDurationMs: closeResponse?.closeDurationMs, + }); + + // Execute Normalization + const normalized = await this.connector.normalize( + extractResponse.data as Parameters[0], + ); + this.broadcastEvent('RESOURCE_NORMALIZED', normalized); + + // Execute Persistence + const tx = await this.storage.beginTransaction(); + await this.storage.saveResource(normalized, tx); + await tx.commit(); + this.broadcastEvent('RESOURCE_PERSISTED', { resourceId: normalized.id }); + + this.scheduler.markCompleted(task.id); + await this.sessionManager.increment('extracted'); + this.broadcastEvent('EXTRACTION_COMPLETED', { + targetUri: task.targetUri, + resourceId: task.id, + }); + } catch (err) { + const errorMsg = String(err); + const updatedTask = this.scheduler.markFailed(task.id, errorMsg); + if (updatedTask?.state === 'failed') { + await this.sessionManager.increment('failed'); + } + await this.sessionManager.increment('totalRetries'); + this.broadcastEvent('RESOURCE_FAILED', { targetUri: task.targetUri, reason: errorMsg }); + } finally { + this.isProcessing = false; + // Clear the "currently processing" marker. Empty string = none + // (exactOptionalPropertyTypes forbids assigning explicit `undefined`). + await this.sessionManager.update({ currentResource: '' }); + } + } + + private broadcastEvent(action: string, payload: unknown): void { + // Also log internally + this.logger.debug(`[EVENT] ${action}`, payload); + // Push to extension messaging + chrome.runtime + .sendMessage({ action: 'SYSTEM_STATUS', data: { stage: action, payload } }) + .catch(() => {}); + } +} diff --git a/apps/extension/src/background/index.ts b/apps/extension/src/background/index.ts index e69de29..6985e05 100644 --- a/apps/extension/src/background/index.ts +++ b/apps/extension/src/background/index.ts @@ -0,0 +1,68 @@ +/** + * Background Worker — Pipeline Orchestrator (Alpha Diagnostic Build) + */ +import { Logger, MetricsCollector, DiagnosticsCollector } from '@knowledge-extractor/shared'; +import { IDiscoveredResource } from '@knowledge-extractor/types'; +import { CrawlController } from './crawl-controller.js'; +import { InstagramConnector } from '@knowledge-extractor/connector-instagram'; +import { InMemoryStorage } from '@knowledge-extractor/storage'; + +const logger = new Logger('BackgroundWorker'); +const metrics = new MetricsCollector(); +const diagnostics = new DiagnosticsCollector(); +const connector = new InstagramConnector(); +const storage = new InMemoryStorage(); +const controller = new CrawlController(metrics, diagnostics, connector, storage); + +// Initialize controller and session +controller.init().catch((err) => logger.error('Failed to init controller', err)); + +// ---- Message Dispatcher ----------------------------------------------------- +chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { + if (message.action === 'START_PIPELINE') { + controller + .startCrawl() + .then((session) => sendResponse({ success: true, session })) + .catch(() => sendResponse({ success: false })); + return true; + } + + if (message.action === 'PAUSE_PIPELINE') { + controller.pauseCrawl().then(() => sendResponse({ success: true })); + return true; + } + + if (message.action === 'RESUME_PIPELINE') { + controller.resumeCrawl().then(() => sendResponse({ success: true })); + return true; + } + + if (message.action === 'CANCEL_PIPELINE') { + controller.cancelCrawl().then(() => sendResponse({ success: true })); + return true; + } + + if (message.action === 'GET_SESSION') { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const sessionManager = (controller as any).sessionManager; + sendResponse(sessionManager?.getSession() || null); + return false; + } + + if (message.action === 'RESOURCES_DISCOVERED') { + controller + .handleDiscoveryBatch( + message.data as Array<{ resource: IDiscoveredResource; fingerprint: string }>, + ) + .catch((err) => logger.error('Discovery batch error', err)); + return false; + } + + if (message.action === 'EXPORT_DIAGNOSTICS') { + const report = diagnostics.buildReport(metrics.snapshot()); + sendResponse(report); + return false; + } + + return false; +}); diff --git a/apps/extension/src/background/scheduler.ts b/apps/extension/src/background/scheduler.ts new file mode 100644 index 0000000..e8e33b3 --- /dev/null +++ b/apps/extension/src/background/scheduler.ts @@ -0,0 +1,98 @@ +import { ICrawlTask, TaskState } from '@knowledge-extractor/types'; +import { Logger } from '@knowledge-extractor/shared'; + +export interface SchedulerOptions { + maxAttempts: number; + baseBackoffMs: number; +} + +export class Scheduler { + private readonly logger = new Logger('Scheduler'); + private tasks = new Map(); + private readonly opts: SchedulerOptions; + + constructor(opts: Partial = {}) { + this.opts = { + maxAttempts: opts.maxAttempts ?? 3, + baseBackoffMs: opts.baseBackoffMs ?? 1000, + }; + } + + enqueue(targetUri: string, priority = 0): ICrawlTask | null { + if (this.tasks.has(targetUri)) { + return null; // Already tracked + } + + const task: ICrawlTask = { + id: targetUri, + targetUri, + state: TaskState.QUEUED, + priority, + attempts: 0, + maxAttempts: this.opts.maxAttempts, + }; + + this.tasks.set(targetUri, task); + this.logger.debug(`Enqueued task: ${targetUri} (Priority: ${priority})`); + return task; + } + + /** + * Retrieves the next task ready for extraction. + * Tasks with nextRetryAt > now are skipped. + * Tasks are sorted by priority (descending). + */ + getNextTask(): ICrawlTask | null { + const now = Date.now(); + + const readyTasks = Array.from(this.tasks.values()).filter( + (t) => t.state === TaskState.QUEUED && (!t.nextRetryAt || t.nextRetryAt <= now), + ); + + if (readyTasks.length === 0) return null; + + readyTasks.sort((a, b) => b.priority - a.priority); + + const task = readyTasks[0]; + task.state = TaskState.OPENING; + return task; + } + + markExtracting(taskId: string): void { + const task = this.tasks.get(taskId); + if (task) task.state = TaskState.EXTRACTING; + } + + markCompleted(taskId: string): void { + const task = this.tasks.get(taskId); + if (task) task.state = TaskState.COMPLETED; + } + + markFailed(taskId: string, error: string): ICrawlTask | null { + const task = this.tasks.get(taskId); + if (!task) return null; + + task.attempts += 1; + task.lastError = error; + + if (task.attempts >= task.maxAttempts) { + task.state = TaskState.FAILED; + this.logger.warn(`Task permanently failed: ${taskId}`); + } else { + task.state = TaskState.QUEUED; + const backoff = this.opts.baseBackoffMs * Math.pow(2, task.attempts - 1); + task.nextRetryAt = Date.now() + backoff; + this.logger.info(`Task scheduled for retry in ${backoff}ms: ${taskId}`); + } + + return task; + } + + getPendingCount(): number { + return Array.from(this.tasks.values()).filter((t) => t.state === TaskState.QUEUED).length; + } + + clear(): void { + this.tasks.clear(); + } +} diff --git a/apps/extension/src/background/session-manager.ts b/apps/extension/src/background/session-manager.ts new file mode 100644 index 0000000..a034cdd --- /dev/null +++ b/apps/extension/src/background/session-manager.ts @@ -0,0 +1,104 @@ +import { ICrawlSession } from '@knowledge-extractor/types'; +import { Logger } from '@knowledge-extractor/shared'; + +/** + * Manages the persistent state of a crawl session using chrome.storage.session. + * The popup dashboard reads this state. + */ +export class SessionManager { + private readonly logger = new Logger('SessionManager'); + private readonly STORAGE_KEY = 'crawl_session'; + private session: ICrawlSession | null = null; + + async init(): Promise { + const data = await chrome.storage.session.get(this.STORAGE_KEY); + if (data[this.STORAGE_KEY]) { + this.session = data[this.STORAGE_KEY] as ICrawlSession; + this.logger.info(`Restored existing session: ${this.session.sessionId}`); + } else { + this.session = this.createEmptySession(); + await this.persist(); + this.logger.info(`Created new session: ${this.session.sessionId}`); + } + } + + startNewSession(): ICrawlSession { + this.session = this.createEmptySession(); + this.session.isRunning = true; + this.persist(); + return this.session; + } + + getSession(): ICrawlSession | null { + return this.session; + } + + async update(patch: Partial): Promise { + if (!this.session) return; + this.session = { ...this.session, ...patch }; + await this.persist(); + } + + async increment( + field: + | 'discovered' + | 'queued' + | 'extracted' + | 'failed' + | 'totalRetries' + | 'scrollFailures' + | 'selectorFailures', + by = 1, + ): Promise { + if (!this.session) return; + this.session[field] += by; + await this.persist(); + } + + async addMetrics(metrics: { + modalOpenLatencyMs?: number; + domStabilizationTimeMs?: number; + extractionDurationMs?: number; + modalCloseDurationMs?: number; + }): Promise { + if (!this.session) return; + if (metrics.modalOpenLatencyMs) + this.session.totalModalOpenLatencyMs += metrics.modalOpenLatencyMs; + if (metrics.domStabilizationTimeMs) + this.session.totalDomStabilizationTimeMs += metrics.domStabilizationTimeMs; + if (metrics.extractionDurationMs) + this.session.totalExtractionDurationMs += metrics.extractionDurationMs; + if (metrics.modalCloseDurationMs) + this.session.totalModalCloseDurationMs += metrics.modalCloseDurationMs; + await this.persist(); + } + + private createEmptySession(): ICrawlSession { + return { + sessionId: crypto.randomUUID(), + startedAt: new Date().toISOString(), + discovered: 0, + queued: 0, + extracted: 0, + failed: 0, + isRunning: false, + isPaused: false, + isCancelled: false, + totalModalOpenLatencyMs: 0, + totalDomStabilizationTimeMs: 0, + totalExtractionDurationMs: 0, + totalModalCloseDurationMs: 0, + totalRetries: 0, + scrollFailures: 0, + selectorFailures: 0, + }; + } + + private async persist(): Promise { + if (!this.session) return; + await chrome.storage.session.set({ [this.STORAGE_KEY]: this.session }); + + // Broadcast status to any open popups + chrome.runtime.sendMessage({ action: 'SESSION_UPDATED', data: this.session }).catch(() => {}); + } +} diff --git a/apps/extension/src/content/index.ts b/apps/extension/src/content/index.ts index e69de29..45e58ee 100644 --- a/apps/extension/src/content/index.ts +++ b/apps/extension/src/content/index.ts @@ -0,0 +1,157 @@ +/** + * Content Script — DOM Adapter Layer (Alpha Diagnostic Build) + * + * Responsibility: DOM access only. It locates the target `
` element and + * delegates ALL parsing to the Instagram Connector (the single runtime extraction + * implementation). It captures a trimmed DOM snapshot on failure for diagnostics. + */ +import { IDiscoveredResource } from '@knowledge-extractor/types'; +import { Logger, featureFlags, FeatureFlag, MetricsCollector } from '@knowledge-extractor/shared'; +import { DiscoveryEngine, InstagramConnector } from '@knowledge-extractor/connector-instagram'; +import { Navigator } from './navigator.js'; + +const logger = new Logger('ContentScript'); +const metrics = new MetricsCollector(); +const engine = new DiscoveryEngine(); +const connector = new InstagramConnector(); +const navigator = new Navigator(); + +const pendingQueue: Array<{ resource: IDiscoveredResource; fingerprint: string }> = []; +let flushTimer: ReturnType | null = null; + +// ---- Message listener ------------------------------------------------------- +chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { + if (message.action === 'RUN_PIPELINE') { + startDiscovery() + .then(() => sendResponse({ success: true })) + .catch((err: unknown) => { + logger.error('Pipeline error', err); + sendResponse({ success: false, error: String(err) }); + }); + return true; + } + + if (message.action === 'STOP_PIPELINE') { + engine.stop(); + sendResponse({ metrics: metrics.snapshot() }); + return true; + } + + if (message.action === 'EXTRACT_RESOURCE') { + const { targetUri } = (message.data ?? {}) as { targetUri: string }; + extractSingleResource(targetUri) + .then(sendResponse) + .catch((err: unknown) => sendResponse({ success: false, error: String(err) })); + return true; + } + + if (message.action === 'NAVIGATE_OPEN') { + const { targetUri } = (message.data ?? {}) as { targetUri: string }; + navigator + .openResource(targetUri) + .then((result) => sendResponse(result)) + .catch((err) => sendResponse({ success: false, error: String(err) })); + return true; + } + + if (message.action === 'NAVIGATE_CLOSE') { + navigator + .closeResource() + .then((result) => sendResponse(result)) + .catch((err) => sendResponse({ success: false, error: String(err) })); + return true; + } + + if (message.action === 'NAVIGATE_SCROLL') { + navigator + .scrollGrid() + .then((result) => sendResponse(result)) + .catch((err) => sendResponse({ success: false, error: String(err) })); + return true; + } + + return false; +}); + +// ---- Discovery orchestration ------------------------------------------------ +async function startDiscovery(): Promise { + if (!featureFlags.isEnabled(FeatureFlag.ENABLE_DISCOVERY)) { + logger.warn('Discovery disabled by feature flag'); + return; + } + + metrics.reset(); + logger.info('Starting DiscoveryEngine'); + + engine.start((resource, fingerprint) => { + metrics.recordDiscovered(); + pendingQueue.push({ resource, fingerprint }); + scheduleFlush(); + }); +} + +function scheduleFlush(): void { + if (flushTimer) return; + flushTimer = setTimeout(() => { + flushQueue(); + flushTimer = null; + }, 150); +} + +function flushQueue(): void { + if (pendingQueue.length === 0) return; + const batch = pendingQueue.splice(0, pendingQueue.length); + logger.debug(`Flushing ${batch.length} discovered resources to background`); + chrome.runtime.sendMessage({ action: 'RESOURCES_DISCOVERED', data: batch }).catch(() => {}); +} + +// ---- Per-resource extraction (DOM location only; parsing delegated) --------- +async function extractSingleResource(targetUri: string): Promise<{ + success: boolean; + data?: unknown; + domSnapshot?: string; + error?: string; +}> { + // DOM concern: locate the target
for this URI (modal or feed), + // falling back to the single-post detail view's lone
. + const target = findArticleForUri(targetUri) ?? document.querySelector('article'); + + if (!target) { + return { + success: false, + domSnapshot: document.body.innerHTML.slice(0, 2000), + error: 'No article element found for URI', + }; + } + + // Trimmed DOM snapshot for failure diagnostics. + const domSnapshot = target.outerHTML.slice(0, 2000); + + try { + // Parsing concern: delegated entirely to the connector's strategy chain. + const parsed = connector.extract(target); + return { success: true, data: parsed }; + } catch (err) { + return { success: false, domSnapshot, error: String(err) }; + } +} + +/** Locates the
in the live DOM whose permalink matches `targetUri`. */ +function findArticleForUri(targetUri: string): Element | undefined { + const pathname = safePathname(targetUri); + return Array.from(document.querySelectorAll('article')).find((a) => { + const link = + a.querySelector('a[href*="/p/"]') ?? + a.querySelector('a[href*="/reel/"]'); + if (!link) return false; + return link.href === targetUri || (pathname !== '' && link.href.includes(pathname)); + }); +} + +function safePathname(uri: string): string { + try { + return new URL(uri).pathname; + } catch { + return ''; + } +} diff --git a/apps/extension/src/content/navigator.ts b/apps/extension/src/content/navigator.ts new file mode 100644 index 0000000..c23b1c6 --- /dev/null +++ b/apps/extension/src/content/navigator.ts @@ -0,0 +1,150 @@ +import { Logger } from '@knowledge-extractor/shared'; + +/** + * Owns all browser state manipulation for the content script. + * Responsible for: scrolling, opening modals, closing modals, waiting. + */ +export class Navigator { + private readonly logger = new Logger('Navigator'); + + /** + * Scrolls the window down by one viewport height and waits for dynamic content to load. + */ + async scrollGrid(): Promise<{ success: boolean; stabilizeMs?: number }> { + const start = performance.now(); + const previousHeight = document.documentElement.scrollHeight; + + // Scroll down by 80% of viewport height to trigger lazy load but keep some overlap + window.scrollBy(0, window.innerHeight * 0.8); + + this.logger.debug('Scrolled down, waiting for DOM stabilization...'); + + // Wait for infinite scroll to trigger and render + await this.sleep(1500); + + const newHeight = document.documentElement.scrollHeight; + + // If the scroll height didn't change, we might be at the bottom + if (newHeight === previousHeight) { + // Try one more small scroll just in case + window.scrollBy(0, window.innerHeight * 0.2); + await this.sleep(1000); + if (document.documentElement.scrollHeight === previousHeight) { + this.logger.info('Reached end of grid (no new height after scroll)'); + return { success: false, stabilizeMs: performance.now() - start }; + } + } + + return { success: true, stabilizeMs: performance.now() - start }; + } + + /** + * Attempts to open the resource specified by targetUri in the current tab. + * Uses Option A (Modal navigation) for grid items. + */ + async openResource(targetUri: string): Promise<{ + success: boolean; + openLatencyMs?: number; + domStabilizeMs?: number; + error?: string; + }> { + // 1. Try to find the thumbnail link in the grid + const links = Array.from( + document.querySelectorAll('a[href*="/p/"], a[href*="/reel/"]'), + ); + const targetLink = links.find( + (l) => l.href === targetUri || l.href.includes(new URL(targetUri).pathname), + ); + + if (targetLink) { + this.logger.debug(`Clicking thumbnail for ${targetUri}`); + + // Ensure element is in view before clicking to avoid some overlay issues + targetLink.scrollIntoView({ block: 'center', behavior: 'instant' }); + await this.sleep(100); + + // We dispatch a click event + const clickTime = performance.now(); + targetLink.click(); + + // Wait for the modal article to appear + const modalLoaded = await this.waitForSelector('article[role="presentation"]', 5000); + const openLatencyMs = performance.now() - clickTime; + + if (!modalLoaded) { + this.logger.warn(`Modal failed to load for ${targetUri}`); + // Attempt to close if it's stuck half-open + await this.closeResource(); + return { success: false, openLatencyMs, error: 'Modal timeout' }; + } + + // Give it a brief moment for dynamic content (images, video) to hydrate + const stabilizeStart = performance.now(); + await this.sleep(500); + const domStabilizeMs = performance.now() - stabilizeStart; + return { success: true, openLatencyMs, domStabilizeMs }; + } + + // 2. If no link is found, we might already be on a feed/detail page where the article is fully loaded + const article = Array.from(document.querySelectorAll('article')).find((a) => { + const link = a.querySelector('a[href*="/p/"], a[href*="/reel/"]'); + return link && (link.href === targetUri || link.href.includes(new URL(targetUri).pathname)); + }); + + if (article) { + this.logger.debug(`Resource already open in feed for ${targetUri}`); + article.scrollIntoView({ block: 'center', behavior: 'instant' }); + const stabilizeStart = performance.now(); + await this.sleep(200); + return { + success: true, + openLatencyMs: 0, + domStabilizeMs: performance.now() - stabilizeStart, + }; + } + + this.logger.error(`Could not locate resource in DOM to open: ${targetUri}`); + return { success: false, error: 'Not found in DOM' }; + } + + /** + * Closes the currently open modal (if any). + */ + async closeResource(): Promise<{ success: boolean; closeDurationMs?: number }> { + const start = performance.now(); + // Instagram modal close button usually has an SVG with aria-label "Close" + const closeBtn = + document.querySelector('svg[aria-label="Close"]')?.closest('button') || + document.querySelector('div[role="dialog"] button'); + + if (closeBtn) { + this.logger.debug('Clicking close button on modal'); + closeBtn.click(); + await this.sleep(300); // Wait for modal to animate out + return { success: true, closeDurationMs: performance.now() - start }; + } else { + // Fallback: If there's a dialog but no close button, try pressing Escape + const dialog = document.querySelector('div[role="dialog"]'); + if (dialog) { + this.logger.debug('No close button found, dispatching Escape key'); + document.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true })); + await this.sleep(300); + return { success: true, closeDurationMs: performance.now() - start }; + } + } + return { success: true, closeDurationMs: 0 }; + } + + private sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + + private async waitForSelector(selector: string, timeoutMs: number): Promise { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + if (document.querySelector(selector)) return true; + await this.sleep(100); + } + return false; + } +} diff --git a/apps/extension/src/popup/index.tsx b/apps/extension/src/popup/index.tsx index e69de29..ff23188 100644 --- a/apps/extension/src/popup/index.tsx +++ b/apps/extension/src/popup/index.tsx @@ -0,0 +1,194 @@ +import React, { useState, useEffect, useCallback } from 'react'; +import { createRoot } from 'react-dom/client'; +import type { ICrawlSession } from '@knowledge-extractor/types'; + +const MetricsBadge = ({ label, value, warn }: { label: string; value: number; warn?: boolean }) => ( +
0 ? '#fff0f0' : '#f0f4ff', + borderRadius: 6, + minWidth: 60, + }} + > +
0 ? '#c00' : '#1a56db' }}> + {value} +
+
{label}
+
+); + +const Popup = () => { + const [session, setSession] = useState(null); + const [events, setEvents] = useState<{ stage: string; payload: unknown; ts: string }[]>([]); + + const refreshSession = useCallback(() => { + chrome.runtime.sendMessage({ action: 'GET_SESSION' }, (resp) => { + if (resp) setSession(resp as ICrawlSession); + }); + }, []); + + useEffect(() => { + // Hydrate on mount + refreshSession(); + + // Subscribe to events + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const listener = (message: any) => { + if (message.action === 'SESSION_UPDATED') { + setSession(message.data); + } else if (message.action === 'SYSTEM_STATUS') { + const { stage, payload } = message.data; + setEvents((prev) => + [{ stage, payload, ts: new Date().toLocaleTimeString() }, ...prev].slice(0, 50), + ); + } + }; + chrome.runtime.onMessage.addListener(listener); + return () => chrome.runtime.onMessage.removeListener(listener); + }, [refreshSession]); + + const sendAction = (action: string) => { + chrome.runtime.sendMessage({ action }, () => refreshSession()); + }; + + const isRunning = session?.isRunning && !session?.isPaused; + const isPaused = session?.isRunning && session?.isPaused; + + return ( +
+
+
Knowledge Extractor Crawler
+
Alpha Diagnostics Build
+
+ +
+ + + +
+ ● {isRunning ? 'RUNNING' : isPaused ? 'PAUSED' : 'IDLE'} +
+
+ +
+ + + + +
+ +
+
+ Event Stream +
+ {events.map((e, i) => ( +
+ {e.ts} + {e.stage} +
+ {JSON.stringify(e.payload)} +
+
+ ))} +
+
+ ); +}; + +const root = createRoot(document.getElementById('root')!); +root.render(); diff --git a/apps/extension/tsconfig.json b/apps/extension/tsconfig.json new file mode 100644 index 0000000..4aec297 --- /dev/null +++ b/apps/extension/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "noEmit": true, + "resolveJsonModule": true, + "jsx": "react-jsx", + "types": ["chrome", "vite/client"] + }, + "include": ["src/**/*", "vite.config.ts"] +} diff --git a/apps/extension/vite.config.ts b/apps/extension/vite.config.ts new file mode 100644 index 0000000..52b1458 --- /dev/null +++ b/apps/extension/vite.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; +import { crx } from '@crxjs/vite-plugin'; +import manifest from './manifest.json'; + +export default defineConfig({ + plugins: [react(), crx({ manifest })], +}); diff --git a/commitlint.config.js b/commitlint.config.js new file mode 100644 index 0000000..2291173 --- /dev/null +++ b/commitlint.config.js @@ -0,0 +1,3 @@ +export default { + extends: ['@commitlint/config-conventional'] +}; diff --git a/connectors/instagram/extractors/reels.ts b/connectors/instagram/extractors/reels.ts deleted file mode 100644 index e69de29..0000000 diff --git a/connectors/instagram/models/post.ts b/connectors/instagram/models/post.ts deleted file mode 100644 index e69de29..0000000 diff --git a/connectors/instagram/package.json b/connectors/instagram/package.json new file mode 100644 index 0000000..05ee678 --- /dev/null +++ b/connectors/instagram/package.json @@ -0,0 +1,22 @@ +{ + "name": "@knowledge-extractor/connector-instagram", + "version": "0.1.0", + "type": "module", + "main": "src/index.ts", + "scripts": { + "test": "vitest run", + "test:watch": "vitest", + "typecheck": "tsc --noEmit", + "lint": "eslint src tests" + }, + "dependencies": { + "@knowledge-extractor/shared": "workspace:*", + "@knowledge-extractor/types": "workspace:*" + }, + "devDependencies": { + "@types/jsdom": "^28.0.3", + "@vitest/coverage-v8": "^2.1.9", + "jsdom": "^29.1.1", + "vitest": "^2.1.9" + } +} diff --git a/connectors/instagram/pipeline/index.ts b/connectors/instagram/pipeline/index.ts deleted file mode 100644 index e69de29..0000000 diff --git a/connectors/instagram/selectors/index.ts b/connectors/instagram/selectors/index.ts deleted file mode 100644 index e69de29..0000000 diff --git a/connectors/instagram/src/discovery-engine.ts b/connectors/instagram/src/discovery-engine.ts new file mode 100644 index 0000000..603df70 --- /dev/null +++ b/connectors/instagram/src/discovery-engine.ts @@ -0,0 +1,123 @@ +import { IDiscoveredResource, IResourceFingerprint } from '@knowledge-extractor/types'; +import { Logger } from '@knowledge-extractor/shared'; +import { ResourceFingerprinter } from './fingerprinter.js'; + +type DiscoveryCallback = (resource: IDiscoveredResource, fingerprint: string) => void; + +/** + * The Discovery Engine monitors the Instagram DOM for post articles and grid items + * using a MutationObserver and maintains a deduplication registry to prevent + * repeated extraction of the same resource during infinite scrolling. + * + * It dynamically adapts to the current layout (Grid vs Feed vs Detail). + */ +export class DiscoveryEngine { + private readonly logger = new Logger('DiscoveryEngine'); + private readonly fingerprinter = new ResourceFingerprinter(); + private readonly seen = new Set(); + private observer: MutationObserver | null = null; + private callback: DiscoveryCallback | null = null; + + start(callback: DiscoveryCallback): void { + this.callback = callback; + this.logger.info('Discovery Engine starting'); + + this.scanDOM(document.body); + + this.observer = new MutationObserver((mutations) => { + let shouldScan = false; + for (const mutation of mutations) { + if (mutation.addedNodes.length > 0) { + shouldScan = true; + break; + } + } + if (shouldScan) { + this.scanDOM(document.body); + } + }); + + this.observer.observe(document.body, { + childList: true, + subtree: true, + }); + + this.logger.info('Discovery Engine active'); + } + + stop(): void { + this.observer?.disconnect(); + this.observer = null; + this.logger.info(`Discovery Engine stopped. Total unique resources found: ${this.seen.size}`); + } + + private scanDOM(root: Element): void { + // 1. Grid layout: Look for post links inside typical grid containers + const gridLinks = Array.from( + root.querySelectorAll('a[href*="/p/"], a[href*="/reel/"]'), + ); + for (const link of gridLinks) { + // Exclude author profile links, explore tags, etc. + if (this.isValidGridLink(link)) { + this.processResource(link.href, link); + } + } + + // 2. Feed / Detail layout: Look for full article elements + const articles = root.matches('article') + ? [root] + : Array.from(root.querySelectorAll('article')); + + for (const article of articles) { + const link = + article.querySelector('a[href*="/p/"]') ?? + article.querySelector('a[href*="/reel/"]'); + if (link) { + this.processResource(link.href, article); + } + } + } + + private isValidGridLink(link: HTMLAnchorElement): boolean { + // Basic heuristic: grid items usually contain an image and aren't inside headers + return link.querySelector('img') !== null && link.closest('header') === null; + } + + private processResource(sourceUri: string, contextEl: Element): void { + // Normalize URL to strip query params + let cleanUri = sourceUri; + try { + const url = new URL(sourceUri); + cleanUri = url.origin + url.pathname; + } catch { + // ignore + } + + const authorEl = contextEl.querySelector('header a'); + const imgs = contextEl.querySelectorAll('img'); + + const fpInput: IResourceFingerprint['inputs'] = { + sourceUri: cleanUri, + mediaCount: imgs.length, + }; + const authorHandle = authorEl?.textContent?.trim(); + if (authorHandle) fpInput.authorHandle = authorHandle; + const captionPreview = contextEl.querySelector('h1, span')?.textContent?.slice(0, 64); + if (captionPreview) fpInput.captionPreview = captionPreview; + + const fp = this.fingerprinter.fingerprint(fpInput); + + if (this.seen.has(fp.hash)) { + return; + } + + this.seen.add(fp.hash); + this.logger.info(`Discovered new resource: ${cleanUri} (fp=${fp.hash})`); + + this.callback?.({ targetUri: cleanUri, providerName: 'instagram' }, fp.hash); + } + + getDiscoveredCount(): number { + return this.seen.size; + } +} diff --git a/connectors/instagram/src/fingerprinter.ts b/connectors/instagram/src/fingerprinter.ts new file mode 100644 index 0000000..33da1b1 --- /dev/null +++ b/connectors/instagram/src/fingerprinter.ts @@ -0,0 +1,30 @@ +import { IResourceFingerprint } from '@knowledge-extractor/types'; + +/** + * Computes a deterministic fingerprint for a raw Instagram DOM node + * to facilitate deduplication before normalization. + * + * Uses a simple, synchronous djb2-style hash since we operate in a content + * script context and cannot use the Web Crypto API without async overhead. + */ +export class ResourceFingerprinter { + fingerprint(inputs: IResourceFingerprint['inputs']): IResourceFingerprint { + const raw = [ + inputs.sourceUri ?? '', + inputs.authorHandle ?? '', + String(inputs.mediaCount ?? 0), + (inputs.captionPreview ?? '').slice(0, 64), + ].join('|'); + + const hash = this.djb2(raw); + return { hash, inputs }; + } + + private djb2(str: string): string { + let h = 5381; + for (let i = 0; i < str.length; i++) { + h = ((h << 5) + h + str.charCodeAt(i)) >>> 0; + } + return h.toString(16).padStart(8, '0'); + } +} diff --git a/connectors/instagram/src/index.ts b/connectors/instagram/src/index.ts new file mode 100644 index 0000000..9bcb994 --- /dev/null +++ b/connectors/instagram/src/index.ts @@ -0,0 +1,78 @@ +import { IConnector, IResource } from '@knowledge-extractor/types'; +import { Logger } from '@knowledge-extractor/shared'; +import { StrategyChain } from './strategy-chain.js'; +import { InstagramParser, InstagramNormalizer } from './parser.js'; +import { + SemanticArticleStrategy, + DataAttributeStrategy, + StructuralHeuristicStrategy, + ArticleElement, +} from './strategies.js'; +import { IInstagramParsedPost } from './types.js'; + +export { DiscoveryEngine } from './discovery-engine.js'; +export { ResourceFingerprinter } from './fingerprinter.js'; +export type { IInstagramParsedPost, InstagramPostLayout } from './types.js'; + +/** + * The Instagram Connector — public entry point and the single runtime + * extraction implementation for Instagram. + * + * Architecture: + * DOM Adapter (Content Script) — provides the raw `
` Element only + * → InstagramConnector.extract() [StrategyChain → InstagramParser.enrich] + * → IInstagramParsedPost (raw, Instagram-shaped) + * → InstagramConnector.normalize() [InstagramNormalizer → IResource] + * + * Extraction (DOM → raw) runs where the DOM lives (content script); normalization + * (raw → domain) is pure and may run anywhere (today: background worker). + */ +export class InstagramConnector implements IConnector { + public readonly providerName = 'instagram'; + + private readonly logger = new Logger('InstagramConnector'); + private readonly chain = new StrategyChain( + 'instagram-article', + [new SemanticArticleStrategy(), new DataAttributeStrategy(), new StructuralHeuristicStrategy()], + ); + private readonly parser = new InstagramParser(); + private readonly normalizer = new InstagramNormalizer(); + + /** + * Validates whether this connector can handle the given URI. + */ + canHandle(uri: string): boolean { + return /(^|\.)instagram\.com$/.test(this.hostOf(uri)) || /\/(p|reel)\//.test(uri); + } + + /** + * Extracts a raw, Instagram-shaped record from a DOM `
` element. + * This is the single runtime extraction path: StrategyChain → Parser.enrich. + * Throws `PlatformError` (PARSE_ERROR) if every strategy is exhausted. + * + * @param article A DOM `
` element from the Instagram page. + */ + extract(article: ArticleElement): IInstagramParsedPost { + this.logger.debug('Extracting article via strategy chain'); + const raw = this.chain.execute(article); + return this.parser.enrich(raw); + } + + /** + * Normalizes a raw `IInstagramParsedPost` into the strict domain `IResource`. + * Idempotent re-enrichment guards against callers passing un-enriched records + * (e.g. fixtures/tests that build the raw shape directly). + */ + async normalize(post: IInstagramParsedPost): Promise { + const enriched = this.parser.enrich(post); + return this.normalizer.normalize(enriched); + } + + private hostOf(uri: string): string { + try { + return new URL(uri).host; + } catch { + return ''; + } + } +} diff --git a/connectors/instagram/src/parser.ts b/connectors/instagram/src/parser.ts new file mode 100644 index 0000000..b42377c --- /dev/null +++ b/connectors/instagram/src/parser.ts @@ -0,0 +1,107 @@ +import { IResource, IMedia, ResourceState, MediaType, BlockType } from '@knowledge-extractor/types'; +import { Logger } from '@knowledge-extractor/shared'; +import { IInstagramParsedPost } from './types.js'; + +/** + * The Instagram Parser understands Instagram-specific DOM and structured data. + * It accepts a parsed `IInstagramParsedPost` and enriches it before + * handing off to the Connector for domain normalization. + * + * Responsibility: Instagram semantics. + * NOT responsible for: domain model creation (that is the Connector's job). + */ +export class InstagramParser { + private readonly logger = new Logger('InstagramParser'); + + /** + * Validates and enriches a parsed post. Returns the same type with + * any correctable fields filled in. + */ + enrich(post: IInstagramParsedPost): IInstagramParsedPost { + this.logger.debug(`Enriching parsed post: ${post.sourceUri}`); + + // Deduplicate media URIs while preserving order + const uniqueMedia = [...new Set(post.mediaUris)]; + + // Detect layout if it came through as 'unknown' + let layout = post.layout; + if (layout === 'unknown') { + if (post.videoUri) layout = 'reel'; + else if (uniqueMedia.length > 1) layout = 'carousel'; + else layout = 'single-image'; + } + + return { ...post, mediaUris: uniqueMedia, layout }; + } +} + +/** + * Normalizes an enriched `IInstagramParsedPost` into a strict domain `IResource`. + * Understands the platform contracts; does not understand Instagram specifics. + */ +export class InstagramNormalizer { + private readonly logger = new Logger('InstagramNormalizer'); + + normalize(post: IInstagramParsedPost): IResource { + this.logger.info(`Normalizing post: ${post.externalId}`); + + const media: IMedia[] = post.mediaUris.map((uri, idx) => ({ + id: `${post.externalId}_media_${idx}`, + type: post.videoUri === uri ? MediaType.VIDEO : MediaType.IMAGE, + sourceUri: uri, + })); + + const children: IResource[] = + post.layout === 'carousel' + ? (post.slideUris ?? post.mediaUris).map((uri, idx) => ({ + id: `ig_${post.externalId}_slide_${idx}`, + kind: 'instagram-slide', + state: ResourceState.EXTRACTED, + completeness: { thumbnail: true, metadata: true, media: true, ocr: false }, + source: { + providerName: 'instagram', + externalId: `${post.externalId}_slide_${idx}`, + originalUri: post.sourceUri, + extractedAt: new Date().toISOString(), + }, + content: [], + media: [{ id: `slide_media_${idx}`, type: MediaType.IMAGE, sourceUri: uri }], + })) + : []; + + const result: IResource = { + id: `ig_${post.externalId || this.djb2(post.sourceUri)}`, + kind: post.layout === 'reel' ? 'instagram-reel' : 'instagram-post', + state: ResourceState.EXTRACTED, + completeness: { thumbnail: true, metadata: true, media: true, ocr: false }, + source: { + providerName: 'instagram', + externalId: post.externalId ?? 'unknown', + originalUri: post.sourceUri, + extractedAt: new Date().toISOString(), + metadata: { layout: post.layout }, + }, + content: post.textContent ? [{ type: BlockType.TEXT, value: post.textContent }] : [], + media, + children, + ...(post.authorHandle + ? { + author: { + handle: post.authorHandle, + ...(post.authorDisplayName ? { displayName: post.authorDisplayName } : {}), + }, + } + : {}), + }; + + return result; + } + + private djb2(str: string): string { + let h = 5381; + for (let i = 0; i < str.length; i++) { + h = ((h << 5) + h + str.charCodeAt(i)) >>> 0; + } + return h.toString(16).padStart(8, '0'); + } +} diff --git a/connectors/instagram/src/strategies.ts b/connectors/instagram/src/strategies.ts new file mode 100644 index 0000000..a9d54d1 --- /dev/null +++ b/connectors/instagram/src/strategies.ts @@ -0,0 +1,188 @@ +import { IExtractionStrategy, IStrategyResult } from '@knowledge-extractor/types'; +import { IInstagramParsedPost } from './types.js'; + +export type ArticleElement = Element; + +// --------------------------------------------------------------------------- +// Strategy A: Semantic article selectors (highest confidence) +// Targets stable landmark roles and well-known data attributes. +// --------------------------------------------------------------------------- +export class SemanticArticleStrategy implements IExtractionStrategy< + ArticleElement, + IInstagramParsedPost +> { + readonly strategyName = 'SemanticArticleStrategy'; + + execute(article: ArticleElement): IStrategyResult { + const link = + article.querySelector('a[href*="/p/"]') ?? + article.querySelector('a[href*="/reel/"]'); + + if (!link) { + return { + applicable: false, + confidence: 0, + failureReason: 'No post/reel permalink found via semantic selector', + }; + } + + const sourceUri = link.href; + const externalId = this.extractId(sourceUri); + const isReel = sourceUri.includes('/reel/'); + + const authorEl = article.querySelector('header a[role="link"]'); + const authorHandle = authorEl?.textContent?.trim() ?? undefined; + const authorDisplayName = + authorEl + ?.closest('header') + ?.querySelector('span:last-child') + ?.textContent?.trim() ?? undefined; + + const captionEl = article.querySelector( + '[data-testid="post-comment-root"] span, h1', + ); + const textContent = captionEl?.textContent?.trim() ?? undefined; + + const timeEl = article.querySelector('time[datetime]'); + const publishedAt = timeEl?.getAttribute('datetime') ?? undefined; + + const videoEl = article.querySelector('video'); + const videoUri = videoEl?.src || videoEl?.querySelector('source')?.src; + + const imgEls = Array.from( + article.querySelectorAll('img[srcset], img[src]'), + ).filter((img) => !img.src.includes('avatar') && img.width > 100); + + const mediaUris = [...new Set(imgEls.map((img) => img.src).filter(Boolean))]; + const slideUris = imgEls.length > 1 ? mediaUris : undefined; + + const dots = article.querySelectorAll('[aria-label*="slide"], [class*="dot"]'); + const layout = isReel + ? 'reel' + : dots.length > 1 || (slideUris && slideUris.length > 1) + ? 'carousel' + : videoUri + ? 'reel' + : 'single-image'; + + const data: IInstagramParsedPost = { + providerName: 'instagram', + sourceUri, + externalId, + mediaUris: videoUri ? [videoUri, ...mediaUris] : mediaUris, + layout, + }; + if (authorHandle) data.authorHandle = authorHandle; + if (authorDisplayName) data.authorDisplayName = authorDisplayName; + if (textContent) data.textContent = textContent; + if (publishedAt) data.publishedAt = publishedAt; + if (slideUris) data.slideUris = slideUris; + if (videoUri) data.videoUri = videoUri; + + return { + applicable: true, + confidence: 0.85, + data, + }; + } + + private extractId(uri: string): string { + const m = uri.match(/\/(?:p|reel)\/([A-Za-z0-9_-]+)/); + return m ? m[1] : ''; + } +} + +// --------------------------------------------------------------------------- +// Strategy B: Data-attribute fallback (medium confidence) +// Targets common class-name fragments Instagram injects at build time. +// --------------------------------------------------------------------------- +export class DataAttributeStrategy implements IExtractionStrategy< + ArticleElement, + IInstagramParsedPost +> { + readonly strategyName = 'DataAttributeStrategy'; + + execute(article: ArticleElement): IStrategyResult { + const link = article.querySelector('a[href]'); + if (!link || !/\/(p|reel)\//.test(link.href)) { + return { + applicable: false, + confidence: 0, + failureReason: 'No post/reel href found via data-attribute strategy', + }; + } + + const sourceUri = link.href; + const externalId = (sourceUri.match(/\/(?:p|reel)\/([A-Za-z0-9_-]+)/) ?? [])[1] ?? ''; + + const imgs = Array.from(article.querySelectorAll('img')).filter( + (img) => img.naturalWidth > 50 || img.src, + ); + const mediaUris = [...new Set(imgs.map((img) => img.src).filter(Boolean))]; + + const textNodes = Array.from(article.querySelectorAll('span')) + .map((s) => s.textContent?.trim()) + .filter(Boolean); + const textContent = textNodes.length > 0 ? textNodes.join(' ') : undefined; + + const data: IInstagramParsedPost = { + providerName: 'instagram', + sourceUri, + externalId, + mediaUris, + layout: 'single-image', + }; + if (textContent) data.textContent = textContent; + + return { + applicable: true, + confidence: 0.5, + data, + }; + } +} + +// --------------------------------------------------------------------------- +// Strategy C: Structural heuristics (low confidence, last resort) +// Attempts extraction from any block containing images and links. +// --------------------------------------------------------------------------- +export class StructuralHeuristicStrategy implements IExtractionStrategy< + ArticleElement, + IInstagramParsedPost +> { + readonly strategyName = 'StructuralHeuristicStrategy'; + + execute(article: ArticleElement): IStrategyResult { + const imgs = article.querySelectorAll('img'); + const links = article.querySelectorAll('a[href]'); + + if (imgs.length === 0) { + return { + applicable: false, + confidence: 0, + failureReason: 'No images found — cannot apply structural heuristic', + }; + } + + const sourceUri = + links.length > 0 ? (links[0] as HTMLAnchorElement).href : window.location.href; + + const data: IInstagramParsedPost = { + providerName: 'instagram', + sourceUri, + externalId: Date.now().toString(), + mediaUris: Array.from(imgs) + .map((img) => (img as HTMLImageElement).src) + .filter(Boolean), + layout: 'unknown', + }; + const textContent = article.textContent?.slice(0, 200).trim(); + if (textContent) data.textContent = textContent; + + return { + applicable: true, + confidence: 0.2, + data, + }; + } +} diff --git a/connectors/instagram/src/strategy-chain.ts b/connectors/instagram/src/strategy-chain.ts new file mode 100644 index 0000000..d031d81 --- /dev/null +++ b/connectors/instagram/src/strategy-chain.ts @@ -0,0 +1,51 @@ +import { + IExtractionStrategy, + IStrategyResult, + ErrorCode, + PlatformError, +} from '@knowledge-extractor/types'; +import { Logger } from '@knowledge-extractor/shared'; + +/** + * Executes an ordered list of strategies, returning the result of the first + * applicable one. If no strategy succeeds, throws a PlatformError. + */ +export class StrategyChain { + private readonly logger: Logger; + private readonly strategies: IExtractionStrategy[]; + + constructor(context: string, strategies: IExtractionStrategy[]) { + this.logger = new Logger(`StrategyChain:${context}`); + this.strategies = strategies; + } + + execute(input: TInput): TOutput { + for (const strategy of this.strategies) { + this.logger.debug(`Trying strategy: ${strategy.strategyName}`); + let result: IStrategyResult; + try { + result = strategy.execute(input); + } catch (err) { + this.logger.warn(`Strategy "${strategy.strategyName}" threw unexpectedly`, err); + continue; + } + + if (result.applicable && result.data !== undefined) { + this.logger.info( + `Strategy "${strategy.strategyName}" succeeded (confidence=${result.confidence.toFixed(2)})`, + ); + return result.data; + } + + this.logger.debug( + `Strategy "${strategy.strategyName}" not applicable: ${result.failureReason ?? 'no reason given'}`, + ); + } + + throw new PlatformError( + 'All extraction strategies exhausted without a successful result.', + ErrorCode.PARSE_ERROR, + false, + ); + } +} diff --git a/connectors/instagram/src/types.ts b/connectors/instagram/src/types.ts new file mode 100644 index 0000000..22c327c --- /dev/null +++ b/connectors/instagram/src/types.ts @@ -0,0 +1,36 @@ +import { IRawSourceResource } from '@knowledge-extractor/types'; + +/** + * The layout classification of an Instagram post as determined by the Parser. + * + * Instagram-specific. This intentionally lives inside the Instagram connector + * and must NOT leak into `@knowledge-extractor/types` (the platform-agnostic + * engine layer). + */ +export type InstagramPostLayout = 'single-image' | 'carousel' | 'reel' | 'unknown'; + +/** + * The normalized intermediate output of the Instagram Parser. + * The Parser understands Instagram's DOM; the Connector understands platform contracts. + * This is the boundary between them. + * + * Extends the platform-agnostic `IRawSourceResource` so the connector can satisfy + * the generic `IConnector` contract. + */ +export interface IInstagramParsedPost extends IRawSourceResource { + /** The detected layout type of the post. */ + layout: InstagramPostLayout; + /** + * For carousels, each slide's media URI in order. + * Overlaps with `mediaUris` but maintains explicit carousel semantics. + */ + slideUris?: string[]; + /** + * For reels, the video source URI. + */ + videoUri?: string; + /** The author's display name if extractable. */ + authorDisplayName?: string; + /** ISO 8601 parsed from the post timestamp element. */ + publishedAt?: string; +} diff --git a/connectors/instagram/tests/connector.test.ts b/connectors/instagram/tests/connector.test.ts new file mode 100644 index 0000000..67bc91f --- /dev/null +++ b/connectors/instagram/tests/connector.test.ts @@ -0,0 +1,99 @@ +import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'fs'; +import { resolve } from 'path'; +import { JSDOM } from 'jsdom'; +import { InstagramConnector } from '../src/index.js'; +import { SemanticArticleStrategy } from '../src/strategies.js'; +import type { IResource } from '@knowledge-extractor/types'; + +const FIXTURES_DIR = resolve(__dirname, './fixtures'); + +function loadFixture(name: string): Element { + const html = readFileSync(resolve(FIXTURES_DIR, `${name}.html`), 'utf-8'); + const dom = new JSDOM(html, { url: 'https://www.instagram.com/' }); + const article = dom.window.document.querySelector('article'); + if (!article) throw new Error(`No
found in fixture: ${name}`); + return article; +} + +function loadExpected(name: string): Partial { + return JSON.parse(readFileSync(resolve(FIXTURES_DIR, `${name}.expected.json`), 'utf-8')); +} + +// Helper: compares the essential fields only (excludes extractedAt timestamps) +function assertResource(actual: IResource, expected: Partial): void { + expect(actual.id).toBe(expected.id); + expect(actual.kind).toBe(expected.kind); + expect(actual.state).toBe(expected.state); + expect(actual.source.providerName).toBe(expected.source?.providerName); + expect(actual.source.externalId).toBe(expected.source?.externalId); + expect(actual.author?.handle).toBe(expected.author?.handle); + expect(actual.content.length).toBe(expected.content?.length ?? 0); + expect(actual.media.length).toBe(expected.media?.length ?? 0); + expect(actual.children?.length ?? 0).toBe(expected.children?.length ?? 0); +} + +describe('Instagram Connector — Fixture Regression Tests', () => { + const connector = new InstagramConnector(); + const strategy = new SemanticArticleStrategy(); + + it('correctly parses a single-image post', async () => { + const article = loadFixture('single-image-post'); + const expected = loadExpected('single-image-post'); + const stratResult = strategy.execute(article); + + expect(stratResult.applicable).toBe(true); + expect(stratResult.confidence).toBeGreaterThanOrEqual(0.8); + expect(stratResult.data?.layout).toBe('single-image'); + + const resource = await connector.normalize(stratResult.data!); + assertResource(resource, expected); + expect(resource.media[0].type).toBe('image'); + }); + + it('correctly parses a carousel post', async () => { + const article = loadFixture('carousel-post'); + const expected = loadExpected('carousel-post'); + const stratResult = strategy.execute(article); + + expect(stratResult.applicable).toBe(true); + expect(stratResult.data?.layout).toBe('carousel'); + + const resource = await connector.normalize(stratResult.data!); + assertResource(resource, expected); + expect(resource.media.length).toBe(3); + expect(resource.children?.length).toBe(3); + resource.children?.forEach((child) => { + expect(child.kind).toBe('instagram-slide'); + }); + }); + + it('correctly parses a reel post', async () => { + const article = loadFixture('reel-post'); + const expected = loadExpected('reel-post'); + const stratResult = strategy.execute(article); + + expect(stratResult.applicable).toBe(true); + expect(stratResult.data?.layout).toBe('reel'); + + const resource = await connector.normalize(stratResult.data!); + assertResource(resource, expected); + expect(resource.kind).toBe('instagram-reel'); + const videoMedia = resource.media.find((m) => m.type === 'video'); + expect(videoMedia).toBeDefined(); + }); + + it('strategy chain falls back when semantic selectors fail', () => { + // Minimal article with no semantic markers + const dom = new JSDOM( + '', + { url: 'https://www.instagram.com/' }, + ); + const article = dom.window.document.querySelector('article')!; + const result = strategy.execute(article); + + // SemanticArticleStrategy should still find the /p/ link + expect(result.applicable).toBe(true); + expect(result.data?.externalId).toBe('fallback123'); + }); +}); diff --git a/connectors/instagram/tests/fixtures/carousel-post.expected.json b/connectors/instagram/tests/fixtures/carousel-post.expected.json new file mode 100644 index 0000000..228c0f5 --- /dev/null +++ b/connectors/instagram/tests/fixtures/carousel-post.expected.json @@ -0,0 +1,93 @@ +{ + "id": "ig_carousel456", + "kind": "instagram-post", + "state": "extracted", + "source": { + "providerName": "instagram", + "externalId": "carousel456", + "originalUri": "https://www.instagram.com/p/carousel456/" + }, + "author": { + "handle": "carousel_user", + "displayName": "Carousel User" + }, + "content": [ + { + "type": "text", + "value": "A trip through the Swiss Alps! 🏔️ #travel #switzerland" + } + ], + "media": [ + { + "id": "carousel456_media_0", + "type": "image", + "sourceUri": "https://cdn.example.com/media/slide_1.jpg" + }, + { + "id": "carousel456_media_1", + "type": "image", + "sourceUri": "https://cdn.example.com/media/slide_2.jpg" + }, + { + "id": "carousel456_media_2", + "type": "image", + "sourceUri": "https://cdn.example.com/media/slide_3.jpg" + } + ], + "children": [ + { + "id": "ig_carousel456_slide_0", + "kind": "instagram-slide", + "state": "extracted", + "source": { + "providerName": "instagram", + "externalId": "carousel456_slide_0", + "originalUri": "https://www.instagram.com/p/carousel456/" + }, + "content": [], + "media": [ + { + "id": "slide_media_0", + "type": "image", + "sourceUri": "https://cdn.example.com/media/slide_1.jpg" + } + ] + }, + { + "id": "ig_carousel456_slide_1", + "kind": "instagram-slide", + "state": "extracted", + "source": { + "providerName": "instagram", + "externalId": "carousel456_slide_1", + "originalUri": "https://www.instagram.com/p/carousel456/" + }, + "content": [], + "media": [ + { + "id": "slide_media_1", + "type": "image", + "sourceUri": "https://cdn.example.com/media/slide_2.jpg" + } + ] + }, + { + "id": "ig_carousel456_slide_2", + "kind": "instagram-slide", + "state": "extracted", + "source": { + "providerName": "instagram", + "externalId": "carousel456_slide_2", + "originalUri": "https://www.instagram.com/p/carousel456/" + }, + "content": [], + "media": [ + { + "id": "slide_media_2", + "type": "image", + "sourceUri": "https://cdn.example.com/media/slide_3.jpg" + } + ] + } + ] +} diff --git a/connectors/instagram/tests/fixtures/carousel-post.html b/connectors/instagram/tests/fixtures/carousel-post.html new file mode 100644 index 0000000..a2d8bed --- /dev/null +++ b/connectors/instagram/tests/fixtures/carousel-post.html @@ -0,0 +1,29 @@ + + +Instagram Carousel Post — Fixture + +
+
+ carousel_user + Carousel User +
+ + View post +
+

A trip through the Swiss Alps! 🏔️ #travel #switzerland

+
+ +
+ + diff --git a/connectors/instagram/tests/fixtures/reel-post.expected.json b/connectors/instagram/tests/fixtures/reel-post.expected.json new file mode 100644 index 0000000..86bf523 --- /dev/null +++ b/connectors/instagram/tests/fixtures/reel-post.expected.json @@ -0,0 +1,33 @@ +{ + "id": "ig_xyz789", + "kind": "instagram-reel", + "state": "extracted", + "source": { + "providerName": "instagram", + "externalId": "xyz789", + "originalUri": "https://www.instagram.com/reel/xyz789/" + }, + "author": { + "handle": "reel_creator", + "displayName": "Reel Creator" + }, + "content": [ + { + "type": "text", + "value": "Morning workout routine 💪 #fitness #motivation" + } + ], + "media": [ + { + "id": "xyz789_media_0", + "type": "video", + "sourceUri": "https://cdn.example.com/media/reel_xyz789.mp4" + }, + { + "id": "xyz789_media_1", + "type": "image", + "sourceUri": "https://cdn.example.com/media/reel_xyz789_thumb.jpg" + } + ], + "children": [] +} diff --git a/connectors/instagram/tests/fixtures/reel-post.html b/connectors/instagram/tests/fixtures/reel-post.html new file mode 100644 index 0000000..717918e --- /dev/null +++ b/connectors/instagram/tests/fixtures/reel-post.html @@ -0,0 +1,23 @@ + + +Instagram Reel — Fixture + +
+
+ reel_creator + Reel Creator +
+
+ + Reel thumbnail +
+ View reel +
+

Morning workout routine 💪 #fitness #motivation

+
+ +
+ + diff --git a/connectors/instagram/tests/fixtures/single-image-post.expected.json b/connectors/instagram/tests/fixtures/single-image-post.expected.json new file mode 100644 index 0000000..418ca3a --- /dev/null +++ b/connectors/instagram/tests/fixtures/single-image-post.expected.json @@ -0,0 +1,28 @@ +{ + "id": "ig_abc123", + "kind": "instagram-post", + "state": "extracted", + "source": { + "providerName": "instagram", + "externalId": "abc123", + "originalUri": "https://www.instagram.com/p/abc123/" + }, + "author": { + "handle": "testuser", + "displayName": "Test User" + }, + "content": [ + { + "type": "text", + "value": "A beautiful sunset over the mountains. #nature #photography" + } + ], + "media": [ + { + "id": "abc123_media_0", + "type": "image", + "sourceUri": "https://cdn.example.com/media/post_abc123.jpg" + } + ], + "children": [] +} diff --git a/connectors/instagram/tests/fixtures/single-image-post.html b/connectors/instagram/tests/fixtures/single-image-post.html new file mode 100644 index 0000000..f8f596f --- /dev/null +++ b/connectors/instagram/tests/fixtures/single-image-post.html @@ -0,0 +1,22 @@ + + +Instagram Single Image Post — Fixture + +
+
+ testuser + Test User +
+
+ Photo by testuser +
+ View post +
+

A beautiful sunset over the mountains. #nature #photography

+
+ +
+ + diff --git a/connectors/instagram/tsconfig.json b/connectors/instagram/tsconfig.json new file mode 100644 index 0000000..63fdcb1 --- /dev/null +++ b/connectors/instagram/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "." + }, + "include": ["src/**/*", "tests/**/*", "vitest.config.ts"] +} diff --git a/connectors/instagram/vitest.config.ts b/connectors/instagram/vitest.config.ts new file mode 100644 index 0000000..a741874 --- /dev/null +++ b/connectors/instagram/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + environment: 'jsdom', + include: ['tests/**/*.test.ts'], + }, +}); diff --git a/docs/architecture/dependency-graph.md b/docs/architecture/dependency-graph.md new file mode 100644 index 0000000..3a1d5ac --- /dev/null +++ b/docs/architecture/dependency-graph.md @@ -0,0 +1,98 @@ +# Architectural Dependency Graph + +This document serves as the strict architectural contract for the Knowledge Extractor monorepo. It defines the allowed dependency flows between packages to prevent tight coupling, circular dependencies, and architectural degradation over time. + +These rules will be programmatically enforced in CI using `dependency-cruiser`. + +--- + +## 1. System Layers + +The repository is structured into hierarchical layers. A package in Layer N may only import from packages in Layer < N. Sibling dependencies are strictly regulated. + +### Layer 0: Primitives + +The absolute foundation. These packages contain no business logic and cannot import from any other package in the monorepo. + +- **`@knowledge-extractor/types`** (`packages/types`) + - **Purpose**: Shared TypeScript interfaces, types, and generic data models (e.g., `NormalizedDocument`). + - **Allowed Imports**: None. + +### Layer 1: Core Utilities + +Shared functionality used across the entire platform. + +- **`@knowledge-extractor/shared`** (`packages/shared`) + - **Purpose**: Cross-cutting concerns like logging, configuration, error classes, and constants. + - **Allowed Imports**: `types` +- **`@knowledge-extractor/utils`** (`packages/utils`) + - **Purpose**: Pure functions, generic data parsing, and string manipulation. + - **Allowed Imports**: `types` + +### Layer 2: Platform Engines + +Independent subsystems that perform specific knowledge platform operations. These engines must not depend on each other unless explicitly passing data through generic interfaces. + +- **`@knowledge-extractor/extractor`** (`packages/extractor`) + - **Purpose**: Validation and normalization pipelines. + - **Allowed Imports**: `types`, `shared`, `utils` +- **`@knowledge-extractor/storage`** (`packages/storage`) + - **Purpose**: Local persistence (IndexedDB, etc.). + - **Allowed Imports**: `types`, `shared` +- **`@knowledge-extractor/exporters`** (`packages/exporters`) + - **Purpose**: Exporting data to JSON, Markdown, CSV. + - **Allowed Imports**: `types`, `shared`, `utils` +- **`@knowledge-extractor/ocr`** (`packages/ocr`) + - **Purpose**: Text extraction from images/frames. + - **Allowed Imports**: `types`, `shared` +- **`@knowledge-extractor/ai`** (`packages/ai`) + - **Purpose**: External LLM integrations and prompt building. + - **Allowed Imports**: `types`, `shared` + +### Layer 3: Domain Connectors + +Source-specific extraction implementations. + +- **`@knowledge-extractor/connector-*`** (`connectors/*`) + - **Purpose**: Transforming source data (e.g., Instagram DOM) into `NormalizedDocument` objects. + - **Allowed Imports**: `types`, `shared`, `utils` + - **Forbidden Imports**: Other connectors, Platform Engines (`storage`, `extractor`, etc.), Apps. + +### Layer 4: Applications (Composition Roots) + +The entry points that wire the platform together. + +- **`@knowledge-extractor/extension`** (`apps/extension`) + - **Purpose**: Browser extension UI and Background Worker orchestration. + - **Allowed Imports**: `types`, `shared`, `utils`, `extractor`, `storage`, `exporters`, `connectors/*` + +--- + +## 2. Forbidden Imports & Strict Rules + +To maintain the modular philosophy, the following rules are non-negotiable: + +1. **No Upward Dependencies**: A package can never import from a layer above it. (e.g., `packages/types` cannot import `packages/shared`). +2. **No Connector Cross-Pollination**: `connectors/instagram` cannot import from `connectors/linkedin`. If logic is shared, it must be abstracted into `packages/utils` or `packages/shared`. +3. **No App Dependencies**: Absolutely no package or connector may import from `apps/*`. The application layer is strictly a consumer. +4. **No UI in Packages**: Business logic packages (`storage`, `extractor`) must not import UI libraries (React, Vue) or browser-specific rendering code. +5. **No Engine Entanglement**: `packages/exporters` must not depend on `packages/storage`. They communicate implicitly because the App layer orchestrates passing data from Storage to the Exporter via the generic `types`. + +--- + +## 3. Future Expansion Strategy + +As new connectors (Reddit, X, YouTube) are added: + +1. They will be created as new independent packages in the `connectors/` directory. +2. They will implement the standard `IConnector` interface defined in `packages/types`. +3. They will be wired into the Background Worker inside `apps/extension`. +4. No changes should be required in `packages/extractor` or `packages/storage` to support a new connector. + +When the platform matures and internal APIs stabilize, we will utilize **API Extractor** to generate `.d.ts` rollups and enforce public API contracts for `packages/types` and `packages/shared`. + +--- + +## 4. Validation + +These rules are translated into code via `dependency-cruiser` in `.dependency-cruiser.js` at the root of the monorepo. The CI pipeline will fail if any of these architectural boundaries are violated. diff --git a/docs/architecture/domain-model.md b/docs/architecture/domain-model.md new file mode 100644 index 0000000..d34d4e9 --- /dev/null +++ b/docs/architecture/domain-model.md @@ -0,0 +1,110 @@ +# Core Domain Model + +This document defines the conceptual language and structural boundaries of the Knowledge Extractor platform. It applies Domain-Driven Design (DDD) principles to establish a universal abstraction layer that transcends source-specific concepts. + +## 1. Core Abstraction Philosophy + +A significant challenge in knowledge extraction is the proliferation of platform-specific terminology: Posts, Reels, Tweets, Threads, Shorts, PDFs, and Articles. + +If we model the system around these specific concepts, the architecture will inevitably bloat into a complex inheritance hierarchy (e.g., `Reel extends InstagramMedia extends SocialPost`). Every new connector would require structural changes to the core system. + +**The Architectural Stance:** +Concepts like "Reel", "Post", or "PDF" **do not exist** as first-class domain objects in the core engine. They are merely presentational nuances of external systems. + +Instead, the core domain relies on highly normalized, universal abstractions: `Document`, `Source`, `Media`, and `Author`. The specific _type_ of external content is preserved purely as metadata within these generic structures. + +--- + +## 2. Ubiquitous Language + +- **Provider**: The external ecosystem originating the data (e.g., Instagram, Reddit, Local File System). +- **Connector**: The boundary layer responsible for translating Provider-specific reality into our core Domain reality. +- **Document**: The universal, autonomous unit of knowledge in our system. A Document is the translation of an external concept (a Reddit Thread, an Instagram Carousel, or a PDF file) into our standardized format. +- **Source**: The provenance of a Document. It answers _where_ the knowledge came from and _when_ it was observed. +- **Media**: Binary assets (images, audio, video) embedded within or attached to a Document. +- **Author**: The external entity (user, organization, or channel) that published the Source. +- **Enrichment**: The act of synthesizing new knowledge from a Document (e.g., performing OCR on Media, or running an LLM to generate a summary). + +--- + +## 3. Aggregate Boundaries & Entities + +The system orbits around a single Aggregate Root: the **Document**. + +### The Document (Aggregate Root - Entity) + +The Document is the primary unit of consistency. You cannot extract, store, or process an isolated comment or an isolated image without its parent Document. + +- **Role**: Contains the structured text, orchestrates its associated Media, and holds its Source provenance. +- **Granularity Challenge**: Is a Reddit Thread one Document, or is the Thread a Document and each Comment a Document? +- **Resolution**: A Document may be hierarchical. A Document can contain `ChildDocuments` (e.g., a Twitter Thread is a parent Document containing child Documents for each reply). This recursive tree structure is far more flexible than rigidly separating "Posts" from "Comments". + +### Media (Entity) + +A distinct asset belonging exclusively to a Document. + +- **Role**: Represents a file (video, image, document scan). +- **Identity**: Identified by a unique hash of its binary contents or a system-generated ID within the scope of the Document. + +--- + +## 4. Value Objects + +Value Objects have no independent identity; their equality is determined by their structural value. + +### Source (Value Object) + +Describes the exact origin of the Document. + +- **Attributes**: `ProviderName` (e.g., "Instagram"), `ExternalId` (the ID assigned by the Provider), `OriginalURI`, `ExtractionTimestamp`. +- **Equality**: Two Source objects are equal if they share the same `ProviderName` and `ExternalId`. + +### Author (Value Object) + +The originator of the content. + +- _Note_: While an "Author" could technically be an Entity if we built a CRM system, for a knowledge extraction pipeline, the Author is immutable historical metadata attached to the Document at the time of extraction. +- **Attributes**: `Handle`, `DisplayName`, `AvatarURI`, `ProviderProfileURI`. + +### ContentBlock (Value Object) + +Instead of a single monolithic "text" field, the body of a Document is an array of `ContentBlocks`. + +- **Attributes**: `Type` (Text, Heading, Quote, Code), `Value`. +- **Why**: A LinkedIn article or a Medium post has structured text. Flattening it destroys semantic meaning. A generic `ContentBlock` array gracefully handles both a simple 140-character Tweet and a complex 10-page PDF. + +--- + +## 5. Lifecycles & State Transitions + +A Document moves through a strict lifecycle pipeline. It cannot skip states. + +1. **Discovered**: The Connector has identified a Source (e.g., intercepted an API response) but has not yet parsed it. +2. **Extracted**: The raw data has been parsed into a Document in memory. Associated Media URIs are known, but the binary data has not been downloaded. +3. **Hydrated**: All Media binaries associated with the Document have been downloaded and localized. +4. **Enriched**: Background processes have executed against the Document. For example, the OCR engine has scanned a Media asset and appended a new `ContentBlock` to the Document containing the transcribed text. +5. **Persisted**: The fully formed Document has been committed to the Storage Engine. +6. **Exported**: The Document has been successfully synchronized to a downstream system (e.g., Notion, Obsidian). + +--- + +## 6. Ownership & Invariants + +- **Ownership Rule 1**: A Document owns its Media. If a Document is deleted from the system, its localized Media binaries must be garbage collected. +- **Ownership Rule 2**: A Document owns its Source. A Source cannot exist independently. +- **Invariant 1 (Identity)**: A Document's system ID must be a deterministic derivative of its `Source` (e.g., a hash of `ProviderName + ExternalId`). This guarantees idempotency. If the engine extracts the same Instagram post twice, it overwrites the existing Document rather than duplicating it. +- **Invariant 2 (Immutability of Source)**: Once a Document is extracted, its `Source` and `Author` metadata cannot be modified by the user or the Enrichment engines. +- **Invariant 3 (Enrichment Additive)**: Enrichment processes (OCR, AI tagging) are strictly additive. They append metadata or `ContentBlocks` but must never destructively overwrite the original extracted text. + +--- + +## 7. Relationship to Future Connectors + +By normalizing "Reel" and "PDF" into the `Document` + `Media` model, adding a new connector requires zero changes to the Domain layer. + +- **Instagram Reel**: Becomes a `Document` with zero text `ContentBlocks`, containing one `Media` (video), with an `Author` (the account). +- **Instagram Carousel**: Becomes a `Document` with multiple `Media` (images), preserving order. +- **PDF**: Becomes a `Document` with `Media` (the PDF file). Upon Enrichment, the OCR engine reads the PDF and populates the `ContentBlocks`. +- **Reddit Thread**: Becomes a parent `Document` containing text `ContentBlocks` (the main post), and a collection of child `Document` aggregates (the comments). + +This guarantees extreme extensibility. The core storage, AI, and export engines only ever operate against this single unified Domain Model. diff --git a/connectors/instagram/extractors/posts.ts b/docs/rfc/.gitkeep similarity index 100% rename from connectors/instagram/extractors/posts.ts rename to docs/rfc/.gitkeep diff --git a/docs/rfc/0001-alpha-stabilization.md b/docs/rfc/0001-alpha-stabilization.md new file mode 100644 index 0000000..4ca1201 --- /dev/null +++ b/docs/rfc/0001-alpha-stabilization.md @@ -0,0 +1,420 @@ +# RFC-0001 — Alpha Stabilization Execution Plan + +| Field | Value | +| --------------- | -------------------------------------------------------------- | +| Status | Proposed | +| Author | Engineering (Staff review follow-up) | +| Date | 2026-06-26 | +| Supersedes | — | +| Source of truth | Staff Engineering Audit (2026-06-26) | +| Scope | All work required to reach a stable, Alpha-ready crawler | +| Constraint | This RFC plans work only. No code is changed by this document. | + +> Finding IDs (`A1`, `D1`, `X1`, `B2`, …) reference the Staff Engineering Audit and are used throughout for traceability. + +--- + +## 1. Executive Summary + +**Architecture maturity: High (design) / Medium (runtime).** +The contract layer (`packages/types`) is well-modeled, single, normalized, and platform-agnostic. Layer boundaries are enforced mechanically by dependency-cruiser (0 violations across 84 modules). The intended pipeline — Discovery → Queue → Scheduler → Navigator → Extractor → Normalizer → Persistence → Diagnostics — exists as named, separated units. + +**Implementation maturity: Low–Medium.** +The runtime wiring diverges from the design in four material ways: (1) extraction is duplicated into the content script while the connector's tested extraction path is dead, (2) diagnostics and metrics collectors are instantiated but never driven from the execution path, (3) infinite scroll exists but is not wired into the crawl loop, and (4) the orchestration loop uses MV3-hostile primitives (`setInterval`) with a non-persistent in-memory queue. + +**Repository health: Not clean.** +CI is red today at Format Check. `turbo run lint` executes zero tasks (no package defines a `lint` script). `turbo run typecheck` covers 1 of 10 packages. The working tree carries macOS duplicate-file artifacts (`* 2.ts`, `tsconfig 2.json`, …), some git-tracked. Tests cover one strategy and the parser/normalizer happy path only. + +**Alpha readiness: Not ready.** +Alpha's mandate is _"do not estimate — measure."_ As wired today, a crawl would produce zeroed metrics and empty failure reports, discovery would be capped at first render (no scroll), and the build does not pass cleanly. **Alpha cannot produce valid engineering evidence until the observability path and crawl loop are connected and the build is restored.** + +**Overall health score (from audit): 5.2 / 10.** Strong skeleton; execution wiring and build hygiene are the gap. + +--- + +## 2. Architecture Status + +### 2.1 Architecture Correct — **FREEZE** + +These are correct, validated, and must not change without a superseding RFC. + +| Area | Evidence | Why frozen | +| ------------------------------------------- | ----------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| Normalized domain model | `IResource`, `IMedia`, `IContentBlock`, `ISource`, `IAuthor` in `packages/types/src/core` | Single, platform-agnostic aggregate; supports children, completeness, lifecycle states. | +| Execution vs. domain split | `ICrawlTask`/`TaskState` (execution) vs. `IResourceCompleteness` (domain) | Clean separation; scheduler owns execution, resource owns completeness. | +| Package boundaries / dependency direction | dependency-cruiser: 0 violations, 84 modules | Layer 0 (`types`) imports nothing; Layer 1 (`shared`, `utils`) imports only Layer 0; connectors isolated; no `apps` imports. | +| Scheduler ownership of the queue | `Scheduler` owns `Map`, retry/backoff | Queue, priority, retry live in one owner; connector has no scheduling role. | +| Storage abstraction | `IStorageEngine`/`ITransaction`; `InMemoryStorage` implements it | Engine is isolated from persistence implementation. | +| Connector isolation | `connectors/instagram/src/*`; depcruise rule `connectors-cannot-import-connectors` | Connectors discover/extract/normalize only. | +| Strategy-chain extraction pattern | `IExtractionStrategy`, `IStrategyResult`, `StrategyChain` | Ordered, confidence-scored, non-throwing strategy fallback. | +| Navigator ownership of browser manipulation | `apps/extension/src/content/navigator.ts` | Scrolling/modal/wait isolated from extraction. | + +### 2.2 Architecture Drift — **CORRECT DURING STABILIZATION** + +| Drift | Audit ID | Why it happened | Impact | Recommended correction | +| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| **Duplicated extraction** — `extractSingleResource()` in `content/index.ts` re-implements `SemanticArticleStrategy`; the connector's `extractArticle`→chain→parser→normalizer path is dead at runtime. | A1, A2 | Vertical-slice expediency: the content script needed _something_ extracting before the connector chain was complete, and the two were never reconciled. | Runtime quality is governed by an untested inline copy; the tested chain (incl. fallback strategies) provides false confidence; adding a connector means editing the extension. | Route content-script extraction through the connector's `extractArticle`/StrategyChain. Delete the inline copy. (Sprint A1) | +| **Diagnostics not wired** — `DiagnosticsCollector` is injected but `recordFailure`/`recordStrategyUsed`/`reset` are never called. | X1, X3 | Collector built before the controller's failure path existed; never connected. | `EXPORT_DIAGNOSTICS` returns empty failures and empty strategy usage — Alpha measurement surface is blank. | Drive the collector from `CrawlController` failure/strategy paths; forward content-script DOM snapshots into `recordFailure`. (Sprint A3) | +| **Metrics not wired** — background `MetricsCollector` never receives `recordExtracted/recordFailed/addExtractionTime`; only `SessionManager` counters move. | X2 | Two collectors in two contexts (content + background) that were never reconciled. | `metrics.snapshot()` feeding the diagnostics report is all-zeros. | Single source of truth for metrics in the background; drive from controller transitions. (Sprint A3) | +| **Connector contract bypass / duplicate `IConnector`** — two conflicting `IConnector` interfaces re-exported from one barrel; `InstagramConnector` implements neither. | D2 | Contract evolved twice (pipeline-style `normalize` vs. capability-style `extract`) without reconciliation; `export *` silently dropped the ambiguous symbol. | The canonical connector contract is unusable; nothing enforces connector shape. | Collapse to one `IConnector`; make `InstagramConnector` implement it. (Sprint A1) | +| **Runtime ownership violation (Content Script owns parsing)** | A2 | Same root as A1. | Content Script exceeds "DOM access only"; Extractor stage has no real home. | Promote Extractor to a real stage owned by the connector; content script supplies DOM only. (Sprint A1) | +| **Crawl loop uses MV3-hostile primitives** — `setInterval(1000)`; in-memory `Scheduler.tasks` not persisted. | §6 | Authored as if a long-lived background page (MV2 mental model) rather than an evictable MV3 service worker. | SW eviction kills the loop and drops the queue mid-crawl; "session restores correctly" fails. | `chrome.alarms`-driven loop; persist queue snapshot to `chrome.storage.session`. (Sprint A2) | +| **Infinite scroll not wired** — `Navigator.scrollGrid()` exists; controller comments "trigger scroll … later." | §6 | Scroll built as a capability before the controller's idle-handling existed. | Discovery is capped at first render; large Saved collections under-discovered. | Controller drives scroll on queue-drain until end-of-grid. (Sprint A2) | +| **Platform leak into Layer 0** — `IInstagramParsedPost`/`InstagramPostLayout` defined in `packages/types`. | D1 | Convenience: the parser's intermediate type was placed next to other connector contracts. | Engine "knows" about Instagram; violates single-normalized-domain rule; pollutes the frozen layer. | Move Instagram-specific intermediate types into `connectors/instagram`. (Sprint A1, low-risk) | +| **Typed event/message bus bypassed** — raw string actions not in `EventAction`/`MessageAction`. | A3 | Events emitted ad hoc as the controller grew. | No compile-time safety on the bus; drift between producers/consumers. | Route all messages through the typed unions. (Sprint A2/P2) | + +--- + +## 3. Stabilization Sprints + +Each sprint is independently completable and has explicit exit criteria. Sprints are ordered by dependency, not by importance alone. + +### Sprint A0 — Repository Stabilization + +**Goals:** Restore a clean, trustworthy build so every later sprint is measured against a stable baseline. +**Deliverables:** + +1. Remove all macOS duplicate artifacts (`* 2.*`, `* 3.*`) from the tree and git index (B1). +2. Remove leftover codemod scripts `fix-exports.js`, `fix-all-imports.js` (B5). +3. Add a `typecheck` script (`tsc --noEmit`) to every package: `types`, `shared`, `storage`, `utils`, `extractor`, `ocr`, `ai`, `exporters`, `extension` (B2). +4. Add a real `lint` script per package (or convert root `lint` to run `eslint .` across the workspace) so `turbo run lint` is non-vacuous; fix the 17 eslint errors / 4 warnings, including stub files not covered by any tsconfig (B3). +5. Make `format:check` pass (B4). +6. Decide and document the fate of the 5 empty stub packages and the empty legacy connector dirs (`extractors/`, `models/`, `selectors/`, `pipeline/`): either delete or mark as intentional placeholders excluded from lint/typecheck (graph hygiene). +7. (Optional, low-risk) Either make project references functional (`composite: true` on leaf configs) or remove the references to avoid implying a `tsc -b` workflow that doesn't work (B6). + **Exit Criteria (all objectively verifiable):** + +- `find . -name "* [0-9]*" -not -path '*/node_modules/*'` returns zero results. +- `pnpm run typecheck` runs for **10/10** packages and exits 0. +- `pnpm run lint` executes a non-zero task count and exits 0; `eslint .` exits 0. +- `pnpm run format:check` exits 0. +- `pnpm run depcruise` exits 0. +- `pnpm run test` exits 0; `pnpm run build` exits 0. + **Dependencies:** None. **This is the root of the graph.** + **Estimated complexity:** Low (mechanical). **Risk:** Low — but must be done first and committed to freeze the tree (a watcher/editor was observed rewriting files mid-audit). + +### Sprint A1 — Runtime Correctness (Extraction Unification) + +**Goals:** Eliminate the duplicated extraction path; restore the connector as the single extraction owner; clean the domain leak and duplicate contract. +**Deliverables:** + +1. Content script supplies the DOM element only; extraction routes through `InstagramConnector.extractArticle` → `StrategyChain` → `InstagramParser` → `InstagramNormalizer`. +2. Delete `extractSingleResource()`'s inline parsing logic (A1, A2). +3. Collapse the two `IConnector` interfaces into one canonical contract; `InstagramConnector` implements it (D2). +4. Move `IInstagramParsedPost`/`InstagramPostLayout` out of `packages/types` into the Instagram connector (D1). +5. Remove the `any` escape hatch in `InstagramNormalizer.normalize` (D4); return a typed `IResource`. + **Exit Criteria:** + +- No extraction/selector logic remains in `apps/extension/src/content`. +- `grep -ri "instagram" packages/types/src` returns only doc-comment examples, no type definitions. +- Exactly one `IConnector` symbol is exported from `@knowledge-extractor/types`; `InstagramConnector` implements it (typechecked). +- A test asserts the content script's produced record is consumed by the connector chain (single path). + **Dependencies:** Requires A0 (clean build to refactor against). + **Estimated complexity:** Medium. **Risk:** Medium — touches the live extraction path; mitigated by existing fixtures plus new single-path test. + +### Sprint A2 — Crawler Correctness (Lifecycle & Scroll) + +**Goals:** Make the crawl loop survive MV3 lifecycle and actually drive discovery to completion. +**Deliverables:** + +1. Replace `setInterval` orchestration with a `chrome.alarms`-driven loop (§6). +2. Persist the Scheduler queue snapshot to `chrome.storage.session`; rehydrate on SW restart so pause/resume/cancel and recovery are real. +3. Wire `Navigator.scrollGrid()` into the controller: on queue drain, scroll until end-of-grid is detected, then finish. +4. Route lifecycle/pipeline messages through the typed `EventAction`/`MessageAction` unions (A3). +5. Remove the `(controller as any).sessionManager` access via a typed accessor (A4). + **Exit Criteria:** + +- Killing/restarting the service worker mid-crawl restores both session stats **and** the pending queue; the crawl continues. +- A crawl over a multi-screen grid discovers items beyond the initial viewport (scroll-driven), and terminates on end-of-grid. +- No raw string message actions remain outside the typed unions (lint/grep check). + **Dependencies:** Requires A1 (single extraction path before persisting/queuing its inputs). Requires A0. + **Estimated complexity:** Medium–High. **Risk:** Medium — MV3 timing is subtle; mitigated by an SW-restart integration test. + +### Sprint A3 — Observability (Diagnostics & Metrics Wiring) + +**Goals:** Connect the measurement surface to the execution path so Alpha can measure rather than estimate. +**Deliverables:** + +1. Drive `DiagnosticsCollector` from `CrawlController`: `reset(pageUrl)` at crawl start, `recordFailure(...)` on every failure with category + DOM snapshot, `recordStrategyUsed(...)` per extraction (X1, X3). +2. Forward content-script DOM failure snapshots into `recordFailure` (X3). +3. Single background `MetricsCollector` updated on every transition: `recordExtracted/recordFailed/recordDuplicate/addExtractionTime/addNormalizationTime` (X2). +4. Ensure `EXPORT_DIAGNOSTICS` → `buildReport` returns populated `metrics`, `failures`, `strategyUsage`, and `memoryUsageMb`. + **Exit Criteria:** + +- After a crawl with at least one induced failure, `EXPORT_DIAGNOSTICS` returns: non-zero `metrics.extracted`, ≥1 `failures` entry with `category` + `domSnapshot`, and non-empty `strategyUsage`. +- `metrics.snapshot()` values match independently counted crawl outcomes (cross-checked against `SessionManager`). + **Dependencies:** Requires A1 (strategy usage/failures only meaningful once extraction is unified) and A2 (failures arise from the real loop). Requires A0. + **Estimated complexity:** Medium. **Risk:** Low — additive wiring, no architectural change. + +### Sprint A4 — Performance (Discovery Scan) + +**Goals:** Remove the discovery scaling ceiling before validating against a real collection. +**Deliverables:** + +1. Debounce the `MutationObserver` callback; coalesce mutation bursts (§9). +2. Scope scans to added subtrees instead of re-scanning all of `document.body`; dedup against `seen` **before** full traversal/fingerprinting (§9). +3. Capture a memory-growth measurement during a large-collection scroll for the Alpha report. + **Exit Criteria:** + +- Discovery cost per mutation batch is bounded by added-node count, not total DOM size (verified by instrumented timing on a synthetic large grid). +- No quadratic growth in scan time as discovered count increases (timing curve recorded). +- Heap usage stays within the Alpha memory limit defined in §8 during a full scroll. + **Dependencies:** Requires A2 (scroll must drive discovery to exercise the hot path). Independent of A3. + **Estimated complexity:** Medium. **Risk:** Medium — observer scoping can miss nodes; mitigated by a dedup-accuracy test (§7). + +### Sprint A5 — Alpha Validation + +**Goals:** Execute the measured validation run against a real Instagram Saved collection and record evidence. +**Deliverables:** + +1. Add regression tests closing the §7 gaps: Scheduler retry/backoff, CrawlController happy+failure paths, DiscoveryEngine dedup, SessionManager restore, fallback strategies, fingerprint collisions. +2. Execute a real crawl; export the diagnostics report. +3. Produce the Alpha Validation report measuring every metric in §8. + **Exit Criteria:** All §8 success metrics met and recorded with real numbers (no estimates). + **Dependencies:** Requires A0–A4 complete. + **Estimated complexity:** Medium. **Risk:** Medium — depends on live Instagram DOM stability; mitigated by fixtures + retry policy. + +--- + +## 4. Dependency Graph Between Fixes + +``` +A0 Repository Stabilization + Requires: nothing (ROOT) + Blocks: A1, A2, A3, A4, A5 (everything) + Independent of: OCR, exporters, AI + +A1 Extraction Unification (A1/A2/D1/D2/D4) + Requires: A0 + Blocks: A2 (persist a single input shape), A3 (strategyUsage/failures), Reddit/LinkedIn/YouTube connectors + Independent of: A4 performance, storage abstraction + +A2 Crawler Lifecycle + Scroll (MV3, queue persistence, scroll wiring) + Requires: A0, A1 + Blocks: A3 (real failures from real loop), A4 (scroll feeds hot path), A5 + Independent of: domain model, exporters + +A3 Diagnostics + Metrics Wiring (X1/X2/X3) + Requires: A0, A1, A2 + Blocks: A5 (Alpha cannot measure without it) + Independent of: A4 performance, OCR + +A4 Discovery Performance (MutationObserver) + Requires: A0, A2 + Blocks: A5 (large-collection viability) + Independent of: A3 diagnostics, A1 contracts + +A5 Alpha Validation + Requires: A0, A1, A2, A3, A4 + Blocks: OCR, AI enrichment, additional connectors (gated on Alpha completion) + Independent of: nothing remaining +``` + +**Permanently independent of the Alpha critical path (do not start until after Alpha):** OCR, video frame extraction, AI enrichment, semantic search, Markdown/Obsidian/Notion exporters, PDF/Reddit/LinkedIn/YouTube/generic connectors. + +--- + +## 5. Critical Path + +The shortest path to **"a stable, Alpha-ready crawler"**: + +``` +A0 → A1 → A2 → A3 → A5 + └→ A4 ─────┘ +``` + +**On the critical path (do this, in order):** + +1. **A0** — clean build (root dependency of everything). +2. **A1** — unify extraction (single tested path; prerequisite for meaningful diagnostics). +3. **A2** — MV3 lifecycle + queue persistence + scroll wiring (a crawl must survive and progress). +4. **A3** — wire diagnostics/metrics (Alpha must measure). +5. **A4** — discovery performance (runs in parallel after A2; required for real-collection viability). +6. **A5** — validate and record. + +**Deferred off the critical path:** typed event-bus migration beyond what A2 requires (P2), project-reference correctness (P2), stub-package cleanup beyond lint-passing (P3), all future features. + +--- + +## 6. Technical Debt Register + +Owner column uses roles, not names (assign at sprint planning). + +### P0 — Must fix before Alpha + +| ID | Description | Impact | Owner | Dependency | Effort | +| --------- | ----------------------------------------------------- | ------------------------------------------------ | --------- | ---------- | ------ | +| B1 | Remove duplicate-file artifacts (tracked + untracked) | Unstable tree; CI noise | Build | none | S | +| B2 | `typecheck` script on all 10 packages | Extension/shared/storage/types never typechecked | Build | none | S | +| B3 | Make `lint` real; fix 17 errors/4 warnings | Lint gate is vacuous; quality unenforced | Build | none | M | +| B4 | Pass `format:check` | CI red today | Build | none | S | +| X1 | Wire `DiagnosticsCollector` into controller | Empty failure reports → Alpha measures nothing | Runtime | A1, A2 | M | +| X2 | Wire background `MetricsCollector` | Zeroed metrics in report | Runtime | A1, A2 | M | +| X3 | Forward DOM snapshots into `recordFailure` | No failure root-cause evidence | Runtime | X1 | S | +| A1/A2 | Unify extraction; remove content-script parsing | Untested runtime path; false test confidence | Runtime | A0 | M | +| §6-loop | `chrome.alarms` loop + queue persistence | SW eviction drops crawl/queue | Runtime | A1 | M–L | +| §6-scroll | Wire scroll into crawl loop | Discovery capped at first render | Runtime | A1 | M | +| §9 | Debounce/scope MutationObserver | Quadratic discovery on large collections | Connector | A2 | M | + +### P1 — Must fix before Beta + +| ID | Description | Impact | Owner | Dependency | Effort | +| ------ | -------------------------------------------------- | ------------------------------------- | --------- | ---------- | ------ | +| D2 | Collapse duplicate `IConnector`; implement it | No enforceable connector contract | Types | A0 | S | +| §7 | Tests: Scheduler/retry, controller, dedup, restore | Retry (an exit criterion) is untested | QA | A1, A2 | M–L | +| A3-bus | Typed event/message bus end-to-end | Producer/consumer drift | Runtime | A2 | M | +| D4 | Remove `any` in normalizer | Domain-type guarantee hole | Connector | A1 | S | + +### P2 — Must fix before v1 + +| ID | Description | Impact | Owner | Dependency | Effort | +| --- | -------------------------------------------- | --------------------------------- | ------- | ---------- | ------ | +| D1 | Move Instagram types out of `packages/types` | Frozen layer pollution | Types | A0 | S | +| B6 | Make project references functional or remove | Misleading `tsc -b` story | Build | none | S | +| §10 | Validate `onMessage` sender | Hardening before external surface | Runtime | none | S | + +### P3 — Can defer indefinitely + +| ID | Description | Impact | Owner | Dependency | Effort | +| --------- | --------------------------------------------------------------------------------------------------- | ------------- | ----- | ---------- | ------ | +| Stub pkgs | Remove `ai`/`exporters`/`extractor`/`ocr`/`utils` empty pkgs and legacy connector dirs until needed | Graph clutter | Build | none | S | +| B5 | Delete codemod scripts | Repo tidiness | Build | none | S | +| Docs | Fill empty `README.md`/`ROADMAP.md` | Onboarding | Docs | none | S | + +--- + +## 7. Future Feature Readiness + +| Feature | Classification | Rationale | +| ---------------------- | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| OCR | **MINOR WORK** | `IOcrEngine`, `IResourceCompleteness.ocr`, `IMedia.localUri` all exist. Blocked only by absence of a **media hydration stage** (today only `sourceUri` is captured, never downloaded). Add hydration, then OCR is a drop-in. | +| Video frame extraction | **MAJOR WORK** | Requires media hydration **plus** binary/video frame sampling pipeline; none exists. | +| AI enrichment | **MINOR WORK** | `ResourceState.ENRICHED` and the content-block model support it; add an engine-side enrichment stage. No contract changes. | +| Semantic search | **MAJOR WORK** | Only `InMemoryStorage` exists; needs durable storage + embeddings/vector index. | +| Markdown export | **MINOR WORK** | `IExporter` + `ExportFormat.MARKDOWN` defined; `exporters` package is empty — implement against the existing contract. | +| Obsidian export | **MINOR WORK** | Builds directly on the Markdown exporter. | +| Notion export | **MINOR WORK** | Contract fits; needs a Notion API/auth client. No domain change. | +| Reddit connector | **MINOR WORK** (after A1) | DOM connector; strategy/connector pattern is reusable. Blocked **only** by the content-script coupling (A1); after unification it is additive. | +| LinkedIn connector | **MINOR WORK** (after A1) | Same as Reddit; auth/session nuances are connector-local. | +| YouTube connector | **MINOR WORK** (after A1) | DOM connector; transcript maps to `BlockType.TRANSCRIPT`. | +| PDF connector | **MAJOR WORK** | Runtime assumes browser DOM + Navigator + content script. PDF is non-DOM; requires decoupling "connector" from "browser extraction." | +| Generic web connector | **MINOR WORK** | `StructuralHeuristicStrategy` is a starting point; generalize selectors. | + +**Gating rule:** No feature in this table begins until **A5 (Alpha) is complete**. DOM-connector readiness additionally requires **A1**. + +--- + +## 8. Architecture Freeze + +### Frozen Decisions — change only via a superseding RFC + +- The normalized domain model (`IResource`, `IMedia`, `IContentBlock`, `ISource`, `IAuthor`) and its lifecycle (`ResourceState`). +- Execution-vs-domain separation: `ICrawlTask`/`TaskState` own execution; `IResourceCompleteness` owns completeness. +- Package boundaries and dependency direction (dependency-cruiser rules). +- Connector isolation: connectors discover/extract/normalize only — no storage, no orchestration. +- `CrawlController` as sole orchestrator. +- `Scheduler` as sole owner of the queue, priority, and retry. +- `Navigator` as sole owner of browser manipulation (scroll/modal/wait). +- Storage abstraction (`IStorageEngine`/`ITransaction`). +- Popup as monitoring-only (no execution). + +### Flexible Decisions — open to iteration without an RFC + +- Extraction implementation details (selectors, strategy internals, confidence thresholds) — **provided extraction stays owned by the connector** post-A1. +- Diagnostics internals (categories, report shape, snapshot size). +- Scheduler algorithm (priority scheme, ordering). +- Retry policy (attempt counts, backoff curve). +- Navigation strategy (modal vs. detail-page, timing constants). +- Performance optimizations (observer scoping, debounce windows, caching). + +--- + +## 9. Success Metrics — Alpha Completion (all objectively measurable) + +**Build & quality gates** + +- `pnpm run depcruise` exits 0. +- `pnpm run typecheck` covers 10/10 packages and exits 0. +- `pnpm run lint` executes ≥1 task per source package and exits 0. +- `pnpm run format:check` exits 0. +- `pnpm run test` exits 0. +- `pnpm run build` exits 0. +- `find . -name "* [0-9]*" -not -path '*/node_modules/*'` returns 0 results. + +**Observability** + +- After a validation crawl, `EXPORT_DIAGNOSTICS` returns `metrics.extracted > 0`, `failures.length` reflecting actual failures (≥1 when failures occur), non-empty `strategyUsage`, and a numeric `memoryUsageMb`. +- Background `metrics.snapshot()` equals the independently counted `SessionManager` outcomes (exact match on discovered/extracted/failed/duplicates). + +**Crawler behavior (against a real Saved collection of ≥100 items)** + +- Successful crawl rate (extracted / discovered) **≥ 95%**. +- Retry success rate: **≥ 90%** of tasks that fail once and are retried eventually reach `COMPLETED` within `maxAttempts`. +- Duplicate-detection accuracy: **0** duplicate `IResource.id` persisted; dedup false-negative rate **= 0** on the validation set. +- Extraction latency: median end-to-end per resource **≤ 1500 ms**; p95 **≤ 4000 ms** (measured from `EXTRACTION_STARTED` to `RESOURCE_PERSISTED`). +- Modal-open latency recorded for every resource (non-null `openLatencyMs`). +- Memory limit: heap usage stays **≤ 300 MB** across a full scroll of the validation collection. +- Session recovery: after a forced service-worker restart mid-crawl, the crawl resumes with the queue intact and completes; **0** resources lost or double-persisted. +- Infinite-scroll stability: discovery continues past the initial viewport and terminates cleanly on end-of-grid with `scrollFailures = 0` for a healthy run. + +--- + +## 10. Risks + +### Technical + +| Risk | Probability | Impact | Mitigation | +| ------------------------------------------------------- | ----------- | ------ | ---------------------------------------------------------------------------------------------- | +| MV3 service-worker eviction breaks the loop/queue | High | High | A2: `chrome.alarms` + persisted queue snapshot; SW-restart integration test in A5. | +| Quadratic discovery scan stalls large collections | High | High | A4: debounce + subtree-scoped scanning + pre-traversal dedup; timing curve recorded. | +| Instagram DOM changes break selectors during validation | Medium | Medium | Strategy chain with fallbacks (restored in A1); fixtures; failure diagnostics for fast triage. | +| Extraction unification regresses parsing | Medium | High | Existing fixtures + new single-path test before deleting the inline copy. | + +### Architectural + +| Risk | Probability | Impact | Mitigation | +| ----------------------------------------------------------------------------------- | ----------- | ------ | ---------------------------------------------------------------------------------------------------------------- | +| Stabilization re-introduces drift (e.g., new logic creeps back into content script) | Medium | High | Architecture Freeze (§8); depcruise; a guard test asserting no extraction logic in `apps/extension/src/content`. | +| Platform concepts re-leak into `packages/types` | Medium | Medium | D1 correction + a lint/grep guard for `instagram`/platform terms in `types`. | +| Duplicate/ambiguous contracts reappear | Low | Medium | D2 fix + single-export assertion. | + +### Product + +| Risk | Probability | Impact | Mitigation | +| --------------------------------------------------------------------------- | ----------- | ------ | ---------------------------------------------------------------------------- | +| Pressure to start OCR/connectors before Alpha completes | Medium | High | Gating rule (§7): no feature work until A5; this RFC is the gate. | +| Alpha metrics look "good enough" while measurement is still partially wired | Low | High | §9 cross-check: background metrics must exactly match SessionManager counts. | + +### Operational + +| Risk | Probability | Impact | Mitigation | +| ---------------------------------------------------------------------------------------------------- | ----------- | ------ | ------------------------------------------------------------------------------------------- | +| Working tree mutates outside version control (watcher/editor rewriting files, observed during audit) | High | Medium | A0 first; commit the cleanup and freeze the tree before any other sprint. | +| CI gives false confidence (vacuous lint, partial typecheck) | High | High | A0: make every gate real; re-run full CI locally before declaring A0 done. | +| Validation depends on a live, authenticated Instagram session | Medium | Medium | Document the validation environment; capture fixtures from the same session for regression. | + +--- + +## 11. Final Recommendation + +1. **Is Alpha implementation ready to continue?** + **No — not as a validation exercise.** The design is ready to _build on_, but the crawler cannot yet produce valid measurements: diagnostics/metrics are unwired, scroll is not driven, the loop is MV3-fragile, and the build is not clean. Stabilization (A0–A4) must precede validation (A5). + +2. **What should be implemented first?** + **Sprint A0 (Repository Stabilization).** It is the root of the dependency graph: remove duplicate files, make typecheck/lint/format real and green, and commit to freeze the tree. Then A1 (extraction unification), because every later correctness and observability gain depends on a single, tested extraction path. + +3. **What should not be touched until after Alpha?** + All future features: OCR, video frame extraction, AI enrichment, semantic search, Markdown/Obsidian/Notion exporters, and the PDF/Reddit/LinkedIn/YouTube/generic connectors. Also defer the typed-bus migration beyond what A2 needs, project-reference correctness, and stub-package removal. + +4. **What should remain permanently frozen?** + The normalized domain model, execution-vs-domain split, package boundaries and dependency direction, connector isolation, CrawlController orchestration ownership, Scheduler queue/retry ownership, Navigator browser-manipulation ownership, the storage abstraction, and Popup-as-monitor. Changes here require a superseding RFC. + +5. **What architectural mistakes are most important to avoid?** + - Re-implementing extraction outside the connector (the original drift); keep parsing in the connector, DOM-only in the content script. + - Leaking platform-specific concepts back into `packages/types`. + - Treating the background as a long-lived page instead of an evictable MV3 worker (persist anything that must survive). + - Building features on an unmeasured crawler — wire observability before declaring anything "validated." + - Trusting vacuous CI gates; every gate must actually execute and fail loudly. + +--- + +_End of RFC-0001. No repository changes were made by this document beyond creating the RFC file itself._ diff --git a/docs/verification/alpha-report.md b/docs/verification/alpha-report.md new file mode 100644 index 0000000..ac6e9e2 --- /dev/null +++ b/docs/verification/alpha-report.md @@ -0,0 +1,177 @@ +# Alpha Stabilization Report — Sprint 3 + +> **Status**: PENDING LIVE VALIDATION +> +> This report template is generated by the engineering platform. Quantitative rows will +> be replaced with real data after executing the extension against a live Instagram +> Saved collection. The instrumentation infrastructure is in place and ready. + +## How to Generate This Report + +1. Run `pnpm run build` inside `apps/extension` +2. Load `apps/extension/dist` as an unpacked extension in Chrome +3. Navigate to `https://www.instagram.com/saved` +4. Click **Start Extraction** in the popup +5. Scroll through the saved collection until the feed ends +6. Click **Export JSON** — this downloads the full `ISessionReport` +7. Replace the placeholder values below with data from the export + +--- + +## Session Summary + +| Field | Value | +| :--------- | :-------------------------------- | +| Session ID | _(from export)_ | +| Page URL | `https://www.instagram.com/saved` | +| Started At | _(from export)_ | +| Ended At | _(from export)_ | +| Heap Usage | _(MB — from export)_ | + +--- + +## Extraction Metrics + +| Metric | Count | Notes | +| :-------------------------------- | :---: | :---------------------------------------------------------- | +| **Discovered** | — | Resources found by `DiscoveryEngine` via `MutationObserver` | +| **Extracted** | — | Resources successfully normalized into `IResource` | +| **Duplicates** | — | Resources skipped by fingerprint deduplication | +| **Skipped** | — | Resources skipped by feature flags | +| **Failed** | — | Resources that produced an `IFailureRecord` | +| **Extraction time (total ms)** | — | Wall-clock time in content script extraction | +| **Normalization time (total ms)** | — | Wall-clock time in `InstagramNormalizer` | +| **Average extraction time (ms)** | — | `extractionTimeMs / extracted` | +| **Success rate** | — | `extracted / discovered * 100` % | + +--- + +## Strategy Usage + +| Strategy | Invocations | Notes | +| :---------------------------- | :---------: | :----------------------------- | +| `SemanticArticleStrategy` | — | Primary path | +| `DataAttributeStrategy` | — | Fallback 1 | +| `StructuralHeuristicStrategy` | — | Fallback 2 (lowest confidence) | + +--- + +## Failure Analysis + +> Failures are automatically classified into four categories by the Background Worker. +> Each failure emits an `IFailureRecord` containing a trimmed DOM snapshot. + +### Summary + +| Category | Count | % of Failures | +| :---------------------- | :---: | :-----------: | +| `selector_failure` | — | — | +| `parsing_failure` | — | — | +| `normalization_failure` | — | — | +| `network_error` | — | — | +| `unknown` | — | — | + +--- + +### Known Failure Modes (Pre-Alpha Hypotheses) + +These are anticipated based on static analysis of the Instagram DOM. Each must be confirmed or +refuted with real-world evidence after the live run. + +--- + +#### FM-001: Dynamic class name obfuscation + +- **Category**: `selector_failure` +- **Root cause**: Instagram uses Webpack-generated class names that rotate on deployments. + The `SemanticArticleStrategy` relies on stable ARIA roles and `href` patterns, not class names, + but the `DataAttributeStrategy` fallback may attempt to query class-based attributes that no + longer exist. +- **DOM Snapshot**: _(attach from live run)_ +- **Proposed fix**: Remove all class-name selectors from strategies. Rely exclusively on ARIA roles, + `href` patterns, `time[datetime]`, and element tag names. +- **Regression test**: `connectors/instagram/tests/fixtures/obfuscated-classes.html` +- **Status**: Hypothesis — requires live confirmation + +--- + +#### FM-002: Infinite scroll boundary — last element not discovered + +- **Category**: `selector_failure` +- **Root cause**: When the Instagram feed reaches the end, the `MutationObserver` may fire a final + mutation that removes rather than adds articles, causing the last batch to be missed. +- **DOM Snapshot**: _(attach from live run)_ +- **Proposed fix**: Add a final `scanDOM(document.body)` call when the observer detects no more + addedNodes after a 2-second idle period. +- **Regression test**: `connectors/instagram/tests/fixtures/feed-end.html` +- **Status**: Hypothesis — requires live confirmation + +--- + +#### FM-003: Protected content — login-gated media + +- **Category**: `network_error` +- **Root cause**: CDN media URLs embedded in `` attributes may expire or return 403 + for non-authenticated requests when accessed outside the page context. +- **Proposed fix**: Record the source URI at extraction time and flag it as `localUri: undefined`. + Do not attempt to fetch the URI during extraction. Deferral to the hydration phase is correct. +- **Regression test**: N/A (the domain model already handles `localUri: undefined`) +- **Status**: Hypothesis — requires live confirmation + +--- + +#### FM-004: Carousel pagination — only first slide captured + +- **Category**: `parsing_failure` +- **Root cause**: Instagram renders carousel slides lazily. Only the first slide's `` is + present in the DOM on initial load; subsequent slides are injected on swipe/click. +- **DOM Snapshot**: _(attach from live run)_ +- **Proposed fix**: After detecting a carousel via the dot-indicator heuristic, dispatch a + synthetic click on the next-slide button in the content script, wait 300ms for the DOM mutation, + and re-scan. Repeat until no next button is present. +- **Regression test**: `connectors/instagram/tests/fixtures/carousel-lazy-slides.html` +- **Status**: Hypothesis — requires live confirmation + +--- + +#### FM-005: Reel video URI missing — video not yet buffered + +- **Category**: `parsing_failure` +- **Root cause**: Reel `