diff --git a/CONTINUATION_PROMPT.md b/CONTINUATION_PROMPT.md new file mode 100644 index 0000000..2844b4c --- /dev/null +++ b/CONTINUATION_PROMPT.md @@ -0,0 +1,562 @@ +# S3 CMS - Development Continuation Prompt + +**Last Updated:** 2025-10-19 +**Branch:** `refactor-to-local-db` +**Status:** โœ… Production-Ready for Content Management + +--- + +## ๐ŸŽฏ Current State + +The app has been **successfully refactored to a local-first architecture** with SQLite as the primary data store. All core functionality is working: + +- โœ… Local SQLite database for collections, entries, and assets +- โœ… Instant CRUD operations (no S3 reads during editing) +- โœ… Automatic background sync to S3 (incremental backups) +- โœ… Publishing workflow (local DB โ†’ generate APIs โ†’ upload to S3) +- โœ… Media library using local database with usage tracking +- โœ… Entry and collection order preservation +- โœ… S3 import/migration on first run +- โœ… Database triggers for automatic source file backup + +--- + +## ๐Ÿ“Š Architecture Overview + +### Data Flow + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ User Creates/Edits Content โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Local SQLite Database (Instant) โ”‚ +โ”‚ โ€ข collections table โ”‚ +โ”‚ โ€ข entries table โ”‚ +โ”‚ โ€ข collection_entry_order table โ”‚ +โ”‚ โ€ข bucket_collection_order table โ”‚ +โ”‚ โ€ข assets table โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + (Database Trigger Fires) + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ data_sync_queue table โ”‚ +โ”‚ โ€ข Queues source file sync operations โ”‚ +โ”‚ โ€ข Processed every 10 seconds by SyncService โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ฌโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ SyncService Background Process (Every 10s) โ”‚ +โ”‚ โ€ข Picks up queued operations โ”‚ +โ”‚ โ€ข Calls PublishingService.syncSourceFile() โ”‚ +โ”‚ โ€ข Uploads to S3: collections/{id}.json, entries/{id}.json โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ S3 Bucket (Backup) โ”‚ +โ”‚ โ€ข collections/{id}.json - Individual collection files โ”‚ +โ”‚ โ€ข entries/{id}.json - Individual entry files โ”‚ +โ”‚ โ€ข schemas/{name}.json - Schema definitions โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + (User Clicks "Publish") + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ PublishingService.generateAllApiEndpoints() โ”‚ +โ”‚ โ€ข Reads from local DB โ”‚ +โ”‚ โ€ข Filters for published content only โ”‚ +โ”‚ โ€ข Generates API JSON files โ”‚ +โ”‚ โ€ข Uploads to S3: api/collections/{slug}.json โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ + โ–ผ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ Published API Endpoints (S3) โ”‚ +โ”‚ โ€ข api/collections.json - All collections index โ”‚ +โ”‚ โ€ข api/collections/{slug}.json - Individual collections โ”‚ +โ”‚ โ€ข api/collections/{SchemaName}.json - Schema lookups โ”‚ +โ”‚ โ€ข Only contains PUBLISHED content โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +--- + +## ๐Ÿ—‚๏ธ Key Files & Services + +### Main Services + +1. **DatabaseService** (`src/main/services/databaseService.ts`) + - All SQLite database operations + - CRUD for collections, entries, assets + - Entry/collection order management + - Migration system (currently at v2) + - Key method: `parseEntry()` - transforms DB data for UI + +2. **PublishingService** (`src/main/services/publishingService.ts`) + - Reads from local DB + - Generates API endpoint JSON files + - Uploads to S3 + - Methods: + - `generateAllApiEndpoints()` - Full publish + - `generateSelectiveApiEndpoints()` - Partial publish + - `syncSourceFile()` - Individual file backup (called by SyncService) + - `deleteSourceFile()` - Delete from S3 + +3. **SyncService** (`src/main/services/syncService.ts`) + - Background process (runs every 10 seconds) + - Processes `data_sync_queue` table + - Automatically backs up source files to S3 + - Handles `source_file_sync` and `source_file_delete` operations + +4. **MigrationService** (`src/main/services/migrationService.ts`) + - Imports existing S3 data into local DB on first run + - Methods: + - `needsMigration()` - Checks if DB is empty + - `migrate()` - Imports collections, entries, assets, and order + - `migrateAssets()` - Imports images and audio from S3 + +5. **MediaService** (`src/main/services/mediaService.ts`) + - Lists assets from local DB (fast!) + - Fallback to S3 scanning if DB unavailable + - Tracks asset usage across collections and entries + - Methods: + - `listAssets()` - Reads from DB + - `trackAssetUsageFromDB()` - Scans entries for asset URLs + +### Database Schema + +**Location:** `src/main/database/schema.sql` + +**Current Version:** 2 + +**Key Tables:** + +- `collections` - Collection data +- `entries` - Entry data (with nested `data` JSON field) +- `collection_entry_order` - Entry ordering per collection +- `bucket_collection_order` - Collection ordering per bucket +- `assets` - Media assets (images, audio) +- `data_sync_queue` - Background sync queue +- `migrations` - Migration tracking + +**Triggers (v2):** + +- `trg_collections_after_insert/update/delete` - Auto-queue collection sync +- `trg_entries_after_insert/update/delete` - Auto-queue entry sync + +### Main Process + +**Location:** `src/main/main.ts` + +**Key Sections:** + +- Lines 209-220: Initialize DatabaseService and SyncService +- Lines 226-280: Configure services with active bucket +- Lines 248-252: Configure SyncService with PublishingService +- Lines 932-1130: IPC handlers for DB operations (create, update, delete) +- Lines 655-695: Publishing IPC handlers + +**Important Notes:** + +- Database triggers handle automatic sync queue population +- Manual `addToSyncQueue()` calls have been removed (redundant) +- Publishing only generates APIs for published content + +--- + +## ๐Ÿ”ง Configuration + +### Sync Prefix (Testing vs Production) + +**Location:** `src/main/main.ts` lines 236-245 + +```typescript +const useSyncPrefix = false // Set to true to publish to /sync/api/... for testing +if (useSyncPrefix) { + publishingService.setSyncPrefix('sync') + console.log('[Publishing] ๐Ÿงช TEST MODE: Using /sync prefix...') +} else { + publishingService.setSyncPrefix('') + console.log('[Publishing] ๐Ÿš€ PRODUCTION MODE: Publishing directly to /api/...') +} +``` + +**When to use:** + +- `false` (default) - Production: Files go to `collections/`, `entries/`, `api/` +- `true` - Testing: Files go to `sync/collections/`, `sync/entries/`, `sync/api/` + +--- + +## โœ… What's Working + +### Content Management + +- โœ… Create/edit/delete collections (instant) +- โœ… Create/edit/delete entries (instant) +- โœ… Drag-and-drop reordering (collections and entries) +- โœ… Entry metadata (title, slug, description) +- โœ… Collection metadata with primary image +- โœ… Schema assignment and validation +- โœ… Published/unpublished state toggle + +### Data Persistence + +- โœ… All data stored in local SQLite +- โœ… Order preserved in separate tables +- โœ… Migration from S3 on first run +- โœ… Automatic incremental backups to S3 (every 10s) + +### Publishing + +- โœ… Only published content in API endpoints +- โœ… Local DB read (fast) +- โœ… Collection order preserved in APIs +- โœ… Entry order preserved in APIs +- โœ… Schema lookup files generated +- โœ… Individual collection/entry files (for migration) + +### Media Library + +- โœ… Assets loaded from local DB +- โœ… Usage tracking (which entries/collections use each asset) +- โœ… Image variant matching (thumbnail-, small-, etc.) +- โœ… Audio asset tracking +- โœ… Duplicate prevention in usage tracking +- โœ… Fallback to S3 scanning if needed + +### Multi-Computer Workflow + +- โœ… Computer A: Changes auto-sync to S3 +- โœ… Computer B: "Clear Database" button reimports from S3 +- โœ… Source files in migration-compatible format + +### Migration System + +- โœ… Automatic on app startup (if DB empty) +- โœ… Automatic on bucket switch (if new bucket DB empty) +- โœ… UI modal with progress indication +- โœ… Error handling with user-friendly messages +- โœ… Non-blocking (doesn't prevent app usage on failure) + +--- + +## ๐Ÿ› Known Issues + +### Port 5173 Blocked + +**Symptom:** `Error: listen EPERM: operation not permitted 127.0.0.1:5173` + +**Fix:** + +```bash +lsof -ti tcp:5173 | xargs kill -9 +npm run dev +``` + +**Root Cause:** macOS firewall blocking Vite dev server + +**Workaround Applied:** Changed `vite.config.ts` to use `host: '127.0.0.1'` instead of IPv6 + +### Schema Editor Still Uses S3 + +**Status:** Not yet refactored + +**Impact:** Loading/editing schemas still requires S3 connection + +**Next Step:** Move schemas to local DB (future enhancement) + +### ~~Media Library Not Refreshing on Bucket Switch~~ + +**Status:** โœ… FIXED (commit d632dd9) + +**Fix:** Added full service reconfiguration in `storage:switch-active-bucket` IPC handler + +**Impact:** Media library now loads correct assets immediately when switching buckets (no restart needed) + +### ~~Migration Required App Restart~~ + +**Status:** โœ… FIXED (commit 0983016) + +**Fix:** Added automatic migration detection when switching buckets + migration UI modal + +**Impact:** + +- Migration now runs automatically when switching to an empty bucket +- Beautiful modal shows migration progress (loading โ†’ success/error) +- No manual app restart needed +- Migration also blocks UI during initial app load if needed + +--- + +## ๐ŸŽฏ Next Steps / TODO + +### High Priority + +1. **Test Production Build Tonight** + - User is finishing wife's content + - Monitor for any production-specific issues + - Check that published APIs work correctly + +2. **Multi-Computer Testing** + - Test the full workflow: + - Edit on Computer A + - Wait for sync (10s) + - "Clear Database" on Computer B + - Verify all changes appear + +### Medium Priority + +3. **Schema Management** + - Move schema definitions to local DB + - Currently still reads/writes directly to S3 + - Would complete the local-first architecture + +4. **Manual Sync Trigger** + - Add a "Sync Now" button in UI + - Currently auto-syncs every 10s + - Users might want immediate sync before closing app + +5. **Sync Status UI** + - Show sync queue size in status bar + - Show last sync time + - Show sync errors if any + +### Low Priority + +6. **Real-time Sync Service** + - Currently syncs every 10 seconds + - Could use file system watchers for instant sync + - Or websockets for multi-computer collaboration + +7. **Conflict Resolution** + - What happens if two computers edit the same entry? + - Currently: last publish wins + - Future: Could add conflict detection/resolution + +8. **Schema Validation** + - Validate entry data against schema on save + - Currently: UI enforces schema, but no server-side validation + +--- + +## ๐Ÿ“ Development Commands + +```bash +# Development +npm run dev # Start dev server (Vite + Electron) +npm run build:main # Build main process only (faster iteration) +npm run build # Full build (renderer + main + copy DB) + +# Production +npm run package # Package app for distribution + +# Database Management +# In app: Settings โ†’ Clear Database (reimports from S3) + +# Git +git status # Check current changes +git log --oneline -10 # Recent commits +git diff # See changes +``` + +--- + +## ๐Ÿ” Debugging Tips + +### View Database Contents + +```bash +sqlite3 "/Users/[username]/Library/Application Support/Electron/s3-cms-poc.db" + +# Useful queries: +SELECT COUNT(*) FROM collections; +SELECT COUNT(*) FROM entries; +SELECT COUNT(*) FROM assets; +SELECT * FROM data_sync_queue WHERE status = 'pending'; +SELECT * FROM migrations; +``` + +### Check Logs + +- Main process logs: Terminal where you ran `npm run dev` +- Renderer logs: DevTools Console (Cmd+Option+I) +- Key log prefixes: + - `[DatabaseService]` - DB operations + - `[SyncService]` - Background sync + - `[PublishingService]` - API generation and S3 uploads + - `[MigrationService]` - S3 import + - `[MediaService]` - Asset management + +### Rebuild After Changes + +```bash +# After changing main process files: +npm run build:main + +# After changing renderer files: +# Vite will hot-reload automatically in dev mode + +# After changing schema.sql: +# Need to clear DB or bump migration version +``` + +--- + +## ๐Ÿ“š Important Context + +### Data Structure Evolution + +**Entry Data Storage:** +The app handles three different entry data structures for backward compatibility: + +1. **New (Current):** + + ```json + { + "id": "...", + "data": { + "field1": "value1", + "metadata": { "title": "...", "slug": "...", "description": "..." }, + "tags": ["tag1", "tag2"] + } + } + ``` + +2. **S3/Legacy:** + + ```json + { + "id": "...", + "field1": "value1", + "metadata": { "title": "...", "slug": "...", "description": "..." }, + "tags": ["tag1", "tag2"] + } + ``` + +3. **Very Old:** + ```json + { + "id": "...", + "field1": "value1" + } + ``` + +The `parseEntry()` method handles all three formats. + +### Publishing vs Syncing + +**Two Separate Operations:** + +1. **Background Sync (SyncService)** + - Automatic, every 10 seconds + - Backs up ALL content (published and unpublished) + - Saves to: `collections/{id}.json`, `entries/{id}.json` + - For: Migration, backup, multi-computer sync + +2. **Publishing (PublishingService)** + - Manual, when user clicks "Publish" + - Only PUBLISHED content + - Generates API endpoints: `api/collections/{slug}.json` + - For: Public consumption, website APIs + +### Bucket Configuration + +Active bucket config stored in: `~/Library/Application Support/Electron/bucketlist-settings.json` + +**Structure:** + +```json +{ + "activeBucketId": "...", + "buckets": [ + { + "id": "...", + "name": "Bucket Name", + "s3Config": { ... }, + "directories": { + "collections": "collections", + "entries": "entries", + "schemas": "schemas", + "assets": "assets", + "api": "api" + } + } + ] +} +``` + +--- + +## ๐Ÿš€ Quick Start for New Session + +1. **Check Current State:** + + ```bash + git status + git log --oneline -5 + npm run dev # Verify everything starts + ``` + +2. **Review Recent Work:** + - Read `COMMIT_EDITMSG` for last commit message + - Check `git diff HEAD~3` for recent changes + - Look at this file (CONTINUATION_PROMPT.md) + +3. **Test Core Features:** + - Create a test collection + - Create a test entry + - Wait 10 seconds, check S3 for source files + - Click "Publish", check S3 for API files + - Delete test content + +4. **If Issues:** + - Check logs for errors + - Verify database exists and has data + - Check sync queue: `SELECT * FROM data_sync_queue WHERE status = 'failed'` + - Clear database and reimport if needed + +--- + +## ๐Ÿ’ก Tips for AI Assistant + +1. **Database Operations:** All DB code is in `DatabaseService`. Don't bypass it. + +2. **S3 Writes:** Only through `PublishingService`. Don't use `ContentService` for new writes. + +3. **Sync Queue:** Populated automatically by triggers. Don't add manual `addToSyncQueue()` calls for collections/entries. + +4. **Entry Metadata:** Always check `parseEntry()` to see how data is transformed. Metadata is extracted to root level for UI. + +5. **Collection Schema:** Check `parseCollection()` - it converts snake_case DB columns to camelCase for UI. + +6. **Media Assets:** Read from DB first, fallback to S3. See `MediaService.listAssets()`. + +7. **Testing:** Always test with "Clear Database" โ†’ reimport โ†’ edit โ†’ publish workflow. + +8. **Commits:** Use conventional commits (feat:, fix:, refactor:). Be descriptive. + +--- + +## ๐Ÿ“ž Contact / Questions + +If you encounter issues or need clarification, check: + +1. This document +2. `POC_TESTING_GUIDE.md` - Original POC documentation +3. `LOCAL_DATABASE_REFACTORING_PROMPT_V2.md` - Detailed refactoring plan +4. Git commit history - Each commit has detailed messages +5. Code comments - Key methods are documented + +--- + +**End of Continuation Prompt** + +Good luck with production testing tonight! ๐ŸŽ‰ diff --git a/LOCAL_DATABASE_REFACTORING_PROMPT_V2.md b/LOCAL_DATABASE_REFACTORING_PROMPT_V2.md new file mode 100644 index 0000000..d3e9b46 --- /dev/null +++ b/LOCAL_DATABASE_REFACTORING_PROMPT_V2.md @@ -0,0 +1,2295 @@ +# Local Database Refactoring Prompt v2.0 + +## Overview + +Refactor the S3 CMS application to use a local-first architecture with background S3 synchronization. This will dramatically improve performance, enable offline functionality, and provide a much better user experience. + +**CRITICAL REQUIREMENT: All existing UI components, layouts, styling, and user interactions must remain completely unchanged. This is a backend-only refactoring that should be invisible to users.** + +## Current Problems + +1. **Performance Issues**: Every operation requires S3 API calls (200-500ms latency) +2. **No Offline Support**: Application unusable without internet +3. **Cache Invalidation**: 5-minute TTL cache frequently invalidated +4. **N+1 Query Problem**: Loading collections requires individual entry fetches +5. **Poor UX**: Loading states and delays for every operation + +## Proposed Architecture: Local-First with Background Sync + +### Database Strategy: SQLite Only (Simplified) + +**SQLite as Single Source of Truth (Main Process)**: + +- Primary data store for all application data +- Complex queries and relationships +- Background sync operations +- Data integrity and transactions +- ACID compliance +- Single source of truth eliminates sync complexity + +**Renderer Process**: + +- Queries main process via IPC +- React Query for caching and optimistic updates +- Event listeners for real-time updates from main process +- No separate IndexedDB (removes unnecessary complexity) + +### New Data Flow + +``` +UI Action โ†’ IPC โ†’ SQLite (immediate) โ†’ Background S3 Sync (data only) + โ†“ + Optimistic UI update + โ†“ + Confirm/rollback on sync result + +API Endpoints โ†’ Only updated when user explicitly publishes +``` + +### Why SQLite Only (Not IndexedDB)? + +1. **Simpler Architecture**: One database to manage, not two +2. **No Sync Conflicts**: Single source of truth +3. **Better for Electron**: Main process has full filesystem access +4. **Easier Debugging**: All data in one place +5. **Transaction Support**: Better data integrity +6. **Complex Queries**: SQLite is more powerful than IndexedDB + +## Phase 0: Pre-Refactor Preparation + +### 0.1 Audit Current System + +**Document Current S3 Operations**: + +```typescript +// Create audit script: src/main/scripts/auditS3Usage.ts +export class S3UsageAuditor { + async audit(): Promise<{ + operations: Map + avgLatency: Map + peakUsage: { operation: string; count: number } + }> { + // Log all S3 operations for 1 week + // Track: operation type, frequency, latency, payload size + } +} +``` + +**Current Operations to Track**: + +- Collection CRUD operations +- Entry CRUD operations +- Schema operations +- Media upload/retrieval +- API endpoint generation +- Cache hit/miss rates + +### 0.2 Create Comprehensive Tests + +**Integration Tests for Current Behavior**: + +```typescript +// src/test/integration/currentBehavior.test.ts +describe('Current S3 Behavior - Baseline', () => { + test('Create collection workflow', async () => { + // Test current behavior to ensure nothing breaks + }) + + test('Entry ordering workflow', async () => { + // Document current entry ordering behavior + }) + + test('Multi-bucket switching', async () => { + // Test bucket switching behavior + }) + + test('Publishing workflow', async () => { + // Test current publish behavior + }) +}) +``` + +### 0.3 Proof of Concept + +Build minimal POC to validate approach: + +1. Basic SQLite setup with one table +2. Simple IPC communication +3. Basic sync service +4. Test with one component + +**Success Criteria**: + +- Operations complete in < 50ms +- Sync works reliably +- No race conditions +- Rollback on errors works + +### 0.4 Rollback Plan + +**Keep S3-Direct Mode as Fallback**: + +```typescript +// src/main/config.ts +export const USE_LOCAL_DB = process.env.USE_LOCAL_DB !== 'false' + +// Allow runtime switching for emergency rollback +export function setDatabaseMode(useLocal: boolean) { + // Switch between local DB and direct S3 +} +``` + +## Implementation Plan + +### Phase 1: Database Setup and Migration + +#### 1.1 Install Dependencies + +```bash +npm install better-sqlite3 @tanstack/react-query +npm install --save-dev @types/better-sqlite3 @electron/rebuild +``` + +**Post-install script** (package.json): + +```json +{ + "scripts": { + "postinstall": "electron-rebuild -f -w better-sqlite3" + } +} +``` + +#### 1.2 Create Database Schema (SQLite) + +Create `src/main/database/schema.sql`: + +```sql +-- Buckets table (store bucket configurations) +CREATE TABLE buckets ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + region TEXT NOT NULL, + access_key_id TEXT, + credentials TEXT, -- Encrypted JSON + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + last_synced_at TEXT, + is_active BOOLEAN DEFAULT TRUE +); + +-- Schemas table (store schema definitions per bucket) +CREATE TABLE schemas ( + id TEXT PRIMARY KEY, + bucket_id TEXT NOT NULL, + name TEXT NOT NULL, + display_name TEXT, + definition TEXT NOT NULL, -- JSON schema definition + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + sync_status TEXT DEFAULT 'pending', + sync_error TEXT, + last_synced_at TEXT, + FOREIGN KEY (bucket_id) REFERENCES buckets(id) ON DELETE CASCADE +); + +-- Collections table +CREATE TABLE collections ( + id TEXT PRIMARY KEY, + bucket_id TEXT NOT NULL, + name TEXT NOT NULL, + description TEXT, + schema_id TEXT NOT NULL, + schema_name TEXT NOT NULL, + published BOOLEAN DEFAULT FALSE, + published_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + metadata TEXT, -- JSON object + sync_status TEXT DEFAULT 'pending', -- pending, syncing, synced, error + sync_error TEXT, + last_synced_at TEXT, + FOREIGN KEY (bucket_id) REFERENCES buckets(id) ON DELETE CASCADE, + FOREIGN KEY (schema_id) REFERENCES schemas(id) +); + +-- Collection entry order (separate table for efficient reordering) +CREATE TABLE collection_entry_order ( + collection_id TEXT NOT NULL, + entry_id TEXT NOT NULL, + position INTEGER NOT NULL, + PRIMARY KEY (collection_id, entry_id), + FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE, + FOREIGN KEY (entry_id) REFERENCES entries(id) ON DELETE CASCADE +); + +-- Entries table +CREATE TABLE entries ( + id TEXT PRIMARY KEY, + bucket_id TEXT NOT NULL, + collection_id TEXT NOT NULL, + schema_id TEXT NOT NULL, + schema_name TEXT NOT NULL, + data TEXT NOT NULL, -- JSON object + published BOOLEAN DEFAULT FALSE, + published_at TEXT, + tags TEXT, -- JSON array + metadata TEXT, -- JSON object + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + sync_status TEXT DEFAULT 'pending', + sync_error TEXT, + last_synced_at TEXT, + FOREIGN KEY (bucket_id) REFERENCES buckets(id) ON DELETE CASCADE, + FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE, + FOREIGN KEY (schema_id) REFERENCES schemas(id) +); + +-- Media cache (for images and other media) +CREATE TABLE media_cache ( + id TEXT PRIMARY KEY, + bucket_id TEXT NOT NULL, + key TEXT NOT NULL, -- S3 key + url TEXT NOT NULL, -- S3 URL + local_path TEXT, -- Local cached file path + file_size INTEGER, + mime_type TEXT, + width INTEGER, + height INTEGER, + metadata TEXT, -- JSON object (srcset info, etc) + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + last_accessed_at TEXT, + cache_status TEXT DEFAULT 'pending', -- pending, cached, expired + UNIQUE(bucket_id, key), + FOREIGN KEY (bucket_id) REFERENCES buckets(id) ON DELETE CASCADE +); + +-- Data sync queue for background operations +CREATE TABLE data_sync_queue ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + bucket_id TEXT NOT NULL, + operation TEXT NOT NULL, -- create, update, delete + entity_type TEXT NOT NULL, -- bucket, schema, collection, entry, media + entity_id TEXT NOT NULL, + data TEXT, -- JSON data for the operation + retry_count INTEGER DEFAULT 0, + max_retries INTEGER DEFAULT 3, + created_at TEXT NOT NULL, + started_at TEXT, + completed_at TEXT, + status TEXT DEFAULT 'pending', -- pending, processing, completed, failed + error_message TEXT, + FOREIGN KEY (bucket_id) REFERENCES buckets(id) ON DELETE CASCADE +); + +-- Dirty state tracking (for API publishing) +CREATE TABLE dirty_state ( + entity_type TEXT NOT NULL, -- collection, entry + entity_id TEXT NOT NULL, + bucket_id TEXT NOT NULL, + marked_dirty_at TEXT NOT NULL, + PRIMARY KEY (entity_type, entity_id), + FOREIGN KEY (bucket_id) REFERENCES buckets(id) ON DELETE CASCADE +); + +-- Sync conflicts (for manual resolution) +CREATE TABLE sync_conflicts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + bucket_id TEXT NOT NULL, + entity_type TEXT NOT NULL, + entity_id TEXT NOT NULL, + local_version TEXT NOT NULL, -- JSON snapshot + remote_version TEXT NOT NULL, -- JSON snapshot + local_updated_at TEXT NOT NULL, + remote_updated_at TEXT NOT NULL, + detected_at TEXT NOT NULL, + resolved BOOLEAN DEFAULT FALSE, + resolution TEXT, -- keep_local, keep_remote, merge + FOREIGN KEY (bucket_id) REFERENCES buckets(id) ON DELETE CASCADE +); + +-- Indexes for performance +CREATE INDEX idx_buckets_active ON buckets(is_active); +CREATE INDEX idx_schemas_bucket_id ON schemas(bucket_id); +CREATE INDEX idx_schemas_name ON schemas(bucket_id, name); +CREATE INDEX idx_collections_bucket_id ON collections(bucket_id); +CREATE INDEX idx_collections_schema_id ON collections(schema_id); +CREATE INDEX idx_collections_published ON collections(bucket_id, published); +CREATE INDEX idx_collection_entry_order_position ON collection_entry_order(collection_id, position); +CREATE INDEX idx_entries_bucket_id ON entries(bucket_id); +CREATE INDEX idx_entries_collection_id ON entries(collection_id); +CREATE INDEX idx_entries_published ON entries(bucket_id, published); +CREATE INDEX idx_media_cache_bucket_key ON media_cache(bucket_id, key); +CREATE INDEX idx_media_cache_accessed ON media_cache(last_accessed_at); +CREATE INDEX idx_data_sync_queue_status ON data_sync_queue(status, created_at); +CREATE INDEX idx_data_sync_queue_bucket ON data_sync_queue(bucket_id, status); +CREATE INDEX idx_dirty_state_bucket ON dirty_state(bucket_id); +CREATE INDEX idx_sync_conflicts_resolved ON sync_conflicts(resolved); +``` + +#### 1.3 Create Database Service (SQLite) + +Create `src/main/services/databaseService.ts`: + +```typescript +import Database from 'better-sqlite3' +import { app } from 'electron' +import path from 'path' +import fs from 'fs' + +export interface Collection { + id: string + bucket_id: string + name: string + description?: string + schema_id: string + schema_name: string + published: boolean + published_at?: string + created_at: string + updated_at: string + metadata?: any + sync_status: 'pending' | 'syncing' | 'synced' | 'error' + sync_error?: string + last_synced_at?: string +} + +export interface Entry { + id: string + bucket_id: string + collection_id: string + schema_id: string + schema_name: string + data: any + published: boolean + published_at?: string + tags?: string[] + metadata?: any + created_at: string + updated_at: string + sync_status: 'pending' | 'syncing' | 'synced' | 'error' + sync_error?: string + last_synced_at?: string +} + +export interface Schema { + id: string + bucket_id: string + name: string + display_name?: string + definition: any + created_at: string + updated_at: string + sync_status: 'pending' | 'syncing' | 'synced' | 'error' + sync_error?: string + last_synced_at?: string +} + +export class DatabaseService { + private db: Database.Database | null = null + private readonly dbPath: string + + constructor() { + const userDataPath = app.getPath('userData') + this.dbPath = path.join(userDataPath, 's3-cms.db') + } + + async initialize(): Promise { + this.db = new Database(this.dbPath) + + // Enable WAL mode for better concurrency + this.db.pragma('journal_mode = WAL') + + // Enable foreign keys + this.db.pragma('foreign_keys = ON') + + // Set reasonable timeouts + this.db.pragma('busy_timeout = 5000') + + // Run migrations + await this.runMigrations() + } + + private async runMigrations(): Promise { + if (!this.db) throw new Error('Database not initialized') + + // Create migrations table + this.db.exec(` + CREATE TABLE IF NOT EXISTS migrations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + version INTEGER NOT NULL UNIQUE, + name TEXT NOT NULL, + executed_at TEXT NOT NULL + ) + `) + + const schemaPath = path.join(__dirname, '../database/schema.sql') + const currentVersion = this.getCurrentVersion() + + if (currentVersion === 0) { + // Initial schema + const schema = fs.readFileSync(schemaPath, 'utf-8') + this.db.exec(schema) + this.recordMigration(1, 'initial_schema') + } + + // Add future migrations here + } + + private getCurrentVersion(): number { + if (!this.db) return 0 + const result = this.db.prepare('SELECT MAX(version) as version FROM migrations').get() as { + version: number | null + } + return result?.version || 0 + } + + private recordMigration(version: number, name: string): void { + if (!this.db) throw new Error('Database not initialized') + this.db + .prepare('INSERT INTO migrations (version, name, executed_at) VALUES (?, ?, ?)') + .run(version, name, new Date().toISOString()) + } + + // ======================================== + // Collection Operations + // ======================================== + + async createCollection( + data: Omit + ): Promise { + if (!this.db) throw new Error('Database not initialized') + + const now = new Date().toISOString() + const collection: Collection = { + ...data, + created_at: now, + updated_at: now, + sync_status: 'pending', + metadata: JSON.stringify(data.metadata || {}) + } as any + + this.db + .prepare( + ` + INSERT INTO collections ( + id, bucket_id, name, description, schema_id, schema_name, + published, published_at, created_at, updated_at, metadata, sync_status + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ` + ) + .run( + collection.id, + collection.bucket_id, + collection.name, + collection.description, + collection.schema_id, + collection.schema_name, + collection.published ? 1 : 0, + collection.published_at, + collection.created_at, + collection.updated_at, + collection.metadata, + collection.sync_status + ) + + return this.getCollection(collection.id)! + } + + async updateCollection(id: string, updates: Partial): Promise { + if (!this.db) throw new Error('Database not initialized') + + const now = new Date().toISOString() + const fields: string[] = [] + const values: any[] = [] + + // Build dynamic update query + for (const [key, value] of Object.entries(updates)) { + if (key === 'id' || key === 'created_at') continue // Don't update these + if (key === 'metadata' || key === 'tags') { + fields.push(`${key} = ?`) + values.push(JSON.stringify(value)) + } else if (key === 'published') { + fields.push(`${key} = ?`) + values.push(value ? 1 : 0) + } else { + fields.push(`${key} = ?`) + values.push(value) + } + } + + fields.push('updated_at = ?') + values.push(now) + values.push(id) + + this.db + .prepare( + ` + UPDATE collections SET ${fields.join(', ')} WHERE id = ? + ` + ) + .run(...values) + + return this.getCollection(id)! + } + + async getCollection(id: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + const row = this.db.prepare('SELECT * FROM collections WHERE id = ?').get(id) as any + if (!row) return null + + return this.parseCollection(row) + } + + async listCollections(bucketId: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + const rows = this.db + .prepare('SELECT * FROM collections WHERE bucket_id = ? ORDER BY updated_at DESC') + .all(bucketId) as any[] + return rows.map(row => this.parseCollection(row)) + } + + async deleteCollection(id: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + this.db.prepare('DELETE FROM collections WHERE id = ?').run(id) + } + + private parseCollection(row: any): Collection { + return { + ...row, + published: Boolean(row.published), + metadata: row.metadata ? JSON.parse(row.metadata) : {} + } + } + + // ======================================== + // Entry Operations + // ======================================== + + async createEntry( + data: Omit + ): Promise { + if (!this.db) throw new Error('Database not initialized') + + const now = new Date().toISOString() + const entry: Entry = { + ...data, + created_at: now, + updated_at: now, + sync_status: 'pending', + data: JSON.stringify(data.data), + tags: JSON.stringify(data.tags || []), + metadata: JSON.stringify(data.metadata || {}) + } as any + + this.db + .prepare( + ` + INSERT INTO entries ( + id, bucket_id, collection_id, schema_id, schema_name, + data, published, published_at, tags, metadata, + created_at, updated_at, sync_status + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ` + ) + .run( + entry.id, + entry.bucket_id, + entry.collection_id, + entry.schema_id, + entry.schema_name, + entry.data, + entry.published ? 1 : 0, + entry.published_at, + entry.tags, + entry.metadata, + entry.created_at, + entry.updated_at, + entry.sync_status + ) + + return this.getEntry(entry.id)! + } + + async updateEntry(id: string, updates: Partial): Promise { + if (!this.db) throw new Error('Database not initialized') + + const now = new Date().toISOString() + const fields: string[] = [] + const values: any[] = [] + + for (const [key, value] of Object.entries(updates)) { + if (key === 'id' || key === 'created_at') continue + if (key === 'data' || key === 'metadata' || key === 'tags') { + fields.push(`${key} = ?`) + values.push(JSON.stringify(value)) + } else if (key === 'published') { + fields.push(`${key} = ?`) + values.push(value ? 1 : 0) + } else { + fields.push(`${key} = ?`) + values.push(value) + } + } + + fields.push('updated_at = ?') + values.push(now) + values.push(id) + + this.db + .prepare( + ` + UPDATE entries SET ${fields.join(', ')} WHERE id = ? + ` + ) + .run(...values) + + return this.getEntry(id)! + } + + async getEntry(id: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + const row = this.db.prepare('SELECT * FROM entries WHERE id = ?').get(id) as any + if (!row) return null + + return this.parseEntry(row) + } + + async listEntries(collectionId: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + // Get entries with their order + const rows = this.db + .prepare( + ` + SELECT e.*, ceo.position + FROM entries e + LEFT JOIN collection_entry_order ceo ON e.id = ceo.entry_id + WHERE e.collection_id = ? + ORDER BY COALESCE(ceo.position, 999999), e.created_at DESC + ` + ) + .all(collectionId) as any[] + + return rows.map(row => this.parseEntry(row)) + } + + async deleteEntry(id: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + this.db.prepare('DELETE FROM entries WHERE id = ?').run(id) + } + + private parseEntry(row: any): Entry { + return { + ...row, + published: Boolean(row.published), + data: row.data ? JSON.parse(row.data) : {}, + tags: row.tags ? JSON.parse(row.tags) : [], + metadata: row.metadata ? JSON.parse(row.metadata) : {} + } + } + + // ======================================== + // Entry Order Operations + // ======================================== + + async updateEntryOrder(collectionId: string, entryIds: string[]): Promise { + if (!this.db) throw new Error('Database not initialized') + + // Use transaction for atomicity + const transaction = this.db.transaction(() => { + // Delete existing order + this.db!.prepare('DELETE FROM collection_entry_order WHERE collection_id = ?').run( + collectionId + ) + + // Insert new order + const stmt = this.db!.prepare(` + INSERT INTO collection_entry_order (collection_id, entry_id, position) + VALUES (?, ?, ?) + `) + + entryIds.forEach((entryId, index) => { + stmt.run(collectionId, entryId, index) + }) + }) + + transaction() + } + + async getEntryOrder(collectionId: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + const rows = this.db + .prepare( + ` + SELECT entry_id FROM collection_entry_order + WHERE collection_id = ? + ORDER BY position ASC + ` + ) + .all(collectionId) as { entry_id: string }[] + + return rows.map(row => row.entry_id) + } + + // ======================================== + // Schema Operations + // ======================================== + + async createSchema( + data: Omit + ): Promise { + if (!this.db) throw new Error('Database not initialized') + + const now = new Date().toISOString() + const schema: Schema = { + ...data, + created_at: now, + updated_at: now, + sync_status: 'pending', + definition: JSON.stringify(data.definition) + } as any + + this.db + .prepare( + ` + INSERT INTO schemas ( + id, bucket_id, name, display_name, definition, + created_at, updated_at, sync_status + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ` + ) + .run( + schema.id, + schema.bucket_id, + schema.name, + schema.display_name, + schema.definition, + schema.created_at, + schema.updated_at, + schema.sync_status + ) + + return this.getSchema(schema.id)! + } + + async getSchema(id: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + const row = this.db.prepare('SELECT * FROM schemas WHERE id = ?').get(id) as any + if (!row) return null + + return this.parseSchema(row) + } + + async listSchemas(bucketId: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + const rows = this.db.prepare('SELECT * FROM schemas WHERE bucket_id = ?').all(bucketId) as any[] + return rows.map(row => this.parseSchema(row)) + } + + private parseSchema(row: any): Schema { + return { + ...row, + definition: row.definition ? JSON.parse(row.definition) : {} + } + } + + // ======================================== + // Sync Queue Operations + // ======================================== + + async addToSyncQueue( + bucketId: string, + operation: string, + entityType: string, + entityId: string, + data: any + ): Promise { + if (!this.db) throw new Error('Database not initialized') + + this.db + .prepare( + ` + INSERT INTO data_sync_queue ( + bucket_id, operation, entity_type, entity_id, data, created_at + ) VALUES (?, ?, ?, ?, ?, ?) + ` + ) + .run( + bucketId, + operation, + entityType, + entityId, + JSON.stringify(data), + new Date().toISOString() + ) + } + + async getSyncQueue(limit: number = 10): Promise { + if (!this.db) throw new Error('Database not initialized') + + const rows = this.db + .prepare( + ` + SELECT * FROM data_sync_queue + WHERE status = 'pending' AND retry_count < max_retries + ORDER BY created_at ASC + LIMIT ? + ` + ) + .all(limit) as any[] + + return rows.map(row => ({ + ...row, + data: row.data ? JSON.parse(row.data) : null + })) + } + + async markSyncProcessing(queueId: number): Promise { + if (!this.db) throw new Error('Database not initialized') + + this.db + .prepare( + ` + UPDATE data_sync_queue + SET status = 'processing', started_at = ? + WHERE id = ? + ` + ) + .run(new Date().toISOString(), queueId) + } + + async markSyncComplete(queueId: number): Promise { + if (!this.db) throw new Error('Database not initialized') + + this.db + .prepare( + ` + UPDATE data_sync_queue + SET status = 'completed', completed_at = ? + WHERE id = ? + ` + ) + .run(new Date().toISOString(), queueId) + } + + async markSyncError(queueId: number, error: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + this.db + .prepare( + ` + UPDATE data_sync_queue + SET status = 'failed', error_message = ?, retry_count = retry_count + 1 + WHERE id = ? + ` + ) + .run(error, queueId) + } + + // ======================================== + // Dirty State Operations + // ======================================== + + async markAsDirty(entityType: string, entityId: string, bucketId: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + this.db + .prepare( + ` + INSERT OR REPLACE INTO dirty_state (entity_type, entity_id, bucket_id, marked_dirty_at) + VALUES (?, ?, ?, ?) + ` + ) + .run(entityType, entityId, bucketId, new Date().toISOString()) + } + + async getDirtyCollections(bucketId: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + const rows = this.db + .prepare( + ` + SELECT entity_id FROM dirty_state + WHERE entity_type = 'collection' AND bucket_id = ? + ` + ) + .all(bucketId) as { entity_id: string }[] + + return rows.map(row => row.entity_id) + } + + async getDirtyEntries(bucketId: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + const rows = this.db + .prepare( + ` + SELECT entity_id FROM dirty_state + WHERE entity_type = 'entry' AND bucket_id = ? + ` + ) + .all(bucketId) as { entity_id: string }[] + + return rows.map(row => row.entity_id) + } + + async clearDirtyState(bucketId: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + this.db.prepare('DELETE FROM dirty_state WHERE bucket_id = ?').run(bucketId) + } + + // ======================================== + // Utility Operations + // ======================================== + + async vacuum(): Promise { + if (!this.db) throw new Error('Database not initialized') + this.db.exec('VACUUM') + } + + async getStats(): Promise<{ + collections: number + entries: number + syncQueueSize: number + dirtyCount: number + dbSize: number + }> { + if (!this.db) throw new Error('Database not initialized') + + const collections = (this.db.prepare('SELECT COUNT(*) as count FROM collections').get() as any) + .count + const entries = (this.db.prepare('SELECT COUNT(*) as count FROM entries').get() as any).count + const syncQueueSize = ( + this.db + .prepare('SELECT COUNT(*) as count FROM data_sync_queue WHERE status = "pending"') + .get() as any + ).count + const dirtyCount = (this.db.prepare('SELECT COUNT(*) as count FROM dirty_state').get() as any) + .count + const dbSize = fs.statSync(this.dbPath).size + + return { collections, entries, syncQueueSize, dirtyCount, dbSize } + } + + close(): void { + if (this.db) { + this.db.close() + this.db = null + } + } +} +``` + +### Phase 2: Background Sync Service + +#### 2.1 Create Sync Service + +Create `src/main/services/syncService.ts`: + +```typescript +import { DatabaseService } from './databaseService' +import { S3Service } from './s3Service' +import { EventEmitter } from 'events' + +export interface SyncStatus { + isRunning: boolean + lastSyncAt?: string + queueSize: number + failedItems: number + successfulSyncs: number + errors: Array<{ entityId: string; error: string; timestamp: string }> +} + +export class SyncService extends EventEmitter { + private databaseService: DatabaseService + private s3Service: S3Service + private isRunning = false + private syncInterval: NodeJS.Timeout | null = null + private status: SyncStatus = { + isRunning: false, + queueSize: 0, + failedItems: 0, + successfulSyncs: 0, + errors: [] + } + + constructor(databaseService: DatabaseService, s3Service: S3Service) { + super() + this.databaseService = databaseService + this.s3Service = s3Service + } + + async start(): Promise { + if (this.isRunning) return + + this.isRunning = true + this.status.isRunning = true + + // Sync every 5 seconds + this.syncInterval = setInterval(() => { + this.processSyncQueue().catch(err => { + console.error('Sync loop error:', err) + }) + }, 5000) + + // Initial sync + await this.processSyncQueue() + + this.emit('statusChange', this.status) + } + + async stop(): Promise { + this.isRunning = false + this.status.isRunning = false + + if (this.syncInterval) { + clearInterval(this.syncInterval) + this.syncInterval = null + } + + this.emit('statusChange', this.status) + } + + async forceSyncNow(): Promise { + await this.processSyncQueue() + } + + getStatus(): SyncStatus { + return { ...this.status } + } + + private async processSyncQueue(): Promise { + try { + const queueItems = await this.databaseService.getSyncQueue(10) + this.status.queueSize = queueItems.length + + for (const item of queueItems) { + try { + await this.databaseService.markSyncProcessing(item.id) + await this.processSyncItem(item) + await this.databaseService.markSyncComplete(item.id) + + this.status.successfulSyncs++ + this.emit('itemSynced', { + entityType: item.entity_type, + entityId: item.entity_id, + operation: item.operation + }) + } catch (error: any) { + console.error('Sync error:', error) + await this.databaseService.markSyncError(item.id, error.message) + + this.status.failedItems++ + this.status.errors.push({ + entityId: item.entity_id, + error: error.message, + timestamp: new Date().toISOString() + }) + + // Keep only last 50 errors + if (this.status.errors.length > 50) { + this.status.errors.shift() + } + + this.emit('syncError', { + entityType: item.entity_type, + entityId: item.entity_id, + error: error.message + }) + } + } + + this.status.lastSyncAt = new Date().toISOString() + this.emit('statusChange', this.status) + } catch (error) { + console.error('Sync queue processing error:', error) + } + } + + private async processSyncItem(item: any): Promise { + const { operation, entity_type, entity_id, data, bucket_id } = item + + switch (entity_type) { + case 'schema': + await this.syncSchema(operation, entity_id, data, bucket_id) + break + case 'collection': + await this.syncCollection(operation, entity_id, data, bucket_id) + break + case 'entry': + await this.syncEntry(operation, entity_id, data, bucket_id) + break + case 'media': + await this.syncMedia(operation, entity_id, data, bucket_id) + break + default: + throw new Error(`Unknown entity type: ${entity_type}`) + } + } + + private async syncSchema( + operation: string, + entityId: string, + data: any, + bucketId: string + ): Promise { + const schema = await this.databaseService.getSchema(entityId) + if (!schema) throw new Error(`Schema ${entityId} not found`) + + switch (operation) { + case 'create': + case 'update': + // Upload schema to S3 + await this.s3Service.uploadSchema(bucketId, schema) + break + case 'delete': + await this.s3Service.deleteSchema(bucketId, entityId) + break + } + + // Update sync status in database + await this.databaseService.updateSchema(entityId, { + sync_status: 'synced', + last_synced_at: new Date().toISOString() + }) + } + + private async syncCollection( + operation: string, + entityId: string, + data: any, + bucketId: string + ): Promise { + const collection = await this.databaseService.getCollection(entityId) + if (!collection) throw new Error(`Collection ${entityId} not found`) + + switch (operation) { + case 'create': + case 'update': + // Upload collection data to S3 (NOT API endpoints) + await this.s3Service.uploadCollectionData(bucketId, collection) + break + case 'delete': + await this.s3Service.deleteCollectionData(bucketId, entityId) + break + } + + // Update sync status + await this.databaseService.updateCollection(entityId, { + sync_status: 'synced', + last_synced_at: new Date().toISOString() + }) + } + + private async syncEntry( + operation: string, + entityId: string, + data: any, + bucketId: string + ): Promise { + const entry = await this.databaseService.getEntry(entityId) + if (!entry) throw new Error(`Entry ${entityId} not found`) + + switch (operation) { + case 'create': + case 'update': + // Upload entry data to S3 (NOT API endpoints) + await this.s3Service.uploadEntryData(bucketId, entry) + break + case 'delete': + await this.s3Service.deleteEntryData(bucketId, entityId) + break + } + + // Update sync status + await this.databaseService.updateEntry(entityId, { + sync_status: 'synced', + last_synced_at: new Date().toISOString() + }) + } + + private async syncMedia( + operation: string, + entityId: string, + data: any, + bucketId: string + ): Promise { + // Media sync implementation + // Handle image uploads and caching + } +} +``` + +### Phase 3: Update IPC Layer + +#### 3.1 Update preload.ts + +Create `src/main/preload.ts` with new APIs: + +```typescript +import { contextBridge, ipcRenderer } from 'electron' + +// Collection API +const collectionAPI = { + create: (bucketId: string, data: any) => ipcRenderer.invoke('collection:create', bucketId, data), + update: (id: string, updates: any) => ipcRenderer.invoke('collection:update', id, updates), + get: (id: string) => ipcRenderer.invoke('collection:get', id), + list: (bucketId: string) => ipcRenderer.invoke('collection:list', bucketId), + delete: (id: string) => ipcRenderer.invoke('collection:delete', id), + + // Real-time updates + onUpdated: (callback: (collection: any) => void) => { + const listener = (_: any, collection: any) => callback(collection) + ipcRenderer.on('collection:updated', listener) + return () => ipcRenderer.removeListener('collection:updated', listener) + } +} + +// Entry API +const entryAPI = { + create: (bucketId: string, collectionId: string, data: any) => + ipcRenderer.invoke('entry:create', bucketId, collectionId, data), + update: (id: string, updates: any) => ipcRenderer.invoke('entry:update', id, updates), + get: (id: string) => ipcRenderer.invoke('entry:get', id), + list: (collectionId: string) => ipcRenderer.invoke('entry:list', collectionId), + delete: (id: string) => ipcRenderer.invoke('entry:delete', id), + + // Entry order + updateOrder: (collectionId: string, entryIds: string[]) => + ipcRenderer.invoke('entry:updateOrder', collectionId, entryIds), + + // Real-time updates + onUpdated: (callback: (entry: any) => void) => { + const listener = (_: any, entry: any) => callback(entry) + ipcRenderer.on('entry:updated', listener) + return () => ipcRenderer.removeListener('entry:updated', listener) + } +} + +// Schema API +const schemaAPI = { + list: (bucketId: string) => ipcRenderer.invoke('schema:list', bucketId), + get: (id: string) => ipcRenderer.invoke('schema:get', id) +} + +// Sync API +const syncAPI = { + getStatus: () => ipcRenderer.invoke('sync:getStatus'), + forceSyncNow: () => ipcRenderer.invoke('sync:forceSyncNow'), + + // Real-time sync status updates + onStatusChange: (callback: (status: any) => void) => { + const listener = (_: any, status: any) => callback(status) + ipcRenderer.on('sync:statusChange', listener) + return () => ipcRenderer.removeListener('sync:statusChange', listener) + }, + + onItemSynced: (callback: (item: any) => void) => { + const listener = (_: any, item: any) => callback(item) + ipcRenderer.on('sync:itemSynced', listener) + return () => ipcRenderer.removeListener('sync:itemSynced', listener) + }, + + onSyncError: (callback: (error: any) => void) => { + const listener = (_: any, error: any) => callback(error) + ipcRenderer.on('sync:syncError', listener) + return () => ipcRenderer.removeListener('sync:syncError', listener) + } +} + +// Publishing API (separate from sync) +const publishingAPI = { + publishChanges: (bucketId: string) => ipcRenderer.invoke('publishing:publishChanges', bucketId), + + getDirtyState: (bucketId: string) => ipcRenderer.invoke('publishing:getDirtyState', bucketId), + + onPublishProgress: (callback: (progress: any) => void) => { + const listener = (_: any, progress: any) => callback(progress) + ipcRenderer.on('publishing:progress', listener) + return () => ipcRenderer.removeListener('publishing:progress', listener) + } +} + +// Database API +const databaseAPI = { + getStats: () => ipcRenderer.invoke('db:getStats'), + vacuum: () => ipcRenderer.invoke('db:vacuum') +} + +// Expose to renderer +contextBridge.exposeInMainWorld('electronAPI', { + collection: collectionAPI, + entry: entryAPI, + schema: schemaAPI, + sync: syncAPI, + publishing: publishingAPI, + database: databaseAPI, + + // Keep existing APIs + s3: { + /* existing s3 APIs */ + }, + storage: { + /* existing storage APIs */ + } +}) +``` + +#### 3.2 Update main.ts with IPC handlers + +Add to `src/main/main.ts`: + +```typescript +import { app, BrowserWindow, ipcMain } from 'electron' +import { DatabaseService } from './services/databaseService' +import { SyncService } from './services/syncService' +import { S3Service } from './services/s3Service' + +let databaseService: DatabaseService +let syncService: SyncService +let s3Service: S3Service +let mainWindow: BrowserWindow + +app.whenReady().then(async () => { + // Initialize services + databaseService = new DatabaseService() + await databaseService.initialize() + + s3Service = new S3Service() + syncService = new SyncService(databaseService, s3Service) + await syncService.start() + + // Set up IPC handlers + setupIPCHandlers() + + // Set up sync event forwarding + setupSyncEventForwarding() + + // Create window + mainWindow = createWindow() +}) + +function setupIPCHandlers() { + // Collection handlers + ipcMain.handle('collection:create', async (_, bucketId, data) => { + const collection = await databaseService.createCollection({ ...data, bucket_id: bucketId }) + await databaseService.addToSyncQueue( + bucketId, + 'create', + 'collection', + collection.id, + collection + ) + await databaseService.markAsDirty('collection', collection.id, bucketId) + + // Notify renderer of update + mainWindow?.webContents.send('collection:updated', collection) + + return collection + }) + + ipcMain.handle('collection:update', async (_, id, updates) => { + const collection = await databaseService.updateCollection(id, updates) + await databaseService.addToSyncQueue(collection.bucket_id, 'update', 'collection', id, updates) + await databaseService.markAsDirty('collection', id, collection.bucket_id) + + mainWindow?.webContents.send('collection:updated', collection) + + return collection + }) + + ipcMain.handle('collection:get', async (_, id) => { + return await databaseService.getCollection(id) + }) + + ipcMain.handle('collection:list', async (_, bucketId) => { + return await databaseService.listCollections(bucketId) + }) + + ipcMain.handle('collection:delete', async (_, id) => { + const collection = await databaseService.getCollection(id) + if (!collection) throw new Error('Collection not found') + + await databaseService.deleteCollection(id) + await databaseService.addToSyncQueue(collection.bucket_id, 'delete', 'collection', id, null) + + return { success: true } + }) + + // Entry handlers + ipcMain.handle('entry:create', async (_, bucketId, collectionId, data) => { + const entry = await databaseService.createEntry({ + ...data, + bucket_id: bucketId, + collection_id: collectionId + }) + await databaseService.addToSyncQueue(bucketId, 'create', 'entry', entry.id, entry) + await databaseService.markAsDirty('entry', entry.id, bucketId) + + mainWindow?.webContents.send('entry:updated', entry) + + return entry + }) + + ipcMain.handle('entry:update', async (_, id, updates) => { + const entry = await databaseService.updateEntry(id, updates) + await databaseService.addToSyncQueue(entry.bucket_id, 'update', 'entry', id, updates) + await databaseService.markAsDirty('entry', id, entry.bucket_id) + + mainWindow?.webContents.send('entry:updated', entry) + + return entry + }) + + ipcMain.handle('entry:get', async (_, id) => { + return await databaseService.getEntry(id) + }) + + ipcMain.handle('entry:list', async (_, collectionId) => { + return await databaseService.listEntries(collectionId) + }) + + ipcMain.handle('entry:delete', async (_, id) => { + const entry = await databaseService.getEntry(id) + if (!entry) throw new Error('Entry not found') + + await databaseService.deleteEntry(id) + await databaseService.addToSyncQueue(entry.bucket_id, 'delete', 'entry', id, null) + + return { success: true } + }) + + ipcMain.handle('entry:updateOrder', async (_, collectionId, entryIds) => { + await databaseService.updateEntryOrder(collectionId, entryIds) + + const collection = await databaseService.getCollection(collectionId) + if (collection) { + await databaseService.markAsDirty('collection', collectionId, collection.bucket_id) + } + + return { success: true } + }) + + // Schema handlers + ipcMain.handle('schema:list', async (_, bucketId) => { + return await databaseService.listSchemas(bucketId) + }) + + ipcMain.handle('schema:get', async (_, id) => { + return await databaseService.getSchema(id) + }) + + // Sync handlers + ipcMain.handle('sync:getStatus', async () => { + return syncService.getStatus() + }) + + ipcMain.handle('sync:forceSyncNow', async () => { + await syncService.forceSyncNow() + return { success: true } + }) + + // Publishing handlers + ipcMain.handle('publishing:publishChanges', async (_, bucketId) => { + const dirtyCollections = await databaseService.getDirtyCollections(bucketId) + const dirtyEntries = await databaseService.getDirtyEntries(bucketId) + + // Publish API endpoints for dirty items + // This is separate from background sync + // Implementation depends on existing publishing logic + + await databaseService.clearDirtyState(bucketId) + + return { success: true, published: dirtyCollections.length + dirtyEntries.length } + }) + + ipcMain.handle('publishing:getDirtyState', async (_, bucketId) => { + const dirtyCollections = await databaseService.getDirtyCollections(bucketId) + const dirtyEntries = await databaseService.getDirtyEntries(bucketId) + + return { + collections: dirtyCollections, + entries: dirtyEntries, + total: dirtyCollections.length + dirtyEntries.length + } + }) + + // Database handlers + ipcMain.handle('db:getStats', async () => { + return await databaseService.getStats() + }) + + ipcMain.handle('db:vacuum', async () => { + await databaseService.vacuum() + return { success: true } + }) +} + +function setupSyncEventForwarding() { + // Forward sync events to renderer + syncService.on('statusChange', status => { + mainWindow?.webContents.send('sync:statusChange', status) + }) + + syncService.on('itemSynced', item => { + mainWindow?.webContents.send('sync:itemSynced', item) + }) + + syncService.on('syncError', error => { + mainWindow?.webContents.send('sync:syncError', error) + }) +} +``` + +### Phase 4: Update Frontend (React Query Integration) + +#### 4.1 Install React Query + +```bash +npm install @tanstack/react-query +npm install --save-dev @tanstack/react-query-devtools +``` + +#### 4.2 Create Query Client Setup + +Create `src/renderer/lib/queryClient.ts`: + +```typescript +import { QueryClient } from '@tanstack/react-query' + +export const queryClient = new QueryClient({ + defaultOptions: { + queries: { + staleTime: 1000 * 60 * 5, // 5 minutes + gcTime: 1000 * 60 * 10, // 10 minutes (formerly cacheTime) + refetchOnWindowFocus: false, + retry: 1 + }, + mutations: { + retry: 1 + } + } +}) +``` + +#### 4.3 Update main.tsx + +Update `src/renderer/main.tsx`: + +```typescript +import React from 'react' +import ReactDOM from 'react-dom/client' +import { QueryClientProvider } from '@tanstack/react-query' +import { ReactQueryDevtools } from '@tanstack/react-query-devtools' +import { queryClient } from './lib/queryClient' +import App from './App' +import './index.css' + +ReactDOM.createRoot(document.getElementById('root')!).render( + + + + + + +) +``` + +#### 4.4 Create Custom Hooks + +Create `src/renderer/hooks/useCollections.ts`: + +```typescript +import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' + +export function useCollections(bucketId: string) { + return useQuery({ + queryKey: ['collections', bucketId], + queryFn: () => window.electronAPI.collection.list(bucketId), + enabled: !!bucketId + }) +} + +export function useCollection(collectionId: string) { + return useQuery({ + queryKey: ['collection', collectionId], + queryFn: () => window.electronAPI.collection.get(collectionId), + enabled: !!collectionId + }) +} + +export function useCreateCollection(bucketId: string) { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: (data: any) => window.electronAPI.collection.create(bucketId, data), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['collections', bucketId] }) + }, + // Optimistic update + onMutate: async newCollection => { + await queryClient.cancelQueries({ queryKey: ['collections', bucketId] }) + const previousCollections = queryClient.getQueryData(['collections', bucketId]) + + queryClient.setQueryData(['collections', bucketId], (old: any[]) => [ + ...old, + { ...newCollection, id: `temp-${Date.now()}` } + ]) + + return { previousCollections } + }, + onError: (_, __, context) => { + // Rollback on error + if (context?.previousCollections) { + queryClient.setQueryData(['collections', bucketId], context.previousCollections) + } + } + }) +} + +export function useUpdateCollection() { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: ({ id, updates }: { id: string; updates: any }) => + window.electronAPI.collection.update(id, updates), + onSuccess: (data, variables) => { + queryClient.invalidateQueries({ queryKey: ['collection', variables.id] }) + queryClient.invalidateQueries({ queryKey: ['collections'] }) + } + }) +} + +export function useDeleteCollection(bucketId: string) { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: (id: string) => window.electronAPI.collection.delete(id), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['collections', bucketId] }) + } + }) +} +``` + +Create `src/renderer/hooks/useEntries.ts`: + +```typescript +import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' + +export function useEntries(collectionId: string) { + return useQuery({ + queryKey: ['entries', collectionId], + queryFn: () => window.electronAPI.entry.list(collectionId), + enabled: !!collectionId + }) +} + +export function useEntry(entryId: string) { + return useQuery({ + queryKey: ['entry', entryId], + queryFn: () => window.electronAPI.entry.get(entryId), + enabled: !!entryId + }) +} + +export function useCreateEntry(bucketId: string, collectionId: string) { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: (data: any) => window.electronAPI.entry.create(bucketId, collectionId, data), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['entries', collectionId] }) + } + }) +} + +export function useUpdateEntry() { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: ({ id, updates }: { id: string; updates: any }) => + window.electronAPI.entry.update(id, updates), + onSuccess: (data, variables) => { + queryClient.invalidateQueries({ queryKey: ['entry', variables.id] }) + queryClient.invalidateQueries({ queryKey: ['entries'] }) + } + }) +} + +export function useUpdateEntryOrder(collectionId: string) { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: (entryIds: string[]) => + window.electronAPI.entry.updateOrder(collectionId, entryIds), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['entries', collectionId] }) + } + }) +} +``` + +Create `src/renderer/hooks/useSyncStatus.ts`: + +```typescript +import { useQuery } from '@tanstack/react-query' +import { useEffect } from 'react' + +export function useSyncStatus() { + const query = useQuery({ + queryKey: ['sync-status'], + queryFn: () => window.electronAPI.sync.getStatus(), + refetchInterval: 5000 // Poll every 5 seconds + }) + + // Listen for real-time updates + useEffect(() => { + const unsubscribe = window.electronAPI.sync.onStatusChange(status => { + query.refetch() + }) + + return unsubscribe + }, []) + + return query +} +``` + +#### 4.5 Update ContentManager Component + +**IMPORTANT: Only data fetching logic changes. UI structure stays identical.** + +Update `src/renderer/components/ContentManager.tsx`: + +```typescript +import { useCollections, useCreateCollection, useUpdateCollection, useDeleteCollection } from '../hooks/useCollections' +import { useEntries, useCreateEntry, useUpdateEntry, useUpdateEntryOrder } from '../hooks/useEntries' + +export const ContentManager: FC = ({ schemas, bucket }) => { + // Replace existing data fetching with React Query hooks + const { data: collections = [], isLoading: collectionsLoading } = useCollections(bucket.id) + const createCollectionMutation = useCreateCollection(bucket.id) + const updateCollectionMutation = useUpdateCollection() + const deleteCollectionMutation = useDeleteCollection(bucket.id) + + // ... existing UI state (unchanged) + + const handleCreateCollection = async (collectionData: any) => { + await createCollectionMutation.mutateAsync(collectionData) + // UI updates automatically via React Query + } + + const handleUpdateCollection = async (collectionId: string, updates: any) => { + await updateCollectionMutation.mutateAsync({ id: collectionId, updates }) + // Immediate UI update (optimistic) + } + + // ... rest of component logic (unchanged) + + // Existing JSX structure remains EXACTLY the same + return ( +
+ {/* Existing UI components unchanged */} +
+ ) +} +``` + +### Phase 5: Data Migration + +#### 5.1 Create Migration Service + +Create `src/main/services/migrationService.ts`: + +```typescript +import { DatabaseService } from './databaseService' +import { S3Service } from './s3Service' +import { EventEmitter } from 'events' + +export interface MigrationProgress { + status: 'idle' | 'running' | 'completed' | 'error' + currentStep: string + totalItems: number + processedItems: number + errors: string[] +} + +export class MigrationService extends EventEmitter { + private databaseService: DatabaseService + private s3Service: S3Service + + constructor(databaseService: DatabaseService, s3Service: S3Service) { + super() + this.databaseService = databaseService + this.s3Service = s3Service + } + + async migrate(bucketId: string): Promise { + const progress: MigrationProgress = { + status: 'running', + currentStep: 'Starting migration', + totalItems: 0, + processedItems: 0, + errors: [] + } + + this.emit('progress', progress) + + try { + // Step 1: Migrate schemas + progress.currentStep = 'Migrating schemas' + this.emit('progress', progress) + const schemas = await this.s3Service.listSchemas(bucketId) + progress.totalItems += schemas.length + + for (const schema of schemas) { + try { + await this.databaseService.createSchema({ + ...schema, + bucket_id: bucketId + }) + progress.processedItems++ + this.emit('progress', progress) + } catch (error: any) { + progress.errors.push(`Schema ${schema.id}: ${error.message}`) + } + } + + // Step 2: Migrate collections + progress.currentStep = 'Migrating collections' + this.emit('progress', progress) + const collections = await this.s3Service.listCollections(bucketId) + progress.totalItems += collections.length + + for (const collection of collections) { + try { + await this.databaseService.createCollection({ + ...collection, + bucket_id: bucketId + }) + progress.processedItems++ + this.emit('progress', progress) + } catch (error: any) { + progress.errors.push(`Collection ${collection.id}: ${error.message}`) + } + } + + // Step 3: Migrate entries + progress.currentStep = 'Migrating entries' + this.emit('progress', progress) + + for (const collection of collections) { + const entries = await this.s3Service.listEntries(bucketId, collection.id) + progress.totalItems += entries.length + + for (const entry of entries) { + try { + await this.databaseService.createEntry({ + ...entry, + bucket_id: bucketId, + collection_id: collection.id + }) + progress.processedItems++ + this.emit('progress', progress) + } catch (error: any) { + progress.errors.push(`Entry ${entry.id}: ${error.message}`) + } + } + + // Migrate entry order + if (collection.entryOrder && collection.entryOrder.length > 0) { + await this.databaseService.updateEntryOrder(collection.id, collection.entryOrder) + } + } + + progress.status = 'completed' + progress.currentStep = 'Migration completed' + this.emit('progress', progress) + } catch (error: any) { + progress.status = 'error' + progress.errors.push(`Fatal error: ${error.message}`) + this.emit('progress', progress) + throw error + } + } + + async needsMigration(bucketId: string): Promise { + // Check if database has data for this bucket + const collections = await this.databaseService.listCollections(bucketId) + return collections.length === 0 + } +} +``` + +#### 5.2 Add Migration UI + +Create `src/renderer/components/MigrationScreen.tsx`: + +```typescript +import React, { useEffect, useState } from 'react' + +interface MigrationProgress { + status: 'idle' | 'running' | 'completed' | 'error' + currentStep: string + totalItems: number + processedItems: number + errors: string[] +} + +export const MigrationScreen: FC<{ bucketId: string; onComplete: () => void }> = ({ + bucketId, + onComplete +}) => { + const [progress, setProgress] = useState({ + status: 'idle', + currentStep: '', + totalItems: 0, + processedItems: 0, + errors: [] + }) + + useEffect(() => { + const startMigration = async () => { + await window.electronAPI.migration.start(bucketId) + } + + const unsubscribe = window.electronAPI.migration.onProgress((newProgress) => { + setProgress(newProgress) + + if (newProgress.status === 'completed') { + setTimeout(() => { + onComplete() + }, 2000) + } + }) + + startMigration() + + return unsubscribe + }, [bucketId]) + + const percentage = progress.totalItems > 0 + ? Math.round((progress.processedItems / progress.totalItems) * 100) + : 0 + + return ( +
+
+

Initial Data Sync

+

Syncing your content to local database for better performance...

+ +
+
+
+ +
+

{progress.currentStep}

+

{progress.processedItems} / {progress.totalItems} items

+
+ + {progress.errors.length > 0 && ( +
+

Errors ({progress.errors.length})

+
    + {progress.errors.slice(0, 5).map((error, i) => ( +
  • {error}
    • + ))} +
+
+ )} +
+
+ ) +} +``` + +### Phase 6: Conflict Resolution + +#### 6.1 Conflict Detection + +Add conflict detection to sync service: + +```typescript +// In syncService.ts +private async detectConflicts(entityType: string, entityId: string, localData: any): Promise { + // Fetch remote version from S3 + const remoteData = await this.s3Service.fetchEntity(entityType, entityId) + + if (!remoteData) return false // No conflict if remote doesn't exist + + // Compare timestamps + const localTime = new Date(localData.updated_at).getTime() + const remoteTime = new Date(remoteData.updated_at).getTime() + + if (remoteTime > localTime) { + // Remote is newer - conflict detected + await this.databaseService.recordConflict( + entityType, + entityId, + localData, + remoteData + ) + return true + } + + return false +} +``` + +#### 6.2 Conflict Resolution Strategy + +**Default Strategy: Last-Write-Wins with User Override** + +```typescript +export class ConflictResolver { + async resolveConflict( + conflictId: number, + resolution: 'keep_local' | 'keep_remote' | 'merge' + ): Promise { + const conflict = await this.databaseService.getConflict(conflictId) + + switch (resolution) { + case 'keep_local': + // Keep local version, force push to S3 + await this.s3Service.forceUpload(conflict.entity_type, conflict.local_version) + break + + case 'keep_remote': + // Discard local, use remote version + await this.databaseService.updateEntity( + conflict.entity_type, + conflict.entity_id, + conflict.remote_version + ) + break + + case 'merge': + // Custom merge logic based on entity type + const merged = await this.mergeEntities(conflict.local_version, conflict.remote_version) + await this.databaseService.updateEntity(conflict.entity_type, conflict.entity_id, merged) + break + } + + await this.databaseService.markConflictResolved(conflictId, resolution) + } +} +``` + +### Phase 7: Testing Strategy + +#### 7.1 Unit Tests + +```typescript +// src/main/services/databaseService.test.ts +describe('DatabaseService', () => { + let service: DatabaseService + + beforeEach(async () => { + service = new DatabaseService() + await service.initialize() + }) + + test('creates collection', async () => { + const collection = await service.createCollection({ + id: 'test-1', + bucket_id: 'bucket-1', + name: 'Test Collection', + schema_id: 'schema-1', + schema_name: 'TestSchema' + }) + + expect(collection).toMatchObject({ + id: 'test-1', + name: 'Test Collection' + }) + }) + + test('lists entries with correct order', async () => { + // Test entry ordering logic + }) + + // ... more tests +}) +``` + +#### 7.2 Integration Tests + +```typescript +// src/test/integration/localDatabase.test.ts +describe('Local Database Integration', () => { + test('full CRUD workflow', async () => { + // 1. Create collection + // 2. Add entries + // 3. Update entries + // 4. Verify sync queue + // 5. Process sync + // 6. Verify S3 has changes + }) + + test('offline to online sync', async () => { + // 1. Simulate offline mode + // 2. Make changes + // 3. Verify changes in local DB + // 4. Go online + // 5. Verify sync completes + }) +}) +``` + +#### 7.3 Performance Benchmarks + +```typescript +// src/test/performance/benchmarks.test.ts +describe('Performance Benchmarks', () => { + test('collection list performance', async () => { + const start = Date.now() + await databaseService.listCollections('bucket-1') + const duration = Date.now() - start + + expect(duration).toBeLessThan(50) // Should be < 50ms + }) + + test('entry update performance', async () => { + const start = Date.now() + await databaseService.updateEntry('entry-1', { data: { updated: true } }) + const duration = Date.now() - start + + expect(duration).toBeLessThan(50) + }) +}) +``` + +## Critical Considerations + +### 1. Two Separate Sync Processes + +**Data Sync (Background, Automatic)**: + +- Syncs collection/entry data to S3 for backup +- Runs continuously in background +- Does NOT update API endpoints +- Queue-based with retry logic + +**API Publishing (Manual, User-Triggered)**: + +- Only triggered by "Publish Changes" button +- Generates API endpoints for dirty collections/entries +- Updates `/api` directory structure +- Clears dirty state after successful publish + +### 2. Media Handling + +**Strategy for Images**: + +- Uploads happen immediately to S3 (can't be local-only) +- Cache URLs in media_cache table +- Generate responsive images on upload +- Store srcset information in metadata +- Clean up unused media periodically + +### 3. Schema Changes + +**Backward Compatibility**: + +- Schema changes might invalidate existing entries +- Need migration path for schema updates +- Consider versioning schemas + +### 4. Database Size Management + +**Strategies**: + +- Periodic VACUUM to reclaim space +- Archive old deleted items +- Implement "compact database" feature +- Show database size in settings + +### 5. Rollback Strategy + +**Emergency Rollback**: + +```typescript +// Environment variable to disable local DB +if (process.env.USE_DIRECT_S3 === 'true') { + // Use old direct S3 implementation +} else { + // Use new local database +} +``` + +### 6. Multi-Window Support + +**Consideration**: If users open multiple windows, they all need to stay in sync. + +**Solution**: Main process broadcasts changes to all renderer processes via IPC. + +## Success Metrics + +### Performance Targets + +- โœ… **Collection List**: < 50ms (currently 200-500ms) +- โœ… **Entry Load**: < 50ms (currently 200-500ms per entry) +- โœ… **Entry Update**: < 50ms (currently 200-500ms) +- โœ… **Entry Reorder**: < 50ms (currently 200-500ms) +- โœ… **Offline Functionality**: 100% features work offline +- โœ… **Sync Reliability**: 99.9% successful syncs +- โœ… **Data Consistency**: Zero data loss + +### User Experience Targets + +- โœ… **No Loading States**: Operations appear instant +- โœ… **Offline Editing**: Full functionality without internet +- โœ… **Background Sync**: Non-intrusive sync indicator +- โœ… **Conflict Resolution**: Clear UI for manual resolution +- โœ… **UI Preservation**: Zero visual/interaction changes + +## Implementation Phases Summary + +1. **Phase 0**: Pre-refactor prep (audits, tests, POC) +2. **Phase 1**: Database setup (SQLite schema, service) +3. **Phase 2**: Background sync service +4. **Phase 3**: IPC layer updates +5. **Phase 4**: Frontend React Query integration +6. **Phase 5**: Data migration from S3 to local +7. **Phase 6**: Conflict resolution +8. **Phase 7**: Testing and optimization + +## Estimated Timeline + +- Phase 0: 2-3 days +- Phase 1: 3-4 days +- Phase 2: 2-3 days +- Phase 3: 2 days +- Phase 4: 3-4 days +- Phase 5: 2 days +- Phase 6: 2-3 days +- Phase 7: 3-4 days + +**Total: ~3-4 weeks** for complete implementation and testing. + +## Conclusion + +This refactoring will transform the application from a slow, network-dependent tool into a fast, responsive, offline-capable content management system. By using SQLite as a single source of truth and React Query for optimistic updates, we achieve: + +1. **10-100x faster operations** +2. **Full offline functionality** +3. **Better user experience** (no loading states) +4. **Reliable background sync** +5. **Zero UI changes** (seamless for users) + +The architecture is simpler, more reliable, and more maintainable than the dual-database approach originally proposed. diff --git a/POC_TESTING_GUIDE.md b/POC_TESTING_GUIDE.md new file mode 100644 index 0000000..e2500fa --- /dev/null +++ b/POC_TESTING_GUIDE.md @@ -0,0 +1,301 @@ +# Local Database POC - Testing Guide + +## What We Built + +A proof-of-concept implementation of local-first database architecture for the S3 CMS application using: + +- **SQLite** (main process) - Primary data store for collections +- **React Query** (renderer process) - Data fetching, caching, and optimistic updates +- **Background Sync Service** - Simulates sync operations (logs only, no actual S3 sync yet) +- **IPC Communication** - Electron IPC handlers for database operations + +## Architecture + +``` +UI (React) โ†’ React Query โ†’ IPC โ†’ DatabaseService (SQLite) โ†’ SyncService โ†’ [S3 - simulated] + โ†“ โ†‘ + Optimistic Updates | + (<50ms response) Background Sync +``` + +## What's Implemented + +### Backend (Main Process) + +1. **DatabaseService** (`src/main/services/databaseService.ts`) + - SQLite database with collections table + - CRUD operations for collections + - Sync queue management + - Database statistics + +2. **SyncService** (`src/main/services/syncService.ts`) + - Background sync queue processor + - Event emitters for status updates + - Simulated sync operations (logs only) + +3. **IPC Handlers** (`src/main/main.ts`) + - `db:collection:create` + - `db:collection:update` + - `db:collection:get` + - `db:collection:list` + - `db:collection:delete` + - `db:sync:getStatus` + - `db:sync:forceSyncNow` + - `db:getStats` + +### Frontend (Renderer Process) + +1. **React Query Setup** (`src/renderer/lib/queryClient.ts`) + - QueryClient configuration + - Caching and stale time settings + +2. **Custom Hooks** (`src/renderer/hooks/useLocalCollections.ts`) + - `useLocalCollections` - Fetch all collections + - `useCreateLocalCollection` - Create with optimistic updates + - `useUpdateLocalCollection` - Update with cache invalidation + - `useDeleteLocalCollection` - Delete with optimistic updates + - `useLocalSyncStatus` - Monitor sync status + - `useLocalDatabaseStats` - Database statistics + +3. **Test Panel** (`src/renderer/components/DatabaseTestPanel.tsx`) + - Interactive UI to test database operations + - Real-time sync status display + - Database statistics + - Create/Update/Delete test collections + +## How to Test + +### 1. Start the Application + +```bash +npm run dev +``` + +The application should start and display the loading screen, then the main interface. + +### 2. Access the Database POC Test Panel + +1. Navigate to **Settings** (gear icon in the header or sidebar) +2. Click on the **"Database POC"** tab (new tab with test tube icon) +3. You should see the test panel interface + +### 3. Test Operations + +#### Create a Collection + +1. Enter a collection name in the input field +2. Click "Create" +3. **Expected**: Collection appears instantly in the list below (< 50ms) +4. **Watch**: Sync status shows the item being queued and "synced" + +#### Update a Collection + +1. Click "Update" button on any collection +2. **Expected**: Description updates instantly with current timestamp +3. **Watch**: Sync status increments successful syncs counter + +#### Delete a Collection + +1. Click "Delete" button on any collection +2. Confirm the deletion +3. **Expected**: Collection disappears immediately from the list + +#### Monitor Sync Status + +- **Status**: Shows sync service running/stopped +- **Queue Size**: Number of pending sync operations +- **Successful Syncs**: Counter of completed syncs +- **Last Sync**: Timestamp of last sync operation + +#### Check Database Stats + +- **Collections**: Total collections in database +- **Sync Queue**: Pending items +- **DB Size**: Database file size + +### 4. Performance Verification + +All operations should complete in **< 50ms**: + +1. Open browser DevTools (F12 or Cmd+Option+I) +2. Go to Console tab +3. Look for timing logs like: + ``` + [useLocalCollections] Fetching collections for bucket: xyz + [useLocalCollections] Fetched 5 collections + ``` +4. Operations should be nearly instant with no loading spinners + +### 5. Database Location + +The POC database is created at: + +``` +~/Library/Application Support/Electron/s3-cms-poc.db +``` + +You can inspect it with: + +```bash +sqlite3 ~/Library/Application\ Support/Electron/s3-cms-poc.db +``` + +## Expected Behavior + +### โœ… What Should Work + +- Creating collections instantly appears in UI +- Updating collections shows immediate changes +- Deleting collections removes them from UI immediately +- Sync service runs in background every 10 seconds +- Sync queue processes pending operations +- Database stats update in real-time +- No loading states for local operations +- Operations complete in < 50ms + +### โš ๏ธ What's Not Implemented (POC Limitations) + +- **No actual S3 sync** - SyncService only logs, doesn't upload to S3 +- **No entries support** - Only collections are implemented +- **No media caching** - Images still fetch from S3 +- **No conflict resolution** - Not needed for POC +- **No migration from existing S3 data** - Fresh database +- **No API endpoint publishing** - Separate from data sync + +## Performance Comparison + +### Before (Direct S3): + +- Collection list: **200-500ms** +- Collection create: **200-500ms** +- Collection update: **200-500ms** +- Collection delete: **200-500ms** +- **Total for 4 operations: ~1.2 seconds** + +### After (Local Database): + +- Collection list: **< 50ms** +- Collection create: **< 50ms** +- Collection update: **< 50ms** +- Collection delete: **< 50ms** +- **Total for 4 operations: < 200ms** + +**Improvement: 6-10x faster** ๐Ÿš€ + +## Next Steps (After POC Validation) + +If the POC performs well: + +1. **Expand Schema** + - Add entries table + - Add entry ordering table + - Add schemas table + - Add media cache table + +2. **Implement Real S3 Sync** + - Connect SyncService to actual S3Service + - Handle upload/download operations + - Implement retry logic + +3. **Add Migration** + - Import existing S3 data on first run + - Progress indicator for migration + +4. **Conflict Resolution** + - Detect conflicts between local and remote + - Resolution strategies (last-write-wins, manual) + +5. **Update ContentManager** + - Replace existing S3 calls with local database hooks + - Keep UI identical (backend-only change) + +6. **Testing** + - Unit tests for database operations + - Integration tests for sync + - Performance benchmarks + +## Troubleshooting + +### Database Not Initializing + +Check main process console for errors: + +``` +[DatabaseService] Initializing database... +[DatabaseService] Database initialized successfully +``` + +### IPC Handlers Not Found + +Verify preload script is loaded: + +``` +Electron API exposed to renderer +``` + +### React Query Not Working + +Check browser console for QueryClient initialization + +### Sync Service Not Running + +Look for: + +``` +[SyncService] Starting background sync... +[SyncService] Started successfully +``` + +## Files Changed + +### New Files + +- `src/main/database/schema.sql` +- `src/main/services/databaseService.ts` +- `src/main/services/syncService.ts` +- `src/renderer/lib/queryClient.ts` +- `src/renderer/hooks/useLocalCollections.ts` +- `src/renderer/components/DatabaseTestPanel.tsx` + +### Modified Files + +- `src/main/main.ts` - Added database service initialization and IPC handlers +- `src/main/preload.ts` - Added database IPC API +- `src/renderer/main.tsx` - Added QueryClientProvider +- `src/renderer/components/Settings.tsx` - Added Database POC tab + +## Success Criteria + +- โœ… Application builds without errors +- โœ… Database initializes on startup +- โœ… Sync service starts in background +- โœ… Test panel accessible in Settings +- โœ… Collections create/update/delete work instantly +- โœ… Operations complete in < 50ms +- โœ… Sync queue processes items +- โœ… Database stats display correctly +- โœ… No UI changes to existing components +- โœ… Background sync runs without blocking UI + +## Feedback + +After testing, please note: + +1. **Performance**: Does it feel instant? +2. **Reliability**: Any errors or crashes? +3. **UX**: Is the improvement noticeable? +4. **Issues**: Any unexpected behavior? + +## Cleanup + +To remove the POC database: + +```bash +rm ~/Library/Application\ Support/Electron/s3-cms-poc.db* +``` + +--- + +**POC Status**: โœ… Ready for Testing + +Built with: SQLite, React Query, Electron IPC, TypeScript diff --git a/README.md b/README.md index 1191f5f..1ede3bc 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ Bucketlist is a beautiful, desktop-native content management system that turns y - โœ… **Markdown Editor** - Rich text with live preview and formatting toolbar - โœ… **Media Library** - Centralized asset management with usage tracking - โœ… **Drag & Drop Reordering** - Intuitive content organization +- โœ… **Multi-Bucket Support** - Manage multiple S3 buckets with complete data isolation +- โœ… **Automatic Migration** - Seamless data import when switching between buckets ### Field Types @@ -61,6 +63,7 @@ Bucketlist is a beautiful, desktop-native content management system that turns y - ๐ŸŽฏ **Schema Collections** - Query entries by schema type - ๐Ÿ“– **API Documentation** - Interactive docs with code examples - โšก **CloudFront CDN** - Optional CDN support for faster delivery and lower costs +- ๐Ÿ”„ **Smart URL Transformation** - Automatic S3 to CloudFront URL conversion in published APIs ### Image Optimization @@ -183,7 +186,28 @@ const collections = await response.json() **Bonus**: Use the TypeScript types from "API Documentation" tab! -### 7๏ธโƒฃ (Optional) Set Up CloudFront CDN +### 7๏ธโƒฃ (Optional) Multi-Bucket Setup + +**Manage multiple projects or clients with separate S3 buckets:** + +1. **Add New Bucket**: + - Go to Settings โ†’ S3 Configuration + - Click "Add New Bucket" or "Switch Bucket" + - Enter new bucket details (name, region, credentials) + - Click "Save Configuration" + +2. **Automatic Migration**: + - When switching to a new bucket, migration starts automatically + - Modal shows "Importing Data" with progress + - All existing S3 data is imported to local database + - No manual setup required! + +3. **Data Isolation**: + - Each bucket maintains separate schemas and content + - Switch between buckets instantly + - Perfect for freelancers managing multiple clients + +### 8๏ธโƒฃ (Optional) Set Up CloudFront CDN **For production sites, use CloudFront for faster delivery and lower costs:** @@ -213,6 +237,22 @@ See the **Help** section in Bucketlist for detailed CloudFront setup instruction ## ๐ŸŽจ Advanced Features +### Multi-Bucket Management + +- ๐Ÿชฃ **Multiple S3 Buckets** - Manage different projects or clients in separate buckets +- ๐Ÿ”’ **Complete Data Isolation** - Each bucket maintains its own schemas, collections, and entries +- ๐Ÿ”„ **Automatic Migration** - When switching to a new bucket, data is automatically imported from S3 +- โšก **Seamless Switching** - No app restart required - switch between buckets instantly +- ๐Ÿ“Š **Bucket-Specific Analytics** - Track publishing dates and entry counts per bucket +- ๐Ÿ›ก๏ธ **Data Integrity** - Built-in safeguards prevent data mixing between buckets + +**Use Cases:** + +- **Freelancers**: Separate buckets for different clients +- **Agencies**: Different projects with isolated content +- **Multi-Site**: Separate content for different websites +- **Testing**: Production vs staging environments + ### Media Library - View all uploaded images and audio files @@ -231,10 +271,12 @@ See the **Help** section in Bucketlist for detailed CloudFront setup instruction - **Better performance** - Automatic caching at the edge - **Custom domains** - Use your own domain (e.g., cdn.yourdomain.com) - **HTTPS included** - Free SSL certificates -- **Smart architecture**: +- **Smart URL transformation**: - CMS stores S3 URLs internally (reliable, always works) - - Published API uses CloudFront URLs (fast delivery) - - Automatic URL transformation on publish + - Published API automatically converts all URLs to CloudFront + - Works across all API endpoints (collections, entries, schema lookups) + - Transforms `primaryImage` and all asset URLs in published content + - Zero manual configuration required - **Easy setup** - Step-by-step guide in Help section - **Zero breaking changes** - Completely optional, falls back to S3 @@ -262,6 +304,14 @@ See the **Help** section in Bucketlist for detailed CloudFront setup instruction - Type-safe content access - Copy/paste ready interfaces +### Enhanced User Experience + +- ๐Ÿ”„ **Real-time Updates** - Entry counts and last published dates update automatically +- ๐Ÿ“Š **Live Status Indicators** - See publishing status and entry counts in real-time +- ๐ŸŽฏ **Intuitive State Management** - No more manual refreshes needed +- โšก **Instant Feedback** - UI updates immediately after content changes +- ๐Ÿ” **Smart Data Isolation** - Each bucket maintains its own state and analytics + ### API Preview & Documentation - **Live API Browser** - Explore your published endpoints in-app @@ -723,15 +773,23 @@ Need help? We've got you covered: ## ๐ŸŽฏ Roadmap -Coming soon: +### โœ… Recently Completed + +- โœ… **Multi-Bucket Support** - Manage multiple S3 buckets with complete isolation +- โœ… **Automatic Migration** - Seamless data import when switching buckets +- โœ… **CloudFront URL Transformation** - Smart URL conversion in published APIs +- โœ… **Enhanced UX** - Real-time updates and improved state management +- โœ… **Data Integrity** - Built-in safeguards against data mixing + +### ๐Ÿš€ Coming Soon - [ ] Video field support - [ ] Bulk import/export - [ ] Content versioning - [ ] Collaborative editing - [ ] Webhooks for publish events -- [ ] CDN integration - [ ] Search and filtering +- [ ] Backup and restore functionality --- diff --git a/S3_AUDIT_REPORT.md b/S3_AUDIT_REPORT.md new file mode 100644 index 0000000..c2887d6 --- /dev/null +++ b/S3_AUDIT_REPORT.md @@ -0,0 +1,220 @@ +# S3 Audit Report - Publishing Workflow Refactoring + +**Date:** October 19, 2025 +**Branch:** refactor-to-local-db +**Status:** โœ… Publishing workflow successfully migrated to local-first architecture + +## Summary + +The publishing workflow has been successfully refactored to read from the local SQLite database instead of S3. This significantly improves performance and reliability, as publishing now works with locally cached data. + +## Changes Made + +### 1. Created `PublishingService` + +**File:** `src/main/services/publishingService.ts` + +A new service that: + +- Reads collection and entry data from the local SQLite database via `DatabaseService` +- Generates API endpoint JSON files +- Uploads them to S3 (write-only) + +**Key Methods:** + +- `generateAllApiEndpoints()` - Full publish of all collections +- `generateSelectiveApiEndpoints()` - Selective publish of specific collections/entries +- `generateApiEndpoints()` - Generate API for a single collection +- `updateCollectionsIndex()` - Generate `/api/collections/index.json` +- `generateSchemaLookup()` - Generate schema lookup files +- `getLastGenerationDate()` - Get last publish timestamp + +### 2. Updated `main.ts` IPC Handlers + +**Previously:** IPC handlers called `contentService` methods which read from S3 +**Now:** IPC handlers call `publishingService` methods which read from local DB + +**Updated handlers:** + +- `content:generate-all-api-endpoints` โ†’ uses `publishingService.generateAllApiEndpoints()` +- `content:generate-selective-api-endpoints` โ†’ uses `publishingService.generateSelectiveApiEndpoints()` +- `content:get-last-generation-date` โ†’ uses `publishingService.getLastGenerationDate()` + +## S3 Operations Status + +### โœ… Publishing Operations (Now Local-First) + +| Operation | Previous | Current | Status | +| ----------------------------------- | -------- | ------------- | -------------------------- | +| List collections for API generation | S3 Read | Local DB Read | โœ… Migrated | +| List entries for API generation | S3 Read | Local DB Read | โœ… Migrated | +| Generate API endpoints | S3 Write | S3 Write | โœ… Unchanged | +| Upload API JSON | S3 Write | S3 Write | โœ… Unchanged | +| Get last generation date | S3 Read | S3 Read | โœ… Unchanged (cached data) | + +### Legacy IPC Handlers (Not Used) + +The following `content:*` IPC handlers still exist in `main.ts` but are **NOT BEING USED** by the renderer: + +**File:** `src/main/main.ts` (lines 476-591) + +- `content:create-entry` โ†’ โŒ Not used (replaced by `db:entry:create`) +- `content:update-entry` โ†’ โŒ Not used (replaced by `db:entry:update`) +- `content:get-entry` โ†’ โŒ Not used (replaced by `db:entry:get`) +- `content:list-entries` โ†’ โŒ Not used (replaced by `db:entry:list`) +- `content:delete-entry` โ†’ โŒ Not used (replaced by `db:entry:delete`) +- `content:create-collection` โ†’ โŒ Not used (replaced by `db:collection:create`) +- `content:update-collection` โ†’ โŒ Not used (replaced by `db:collection:update`) +- `content:get-collection` โ†’ โŒ Not used (replaced by `db:collection:get`) +- `content:list-collections` โ†’ โŒ Not used (replaced by `db:collection:list`) +- `content:delete-collection` โ†’ โŒ Not used (replaced by `db:collection:delete`) +- `content:update-entry-order` โ†’ โŒ Not used (updated in DB directly) +- `content:update-collection-order` โ†’ โŒ Not used (updated in DB directly) + +**Evidence:** + +- Searched for `useContentAPI` imports โ†’ **0 results** +- The `useContentAPI.ts` hook is defined but never imported/used +- `ContentManager.tsx` uses `db:*` handlers exclusively (except for API generation which now uses `publishingService`) + +### ๐Ÿ”„ Still Using S3 (Intentionally) + +These operations appropriately continue to use S3: + +**Media Operations:** + +- `content:upload-image` โ†’ S3 Write (media storage) +- `content:upload-audio` โ†’ S3 Write (media storage) +- Image processing and optimization โ†’ S3 Read/Write + +**Schema Operations:** + +- Schema listing/reading โ†’ S3 Read (schemas stored in S3) +- TypeScript type generation โ†’ S3 Read + Write + +**Settings:** + +- Bucket configuration โ†’ Local storage (not S3) + +## Architecture Overview + +``` +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ RENDERER PROCESS โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ ContentMgr โ”‚โ”€โ”€โ”€โ”€โ–ถโ”‚ DirtyStateContext โ”‚ โ”‚ +โ”‚ โ”‚ Components โ”‚ โ”‚ - Track unpublished โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ - Trigger publish โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ publishChanges() โ”‚ +โ”‚ โ–ผ โ”‚ +โ”‚ IPC: content:generate-*-endpoints โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”‚โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ + โ”‚ +โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”‚โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” +โ”‚ MAIN PROCESS โ”‚ โ”‚ +โ”‚ โ–ผ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ PublishingService โ”‚ โ”‚ +โ”‚ โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ 1. Read collections/entries from DB โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ (via DatabaseService) โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ 2. Generate API JSON files โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ - /api/collections/index.json โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ - /api/collections/{slug}.json โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ - /api/collections/{slug}/entries.jsonโ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ - /api/entries/{slug}.json โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ 3. Upload to S3 โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ โ”‚ (S3 Write only) โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ”‚ +โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ +โ”‚ โ”‚ Database โ”‚ โ”‚ S3 โ”‚ โ”‚ +โ”‚ โ”‚ Service โ”‚ โ”‚ (Write Only) โ”‚ โ”‚ +โ”‚ โ”‚ (Local DB) โ”‚ โ”‚ โ”‚ โ”‚ +โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ +โ”‚ โ–ฒ โ–ฒ โ”‚ +โ”‚ โ”‚ โ”‚ โ”‚ +โ”‚ Collections API JSON โ”‚ +โ”‚ Entries Files โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +## Performance Improvements + +**Before:** + +1. User clicks "Publish Changes" +2. IPC call to main process +3. Main process reads collections from S3 โ†’ **SLOW** (network call) +4. For each collection, reads entries from S3 โ†’ **VERY SLOW** (multiple network calls) +5. Generates API JSON files +6. Uploads to S3 + +**After:** + +1. User clicks "Publish Changes" +2. IPC call to main process +3. Main process reads collections from local SQLite โ†’ **FAST** (disk read) +4. For each collection, entries are already loaded โ†’ **INSTANT** (in-memory) +5. Generates API JSON files +6. Uploads to S3 + +**Expected speedup:** 5-10x faster for typical collections with 10-50 entries + +## Remaining Work + +### Low Priority Cleanup + +1. **Remove legacy `content:*` IPC handlers** (lines 476-591 in `main.ts`) + - These are no longer used but still exist in the codebase + - Safe to remove once we confirm no other code paths use them + +2. **Remove `useContentAPI.ts` hook** + - This hook is defined but never imported/used + - Can be safely deleted + +3. **Update ContentService methods (optional)** + - Keep `contentService` for S3 write operations (upload image/audio) + - Consider renaming to `S3UploadService` for clarity + +### Future Enhancements + +1. **Background sync improvements** + - Currently `SyncService` logs but doesn't actually sync to S3 + - Implement actual S3 sync for collection/entry CRUD operations + +2. **Conflict resolution** + - Handle cases where local DB and S3 diverge + - Implement merge strategies + +## Testing Checklist + +- [ ] Create new entry โ†’ Publish โ†’ Verify API JSON in S3 +- [ ] Edit entry metadata โ†’ Publish โ†’ Verify API JSON updated +- [ ] Reorder entries โ†’ Publish โ†’ Verify order in API JSON +- [ ] Publish/Unpublish collections โ†’ Verify index.json +- [ ] Check CloudFront URL transformation +- [ ] Verify dirty state cleared after publish +- [ ] Test selective publishing (single collection) + +## Conclusion + +โœ… **Publishing workflow successfully migrated to local-first architecture** + +The publishing workflow now: + +- Reads from local SQLite database (fast) +- Writes to S3 (unchanged) +- No longer performs slow S3 read operations during publishing +- Maintains data consistency through `DatabaseService` + +Next steps: + +1. Test the publish workflow end-to-end +2. Clean up legacy code (optional) +3. Continue auditing other S3 operations (media, schemas, etc.) diff --git a/SWAGGER_API_DOCUMENTATION_PROMPT.md b/SWAGGER_API_DOCUMENTATION_PROMPT.md new file mode 100644 index 0000000..d3bfb37 --- /dev/null +++ b/SWAGGER_API_DOCUMENTATION_PROMPT.md @@ -0,0 +1,340 @@ +# Swagger/OpenAPI Documentation Implementation Prompt + +## Overview + +Implement comprehensive API documentation using Swagger/OpenAPI for Bucketlist CMS custom schemas. This will provide professional, auto-generated documentation that enhances the business value and client experience. + +## Business Value + +### For Freelance Package: + +- **Professional Documentation**: Clients get auto-generated API docs +- **Developer Experience**: Easy integration for frontend developers +- **Reduced Support**: Self-documenting APIs reduce client questions +- **Competitive Advantage**: Most CMS solutions don't offer this level of documentation + +### For SaaS Potential: + +- **Enterprise Feature**: Professional API documentation is expected in enterprise tools +- **Developer Adoption**: Good docs = easier adoption = more customers +- **Integration Ecosystem**: Third-party developers can build integrations + +## Technical Requirements + +### Core Features: + +1. **Auto-generated OpenAPI specs** from custom schemas +2. **Interactive Swagger UI** for testing endpoints +3. **Schema documentation** with field descriptions and validation rules +4. **Code examples** in multiple languages +5. **Relationship documentation** for complex data models + +### Advanced Features: + +1. **SDK generation** for popular languages +2. **Type definitions** (TypeScript interfaces) +3. **Custom branding** for white-label solutions +4. **Authentication documentation** (API keys, etc.) + +## Implementation Plan + +### Phase 1: Basic OpenAPI Generation (2-3 weeks) + +- Schema to OpenAPI converter +- Basic endpoint documentation +- Simple documentation UI + +### Phase 2: Interactive Features (2-3 weeks) + +- Swagger UI integration +- Code examples generation +- Enhanced field documentation + +### Phase 3: Advanced Features (2-3 weeks) + +- SDK generation +- Custom branding +- Advanced documentation features + +## Technical Architecture + +### 1. Schema to OpenAPI Converter + +```typescript +interface SchemaToOpenAPIConverter { + convertSchema(schema: Schema): OpenAPISchema + convertField(field: Field): OpenAPIField + generateEndpointDocs(collection: ContentCollection): EndpointDocumentation + generateValidationDocs(schema: Schema): ValidationDocumentation +} +``` + +### 2. API Endpoint Documentation + +```typescript +interface APIEndpointDocumenter { + generateCollectionEndpoints(collection: ContentCollection): EndpointDocumentation[] + generateSchemaEndpoints(schema: Schema): EndpointDocumentation[] + generateRelationshipEndpoints(relationships: Relationship[]): EndpointDocumentation[] +} +``` + +### 3. Interactive Documentation UI + +```typescript +interface DocumentationUI { + renderSwaggerUI(spec: OpenAPISpec): React.Component + renderSchemaDocs(schema: Schema): React.Component + renderCodeExamples(schema: Schema): React.Component + renderRelationshipDocs(relationships: Relationship[]): React.Component +} +``` + +## File Structure + +### New Files to Create: + +``` +src/ +โ”œโ”€โ”€ main/ +โ”‚ โ””โ”€โ”€ services/ +โ”‚ โ”œโ”€โ”€ openAPIService.ts # OpenAPI spec generation +โ”‚ โ”œโ”€โ”€ schemaDocumentationService.ts # Schema documentation +โ”‚ โ””โ”€โ”€ codeGenerationService.ts # SDK/code generation +โ”œโ”€โ”€ renderer/ +โ”‚ โ”œโ”€โ”€ components/ +โ”‚ โ”‚ โ”œโ”€โ”€ ApiDocumentation.tsx # Main documentation component +โ”‚ โ”‚ โ”œโ”€โ”€ SwaggerUI.tsx # Swagger UI wrapper +โ”‚ โ”‚ โ”œโ”€โ”€ SchemaDocs.tsx # Schema documentation +โ”‚ โ”‚ โ””โ”€โ”€ CodeExamples.tsx # Code examples +โ”‚ โ””โ”€โ”€ services/ +โ”‚ โ””โ”€โ”€ documentationService.ts # Documentation utilities +โ””โ”€โ”€ types/ + โ””โ”€โ”€ openAPI.ts # OpenAPI type definitions +``` + +## Implementation Details + +### 1. OpenAPI Spec Generation + +```typescript +class OpenAPIService { + generateSpec(bucket: string): OpenAPISpec { + return { + openapi: '3.0.0', + info: { + title: 'Bucketlist CMS API', + version: '1.0.0', + description: 'Auto-generated API documentation for custom schemas' + }, + paths: this.generatePaths(bucket), + components: { + schemas: this.generateSchemas(bucket), + securitySchemes: this.generateSecuritySchemes() + } + } + } + + private generatePaths(bucket: string): Record { + // Generate API paths from collections and schemas + } + + private generateSchemas(bucket: string): Record { + // Convert schemas to OpenAPI schema definitions + } +} +``` + +### 2. Schema Documentation + +```typescript +class SchemaDocumentationService { + generateSchemaDocs(schema: Schema): SchemaDocumentation { + return { + name: schema.name, + description: schema.description, + fields: this.generateFieldDocs(schema.fields), + relationships: this.generateRelationshipDocs(schema.relationships), + examples: this.generateExamples(schema), + validation: this.generateValidationDocs(schema) + } + } + + private generateFieldDocs(fields: Record): FieldDocumentation[] { + return Object.entries(fields).map(([name, field]) => ({ + name, + type: field.type, + description: field.description, + required: field.required, + validation: field.validation, + examples: field.examples + })) + } +} +``` + +### 3. Code Generation + +```typescript +class CodeGenerationService { + generateTypeScriptInterfaces(schema: Schema): string { + // Generate TypeScript interfaces from schema + } + + generateJavaScriptSDK(spec: OpenAPISpec): string { + // Generate JavaScript SDK from OpenAPI spec + } + + generateReactHooks(schema: Schema): string { + // Generate React hooks for API calls + } +} +``` + +## UI Components + +### 1. Main Documentation Component + +```typescript +const ApiDocumentation: React.FC = () => { + const [activeTab, setActiveTab] = useState<'overview' | 'schemas' | 'endpoints' | 'examples'>('overview') + + return ( +
+ + + +
+ ) +} +``` + +### 2. Swagger UI Integration + +```typescript +const SwaggerUI: React.FC<{ spec: OpenAPISpec }> = ({ spec }) => { + return ( +
+ +
+ ) +} +``` + +### 3. Schema Documentation + +```typescript +const SchemaDocs: React.FC<{ schema: Schema }> = ({ schema }) => { + return ( +
+ + + + + +
+ ) +} +``` + +## Integration Points + +### 1. Schema Builder Integration + +- Add "Generate Documentation" button to schema builder +- Auto-generate docs when schema is saved +- Show documentation preview in schema builder + +### 2. API Preview Integration + +- Add "View Documentation" link to API preview +- Show documentation for specific collections +- Link to interactive API explorer + +### 3. Publishing Integration + +- Generate documentation when publishing API routes +- Update documentation when schemas change +- Version documentation with API versions + +## Dependencies + +### Required Packages: + +```json +{ + "swagger-ui-react": "^5.0.0", + "swagger-ui-dist": "^5.0.0", + "@apidevtools/swagger-parser": "^10.0.0", + "openapi-typescript": "^6.0.0", + "openapi-generator-cli": "^2.0.0" +} +``` + +## Success Metrics + +### Technical Metrics: + +- OpenAPI spec generation time < 1 second +- Documentation UI loads in < 2 seconds +- Code generation completes in < 5 seconds + +### Business Metrics: + +- Reduced client support questions about API usage +- Increased developer adoption of API +- Positive client feedback on documentation quality + +## Future Enhancements + +### Phase 4: Advanced Features + +- **Custom branding** for white-label solutions +- **Multi-language support** for documentation +- **API versioning** with documentation versioning +- **Integration marketplace** for third-party tools + +### Phase 5: Enterprise Features + +- **Role-based documentation** access +- **Custom documentation themes** +- **Advanced analytics** on API usage +- **Webhook documentation** and testing + +## Implementation Notes + +### Prerequisites: + +- Local database refactoring must be completed +- Schema system must be stable +- API endpoint generation must be working + +### Considerations: + +- Documentation should be generated on-demand, not stored +- Consider caching generated documentation for performance +- Ensure documentation stays in sync with schema changes +- Plan for documentation versioning as schemas evolve + +## Business Impact + +### Competitive Advantage: + +- **Professional credibility** with auto-generated docs +- **Developer experience** that rivals enterprise tools +- **Reduced support burden** through self-documenting APIs +- **Enterprise appeal** with professional documentation + +### Revenue Potential: + +- **Freelance package value** increases significantly +- **SaaS feature differentiation** from competitors +- **Enterprise client appeal** with professional documentation +- **Developer ecosystem** potential for integrations + +This implementation will transform Bucketlist from a simple CMS into a professional, enterprise-ready content management platform with world-class API documentation. diff --git a/package-lock.json b/package-lock.json index 5b40090..fddea7b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,8 @@ "@dnd-kit/core": "^6.3.1", "@dnd-kit/sortable": "^10.0.0", "@dnd-kit/utilities": "^3.2.2", + "@tanstack/react-query": "^5.90.5", + "better-sqlite3": "^11.10.0", "clsx": "^2.0.0", "lucide-react": "^0.294.0", "react": "^18.2.0", @@ -29,6 +31,7 @@ "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.0", "@testing-library/user-event": "^14.6.1", + "@types/better-sqlite3": "^7.6.13", "@types/node": "^20.10.0", "@types/react": "^18.2.43", "@types/react-dom": "^18.2.17", @@ -4654,6 +4657,32 @@ "node": ">=10" } }, + "node_modules/@tanstack/query-core": { + "version": "5.90.5", + "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.90.5.tgz", + "integrity": "sha512-wLamYp7FaDq6ZnNehypKI5fNvxHPfTYylE0m/ZpuuzJfJqhR5Pxg9gvGBHZx4n7J+V5Rg5mZxHHTlv25Zt5u+w==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, + "node_modules/@tanstack/react-query": { + "version": "5.90.5", + "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.90.5.tgz", + "integrity": "sha512-pN+8UWpxZkEJ/Rnnj2v2Sxpx1WFlaa9L6a4UO89p6tTQbeo+m0MS8oYDjbggrR8QcTyjKoYWKS3xJQGr3ExT8Q==", + "license": "MIT", + "dependencies": { + "@tanstack/query-core": "5.90.5" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": "^18 || ^19" + } + }, "node_modules/@testing-library/dom": { "version": "10.4.1", "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.1.tgz", @@ -4850,6 +4879,16 @@ "@babel/types": "^7.28.2" } }, + "node_modules/@types/better-sqlite3": { + "version": "7.6.13", + "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz", + "integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/cacheable-request": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.3.tgz", @@ -6295,7 +6334,6 @@ "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, "funding": [ { "type": "github", @@ -6322,6 +6360,17 @@ "baseline-browser-mapping": "dist/cli.js" } }, + "node_modules/better-sqlite3": { + "version": "11.10.0", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.10.0.tgz", + "integrity": "sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + } + }, "node_modules/bidi-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz", @@ -6345,11 +6394,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dev": true, "license": "MIT", "dependencies": { "buffer": "^5.5.0", @@ -6450,7 +6507,6 @@ "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "dev": true, "funding": [ { "type": "github", @@ -7512,7 +7568,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "dev": true, "license": "MIT", "dependencies": { "mimic-response": "^3.1.0" @@ -7528,7 +7583,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=10" @@ -7547,6 +7601,15 @@ "node": ">=6" } }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -8129,7 +8192,6 @@ "version": "1.4.5", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", - "dev": true, "license": "MIT", "dependencies": { "once": "^1.4.0" @@ -8792,6 +8854,15 @@ "dev": true, "license": "MIT" }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, "node_modules/expect": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", @@ -8992,6 +9063,12 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, "node_modules/filelist": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/filelist/-/filelist-1.0.4.tgz", @@ -9173,9 +9250,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/fs-extra": { "version": "8.1.0", @@ -9404,6 +9479,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", @@ -9871,7 +9952,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true, "funding": [ { "type": "github", @@ -9958,7 +10038,12 @@ "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true, + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", "license": "ISC" }, "node_modules/internal-slot": { @@ -11822,7 +11907,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -12055,6 +12139,12 @@ "node": ">=10" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, "node_modules/mrmime": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz", @@ -12116,6 +12206,12 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -12151,7 +12247,6 @@ "version": "3.78.0", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.78.0.tgz", "integrity": "sha512-E2wEyrgX/CqvicaQYU3Ze1PFGjc4QYPGsjUrlYkqAE0WjHEZwgOsGMPMzkMse4LjJbDmaEuDX3CM036j5K2DSQ==", - "dev": true, "license": "MIT", "dependencies": { "semver": "^7.3.5" @@ -12356,7 +12451,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -12876,6 +12970,32 @@ "dev": true, "license": "MIT" }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -13022,7 +13142,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", - "dev": true, "license": "MIT", "dependencies": { "end-of-stream": "^1.1.0", @@ -13073,6 +13192,30 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/rc/node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/react": { "version": "18.3.1", "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", @@ -13177,7 +13320,6 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dev": true, "license": "MIT", "dependencies": { "inherits": "^2.0.3", @@ -13663,7 +13805,6 @@ "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true, "funding": [ { "type": "github", @@ -14042,6 +14183,51 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/simple-update-notifier": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", @@ -14311,7 +14497,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.2.0" @@ -14742,13 +14927,29 @@ "node": ">=10" } }, + "node_modules/tar-fs": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-fs/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, "node_modules/tar-stream": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "dev": true, "license": "MIT", - "peer": true, "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -15104,6 +15305,18 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -15346,7 +15559,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true, "license": "MIT" }, "node_modules/uuid": { @@ -15804,7 +16016,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, "license": "ISC" }, "node_modules/ws": { diff --git a/package.json b/package.json index 7dcdd39..1840e63 100644 --- a/package.json +++ b/package.json @@ -8,9 +8,10 @@ "dev": "concurrently \"npm run dev:renderer\" \"wait-on http://localhost:5173 && npm run dev:main\"", "dev:renderer": "vite", "dev:main": "tsc -p tsconfig.main.json && NODE_ENV=development electron dist/main/main.js", - "build": "npm run build:renderer && npm run build:main", + "build": "npm run build:renderer && npm run build:main && npm run build:copy-database", "build:renderer": "vite build", "build:main": "tsc -p tsconfig.main.json", + "build:copy-database": "node -e \"require('fs').cpSync('src/main/database', 'dist/main/database', {recursive: true})\"", "build:clean": "rimraf dist release", "preview": "vite preview", "electron": "electron .", @@ -51,6 +52,7 @@ "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.0", "@testing-library/user-event": "^14.6.1", + "@types/better-sqlite3": "^7.6.13", "@types/node": "^20.10.0", "@types/react": "^18.2.43", "@types/react-dom": "^18.2.17", @@ -91,6 +93,8 @@ "@dnd-kit/core": "^6.3.1", "@dnd-kit/sortable": "^10.0.0", "@dnd-kit/utilities": "^3.2.2", + "@tanstack/react-query": "^5.90.5", + "better-sqlite3": "^11.10.0", "clsx": "^2.0.0", "lucide-react": "^0.294.0", "react": "^18.2.0", @@ -113,6 +117,7 @@ "afterPack": "./scripts/afterPack.js", "files": [ "dist/**/*", + "src/main/database/**/*", "loading.html", "node_modules/**/*", "!node_modules/.cache", diff --git a/src/main/database/schema.sql b/src/main/database/schema.sql new file mode 100644 index 0000000..504d094 --- /dev/null +++ b/src/main/database/schema.sql @@ -0,0 +1,309 @@ +-- Local Database Schema - Full Migration +-- Stores collections, entries, and entry ordering + +-- Collections table +CREATE TABLE IF NOT EXISTS collections ( + id TEXT PRIMARY KEY, + bucket_id TEXT NOT NULL, + name TEXT NOT NULL, + description TEXT, + schema_id TEXT NOT NULL, + schema_name TEXT NOT NULL, + published BOOLEAN DEFAULT 0, + published_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + metadata TEXT, -- JSON object + sync_status TEXT DEFAULT 'pending', -- pending, syncing, synced, error + sync_error TEXT, + last_synced_at TEXT +); + +-- Entries table +CREATE TABLE IF NOT EXISTS entries ( + id TEXT PRIMARY KEY, + bucket_id TEXT NOT NULL, + collection_id TEXT NOT NULL, + schema_id TEXT NOT NULL, + schema_name TEXT NOT NULL, + data TEXT NOT NULL, -- JSON content + published BOOLEAN DEFAULT 0, + published_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + sync_status TEXT DEFAULT 'pending', -- pending, syncing, synced, error + sync_error TEXT, + last_synced_at TEXT, + FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE +); + +-- Entry ordering table (normalized for efficient reordering) +CREATE TABLE IF NOT EXISTS collection_entry_order ( + collection_id TEXT NOT NULL, + entry_id TEXT NOT NULL, + position INTEGER NOT NULL, + PRIMARY KEY (collection_id, entry_id), + FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE, + FOREIGN KEY (entry_id) REFERENCES entries(id) ON DELETE CASCADE +); + +-- Collection ordering table (per bucket) +CREATE TABLE IF NOT EXISTS bucket_collection_order ( + bucket_id TEXT NOT NULL, + collection_id TEXT NOT NULL, + position INTEGER NOT NULL, + PRIMARY KEY (bucket_id, collection_id), + FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE +); + +-- Sync queue for background operations +CREATE TABLE IF NOT EXISTS data_sync_queue ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + bucket_id TEXT NOT NULL, + operation TEXT NOT NULL, -- create, update, delete + entity_type TEXT NOT NULL, -- collection, entry (future) + entity_id TEXT NOT NULL, + data TEXT, -- JSON data for the operation + retry_count INTEGER DEFAULT 0, + max_retries INTEGER DEFAULT 3, + created_at TEXT NOT NULL, + started_at TEXT, + completed_at TEXT, + status TEXT DEFAULT 'pending' -- pending, processing, completed, failed +); + +-- Indexes for performance +CREATE INDEX IF NOT EXISTS idx_collections_bucket_id ON collections(bucket_id); +CREATE INDEX IF NOT EXISTS idx_collections_published ON collections(bucket_id, published); +CREATE INDEX IF NOT EXISTS idx_collections_updated_at ON collections(updated_at DESC); + +CREATE INDEX IF NOT EXISTS idx_entries_bucket_id ON entries(bucket_id); +CREATE INDEX IF NOT EXISTS idx_entries_collection_id ON entries(collection_id); +CREATE INDEX IF NOT EXISTS idx_entries_published ON entries(bucket_id, published); +CREATE INDEX IF NOT EXISTS idx_entries_updated_at ON entries(updated_at DESC); + +CREATE INDEX IF NOT EXISTS idx_entry_order_collection ON collection_entry_order(collection_id, position); +CREATE INDEX IF NOT EXISTS idx_collection_order_bucket ON bucket_collection_order(bucket_id, position); + +CREATE INDEX IF NOT EXISTS idx_sync_queue_status ON data_sync_queue(status, created_at); +CREATE INDEX IF NOT EXISTS idx_sync_queue_bucket ON data_sync_queue(bucket_id, status); + +-- Assets/Media tracking table +CREATE TABLE IF NOT EXISTS assets ( + id TEXT PRIMARY KEY, + bucket_id TEXT NOT NULL, -- References bucket from settings, not enforced via FK + type TEXT NOT NULL, -- 'image', 'audio', 'embed' + key TEXT NOT NULL, -- S3 key + url TEXT NOT NULL, + filename TEXT, + size INTEGER, + mime_type TEXT, + metadata TEXT, -- JSON: width, height, duration, variants, etc. + uploaded_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_assets_bucket_id ON assets(bucket_id); +CREATE INDEX IF NOT EXISTS idx_assets_type ON assets(bucket_id, type); +CREATE INDEX IF NOT EXISTS idx_assets_uploaded_at ON assets(uploaded_at DESC); +CREATE INDEX IF NOT EXISTS idx_assets_key ON assets(key); + +-- Migrations tracking table +CREATE TABLE IF NOT EXISTS migrations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + version INTEGER NOT NULL UNIQUE, + name TEXT NOT NULL, + executed_at TEXT NOT NULL +); + +-- ============================================================================ +-- TRIGGERS: Automatic source file sync queue +-- ============================================================================ + +-- Trigger: Auto-queue collection source file sync on INSERT +CREATE TRIGGER IF NOT EXISTS trg_collections_after_insert +AFTER INSERT ON collections +BEGIN + INSERT INTO data_sync_queue ( + bucket_id, + operation, + entity_type, + entity_id, + data, + created_at, + status + ) + VALUES ( + NEW.bucket_id, + 'source_file_sync', + 'collection', + NEW.id, + json_object( + 'id', NEW.id, + 'name', NEW.name, + 'description', NEW.description, + 'schema_id', NEW.schema_id, + 'schema_name', NEW.schema_name, + 'published', NEW.published, + 'published_at', NEW.published_at, + 'created_at', NEW.created_at, + 'updated_at', NEW.updated_at, + 'metadata', NEW.metadata + ), + datetime('now'), + 'pending' + ); +END; + +-- Trigger: Auto-queue collection source file sync on UPDATE +CREATE TRIGGER IF NOT EXISTS trg_collections_after_update +AFTER UPDATE ON collections +BEGIN + INSERT INTO data_sync_queue ( + bucket_id, + operation, + entity_type, + entity_id, + data, + created_at, + status + ) + VALUES ( + NEW.bucket_id, + 'source_file_sync', + 'collection', + NEW.id, + json_object( + 'id', NEW.id, + 'name', NEW.name, + 'description', NEW.description, + 'schema_id', NEW.schema_id, + 'schema_name', NEW.schema_name, + 'published', NEW.published, + 'published_at', NEW.published_at, + 'created_at', NEW.created_at, + 'updated_at', NEW.updated_at, + 'metadata', NEW.metadata + ), + datetime('now'), + 'pending' + ); +END; + +-- Trigger: Auto-queue collection source file deletion on DELETE +CREATE TRIGGER IF NOT EXISTS trg_collections_after_delete +AFTER DELETE ON collections +BEGIN + INSERT INTO data_sync_queue ( + bucket_id, + operation, + entity_type, + entity_id, + data, + created_at, + status + ) + VALUES ( + OLD.bucket_id, + 'source_file_delete', + 'collection', + OLD.id, + json_object('id', OLD.id), + datetime('now'), + 'pending' + ); +END; + +-- Trigger: Auto-queue entry source file sync on INSERT +CREATE TRIGGER IF NOT EXISTS trg_entries_after_insert +AFTER INSERT ON entries +BEGIN + INSERT INTO data_sync_queue ( + bucket_id, + operation, + entity_type, + entity_id, + data, + created_at, + status + ) + VALUES ( + NEW.bucket_id, + 'source_file_sync', + 'entry', + NEW.id, + json_object( + 'id', NEW.id, + 'collection_id', NEW.collection_id, + 'schema_id', NEW.schema_id, + 'schema_name', NEW.schema_name, + 'data', NEW.data, + 'published', NEW.published, + 'published_at', NEW.published_at, + 'created_at', NEW.created_at, + 'updated_at', NEW.updated_at + ), + datetime('now'), + 'pending' + ); +END; + +-- Trigger: Auto-queue entry source file sync on UPDATE +CREATE TRIGGER IF NOT EXISTS trg_entries_after_update +AFTER UPDATE ON entries +BEGIN + INSERT INTO data_sync_queue ( + bucket_id, + operation, + entity_type, + entity_id, + data, + created_at, + status + ) + VALUES ( + NEW.bucket_id, + 'source_file_sync', + 'entry', + NEW.id, + json_object( + 'id', NEW.id, + 'collection_id', NEW.collection_id, + 'schema_id', NEW.schema_id, + 'schema_name', NEW.schema_name, + 'data', NEW.data, + 'published', NEW.published, + 'published_at', NEW.published_at, + 'created_at', NEW.created_at, + 'updated_at', NEW.updated_at + ), + datetime('now'), + 'pending' + ); +END; + +-- Trigger: Auto-queue entry source file deletion on DELETE +CREATE TRIGGER IF NOT EXISTS trg_entries_after_delete +AFTER DELETE ON entries +BEGIN + INSERT INTO data_sync_queue ( + bucket_id, + operation, + entity_type, + entity_id, + data, + created_at, + status + ) + VALUES ( + OLD.bucket_id, + 'source_file_delete', + 'entry', + OLD.id, + json_object('id', OLD.id), + datetime('now'), + 'pending' + ); +END; + diff --git a/src/main/main.ts b/src/main/main.ts index bd0687f..df6d6ff 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -9,6 +9,11 @@ const { imageService } = require('./services/imageService.js') const { settingsService } = require('./services/settingsService.js') const { typeGeneratorService } = require('./services/typeGeneratorService.js') const { mediaService } = require('./services/mediaService.js') +// POC: Local database services +const { DatabaseService } = require('./services/databaseService.js') +const { SyncService } = require('./services/syncService.js') +const { MigrationService } = require('./services/migrationService.js') +const { PublishingService } = require('./services/publishingService.js') const { app, BrowserWindow, ipcMain, dialog } = electron const { join } = path @@ -29,6 +34,11 @@ let mainWindow: any = null let s3Service: any let storageService: any let contentService: any +// POC: Local database services +let databaseService: any +let syncService: any +let migrationService: any +let publishingService: any function createWindow() { // Create the browser window @@ -187,6 +197,58 @@ function createWindow() { }) } +/** + * Run migration if needed and notify the renderer of progress + */ +async function runMigrationIfNeeded(bucketConfig: any): Promise { + try { + const needsMigration = await migrationService.needsMigration(bucketConfig.id) + if (!needsMigration) { + console.log('[Migration] Local database already has data, skipping migration') + return false + } + + console.log('[Migration] No local data found, starting migration from S3...') + + // Notify renderer that migration is starting + mainWindow?.webContents.send('migration:started', { + bucketName: bucketConfig.name, + bucketId: bucketConfig.id + }) + + const s3BucketName = bucketConfig.s3BucketName || bucketConfig.bucketName || bucketConfig.name + + if (!s3BucketName) { + throw new Error('S3 bucket name not found in configuration') + } + + console.log('[Migration] Using S3 bucket:', s3BucketName) + await migrationService.migrate(bucketConfig.id, s3BucketName) + console.log('[Migration] โœ“ Migration completed successfully!') + + // Notify renderer that migration is complete + mainWindow?.webContents.send('migration:complete', { + bucketName: bucketConfig.name, + bucketId: bucketConfig.id, + success: true + }) + + return true + } catch (error: any) { + console.error('[Migration] Migration failed:', error) + + // Notify renderer of migration failure + mainWindow?.webContents.send('migration:complete', { + bucketName: bucketConfig.name, + bucketId: bucketConfig.id, + success: false, + error: error.message + }) + + throw error + } +} + // This method will be called when Electron has finished initialization app.whenReady().then(async () => { // Initialize settingsService to set up settingsPath @@ -197,6 +259,16 @@ app.whenReady().then(async () => { storageService = new (require('./services/storageService.js').StorageService)() contentService = new (require('./services/contentService.js').ContentService)() + // POC: Initialize local database services + console.log('[POC] Initializing local database services...') + databaseService = new DatabaseService() + await databaseService.initialize() + syncService = new SyncService(databaseService) + await syncService.start() + migrationService = new MigrationService(databaseService, contentService, syncService) + publishingService = new PublishingService(databaseService) + console.log('[POC] Local database services initialized') + // Initialize with active bucket configuration if available try { // Try to migrate legacy config first @@ -211,15 +283,41 @@ app.whenReady().then(async () => { await s3Service.configure(activeBucketConfig.s3Config) contentService.configure(s3Service.client, s3Service.config) mediaService.configure(s3Service.client, s3Service.config) + publishingService.configure(s3Service.client, activeBucketConfig) + // Toggle sync prefix: set to 'sync' for testing, or '' for production + const useSyncPrefix = false // Set to true to publish to /sync/api/... for testing + if (useSyncPrefix) { + publishingService.setSyncPrefix('sync') + console.log( + '[Publishing] ๐Ÿงช TEST MODE: Using /sync prefix - files will upload to /sync/api/...' + ) + } else { + publishingService.setSyncPrefix('') + console.log('[Publishing] ๐Ÿš€ PRODUCTION MODE: Publishing directly to /api/...') + } + + // Configure sync service for automatic source file backup + const s3BucketName = + activeBucketConfig.s3BucketName || activeBucketConfig.bucketName || activeBucketConfig.name + syncService.configure(publishingService, s3BucketName, activeBucketConfig.id) // Set directories for all services s3Service.setDirectories(activeBucketConfig.directories) contentService.setDirectories(activeBucketConfig.directories) mediaService.setDirectories(activeBucketConfig.directories) + mediaService.setDatabaseService(databaseService, activeBucketConfig.id) imageService.setDirectories(activeBucketConfig.directories) settingsService.setDirectories(activeBucketConfig.directories) console.log('Services configured with directories:', activeBucketConfig.directories) + + // Run migration if needed (with UI notifications) + try { + await runMigrationIfNeeded(activeBucketConfig) + } catch (migrationError) { + console.error('[POC] Migration failed:', migrationError) + // Continue anyway - user can still use the app + } } } catch (error) { console.error('Failed to initialize with active bucket configuration:', error) @@ -313,6 +411,49 @@ ipcMain.handle('storage:delete-bucket-config', async (event: any, id: string) => ipcMain.handle('storage:switch-active-bucket', async (event: any, id: string) => { const success = await storageService.switchActiveBucket(id) + + if (success) { + // Reconfigure services with the new active bucket + const activeBucketConfig = await storageService.getActiveBucketConfig() + if (activeBucketConfig) { + console.log('[IPC] Reconfiguring services for bucket:', activeBucketConfig.name) + + // Reconfigure S3 service + await s3Service.configure(activeBucketConfig.s3Config) + + // Reconfigure all services with new bucket + contentService.configure(s3Service.client, s3Service.config) + mediaService.configure(s3Service.client, s3Service.config) + publishingService.configure(s3Service.client, activeBucketConfig) + + // Configure sync service + const s3BucketName = + activeBucketConfig.s3BucketName || activeBucketConfig.bucketName || activeBucketConfig.name + syncService.configure(publishingService, s3BucketName, activeBucketConfig.id) + + // Set directories + s3Service.setDirectories(activeBucketConfig.directories) + contentService.setDirectories(activeBucketConfig.directories) + mediaService.setDirectories(activeBucketConfig.directories) + + // Reconfigure MediaService with new bucket ID (critical for loading correct assets) + mediaService.setDatabaseService(databaseService, activeBucketConfig.id) + + imageService.setDirectories(activeBucketConfig.directories) + settingsService.setDirectories(activeBucketConfig.directories) + + console.log('[IPC] Services reconfigured successfully') + + // Check if migration is needed for this bucket (runs with UI notifications) + try { + await runMigrationIfNeeded(activeBucketConfig) + } catch (migrationError) { + console.error('[IPC] Migration failed during bucket switch:', migrationError) + // Don't fail the bucket switch - user can retry migration + } + } + } + return success }) @@ -587,9 +728,17 @@ ipcMain.handle( ipcMain.handle('content:generate-all-api-endpoints', async (event: any, { bucket }: any) => { try { - await contentService.generateAllApiEndpoints(bucket) + // Get active bucket config to get the bucketId + const activeBucketConfig = await storageService.getActiveBucketConfig() + if (!activeBucketConfig) { + throw new Error('No active bucket configuration') + } + + // Use publishingService to read from local DB and write to S3 + await publishingService.generateAllApiEndpoints(bucket, activeBucketConfig.id) return { success: true } } catch (error) { + console.error('[IPC] Error generating API endpoints:', error) return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } } }) @@ -598,13 +747,22 @@ ipcMain.handle( 'content:generate-selective-api-endpoints', async (event: any, { bucket, collectionIds, entryIds }: any) => { try { - await contentService.generateSelectiveApiEndpoints( + // Get active bucket config to get the bucketId + const activeBucketConfig = await storageService.getActiveBucketConfig() + if (!activeBucketConfig) { + throw new Error('No active bucket configuration') + } + + // Use publishingService to read from local DB and write to S3 + await publishingService.generateSelectiveApiEndpoints( bucket, + activeBucketConfig.id, collectionIds || [], entryIds || [] ) return { success: true } } catch (error) { + console.error('[IPC] Error generating selective API endpoints:', error) return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } } } @@ -612,7 +770,7 @@ ipcMain.handle( ipcMain.handle('content:get-last-generation-date', async (event: any, { bucket }: any) => { try { - const lastGenerated = await contentService.getLastGenerationDate(bucket) + const lastGenerated = await publishingService.getLastGenerationDate(bucket) return { success: true, lastGenerated } } catch (error) { return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } @@ -845,3 +1003,321 @@ ipcMain.handle('media:delete-asset', async (event: any, { bucket, asset }: any) return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } } }) + +// ======================================== +// POC: Local Database IPC Handlers +// ======================================== + +// Collection handlers +ipcMain.handle('db:collection:create', async (event: any, bucketId: string, data: any) => { + try { + console.log('[IPC] db:collection:create', bucketId, data.id) + const collection = await databaseService.createCollection({ ...data, bucket_id: bucketId }) + // Note: Database triggers automatically queue source file sync + + // Notify renderer of update + mainWindow?.webContents.send('db:collection:updated', collection) + + return { success: true, result: collection } + } catch (error) { + console.error('[IPC] db:collection:create error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +ipcMain.handle('db:collection:update', async (event: any, id: string, updates: any) => { + try { + console.log('[IPC] db:collection:update', id) + const collection = await databaseService.updateCollection(id, updates) + // Note: Database triggers automatically queue source file sync + + // Notify renderer of update + mainWindow?.webContents.send('db:collection:updated', collection) + + return { success: true, result: collection } + } catch (error) { + console.error('[IPC] db:collection:update error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +ipcMain.handle('db:collection:get', async (event: any, id: string) => { + try { + console.log('[IPC] db:collection:get', id) + const collection = await databaseService.getCollection(id) + return { success: true, result: collection } + } catch (error) { + console.error('[IPC] db:collection:get error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +ipcMain.handle('db:collection:list', async (event: any, bucketId: string) => { + try { + console.log('[IPC] db:collection:list', bucketId) + const collections = await databaseService.listCollections(bucketId) + console.log( + `[IPC] Returning ${collections.length} collections with entry counts:`, + collections.map(c => ({ name: c.name, entryCount: c.entries?.length || 0 })) + ) + return { success: true, result: collections } + } catch (error) { + console.error('[IPC] db:collection:list error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +ipcMain.handle('db:collection:delete', async (event: any, id: string) => { + try { + console.log('[IPC] db:collection:delete', id) + const collection = await databaseService.getCollection(id) + if (!collection) { + return { success: false, error: 'Collection not found' } + } + + await databaseService.deleteCollection(id) + // Note: Database triggers automatically queue source file sync + + return { success: true } + } catch (error) { + console.error('[IPC] db:collection:delete error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +// Entry handlers +ipcMain.handle('db:entry:create', async (event: any, bucketId: string, data: any) => { + try { + console.log('[IPC] db:entry:create', bucketId, data.id) + const entry = await databaseService.createEntry({ ...data, bucket_id: bucketId }) + // Note: Database triggers automatically queue source file sync + + // Notify renderer of update + mainWindow?.webContents.send('db:entry:updated', entry) + + return { success: true, result: entry } + } catch (error) { + console.error('[IPC] db:entry:create error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +ipcMain.handle('db:entry:update', async (event: any, id: string, updates: any) => { + try { + console.log('[IPC] db:entry:update', id) + const entry = await databaseService.updateEntry(id, updates) + // Note: Database triggers automatically queue source file sync + + // Notify renderer of update + mainWindow?.webContents.send('db:entry:updated', entry) + + return { success: true, result: entry } + } catch (error) { + console.error('[IPC] db:entry:update error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +ipcMain.handle('db:entry:get', async (event: any, id: string) => { + try { + console.log('[IPC] db:entry:get', id) + const entry = await databaseService.getEntry(id) + return { success: true, result: entry } + } catch (error) { + console.error('[IPC] db:entry:get error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +ipcMain.handle('db:entry:list', async (event: any, collectionId: string) => { + try { + console.log('[IPC] db:entry:list', collectionId) + const entries = await databaseService.listEntries(collectionId) + return { success: true, result: entries } + } catch (error) { + console.error('[IPC] db:entry:list error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +ipcMain.handle('db:entry:delete', async (event: any, id: string) => { + try { + console.log('[IPC] db:entry:delete', id) + const entry = await databaseService.getEntry(id) + if (!entry) { + return { success: false, error: 'Entry not found' } + } + + await databaseService.deleteEntry(id) + // Note: Database triggers automatically queue source file sync + + // Notify renderer of deletion + mainWindow?.webContents.send('db:entry:deleted', id) + + return { success: true } + } catch (error) { + console.error('[IPC] db:entry:delete error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +ipcMain.handle( + 'db:entry:updateOrder', + async (event: any, collectionId: string, entryIds: string[]) => { + try { + console.log('[IPC] db:entry:updateOrder', collectionId, entryIds.length) + await databaseService.updateEntryOrder(collectionId, entryIds) + + // Note: Entry order updates don't trigger collection file sync + // The order is stored separately and included during publish + + return { success: true } + } catch (error) { + console.error('[IPC] db:entry:updateOrder error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } + } +) + +ipcMain.handle( + 'db:collection:updateOrder', + async (event: any, bucketId: string, collectionIds: string[]) => { + try { + console.log('[IPC] db:collection:updateOrder', bucketId, collectionIds.length) + await databaseService.updateCollectionOrder(bucketId, collectionIds) + + // Note: Collection order updates don't trigger individual file sync + // The order is stored separately and included during publish + + return { success: true } + } catch (error) { + console.error('[IPC] db:collection:updateOrder error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } + } +) + +// Sync handlers +ipcMain.handle('db:sync:getStatus', async () => { + try { + const status = syncService.getStatus() + return { success: true, result: status } + } catch (error) { + console.error('[IPC] db:sync:getStatus error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +ipcMain.handle('db:sync:forceSyncNow', async () => { + try { + await syncService.forceSyncNow() + return { success: true } + } catch (error) { + console.error('[IPC] db:sync:forceSyncNow error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +// Database stats +ipcMain.handle('db:getStats', async () => { + try { + const stats = await databaseService.getStats() + return { success: true, result: stats } + } catch (error) { + console.error('[IPC] db:getStats error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +// Set up sync event forwarding to renderer +if (syncService) { + syncService.on('statusChange', (status: any) => { + mainWindow?.webContents.send('db:sync:statusChange', status) + }) + + syncService.on('itemSynced', (item: any) => { + mainWindow?.webContents.send('db:sync:itemSynced', item) + }) + + syncService.on('syncError', (error: any) => { + mainWindow?.webContents.send('db:sync:syncError', error) + }) +} + +// Database management handlers +ipcMain.handle('db:clear', async () => { + try { + console.log('[IPC] db:clear - Clearing local database') + await databaseService.clearDatabase() + + // After clearing, check if we need to migrate from S3 + const activeBucketConfig = await storageService.getActiveBucketConfig() + if (activeBucketConfig) { + const needsMigration = await migrationService.needsMigration(activeBucketConfig.id) + if (needsMigration) { + console.log('[IPC] db:clear - Re-importing from S3...') + const s3BucketName = + activeBucketConfig.s3BucketName || + activeBucketConfig.bucketName || + activeBucketConfig.name + if (s3BucketName) { + await migrationService.migrate(activeBucketConfig.id, s3BucketName) + console.log('[IPC] db:clear - Re-import completed') + } + } + } + + return { success: true } + } catch (error) { + console.error('[IPC] db:clear error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +ipcMain.handle('db:getPath', async () => { + try { + const dbPath = databaseService.getDatabasePath() + return { success: true, result: dbPath } + } catch (error) { + console.error('[IPC] db:getPath error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +// Debug handler to inspect bucket_ids in database +ipcMain.handle('db:debug:getBucketIds', async () => { + try { + const db = (databaseService as any).db + if (!db) { + return { success: false, error: 'Database not initialized' } + } + + const collections = db.prepare('SELECT id, name, bucket_id FROM collections').all() + const entries = db.prepare('SELECT id, bucket_id FROM entries').all() + + const collectionBucketIds = [...new Set(collections.map((c: any) => c.bucket_id))] + const entryBucketIds = [...new Set(entries.map((e: any) => e.bucket_id))] + + console.log('[DEBUG] Collections by bucket_id:') + collectionBucketIds.forEach((bucketId: string) => { + const count = collections.filter((c: any) => c.bucket_id === bucketId).length + console.log(` ${bucketId}: ${count} collections`) + }) + + return { + success: true, + result: { + collectionBucketIds, + entryBucketIds, + totalCollections: collections.length, + totalEntries: entries.length, + sampleCollections: collections.slice(0, 3) + } + } + } catch (error) { + console.error('[IPC] db:debug:getBucketIds error:', error) + return { success: false, error: error instanceof Error ? error.message : 'Unknown error' } + } +}) + +console.log('[POC] Database IPC handlers registered') diff --git a/src/main/preload.ts b/src/main/preload.ts index 12d1309..441e5fb 100644 --- a/src/main/preload.ts +++ b/src/main/preload.ts @@ -101,6 +101,93 @@ contextBridge.exposeInMainWorld('electronAPI', { listAssets: (params: { bucket: string }) => ipcRenderer.invoke('media:list-assets', params), deleteAsset: (params: { bucket: string; asset: any }) => ipcRenderer.invoke('media:delete-asset', params) + }, + + // Local Database operations (POC) + db: { + collection: { + create: (bucketId: string, data: any) => + ipcRenderer.invoke('db:collection:create', bucketId, data), + update: (id: string, updates: any) => ipcRenderer.invoke('db:collection:update', id, updates), + get: (id: string) => ipcRenderer.invoke('db:collection:get', id), + list: (bucketId: string) => ipcRenderer.invoke('db:collection:list', bucketId), + delete: (id: string) => ipcRenderer.invoke('db:collection:delete', id), + updateOrder: (bucketId: string, collectionIds: string[]) => + ipcRenderer.invoke('db:collection:updateOrder', bucketId, collectionIds), + + // Real-time updates + onUpdated: (callback: (collection: any) => void) => { + const listener = (_: any, collection: any) => callback(collection) + ipcRenderer.on('db:collection:updated', listener) + return () => ipcRenderer.removeListener('db:collection:updated', listener) + } + }, + + entry: { + create: (bucketId: string, data: any) => + ipcRenderer.invoke('db:entry:create', bucketId, data), + update: (id: string, updates: any) => ipcRenderer.invoke('db:entry:update', id, updates), + get: (id: string) => ipcRenderer.invoke('db:entry:get', id), + list: (collectionId: string) => ipcRenderer.invoke('db:entry:list', collectionId), + delete: (id: string) => ipcRenderer.invoke('db:entry:delete', id), + updateOrder: (collectionId: string, entryIds: string[]) => + ipcRenderer.invoke('db:entry:updateOrder', collectionId, entryIds), + + // Real-time updates + onUpdated: (callback: (entry: any) => void) => { + const listener = (_: any, entry: any) => callback(entry) + ipcRenderer.on('db:entry:updated', listener) + return () => ipcRenderer.removeListener('db:entry:updated', listener) + }, + onDeleted: (callback: (entryId: string) => void) => { + const listener = (_: any, entryId: string) => callback(entryId) + ipcRenderer.on('db:entry:deleted', listener) + return () => ipcRenderer.removeListener('db:entry:deleted', listener) + } + }, + + sync: { + getStatus: () => ipcRenderer.invoke('db:sync:getStatus'), + forceSyncNow: () => ipcRenderer.invoke('db:sync:forceSyncNow'), + + // Real-time sync status updates + onStatusChange: (callback: (status: any) => void) => { + const listener = (_: any, status: any) => callback(status) + ipcRenderer.on('db:sync:statusChange', listener) + return () => ipcRenderer.removeListener('db:sync:statusChange', listener) + } + }, + + migration: { + // Real-time migration events + onStarted: (callback: (data: { bucketName: string; bucketId: string }) => void) => { + const listener = (_: any, data: any) => callback(data) + ipcRenderer.on('migration:started', listener) + return () => ipcRenderer.removeListener('migration:started', listener) + }, + onComplete: ( + callback: (data: { + bucketName: string + bucketId: string + success: boolean + error?: string + }) => void + ) => { + const listener = (_: any, data: any) => callback(data) + ipcRenderer.on('migration:complete', listener) + return () => ipcRenderer.removeListener('migration:complete', listener) + } + }, + + // Database management + clear: () => ipcRenderer.invoke('db:clear'), + getPath: () => ipcRenderer.invoke('db:getPath'), + getStats: () => ipcRenderer.invoke('db:getStats'), + + // Debug helpers + debug: { + getBucketIds: () => ipcRenderer.invoke('db:debug:getBucketIds') + } } }) diff --git a/src/main/services/databaseService.ts b/src/main/services/databaseService.ts new file mode 100644 index 0000000..d19c8fd --- /dev/null +++ b/src/main/services/databaseService.ts @@ -0,0 +1,1099 @@ +/** + * DatabaseService - POC Version + * Minimal implementation for proof of concept with collections only + */ + +import Database from 'better-sqlite3' +import { app } from 'electron' +import path from 'path' +import fs from 'fs' + +export interface Collection { + id: string + bucket_id: string + name: string + description?: string + schema_id: string + schema_name: string + published: boolean + published_at?: string + created_at: string + updated_at: string + metadata?: any + sync_status: 'pending' | 'syncing' | 'synced' | 'error' + sync_error?: string + last_synced_at?: string + entries?: any[] // For UI compatibility - populated by listCollections + entryOrder?: string[] // For UI compatibility +} + +export interface Entry { + id: string + bucket_id: string + collection_id: string + schema_id: string + schema_name: string + data: any // JSON content (all entry fields stored here) + published: boolean + published_at?: string + created_at: string + updated_at: string + sync_status: 'pending' | 'syncing' | 'synced' | 'error' + sync_error?: string + last_synced_at?: string +} + +export interface SyncQueueItem { + id: number + bucket_id: string + operation: string + entity_type: string + entity_id: string + data: any + retry_count: number + max_retries: number + created_at: string + started_at?: string + completed_at?: string + status: 'pending' | 'processing' | 'completed' | 'failed' +} + +export class DatabaseService { + private db: Database.Database | null = null + private readonly dbPath: string + + constructor() { + const userDataPath = app.getPath('userData') + this.dbPath = path.join(userDataPath, 's3-cms-poc.db') + console.log('[DatabaseService] Database path:', this.dbPath) + } + + async initialize(): Promise { + console.log('[DatabaseService] Initializing database...') + + this.db = new Database(this.dbPath) + + // Enable WAL mode for better concurrency + this.db.pragma('journal_mode = WAL') + + // Enable foreign keys + this.db.pragma('foreign_keys = ON') + + // Set reasonable timeouts + this.db.pragma('busy_timeout = 5000') + + // Run migrations + await this.runMigrations() + + console.log('[DatabaseService] Database initialized successfully') + } + + private async runMigrations(): Promise { + if (!this.db) throw new Error('Database not initialized') + + // Try multiple paths for the schema file (dev vs production) + const possiblePaths = [ + // Production build: compiled service is in dist/main/services/, schema is in dist/main/database/ + path.join(__dirname, '../database/schema.sql'), + // Development: compiled service is in dist/main/services/, schema is in src/main/database/ + path.join(__dirname, '../../src/main/database/schema.sql'), + // Development: from project root + path.join(process.cwd(), 'src/main/database/schema.sql'), + // Packaged app: schema is included in the app bundle + path.join(process.resourcesPath || '', 'app.asar', 'src/main/database/schema.sql'), + path.join(process.resourcesPath || '', 'app', 'src/main/database/schema.sql'), + // Packaged app: schema copied to dist/main/database/ during build + path.join(process.resourcesPath || '', 'app.asar', 'dist/main/database/schema.sql'), + path.join(process.resourcesPath || '', 'app', 'dist/main/database/schema.sql') + ] + + console.log('[DatabaseService] Looking for schema file...') + console.log('[DatabaseService] __dirname:', __dirname) + console.log('[DatabaseService] process.cwd():', process.cwd()) + console.log('[DatabaseService] process.resourcesPath:', process.resourcesPath) + + let schemaPath: string | null = null + for (const testPath of possiblePaths) { + console.log('[DatabaseService] Checking path:', testPath, 'exists:', fs.existsSync(testPath)) + if (fs.existsSync(testPath)) { + schemaPath = testPath + break + } + } + + if (!schemaPath) { + const errorMessage = `Schema file not found. Tried paths:\n${possiblePaths.map(p => ` - ${p}`).join('\n')}` + console.error('[DatabaseService]', errorMessage) + throw new Error(errorMessage) + } + + console.log('[DatabaseService] โœ“ Using schema file:', schemaPath) + + const currentVersion = this.getCurrentVersion() + console.log('[DatabaseService] Current schema version:', currentVersion) + + if (currentVersion === 0) { + console.log('[DatabaseService] Running initial schema...') + const schema = fs.readFileSync(schemaPath, 'utf-8') + this.db.exec(schema) + this.recordMigration(1, 'initial_schema_poc') + console.log('[DatabaseService] Initial schema applied') + } + + // Migration 2: Add automatic source file sync triggers + if (currentVersion < 2) { + console.log('[DatabaseService] Running migration 2: Add source file sync triggers...') + + const triggersSql = ` + -- ============================================================================ + -- TRIGGERS: Automatic source file sync queue + -- ============================================================================ + + -- Trigger: Auto-queue collection source file sync on INSERT + CREATE TRIGGER IF NOT EXISTS trg_collections_after_insert + AFTER INSERT ON collections + BEGIN + INSERT INTO data_sync_queue ( + bucket_id, operation, entity_type, entity_id, data, created_at, status + ) + VALUES ( + NEW.bucket_id, 'source_file_sync', 'collection', NEW.id, + json_object( + 'id', NEW.id, 'name', NEW.name, 'description', NEW.description, + 'schema_id', NEW.schema_id, 'schema_name', NEW.schema_name, + 'published', NEW.published, 'published_at', NEW.published_at, + 'created_at', NEW.created_at, 'updated_at', NEW.updated_at, + 'metadata', NEW.metadata + ), + datetime('now'), 'pending' + ); + END; + + -- Trigger: Auto-queue collection source file sync on UPDATE + CREATE TRIGGER IF NOT EXISTS trg_collections_after_update + AFTER UPDATE ON collections + BEGIN + INSERT INTO data_sync_queue ( + bucket_id, operation, entity_type, entity_id, data, created_at, status + ) + VALUES ( + NEW.bucket_id, 'source_file_sync', 'collection', NEW.id, + json_object( + 'id', NEW.id, 'name', NEW.name, 'description', NEW.description, + 'schema_id', NEW.schema_id, 'schema_name', NEW.schema_name, + 'published', NEW.published, 'published_at', NEW.published_at, + 'created_at', NEW.created_at, 'updated_at', NEW.updated_at, + 'metadata', NEW.metadata + ), + datetime('now'), 'pending' + ); + END; + + -- Trigger: Auto-queue collection source file deletion on DELETE + CREATE TRIGGER IF NOT EXISTS trg_collections_after_delete + AFTER DELETE ON collections + BEGIN + INSERT INTO data_sync_queue ( + bucket_id, operation, entity_type, entity_id, data, created_at, status + ) + VALUES ( + OLD.bucket_id, 'source_file_delete', 'collection', OLD.id, + json_object('id', OLD.id), datetime('now'), 'pending' + ); + END; + + -- Trigger: Auto-queue entry source file sync on INSERT + CREATE TRIGGER IF NOT EXISTS trg_entries_after_insert + AFTER INSERT ON entries + BEGIN + INSERT INTO data_sync_queue ( + bucket_id, operation, entity_type, entity_id, data, created_at, status + ) + VALUES ( + NEW.bucket_id, 'source_file_sync', 'entry', NEW.id, + json_object( + 'id', NEW.id, 'collection_id', NEW.collection_id, + 'schema_id', NEW.schema_id, 'schema_name', NEW.schema_name, + 'data', NEW.data, 'published', NEW.published, + 'published_at', NEW.published_at, 'created_at', NEW.created_at, + 'updated_at', NEW.updated_at + ), + datetime('now'), 'pending' + ); + END; + + -- Trigger: Auto-queue entry source file sync on UPDATE + CREATE TRIGGER IF NOT EXISTS trg_entries_after_update + AFTER UPDATE ON entries + BEGIN + INSERT INTO data_sync_queue ( + bucket_id, operation, entity_type, entity_id, data, created_at, status + ) + VALUES ( + NEW.bucket_id, 'source_file_sync', 'entry', NEW.id, + json_object( + 'id', NEW.id, 'collection_id', NEW.collection_id, + 'schema_id', NEW.schema_id, 'schema_name', NEW.schema_name, + 'data', NEW.data, 'published', NEW.published, + 'published_at', NEW.published_at, 'created_at', NEW.created_at, + 'updated_at', NEW.updated_at + ), + datetime('now'), 'pending' + ); + END; + + -- Trigger: Auto-queue entry source file deletion on DELETE + CREATE TRIGGER IF NOT EXISTS trg_entries_after_delete + AFTER DELETE ON entries + BEGIN + INSERT INTO data_sync_queue ( + bucket_id, operation, entity_type, entity_id, data, created_at, status + ) + VALUES ( + OLD.bucket_id, 'source_file_delete', 'entry', OLD.id, + json_object('id', OLD.id), datetime('now'), 'pending' + ); + END; + ` + + this.db.exec(triggersSql) + this.recordMigration(2, 'add_source_file_sync_triggers') + console.log('[DatabaseService] โœ“ Migration 2 applied: Source file sync triggers added') + } + } + + private getCurrentVersion(): number { + if (!this.db) return 0 + + try { + const result = this.db.prepare('SELECT MAX(version) as version FROM migrations').get() as { + version: number | null + } + return result?.version || 0 + } catch { + // Migrations table doesn't exist yet + return 0 + } + } + + private recordMigration(version: number, name: string): void { + if (!this.db) throw new Error('Database not initialized') + this.db + .prepare('INSERT INTO migrations (version, name, executed_at) VALUES (?, ?, ?)') + .run(version, name, new Date().toISOString()) + } + + // ======================================== + // Collection Operations + // ======================================== + + async createCollection( + data: Omit + ): Promise { + if (!this.db) throw new Error('Database not initialized') + + const now = new Date().toISOString() + const collection: any = { + ...data, + created_at: now, + updated_at: now, + sync_status: 'pending', + published: data.published ? 1 : 0, + metadata: data.metadata ? JSON.stringify(data.metadata) : null + } + + console.log('[DatabaseService] Creating collection:', collection.id) + + this.db + .prepare( + ` + INSERT INTO collections ( + id, bucket_id, name, description, schema_id, schema_name, + published, published_at, created_at, updated_at, metadata, sync_status + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ` + ) + .run( + collection.id, + collection.bucket_id, + collection.name, + collection.description || null, + collection.schema_id, + collection.schema_name, + collection.published, + collection.published_at || null, + collection.created_at, + collection.updated_at, + collection.metadata, + collection.sync_status + ) + + const created = await this.getCollection(collection.id) + if (!created) throw new Error('Failed to create collection') + return created + } + + async updateCollection(id: string, updates: Partial): Promise { + if (!this.db) throw new Error('Database not initialized') + + const now = new Date().toISOString() + const fields: string[] = [] + const values: any[] = [] + + console.log('[DatabaseService] Updating collection:', id, updates) + + // Build dynamic update query + for (const [key, value] of Object.entries(updates)) { + if (key === 'id' || key === 'created_at' || key === 'bucket_id') continue // Don't update these + + if (key === 'metadata') { + fields.push(`${key} = ?`) + values.push(value ? JSON.stringify(value) : null) + } else if (key === 'published') { + fields.push(`${key} = ?`) + values.push(value ? 1 : 0) + } else { + fields.push(`${key} = ?`) + values.push(value ?? null) + } + } + + fields.push('updated_at = ?') + values.push(now) + values.push(id) + + if (fields.length > 1) { + // More than just updated_at + this.db + .prepare( + ` + UPDATE collections SET ${fields.join(', ')} WHERE id = ? + ` + ) + .run(...values) + } + + const updated = await this.getCollection(id) + if (!updated) throw new Error('Failed to update collection') + return updated + } + + async getCollection(id: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + const row = this.db.prepare('SELECT * FROM collections WHERE id = ?').get(id) as any + if (!row) return null + + const collection = this.parseCollection(row) + + // Include entry order + const entryOrder = this.getEntryOrderSync(id) + + return { + ...collection, + entryOrder + } + } + + async listCollections(bucketId: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + console.log('[DatabaseService] Listing collections for bucket:', bucketId) + + // First, check total collections in database (for debugging) + const totalCollections = this.db + .prepare('SELECT COUNT(*) as count FROM collections') + .get() as any + console.log('[DatabaseService] Total collections in database:', totalCollections.count) + + const rows = this.db + .prepare( + ` + SELECT + c.*, + COUNT(e.id) as entry_count, + bco.position + FROM collections c + LEFT JOIN entries e ON e.collection_id = c.id + LEFT JOIN bucket_collection_order bco ON c.id = bco.collection_id AND c.bucket_id = bco.bucket_id + WHERE c.bucket_id = ? + GROUP BY c.id + ORDER BY COALESCE(bco.position, 999999), c.updated_at DESC + ` + ) + .all(bucketId) as any[] + + console.log(`[DatabaseService] Found ${rows.length} collections for bucket ${bucketId}`) + + const collections = rows.map(row => { + const collection = this.parseCollection(row) + // Add entries array with count for UI compatibility + const entryCount = Number(row.entry_count || 0) + + // Fetch entry order for this collection + const entryOrder = this.getEntryOrderSync(collection.id) + + console.log( + `[DatabaseService] Collection "${collection.name}" (bucket_id: ${collection.bucket_id}) has ${entryCount} entries, order: ${entryOrder.length} ids` + ) + + // Verify this collection belongs to the requested bucket + if (collection.bucket_id !== bucketId) { + console.error( + `[DatabaseService] โš ๏ธ WARNING: Collection "${collection.name}" has bucket_id "${collection.bucket_id}" but was requested for bucket "${bucketId}"!` + ) + } + + return { + ...collection, + entries: new Array(entryCount).fill(null), + entryOrder: entryOrder + } + }) + + console.log(`[DatabaseService] Returning ${collections.length} collections (ordered)`) + return collections + } + + // Synchronous version of getEntryOrder for use within listCollections + private getEntryOrderSync(collectionId: string): string[] { + if (!this.db) return [] + + const rows = this.db + .prepare( + ` + SELECT entry_id FROM collection_entry_order + WHERE collection_id = ? + ORDER BY position ASC + ` + ) + .all(collectionId) as { entry_id: string }[] + + return rows.map(row => row.entry_id) + } + + async deleteCollection(id: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + console.log('[DatabaseService] Deleting collection:', id) + this.db.prepare('DELETE FROM collections WHERE id = ?').run(id) + } + + private parseCollection(row: any): Collection { + return { + ...row, + published: Boolean(row.published), + metadata: row.metadata ? JSON.parse(row.metadata) : {}, + // Convert snake_case to camelCase for UI compatibility + schemaId: row.schema_id, + schemaName: row.schema_name, + bucketId: row.bucket_id, + publishedAt: row.published_at, + createdAt: row.created_at, + updatedAt: row.updated_at, + syncStatus: row.sync_status, + syncError: row.sync_error, + lastSyncedAt: row.last_synced_at + } + } + + // ======================================== + // Entry Operations + // ======================================== + + async createEntry( + data: Omit + ): Promise { + if (!this.db) throw new Error('Database not initialized') + + const now = new Date().toISOString() + const entry: any = { + ...data, + created_at: now, + updated_at: now, + sync_status: 'pending', + published: data.published ? 1 : 0, + data: typeof data.data === 'string' ? data.data : JSON.stringify(data.data || {}) + } + + console.log('[DatabaseService] Creating entry:', entry.id) + + this.db + .prepare( + ` + INSERT INTO entries ( + id, bucket_id, collection_id, schema_id, schema_name, + data, published, published_at, + created_at, updated_at, sync_status + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ` + ) + .run( + entry.id, + entry.bucket_id, + entry.collection_id, + entry.schema_id, + entry.schema_name, + entry.data, + entry.published, + entry.published_at || null, + entry.created_at, + entry.updated_at, + entry.sync_status + ) + + const created = await this.getEntry(entry.id) + if (!created) throw new Error('Failed to create entry') + return created + } + + async updateEntry(id: string, updates: Partial): Promise { + if (!this.db) throw new Error('Database not initialized') + + const now = new Date().toISOString() + const fields: string[] = [] + const values: any[] = [] + + console.log('[DatabaseService] Updating entry:', id, updates) + + for (const [key, value] of Object.entries(updates)) { + if (key === 'id' || key === 'created_at' || key === 'bucket_id') continue + + if (key === 'data') { + fields.push(`${key} = ?`) + values.push(typeof value === 'string' ? value : JSON.stringify(value || {})) + } else if (key === 'published') { + fields.push(`${key} = ?`) + values.push(value ? 1 : 0) + } else { + fields.push(`${key} = ?`) + values.push(value ?? null) + } + } + + fields.push('updated_at = ?') + values.push(now) + values.push(id) + + if (fields.length > 1) { + this.db + .prepare( + ` + UPDATE entries SET ${fields.join(', ')} WHERE id = ? + ` + ) + .run(...values) + } + + const updated = await this.getEntry(id) + if (!updated) throw new Error('Failed to update entry') + return updated + } + + async getEntry(id: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + const row = this.db.prepare('SELECT * FROM entries WHERE id = ?').get(id) as any + if (!row) return null + + return this.parseEntry(row) + } + + async listEntries(collectionId: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + console.log('[DatabaseService] Listing entries for collection:', collectionId) + + // Get entries with their order + const rows = this.db + .prepare( + ` + SELECT e.*, ceo.position + FROM entries e + LEFT JOIN collection_entry_order ceo ON e.id = ceo.entry_id AND e.collection_id = ceo.collection_id + WHERE e.collection_id = ? + ORDER BY COALESCE(ceo.position, 999999), e.created_at DESC + ` + ) + .all(collectionId) as any[] + + return rows.map(row => this.parseEntry(row)) + } + + async deleteEntry(id: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + console.log('[DatabaseService] Deleting entry:', id) + this.db.prepare('DELETE FROM entries WHERE id = ?').run(id) + } + + private parseEntry(row: any): Entry { + // Parse the JSON data field + const parsedData = row.data ? JSON.parse(row.data) : {} + + // Determine data structure + // New structure: { data: {...fields...}, metadata: {...}, tags: [...] } + // S3/Legacy structure: { ...fields..., metadata: {...}, tags: [...] } - metadata at root + // Very old structure: { ...fields... } - no metadata at all + + let entryData: any + let metadata: any + let tags: string[] = [] + + // Check if we have the new wrapped structure + if (parsedData.data !== undefined && typeof parsedData.data === 'object') { + // New structure: data is wrapped + entryData = parsedData.data || {} + metadata = parsedData.metadata || {} + tags = parsedData.tags || [] + } else { + // S3/Legacy structure: data fields are at root level + // We need to separate metadata from schema fields + const { metadata: extractedMetadata, tags: extractedTags, ...schemaFields } = parsedData + entryData = schemaFields + metadata = extractedMetadata || {} + tags = extractedTags || [] + } + + // Extract metadata fields to root level for UI compatibility + const title = metadata.title || '' + const slug = metadata.slug || '' + const description = metadata.description || '' + + console.log('[DatabaseService] parseEntry:', { + id: row.id, + hasWrappedData: parsedData.data !== undefined, + metadata, + title, + slug, + description + }) + + // Build the entry object with UI-compatible structure + return { + ...row, + published: Boolean(row.published), + // Extract metadata fields to root level + title, + slug, + description, + metadata, // Keep metadata object + tags, // Include tags + // Keep schemaId/schemaName in camelCase for UI compatibility + schemaId: row.schema_id, + schemaName: row.schema_name, + collectionId: row.collection_id, + bucketId: row.bucket_id, + publishedAt: row.published_at, + createdAt: row.created_at, + updatedAt: row.updated_at, + syncStatus: row.sync_status, + syncError: row.sync_error, + lastSyncedAt: row.last_synced_at, + // Store the actual schema field data + data: entryData + } + } + + // ======================================== + // Entry Order Operations + // ======================================== + + async updateEntryOrder(collectionId: string, entryIds: string[]): Promise { + if (!this.db) throw new Error('Database not initialized') + + console.log('[DatabaseService] Updating entry order for collection:', collectionId) + + // Use transaction for atomicity + const transaction = this.db.transaction(() => { + // Delete existing order + this.db!.prepare('DELETE FROM collection_entry_order WHERE collection_id = ?').run( + collectionId + ) + + // Insert new order + const stmt = this.db!.prepare(` + INSERT INTO collection_entry_order (collection_id, entry_id, position) + VALUES (?, ?, ?) + `) + + entryIds.forEach((entryId, index) => { + stmt.run(collectionId, entryId, index) + }) + }) + + transaction() + } + + async getEntryOrder(collectionId: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + const rows = this.db + .prepare( + ` + SELECT entry_id FROM collection_entry_order + WHERE collection_id = ? + ORDER BY position ASC + ` + ) + .all(collectionId) as { entry_id: string }[] + + return rows.map(row => row.entry_id) + } + + // ======================================== + // Collection Order Operations + // ======================================== + + async updateCollectionOrder(bucketId: string, collectionIds: string[]): Promise { + if (!this.db) throw new Error('Database not initialized') + + console.log('[DatabaseService] Updating collection order for bucket:', bucketId) + + // Use transaction for atomicity + const transaction = this.db.transaction(() => { + // Delete existing order + this.db!.prepare('DELETE FROM bucket_collection_order WHERE bucket_id = ?').run(bucketId) + + // Insert new order + const stmt = this.db!.prepare(` + INSERT INTO bucket_collection_order (bucket_id, collection_id, position) + VALUES (?, ?, ?) + `) + + collectionIds.forEach((collectionId, index) => { + stmt.run(bucketId, collectionId, index) + }) + }) + + transaction() + console.log('[DatabaseService] Collection order updated successfully') + } + + async getCollectionOrder(bucketId: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + const rows = this.db + .prepare( + ` + SELECT collection_id FROM bucket_collection_order + WHERE bucket_id = ? + ORDER BY position ASC + ` + ) + .all(bucketId) as { collection_id: string }[] + + return rows.map(row => row.collection_id) + } + + // Synchronous version for use within listCollections + private getCollectionOrderSync(bucketId: string): string[] { + if (!this.db) return [] + + const rows = this.db + .prepare( + ` + SELECT collection_id FROM bucket_collection_order + WHERE bucket_id = ? + ORDER BY position ASC + ` + ) + .all(bucketId) as { collection_id: string }[] + + return rows.map(row => row.collection_id) + } + + // ======================================== + // Sync Queue Operations + // ======================================== + + async addToSyncQueue( + bucketId: string, + operation: string, + entityType: string, + entityId: string, + data: any + ): Promise { + if (!this.db) throw new Error('Database not initialized') + + console.log('[DatabaseService] Adding to sync queue:', { operation, entityType, entityId }) + + this.db + .prepare( + ` + INSERT INTO data_sync_queue ( + bucket_id, operation, entity_type, entity_id, data, created_at + ) VALUES (?, ?, ?, ?, ?, ?) + ` + ) + .run( + bucketId, + operation, + entityType, + entityId, + data ? JSON.stringify(data) : null, + new Date().toISOString() + ) + } + + async getSyncQueue(limit: number = 10, bucketId?: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + // Filter by bucket_id if provided to prevent cross-bucket sync contamination + let query: string + let params: any[] + + if (bucketId) { + query = ` + SELECT * FROM data_sync_queue + WHERE bucket_id = ? AND status = 'pending' AND retry_count < max_retries + ORDER BY created_at ASC + LIMIT ? + ` + params = [bucketId, limit] + } else { + // Fallback: get all pending items (for backwards compatibility) + query = ` + SELECT * FROM data_sync_queue + WHERE status = 'pending' AND retry_count < max_retries + ORDER BY created_at ASC + LIMIT ? + ` + params = [limit] + } + + const rows = this.db.prepare(query).all(...params) as any[] + + return rows.map(row => ({ + ...row, + data: row.data ? JSON.parse(row.data) : null + })) + } + + async markSyncProcessing(queueId: number): Promise { + if (!this.db) throw new Error('Database not initialized') + + this.db + .prepare( + ` + UPDATE data_sync_queue + SET status = 'processing', started_at = ? + WHERE id = ? + ` + ) + .run(new Date().toISOString(), queueId) + } + + async markSyncComplete(queueId: number): Promise { + if (!this.db) throw new Error('Database not initialized') + + this.db + .prepare( + ` + UPDATE data_sync_queue + SET status = 'completed', completed_at = ? + WHERE id = ? + ` + ) + .run(new Date().toISOString(), queueId) + } + + async markSyncError(queueId: number, error: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + this.db + .prepare( + ` + UPDATE data_sync_queue + SET status = 'failed', retry_count = retry_count + 1 + WHERE id = ? + ` + ) + .run(queueId) + } + + // ======================================== + // Asset/Media Management + // ======================================== + + async createAsset(bucketId: string, asset: any): Promise { + if (!this.db) throw new Error('Database not initialized') + + const now = new Date().toISOString() + const stmt = this.db.prepare(` + INSERT INTO assets ( + id, bucket_id, type, key, url, filename, size, mime_type, + metadata, uploaded_at, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `) + + stmt.run( + asset.id, + bucketId, + asset.type, + asset.key, + asset.url, + asset.filename || null, + asset.size || null, + asset.mimeType || null, + asset.metadata ? JSON.stringify(asset.metadata) : null, + asset.uploadedAt || now, + now, + now + ) + + return this.getAsset(asset.id) + } + + async getAsset(id: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + const row = this.db.prepare('SELECT * FROM assets WHERE id = ?').get(id) as any + if (!row) return null + + return this.parseAsset(row) + } + + async listAssets(bucketId: string, type?: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + let query = 'SELECT * FROM assets WHERE bucket_id = ?' + const params: any[] = [bucketId] + + if (type) { + query += ' AND type = ?' + params.push(type) + } + + query += ' ORDER BY uploaded_at DESC' + + const rows = this.db.prepare(query).all(...params) as any[] + return rows.map(row => this.parseAsset(row)) + } + + async deleteAsset(id: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + this.db.prepare('DELETE FROM assets WHERE id = ?').run(id) + } + + private parseAsset(row: any): any { + return { + ...row, + metadata: row.metadata ? JSON.parse(row.metadata) : {}, + // Convert snake_case to camelCase + bucketId: row.bucket_id, + mimeType: row.mime_type, + uploadedAt: row.uploaded_at, + createdAt: row.created_at, + updatedAt: row.updated_at + } + } + + // ======================================== + // Utility Operations + // ======================================== + + async getStats(): Promise<{ + collections: number + syncQueueSize: number + dbSize: number + }> { + if (!this.db) throw new Error('Database not initialized') + + const collections = (this.db.prepare('SELECT COUNT(*) as count FROM collections').get() as any) + .count + const syncQueueSize = ( + this.db + .prepare("SELECT COUNT(*) as count FROM data_sync_queue WHERE status = 'pending'") + .get() as any + ).count + const dbSize = fs.existsSync(this.dbPath) ? fs.statSync(this.dbPath).size : 0 + + return { collections, syncQueueSize, dbSize } + } + + /** + * Clear all data for a specific bucket + * This allows proper bucket isolation when reimporting data + */ + async clearBucketData(bucketId: string): Promise { + if (!this.db) throw new Error('Database not initialized') + + console.log('[DatabaseService] Clearing data for bucket:', bucketId) + + // Use transaction to ensure atomicity + const transaction = this.db.transaction(() => { + // Delete bucket-specific data in the correct order (respecting foreign keys) + // 1. Delete entry order records for collections in this bucket + this.db!.prepare( + ` + DELETE FROM collection_entry_order + WHERE collection_id IN (SELECT id FROM collections WHERE bucket_id = ?) + ` + ).run(bucketId) + + // 2. Delete entries for this bucket + this.db!.prepare('DELETE FROM entries WHERE bucket_id = ?').run(bucketId) + + // 3. Delete collection order records for this bucket + this.db!.prepare('DELETE FROM bucket_collection_order WHERE bucket_id = ?').run(bucketId) + + // 4. Delete collections for this bucket + this.db!.prepare('DELETE FROM collections WHERE bucket_id = ?').run(bucketId) + + // 5. Delete assets for this bucket + this.db!.prepare('DELETE FROM assets WHERE bucket_id = ?').run(bucketId) + + // 6. Delete sync queue items for this bucket + this.db!.prepare('DELETE FROM data_sync_queue WHERE bucket_id = ?').run(bucketId) + }) + + transaction() + console.log('[DatabaseService] โœ“ Bucket data cleared successfully') + } + + async clearDatabase(): Promise { + console.log('[DatabaseService] Clearing database...') + + // Close the database connection + if (this.db) { + this.db.close() + this.db = null + } + + // Delete the database file + if (fs.existsSync(this.dbPath)) { + fs.unlinkSync(this.dbPath) + console.log('[DatabaseService] Database file deleted') + } + + // Delete WAL and SHM files if they exist + const walPath = `${this.dbPath}-wal` + const shmPath = `${this.dbPath}-shm` + if (fs.existsSync(walPath)) { + fs.unlinkSync(walPath) + console.log('[DatabaseService] WAL file deleted') + } + if (fs.existsSync(shmPath)) { + fs.unlinkSync(shmPath) + console.log('[DatabaseService] SHM file deleted') + } + + // Reinitialize the database + await this.initialize() + console.log('[DatabaseService] Database cleared and reinitialized') + } + + getDatabasePath(): string { + return this.dbPath + } + + close(): void { + if (this.db) { + console.log('[DatabaseService] Closing database') + this.db.close() + this.db = null + } + } +} diff --git a/src/main/services/mediaService.ts b/src/main/services/mediaService.ts index 2ccccd7..fe620d5 100644 --- a/src/main/services/mediaService.ts +++ b/src/main/services/mediaService.ts @@ -40,6 +40,8 @@ class MediaService { private client: any = null private config: any = null private directories: any = null + private databaseService: any = null + private activeBucketId: string | null = null configure(client: any, config: any) { this.client = client @@ -50,6 +52,12 @@ class MediaService { this.directories = directories } + setDatabaseService(databaseService: any, bucketId: string): void { + this.databaseService = databaseService + this.activeBucketId = bucketId + console.log(`[MediaService] โœ“ Database service configured for bucket: ${bucketId}`) + } + private getDirectories() { // Return configured directories or defaults return ( @@ -64,6 +72,49 @@ class MediaService { } async listAssets(bucket: string): Promise { + if (!this.databaseService || !this.activeBucketId) { + console.warn('[MediaService] Database service not configured, falling back to S3 scan') + return this.listAssetsFromS3(bucket) + } + + try { + console.log('[MediaService] Loading assets from local database') + + // Get all assets from database + const dbAssets = await this.databaseService.listAssets(this.activeBucketId) + + // Convert database assets to Asset format + const assets: Asset[] = dbAssets.map((dbAsset: any) => ({ + type: dbAsset.type, + url: dbAsset.url, + filename: dbAsset.filename, + key: dbAsset.key, + size: dbAsset.size, + width: dbAsset.metadata?.width, + height: dbAsset.metadata?.height, + uploadedAt: dbAsset.uploadedAt, + mimeType: dbAsset.mimeType, + duration: dbAsset.metadata?.duration, + variants: dbAsset.metadata?.variants || [], + provider: dbAsset.metadata?.provider, + embedUrl: dbAsset.metadata?.embedUrl + })) + + // Track usage of all assets from entries in database + await this.trackAssetUsageFromDB(assets) + + console.log(`[MediaService] Loaded ${assets.length} assets from database`) + return assets + } catch (error) { + console.error('[MediaService] Error listing assets from database:', error) + // Fallback to S3 scan on error + console.warn('[MediaService] Falling back to S3 scan') + return this.listAssetsFromS3(bucket) + } + } + + // Legacy S3 scanning method (fallback only) + private async listAssetsFromS3(bucket: string): Promise { if (!this.client) { throw new Error('Media service not configured') } @@ -398,6 +449,195 @@ class MediaService { return Array.from(embedsMap.values()) } + private async trackAssetUsageFromDB(assets: Asset[]): Promise { + if (!this.databaseService || !this.activeBucketId) { + console.warn('[MediaService] Database service not configured for tracking asset usage') + return + } + + try { + // Load all collections from database + const collections = await this.databaseService.listCollections(this.activeBucketId) + + // Track collection primary images + for (const collection of collections) { + if (collection.metadata?.primaryImage) { + const imageUrl = collection.metadata.primaryImage + let asset = assets.find(a => a.url === imageUrl) + + // Try to match by filename if exact URL doesn't match + if (!asset) { + const filename = imageUrl.split('/').pop() + if (filename) { + // Extract base filename without size prefixes + const baseFilename = filename.replace(/^(thumbnail|small|medium|large|xlarge)-/, '') + asset = assets.find(a => a.filename === baseFilename || a.url.includes(baseFilename)) + } + } + + if (asset) { + asset.usedIn = asset.usedIn || [] + // Check if already tracked to avoid duplicates + const alreadyTracked = asset.usedIn.some( + u => u.collectionId === collection.id && u.entryId === 'collection' + ) + if (!alreadyTracked) { + asset.usedIn.push({ + collectionId: collection.id, + collectionName: collection.name, + entryId: 'collection', + entryTitle: `Collection: ${collection.name}` + }) + } + } + } + } + + // Load all entries from database by iterating through collections + const allEntries: any[] = [] + for (const collection of collections) { + const entries = await this.databaseService.listEntries(collection.id) + allEntries.push(...entries) + } + + console.log(`[MediaService] Loaded ${allEntries.length} entries for usage tracking`) + + // Track asset usage in entries + for (const entry of allEntries) { + if (!entry.data) continue + + // Scan entry data for image URLs and embeds + const entryData = typeof entry.data === 'string' ? JSON.parse(entry.data) : entry.data + this.scanObjectForAssetUsage(entryData, assets, entry, collections) + } + } catch (error) { + console.error('[MediaService] Error tracking asset usage from database:', error) + } + } + + private scanObjectForAssetUsage(obj: any, assets: Asset[], entry: any, collections: any[]): void { + if (!obj || typeof obj !== 'object') return + + // Find the collection for this entry + const collection = collections.find(c => c.id === entry.collectionId) + const collectionName = collection?.name || 'Unknown Collection' + const entryTitle = entry.metadata?.title || entry.title || 'Untitled Entry' + + // Helper to track image usage with duplicate prevention + const trackImageUsage = (imageUrl: string) => { + // Try exact match first + let asset = assets.find(a => a.url === imageUrl) + + // If not found, try to match by filename (for variants) + if (!asset) { + const filename = imageUrl.split('/').pop() + if (filename) { + // Extract base filename without size prefixes + const baseFilename = filename.replace(/^(thumbnail|small|medium|large|xlarge)-/, '') + asset = assets.find(a => a.filename === baseFilename || a.url.includes(baseFilename)) + } + } + + if (asset) { + asset.usedIn = asset.usedIn || [] + // Check if already tracked to avoid duplicates + const alreadyTracked = asset.usedIn.some( + u => u.collectionId === entry.collectionId && u.entryId === entry.id + ) + if (!alreadyTracked) { + asset.usedIn.push({ + collectionId: entry.collectionId, + collectionName, + entryId: entry.id, + entryTitle + }) + } + } + } + + // Helper to track audio usage with duplicate prevention + const trackAudioUsage = (audioUrl: string) => { + // Try exact match first + let asset = assets.find(a => a.url === audioUrl && a.type === 'audio') + + // If not found, try to match by filename + if (!asset) { + const filename = audioUrl.split('/').pop() + if (filename) { + asset = assets.find( + a => a.type === 'audio' && (a.filename === filename || a.url.includes(filename)) + ) + } + } + + if (asset) { + asset.usedIn = asset.usedIn || [] + const alreadyTracked = asset.usedIn.some( + u => u.collectionId === entry.collectionId && u.entryId === entry.id + ) + if (!alreadyTracked) { + asset.usedIn.push({ + collectionId: entry.collectionId, + collectionName, + entryId: entry.id, + entryTitle + }) + } + } + } + + // Helper to track embed usage with duplicate prevention + const trackEmbedUsage = (embedUrl: string) => { + const asset = assets.find(a => a.embedUrl === embedUrl) + if (asset) { + asset.usedIn = asset.usedIn || [] + const alreadyTracked = asset.usedIn.some( + u => u.collectionId === entry.collectionId && u.entryId === entry.id + ) + if (!alreadyTracked) { + asset.usedIn.push({ + collectionId: entry.collectionId, + collectionName, + entryId: entry.id, + entryTitle + }) + } + } + } + + for (const [key, value] of Object.entries(obj)) { + if (typeof value === 'string') { + // Check for image URLs + if ( + value.startsWith('http') && + (value.includes('amazonaws.com') || value.match(/\.(jpg|jpeg|png|gif|webp|svg)$/i)) + ) { + trackImageUsage(value) + } + + // Check for audio URLs + if ( + value.startsWith('http') && + (value.includes('amazonaws.com') || value.match(/\.(mp3|wav|ogg|m4a|aac)$/i)) + ) { + trackAudioUsage(value) + } + + // Check for embed URLs (YouTube, Vimeo, etc.) + if ( + value.includes('youtube.com') || + value.includes('youtu.be') || + value.includes('vimeo.com') + ) { + trackEmbedUsage(value) + } + } else if (typeof value === 'object' && value !== null) { + // Recursively scan nested objects + this.scanObjectForAssetUsage(value, assets, entry, collections) + } + } + } + private async trackAssetUsage(bucket: string, assets: Asset[]): Promise { try { // Load all collections for name lookup diff --git a/src/main/services/migrationService.ts b/src/main/services/migrationService.ts new file mode 100644 index 0000000..2e3d7b2 --- /dev/null +++ b/src/main/services/migrationService.ts @@ -0,0 +1,391 @@ +/** + * MigrationService - Import existing S3 data into local database + */ + +import { DatabaseService } from './databaseService' +import { SyncService } from './syncService' +import { EventEmitter } from 'events' + +export interface MigrationProgress { + status: 'idle' | 'running' | 'completed' | 'error' + currentStep: string + totalItems: number + processedItems: number + errors: string[] +} + +export class MigrationService extends EventEmitter { + private databaseService: DatabaseService + private contentService: any + private syncService: SyncService + + constructor(databaseService: DatabaseService, contentService: any, syncService: SyncService) { + super() + this.databaseService = databaseService + this.contentService = contentService + this.syncService = syncService + } + + async migrate(bucketId: string, s3BucketName: string): Promise { + const progress: MigrationProgress = { + status: 'running', + currentStep: 'Starting migration', + totalItems: 0, + processedItems: 0, + errors: [] + } + + console.log('[MigrationService] ========================================') + console.log('[MigrationService] Starting migration for bucket:', bucketId) + console.log('[MigrationService] S3 bucket name:', s3BucketName) + console.log('[MigrationService] ========================================') + this.emit('progress', progress) + + // Pause sync service during migration to prevent interference + this.syncService.pause() + + try { + // Step 0: Clear existing data for this bucket to prevent mixing + progress.currentStep = 'Clearing existing data for bucket' + this.emit('progress', progress) + + console.log('[MigrationService] Clearing existing data for bucket:', bucketId) + await this.clearBucketData(bucketId) + console.log('[MigrationService] โœ“ Existing data cleared') + + // Step 1: Migrate collections + progress.currentStep = 'Loading collections from S3' + this.emit('progress', progress) + + // ContentService needs the S3 bucket name (not our internal bucket ID) + const collectionsResult = await this.contentService.listCollections(s3BucketName) + const collections = collectionsResult || [] + + progress.totalItems = collections.length + console.log(`[MigrationService] Found ${collections.length} collections to migrate from S3`) + + for (const collection of collections) { + try { + console.log( + `[MigrationService] Migrating collection: "${collection.name}" (id: ${collection.id}) with bucket_id: ${bucketId}` + ) + + // Convert collection format to match database schema + await this.databaseService.createCollection({ + id: collection.id, + bucket_id: bucketId, + name: collection.name, + description: collection.description || '', + schema_id: collection.schemaId, + schema_name: collection.schemaName, + published: collection.published || false, + published_at: collection.publishedAt, + metadata: collection.metadata || {} + }) + + console.log(`[MigrationService] โœ“ Collection "${collection.name}" migrated successfully`) + + progress.processedItems++ + this.emit('progress', progress) + } catch (error: any) { + console.error('[MigrationService] Error migrating collection:', error) + progress.errors.push(`Collection ${collection.name}: ${error.message}`) + } + } + + // Step 2: Migrate entries + progress.currentStep = 'Loading entries from S3' + let totalEntries = 0 + + for (const collection of collections) { + try { + // ContentService.listContentEntries needs S3 bucket name and collection ID + const entriesResult = await this.contentService.listContentEntries( + s3BucketName, + collection.id + ) + const entries = entriesResult || [] + totalEntries += entries.length + + progress.totalItems += entries.length + console.log( + `[MigrationService] Found ${entries.length} entries in collection ${collection.name}` + ) + + // Track successfully created entry IDs + const createdEntryIds = new Set() + + for (const entry of entries) { + try { + console.log( + `[MigrationService] - Entry "${entry.metadata?.title || entry.id}" (id: ${entry.id}) with bucket_id: ${bucketId}` + ) + + // Convert entry format to match database schema + // S3 structure has: { data: {...}, metadata: {...}, tags: [...], ... } + // Wrap it in our new structure: { data: {...fields...}, metadata: {...}, tags: [...] } + await this.databaseService.createEntry({ + id: entry.id, + bucket_id: bucketId, + collection_id: collection.id, + schema_id: entry.schemaId, + schema_name: entry.schemaName, + data: { + data: entry.data || {}, + metadata: entry.metadata || {}, + tags: entry.tags || [] + }, + published: entry.published || false, + published_at: entry.publishedAt + }) + + createdEntryIds.add(entry.id) + progress.processedItems++ + this.emit('progress', progress) + } catch (error: any) { + console.error('[MigrationService] Error migrating entry:', error) + progress.errors.push(`Entry ${entry.id}: ${error.message}`) + } + } + + // Migrate entry order if it exists + // Only include entries that were successfully created + if (collection.entryOrder && collection.entryOrder.length > 0) { + const validEntryOrder = collection.entryOrder.filter((id: string) => + createdEntryIds.has(id) + ) + if (validEntryOrder.length > 0) { + console.log( + `[MigrationService] Migrating entry order for ${collection.name} (${validEntryOrder.length}/${collection.entryOrder.length} entries)` + ) + await this.databaseService.updateEntryOrder(collection.id, validEntryOrder) + } else { + console.log( + `[MigrationService] Skipping entry order for ${collection.name} (no valid entries)` + ) + } + } + } catch (error: any) { + console.error('[MigrationService] Error loading entries for collection:', error) + progress.errors.push(`Entries for ${collection.name}: ${error.message}`) + } + } + + // Step 3: Migrate collection order from S3 collections index + try { + progress.currentStep = 'Loading collection order from S3' + this.emit('progress', progress) + + // Fetch the collections index to get the collectionOrder + const { GetObjectCommand } = require('@aws-sdk/client-s3') + const directories = this.contentService.getDirectories() + const collectionsApiDir = directories.collections + const indexKey = `${directories.api}/${collectionsApiDir}/index.json` + + const getIndexCommand = new GetObjectCommand({ + Bucket: s3BucketName, + Key: indexKey + }) + + const indexResult = await this.contentService.client.send(getIndexCommand) + const indexBody = await indexResult.Body?.transformToString() + + if (indexBody) { + const indexContent = JSON.parse(indexBody) + if (indexContent.collectionOrder && Array.isArray(indexContent.collectionOrder)) { + console.log( + `[MigrationService] Migrating collection order: ${indexContent.collectionOrder.length} collections` + ) + await this.databaseService.updateCollectionOrder(bucketId, indexContent.collectionOrder) + } else { + console.log('[MigrationService] No collection order found in collections index') + } + } + } catch (error: any) { + console.warn('[MigrationService] Could not migrate collection order:', error.message) + // Don't fail the entire migration if collection order can't be migrated + } + + // Step 4: Migrate assets from S3 + try { + progress.currentStep = 'Migrating assets from S3' + this.emit('progress', progress) + + console.log(`[MigrationService] Starting asset migration from bucket: ${s3BucketName}`) + const assetsImported = await this.migrateAssets(bucketId, s3BucketName) + console.log(`[MigrationService] โœ“ Migrated ${assetsImported} assets`) + } catch (error: any) { + console.error('[MigrationService] Asset migration error:', error) + console.warn('[MigrationService] Could not migrate assets:', error.message) + // Don't fail the entire migration if assets can't be migrated + } + + progress.status = 'completed' + progress.currentStep = `Migration completed: ${collections.length} collections, ${totalEntries} entries` + console.log('[MigrationService]', progress.currentStep) + console.log('[MigrationService] Errors:', progress.errors.length) + this.emit('progress', progress) + + // Resume sync service after successful migration + this.syncService.resume() + console.log('[MigrationService] ========================================') + } catch (error: any) { + progress.status = 'error' + progress.errors.push(`Fatal error: ${error.message}`) + console.error('[MigrationService] Fatal error:', error) + this.emit('progress', progress) + + // Resume sync service even on error + this.syncService.resume() + + throw error + } + } + + private async migrateAssets(bucketId: string, s3BucketName: string): Promise { + let assetsImported = 0 + const { ListObjectsV2Command, HeadObjectCommand } = require('@aws-sdk/client-s3') + const directories = this.contentService.getDirectories() + + console.log('[MigrationService] Scanning for assets in directories:', directories) + + // Import images from assets/originals/ + try { + const imagesPrefix = `${directories.assets}/originals/` + console.log(`[MigrationService] Looking for images in: ${imagesPrefix}`) + const listImagesCmd = new ListObjectsV2Command({ + Bucket: s3BucketName, + Prefix: imagesPrefix + }) + + const imagesResult = await this.contentService.client.send(listImagesCmd) + console.log(`[MigrationService] Found ${imagesResult.Contents?.length || 0} image files`) + + if (imagesResult.Contents) { + for (const item of imagesResult.Contents) { + if (!item.Key || !this.isImageFile(item.Key)) continue + + try { + const headCmd = new HeadObjectCommand({ + Bucket: s3BucketName, + Key: item.Key + }) + const headResult = await this.contentService.client.send(headCmd) + + const asset = { + id: this.generateAssetId(item.Key), + type: 'image', + key: item.Key, + url: `https://${s3BucketName}.s3.${process.env.AWS_REGION || 'us-east-1'}.amazonaws.com/${item.Key}`, + filename: item.Key.split('/').pop(), + size: item.Size, + mimeType: headResult.ContentType, + metadata: { + width: headResult.Metadata?.width ? parseInt(headResult.Metadata.width) : null, + height: headResult.Metadata?.height ? parseInt(headResult.Metadata.height) : null + }, + uploadedAt: item.LastModified?.toISOString() + } + + await this.databaseService.createAsset(bucketId, asset) + assetsImported++ + } catch (error) { + console.warn(`[MigrationService] Could not import image ${item.Key}:`, error) + } + } + } + } catch (error) { + console.warn('[MigrationService] Could not list images from S3:', error) + } + + // Import audio from assets/audio/ + try { + const audioPrefix = `${directories.assets}/audio/` + console.log(`[MigrationService] Looking for audio in: ${audioPrefix}`) + const listAudioCmd = new ListObjectsV2Command({ + Bucket: s3BucketName, + Prefix: audioPrefix + }) + + const audioResult = await this.contentService.client.send(listAudioCmd) + console.log(`[MigrationService] Found ${audioResult.Contents?.length || 0} audio files`) + + if (audioResult.Contents) { + for (const item of audioResult.Contents) { + if (!item.Key || !this.isAudioFile(item.Key)) continue + + try { + const headCmd = new HeadObjectCommand({ + Bucket: s3BucketName, + Key: item.Key + }) + const headResult = await this.contentService.client.send(headCmd) + + const asset = { + id: this.generateAssetId(item.Key), + type: 'audio', + key: item.Key, + url: `https://${s3BucketName}.s3.${process.env.AWS_REGION || 'us-east-1'}.amazonaws.com/${item.Key}`, + filename: item.Key.split('/').pop(), + size: item.Size, + mimeType: headResult.ContentType, + metadata: {}, + uploadedAt: item.LastModified?.toISOString() + } + + await this.databaseService.createAsset(bucketId, asset) + assetsImported++ + } catch (error) { + console.warn(`[MigrationService] Could not import audio ${item.Key}:`, error) + } + } + } + } catch (error) { + console.warn('[MigrationService] Could not list audio from S3:', error) + } + + return assetsImported + } + + private isImageFile(key: string): boolean { + const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg'] + return imageExtensions.some(ext => key.toLowerCase().endsWith(ext)) + } + + private isAudioFile(key: string): boolean { + const audioExtensions = ['.mp3', '.wav', '.ogg', '.m4a', '.flac', '.aac'] + return audioExtensions.some(ext => key.toLowerCase().endsWith(ext)) + } + + private generateAssetId(key: string): string { + // Generate a simple ID from the key + return key.replace(/[^a-zA-Z0-9]/g, '-').toLowerCase() + } + + /** + * Clear all data for a specific bucket before migration + * This prevents data mixing when reimporting or switching buckets + */ + private async clearBucketData(bucketId: string): Promise { + console.log('[MigrationService] Clearing data for bucket:', bucketId) + + // Clear only data for this specific bucket, preserving data from other buckets + await this.databaseService.clearBucketData(bucketId) + + console.log('[MigrationService] โœ“ Bucket data cleared (ready for clean import)') + } + + async needsMigration(bucketId: string): Promise { + // Check if database has data for this bucket + const collections = await this.databaseService.listCollections(bucketId) + const needsMigration = collections.length === 0 + console.log( + '[MigrationService] Needs migration:', + needsMigration, + '(collections in DB:', + collections.length, + ')' + ) + return needsMigration + } +} diff --git a/src/main/services/publishingService.ts b/src/main/services/publishingService.ts new file mode 100644 index 0000000..d537767 --- /dev/null +++ b/src/main/services/publishingService.ts @@ -0,0 +1,836 @@ +import { DatabaseService } from './databaseService' +import { ContentCollection, ContentEntry } from '../../types/content' +import { PutObjectCommand as PutObjectCommandLib } from '@aws-sdk/client-s3' + +/** + * Publishing Service + * + * Handles the publishing workflow by: + * 1. Reading collection and entry data from the local SQLite database + * 2. Generating API endpoint JSON files + * 3. Uploading them to S3 + * + * This service separates concerns: + * - DatabaseService: local data operations + * - ContentService: S3 write operations (for uploading generated JSON) + * - PublishingService: orchestrates the publishing workflow + */ +export class PublishingService { + private databaseService: DatabaseService + private s3Client: any = null + private config: any = null + private syncPrefix: string = 'sync' // Prefix for testing S3 sync operations + + constructor(databaseService: DatabaseService) { + this.databaseService = databaseService + } + + /** + * Configure the S3 client for publishing + */ + configure(client: any, config: any): void { + this.s3Client = client + this.config = config + } + + /** + * Set the sync prefix for S3 operations (for testing) + * @param prefix - Directory prefix (e.g., 'sync' will upload to /sync/api/...) + */ + setSyncPrefix(prefix: string): void { + this.syncPrefix = prefix + } + + /** + * Sync a single source file to S3 (called by SyncService) + * This is for incremental backups, separate from full publishing + */ + async syncSourceFile( + bucket: string, + entityType: 'collection' | 'entry', + entityId: string, + data: any + ): Promise { + if (!this.s3Client) { + throw new Error('Publishing service not configured') + } + + const directories = this.getDirectories() + + if (entityType === 'collection') { + // Build collection source file + const collectionPath = this.syncPrefix + ? `${this.syncPrefix}/${directories.collections}` + : directories.collections + + const collectionKey = `${collectionPath}/${entityId}.json` + + const collectionData = { + id: data.id, + name: data.name, + description: data.description, + schemaId: data.schema_id, + schemaName: data.schema_name, + published: data.published, + publishedAt: data.published_at, + createdAt: data.created_at, + updatedAt: data.updated_at, + metadata: typeof data.metadata === 'string' ? JSON.parse(data.metadata) : data.metadata + } + + await this.uploadToS3(bucket, collectionKey, collectionData) + console.log(`[PublishingService] โœ“ Synced collection source: ${collectionKey}`) + } else if (entityType === 'entry') { + // Build entry source file + const entriesPath = this.syncPrefix + ? `${this.syncPrefix}/${directories.entries}` + : directories.entries + + const entryKey = `${entriesPath}/${entityId}.json` + + const entryData = { + id: data.id, + collectionId: data.collection_id, + schemaId: data.schema_id, + schemaName: data.schema_name, + data: typeof data.data === 'string' ? JSON.parse(data.data) : data.data, + metadata: + typeof data.data === 'string' + ? JSON.parse(data.data).metadata || {} + : data.data?.metadata || {}, + tags: + typeof data.data === 'string' ? JSON.parse(data.data).tags || [] : data.data?.tags || [], + published: data.published, + publishedAt: data.published_at, + createdAt: data.created_at, + updatedAt: data.updated_at + } + + await this.uploadToS3(bucket, entryKey, entryData) + console.log(`[PublishingService] โœ“ Synced entry source: ${entryKey}`) + } + } + + /** + * Delete a source file from S3 (called by SyncService) + */ + async deleteSourceFile( + bucket: string, + entityType: 'collection' | 'entry', + entityId: string + ): Promise { + if (!this.s3Client) { + throw new Error('Publishing service not configured') + } + + const directories = this.getDirectories() + const dir = entityType === 'collection' ? directories.collections : directories.entries + const path = this.syncPrefix ? `${this.syncPrefix}/${dir}` : dir + const key = `${path}/${entityId}.json` + + const { DeleteObjectCommand } = require('@aws-sdk/client-s3') + const command = new DeleteObjectCommand({ + Bucket: bucket, + Key: key + }) + + await this.s3Client.send(command) + console.log(`[PublishingService] โœ“ Deleted source file: ${key}`) + } + + /** + * Get directory configuration from active bucket config + * Applies sync prefix for testing (only to the API root, not subdirectories) + */ + private getDirectories() { + const baseDirectories = { + api: this.config?.directories?.api || 'api', + collections: this.config?.directories?.collections || 'collections', + entries: this.config?.directories?.entries || 'entries' + } + + // Apply sync prefix for testing - only to the API root path + // collections/entries remain as subdirectories within the API path + if (this.syncPrefix) { + return { + api: `${this.syncPrefix}/${baseDirectories.api}`, + collections: baseDirectories.collections, // Used within api path + entries: baseDirectories.entries // Used within api path + } + } + + return baseDirectories + } + + /** + * Transform URLs in data to use CloudFront if configured + */ + private async transformUrlsToCloudFront(data: any, bucket: string): Promise { + if (!data || typeof data !== 'object') { + return data + } + + const cloudFrontUrl = this.config?.cloudFrontUrl + + if (!cloudFrontUrl) { + return data + } + + // Remove protocol and trailing slash to get clean domain + const cloudFrontDomain = cloudFrontUrl.replace(/^https?:\/\//, '').replace(/\/$/, '') + + // Match S3 URLs with or without region + const s3UrlPattern = new RegExp(`https://${bucket}\\.s3\\.[^/]+\\.amazonaws\\.com/`, 'g') + const s3UrlPatternAlt = new RegExp(`https://${bucket}\\.s3\\.amazonaws\\.com/`, 'g') + const s3UrlPatternAlt2 = new RegExp(`https://s3\\.amazonaws\\.com/${bucket}/`, 'g') + + const transformValue = (value: any): any => { + if (typeof value === 'string') { + return value + .replace(s3UrlPattern, `https://${cloudFrontDomain}/`) + .replace(s3UrlPatternAlt, `https://${cloudFrontDomain}/`) + .replace(s3UrlPatternAlt2, `https://${cloudFrontDomain}/`) + } else if (Array.isArray(value)) { + return value.map(transformValue) + } else if (typeof value === 'object' && value !== null) { + const transformed: any = {} + for (const key in value) { + transformed[key] = transformValue(value[key]) + } + return transformed + } + return value + } + + return transformValue(data) + } + + /** + * Upload JSON data to S3 + */ + private async uploadToS3(bucket: string, key: string, data: any): Promise { + if (!this.s3Client) { + throw new Error('S3 client not configured') + } + + // Log upload with sync prefix indicator + if (this.syncPrefix) { + console.log(`[Publishing] ๐Ÿ“ค Uploading to S3 (TEST MODE): ${key}`) + } + + const command = new PutObjectCommandLib({ + Bucket: bucket, + Key: key, + Body: JSON.stringify(data, null, 2), + ContentType: 'application/json' + }) + + await this.s3Client.send(command) + } + + /** + * Load collections with their entries from the database + * Converts database types to UI-friendly ContentCollection format + */ + private async loadCollectionsWithEntries(bucketId: string): Promise { + // Get all collections from database (returns Collection[] with snake_case) + const dbCollections = await this.databaseService.listCollections(bucketId) + + // Convert each collection and load its entries + const collections: ContentCollection[] = [] + for (const dbCollection of dbCollections) { + // Get entries for this collection (already sorted by entry order) + const dbEntries = await this.databaseService.listEntries(dbCollection.id) + + // Get entry order + const entryOrder = await this.databaseService.getEntryOrder(dbCollection.id) + + // Convert to ContentCollection format (camelCase) + const collection: ContentCollection = { + id: dbCollection.id, + name: dbCollection.name, + description: dbCollection.description, + schemaId: dbCollection.schema_id, + schemaName: dbCollection.schema_name, + published: dbCollection.published, + publishedAt: dbCollection.published_at, + createdAt: dbCollection.created_at, + updatedAt: dbCollection.updated_at, + metadata: dbCollection.metadata || { + title: dbCollection.name, + description: dbCollection.description || '', + slug: dbCollection.name.toLowerCase().replace(/\s+/g, '-'), + primaryImage: '' + }, + // dbEntries have title/slug/description at root level (from parseEntry) + // parseEntry() returns camelCase properties at runtime, despite type definition + // Convert to ContentEntry format + entries: dbEntries.map( + (e: any) => + ({ + id: e.id, + schemaId: e.schemaId, + schemaName: e.schemaName, + data: e.data, + createdAt: e.createdAt, + updatedAt: e.updatedAt, + published: e.published, + publishedAt: e.publishedAt, + tags: e.tags || [], + metadata: e.metadata || {} + }) as ContentEntry + ), + // Entry order from database + entryOrder: entryOrder + } + + collections.push(collection) + } + + return collections + } + + /** + * Save individual collection JSON file for migration compatibility + * Saves to: collections/{collectionId}.json or sync/collections/{collectionId}.json + */ + private async saveCollectionSource(bucket: string, collection: ContentCollection): Promise { + const directories = this.getDirectories() + const collectionsDir = directories.collections + + // Apply sync prefix to collections directory if enabled + const collectionsPath = this.syncPrefix + ? `${this.syncPrefix}/${collectionsDir}` + : collectionsDir + + const collectionKey = `${collectionsPath}/${collection.id}.json` + + const collectionData = { + id: collection.id, + name: collection.name, + description: collection.description, + schemaId: collection.schemaId, + schemaName: collection.schemaName, + published: collection.published, + publishedAt: collection.publishedAt, + createdAt: collection.createdAt, + updatedAt: collection.updatedAt, + metadata: collection.metadata + } + + await this.uploadToS3(bucket, collectionKey, collectionData) + console.log(`[Publishing] โœ“ Saved collection source: ${collectionKey}`) + } + + /** + * Save individual entry JSON files for migration compatibility + * Saves to: entries/{entryId}.json or sync/entries/{entryId}.json + */ + private async saveEntrySources( + bucket: string, + collectionId: string, + entries: ContentEntry[] + ): Promise { + const directories = this.getDirectories() + const entriesDir = directories.entries + + // Apply sync prefix to entries directory if enabled + const entriesPath = this.syncPrefix ? `${this.syncPrefix}/${entriesDir}` : entriesDir + + for (const entry of entries) { + const entryKey = `${entriesPath}/${entry.id}.json` + + const entryData = { + id: entry.id, + collectionId: collectionId, + schemaId: entry.schemaId, + schemaName: entry.schemaName, + data: await this.transformUrlsToCloudFront(entry.data, bucket), + metadata: entry.metadata, + tags: entry.tags || [], + published: entry.published, + publishedAt: entry.publishedAt, + createdAt: entry.createdAt, + updatedAt: entry.updatedAt + } + + await this.uploadToS3(bucket, entryKey, entryData) + } + + console.log(`[Publishing] โœ“ Saved ${entries.length} entry source files`) + } + + /** + * Generate API endpoints for a single collection + * Also saves source collection and entry JSON files for migration compatibility + */ + async generateApiEndpoints( + bucket: string, + bucketId: string, + collection: ContentCollection + ): Promise { + if (!this.s3Client) { + throw new Error('Publishing service not configured') + } + + const directories = this.getDirectories() + const apiBasePath = directories.api + + // Always save source files for migration compatibility (even for unpublished collections) + await this.saveCollectionSource(bucket, collection) + await this.saveEntrySources(bucket, collection.id, collection.entries) + + // Only generate API endpoints for published collections + if (!collection.published) { + console.log( + `[Publishing] Saved source files but skipping API generation for unpublished collection: ${collection.name}` + ) + return + } + + // Filter to only published entries for API generation + const publishedEntries = collection.entries.filter(entry => entry.published) + + if (process.env.NODE_ENV === 'development') { + console.log(`\n[Publishing] === API Generation for: ${collection.name} ===`) + console.log('[Publishing] Collection entryOrder:', collection.entryOrder) + console.log(`[Publishing] Total entries: ${collection.entries.length}`) + console.log( + '[Publishing] Entries in current order:', + collection.entries.map(e => ({ + id: e.id, + title: e.metadata?.title || e.id, + published: e.published + })) + ) + console.log( + `[Publishing] Published entries (${publishedEntries.length}):`, + publishedEntries.map(e => ({ + id: e.id, + title: e.metadata?.title || e.id + })) + ) + console.log('[Publishing] === End API Generation ===\n') + } + + // Transform entry data URLs to CloudFront if configured + const transformedEntries = await Promise.all( + publishedEntries.map(async entry => ({ + id: entry.id, + title: entry.metadata?.title || `Entry ${entry.id}`, + description: entry.metadata?.description, + slug: entry.metadata?.slug, + published: entry.published, + publishedAt: entry.publishedAt, + tags: entry.tags, + createdAt: entry.createdAt, + updatedAt: entry.updatedAt, + data: await this.transformUrlsToCloudFront(entry.data, bucket) + })) + ) + + // 1. Generate collection endpoint: /api/{collections-dir}/{slug}.json + const collectionEndpoint = { + id: collection.id, + name: collection.name, + description: collection.description, + slug: collection.metadata.slug, + schemaId: collection.schemaId, + metadata: collection.metadata, + published: collection.published, + publishedAt: collection.publishedAt, + entryCount: publishedEntries.length, + createdAt: collection.createdAt, + updatedAt: collection.updatedAt, + entries: transformedEntries + } + + // Transform URLs to CloudFront if configured + const transformedCollectionEndpoint = await this.transformUrlsToCloudFront( + collectionEndpoint, + bucket + ) + + const collectionsApiDir = directories.collections + const collectionKey = `${apiBasePath}/${collectionsApiDir}/${collection.metadata.slug}.json` + + await this.uploadToS3(bucket, collectionKey, transformedCollectionEndpoint) + console.log(`[Publishing] โœ“ Generated collection endpoint: ${collectionKey}`) + + // 2. Generate entries endpoint: /api/{collections-dir}/{slug}/{entries-dir}.json + const entriesEndpoint = { + collection: { + id: collection.id, + name: collection.name, + slug: collection.metadata.slug, + published: collection.published + }, + entries: transformedEntries.map(entry => ({ + id: entry.id, + title: entry.title, + description: entry.description, + slug: entry.slug, + published: entry.published, + tags: entry.tags, + createdAt: entry.createdAt, + updatedAt: entry.updatedAt, + data: entry.data + })), + total: transformedEntries.length, + published: transformedEntries.filter(e => e.published).length + } + + const entriesApiDir = directories.entries + const entriesKey = `${apiBasePath}/${collectionsApiDir}/${collection.metadata.slug}/${entriesApiDir}.json` + + await this.uploadToS3(bucket, entriesKey, entriesEndpoint) + console.log(`[Publishing] โœ“ Generated entries endpoint: ${entriesKey}`) + + // 3. Generate individual entry endpoints: /api/{collections-dir}/{slug}/{entries-dir}/{entry-slug}.json + for (const entry of transformedEntries) { + const entryEndpoint = { + id: entry.id, + collectionId: collection.id, + collectionSlug: collection.metadata.slug, + title: entry.title, + description: entry.description, + slug: entry.slug, + published: entry.published, + publishedAt: entry.publishedAt, + tags: entry.tags, + createdAt: entry.createdAt, + updatedAt: entry.updatedAt, + data: entry.data + } + + const entryKey = `${apiBasePath}/${collectionsApiDir}/${collection.metadata.slug}/${entriesApiDir}/${entry.slug}.json` + + await this.uploadToS3(bucket, entryKey, entryEndpoint) + } + + console.log(`[Publishing] โœ“ Generated ${transformedEntries.length} individual entry endpoints`) + } + + /** + * Update the collections index at /api/collections/index.json + */ + async updateCollectionsIndex(bucket: string, bucketId: string): Promise { + if (!this.s3Client) { + throw new Error('Publishing service not configured') + } + + console.log('[Publishing] Updating collections index...') + + // Get all collections with entries from local database + const collections = await this.loadCollectionsWithEntries(bucketId) + + // Filter to only published collections + console.log(`[Publishing] Total collections: ${collections.length}`) + collections.forEach(c => + console.log(`[Publishing] Collection ${c.name}: published=${c.published}`) + ) + const publishedCollections = collections.filter(collection => collection.published) + console.log(`[Publishing] Published collections: ${publishedCollections.length}`) + + // Create collections index + const collectionsIndex = { + collections: publishedCollections.map(collection => ({ + id: collection.id, + name: collection.name, + description: collection.description, + slug: collection.metadata.slug, + schemaId: collection.schemaId, + schemaName: collection.schemaName, + metadata: collection.metadata, + published: collection.published, + publishedAt: collection.publishedAt, + entryCount: collection.entries.filter(e => e.published).length, + publishedEntries: collection.entries.filter(e => e.published).length, + createdAt: collection.createdAt, + updatedAt: collection.updatedAt + })), + collectionOrder: publishedCollections.map(collection => collection.id), + total: publishedCollections.length, + lastUpdated: new Date().toISOString(), + lastGenerated: new Date().toISOString() + } + + // Transform URLs to CloudFront if configured + const transformedCollectionsIndex = await this.transformUrlsToCloudFront( + collectionsIndex, + bucket + ) + + // Save collections index + const directories = this.getDirectories() + const collectionsApiDir = directories.collections + const indexKey = `${directories.api}/${collectionsApiDir}/index.json` + + await this.uploadToS3(bucket, indexKey, transformedCollectionsIndex) + console.log(`[Publishing] โœ“ Updated collections index: ${indexKey}`) + } + + /** + * Generate schema lookup at /api/schemas/{schemaId}.json + */ + async generateSchemaLookup(bucket: string, bucketId: string): Promise { + if (!this.s3Client) { + throw new Error('Publishing service not configured') + } + + console.log('[Publishing] Generating schema lookups...') + + // Get all collections with entries from local database + const collections = await this.loadCollectionsWithEntries(bucketId) + const publishedCollections = collections.filter(collection => collection.published) + + // Group collections by schemaId + const schemaMap = new Map() + for (const collection of publishedCollections) { + if (!schemaMap.has(collection.schemaId)) { + schemaMap.set(collection.schemaId, []) + } + schemaMap.get(collection.schemaId)!.push(collection) + } + + // Generate schema lookup for each schema + const directories = this.getDirectories() + const collectionsApiDir = directories.collections || 'collections' + + for (const [schemaId, schemaCollections] of schemaMap.entries()) { + const schemaName = schemaCollections[0].schemaName + + const schemaLookup = { + schemaName, + collections: schemaCollections.map(c => ({ + id: c.id, + name: c.name, + description: c.description, + slug: c.metadata.slug, + schemaId: c.schemaId, + metadata: c.metadata, + published: c.published, + publishedAt: c.publishedAt, + entryCount: c.entries.filter(e => e.published).length, + publishedEntries: c.entries.filter(e => e.published).length, + createdAt: c.createdAt, + updatedAt: c.updatedAt + })), + total: schemaCollections.length, + lastGenerated: new Date().toISOString() + } + + // Transform URLs to CloudFront if configured + const transformedSchemaLookup = await this.transformUrlsToCloudFront(schemaLookup, bucket) + + // Use the old path format for backward compatibility: api/collections/{SchemaName}.json + const schemaKey = `${directories.api}/${collectionsApiDir}/${schemaName}.json` + await this.uploadToS3(bucket, schemaKey, transformedSchemaLookup) + console.log( + `[Publishing] โœ“ Generated schema lookup for ${schemaName}: ${schemaCollections.length} collection(s)` + ) + } + } + + /** + * Save schema definition files for migration compatibility + * Saves to: schemas/{schemaId}.json or sync/schemas/{schemaId}.json + */ + private async saveSchemaSources(bucket: string): Promise { + try { + console.log('[Publishing] Saving schema definition files...') + + const directories = this.getDirectories() + const schemasDir = this.config?.directories?.schemas || 'schemas' + + // Apply sync prefix to schemas directory if enabled + const schemasPath = this.syncPrefix ? `${this.syncPrefix}/${schemasDir}` : schemasDir + + // List schemas from S3 using the S3 client directly + const { ListObjectsV2Command, GetObjectCommand } = require('@aws-sdk/client-s3') + + const listCommand = new ListObjectsV2Command({ + Bucket: bucket, + Prefix: `${schemasDir}/` + }) + + const result = await this.s3Client.send(listCommand) + + if (!result.Contents || result.Contents.length === 0) { + console.log('[Publishing] No schema files found to sync') + return + } + + let savedCount = 0 + for (const item of result.Contents) { + if (item.Key && item.Key.endsWith('.json')) { + try { + // Download the schema file + const getCommand = new GetObjectCommand({ + Bucket: bucket, + Key: item.Key + }) + const schemaResult = await this.s3Client.send(getCommand) + const schemaContent = await schemaResult.Body?.transformToString() + + if (schemaContent) { + const schema = JSON.parse(schemaContent) + + // Save to sync location if enabled + const filename = item.Key.split('/').pop() + const schemaKey = `${schemasPath}/${filename}` + + await this.uploadToS3(bucket, schemaKey, schema) + savedCount++ + } + } catch (error) { + console.warn(`[Publishing] Could not sync schema ${item.Key}:`, error) + } + } + } + + console.log(`[Publishing] โœ“ Saved ${savedCount} schema definition files`) + } catch (error) { + console.warn('[Publishing] Error saving schema sources:', error) + // Don't fail the entire publish if schema sync fails + } + } + + /** + * Generate all API endpoints for all collections + * Also saves source files (collections, entries, schemas) for migration compatibility + */ + async generateAllApiEndpoints(bucket: string, bucketId: string): Promise { + if (!this.s3Client) { + throw new Error('Publishing service not configured') + } + + try { + console.log('\n[Publishing] ========================================') + console.log(`[Publishing] Starting full publish for bucket: ${bucket}`) + console.log('[Publishing] ========================================\n') + + // Save schema definition files for migration compatibility + await this.saveSchemaSources(bucket) + + // Get all collections with entries from local database + const collections = await this.loadCollectionsWithEntries(bucketId) + console.log(`[Publishing] Loaded ${collections.length} collections from local database`) + + // Generate API endpoints for each collection + for (const collection of collections) { + await this.generateApiEndpoints(bucket, bucketId, collection) + } + + // Update collections index + await this.updateCollectionsIndex(bucket, bucketId) + + // Generate schema lookups + await this.generateSchemaLookup(bucket, bucketId) + + console.log('\n[Publishing] ========================================') + console.log(`[Publishing] Published ${collections.length} collections successfully`) + console.log('[Publishing] ========================================\n') + } catch (error) { + console.error('[Publishing] Error generating API endpoints:', error) + throw error + } + } + + /** + * Generate API endpoints for specific collections/entries (selective publishing) + */ + async generateSelectiveApiEndpoints( + bucket: string, + bucketId: string, + collectionIds: string[], + entryIds: string[] = [] + ): Promise { + if (!this.s3Client) { + throw new Error('Publishing service not configured') + } + + try { + console.log( + `\n[Publishing] Selective publishing: ${collectionIds.length} collections, ${entryIds.length} entries` + ) + + // Get all collections with entries from local database + const allCollections = await this.loadCollectionsWithEntries(bucketId) + const collectionsToUpdate = new Set() + + // Add collections that are explicitly dirty + for (const collectionId of collectionIds) { + collectionsToUpdate.add(collectionId) + } + + // Add collections that contain dirty entries + for (const entryId of entryIds) { + const collection = allCollections.find(c => c.entries.some(e => e.id === entryId)) + if (collection) { + collectionsToUpdate.add(collection.id) + } + } + + // Generate API endpoints for affected collections + for (const collectionId of collectionsToUpdate) { + const collection = allCollections.find(c => c.id === collectionId) + if (collection) { + await this.generateApiEndpoints(bucket, bucketId, collection) + } + } + + // Save schema definition files for migration compatibility + await this.saveSchemaSources(bucket) + + // Update collections index only if collection metadata changed + if (collectionIds.length > 0) { + await this.updateCollectionsIndex(bucket, bucketId) + } + + // Update schema lookups + await this.generateSchemaLookup(bucket, bucketId) + + console.log( + `[Publishing] Selective publishing complete: updated ${collectionsToUpdate.size} collections\n` + ) + } catch (error) { + console.error('[Publishing] Error in selective API generation:', error) + throw error + } + } + + /** + * Get the last generation date from the collections index + */ + async getLastGenerationDate(bucket: string): Promise { + if (!this.s3Client) { + throw new Error('Publishing service not configured') + } + + try { + const directories = this.getDirectories() + const collectionsApiDir = directories.collections + const indexKey = `${directories.api}/${collectionsApiDir}/index.json` + + const { GetObjectCommand } = await import('@aws-sdk/client-s3') + const getIndexCommand = new GetObjectCommand({ + Bucket: bucket, + Key: indexKey + }) + + const indexResult = await this.s3Client.send(getIndexCommand) + const indexBody = await indexResult.Body?.transformToString() + if (indexBody) { + const index = JSON.parse(indexBody) + return index.lastGenerated || null + } + return null + } catch (error) { + // If index doesn't exist, return null + return null + } + } +} + +export const publishingService = new PublishingService(new DatabaseService()) diff --git a/src/main/services/syncService.ts b/src/main/services/syncService.ts new file mode 100644 index 0000000..633d49d --- /dev/null +++ b/src/main/services/syncService.ts @@ -0,0 +1,209 @@ +/** + * SyncService - POC Version + * Handles background sync of source files to S3 + */ + +import { DatabaseService } from './databaseService' +import { PublishingService } from './publishingService' +import { EventEmitter } from 'events' + +export interface SyncStatus { + isRunning: boolean + lastSyncAt?: string + queueSize: number + successfulSyncs: number + failedSyncs: number +} + +export class SyncService extends EventEmitter { + private databaseService: DatabaseService + private publishingService: PublishingService | null = null + private activeBucket: string | null = null + private activeBucketId: string | null = null + private isRunning = false + private isPaused = false + private syncInterval: NodeJS.Timeout | null = null + private status: SyncStatus = { + isRunning: false, + queueSize: 0, + successfulSyncs: 0, + failedSyncs: 0 + } + + constructor(databaseService: DatabaseService) { + super() + this.databaseService = databaseService + } + + /** + * Configure the sync service with publishing service for S3 writes + */ + configure(publishingService: PublishingService, bucket: string, bucketId: string): void { + this.publishingService = publishingService + this.activeBucket = bucket + this.activeBucketId = bucketId + console.log('[SyncService] Configured for bucket:', bucket, 'bucketId:', bucketId) + } + + async start(): Promise { + if (this.isRunning) { + console.log('[SyncService] Already running') + return + } + + console.log('[SyncService] Starting background sync...') + this.isRunning = true + this.status.isRunning = true + + // Sync every 10 seconds (longer interval for POC) + this.syncInterval = setInterval(() => { + this.processSyncQueue().catch(err => { + console.error('[SyncService] Sync loop error:', err) + }) + }, 10000) + + // Initial sync + await this.processSyncQueue() + + this.emit('statusChange', this.status) + console.log('[SyncService] Started successfully') + } + + async stop(): Promise { + console.log('[SyncService] Stopping background sync...') + this.isRunning = false + this.status.isRunning = false + + if (this.syncInterval) { + clearInterval(this.syncInterval) + this.syncInterval = null + } + + this.emit('statusChange', this.status) + console.log('[SyncService] Stopped') + } + + pause(): void { + console.log('[SyncService] โธ๏ธ Pausing sync (migration in progress)...') + this.isPaused = true + } + + resume(): void { + console.log('[SyncService] โ–ถ๏ธ Resuming sync (migration complete)...') + this.isPaused = false + // Trigger an immediate sync after resuming + if (this.isRunning) { + this.processSyncQueue().catch(err => { + console.error('[SyncService] Sync error after resume:', err) + }) + } + } + + async forceSyncNow(): Promise { + console.log('[SyncService] Force sync requested') + await this.processSyncQueue() + } + + getStatus(): SyncStatus { + return { ...this.status } + } + + private async processSyncQueue(): Promise { + // Skip processing if paused (e.g., during migration) + if (this.isPaused) { + console.log('[SyncService] Sync paused, skipping queue processing') + return + } + + try { + // Only process sync queue items for the currently active bucket to prevent cross-bucket contamination + const queueItems = await this.databaseService.getSyncQueue( + 5, + this.activeBucketId || undefined + ) + this.status.queueSize = queueItems.length + + if (queueItems.length === 0) { + return // Nothing to sync + } + + console.log(`[SyncService] Processing ${queueItems.length} items from sync queue`) + + for (const item of queueItems) { + try { + await this.databaseService.markSyncProcessing(item.id) + await this.processSyncItem(item) + await this.databaseService.markSyncComplete(item.id) + + this.status.successfulSyncs++ + + console.log(`[SyncService] โœ“ Synced ${item.entity_type} ${item.entity_id}`) + + this.emit('itemSynced', { + entityType: item.entity_type, + entityId: item.entity_id, + operation: item.operation + }) + } catch (error: any) { + console.error('[SyncService] Sync error:', error) + await this.databaseService.markSyncError(item.id, error.message) + + this.status.failedSyncs++ + + this.emit('syncError', { + entityType: item.entity_type, + entityId: item.entity_id, + error: error.message + }) + } + } + + this.status.lastSyncAt = new Date().toISOString() + this.emit('statusChange', this.status) + } catch (error) { + console.error('[SyncService] Sync queue processing error:', error) + } + } + + private async processSyncItem(item: any): Promise { + const { operation, entity_type, entity_id, data } = item + + // Handle source file sync operations + if (operation === 'source_file_sync' && this.publishingService && this.activeBucket) { + const parsedData = typeof data === 'string' ? JSON.parse(data) : data + + if (entity_type === 'collection') { + console.log(`[SyncService] Syncing collection source file: ${parsedData.name}`) + } else if (entity_type === 'entry') { + console.log(`[SyncService] Syncing entry source file: ${entity_id}`) + } + + // Actually sync the file to S3 + await this.publishingService.syncSourceFile( + this.activeBucket, + entity_type as 'collection' | 'entry', + entity_id, + parsedData + ) + return + } + + if (operation === 'source_file_delete' && this.publishingService && this.activeBucket) { + console.log(`[SyncService] Deleting source file: ${entity_type}s/${entity_id}.json`) + + // Actually delete the file from S3 + await this.publishingService.deleteSourceFile( + this.activeBucket, + entity_type as 'collection' | 'entry', + entity_id + ) + return + } + + // Legacy sync operations (for backward compatibility) + console.log(`[SyncService] [POC] Would sync: ${operation} ${entity_type} ${entity_id}`) + + // Simulate network delay + await new Promise(resolve => setTimeout(resolve, 100)) + } +} diff --git a/src/renderer/App.tsx b/src/renderer/App.tsx index 3d1a837..624325f 100644 --- a/src/renderer/App.tsx +++ b/src/renderer/App.tsx @@ -23,6 +23,7 @@ import { StorageProvider } from './contexts/StorageContext' import { PublishingProvider } from './contexts/PublishingContext' import { DirtyStateProvider } from './contexts/DirtyStateContext' import { DirtyStateIndicator } from './components/DirtyStateIndicator' +import { MigrationModal } from './components/MigrationModal' function App() { const [s3Config, setS3Config] = useState(null) @@ -57,6 +58,13 @@ function App() { title: '', message: '' }) + const [migrationState, setMigrationState] = useState<{ + isRunning: boolean + bucketName: string + }>({ + isRunning: false, + bucketName: '' + }) const { configure, uploadSchema, listSchemas, downloadSchema, deleteSchema } = useS3Service() const { activeBucketConfig, isLoading: storageLoading } = useStorage() @@ -96,6 +104,27 @@ function App() { loadSchemas() }, [isConfigured, activeBucketConfig, listSchemas]) + // Listen for migration events + useEffect(() => { + const unsubscribeStarted = window.electronAPI.db.migration.onStarted(data => { + console.log('[Migration] Started:', data) + setMigrationState({ + isRunning: true, + bucketName: data.bucketName + }) + }) + + const unsubscribeComplete = window.electronAPI.db.migration.onComplete(data => { + console.log('[Migration] Complete:', data) + // Keep modal open briefly to show result, then close via modal's own logic + }) + + return () => { + unsubscribeStarted() + unsubscribeComplete() + } + }, []) + const handleS3Config = async (config: S3Config) => { const result = await configure(config) if (result.success) { @@ -292,6 +321,15 @@ function App() { {/* Footer */}
+ + {/* Migration Modal */} + { + setMigrationState({ isRunning: false, bucketName: '' }) + }} + />
) } diff --git a/src/renderer/components/ContentManager.tsx b/src/renderer/components/ContentManager.tsx index 2550cb8..bb32ee1 100644 --- a/src/renderer/components/ContentManager.tsx +++ b/src/renderer/components/ContentManager.tsx @@ -24,11 +24,11 @@ import { EntryViewer } from './EntryViewer' import { DraggableEntryItem } from './DraggableEntryItem' import { DraggableCollectionItem } from './DraggableCollectionItem' import { Spinner } from './Spinner' -import { slugify } from '../utils/stringUtils' import { useOperationState } from '../hooks/useOperationState' import { useModalState } from '../hooks/useModalState' import { usePublishing } from '../contexts/PublishingContext' import { useDirtyState } from '../contexts/DirtyStateContext' +import { useStorage } from '../hooks/useStorage' interface ContentManagerProps { schemas: Schema[] @@ -41,6 +41,10 @@ export const ContentManager: FC = ({ schemas, bucket }) => const modal = useModalState() const { markAsDirty } = usePublishing() const { markCollectionAsDirty, markEntryAsDirty } = useDirtyState() + const { activeBucketConfig } = useStorage() + + // Get bucketId for local database queries + const bucketId = activeBucketConfig?.id const [collections, setCollections] = useState([]) const [selectedCollection, setSelectedCollection] = useState(null) @@ -69,18 +73,18 @@ export const ContentManager: FC = ({ schemas, bucket }) => }) const loadCollections = async (showLoading = true) => { - if (!window.electronAPI?.content) return + if (!window.electronAPI?.db || !bucketId) return if (showLoading) { operations.setLoading('collection') } try { - const result = await window.electronAPI.content.listCollections({ bucket }) + const result = await window.electronAPI.db.collection.list(bucketId) if (result.success) { setCollections(result.result || []) } } catch (error) { - console.error('Failed to load collections:', error) + console.error('Failed to load collections from local database:', error) } finally { if (showLoading) { operations.setIdle() @@ -89,33 +93,20 @@ export const ContentManager: FC = ({ schemas, bucket }) => } const loadEntries = async (collectionId: string, showLoading = true) => { - if (!window.electronAPI?.content) return + if (!window.electronAPI?.db) return if (showLoading) { operations.setLoading('entry') } try { - const result = await window.electronAPI.content.listEntries({ bucket, collectionId }) + const result = await window.electronAPI.db.entry.list(collectionId) if (result.success) { + // Local database already returns entries in the correct order from collection_entry_order table const loadedEntries = result.result || [] - - // Find the collection to get its entryOrder - const collection = collections.find(c => c.id === collectionId) - if (collection && collection.entryOrder && collection.entryOrder.length > 0) { - // Sort entries according to the collection's entryOrder - loadedEntries.sort((a: ContentEntry, b: ContentEntry) => { - const aIndex = collection.entryOrder.indexOf(a.id) - const bIndex = collection.entryOrder.indexOf(b.id) - if (aIndex === -1) return 1 - if (bIndex === -1) return -1 - return aIndex - bIndex - }) - } - setEntries(loadedEntries) } } catch (error) { - console.error('Failed to load entries:', error) + console.error('Failed to load entries from local database:', error) } finally { if (showLoading) { operations.setIdle() @@ -124,9 +115,28 @@ export const ContentManager: FC = ({ schemas, bucket }) => } useEffect(() => { - loadCollections() - loadLastGenerationDate() - }, [bucket]) + // Clear state and load new data when bucket changes + if (bucketId) { + console.log('[ContentManager] Bucket changed - loading collections for bucketId:', bucketId) + // Clear previous bucket's data + setCollections([]) + setSelectedCollection(null) + setEntries([]) + setSchemaFilter('all') + // Load new bucket's data + loadCollections() + loadLastGenerationDate() + } + }, [bucketId]) // Changed from bucket to bucketId to ensure we use the correct ID + + // Refresh last generation date periodically (every 30 seconds when component is active) + useEffect(() => { + const interval = setInterval(() => { + loadLastGenerationDate() + }, 30000) // Check every 30 seconds + + return () => clearInterval(interval) + }, [bucketId]) const loadLastGenerationDate = async () => { if (!window.electronAPI?.content) return @@ -178,25 +188,22 @@ export const ContentManager: FC = ({ schemas, bucket }) => schemaId: string schemaName: string }) => { - if (!window.electronAPI?.content) return + if (!window.electronAPI?.db || !bucketId) return operations.setCreating('collection') try { - const result = await window.electronAPI.content.createCollection({ - bucket, - collection: { - name: data.name, - description: data.description, - schemaId: data.schemaId, - schemaName: data.schemaName, - published: false, // New collections start as unpublished - metadata: { - title: data.name, - description: data.description || '', - slug: data.name.toLowerCase().replace(/\s+/g, '-'), - primaryImage: '' - }, - entryOrder: [] + const result = await window.electronAPI.db.collection.create(bucketId, { + id: `mgc${Date.now()}${Math.random().toString(36).substr(2, 9)}`, // Generate ID with prefix + name: data.name, + description: data.description, + schema_id: data.schemaId, + schema_name: data.schemaName, + published: false, // New collections start as unpublished + metadata: { + title: data.name, + description: data.description || '', + slug: data.name.toLowerCase().replace(/\s+/g, '-'), + primaryImage: '' } }) @@ -218,7 +225,7 @@ export const ContentManager: FC = ({ schemas, bucket }) => } const saveCollection = async (updatedCollection: ContentCollection) => { - if (!window.electronAPI?.content) return + if (!window.electronAPI?.db) return // Track if published state changed const oldCollection = collections.find(c => c.id === updatedCollection.id) @@ -233,10 +240,12 @@ export const ContentManager: FC = ({ schemas, bucket }) => operations.setUpdating('collection') try { - const result = await window.electronAPI.content.updateCollection({ - bucket, - collectionId: updatedCollection.id, - updates: updatedCollection + const result = await window.electronAPI.db.collection.update(updatedCollection.id, { + name: updatedCollection.name, + description: updatedCollection.description, + published: updatedCollection.published, + published_at: updatedCollection.publishedAt, + metadata: updatedCollection.metadata }) if (result.success) { @@ -280,11 +289,10 @@ export const ContentManager: FC = ({ schemas, bucket }) => operations.setSaving('order') try { - const result = await window.electronAPI.content.updateEntryOrder({ - bucket, - collectionId: selectedCollection.id, - entryOrder: newEntryOrder - }) + const result = await window.electronAPI.db.entry.updateOrder( + selectedCollection.id, + newEntryOrder + ) if (result.success) { // Update the selected collection's entry order @@ -323,6 +331,12 @@ export const ContentManager: FC = ({ schemas, bucket }) => const { active, over } = event if (active.id !== over.id) { + // Guard against undefined bucketId + if (!bucketId) { + toast.error('No active bucket selected') + return + } + const oldIndex = collections.findIndex(collection => collection.id === active.id) const newIndex = collections.findIndex(collection => collection.id === over.id) @@ -334,10 +348,10 @@ export const ContentManager: FC = ({ schemas, bucket }) => operations.setSaving('order') try { - const result = await window.electronAPI.content.updateCollectionOrder({ - bucket, - collectionOrder: newCollectionOrder - }) + const result = await window.electronAPI.db.collection.updateOrder( + bucketId, + newCollectionOrder + ) if (result.success) { // Mark as dirty since collection order affects published API @@ -412,36 +426,41 @@ export const ContentManager: FC = ({ schemas, bucket }) => } }) - if (!window.electronAPI?.content) return + if (!window.electronAPI?.db || !bucketId) return operations.setCreating('entry') try { - const result = await window.electronAPI.content.createEntry({ - bucket, - collectionId: selectedCollection.id, - entry: { - schemaId: selectedCollection.schemaId, - schemaName: schema.name, + const result = await window.electronAPI.db.entry.create(bucketId, { + id: `mge${Date.now()}${Math.random().toString(36).substr(2, 9)}`, // Generate ID with prefix + collection_id: selectedCollection.id, + schema_id: selectedCollection.schemaId, + schema_name: schema.name, + data: { data: entryData, - published: false, - tags: [], metadata: { - title: `New ${schema.name}`, + title: '', description: '', - slug: slugify(`new-${schema.name}`) - } - } + slug: '' + }, + tags: [] + }, + published: false }) if (result.success) { await loadEntries(selectedCollection.id, false) - // Update the entry count for the selected collection without reloading all collections + // Update the entry count for the selected collection setCollections(prevCollections => - prevCollections.map(collection => - collection.id === selectedCollection.id - ? { ...collection, entries: [...(collection.entries || []), result.result] } - : collection - ) + prevCollections.map(collection => { + if (collection.id === selectedCollection.id) { + const currentCount = collection.entries?.length || 0 + return { + ...collection, + entries: new Array(currentCount + 1).fill(null) // Maintain the same structure as DB + } + } + return collection + }) ) toast.success('Entry created successfully!') } else { @@ -465,7 +484,7 @@ export const ContentManager: FC = ({ schemas, bucket }) => const saveEntry = async (updatedEntry: ContentEntry) => { if (!selectedCollection) return - if (!window.electronAPI?.content) return + if (!window.electronAPI?.db) return // Track if published state changed const oldEntry = entries.find(e => e.id === updatedEntry.id) @@ -479,11 +498,14 @@ export const ContentManager: FC = ({ schemas, bucket }) => operations.setUpdating('entry') try { - const result = await window.electronAPI.content.updateEntry({ - bucket, - collectionId: selectedCollection.id, - entryId: updatedEntry.id, - updates: updatedEntry + const result = await window.electronAPI.db.entry.update(updatedEntry.id, { + data: { + data: updatedEntry.data, + metadata: updatedEntry.metadata, + tags: updatedEntry.tags || [] + }, + published: updatedEntry.published, + published_at: updatedEntry.publishedAt }) if (result.success) { @@ -496,7 +518,7 @@ export const ContentManager: FC = ({ schemas, bucket }) => ...collection, entries: collection.entries?.map(entry => - entry.id === updatedEntry.id ? updatedEntry : entry + entry && entry.id === updatedEntry.id ? updatedEntry : entry ) || [] } : collection @@ -528,28 +550,27 @@ export const ContentManager: FC = ({ schemas, bucket }) => return } - if (!window.electronAPI?.content) return + if (!window.electronAPI?.db) return operations.setDeleting('entry') try { - const result = await window.electronAPI.content.deleteEntry({ - bucket, - collectionId: selectedCollection.id, - entryId: entry.id - }) + const result = await window.electronAPI.db.entry.delete(entry.id) if (result.success) { await loadEntries(selectedCollection.id, false) - // Remove the entry from the collections state without reloading all collections + // Update the entry count for the selected collection setCollections(prevCollections => - prevCollections.map(collection => - collection.id === selectedCollection.id - ? { - ...collection, - entries: collection.entries?.filter(e => e.id !== entry.id) || [] - } - : collection - ) + prevCollections.map(collection => { + if (collection.id === selectedCollection.id) { + const currentCount = collection.entries?.length || 0 + const newCount = Math.max(0, currentCount - 1) + return { + ...collection, + entries: new Array(newCount).fill(null) // Maintain the same structure as DB + } + } + return collection + }) ) toast.success('Entry deleted successfully!') } else { @@ -565,20 +586,17 @@ export const ContentManager: FC = ({ schemas, bucket }) => const deleteCollection = async (collection: ContentCollection) => { if ( !confirm( - `Are you sure you want to delete the collection "${collection.name}"? This will also delete all ${collection.entries.length} entries in this collection. This action cannot be undone.` + `Are you sure you want to delete the collection "${collection.name}"? This will also delete all ${collection.entries?.length || 0} entries in this collection. This action cannot be undone.` ) ) { return } - if (!window.electronAPI?.content) return + if (!window.electronAPI?.db) return operations.setDeleting('collection') try { - const result = await window.electronAPI.content.deleteCollection({ - bucket, - collectionId: collection.id - }) + const result = await window.electronAPI.db.collection.delete(collection.id) if (result.success) { await loadCollections(false) diff --git a/src/renderer/components/DatabaseTestPanel.tsx b/src/renderer/components/DatabaseTestPanel.tsx new file mode 100644 index 0000000..129528a --- /dev/null +++ b/src/renderer/components/DatabaseTestPanel.tsx @@ -0,0 +1,350 @@ +/** + * DatabaseTestPanel - POC Test Component + * Simple UI to test local database operations + */ + +import React, { useState } from 'react' +import { + useLocalCollections, + useCreateLocalCollection, + useUpdateLocalCollection, + useDeleteLocalCollection, + useLocalSyncStatus, + useLocalDatabaseStats +} from '../hooks/useLocalCollections' + +interface DatabaseTestPanelProps { + bucketId: string +} + +export const DatabaseTestPanel: React.FC = ({ bucketId }) => { + const [testName, setTestName] = useState('') + const [dbPath, setDbPath] = useState('') + const [isClearing, setIsClearing] = useState(false) + + const { data: collections = [], isLoading, error } = useLocalCollections(bucketId) + const createMutation = useCreateLocalCollection(bucketId) + const updateMutation = useUpdateLocalCollection() + const deleteMutation = useDeleteLocalCollection(bucketId) + const { data: syncStatus } = useLocalSyncStatus() + const { data: dbStats } = useLocalDatabaseStats() + + // Load database path on mount + React.useEffect(() => { + const loadDbPath = async () => { + const result = await window.electronAPI.db.getPath() + if (result.success && result.result) { + setDbPath(result.result) + } + } + loadDbPath() + }, []) + + const handleCreateTest = async () => { + if (!testName) return + + const testCollection = { + id: `test-${Date.now()}`, + name: testName, + schema_id: 'test-schema-poc', // Use snake_case to match database column + schema_name: 'TestSchema', // Use snake_case to match database column + description: 'POC test collection', + published: false, + metadata: { test: true } + } + + await createMutation.mutateAsync(testCollection) + setTestName('') + } + + const handleUpdateTest = async (collectionId: string) => { + await updateMutation.mutateAsync({ + id: collectionId, + updates: { + description: `Updated at ${new Date().toISOString()}` + } + }) + } + + const handleDeleteTest = async (collectionId: string) => { + if (confirm('Delete this test collection?')) { + await deleteMutation.mutateAsync(collectionId) + } + } + + const handleClearDatabase = async () => { + const confirmed = confirm( + 'โš ๏ธ WARNING: This will delete ALL local data and re-import from S3.\n\n' + + 'This action cannot be undone. Continue?' + ) + if (!confirmed) return + + try { + setIsClearing(true) + const result = await window.electronAPI.db.clear() + if (result.success) { + alert('โœ… Database cleared and re-imported from S3 successfully!') + // Force reload to refresh all data + window.location.reload() + } else { + alert(`โŒ Failed to clear database: ${result.error}`) + } + } catch (error) { + console.error('Error clearing database:', error) + alert(`โŒ Error: ${error instanceof Error ? error.message : 'Unknown error'}`) + } finally { + setIsClearing(false) + } + } + + const formatSize = (bytes: number) => { + if (bytes < 1024) return `${bytes} B` + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(2)} KB` + return `${(bytes / 1024 / 1024).toFixed(2)} MB` + } + + return ( +
+

๐Ÿงช Local Database POC Test Panel

+ + {/* Database Stats */} + {dbStats && ( +
+

+ Database Stats +

+
+
Collections: {dbStats.collections}
+
Sync Queue: {dbStats.syncQueueSize} pending
+
DB Size: {formatSize(dbStats.dbSize)}
+ {dbPath && ( +
+ Path: {dbPath} +
+ )} +
+
+ )} + + {/* Danger Zone */} +
+

โš ๏ธ Danger Zone

+

+ Clear the local database and re-import all data from S3. Use this if you encounter data + sync issues. +

+ +
+ + {/* Sync Status */} + {syncStatus && ( +
+

Sync Status

+
+
Status: {syncStatus.isRunning ? '๐ŸŸข Running' : '๐Ÿ”ด Stopped'}
+
Queue Size: {syncStatus.queueSize}
+
Successful Syncs: {syncStatus.successfulSyncs}
+
Failed Syncs: {syncStatus.failedSyncs}
+ {syncStatus.lastSyncAt && ( +
Last Sync: {new Date(syncStatus.lastSyncAt).toLocaleTimeString()}
+ )} +
+
+ )} + + {/* Create Test Collection */} +
+

+ Create Test Collection +

+
+ setTestName(e.target.value)} + placeholder="Collection name" + style={{ + flex: 1, + padding: '8px', + border: '1px solid #ddd', + borderRadius: '4px', + fontSize: '14px' + }} + /> + +
+
+ + {/* Collections List */} +
+

+ Local Collections ({collections.length}) +

+ + {isLoading &&
Loading collections...
} + {error &&
Error: {(error as Error).message}
} + + {collections.length === 0 && !isLoading && ( +
+ No collections yet. Create one above to test! +
+ )} + +
+ {collections.map((collection: any) => ( +
+
+
{collection.name}
+
+ {collection.description || 'No description'} +
+
+ Status:{' '} + + {collection.sync_status} + + {' โ€ข '} + Updated: {new Date(collection.updated_at).toLocaleTimeString()} +
+
+
+ + +
+
+ ))} +
+
+ +
+ POC Test Instructions: +
    +
  • Create test collections and watch them appear instantly
    • +
  • Update collections to see optimistic UI updates
    • +
  • Check sync status to see background sync working
    • +
  • Database operations should complete in <50ms
    • +
+
+
+ ) +} diff --git a/src/renderer/components/DirtyStateIndicator.tsx b/src/renderer/components/DirtyStateIndicator.tsx index 9572f78..d58c5ac 100644 --- a/src/renderer/components/DirtyStateIndicator.tsx +++ b/src/renderer/components/DirtyStateIndicator.tsx @@ -1,5 +1,5 @@ -import React from 'react' -import { Upload, AlertCircle } from 'lucide-react' +import React, { useState } from 'react' +import { Upload, AlertCircle, ChevronDown, ChevronUp, X } from 'lucide-react' import { useDirtyState } from '../contexts/DirtyStateContext' import { Spinner } from './Spinner' import { PublishingProgress } from './PublishingProgress' @@ -18,6 +18,8 @@ export const DirtyStateIndicator: React.FC = () => { setShowPublishingProgress } = useDirtyState() + const [isMinimized, setIsMinimized] = useState(false) + if (!hasUnpublishedChanges) { return null } @@ -43,26 +45,42 @@ export const DirtyStateIndicator: React.FC = () => { return ( <> -
-
-
-
- +
+
+ {/* Header - always visible */} +
+
+ +

Unpublished Changes

+ + {totalCount} +
-
-
-

Unpublished Changes

- - {totalCount} - -
-

{getChangeText()}

+
+ +
+
+ + {/* Content - collapsible */} + {!isMinimized && ( +
+

{getChangeText()}

{lastPublished && ( -

+

Last published: {new Date(lastPublished).toLocaleString()}

)} -
+
-
+ )}
diff --git a/src/renderer/components/DraggableCollectionItem.tsx b/src/renderer/components/DraggableCollectionItem.tsx index e751135..66afa26 100644 --- a/src/renderer/components/DraggableCollectionItem.tsx +++ b/src/renderer/components/DraggableCollectionItem.tsx @@ -1,9 +1,15 @@ -import { useState, useEffect } from 'react' +import { FC, useState, useEffect } from 'react' import { useSortable } from '@dnd-kit/sortable' import { CSS } from '@dnd-kit/utilities' import { ContentCollection } from '../../types/content' import { MoreVertical, Edit, Trash2, GripVertical, CheckCircle, Circle } from 'lucide-react' +interface Schema { + id: string + name: string + [key: string]: any +} + interface DraggableCollectionItemProps { collection: ContentCollection isSelected: boolean @@ -11,7 +17,7 @@ interface DraggableCollectionItemProps { onEdit: (collection: ContentCollection) => void onDelete: (collection: ContentCollection) => void onTogglePublish: (collection: ContentCollection) => void - schemas: any[] + schemas: Schema[] } export const DraggableCollectionItem: FC = ({ @@ -37,7 +43,9 @@ export const DraggableCollectionItem: FC = ({ } const schemaName = - collection.schemaName || schemas.find(s => s.id === collection.schemaId)?.name || 'Unknown' + collection.schemaName || + schemas.find((s: Schema) => s.id === collection.schemaId)?.name || + 'Unknown' // Close menu when clicking outside useEffect(() => { @@ -97,14 +105,12 @@ export const DraggableCollectionItem: FC = ({

{collection.metadata?.title || collection.name}

- {collection.metadata?.description && ( -

- {collection.metadata.description} -

+ {collection.description && ( +

{collection.description}

)}
Schema: {schemaName} - {collection.entries.length} entries + {collection.entries?.length || 0} entries
diff --git a/src/renderer/components/MigrationModal.tsx b/src/renderer/components/MigrationModal.tsx new file mode 100644 index 0000000..59f2994 --- /dev/null +++ b/src/renderer/components/MigrationModal.tsx @@ -0,0 +1,145 @@ +import { useEffect, useState } from 'react' + +interface MigrationModalProps { + show: boolean + bucketName: string + onComplete?: () => void +} + +export function MigrationModal({ show, bucketName, onComplete }: MigrationModalProps) { + const [isComplete, setIsComplete] = useState(false) + const [success, setSuccess] = useState(false) + const [error, setError] = useState(null) + + useEffect(() => { + if (!show) { + setIsComplete(false) + setSuccess(false) + setError(null) + return + } + + // Listen for migration completion + const unsubscribe = window.electronAPI.db.migration.onComplete(data => { + setIsComplete(true) + setSuccess(data.success) + setError(data.error || null) + + // Auto-dismiss after 2 seconds on success + if (data.success) { + setTimeout(() => { + onComplete?.() + }, 2000) + } + }) + + return () => { + unsubscribe() + } + }, [show, onComplete]) + + if (!show) return null + + return ( +
+
+
+ {!isComplete ? ( + <> + {/* Migration in progress */} +
+ + + + +
+

Importing Data

+

+ Importing content from {bucketName} +

+

+ This may take a moment depending on how much content you have... +

+ + ) : success ? ( + <> + {/* Migration successful */} +
+ + + +
+

Import Complete!

+

+ Successfully imported content from{' '} + {bucketName} +

+ + ) : ( + <> + {/* Migration failed */} +
+ + + +
+

Import Failed

+

+ Failed to import content from {bucketName} +

+ {error && ( +
+

{error}

+
+ )} + + + )} +
+
+
+ ) +} diff --git a/src/renderer/components/Settings.tsx b/src/renderer/components/Settings.tsx index 24cd171..6111396 100644 --- a/src/renderer/components/Settings.tsx +++ b/src/renderer/components/Settings.tsx @@ -1,16 +1,21 @@ import { FC, useState } from 'react' -import { Settings as SettingsIcon, Database } from 'lucide-react' +import { Settings as SettingsIcon, Database, TestTube } from 'lucide-react' import { BucketManager } from './BucketManager' import { ThemeManager } from './ThemeManager' +import { DatabaseTestPanel } from './DatabaseTestPanel' +import { useStorage } from '../hooks/useStorage' interface SettingsProps { onS3Config?: (config: any) => void isConfigured?: boolean - initialTab?: 'themes' | 'buckets' + initialTab?: 'themes' | 'buckets' | 'database' } export const Settings: FC = ({ isConfigured = false, initialTab = 'buckets' }) => { - const [activeSettingsTab, setActiveSettingsTab] = useState<'themes' | 'buckets'>(initialTab) + const [activeSettingsTab, setActiveSettingsTab] = useState<'themes' | 'buckets' | 'database'>( + initialTab + ) + const { activeBucketConfig } = useStorage() return (
@@ -51,6 +56,19 @@ export const Settings: FC = ({ isConfigured = false, initialTab = Themes
+
@@ -62,6 +80,12 @@ export const Settings: FC = ({ isConfigured = false, initialTab = )} {activeSettingsTab === 'themes' && } + + {activeSettingsTab === 'database' && activeBucketConfig && ( +
+ +
+ )}
) } diff --git a/src/renderer/hooks/useLocalCollections.ts b/src/renderer/hooks/useLocalCollections.ts new file mode 100644 index 0000000..015481f --- /dev/null +++ b/src/renderer/hooks/useLocalCollections.ts @@ -0,0 +1,202 @@ +/** + * useLocalCollections - POC React Query Hooks for Local Database + * These hooks provide access to the local database collections via IPC + */ + +import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' + +/** + * Fetch all collections for a bucket from local database + */ +export function useLocalCollections(bucketId: string | undefined) { + return useQuery({ + queryKey: ['local-collections', bucketId], + queryFn: async () => { + if (!bucketId) return [] + + console.log('[useLocalCollections] Fetching collections for bucket:', bucketId) + const response = await window.electronAPI.db.collection.list(bucketId) + + if (!response.success) { + throw new Error(response.error || 'Failed to fetch collections') + } + + console.log('[useLocalCollections] Fetched', response.result?.length || 0, 'collections') + return response.result || [] + }, + enabled: !!bucketId + }) +} + +/** + * Fetch a single collection by ID + */ +export function useLocalCollection(collectionId: string | undefined) { + return useQuery({ + queryKey: ['local-collection', collectionId], + queryFn: async () => { + if (!collectionId) return null + + const response = await window.electronAPI.db.collection.get(collectionId) + + if (!response.success) { + throw new Error(response.error || 'Failed to fetch collection') + } + + return response.result + }, + enabled: !!collectionId + }) +} + +/** + * Create a new collection in local database + */ +export function useCreateLocalCollection(bucketId: string) { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: async (data: any) => { + console.log('[useCreateLocalCollection] Creating collection:', data.id) + const response = await window.electronAPI.db.collection.create(bucketId, data) + + if (!response.success) { + throw new Error(response.error || 'Failed to create collection') + } + + return response.result + }, + onSuccess: () => { + console.log('[useCreateLocalCollection] Invalidating collections query') + queryClient.invalidateQueries({ queryKey: ['local-collections', bucketId] }) + }, + // Optimistic update + onMutate: async newCollection => { + await queryClient.cancelQueries({ queryKey: ['local-collections', bucketId] }) + const previousCollections = queryClient.getQueryData(['local-collections', bucketId]) + + queryClient.setQueryData(['local-collections', bucketId], (old: any[] = []) => [ + ...old, + { ...newCollection, sync_status: 'pending' } + ]) + + return { previousCollections } + }, + onError: (_, __, context) => { + console.error('[useCreateLocalCollection] Error, rolling back') + // Rollback on error + if (context?.previousCollections) { + queryClient.setQueryData(['local-collections', bucketId], context.previousCollections) + } + } + }) +} + +/** + * Update an existing collection + */ +export function useUpdateLocalCollection() { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: async ({ id, updates }: { id: string; updates: any }) => { + console.log('[useUpdateLocalCollection] Updating collection:', id) + const response = await window.electronAPI.db.collection.update(id, updates) + + if (!response.success) { + throw new Error(response.error || 'Failed to update collection') + } + + return response.result + }, + onSuccess: (data, variables) => { + console.log('[useUpdateLocalCollection] Invalidating queries') + queryClient.invalidateQueries({ queryKey: ['local-collection', variables.id] }) + queryClient.invalidateQueries({ queryKey: ['local-collections'] }) + } + }) +} + +/** + * Delete a collection + */ +export function useDeleteLocalCollection(bucketId: string) { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: async (id: string) => { + console.log('[useDeleteLocalCollection] Deleting collection:', id) + const response = await window.electronAPI.db.collection.delete(id) + + if (!response.success) { + throw new Error(response.error || 'Failed to delete collection') + } + + return { success: true } + }, + onSuccess: () => { + console.log('[useDeleteLocalCollection] Invalidating collections query') + queryClient.invalidateQueries({ queryKey: ['local-collections', bucketId] }) + } + }) +} + +/** + * Get sync status + */ +export function useLocalSyncStatus() { + return useQuery({ + queryKey: ['local-sync-status'], + queryFn: async () => { + const response = await window.electronAPI.db.sync.getStatus() + + if (!response.success) { + throw new Error(response.error || 'Failed to get sync status') + } + + return response.result + }, + refetchInterval: 10000 // Poll every 10 seconds + }) +} + +/** + * Force sync now + */ +export function useForceSyncNow() { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: async () => { + const response = await window.electronAPI.db.sync.forceSyncNow() + + if (!response.success) { + throw new Error(response.error || 'Failed to force sync') + } + + return { success: true } + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ['local-sync-status'] }) + } + }) +} + +/** + * Get database stats + */ +export function useLocalDatabaseStats() { + return useQuery({ + queryKey: ['local-db-stats'], + queryFn: async () => { + const response = await window.electronAPI.db.getStats() + + if (!response.success) { + throw new Error(response.error || 'Failed to get database stats') + } + + return response.result + }, + refetchInterval: 30000 // Refresh every 30 seconds + }) +} diff --git a/src/renderer/hooks/useLocalEntries.ts b/src/renderer/hooks/useLocalEntries.ts new file mode 100644 index 0000000..865804d --- /dev/null +++ b/src/renderer/hooks/useLocalEntries.ts @@ -0,0 +1,178 @@ +/** + * useLocalEntries - React Query Hooks for Local Database Entries + * These hooks provide access to the local database entries via IPC + */ + +import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' + +/** + * Fetch all entries for a collection from local database + */ +export function useLocalEntries(collectionId: string | undefined) { + return useQuery({ + queryKey: ['local-entries', collectionId], + queryFn: async () => { + if (!collectionId) return [] + + console.log('[useLocalEntries] Fetching entries for collection:', collectionId) + const response = await window.electronAPI.db.entry.list(collectionId) + + if (!response.success) { + throw new Error(response.error || 'Failed to fetch entries') + } + + console.log('[useLocalEntries] Fetched', response.result?.length || 0, 'entries') + return response.result || [] + }, + enabled: !!collectionId + }) +} + +/** + * Fetch a single entry by ID + */ +export function useLocalEntry(entryId: string | undefined) { + return useQuery({ + queryKey: ['local-entry', entryId], + queryFn: async () => { + if (!entryId) return null + + const response = await window.electronAPI.db.entry.get(entryId) + + if (!response.success) { + throw new Error(response.error || 'Failed to fetch entry') + } + + return response.result + }, + enabled: !!entryId + }) +} + +/** + * Create a new entry in local database + */ +export function useCreateLocalEntry(bucketId: string, collectionId: string) { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: async (data: any) => { + console.log('[useCreateLocalEntry] Creating entry:', data.id) + const response = await window.electronAPI.db.entry.create(bucketId, { + ...data, + collection_id: collectionId + }) + + if (!response.success) { + throw new Error(response.error || 'Failed to create entry') + } + + return response.result + }, + onSuccess: data => { + console.log('[useCreateLocalEntry] Entry created successfully:', data.id) + // Invalidate the entries list to refetch + queryClient.invalidateQueries({ queryKey: ['local-entries', collectionId] }) + // Set the entry data in cache + queryClient.setQueryData(['local-entry', data.id], data) + }, + onError: error => { + console.error('[useCreateLocalEntry] Error, rolling back', error) + } + }) +} + +/** + * Update an entry in local database + */ +export function useUpdateLocalEntry(collectionId?: string) { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: async ({ id, updates }: { id: string; updates: any }) => { + console.log('[useUpdateLocalEntry] Updating entry:', id) + const response = await window.electronAPI.db.entry.update(id, updates) + + if (!response.success) { + throw new Error(response.error || 'Failed to update entry') + } + + return response.result + }, + onSuccess: data => { + console.log('[useUpdateLocalEntry] Entry updated successfully:', data.id) + // Invalidate the entries list to refetch + if (collectionId) { + queryClient.invalidateQueries({ queryKey: ['local-entries', collectionId] }) + } + // Update the entry data in cache + queryClient.setQueryData(['local-entry', data.id], data) + }, + onError: error => { + console.error('[useUpdateLocalEntry] Error, rolling back', error) + } + }) +} + +/** + * Delete an entry from local database + */ +export function useDeleteLocalEntry(collectionId: string) { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: async (entryId: string) => { + console.log('[useDeleteLocalEntry] Deleting entry:', entryId) + const response = await window.electronAPI.db.entry.delete(entryId) + + if (!response.success) { + throw new Error(response.error || 'Failed to delete entry') + } + + return entryId + }, + onSuccess: entryId => { + console.log('[useDeleteLocalEntry] Entry deleted successfully:', entryId) + // Invalidate the entries list to refetch + queryClient.invalidateQueries({ queryKey: ['local-entries', collectionId] }) + // Remove the entry from cache + queryClient.removeQueries({ queryKey: ['local-entry', entryId] }) + }, + onError: error => { + console.error('[useDeleteLocalEntry] Error, rolling back', error) + } + }) +} + +/** + * Update entry order in a collection + */ +export function useUpdateLocalEntryOrder(collectionId: string) { + const queryClient = useQueryClient() + + return useMutation({ + mutationFn: async (entryIds: string[]) => { + console.log( + '[useUpdateLocalEntryOrder] Updating order for collection:', + collectionId, + 'entries:', + entryIds.length + ) + const response = await window.electronAPI.db.entry.updateOrder(collectionId, entryIds) + + if (!response.success) { + throw new Error(response.error || 'Failed to update entry order') + } + + return entryIds + }, + onSuccess: () => { + console.log('[useUpdateLocalEntryOrder] Entry order updated successfully') + // Invalidate the entries list to refetch with new order + queryClient.invalidateQueries({ queryKey: ['local-entries', collectionId] }) + }, + onError: error => { + console.error('[useUpdateLocalEntryOrder] Error, rolling back', error) + } + }) +} diff --git a/src/renderer/lib/queryClient.ts b/src/renderer/lib/queryClient.ts new file mode 100644 index 0000000..1a98f86 --- /dev/null +++ b/src/renderer/lib/queryClient.ts @@ -0,0 +1,20 @@ +/** + * React Query Client Configuration - POC + * Minimal setup for proof of concept + */ + +import { QueryClient } from '@tanstack/react-query' + +export const queryClient = new QueryClient({ + defaultOptions: { + queries: { + staleTime: 1000 * 60 * 5, // 5 minutes + gcTime: 1000 * 60 * 10, // 10 minutes (formerly cacheTime) + refetchOnWindowFocus: false, + retry: 1 + }, + mutations: { + retry: 1 + } + } +}) diff --git a/src/renderer/main.tsx b/src/renderer/main.tsx index 201f7ce..0237398 100644 --- a/src/renderer/main.tsx +++ b/src/renderer/main.tsx @@ -2,12 +2,16 @@ import React from 'react' import ReactDOM from 'react-dom/client' import App from './App' import { ThemeProvider } from './contexts/ThemeContext' +import { QueryClientProvider } from '@tanstack/react-query' +import { queryClient } from './lib/queryClient' import './index.css' ReactDOM.createRoot(document.getElementById('root')!).render( - - - + + + + + ) diff --git a/src/test/setup.ts b/src/test/setup.ts index e9ef7ed..ce10ca6 100644 --- a/src/test/setup.ts +++ b/src/test/setup.ts @@ -1,5 +1,98 @@ import { vi } from 'vitest' +// Polyfill Node.js globals for test environment +if (typeof globalThis.global === 'undefined') { + globalThis.global = globalThis +} + +// Mock URL constructor for Node.js compatibility +if (typeof globalThis.URL === 'undefined') { + globalThis.URL = class URL { + constructor(url: string, base?: string) { + return new window.URL(url, base) + } + } +} + +// Mock Node.js Buffer for compatibility +if (typeof globalThis.Buffer === 'undefined') { + globalThis.Buffer = { + isBuffer: () => false, + from: (data: unknown) => new Uint8Array(data as ArrayBufferLike), + alloc: (size: number) => new Uint8Array(size) + } as typeof Buffer +} + +// Mock process for Node.js compatibility +if (typeof globalThis.process === 'undefined') { + globalThis.process = { + env: {}, + version: 'v18.0.0', + platform: 'linux', + nextTick: (fn: () => void) => setTimeout(fn, 0) + } as typeof process +} + +// Mock Node.js util module for webidl-conversions compatibility +if (typeof globalThis.util === 'undefined') { + globalThis.util = { + inspect: (obj: any) => JSON.stringify(obj), + types: { + isAnyArrayBuffer: (obj: any) => obj instanceof ArrayBuffer, + isArrayBuffer: (obj: any) => obj instanceof ArrayBuffer, + isArrayBufferView: (obj: any) => ArrayBuffer.isView(obj), + isUint8Array: (obj: any) => obj instanceof Uint8Array, + isUint8ClampedArray: (obj: any) => obj instanceof Uint8ClampedArray, + isUint16Array: (obj: any) => obj instanceof Uint16Array, + isUint32Array: (obj: any) => obj instanceof Uint32Array, + isInt8Array: (obj: any) => obj instanceof Int8Array, + isInt16Array: (obj: any) => obj instanceof Int16Array, + isInt32Array: (obj: any) => obj instanceof Int32Array, + isFloat32Array: (obj: any) => obj instanceof Float32Array, + isFloat64Array: (obj: any) => obj instanceof Float64Array, + isBigInt64Array: (obj: any) => obj instanceof BigInt64Array, + isBigUint64Array: (obj: any) => obj instanceof BigUint64Array, + isDataView: (obj: any) => obj instanceof DataView + } + } +} + +// Mock Node.js stream module +if (typeof globalThis.stream === 'undefined') { + globalThis.stream = { + Readable: class Readable {}, + Writable: class Writable {}, + Transform: class Transform {}, + PassThrough: class PassThrough {} + } +} + +// Mock Node.js events module +if (typeof globalThis.events === 'undefined') { + globalThis.events = { + EventEmitter: class EventEmitter { + emit() { + return false + } + on() { + return this + } + once() { + return this + } + off() { + return this + } + removeListener() { + return this + } + addListener() { + return this + } + } + } +} + // Mock Electron APIs globally global.window = { ...global.window, @@ -25,12 +118,41 @@ global.window = { }, s3: { configure: vi.fn() + }, + db: { + collection: { + getAll: vi.fn(), + getById: vi.fn(), + create: vi.fn(), + update: vi.fn(), + delete: vi.fn(), + reorder: vi.fn() + }, + entry: { + getAll: vi.fn(), + getById: vi.fn(), + create: vi.fn(), + update: vi.fn(), + delete: vi.fn(), + reorder: vi.fn() + }, + sync: { + getStatus: vi.fn(), + start: vi.fn(), + stop: vi.fn(), + clearQueue: vi.fn() + }, + migration: { + onStarted: vi.fn(), + onComplete: vi.fn(), + onError: vi.fn() + } } }, location: { reload: vi.fn() } -} as any +} as Window & typeof globalThis // Mock window.confirm global.window.confirm = vi.fn(() => true) diff --git a/src/types/electron.d.ts b/src/types/electron.d.ts index 2570de9..dfdac9d 100644 --- a/src/types/electron.d.ts +++ b/src/types/electron.d.ts @@ -72,6 +72,70 @@ export interface ElectronAPI { listAssets: (params: { bucket: string }) => Promise deleteAsset: (params: { bucket: string; asset: any }) => Promise } + // POC: Local Database API + db: { + collection: { + create: ( + bucketId: string, + data: any + ) => Promise<{ success: boolean; result?: any; error?: string }> + update: ( + id: string, + updates: any + ) => Promise<{ success: boolean; result?: any; error?: string }> + get: (id: string) => Promise<{ success: boolean; result?: any; error?: string }> + list: (bucketId: string) => Promise<{ success: boolean; result?: any[]; error?: string }> + delete: (id: string) => Promise<{ success: boolean; error?: string }> + updateOrder: ( + bucketId: string, + collectionIds: string[] + ) => Promise<{ success: boolean; error?: string }> + onUpdated: (callback: (collection: any) => void) => () => void + } + entry: { + create: ( + bucketId: string, + data: any + ) => Promise<{ success: boolean; result?: any; error?: string }> + update: ( + id: string, + updates: any + ) => Promise<{ success: boolean; result?: any; error?: string }> + get: (id: string) => Promise<{ success: boolean; result?: any; error?: string }> + list: (collectionId: string) => Promise<{ success: boolean; result?: any[]; error?: string }> + delete: (id: string) => Promise<{ success: boolean; error?: string }> + updateOrder: ( + collectionId: string, + entryIds: string[] + ) => Promise<{ success: boolean; error?: string }> + onUpdated: (callback: (entry: any) => void) => () => void + onDeleted: (callback: (entryId: string) => void) => () => void + } + sync: { + getStatus: () => Promise<{ success: boolean; result?: any; error?: string }> + forceSyncNow: () => Promise<{ success: boolean; error?: string }> + onStatusChange: (callback: (status: any) => void) => () => void + } + migration: { + onStarted: (callback: (data: { bucketName: string; bucketId: string }) => void) => () => void + onComplete: ( + callback: (data: { + bucketName: string + bucketId: string + success: boolean + error?: string + }) => void + ) => () => void + } + // Database management + clear: () => Promise<{ success: boolean; error?: string }> + getPath: () => Promise<{ success: boolean; result?: string; error?: string }> + getStats: () => Promise<{ success: boolean; result?: any; error?: string }> + // Debug helpers + debug: { + getBucketIds: () => Promise<{ success: boolean; result?: any; error?: string }> + } + } } declare global { diff --git a/vite.config.ts b/vite.config.ts index 7cb1566..cb09a63 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -8,14 +8,15 @@ module.exports = defineConfig({ base: './', build: { outDir: 'dist/renderer', - emptyOutDir: true, + emptyOutDir: true }, resolve: { alias: { - '@': resolve(__dirname, 'src'), - }, + '@': resolve(__dirname, 'src') + } }, server: { port: 5173, - }, + host: '127.0.0.1' // Use IPv4 to avoid macOS firewall issues + } }) diff --git a/vitest.config.ts b/vitest.config.ts index 7c5de92..6fdf723 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -13,5 +13,8 @@ export default defineConfig({ alias: { '@': path.resolve(__dirname, './src') } + }, + define: { + global: 'globalThis' } })