From b89b22fa28d0e14a73c4dfd805c49cd67f13e1a2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 21 Aug 2026 11:38:42 +0000 Subject: [PATCH 1/3] feat(import): CES-70 replace-mode ZIP import (tests outstanding) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Settings → Import data restores a self-produced export ZIP with replace semantics. Header constants are shared from client/lib/export/. Automated tests from spec § Test expectations are not written yet — do not mark CES-70 Done or unblock CES-71 until they land on main. Co-authored-by: JMNofziger --- client/README.md | 7 +- client/lib/app/pages/import_data_section.dart | 495 +++++++++++ client/lib/app/pages/settings_page.dart | 14 +- client/lib/import/apply.dart | 407 +++++++++ client/lib/import/csv_parse.dart | 280 ++++++ client/lib/import/import_errors.dart | 92 ++ client/lib/import/import_service.dart | 170 ++++ client/lib/import/plan.dart | 212 +++++ client/lib/import/validate.dart | 828 ++++++++++++++++++ client/lib/import/zip_read.dart | 201 +++++ client/pubspec.lock | 16 + client/pubspec.yaml | 4 + docs/product/README.md | 2 +- docs/product/delivery-plan-v1.md | 21 +- docs/product/prompts/ces-70-import.md | 66 +- docs/product/ux/UX_IMPLEMENTATION_GAPS.md | 4 +- docs/product/ux/cestovni-views.md | 5 +- docs/specs/ARCHITECTURE.md | 2 +- docs/specs/README.md | 2 +- docs/specs/export-import.md | 8 +- docs/specs/export-v1.md | 2 +- docs/specs/sync-protocol.md | 2 +- tests/export/README.md | 2 +- tests/import/README.md | 26 + 24 files changed, 2809 insertions(+), 59 deletions(-) create mode 100644 client/lib/app/pages/import_data_section.dart create mode 100644 client/lib/import/apply.dart create mode 100644 client/lib/import/csv_parse.dart create mode 100644 client/lib/import/import_errors.dart create mode 100644 client/lib/import/import_service.dart create mode 100644 client/lib/import/plan.dart create mode 100644 client/lib/import/validate.dart create mode 100644 client/lib/import/zip_read.dart create mode 100644 tests/import/README.md diff --git a/client/README.md b/client/README.md index d140eea..69d2caf 100644 --- a/client/README.md +++ b/client/README.md @@ -1,7 +1,7 @@ # Cestovni — mobile client **Stack:** Flutter + Drift (ADR 003). -**Milestones:** M0 closed (CES-36, CES-37). **Android M1 closed** on `main` (`bb1d5d5`, 2026-08-16) — CES-38 math, CES-39 Log/History/vehicles, CES-57/65 prefs, CES-66 Metrics, CES-67 Maint, CES-40 photos. **CES-41 export** on this branch. Next coding: **CES-70** import. See [`docs/product/delivery-plan-v1.md`](../docs/product/delivery-plan-v1.md). +**Milestones:** M0 closed (CES-36, CES-37). **Android M1 closed** on `main` (`bb1d5d5`, 2026-08-16) — CES-38 math, CES-39 Log/History/vehicles, CES-57/65 prefs, CES-66 Metrics, CES-67 Maint, CES-40 photos. **CES-41 export** on `main`. **CES-70 import** implemented (Settings → Import data, `client/lib/import/`); automated tests still outstanding. See [`docs/product/delivery-plan-v1.md`](../docs/product/delivery-plan-v1.md). ## Quick start @@ -28,7 +28,8 @@ client/ log_page.dart # fill-up form + drafts (CES-39) + photos (CES-40) history_page.dart # fuel + maint timeline (CES-39 / CES-67) vehicle_form_page.dart # add/edit vehicle (CES-39) - settings_page.dart # vehicle CRUD + prefs (CES-57) + settings_page.dart # vehicle CRUD + prefs (CES-57) + export/import + import_data_section.dart # CES-70 ZIP import (replace) metrics_page.dart # aggregates + cost chart (CES-66) maintenance_page.dart # maint entry + history (CES-67) debug_page.dart @@ -36,6 +37,7 @@ client/ consumption/ # CES-38 math + validation photos/ # CES-40 receipt photo pipeline export/ # CES-41 ZIP export (CSV + STORE zip + share) + import/ # CES-70 ZIP import (replace; tests outstanding) metrics/ # CES-66 aggregation maintenance/ # CES-67 date-only + history ledger db/ @@ -48,6 +50,7 @@ client/ consumption/ # golden fixtures + module purity photos/ # EXIF strip, TTL, cleanup, no-upload invariant export/ # ZIP golden, streaming, photos excluded + import/ # CES-70 — not written yet (see spec § Test expectations) db/ shell_smoke_test.dart ``` diff --git a/client/lib/app/pages/import_data_section.dart b/client/lib/app/pages/import_data_section.dart new file mode 100644 index 0000000..1069fd0 --- /dev/null +++ b/client/lib/app/pages/import_data_section.dart @@ -0,0 +1,495 @@ +/// Settings → Import data (CES-70). +/// +/// Spec: `docs/specs/export-import.md` § UX. Sits directly under +/// **Export data** in the Backup section. +/// +/// Flow: pick a `.zip` → validate with no writes → confirm dialog showing +/// what comes in and what goes out → apply → summary. Foreground-only, +/// matching export amendment A5. +/// +/// User-facing wording comes from the spec's § User-facing explanation +/// rather than being invented here. +library; + +import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; + +import '../../db/app_database.dart'; +import '../../db/repositories/settings_repository.dart'; +import '../../export/export_service.dart'; +import '../../import/apply.dart'; +import '../../import/import_errors.dart'; +import '../../import/import_service.dart'; +import '../active_vehicle.dart'; +import '../theme/cestovni_primitives.dart'; +import '../theme/cestovni_tokens.dart'; +import '../theme/cestovni_typography.dart'; + +class ImportDataSection extends StatefulWidget { + const ImportDataSection({super.key, required this.db, this.service}); + + final AppDatabase db; + + /// Test hook. Production leaves this null and builds an + /// [ImportService] that uses the platform picker. + final ImportService? service; + + @override + State createState() => _ImportDataSectionState(); +} + +class _ImportDataSectionState extends State { + bool _busy = false; + String? _error; + + Future _run() async { + if (_busy) return; + final active = ActiveVehicleScope.of(context); + final service = widget.service ?? ImportService(db: widget.db); + + setState(() { + _busy = true; + _error = null; + }); + + try { + final Uint8List? bytes = await service.pickArchive(); + if (bytes == null) return; + + final preview = await service.preview(bytes); + if (!mounted) return; + + final typed = await showDialog( + context: context, + barrierDismissible: false, + builder: (_) => _ImportConfirmDialog(db: widget.db, preview: preview), + ); + if (typed == null || !mounted) return; + + final outcome = await service.commit( + preview, + typedConfirmation: typed, + ); + + await _reseedActiveVehicle(active); + if (!mounted) return; + + await showDialog( + context: context, + builder: (_) => _ImportSummaryDialog(outcome: outcome), + ); + } on ImportException catch (e) { + if (mounted) setState(() => _error = e.display); + } catch (_) { + if (mounted) { + setState(() => _error = 'Import failed. Nothing was changed.'); + } + } finally { + if (mounted) setState(() => _busy = false); + } + } + + /// The previously active vehicle id may have been destroyed by the + /// replace, so re-run the shell's seeding rule (CES-57: the persisted + /// default wins when it resolves to a live vehicle). + Future _reseedActiveVehicle(ActiveVehicle active) async { + final live = await VehiclesRepository(widget.db).liveOnce(); + if (live.isEmpty) { + active.setVehicleId(null); + return; + } + final settings = await SettingsRepository(widget.db).getOrBootstrap(); + final defaultId = settings.defaultVehicleId; + final defaultIsLive = + defaultId != null && live.any((v) => v.id == defaultId); + active.setVehicleId(defaultIsLive ? defaultId : live.first.id); + } + + @override + Widget build(BuildContext context) { + final colors = context.cestovniColors; + return Padding( + padding: const EdgeInsets.fromLTRB( + CestovniMetrics.pagePadding, + 0, + CestovniMetrics.pagePadding, + CestovniMetrics.tilePadding, + ), + child: LedgerTile( + onTap: _busy ? null : _run, + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + 'IMPORT', + style: CestovniTypography.labelMono( + color: colors.mutedForeground, + ), + ), + const SizedBox(height: 6), + Text( + 'Import data', + style: Theme.of(context).textTheme.titleMedium, + ), + const SizedBox(height: 4), + Text( + "Replaces this device's history with a backup file.", + style: Theme.of(context).textTheme.bodySmall?.copyWith( + color: colors.mutedForeground, + ), + ), + if (_busy) ...[ + const SizedBox(height: 12), + const LinearProgressIndicator(minHeight: 2), + ], + if (_error != null) ...[ + const SizedBox(height: 8), + Text( + _error!, + style: Theme.of(context).textTheme.bodySmall?.copyWith( + color: colors.destructive, + ), + ), + ], + ], + ), + ), + ); + } +} + +/// Shows what comes in and what goes out side by side, then gates the +/// destructive path behind a typed keyword. +class _ImportConfirmDialog extends StatefulWidget { + const _ImportConfirmDialog({required this.db, required this.preview}); + + final AppDatabase db; + final ImportPreview preview; + + @override + State<_ImportConfirmDialog> createState() => _ImportConfirmDialogState(); +} + +class _ImportConfirmDialogState extends State<_ImportConfirmDialog> { + final _keywordController = TextEditingController(); + bool _exporting = false; + String? _exportNote; + + @override + void dispose() { + _keywordController.dispose(); + super.dispose(); + } + + bool get _confirmed { + if (!widget.preview.requiresTypedConfirmation) return true; + return _keywordController.text == importConfirmationKeyword; + } + + Future _exportFirst() async { + if (_exporting) return; + setState(() { + _exporting = true; + _exportNote = null; + }); + try { + await ExportService(db: widget.db).exportAndShare(); + if (mounted) setState(() => _exportNote = 'Current data exported.'); + } catch (_) { + if (mounted) { + setState(() => _exportNote = 'Export failed. Try again before ' + 'importing.'); + } + } finally { + if (mounted) setState(() => _exporting = false); + } + } + + @override + Widget build(BuildContext context) { + final colors = context.cestovniColors; + final theme = Theme.of(context); + final preview = widget.preview; + final plan = preview.plan; + final footprint = preview.footprint; + final destructive = preview.requiresTypedConfirmation; + + return AlertDialog( + title: Text( + destructive + ? "Replace this device's history?" + : 'Import this backup?', + ), + content: SingleChildScrollView( + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + mainAxisSize: MainAxisSize.min, + children: [ + Text( + destructive + ? 'Importing is a restore, not a merge. This device will ' + 'match the backup exactly, and there is no undo.' + : 'This device has no records yet, so nothing will be ' + 'lost.', + style: theme.textTheme.bodyMedium, + ), + const SizedBox(height: 16), + _CountBlock( + label: 'COMING IN', + counts: plan.incomingCounts, + ), + if (destructive) ...[ + const SizedBox(height: 12), + _CountBlock( + label: 'BEING REPLACED', + counts: footprint.rowCounts, + emphasize: true, + ), + ], + const SizedBox(height: 12), + _MetaLine( + label: 'Backup made', + value: plan.manifest.exportedAtUtc, + ), + _MetaLine( + label: 'Archive key', + value: plan.manifest.userKeyHash, + ), + if (footprint.queuedChanges > 0) + _MetaLine( + label: 'Queued changes discarded', + value: '${footprint.queuedChanges}', + ), + if (footprint.draftsAtRisk > 0) + _MetaLine( + label: 'Unsaved fill-ups discarded', + value: '${footprint.draftsAtRisk}', + ), + if (preview.warnings.isNotEmpty) ...[ + const SizedBox(height: 12), + for (final warning in preview.warnings) + Padding( + padding: const EdgeInsets.only(bottom: 4), + child: Text( + warning.message, + style: theme.textTheme.bodySmall?.copyWith( + color: colors.mutedForeground, + ), + ), + ), + ], + if (destructive) ...[ + const SizedBox(height: 16), + OutlinedButton.icon( + onPressed: _exporting ? null : _exportFirst, + icon: const Icon(Icons.ios_share_outlined, size: 18), + label: const Text('Export current data first'), + ), + if (_exportNote != null) + Padding( + padding: const EdgeInsets.only(top: 6), + child: Text( + _exportNote!, + style: theme.textTheme.bodySmall?.copyWith( + color: colors.mutedForeground, + ), + ), + ), + const SizedBox(height: 16), + Text( + 'Type $importConfirmationKeyword to continue.', + style: theme.textTheme.bodySmall, + ), + const SizedBox(height: 6), + TextField( + controller: _keywordController, + autocorrect: false, + enableSuggestions: false, + textCapitalization: TextCapitalization.characters, + inputFormatters: [ + LengthLimitingTextInputFormatter( + importConfirmationKeyword.length, + ), + ], + decoration: const InputDecoration( + isDense: true, + border: OutlineInputBorder(), + ), + onChanged: (_) => setState(() {}), + ), + ], + ], + ), + ), + actions: [ + TextButton( + onPressed: () => Navigator.of(context).pop(), + child: const Text('Cancel'), + ), + FilledButton( + onPressed: _confirmed + // The service only enforces the keyword when the preview + // says it is required, so passing it unconditionally is + // safe and keeps the dialog contract to a single String. + ? () => Navigator.of(context).pop(importConfirmationKeyword) + : null, + child: Text(destructive ? 'Replace' : 'Import'), + ), + ], + ); + } +} + +class _ImportSummaryDialog extends StatelessWidget { + const _ImportSummaryDialog({required this.outcome}); + + final ImportOutcome outcome; + + @override + Widget build(BuildContext context) { + final theme = Theme.of(context); + return AlertDialog( + title: const Text('Import complete'), + content: SingleChildScrollView( + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + mainAxisSize: MainAxisSize.min, + children: [ + _CountBlock(label: 'WRITTEN', counts: outcome.rowsWritten), + if (outcome.totalReplaced > 0) ...[ + const SizedBox(height: 12), + _CountBlock(label: 'REPLACED', counts: outcome.rowsReplaced), + ], + const SizedBox(height: 12), + if (outcome.queueDiscarded > 0) + _MetaLine( + label: 'Queued changes discarded', + value: '${outcome.queueDiscarded}', + ), + if (outcome.draftsDiscarded > 0) + _MetaLine( + label: 'Unsaved fill-ups discarded', + value: '${outcome.draftsDiscarded}', + ), + if (outcome.photoIdsToDelete.isNotEmpty) + _MetaLine( + label: 'Receipt photos removed', + value: '${outcome.photoIdsToDelete.length}', + ), + const SizedBox(height: 8), + Text( + 'Receipt photos are never part of a backup, so none were ' + 'imported.', + style: theme.textTheme.bodySmall, + ), + ], + ), + ), + actions: [ + FilledButton( + onPressed: () => Navigator.of(context).pop(), + child: const Text('Done'), + ), + ], + ); + } +} + +class _CountBlock extends StatelessWidget { + const _CountBlock({ + required this.label, + required this.counts, + this.emphasize = false, + }); + + final String label; + final Map counts; + final bool emphasize; + + @override + Widget build(BuildContext context) { + final colors = context.cestovniColors; + final theme = Theme.of(context); + return Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + label, + style: CestovniTypography.labelMono( + color: emphasize ? colors.destructive : colors.mutedForeground, + ), + ), + const SizedBox(height: 4), + for (final entry in counts.entries) + Padding( + padding: const EdgeInsets.only(top: 2), + child: Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + Text( + _humanTable(entry.key), + style: theme.textTheme.bodySmall, + ), + Text( + '${entry.value}', + style: theme.textTheme.bodySmall, + ), + ], + ), + ), + ], + ); + } + + static String _humanTable(String table) { + switch (table) { + case 'vehicles': + return 'Vehicles'; + case 'fill_ups': + return 'Fill-ups'; + case 'maintenance_rules': + return 'Reminders'; + case 'maintenance_events': + return 'Maintenance'; + case 'settings': + return 'Preferences'; + default: + return table; + } + } +} + +class _MetaLine extends StatelessWidget { + const _MetaLine({required this.label, required this.value}); + + final String label; + final String value; + + @override + Widget build(BuildContext context) { + final colors = context.cestovniColors; + final theme = Theme.of(context); + return Padding( + padding: const EdgeInsets.only(top: 2), + child: Row( + mainAxisAlignment: MainAxisAlignment.spaceBetween, + children: [ + Text( + label, + style: theme.textTheme.bodySmall?.copyWith( + color: colors.mutedForeground, + ), + ), + const SizedBox(width: 12), + Flexible( + child: Text( + value, + textAlign: TextAlign.right, + style: theme.textTheme.bodySmall, + ), + ), + ], + ), + ); + } +} diff --git a/client/lib/app/pages/settings_page.dart b/client/lib/app/pages/settings_page.dart index 8a7f0db..4ec93c7 100644 --- a/client/lib/app/pages/settings_page.dart +++ b/client/lib/app/pages/settings_page.dart @@ -3,11 +3,13 @@ import 'package:flutter/material.dart'; import '../../db/app_database.dart'; import '../../db/repositories/settings_repository.dart'; import '../../export/export_service.dart'; +import '../../import/import_service.dart'; import '../active_vehicle.dart'; import '../theme/cestovni_primitives.dart'; import '../theme/cestovni_tokens.dart'; import '../theme/cestovni_typography.dart'; import 'debug_page.dart'; +import 'import_data_section.dart'; import 'vehicle_form_page.dart'; /// Settings — pushed route from the shell header gear icon (CES-56). @@ -17,13 +19,22 @@ import 'vehicle_form_page.dart'; /// default vehicle) to [SettingsRepository]. Debug stays reachable /// from inside Settings until the rollback tooling lands (CES-50). class SettingsPage extends StatelessWidget { - const SettingsPage({super.key, required this.db, this.onExport}); + const SettingsPage({ + super.key, + required this.db, + this.onExport, + this.importService, + }); final AppDatabase db; /// Test hook. Production leaves this null and uses [ExportService]. final Future Function()? onExport; + /// Test hook. Production leaves this null so [ImportDataSection] + /// builds an [ImportService] backed by the platform file picker. + final ImportService? importService; + @override Widget build(BuildContext context) { final colors = context.cestovniColors; @@ -48,6 +59,7 @@ class SettingsPage extends StatelessWidget { subtitle: Text('Offline — sign in lands in M3.'), ), _ExportDataSection(db: db, onExport: onExport), + ImportDataSection(db: db, service: importService), const HairlineDivider(), const _SectionLabel(text: 'Developer'), ListTile( diff --git a/client/lib/import/apply.dart b/client/lib/import/apply.dart new file mode 100644 index 0000000..fc377d0 --- /dev/null +++ b/client/lib/import/apply.dart @@ -0,0 +1,407 @@ +/// Applies a validated [ImportPlan] with **replace** semantics (CES-70). +/// +/// Spec: `docs/specs/export-import.md` § Replace semantics. Mode is +/// `replace` and merge is not built — see that spec's § Product +/// decisions for why (export omits tombstones, so a merge could never +/// delete anything). +/// +/// Everything here runs inside one Drift transaction, so any failure +/// leaves the database byte-identical. The single deliberate exception is +/// photo **file** deletion, which the caller performs *after* commit; +/// see [ImportOutcome.photoIdsToDelete]. +library; + +import 'package:drift/drift.dart'; + +import '../db/app_database.dart'; +import '../db/repositories/protocol_writes.dart'; +import 'import_errors.dart'; +import 'plan.dart'; + +/// What an import removed and wrote. Drives the post-import summary. +class ImportOutcome { + const ImportOutcome({ + required this.rowsWritten, + required this.rowsReplaced, + required this.queueDiscarded, + required this.draftsDiscarded, + required this.photoIdsToDelete, + required this.defaultVehicleAdopted, + }); + + /// Rows inserted per table (plus `settings: 1`, updated in place). + final Map rowsWritten; + + /// Rows destroyed per table. + final Map rowsReplaced; + + /// Pending outbox entries cleared. + final int queueDiscarded; + + /// Drafts removed because their vehicle is not in the archive. + final int draftsDiscarded; + + /// `photo_refs.id`s whose rows were deleted; their JPEGs must be + /// removed **after** the transaction commits. + final List photoIdsToDelete; + + /// Whether `settings.default_vehicle_id` from the archive survived + /// validation against the imported vehicles. + final bool defaultVehicleAdopted; + + int get totalWritten => + rowsWritten.values.fold(0, (sum, value) => sum + value); + + int get totalReplaced => + rowsReplaced.values.fold(0, (sum, value) => sum + value); +} + +/// Counts of what a replace would destroy, for the confirm dialog. +class LocalFootprint { + const LocalFootprint({ + required this.rowCounts, + required this.queuedChanges, + required this.draftsAtRisk, + }); + + final Map rowCounts; + final int queuedChanges; + + /// Drafts that would be discarded — computed against the archive's + /// vehicle ids, so this is only meaningful for a specific plan. + final int draftsAtRisk; + + int get totalRows => rowCounts.values.fold(0, (sum, value) => sum + value); + + /// Whether there is any history to lose. Drives whether the typed + /// confirmation is required at all. + bool get isEmpty => totalRows == 0; +} + +class ImportApplier { + ImportApplier( + this.db, { + String Function()? newId, + }) : _newId = newId ?? newUuid; + + final AppDatabase db; + final String Function() _newId; + + /// What replace would destroy, measured against [plan]. + /// + /// Counts every row including soft-deleted ones: a hard `DELETE` takes + /// those too, and the user is entitled to know. + Future measure(ImportPlan plan) async { + final counts = { + 'vehicles': await _countAll(db.vehicles), + 'fill_ups': await _countAll(db.fillUps), + 'maintenance_rules': await _countAll(db.maintenanceRules), + 'maintenance_events': await _countAll(db.maintenanceEvents), + }; + final queued = await _countAll(db.outbox); + final doomed = await _draftsWithoutVehicle(plan.vehicleIds); + return LocalFootprint( + rowCounts: counts, + queuedChanges: queued, + draftsAtRisk: doomed.length, + ); + } + + /// Replace local history with [plan]. + /// + /// Ordering (spec § Replace semantics → Ordering): + /// 1. delete children before parents, + /// 2. clear the outbox, + /// 3. insert parents before children, + /// 4. update `settings` in place — never delete it, or `settings.id` + /// (the local identity) would be destroyed, + /// 5. reconcile drafts against the imported vehicles. + Future apply(ImportPlan plan) async { + try { + return await db.transaction(() async { + final replaced = { + 'maintenance_events': await db.delete(db.maintenanceEvents).go(), + 'fill_ups': await db.delete(db.fillUps).go(), + 'maintenance_rules': await db.delete(db.maintenanceRules).go(), + 'vehicles': await db.delete(db.vehicles).go(), + }; + + // Every outbox row describes a mutation on one of the four tables + // just cleared (the `table` CHECK admits nothing else), so + // keeping them would later push rows the user replaced. + final queueDiscarded = await db.delete(db.outbox).go(); + + await _insertVehicles(plan.vehicles); + await _insertRules(plan.maintenanceRules); + await _insertFillUps(plan.fillUps); + await _insertEvents(plan.maintenanceEvents); + + final adopted = await _updateSettings(plan); + + final reconciled = await _reconcileDrafts(plan.vehicleIds); + + return ImportOutcome( + rowsWritten: { + 'vehicles': plan.vehicles.length, + 'fill_ups': plan.fillUps.length, + 'maintenance_rules': plan.maintenanceRules.length, + 'maintenance_events': plan.maintenanceEvents.length, + 'settings': 1, + }, + rowsReplaced: replaced, + queueDiscarded: queueDiscarded, + draftsDiscarded: reconciled.draftsDeleted, + photoIdsToDelete: reconciled.photoIds, + defaultVehicleAdopted: adopted, + ); + }); + } on ImportException { + rethrow; + } catch (error) { + throw ImportException( + ImportErrorCode.txnFailed, + 'The import could not be applied, so nothing was changed: $error', + ); + } + } + + // ───────────────────────────────────────── inserts + + /// Imported rows are **never-synced**: `user_id` and `row_version` stay + /// null (the server assigns them on first write), `deleted_at` is null + /// because soft-deleted rows never leave in an export, `updated_at` is + /// preserved from the archive, and `mutation_id` is freshly generated + /// because it is local bookkeeping the archive does not carry. + Future _insertVehicles(List rows) async { + await db.batch((batch) { + for (final row in rows) { + batch.insert( + db.vehicles, + VehiclesCompanion.insert( + id: row.id, + userId: const Value(null), + rowVersion: const Value(null), + updatedAt: row.updatedAt, + deletedAt: const Value(null), + mutationId: _newId(), + name: row.name, + make: Value(row.make), + model: Value(row.model), + year: Value(row.year), + vin: Value(row.vin), + fuelType: row.fuelType, + tankCapacityUL: Value(row.tankCapacityUL), + archivedAt: Value(row.archivedAt), + ), + ); + } + }); + } + + Future _insertRules(List rows) async { + await db.batch((batch) { + for (final row in rows) { + batch.insert( + db.maintenanceRules, + MaintenanceRulesCompanion.insert( + id: row.id, + userId: const Value(null), + rowVersion: const Value(null), + updatedAt: row.updatedAt, + deletedAt: const Value(null), + mutationId: _newId(), + vehicleId: row.vehicleId, + name: row.name, + // Canonical meters, verbatim. The column is still named + // `cadence_km` until CES-71 renames it. + cadenceKm: Value(row.cadenceMeters), + cadenceDays: Value(row.cadenceDays), + enabled: Value(row.enabled), + notes: Value(row.notes), + ), + ); + } + }); + } + + Future _insertFillUps(List rows) async { + await db.batch((batch) { + for (final row in rows) { + batch.insert( + db.fillUps, + FillUpsCompanion.insert( + id: row.id, + userId: const Value(null), + rowVersion: const Value(null), + updatedAt: row.updatedAt, + deletedAt: const Value(null), + mutationId: _newId(), + vehicleId: row.vehicleId, + filledAt: row.filledAt, + odometerM: row.odometerM, + volumeUL: row.volumeUL, + totalPriceCents: row.totalPriceCents, + currencyCode: row.currencyCode, + isFull: row.isFull, + missedBefore: Value(row.missedBefore), + odometerReset: Value(row.odometerReset), + notes: Value(row.notes), + ), + ); + } + }); + } + + Future _insertEvents(List rows) async { + await db.batch((batch) { + for (final row in rows) { + batch.insert( + db.maintenanceEvents, + MaintenanceEventsCompanion.insert( + id: row.id, + userId: const Value(null), + rowVersion: const Value(null), + updatedAt: row.updatedAt, + deletedAt: const Value(null), + mutationId: _newId(), + vehicleId: row.vehicleId, + ruleId: Value(row.ruleId), + performedAt: row.performedAt, + odometerM: Value(row.odometerM), + // `cost_cents` and `category` carry SQL defaults (0 and + // 'other'), so Drift models them as optional on insert. + costCents: Value(row.costCents), + currencyCode: row.currencyCode, + category: Value(row.category), + shop: Value(row.shop), + notes: Value(row.notes), + ), + ); + } + }); + } + + // ───────────────────────────────────────── settings + + /// Adopt the archive's display preferences, leaving identity alone. + /// + /// `settings` is updated, never deleted and re-inserted: the row's `id` + /// **is** the local user id, and the archive deliberately omits it + /// (`export-v1.md` § A1). `row_version` and `user_id` are likewise + /// untouched. + Future _updateSettings(ImportPlan plan) async { + final existing = await db.select(db.appSettings).getSingleOrNull(); + final incoming = plan.settings; + + // Only honour a default vehicle that actually arrived; CES-57 + // re-validates against live vehicles anyway, but persisting a + // dangling id here would be storing known-bad data. + final requested = incoming.defaultVehicleId; + final adopted = requested != null && plan.vehicleIds.contains(requested); + + if (existing == null) { + // A fresh install that has not booted the Settings bootstrap yet. + // Generate the local identity here rather than taking one from the + // archive, which does not carry it. + final id = _newId(); + await db.into(db.appSettings).insert( + AppSettingsCompanion.insert( + id: id, + userId: const Value(null), + rowVersion: const Value(null), + updatedAt: incoming.updatedAt, + deletedAt: const Value(null), + mutationId: _newId(), + preferredDistanceUnit: incoming.preferredDistanceUnit, + preferredVolumeUnit: incoming.preferredVolumeUnit, + currencyCode: incoming.currencyCode, + timezone: incoming.timezone, + defaultVehicleId: Value(adopted ? requested : null), + ), + ); + return adopted; + } + + await (db.update(db.appSettings) + ..where((s) => s.id.equals(existing.id))) + .write( + AppSettingsCompanion( + preferredDistanceUnit: Value(incoming.preferredDistanceUnit), + preferredVolumeUnit: Value(incoming.preferredVolumeUnit), + currencyCode: Value(incoming.currencyCode), + timezone: Value(incoming.timezone), + defaultVehicleId: Value(adopted ? requested : null), + updatedAt: Value(incoming.updatedAt), + mutationId: Value(_newId()), + ), + ); + return adopted; + } + + // ───────────────────────────────────────── drafts + photos + + /// Drafts are unsaved typing, not history, so replace does not clear + /// them wholesale. But drafts are looked up by vehicle + /// (`DraftsRepository.openDraftForVehicle`), so a draft whose vehicle + /// the import destroyed is unreachable by construction — and would + /// resurface if that vehicle id ever returned in a later import. + /// Because UUIDs are stable across devices, importing your own archive + /// normally keeps every draft. + /// + /// `photo_refs.draft_id` is a foreign key with no `ON DELETE CASCADE`, + /// so its rows go first. + Future<_DraftReconcile> _reconcileDrafts(Set keptVehicleIds) async { + final doomed = await _draftsWithoutVehicle(keptVehicleIds); + if (doomed.isEmpty) { + return const _DraftReconcile(draftsDeleted: 0, photoIds: []); + } + + final photoIds = []; + for (final draftId in doomed) { + final refs = await (db.select(db.photoRefs) + ..where((p) => p.draftId.equals(draftId))) + .get(); + photoIds.addAll(refs.map((r) => r.id)); + await (db.delete(db.photoRefs) + ..where((p) => p.draftId.equals(draftId))) + .go(); + } + + var deleted = 0; + for (final draftId in doomed) { + deleted += + await (db.delete(db.drafts)..where((d) => d.id.equals(draftId))).go(); + } + + return _DraftReconcile(draftsDeleted: deleted, photoIds: photoIds); + } + + /// Ids of drafts whose `vehicle_id` is absent from [keptVehicleIds]. + /// + /// A draft with a null `vehicle_id` is left alone: it is reachable + /// again as soon as the user picks a vehicle, so it is not orphaned. + Future> _draftsWithoutVehicle(Set keptVehicleIds) async { + final drafts = await db.select(db.drafts).get(); + return drafts + .where((d) { + final vehicleId = d.vehicleId; + return vehicleId != null && !keptVehicleIds.contains(vehicleId); + }) + .map((d) => d.id) + .toList(); + } + + Future _countAll(TableInfo table) async { + final expression = countAll(); + final row = + await (db.selectOnly(table)..addColumns([expression])).getSingle(); + return row.read(expression) ?? 0; + } +} + +class _DraftReconcile { + const _DraftReconcile({required this.draftsDeleted, required this.photoIds}); + + final int draftsDeleted; + final List photoIds; +} diff --git a/client/lib/import/csv_parse.dart b/client/lib/import/csv_parse.dart new file mode 100644 index 0000000..d5aafc3 --- /dev/null +++ b/client/lib/import/csv_parse.dart @@ -0,0 +1,280 @@ +/// RFC 4180 CSV reader + strict scalar coercion for CES-70 import. +/// +/// Spec: `docs/specs/export-import.md` § Input contract → CSV parsing +/// rules. Read-direction mirror of `client/lib/export/csv.dart`. +/// +/// A ZIP is a user-editable file, so coercion is deliberately strict: +/// export writes bare digits with no grouping (`export/derived.dart`), +/// therefore anything else means the cell was edited and we would rather +/// fail loudly than guess. The one concession is boolean case, because +/// spreadsheets upper-case `true`/`false` on save. +/// +/// Pure module: no Flutter, no Drift, no `dart:io`. +library; + +import 'import_errors.dart'; + +/// One parsed CSV record. [line] is the 1-based physical line the record +/// started on, so an error can point at the offending row even when a +/// quoted `notes` field spans several lines. +class CsvRecord { + const CsvRecord({required this.fields, required this.line}); + + final List fields; + final int line; +} + +/// Strip a leading UTF-8 BOM, if present. +String stripBom(String text) => + text.startsWith('\uFEFF') ? text.substring(1) : text; + +/// Split [text] into records. +/// +/// Accepts CRLF **and** LF terminators (a tool may normalize line +/// endings) and handles quoted fields containing either — export quotes +/// any field with a comma, quote, CR or LF, so a single record can span +/// multiple physical lines. +List parseCsv(String text, {required String file}) { + final source = stripBom(text); + final records = []; + final length = source.length; + var index = 0; + var line = 1; + + while (index < length) { + final startLine = line; + final fields = []; + final buffer = StringBuffer(); + var inQuotes = false; + var recordDone = false; + + while (index < length && !recordDone) { + final char = source[index]; + + if (inQuotes) { + if (char == '"') { + if (index + 1 < length && source[index + 1] == '"') { + buffer.write('"'); + index += 2; + continue; + } + inQuotes = false; + index++; + continue; + } + if (char == '\n') line++; + buffer.write(char); + index++; + continue; + } + + if (char == '"' && buffer.isEmpty) { + inQuotes = true; + index++; + continue; + } + if (char == ',') { + fields.add(buffer.toString()); + buffer.clear(); + index++; + continue; + } + if (char == '\r') { + index += (index + 1 < length && source[index + 1] == '\n') ? 2 : 1; + line++; + recordDone = true; + continue; + } + if (char == '\n') { + index++; + line++; + recordDone = true; + continue; + } + buffer.write(char); + index++; + } + + if (inQuotes) { + throw ImportException( + ImportErrorCode.rowMalformed, + 'Unterminated quoted field.', + file: file, + line: startLine, + ); + } + + fields.add(buffer.toString()); + + // A lone empty field means a blank physical line. Export never emits + // one, but a text tool may append it; skipping is tolerant without + // accepting ambiguous data. + final isBlankLine = fields.length == 1 && fields.single.isEmpty; + if (!isBlankLine) { + records.add(CsvRecord(fields: fields, line: startLine)); + } + } + + return records; +} + +// ───────────────────────────────────────────── scalar coercion + +/// Bare optionally-negative integer. Rejects `1.0`, `1,234`, `1 234`, +/// `+5`, and padded forms like `007`. +final RegExp _integerPattern = RegExp(r'^-?(?:0|[1-9][0-9]*)$'); + +/// Requires an explicit UTC designator or numeric offset. Without this +/// an offset-less timestamp would be read as device-local and silently +/// shifted, which is a data-corruption bug rather than a parse error. +final RegExp _timestampPattern = RegExp( + r'^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})$', +); + +Never _invalid( + String message, { + required String file, + required int line, + required String column, +}) { + throw ImportException( + ImportErrorCode.valueInvalid, + message, + file: file, + line: line, + column: column, + ); +} + +int? readNullableInt( + String raw, { + required String file, + required int line, + required String column, +}) { + if (raw.isEmpty) return null; + if (!_integerPattern.hasMatch(raw)) { + _invalid( + 'Expected a whole number with no separators, got "$raw".', + file: file, + line: line, + column: column, + ); + } + return int.parse(raw); +} + +int readRequiredInt( + String raw, { + required String file, + required int line, + required String column, +}) { + final value = readNullableInt(raw, file: file, line: line, column: column); + if (value == null) { + _invalid( + 'Required value is empty.', + file: file, + line: line, + column: column, + ); + } + return value; +} + +/// `true` / `false`, case-insensitive. `1`, `0`, `yes`, `no` are +/// rejected on purpose. +bool readRequiredBool( + String raw, { + required String file, + required int line, + required String column, +}) { + switch (raw.toLowerCase()) { + case 'true': + return true; + case 'false': + return false; + } + _invalid( + 'Expected true or false, got "$raw".', + file: file, + line: line, + column: column, + ); +} + +String? readNullableText(String raw) => raw.isEmpty ? null : raw; + +String readRequiredText( + String raw, { + required String file, + required int line, + required String column, +}) { + if (raw.isEmpty) { + _invalid( + 'Required value is empty.', + file: file, + line: line, + column: column, + ); + } + return raw; +} + +/// Parse an ISO-8601 instant and re-serialize it the way the +/// repositories do (`nowIsoUtc()` → `toUtc().toIso8601String()`). +/// +/// Normalizing matters beyond tidiness: export writes second precision +/// (`...T00:00:00Z`) while local writes carry milliseconds +/// (`...T00:00:00.000Z`). Those two forms sort differently as strings, +/// and History orders by `filled_at` as text — so storing the export +/// form verbatim would interleave imported and locally-created rows +/// incorrectly when their instants tie. +String readTimestampUtc( + String raw, { + required String file, + required int line, + required String column, +}) { + if (raw.isEmpty) { + _invalid( + 'Required timestamp is empty.', + file: file, + line: line, + column: column, + ); + } + if (!_timestampPattern.hasMatch(raw)) { + _invalid( + 'Expected an ISO-8601 timestamp in UTC (for example ' + '2026-01-31T08:15:00Z), got "$raw".', + file: file, + line: line, + column: column, + ); + } + final DateTime parsed; + try { + parsed = DateTime.parse(raw); + } on FormatException { + _invalid( + 'Not a valid timestamp: "$raw".', + file: file, + line: line, + column: column, + ); + } + return parsed.toUtc().toIso8601String(); +} + +String? readNullableTimestampUtc( + String raw, { + required String file, + required int line, + required String column, +}) { + if (raw.isEmpty) return null; + return readTimestampUtc(raw, file: file, line: line, column: column); +} diff --git a/client/lib/import/import_errors.dart b/client/lib/import/import_errors.dart new file mode 100644 index 0000000..a44a435 --- /dev/null +++ b/client/lib/import/import_errors.dart @@ -0,0 +1,92 @@ +/// Typed failures and warnings for CES-70 ZIP import. +/// +/// Spec: `docs/specs/export-import.md` § Error handling. Every +/// [ImportErrorCode] aborts before or inside the transaction and leaves +/// the database unchanged; warnings are surfaced in the confirm dialog +/// or the post-import summary and never block. +/// +/// Pure module: no Flutter, no Drift, no `dart:io`. +library; + +enum ImportErrorCode { + notAZip('E_NOT_A_ZIP'), + missingManifest('E_MISSING_MANIFEST'), + manifestInvalid('E_MANIFEST_INVALID'), + schemaVersionUnsupported('E_SCHEMA_VERSION_UNSUPPORTED'), + photosPresent('E_PHOTOS_PRESENT'), + rowVersionPresent('E_ROW_VERSION_PRESENT'), + missingCsv('E_MISSING_CSV'), + headerMismatch('E_HEADER_MISMATCH'), + rowMalformed('E_ROW_MALFORMED'), + valueInvalid('E_VALUE_INVALID'), + cadenceMissing('E_CADENCE_MISSING'), + fkOrphan('E_FK_ORPHAN'), + duplicateId('E_DUPLICATE_ID'), + settingsRowCount('E_SETTINGS_ROW_COUNT'), + countMismatch('E_COUNT_MISMATCH'), + notConfirmed('E_NOT_CONFIRMED'), + txnFailed('E_TXN_FAILED'); + + const ImportErrorCode(this.wire); + + final String wire; +} + +/// A rejected import. [file], [line], and [column] are populated where +/// the spec asks for them so a user can find the offending cell. +class ImportException implements Exception { + const ImportException( + this.code, + this.message, { + this.file, + this.line, + this.column, + }); + + final ImportErrorCode code; + final String message; + + /// ZIP entry name, e.g. `fill_ups.csv`. + final String? file; + + /// 1-based physical line of the record's first line. + final int? line; + + final String? column; + + /// Short user-facing sentence. Deliberately not the raw code. + String get display { + final where = [ + ?file, + if (line != null) 'line $line', + if (column != null) 'column $column', + ].join(', '); + return where.isEmpty ? message : '$message ($where)'; + } + + @override + String toString() => 'ImportException(${code.wire}): $display'; +} + +enum ImportWarningCode { + unknownEntry('W_UNKNOWN_ENTRY'), + differentSourceKey('W_DIFFERENT_SOURCE_KEY'), + sourceHadPendingOutbox('W_SOURCE_HAD_PENDING_OUTBOX'), + localDataReplaced('W_LOCAL_DATA_REPLACED'), + queueDiscarded('W_QUEUE_DISCARDED'), + draftsDiscarded('W_DRAFTS_DISCARDED'); + + const ImportWarningCode(this.wire); + + final String wire; +} + +class ImportWarning { + const ImportWarning(this.code, this.message); + + final ImportWarningCode code; + final String message; + + @override + String toString() => '${code.wire}: $message'; +} diff --git a/client/lib/import/import_service.dart b/client/lib/import/import_service.dart new file mode 100644 index 0000000..de20f51 --- /dev/null +++ b/client/lib/import/import_service.dart @@ -0,0 +1,170 @@ +/// Orchestrates pick → read → validate → confirm → apply (CES-70). +/// +/// Spec: `docs/specs/export-import.md` § UX, § Replace semantics. +/// +/// Foreground-only, matching export amendment A5: no background service, +/// no completion notification, no new runtime permission. This is the one +/// impure file in `client/lib/import/` — it owns `dart:io`, the platform +/// picker and the photo sandbox so the parser, validator and planner stay +/// testable without a device. +library; + +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:file_picker/file_picker.dart'; + +import '../db/app_database.dart'; +import '../export/user_key_hash.dart'; +import '../photos/photo_store.dart'; +import 'apply.dart'; +import 'import_errors.dart'; +import 'plan.dart'; +import 'validate.dart'; +import 'zip_read.dart'; + +/// Word the user types to confirm a destructive import. +/// +/// English and un-localized because the client has no i18n in v1. If +/// localization lands this must be localized or replaced with a non-text +/// affordance — an English-only destructive gate in a translated UI is a +/// trap (spec § Replace semantics → Confirmation). +const String importConfirmationKeyword = 'REPLACE'; + +/// Picks an archive and returns its bytes, or null when the user +/// cancels. Injectable so widget tests never reach the plugin. +typedef ArchivePicker = Future Function(); + +/// A validated archive plus what applying it would destroy. Produced +/// without writing anything. +class ImportPreview { + const ImportPreview({required this.plan, required this.footprint}); + + final ImportPlan plan; + final LocalFootprint footprint; + + /// Whether the user must type [importConfirmationKeyword]. + /// + /// False on a device with no history — the new-phone path, where there + /// is nothing to lose and friction buys no safety. + bool get requiresTypedConfirmation => !footprint.isEmpty; + + List get warnings => plan.warnings; +} + +class ImportService { + ImportService({ + required this.db, + ArchivePicker? picker, + PhotoStore? photoStore, + ImportApplier? applier, + }) : _picker = picker, + _photoStore = photoStore, + _applier = applier; + + final AppDatabase db; + final ArchivePicker? _picker; + final PhotoStore? _photoStore; + final ImportApplier? _applier; + + ImportApplier get _apply => _applier ?? ImportApplier(db); + + /// Prompt for a `.zip`. Returns null when the user cancels. + Future pickArchive() async { + final injected = _picker; + if (injected != null) return injected(); + + final result = await FilePicker.platform.pickFiles( + type: FileType.custom, + allowedExtensions: const ['zip'], + withData: true, + ); + if (result == null || result.files.isEmpty) return null; + + final picked = result.files.single; + final bytes = picked.bytes; + if (bytes != null) return bytes; + + final path = picked.path; + if (path == null) { + throw const ImportException( + ImportErrorCode.notAZip, + 'That file could not be read.', + ); + } + return File(path).readAsBytes(); + } + + /// Validate [bytes] and measure the local data a replace would + /// destroy. Performs **no** writes. + Future preview(Uint8List bytes) async { + final entries = readZipEntries(bytes, inflate: _inflateRaw); + final plan = buildImportPlan( + entries, + localUserKeyHash: await _localUserKeyHash(), + ); + final footprint = await _apply.measure(plan); + return ImportPreview(plan: plan, footprint: footprint); + } + + /// Apply [preview] with replace semantics. + /// + /// Throws [ImportErrorCode.notConfirmed] when the archive would + /// destroy local history and [typedConfirmation] is not exactly + /// [importConfirmationKeyword]. + /// + /// Photo files for discarded drafts are deleted **after** the + /// transaction commits. That ordering is deliberate: an interruption + /// leaves files with no row, which `PhotoService.sweep` already + /// collects as orphan files. Deleting first would leave rows pointing + /// at missing files if the transaction rolled back. + Future commit( + ImportPreview preview, { + String? typedConfirmation, + }) async { + if (preview.requiresTypedConfirmation && + typedConfirmation != importConfirmationKeyword) { + throw const ImportException( + ImportErrorCode.notConfirmed, + 'Type $importConfirmationKeyword to confirm replacing this ' + "device's history.", + ); + } + + final outcome = await _apply.apply(preview.plan); + await _deleteOrphanedPhotoFiles(outcome.photoIdsToDelete); + return outcome; + } + + Future _deleteOrphanedPhotoFiles(List photoIds) async { + if (photoIds.isEmpty) return; + final store = _photoStore ?? PhotoStore.appSandbox(); + for (final id in photoIds) { + try { + await store.delete(id); + } catch (_) { + // The rows are already gone, so a file left behind is a + // harmless orphan the next photo sweep collects. Never fail a + // committed import over cleanup. + } + } + } + + /// Local `user_key_hash` for the archive-vs-device comparison. + /// + /// Read-only on purpose — `preview` must not write, so this does not + /// bootstrap the settings row. An empty string means "no local + /// identity yet", which suppresses the mismatch warning. + Future _localUserKeyHash() async { + final settings = await db.select(db.appSettings).getSingleOrNull(); + if (settings == null) return ''; + return userKeyHashFromSettingsId(settings.id); + } +} + +/// Raw DEFLATE inflater backed by `dart:io`. Injected into +/// [readZipEntries] so the reader itself stays pure. +Uint8List _inflateRaw(Uint8List deflated, int expectedSize) { + final decoded = ZLibDecoder(raw: true).convert(deflated); + return decoded is Uint8List ? decoded : Uint8List.fromList(decoded); +} diff --git a/client/lib/import/plan.dart b/client/lib/import/plan.dart new file mode 100644 index 0000000..f831478 --- /dev/null +++ b/client/lib/import/plan.dart @@ -0,0 +1,212 @@ +/// Validated, DB-free representation of an export ZIP (CES-70). +/// +/// Spec: `docs/specs/export-import.md` § Suggested layout — `plan.dart` +/// holds parsed + validated rows, counts and warnings, and knows nothing +/// about Drift. `apply.dart` maps these models onto companions. +/// +/// Only **canonical** columns appear here. Derived columns +/// (`odometer_km`, `volume_L`, `*_major`, …) and `*_local` timestamps are +/// required to be present in the header but their values are never read, +/// so there is deliberately nowhere to put them. +/// +/// Pure module: no Flutter, no Drift, no `dart:io`. +library; + +import 'import_errors.dart'; + +class ImportedVehicle { + const ImportedVehicle({ + required this.id, + required this.name, + required this.fuelType, + required this.updatedAt, + this.make, + this.model, + this.year, + this.vin, + this.tankCapacityUL, + this.archivedAt, + }); + + final String id; + final String name; + final String fuelType; + final String updatedAt; + final String? make; + final String? model; + final int? year; + final String? vin; + final int? tankCapacityUL; + final String? archivedAt; +} + +class ImportedFillUp { + const ImportedFillUp({ + required this.id, + required this.vehicleId, + required this.filledAt, + required this.odometerM, + required this.volumeUL, + required this.totalPriceCents, + required this.currencyCode, + required this.isFull, + required this.missedBefore, + required this.odometerReset, + required this.updatedAt, + this.notes, + }); + + final String id; + final String vehicleId; + final String filledAt; + final int odometerM; + final int volumeUL; + final int totalPriceCents; + final String currencyCode; + final bool isFull; + final bool missedBefore; + final bool odometerReset; + final String updatedAt; + final String? notes; +} + +class ImportedMaintenanceRule { + const ImportedMaintenanceRule({ + required this.id, + required this.vehicleId, + required this.name, + required this.enabled, + required this.updatedAt, + this.cadenceMeters, + this.cadenceDays, + this.notes, + }); + + final String id; + final String vehicleId; + final String name; + + /// The `cadence_km` column, which carries canonical **meters** despite + /// its name (`export-v1.md` § A3). Imported verbatim, never converted. + /// Named `cadenceMeters` here so no caller can misread it; the column + /// name is restored in `apply.dart`. + final int? cadenceMeters; + + final int? cadenceDays; + final bool enabled; + final String updatedAt; + final String? notes; +} + +class ImportedMaintenanceEvent { + const ImportedMaintenanceEvent({ + required this.id, + required this.vehicleId, + required this.performedAt, + required this.costCents, + required this.currencyCode, + required this.category, + required this.updatedAt, + this.ruleId, + this.odometerM, + this.shop, + this.notes, + }); + + final String id; + final String vehicleId; + final String? ruleId; + final String performedAt; + final int? odometerM; + final int costCents; + final String currencyCode; + final String category; + final String? shop; + final String? notes; + final String updatedAt; +} + +/// The single `settings` row. Carries no `id`: export omits it because it +/// equals the user id, which is exactly what makes local identity safe to +/// preserve on import. +class ImportedSettings { + const ImportedSettings({ + required this.preferredDistanceUnit, + required this.preferredVolumeUnit, + required this.currencyCode, + required this.timezone, + required this.updatedAt, + this.defaultVehicleId, + }); + + final String preferredDistanceUnit; + final String preferredVolumeUnit; + final String currencyCode; + final String timezone; + final String? defaultVehicleId; + final String updatedAt; +} + +/// `manifest.json`, after the gates in `validate.dart` have passed. +class ImportedManifest { + const ImportedManifest({ + required this.schemaVersion, + required this.exportedAtUtc, + required this.appVersion, + required this.appPlatform, + required this.timezone, + required this.userKeyHash, + required this.outboxPendingCount, + required this.rowCounts, + }); + + final int schemaVersion; + final String exportedAtUtc; + final String appVersion; + final String appPlatform; + final String timezone; + final String userKeyHash; + final int outboxPendingCount; + final Map rowCounts; +} + +/// Everything needed to apply an import, plus everything needed to +/// describe it to the user first. Building a plan performs **no** writes. +class ImportPlan { + ImportPlan({ + required this.manifest, + required this.vehicles, + required this.fillUps, + required this.maintenanceRules, + required this.maintenanceEvents, + required this.settings, + required this.warnings, + }); + + final ImportedManifest manifest; + final List vehicles; + final List fillUps; + final List maintenanceRules; + final List maintenanceEvents; + final ImportedSettings settings; + final List warnings; + + /// Incoming row counts, keyed by table name. + Map get incomingCounts => { + 'vehicles': vehicles.length, + 'fill_ups': fillUps.length, + 'maintenance_rules': maintenanceRules.length, + 'maintenance_events': maintenanceEvents.length, + 'settings': 1, + }; + + int get totalIncomingRows => + vehicles.length + + fillUps.length + + maintenanceRules.length + + maintenanceEvents.length; + + /// Vehicle ids the archive brings. Used to reconcile local drafts and + /// to validate `settings.default_vehicle_id`. + Set get vehicleIds => vehicles.map((v) => v.id).toSet(); +} diff --git a/client/lib/import/validate.dart b/client/lib/import/validate.dart new file mode 100644 index 0000000..b6d63b8 --- /dev/null +++ b/client/lib/import/validate.dart @@ -0,0 +1,828 @@ +/// Manifest gates, header strictness and per-column validation (CES-70). +/// +/// Spec: `docs/specs/export-import.md` § Input contract, § Invariants, +/// § Value validation, § Error handling. +/// +/// Nothing here writes. A plan is only produced when every gate passes, +/// which is what makes "either a valid new state or the database is +/// byte-identical" achievable — by the time `apply.dart` runs, the only +/// remaining failure modes are disk and constraint errors. +/// +/// Pure module: no Flutter, no Drift, no `dart:io`. +library; + +import 'dart:convert'; +import 'dart:typed_data'; + +import '../export/headers.dart'; +import '../export/manifest.dart' show exportSchemaVersion; +import '../photos/photo_export_guard.dart'; +import 'csv_parse.dart'; +import 'import_errors.dart'; +import 'plan.dart'; + +/// Manifest `schema_version` this build understands. +/// +/// Taken from the export module rather than restated, so bumping the +/// export format cannot leave import silently accepting the old one. +const int supportedImportSchemaVersion = exportSchemaVersion; + +const String manifestEntryName = 'manifest.json'; +const String readmeEntryName = 'README_export.txt'; + +const Set _fuelTypes = { + 'gasoline', + 'diesel', + 'lpg', + 'cng', + 'ev_kwh', + 'other', +}; + +const Set _maintenanceCategories = { + 'oil', + 'tires', + 'brakes', + 'inspection', + 'battery', + 'fluid', + 'other', +}; + +const Set _distanceUnits = {'km', 'mi'}; +const Set _volumeUnits = {'L', 'gal'}; + +const Set _imageExtensions = { + '.jpg', + '.jpeg', + '.png', + '.heic', + '.heif', + '.webp', + '.gif', +}; + +final RegExp _uuidPattern = RegExp( + r'^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-' + r'[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$', +); + +final RegExp _currencyPattern = RegExp(r'^[A-Z]{3}$'); + +/// CSV entry name → its authoritative header, taken from the **export** +/// constants so the two contracts cannot drift. +const Map importCsvHeaders = { + 'vehicles.csv': vehiclesCsvHeader, + 'fill_ups.csv': fillUpsCsvHeader, + 'maintenance_rules.csv': maintenanceRulesCsvHeader, + 'maintenance_events.csv': maintenanceEventsCsvHeader, + 'settings.csv': settingsCsvHeader, +}; + +/// Build a validated [ImportPlan] from raw ZIP entries. +/// +/// [localUserKeyHash] is only used to raise +/// [ImportWarningCode.differentSourceKey]; a mismatch never rejects, +/// because device-to-device transfer is the normal case. +ImportPlan buildImportPlan( + Map entries, { + required String localUserKeyHash, +}) { + final warnings = []; + + _guardPhotoContent(entries); + _guardEntrySet(entries, warnings); + + final manifest = _readManifest(entries[manifestEntryName]!); + + // A differing key is the normal device-to-device case, so it warns and + // never rejects (spec § Product decisions → Cross-account imports). An + // empty local hash means this device has no identity yet, so there is + // nothing to compare. + if (localUserKeyHash.isNotEmpty && + manifest.userKeyHash != localUserKeyHash) { + warnings.add(ImportWarning( + ImportWarningCode.differentSourceKey, + 'This archive came from key ${manifest.userKeyHash}; this device is ' + '$localUserKeyHash.', + )); + } + if (manifest.outboxPendingCount > 0) { + warnings.add(ImportWarning( + ImportWarningCode.sourceHadPendingOutbox, + '${manifest.outboxPendingCount} change(s) on the source device had ' + 'not been saved to a server when this archive was made.', + )); + } + + final vehicles = _readVehicles(_records(entries, 'vehicles.csv')); + final rules = _readRules(_records(entries, 'maintenance_rules.csv')); + final fillUps = _readFillUps(_records(entries, 'fill_ups.csv')); + final events = _readEvents(_records(entries, 'maintenance_events.csv')); + final settings = _readSettings(_records(entries, 'settings.csv')); + + _assertCounts(manifest, { + 'vehicles': vehicles.length, + 'fill_ups': fillUps.length, + 'maintenance_rules': rules.length, + 'maintenance_events': events.length, + 'settings': 1, + }); + + _assertReferences( + vehicles: vehicles, + rules: rules, + fillUps: fillUps, + events: events, + ); + + return ImportPlan( + manifest: manifest, + vehicles: vehicles, + fillUps: fillUps, + maintenanceRules: rules, + maintenanceEvents: events, + settings: settings, + warnings: warnings, + ); +} + +// ───────────────────────────────────────────── entry-level gates + +/// Fail closed on anything photo-shaped. Photos are never exported and +/// must never be imported; the guard from CES-40 decides what counts as +/// photo content so import does not re-litigate it. +void _guardPhotoContent(Map entries) { + for (final entry in entries.entries) { + final name = entry.key; + if (isPhotoSandboxPath(name)) { + throw ImportException( + ImportErrorCode.photosPresent, + 'Archive contains photo content, which is never part of an ' + 'export.', + file: name, + ); + } + if (name == 'photo_refs.csv') { + throw ImportException( + ImportErrorCode.photosPresent, + 'Archive contains a photo index, which is never part of an ' + 'export.', + file: name, + ); + } + final lower = name.toLowerCase(); + if (_imageExtensions.any(lower.endsWith)) { + throw ImportException( + ImportErrorCode.photosPresent, + 'Archive contains an image file.', + file: name, + ); + } + if (_looksLikeImageBytes(entry.value)) { + throw ImportException( + ImportErrorCode.photosPresent, + 'Archive contains image data.', + file: name, + ); + } + } +} + +bool _looksLikeImageBytes(Uint8List bytes) { + if (bytes.length >= 3 && + bytes[0] == 0xFF && + bytes[1] == 0xD8 && + bytes[2] == 0xFF) { + return true; + } + return bytes.length >= 4 && + bytes[0] == 0x89 && + bytes[1] == 0x50 && + bytes[2] == 0x4E && + bytes[3] == 0x47; +} + +void _guardEntrySet( + Map entries, + List warnings, +) { + if (!entries.containsKey(manifestEntryName)) { + throw const ImportException( + ImportErrorCode.missingManifest, + 'This file is missing manifest.json, so it is not a Cestovni ' + 'export.', + file: manifestEntryName, + ); + } + for (final csv in importCsvHeaders.keys) { + if (!entries.containsKey(csv)) { + throw ImportException( + ImportErrorCode.missingCsv, + 'Archive is missing $csv.', + file: csv, + ); + } + } + + // Extra entries are ignored rather than rejected: cloud tools add + // `__MACOSX/`, and users add notes next to their backup. Anything + // photo-shaped already failed closed above. + final known = { + manifestEntryName, + readmeEntryName, + ...importCsvHeaders.keys, + }; + for (final name in entries.keys) { + if (known.contains(name)) continue; + warnings.add(ImportWarning( + ImportWarningCode.unknownEntry, + 'Ignored unexpected file in the archive: $name', + )); + } +} + +// ───────────────────────────────────────────── manifest + +ImportedManifest _readManifest(Uint8List bytes) { + final Object? decoded; + try { + decoded = jsonDecode(utf8.decode(bytes)); + } catch (_) { + throw const ImportException( + ImportErrorCode.manifestInvalid, + 'manifest.json is not readable JSON.', + file: manifestEntryName, + ); + } + if (decoded is! Map) { + throw const ImportException( + ImportErrorCode.manifestInvalid, + 'manifest.json is not a JSON object.', + file: manifestEntryName, + ); + } + + final schemaVersion = decoded['schema_version']; + if (schemaVersion is! int) { + throw const ImportException( + ImportErrorCode.manifestInvalid, + 'manifest.json has no schema_version.', + file: manifestEntryName, + ); + } + if (schemaVersion != supportedImportSchemaVersion) { + throw ImportException( + ImportErrorCode.schemaVersionUnsupported, + 'This archive uses export format $schemaVersion; this version of ' + 'Cestovni reads format $supportedImportSchemaVersion. Update the ' + 'app and try again.', + file: manifestEntryName, + ); + } + + final photos = decoded['photos_in_export']; + if (photos != false) { + throw const ImportException( + ImportErrorCode.photosPresent, + 'manifest.json does not declare photos_in_export: false.', + file: manifestEntryName, + ); + } + + if (decoded['max_row_version_seen'] != null) { + throw const ImportException( + ImportErrorCode.rowVersionPresent, + 'This archive carries server row versions, which this version of ' + 'Cestovni cannot import.', + file: manifestEntryName, + ); + } + + final rawCounts = decoded['row_counts']; + if (rawCounts is! Map) { + throw const ImportException( + ImportErrorCode.manifestInvalid, + 'manifest.json has no row_counts.', + file: manifestEntryName, + ); + } + final counts = {}; + for (final entry in rawCounts.entries) { + final value = entry.value; + if (value is! int) { + throw ImportException( + ImportErrorCode.manifestInvalid, + 'row_counts.${entry.key} is not a whole number.', + file: manifestEntryName, + ); + } + counts[entry.key] = value; + } + + final pending = decoded['outbox_pending_count']; + + return ImportedManifest( + schemaVersion: schemaVersion, + exportedAtUtc: _manifestString(decoded, 'exported_at_utc'), + appVersion: _manifestString(decoded, 'app_version'), + appPlatform: _manifestString(decoded, 'app_platform'), + timezone: _manifestString(decoded, 'timezone'), + userKeyHash: _manifestString(decoded, 'user_key_hash'), + outboxPendingCount: pending is int ? pending : 0, + rowCounts: counts, + ); +} + +String _manifestString(Map json, String key) { + final value = json[key]; + if (value is! String || value.isEmpty) { + throw ImportException( + ImportErrorCode.manifestInvalid, + 'manifest.json has no $key.', + file: manifestEntryName, + ); + } + return value; +} + +void _assertCounts(ImportedManifest manifest, Map actual) { + for (final entry in actual.entries) { + final declared = manifest.rowCounts[entry.key]; + if (declared == null) { + throw ImportException( + ImportErrorCode.manifestInvalid, + 'row_counts is missing ${entry.key}.', + file: manifestEntryName, + ); + } + if (declared != entry.value) { + throw ImportException( + ImportErrorCode.countMismatch, + 'Archive declares $declared ${entry.key} row(s) but contains ' + '${entry.value}. The file appears to have been edited.', + file: '${entry.key}.csv', + ); + } + } +} + +// ───────────────────────────────────────────── header + record access + +/// Header-checked records for [file], excluding the header row itself. +_Table _records(Map entries, String file) { + final expected = importCsvHeaders[file]!; + final String text; + try { + text = utf8.decode(entries[file]!); + } on FormatException { + throw ImportException( + ImportErrorCode.rowMalformed, + '$file is not valid UTF-8.', + file: file, + ); + } + + final records = parseCsv(text, file: file); + if (records.isEmpty) { + throw ImportException( + ImportErrorCode.headerMismatch, + '$file is empty.', + file: file, + ); + } + + final actualHeader = records.first.fields.join(','); + if (actualHeader != expected) { + throw ImportException( + ImportErrorCode.headerMismatch, + 'Unexpected columns in $file.\nExpected: $expected\nFound: ' + '$actualHeader', + file: file, + line: records.first.line, + ); + } + + return _Table( + file: file, + columns: expected.split(','), + rows: records.skip(1).toList(), + ); +} + +/// A header-validated CSV: fixed column order plus its data records. +class _Table { + _Table({required this.file, required this.columns, required this.rows}) + : _index = { + for (var i = 0; i < columns.length; i++) columns[i]: i, + }; + + final String file; + final List columns; + final List rows; + final Map _index; + + _Row row(CsvRecord record) { + if (record.fields.length != columns.length) { + throw ImportException( + ImportErrorCode.rowMalformed, + 'Expected ${columns.length} values but found ' + '${record.fields.length}.', + file: file, + line: record.line, + ); + } + return _Row(table: this, record: record); + } +} + +/// One data record, addressed by column name. +class _Row { + const _Row({required this.table, required this.record}); + + final _Table table; + final CsvRecord record; + + String get file => table.file; + int get line => record.line; + + String raw(String column) => record.fields[table._index[column]!]; + + /// `row_version` must always be empty: the client never assigns one + /// before M3, so a populated cell means the archive is not something + /// this build produced. + void assertRowVersionEmpty() { + if (raw('row_version').isNotEmpty) { + throw ImportException( + ImportErrorCode.rowVersionPresent, + 'This archive carries server row versions, which this version of ' + 'Cestovni cannot import.', + file: file, + line: line, + column: 'row_version', + ); + } + } + + String uuid(String column) { + final value = readRequiredText( + raw(column), + file: file, + line: line, + column: column, + ); + if (!_uuidPattern.hasMatch(value)) { + throw ImportException( + ImportErrorCode.valueInvalid, + 'Expected a UUID, got "$value".', + file: file, + line: line, + column: column, + ); + } + return value; + } + + String? nullableUuid(String column) { + if (raw(column).isEmpty) return null; + return uuid(column); + } + + String text(String column, {required int min, required int max}) { + final value = readRequiredText( + raw(column), + file: file, + line: line, + column: column, + ); + _assertLength(value, column: column, min: min, max: max); + return value; + } + + String? nullableText(String column, {int min = 0, required int max}) { + final value = readNullableText(raw(column)); + if (value == null) return null; + _assertLength(value, column: column, min: min, max: max); + return value; + } + + void _assertLength( + String value, { + required String column, + required int min, + required int max, + }) { + if (value.length < min || value.length > max) { + throw ImportException( + ImportErrorCode.valueInvalid, + 'Expected between $min and $max characters, got ${value.length}.', + file: file, + line: line, + column: column, + ); + } + } + + String oneOf(String column, Set allowed) { + final value = readRequiredText( + raw(column), + file: file, + line: line, + column: column, + ); + if (!allowed.contains(value)) { + throw ImportException( + ImportErrorCode.valueInvalid, + 'Expected one of ${allowed.join(', ')}, got "$value".', + file: file, + line: line, + column: column, + ); + } + return value; + } + + String currency(String column) { + final value = readRequiredText( + raw(column), + file: file, + line: line, + column: column, + ); + if (!_currencyPattern.hasMatch(value)) { + throw ImportException( + ImportErrorCode.valueInvalid, + 'Expected a three-letter uppercase currency code, got "$value".', + file: file, + line: line, + column: column, + ); + } + return value; + } + + int nonNegativeInt(String column) { + final value = + readRequiredInt(raw(column), file: file, line: line, column: column); + return _assertAtLeast(value, 0, column); + } + + int? nullableNonNegativeInt(String column) { + final value = + readNullableInt(raw(column), file: file, line: line, column: column); + if (value == null) return null; + return _assertAtLeast(value, 0, column); + } + + int? nullablePositiveInt(String column) { + final value = + readNullableInt(raw(column), file: file, line: line, column: column); + if (value == null) return null; + return _assertAtLeast(value, 1, column); + } + + int? nullableIntInRange(String column, {required int min, required int max}) { + final value = + readNullableInt(raw(column), file: file, line: line, column: column); + if (value == null) return null; + if (value < min || value > max) { + throw ImportException( + ImportErrorCode.valueInvalid, + 'Expected a value between $min and $max, got $value.', + file: file, + line: line, + column: column, + ); + } + return value; + } + + int _assertAtLeast(int value, int min, String column) { + if (value < min) { + throw ImportException( + ImportErrorCode.valueInvalid, + 'Expected a value of at least $min, got $value.', + file: file, + line: line, + column: column, + ); + } + return value; + } + + bool boolean(String column) => + readRequiredBool(raw(column), file: file, line: line, column: column); + + String timestamp(String column) => + readTimestampUtc(raw(column), file: file, line: line, column: column); + + String? nullableTimestamp(String column) => readNullableTimestampUtc( + raw(column), + file: file, + line: line, + column: column, + ); +} + +/// Reject a repeated `id` inside one CSV — it would make the import +/// non-deterministic and, under replace, silently drop a row. +void _assertUniqueId(Set seen, String id, _Row row) { + if (!seen.add(id)) { + throw ImportException( + ImportErrorCode.duplicateId, + 'Duplicate id "$id" in ${row.file}.', + file: row.file, + line: row.line, + column: 'id', + ); + } +} + +// ───────────────────────────────────────────── per-table readers + +List _readVehicles(_Table table) { + final seen = {}; + final out = []; + for (final record in table.rows) { + final row = table.row(record); + row.assertRowVersionEmpty(); + final id = row.uuid('id'); + _assertUniqueId(seen, id, row); + out.add(ImportedVehicle( + id: id, + name: row.text('name', min: 1, max: 80), + make: row.nullableText('make', max: 80), + model: row.nullableText('model', max: 80), + year: row.nullableIntInRange('year', min: 1900, max: 2100), + vin: row.nullableText('vin', max: 32), + fuelType: row.oneOf('fuel_type', _fuelTypes), + tankCapacityUL: row.nullableNonNegativeInt('tank_capacity_uL'), + archivedAt: row.nullableTimestamp('archived_at_utc'), + updatedAt: row.timestamp('updated_at_utc'), + )); + } + return out; +} + +List _readFillUps(_Table table) { + final seen = {}; + final out = []; + for (final record in table.rows) { + final row = table.row(record); + row.assertRowVersionEmpty(); + final id = row.uuid('id'); + _assertUniqueId(seen, id, row); + out.add(ImportedFillUp( + id: id, + vehicleId: row.uuid('vehicle_id'), + filledAt: row.timestamp('filled_at_utc'), + odometerM: row.nonNegativeInt('odometer_m'), + volumeUL: row.nonNegativeInt('volume_uL'), + totalPriceCents: row.nonNegativeInt('total_price_cents'), + currencyCode: row.currency('currency_code'), + isFull: row.boolean('is_full'), + missedBefore: row.boolean('missed_before'), + odometerReset: row.boolean('odometer_reset'), + notes: row.nullableText('notes', max: 500), + updatedAt: row.timestamp('updated_at_utc'), + )); + } + return out; +} + +List _readRules(_Table table) { + final seen = {}; + final out = []; + for (final record in table.rows) { + final row = table.row(record); + row.assertRowVersionEmpty(); + final id = row.uuid('id'); + _assertUniqueId(seen, id, row); + + // `cadence_km` holds canonical meters (export-v1.md § A3). Read + // verbatim — no conversion in either direction. + final cadenceMeters = row.nullablePositiveInt('cadence_km'); + final cadenceDays = row.nullablePositiveInt('cadence_days'); + if (cadenceMeters == null && cadenceDays == null) { + throw ImportException( + ImportErrorCode.cadenceMissing, + 'A maintenance rule needs a distance or a time interval.', + file: row.file, + line: row.line, + ); + } + + out.add(ImportedMaintenanceRule( + id: id, + vehicleId: row.uuid('vehicle_id'), + name: row.text('name', min: 1, max: 80), + cadenceMeters: cadenceMeters, + cadenceDays: cadenceDays, + enabled: row.boolean('enabled'), + notes: row.nullableText('notes', max: 500), + updatedAt: row.timestamp('updated_at_utc'), + )); + } + return out; +} + +List _readEvents(_Table table) { + final seen = {}; + final out = []; + for (final record in table.rows) { + final row = table.row(record); + row.assertRowVersionEmpty(); + final id = row.uuid('id'); + _assertUniqueId(seen, id, row); + out.add(ImportedMaintenanceEvent( + id: id, + vehicleId: row.uuid('vehicle_id'), + ruleId: row.nullableUuid('rule_id'), + performedAt: row.timestamp('performed_at_utc'), + odometerM: row.nullableNonNegativeInt('odometer_m'), + costCents: row.nonNegativeInt('cost_cents'), + currencyCode: row.currency('currency_code'), + category: row.oneOf('category', _maintenanceCategories), + shop: row.nullableText('shop', min: 1, max: 120), + notes: row.nullableText('notes', max: 500), + updatedAt: row.timestamp('updated_at_utc'), + )); + } + return out; +} + +ImportedSettings _readSettings(_Table table) { + if (table.rows.length != 1) { + throw ImportException( + ImportErrorCode.settingsRowCount, + 'Expected exactly one settings row, found ${table.rows.length}.', + file: table.file, + ); + } + final row = table.row(table.rows.single); + row.assertRowVersionEmpty(); + return ImportedSettings( + preferredDistanceUnit: row.oneOf('preferred_distance_unit', _distanceUnits), + preferredVolumeUnit: row.oneOf('preferred_volume_unit', _volumeUnits), + currencyCode: row.currency('currency_code'), + timezone: row.text('timezone', min: 1, max: 64), + defaultVehicleId: row.nullableUuid('default_vehicle_id'), + updatedAt: row.timestamp('updated_at_utc'), + ); +} + +// ───────────────────────────────────────────── referential integrity + +/// Under replace the four history tables are cleared first, so every +/// reference must resolve **within the archive** — there is no +/// "already live locally" fallback. An orphan would be invisible in +/// History (which filters by vehicle), i.e. silent data loss reported as +/// success. +void _assertReferences({ + required List vehicles, + required List rules, + required List fillUps, + required List events, +}) { + final vehicleIds = vehicles.map((v) => v.id).toSet(); + final ruleIds = rules.map((r) => r.id).toSet(); + + for (final rule in rules) { + if (!vehicleIds.contains(rule.vehicleId)) { + throw ImportException( + ImportErrorCode.fkOrphan, + 'Maintenance rule "${rule.name}" refers to a vehicle that is not ' + 'in this archive.', + file: 'maintenance_rules.csv', + ); + } + } + for (final fillUp in fillUps) { + if (!vehicleIds.contains(fillUp.vehicleId)) { + throw ImportException( + ImportErrorCode.fkOrphan, + 'A fill-up refers to a vehicle that is not in this archive.', + file: 'fill_ups.csv', + ); + } + } + for (final event in events) { + if (!vehicleIds.contains(event.vehicleId)) { + throw ImportException( + ImportErrorCode.fkOrphan, + 'A maintenance record refers to a vehicle that is not in this ' + 'archive.', + file: 'maintenance_events.csv', + ); + } + final ruleId = event.ruleId; + if (ruleId != null && !ruleIds.contains(ruleId)) { + throw ImportException( + ImportErrorCode.fkOrphan, + 'A maintenance record refers to a reminder rule that is not in ' + 'this archive.', + file: 'maintenance_events.csv', + ); + } + } +} diff --git a/client/lib/import/zip_read.dart b/client/lib/import/zip_read.dart new file mode 100644 index 0000000..8896f07 --- /dev/null +++ b/client/lib/import/zip_read.dart @@ -0,0 +1,201 @@ +/// Central-directory ZIP reader for CES-70 import. +/// +/// Spec: `docs/specs/export-import.md` § Input contract → ZIP shape. +/// +/// Sizes come from the **central directory**, never the local headers: +/// CES-41 writes with the data-descriptor flag (GP bit 3) set, so the +/// local-header CRC and size fields are zero. Promoted from the CES-41 +/// test helper `client/test/export/zip_read.dart`, which already got +/// this right, and extended with DEFLATE support. +/// +/// Our own exports are always STORE. DEFLATE is accepted defensively so +/// an archive that passed through a cloud-storage tool still opens; the +/// inflate callback is **injected** so this file stays pure (no +/// `dart:io`, no Flutter, no Drift). +library; + +import 'dart:convert'; +import 'dart:typed_data'; + +import 'import_errors.dart'; + +/// Raw-DEFLATE inflater. [expectedSize] is the uncompressed length from +/// the central directory, for implementations that want to pre-size. +typedef Inflate = Uint8List Function(Uint8List deflated, int expectedSize); + +const int _methodStore = 0; +const int _methodDeflate = 8; + +const int _sigEocd = 0x06054b50; +const int _sigCentral = 0x02014b50; +const int _sigLocal = 0x04034b50; + +/// Decoded ZIP entries by name, in central-directory order. +/// +/// Directory entries (names ending in `/`) are skipped. Throws +/// [ImportException] with [ImportErrorCode.notAZip] on any structural +/// problem — a truncated file, a missing EOCD, a bad signature, or a +/// compression method we do not support. +Map readZipEntries( + Uint8List bytes, { + Inflate? inflate, +}) { + final eocd = _findEocd(bytes); + final entryCount = _u16(bytes, eocd + 10); + final cdOffset = _u32(bytes, eocd + 16); + if (cdOffset >= bytes.length) { + throw const ImportException( + ImportErrorCode.notAZip, + 'Central directory offset is outside the file.', + ); + } + + final out = {}; + var pos = cdOffset; + for (var i = 0; i < entryCount; i++) { + if (pos + 46 > bytes.length || _u32(bytes, pos) != _sigCentral) { + throw const ImportException( + ImportErrorCode.notAZip, + 'Damaged central directory entry.', + ); + } + final method = _u16(bytes, pos + 10); + final compressedSize = _u32(bytes, pos + 20); + final uncompressedSize = _u32(bytes, pos + 24); + final nameLen = _u16(bytes, pos + 28); + final extraLen = _u16(bytes, pos + 30); + final commentLen = _u16(bytes, pos + 32); + final localOffset = _u32(bytes, pos + 42); + + if (pos + 46 + nameLen > bytes.length) { + throw const ImportException( + ImportErrorCode.notAZip, + 'Entry name runs past the end of the file.', + ); + } + final name = _decodeName(bytes, pos + 46, nameLen); + + if (!name.endsWith('/')) { + out[name] = _readEntryData( + bytes, + name: name, + localOffset: localOffset, + method: method, + compressedSize: compressedSize, + uncompressedSize: uncompressedSize, + inflate: inflate, + ); + } + + pos += 46 + nameLen + extraLen + commentLen; + } + return out; +} + +Uint8List _readEntryData( + Uint8List bytes, { + required String name, + required int localOffset, + required int method, + required int compressedSize, + required int uncompressedSize, + required Inflate? inflate, +}) { + if (localOffset + 30 > bytes.length || + _u32(bytes, localOffset) != _sigLocal) { + throw ImportException( + ImportErrorCode.notAZip, + 'Damaged local header for "$name".', + ); + } + final localNameLen = _u16(bytes, localOffset + 26); + final localExtraLen = _u16(bytes, localOffset + 28); + final start = localOffset + 30 + localNameLen + localExtraLen; + final end = start + compressedSize; + if (start > bytes.length || end > bytes.length) { + throw ImportException( + ImportErrorCode.notAZip, + 'Entry "$name" is truncated.', + ); + } + final raw = Uint8List.sublistView(bytes, start, end); + + switch (method) { + case _methodStore: + return Uint8List.fromList(raw); + case _methodDeflate: + if (inflate == null) { + throw ImportException( + ImportErrorCode.notAZip, + 'Entry "$name" is DEFLATE-compressed and no inflater was ' + 'provided.', + ); + } + final result = inflate(Uint8List.fromList(raw), uncompressedSize); + if (uncompressedSize != 0 && result.length != uncompressedSize) { + throw ImportException( + ImportErrorCode.notAZip, + 'Entry "$name" did not decompress to its declared size.', + ); + } + return result; + default: + throw ImportException( + ImportErrorCode.notAZip, + 'Entry "$name" uses an unsupported compression method ($method).', + ); + } +} + +/// Scan backwards for the end-of-central-directory record. Scanning from +/// the tail is required because the record is followed by a +/// variable-length comment; we additionally check the declared comment +/// length so a byte sequence inside a comment cannot masquerade as EOCD. +int _findEocd(Uint8List bytes) { + if (bytes.length < 22) { + throw const ImportException( + ImportErrorCode.notAZip, + 'File is too small to be a ZIP archive.', + ); + } + for (var i = bytes.length - 22; i >= 0; i--) { + if (_u32(bytes, i) != _sigEocd) continue; + final commentLen = _u16(bytes, i + 20); + if (i + 22 + commentLen == bytes.length) return i; + } + throw const ImportException( + ImportErrorCode.notAZip, + 'Not a ZIP archive (no end-of-central-directory record).', + ); +} + +String _decodeName(Uint8List bytes, int offset, int length) { + try { + return utf8.decode(Uint8List.sublistView(bytes, offset, offset + length)); + } on FormatException { + throw const ImportException( + ImportErrorCode.notAZip, + 'Entry name is not valid UTF-8.', + ); + } +} + +int _u16(Uint8List b, int o) { + if (o + 2 > b.length) { + throw const ImportException( + ImportErrorCode.notAZip, + 'Unexpected end of archive.', + ); + } + return b[o] | (b[o + 1] << 8); +} + +int _u32(Uint8List b, int o) { + if (o + 4 > b.length) { + throw const ImportException( + ImportErrorCode.notAZip, + 'Unexpected end of archive.', + ); + } + return b[o] | (b[o + 1] << 8) | (b[o + 2] << 16) | (b[o + 3] << 24); +} diff --git a/client/pubspec.lock b/client/pubspec.lock index 05395e7..3598a9a 100644 --- a/client/pubspec.lock +++ b/client/pubspec.lock @@ -201,6 +201,14 @@ packages: url: "https://pub.dev" source: hosted version: "3.1.7" + dbus: + dependency: transitive + description: + name: dbus + sha256: a48d5da28e89bd02196e80d81ed8d7954923d00a0f4a68cc20b575038f023383 + url: "https://pub.dev" + source: hosted + version: "0.7.15" drift: dependency: "direct main" description: @@ -241,6 +249,14 @@ packages: url: "https://pub.dev" source: hosted version: "7.0.1" + file_picker: + dependency: "direct main" + description: + name: file_picker + sha256: "57d9a1dd5063f85fa3107fb42d1faffda52fdc948cefd5fe5ea85267a5fc7343" + url: "https://pub.dev" + source: hosted + version: "10.3.10" file_selector_linux: dependency: transitive description: diff --git a/client/pubspec.yaml b/client/pubspec.yaml index 89fe476..6790779 100644 --- a/client/pubspec.yaml +++ b/client/pubspec.yaml @@ -21,6 +21,10 @@ dependencies: uuid: ^4.4.0 # Share sheet for CES-41 ZIP export (sandbox file → user-chosen dest). share_plus: ^11.0.0 + # ZIP picker for CES-70 import. Only the platform picker lives behind + # this plugin; parsing and applying an archive never touch it, so the + # import module stays testable without a device. + file_picker: ^10.0.0 # Visual system fonts (CES-55) — Fraunces (serif), Inter (sans), # JetBrains Mono (mono). Per cestovni-styling.md §2 / §14 with diff --git a/docs/product/README.md b/docs/product/README.md index 0b7296c..2eb42b2 100644 --- a/docs/product/README.md +++ b/docs/product/README.md @@ -24,7 +24,7 @@ Store listing skeleton, privacy policy outline, in-app Data & privacy bullets: ` ## Delivery (Phase 3 / Stage 5) -Active engineering breakdown, milestone spine, per-vertical backlog with `Spec:` paths, test strategy, and **repo progress**: `[delivery-plan-v1.md](delivery-plan-v1.md)` (RYG checklist is source of truth). Linear epic **[CES-35](https://linear.app/personal-interests-llc/issue/CES-35)**. **M0 closed** (**CES-36** / **CES-37**). **Android M1 closed** on `main` (`bb1d5d5`, 2026-08-16): **CES-38** / **CES-39** / **CES-57** / **CES-65** / **CES-66** / **CES-67** / **CES-40** **Done**. Next Android coding: **CES-41** export — prompt [`prompts/ces-41-export.md`](prompts/ces-41-export.md). Prerequisites **CES-53**–**CES-56** **Done**. Linear MCP/API: [`../linear/mcp-setup.md`](../linear/mcp-setup.md). +Active engineering breakdown, milestone spine, per-vertical backlog with `Spec:` paths, test strategy, and **repo progress**: `[delivery-plan-v1.md](delivery-plan-v1.md)` (RYG checklist is source of truth). Linear epic **[CES-35](https://linear.app/personal-interests-llc/issue/CES-35)**. **M0 closed** (**CES-36** / **CES-37**). **Android M1 closed** on `main` (`bb1d5d5`, 2026-08-16): **CES-38** / **CES-39** / **CES-57** / **CES-65** / **CES-66** / **CES-67** / **CES-40** **Done**. **CES-41** export shipped. In flight: **[CES-70](https://linear.app/personal-interests-llc/issue/CES-70)** ZIP import — prompt [`prompts/ces-70-import.md`](prompts/ces-70-import.md); tests outstanding. Prerequisites **CES-53**–**CES-56** **Done**. Linear MCP/API: [`../linear/mcp-setup.md`](../linear/mcp-setup.md). ## Specs diff --git a/docs/product/delivery-plan-v1.md b/docs/product/delivery-plan-v1.md index 12641b7..e46712f 100644 --- a/docs/product/delivery-plan-v1.md +++ b/docs/product/delivery-plan-v1.md @@ -12,18 +12,18 @@ Stage 5 exit (copied from workflow): **running build with test strategy tied to ## Current focus -**Recommended next coding work:** **[CES-70](https://linear.app/personal-interests-llc/issue/CES-70) ZIP import** — device-to-device restore of the CES-41 archive (merge vs replace is a product lock; spec stub still TBD). **Do not start M3 (CES-42–45)** unless product redirects. Parallel ops: **[CES-63](https://linear.app/personal-interests-llc/issue/CES-63)** iPhone install-doc + T1; **[CES-68](https://linear.app/personal-interests-llc/issue/CES-68)** Android APK anytime for demo distribution. +**In flight:** **[CES-70](https://linear.app/personal-interests-llc/issue/CES-70) ZIP import** — implementation on `main`-bound branch; **automated tests still outstanding** (see M2 rollup). Product locks resolved 2026-08-16: mode is **replace**, cross-account imports stay a warning. **Do not start M3 (CES-42–45)** unless product redirects. Parallel ops: **[CES-63](https://linear.app/personal-interests-llc/issue/CES-63)** iPhone install-doc + T1; **[CES-68](https://linear.app/personal-interests-llc/issue/CES-68)** Android APK anytime for demo distribution. | Track | Issue | Why now | Done when | | ----- | ----- | ------- | --------- | -| **A — M2 import** | **[CES-70](https://linear.app/personal-interests-llc/issue/CES-70)** | CES-41 export shipped; import is the matching restore path | Spec (merge vs replace) + restore of vehicles / fill-ups / maint / settings; photos never imported | +| **A — M2 import** | **[CES-70](https://linear.app/personal-interests-llc/issue/CES-70)** | CES-41 export shipped; import is the matching restore path | Replace-mode restore of vehicles / fill-ups / maint / settings; photos never imported; spec § Test expectations green | | **B — M-dist (ops)** | **[CES-63](https://linear.app/personal-interests-llc/issue/CES-63)** | CI deploy live; install doc + phone T1 still open | [`install-ios.md`](install-ios.md) finalized; iPhone T1 on **CES-62** | **Shipped this cycle:** **CES-41** (Export ZIP — Settings → Export data, STORE ZIP, photos excluded). Earlier: **CES-67** (Maintenance) and **CES-40** (receipt photos). **M3** stays on the spine after import, not before. -**Prompts (executed):** [`prompts/ces-67-maintenance.md`](prompts/ces-67-maintenance.md) · [`prompts/ces-40-photo-pipeline.md`](prompts/ces-40-photo-pipeline.md) · [`prompts/ces-41-export.md`](prompts/ces-41-export.md) +**Prompts (executed):** [`prompts/ces-67-maintenance.md`](prompts/ces-67-maintenance.md) · [`prompts/ces-40-photo-pipeline.md`](prompts/ces-40-photo-pipeline.md) · [`prompts/ces-41-export.md`](prompts/ces-41-export.md) · [`prompts/ces-70-import.md`](prompts/ces-70-import.md) -**Prompt (next coding):** **CES-70 import** — needs a spec (merge vs replace, id collisions). Do **not** start M3 (CES-42–45) until product redirects. +**Spec:** [`../specs/export-import.md`](../specs/export-import.md) — **Complete (v1)**, replace mode locked. Do **not** start M3 (CES-42–45) until product redirects. **Manual checklist (CES-40):** [`ces-40-manual-test.md`](ces-40-manual-test.md) — needs a physical Android device with a camera; not runnable in CI or on the Cloud VM. CES-41 device timing (10k rows / 30 s) rides along with **[CES-68](https://linear.app/personal-interests-llc/issue/CES-68)**. @@ -96,7 +96,7 @@ Rollup mirrors milestones **M0→M5** and verticals **CES-36..CES-47** ([epic CE ### M1 — Local logging + math -- 🟩 **M1 rollup (closed 2026-08-16)** — offline logging usable end-to-end without a server. Log / History / vehicles / settings / metrics / Maint / photos on `main` (`bb1d5d5`). **CES-41 export shipped** (M2); next coding is **CES-70** import, not M3. +- 🟩 **M1 rollup (closed 2026-08-16)** — offline logging usable end-to-end without a server. Log / History / vehicles / settings / metrics / Maint / photos on `main` (`bb1d5d5`). **CES-41 export shipped** (M2). **CES-70 import** is the remaining M2 work (implemented, tests outstanding) — not M3. - 🟩 **CES-38 — Consumption math + golden tests** — **Done in repo on `main`** (2026-05): `client/lib/consumption/`, auto-discovery runner over 20 `tests/math/fixtures/`, module-purity test, validation wired in Log/History save paths. Follow-ups: **CES-51** / **CES-52** (non-blocking). - 🟩 **CES-39 — Fill-up + vehicle UI (core)** — **Done** (repo 2026-05, Linear 2026-07-17): repos + vehicle CRUD + Log/History UI; 121+ tests. Out of scope → **CES-65** / **CES-66** / **CES-67** / **CES-40**. - 🟩 **CES-57 — Settings prefs + default vehicle** — **Done** (PR #9, Linear Done). Display wiring closed as **CES-65**. @@ -106,14 +106,15 @@ Rollup mirrors milestones **M0→M5** and verticals **CES-36..CES-47** ([epic CE - 🟩 **CES-40 — Photo pipeline** — **Done** (2026-08-15): `client/lib/photos/` (orientation bake → 1600 px long edge → JPEG q80 → empty EXIF container → SHA-256), `PhotoRefsRepository` with the 5-per-draft cap, sandbox `photos/.jpg`, TTL sweep (30 d capture / 7 d post-complete / orphan rows + files), Log attach + preview + delete, `android:allowBackup="false"`. Invariant tests: photo bytes never in an outbox payload, `photo_refs` rejected by the outbox `table` CHECK, export path guard. Out of scope: OCR, upload, ZIP export (**CES-41**), PWA-lite photos, `Telemetry.emit` (**CES-46**). Known gaps: iOS backup exclusion is documented, not implemented (ADR 005 — iPhone runs PWA-lite); no "had a receipt" indicator on completed fill-ups (would need a `fill_ups` column). - 🟩 **M1 UX gap closure (blocks CES-39):** 🟩 **[CES-53](https://linear.app/personal-interests-llc/issue/CES-53)** maintenance contract — **repo Done** · 🟩 **[CES-54](https://linear.app/personal-interests-llc/issue/CES-54)** date-only vs `TIMESTAMPTZ` — **repo Done** (2026-04-24) · 🟩 **[CES-55](https://linear.app/personal-interests-llc/issue/CES-55)** visual bootstrap — **repo Done** (2026-04-25) · 🟩 **[CES-56](https://linear.app/personal-interests-llc/issue/CES-56)** shell + active vehicle — **repo Done** (2026-04-25) — parent epic **[CES-35](https://linear.app/personal-interests-llc/issue/CES-35)**. -### M2 — Export +### M2 — Export + import -- 🟩 **M2 rollup** — on-device ZIP export exists; import is **CES-70** (next coding, spec TBD). +- 🟨 **M2 rollup** — on-device ZIP export shipped; import implemented, tests outstanding. - 🟩 **CES-41 — Export ZIP** — **Done in repo** (`client/lib/export/`, Settings → Export data). STORE ZIP (no `archive` write path), A1 headers, `photos_in_export: false`, streaming test over 1 000 lazy rows. Device 10k timing deferred to CES-68. Tests: `client/test/export/` + pointer [`tests/export/README.md`](../../tests/export/README.md). + - 🟨 **CES-70 — ZIP import** — **implemented, not yet tested.** `client/lib/import/` (`zip_read` · `csv_parse` · `validate` · `plan` · `apply` · `import_service`) + Settings → **Import data** (`client/lib/app/pages/import_data_section.dart`). **Replace** semantics per [`export-import.md`](../specs/export-import.md) § Replace semantics: hard-delete the four history tables and re-insert, `settings` updated in place, outbox cleared, drafts kept only when their vehicle survives, photo files deleted post-commit. Header constants shared from `client/lib/export/headers.dart`. **Outstanding before this goes 🟩:** the 17 cases in spec § Test expectations — none are written yet (development-only pass, testing deferred to a manual run by product). ### M3 — Backup + restore -- 🟥 **M3 rollup** — ADR 002 + `sync-protocol` closed in running code + tests. *(Fill-up outbox gate slice + `server/dev-sync-stub/` already on `main`. Remaining: real Postgres/RLS, production API, vehicles/settings/maint enqueue, restore UX. **Do not start until CES-70 is scoped or product redirects.**)* + - 🟥 **M3 rollup** — ADR 002 + `sync-protocol` closed in running code + tests. *(Fill-up outbox gate slice + `server/dev-sync-stub/` already on `main`. Remaining: real Postgres/RLS, production API, vehicles/settings/maint enqueue, restore UX. **Do not start until CES-70 is on `main` or product redirects.**)* - 🟥 **CES-42 — Server Postgres + RLS migrations** — `[tests/rls/](../../tests/rls/)`, `[ci/rls-regression.yml](../../ci/rls-regression.yml)`. - 🟥 **CES-43 — Server API + auth** — contract tests `[tests/contract/](../../tests/contract/)` (managed + self-host). Dev stub on `main` is **not** this ticket. - 🟥 **CES-44 — Backup / outbox (client)** — fill-up enqueue/flush **gate slice on `main`**; vehicles/settings/maint still do not enqueue. Depends on CES-37 + real CES-43. @@ -210,7 +211,7 @@ Leading emoji tracks **exit** state (independent of per-vertical RYG above, but - 🟩 Every vertical above has a Linear issue with a `Spec:` line. *(CES-35 epic + CES-36..CES-47; M0 follow-ups CES-48/49/50 created 2026-04-22, linked to their downstream verticals via `blocks`.)* - 🟨 M0 + M1 land: offline app runs, fill-up works end-to-end, golden math tests green. *(**M0 closed**. **M1 closed** — CES-38/39/57 + CES-65/66/67 + CES-40 all Done. **CES-41 export shipped.** **Next:** CES-70 import — see [Current focus](#current-focus).)* -- 🟩 M2 lands: ZIP export round-trips for a representative fixture. *(CES-41. Import is CES-70.)* +- 🟨 M2 lands: ZIP export round-trips for a representative fixture. *(CES-41 **Done**. CES-70 import implemented; the round-trip proof is one of its outstanding tests.)* - 🟥 M3 lands: backup/restore passes `tests/contract/` + integration fixtures; RLS regression green. - 🟥 M4 lands: `ci/telemetry-gate.`* green; client emits only allow-listed events. - 🟥 M5 lands: migration rollback spec real (not stub); rollback tooling proven against fixture. @@ -233,5 +234,5 @@ When every exit bullet above is 🟩, Stage 5 exit is met — flip workflow perc - `[launch-copy-v1.md](launch-copy-v1.md)` — Stage 4 copy; feeds Stage 6. - `[../specs/platform-compliance-v1.md](../specs/platform-compliance-v1.md)` — compliance posture already signed off. -*Last updated: 2026-08-16 — CES-41 Export ZIP. Next coding: CES-70 import.* +*Last updated: 2026-08-21 — CES-70 ZIP import implemented (replace mode); its automated tests are the remaining M2 work.* diff --git a/docs/product/prompts/ces-70-import.md b/docs/product/prompts/ces-70-import.md index 747eb85..bd26edd 100644 --- a/docs/product/prompts/ces-70-import.md +++ b/docs/product/prompts/ces-70-import.md @@ -1,13 +1,13 @@ # Cursor execution prompt — CES-70 ZIP import -> **Status: READY** (2026-08-16). All product locks resolved — **mode is `replace`**. No prerequisites outstanding. -> **CES-41 is merged** to `main` ([PR #21](https://github.com/JMNofziger/cestovni/pull/21)), so `client/lib/export/` is available to share from. -> Linear **[CES-70](https://linear.app/personal-interests-llc/issue/CES-70)** — **Todo** → set **In Progress** when you start. -> Do **not** pick up M3 (CES-42–45), CES-51, CES-71, or PWA-lite unless the user explicitly redirects. +> **Status: IMPLEMENTED, tests outstanding** (2026-08-21). Mode is `replace`. Code is on `cursor/ces-70-zip-import-40e4` in `client/lib/import/` + Settings → **Import data**. +> **Do not re-implement.** The remaining work is the 17 cases in spec § Test expectations (`client/test/import/` does not exist yet). +> Linear **[CES-70](https://linear.app/personal-interests-llc/issue/CES-70)** — **In Progress** until those tests land on `main`. GitHub PR automation will flip it Done; that is wrong until tests exist. +> Do **not** pick up M3 (CES-42–45), CES-51, CES-71, or PWA-lite unless the user explicitly redirects. **Do not unblock CES-71** until import is on `main` and round-trips `cadence_km`. -**Branch:** cut `cursor/ces-70-import-` from **`main`** -**Spec (normative):** [`docs/specs/export-import.md`](../../specs/export-import.md) — read it end to end before writing code. It is complete; do not re-litigate its decisions. -**Also read:** [`docs/specs/export-v1.md`](../../specs/export-v1.md) § v1 amendments · [`docs/specs/data-model.md`](../../specs/data-model.md) · [`docs/specs/si-units.md`](../../specs/si-units.md) · [`docs/specs/photo-pipeline.md`](../../specs/photo-pipeline.md) · `client/lib/export/` · [`client/lib/photos/photo_export_guard.dart`](../../../client/lib/photos/photo_export_guard.dart) · [`AGENTS.md`](../../../AGENTS.md) +**Branch:** `cursor/ces-70-zip-import-40e4` (cut from `main` at `7e7ae1b`, which includes CES-41 `client/lib/export/`) +**Spec (normative):** [`docs/specs/export-import.md`](../../specs/export-import.md) +**Also read:** [`docs/specs/export-v1.md`](../../specs/export-v1.md) § v1 amendments · `client/lib/export/` · [`client/lib/photos/photo_export_guard.dart`](../../../client/lib/photos/photo_export_guard.dart) --- @@ -15,14 +15,13 @@ | Item | State | |------|-------| -| Last coding | **CES-41** export — merged to `main` ([PR #21](https://github.com/JMNofziger/cestovni/pull/21)); code in `client/lib/export/` | +| Last coding | **CES-70** import — implemented, **not on `main`**, **no tests** | | M1 | **Closed.** Log / History / Metrics / Maint / photos ship on Android | -| M2 | **CES-41 done.** This ticket is the import half | -| Prerequisite | None outstanding — import imports `client/lib/export/headers.dart`, never copies it | +| M2 | **CES-41 done** on `main`. CES-70 import implemented; 17 spec tests remaining | +| Prerequisite | Header constants imported from `client/lib/export/headers.dart` (never copied) | | Parallel (do not do here) | CES-63 iPhone install-doc · CES-68 APK · M3 CES-42–45 · CES-71 cadence rename | -**Git start:** `git fetch origin && git checkout -b cursor/ces-70-import- origin/main` - +**Next coding:** land `client/test/import/` covering spec § Test expectations. Do **not** cut a second implementation branch off stale spec history — that would delete `client/lib/export/`. --- ## Goal @@ -82,14 +81,14 @@ Validate → `DELETE` children before parents (`maintenance_events` → `fill_up --- -## Scope (in) +## Scope (in) — code done; tests remaining -1. `client/lib/import/` split per spec § Suggested layout — pure `csv_parse` / `validate` / `plan`, Drift only in `apply`, IO only in `import_service`. -2. Promote `client/test/export/zip_read.dart` into `client/lib/import/zip_read.dart` (central-directory reader; CES-41 sets the data-descriptor bit, so local-header sizes are zero). Add DEFLATE via an **injected** inflate callback so the parser stays pure. -3. Strict CSV coercion: BOM strip, CRLF **and** LF, RFC 4180, empty = null, booleans case-insensitive `true`/`false` only, integers reject `1.0` / grouping / `+5`. -4. Typed error codes per spec § Error handling (17 codes), each carrying file + line + column where applicable, plus the six warning codes. -5. Settings UI: **Import data** `LedgerTile` under **Export data**; confirm dialog showing incoming counts, destroyed counts, discarded queue count, affected drafts, both hashes, and the typed field when needed; summary afterwards. User-facing wording is drafted in spec § User-facing explanation — reuse it, do not invent new copy. -6. Tests per spec § Test expectations (all 17). +1. ✅ `client/lib/import/` split per spec § Suggested layout — pure `csv_parse` / `validate` / `plan`, Drift only in `apply`, IO only in `import_service`. +2. ✅ `client/lib/import/zip_read.dart` (central-directory reader; injected inflate). +3. ✅ Strict CSV coercion. +4. ✅ 17 error codes + 6 warning codes. +5. ✅ Settings UI: **Import data** under **Export data**. +6. ❌ Tests per spec § Test expectations (all 17) — **this is the remaining work.** ## Scope (out) @@ -109,6 +108,8 @@ Validate → `DELETE` children before parents (`maintenance_events` → `fill_up ## Acceptance +- [x] Implementation in `client/lib/import/` + Settings → Import data (replace) +- [x] `delivery-plan-v1.md` M2 row + Current focus updated (honest: tests outstanding) - [ ] Golden round-trip: export fixture → import into empty DB → canonical columns equal row for row - [ ] Importing the same ZIP twice yields identical state (idempotent, no duplicate `id`s) - [ ] Replace clears prior history: populated DB + disjoint ZIP → exactly the ZIP's rows remain @@ -120,19 +121,20 @@ Validate → `DELETE` children before parents (`maintenance_events` → `fill_up - [ ] Imported rows have `row_version IS NULL`; nothing enqueued - [ ] Atomicity: an induced mid-write failure leaves pre-existing rows intact - [ ] Module-purity + streaming tests present (device timing deferred to CES-68 per export A4) +- [ ] Header-constant drift test (import expected set *is* the export constant set) - [ ] `flutter analyze` + `flutter test --no-pub` + `python3 ci/telemetry-gate.py` green -- [ ] `delivery-plan-v1.md` M2 row + Current focus updated; CES-71 unblocked -- [ ] Linear CES-70 Done + closeout comment - -## Status report (required) - -1. How the ZIP reader handles STORE vs DEFLATE and where the inflate is injected -2. Confirm-dialog contents and how the typed-keyword gate is bypassed on an empty DB -3. How header constants are shared with `client/lib/export/` and how the drift test asserts it -4. Draft/photo reconcile: how post-commit file deletion stays crash-safe -5. Error + warning codes implemented vs spec, and how partial import is prevented -6. Tests added, including the streaming and module-purity proofs -7. Known limits — call out device timing as deferred to CES-68 -8. PR URL + Linear CES-70 state +- [ ] CES-71 unblocked — **only after this is on `main` with a working round-trip** +- [ ] Linear CES-70 Done + closeout comment — **not before tests** + +## Implementation status (2026-08-21) + +1. **ZIP reader.** Central-directory sizes (`client/lib/import/zip_read.dart`). STORE is native; DEFLATE via injected `Inflate`. Production inflater is `ZLibDecoder(raw: true)` in `import_service.dart` (`dart:io`), so the parser stays pure. +2. **Confirm dialog.** Incoming vs replaced counts, both `user_key_hash` values, export-first button. Typed keyword `REPLACE` (`importConfirmationKeyword`). Service enforces the keyword only when `requiresTypedConfirmation` (local history non-empty). Empty DB still shows the dialog; the keyword is not required. +3. **Headers.** `validate.dart` imports `client/lib/export/headers.dart`. **Drift test not written.** +4. **Drafts/photos.** Apply deletes `photo_refs` then drafts inside the txn; returns `photoIdsToDelete`. `ImportService.commit` deletes files **after** commit. Failures are swallowed — `PhotoService.sweep` collects orphans. +5. **Errors.** 17 `ImportErrorCode` values + 6 `ImportWarningCode` values. Validation happens before the txn; apply is one Drift transaction (`E_TXN_FAILED` on failure). +6. **Tests.** None. `client/test/import/` does not exist. Pointer: [`tests/import/README.md`](../../../tests/import/README.md). +7. **Limits.** Device timing deferred to CES-68. Keyword is English-only. Pre-M3 every user with fill-ups has a non-empty outbox — keep "queued changes discarded" quiet. Confirm dialog currently returns the keyword even on empty DB (service skips the check). +8. **PR / Linear.** Filled in on the PR once opened. CES-70 stays **In Progress**. CES-71 stays **Backlog**. Tag: `CES-70 — ZIP import`. diff --git a/docs/product/ux/UX_IMPLEMENTATION_GAPS.md b/docs/product/ux/UX_IMPLEMENTATION_GAPS.md index 354a232..631c814 100644 --- a/docs/product/ux/UX_IMPLEMENTATION_GAPS.md +++ b/docs/product/ux/UX_IMPLEMENTATION_GAPS.md @@ -2,9 +2,9 @@ **Purpose:** Track documentation and product gaps discovered before M1 UI execution so they do not leak into implementation as silent contradictions. -**Gate (closed):** Critical-gap rows **Done** (repo + Linear). **CES-39 Done** (2026-07-17). **CES-65 + CES-66 Done** (repo + Linear 2026-07-22). **CES-67 Done** (2026-08-15). **CES-40 Done** (receipt photos, 2026-08-15) — **M1 verticals all closed.** Next spine item: export (**CES-41**, M2). +**Gate (closed):** Critical-gap rows **Done** (repo + Linear). **CES-39 Done** (2026-07-17). **CES-65 + CES-66 Done** (repo + Linear 2026-07-22). **CES-67 Done** (2026-08-15). **CES-40 Done** (receipt photos, 2026-08-15) — **M1 verticals all closed.** **CES-41** export shipped. Next spine item: import (**CES-70**, M2) — implemented, tests outstanding. -**Last reviewed:** 2026-08-16 (hygiene after `main` `bb1d5d5`; next coding = CES-41 export) +**Last reviewed:** 2026-08-21 (hygiene: CES-70 import implemented, tests outstanding) --- diff --git a/docs/product/ux/cestovni-views.md b/docs/product/ux/cestovni-views.md index 184d04e..a72873b 100644 --- a/docs/product/ux/cestovni-views.md +++ b/docs/product/ux/cestovni-views.md @@ -152,13 +152,14 @@ Screenshot: `screenshots/dark-midnight/settings.png` - Preferences: theme, units, currency, default vehicle. - Vehicle management list with add/edit/delete. -- Data actions (export, destructive reset) are visible and explicit. +- Data actions (export, import, destructive reset) are visible and explicit. **Current implementation anchors** - Preferences currently implemented in `settings_page.dart` (distance/volume/currency/timezone). - Vehicle CRUD is implemented across `vehicle_list_page.dart`, `vehicle_detail_page.dart`, and `vehicle_form_page.dart`. -- Export is **shipped (CES-41)** — Settings → Backup → **Export data** (`client/lib/export/`, photos excluded). Destructive reset remains Later. +- Export is **shipped (CES-41)** — Settings → Backup → **Export data** (`client/lib/export/`, photos excluded). +- Import is **implemented (CES-70)** — Settings → Backup → **Import data** (`client/lib/import/` + `import_data_section.dart`), replace mode, typed `REPLACE` when local history is non-empty. Automated tests outstanding; not on `main` yet. Destructive reset remains Later. **Scope gate** diff --git a/docs/specs/ARCHITECTURE.md b/docs/specs/ARCHITECTURE.md index b6a2a3a..efe428d 100644 --- a/docs/specs/ARCHITECTURE.md +++ b/docs/specs/ARCHITECTURE.md @@ -77,7 +77,7 @@ Parent **CES-22**; children **CES-26–CES-32** (see [`README.md`](README.md)). ## Implementation status (Stage 5) -- **M0 (2026-04-18):** Mobile shell + local Drift database on `main` — repository root [`client/`](../../client/). Matches **ADR 003** (Flutter + Drift) and **`data-model.md`** client tables (`schema_version` **3** as of 2026-06-30: `0001_init` + `0002_add_maintenance_events_category_shop` + `0003_settings_default_vehicle_id`; fresh `onCreate` uses `m.createAll()`). Not yet: export (M2), production server backup (M3), Sentry wiring (M4). CI: [`ci/client-build.yml`](../../ci/client-build.yml); telemetry allow-list gate includes a **Dart source scan** for literal `Telemetry.emit` event names ([`ci/telemetry-gate.py`](../../ci/telemetry-gate.py)). +- **M0 (2026-04-18):** Mobile shell + local Drift database on `main` — repository root [`client/`](../../client/). Matches **ADR 003** (Flutter + Drift) and **`data-model.md`** client tables (`schema_version` **3** as of 2026-06-30: `0001_init` + `0002_add_maintenance_events_category_shop` + `0003_settings_default_vehicle_id`; fresh `onCreate` uses `m.createAll()`). **CES-41 export** is on `main` (`client/lib/export/`). **CES-70 import** is implemented (`client/lib/import/`) with tests outstanding. Not yet: production server backup (M3), Sentry wiring (M4). CI: [`ci/client-build.yml`](../../ci/client-build.yml); telemetry allow-list gate includes a **Dart source scan** for literal `Telemetry.emit` event names ([`ci/telemetry-gate.py`](../../ci/telemetry-gate.py)). ## Related compliance / ops diff --git a/docs/specs/README.md b/docs/specs/README.md index 1e4b8a1..ec39f11 100644 --- a/docs/specs/README.md +++ b/docs/specs/README.md @@ -19,7 +19,7 @@ Each Linear issue should include a **`Spec:`** line pointing to the relevant fil | [`consumption-math.md`](consumption-math.md) | Math, segments, fill-up flags | CES-26 | Complete (v1) | | [`si-units.md`](si-units.md) | Canonical INT storage + conversions | CES-27 | Complete (v1) | | [`export-v1.md`](export-v1.md) | ZIP / CSV export contract | CES-28 | Complete (v1) | -| [`export-import.md`](export-import.md) | Device-to-device ZIP import (zero-server) | CES-70 | Complete (v1) — replace mode locked | +| [`export-import.md`](export-import.md) | Device-to-device ZIP import (zero-server) | CES-70 | Complete (v1) — implemented; tests outstanding | | [`telemetry-allowlist.md`](telemetry-allowlist.md) | Crash + product event allow-list | CES-29 | Complete (v1) | | [`telemetry-events.v1.yaml`](telemetry-events.v1.yaml) | Machine-readable event catalogue | CES-29 | Complete (v1) | | [`photo-pipeline.md`](photo-pipeline.md) | Ephemeral photos / deferred entry | CES-30 | Complete (v1) | diff --git a/docs/specs/export-import.md b/docs/specs/export-import.md index ea718f2..9010733 100644 --- a/docs/specs/export-import.md +++ b/docs/specs/export-import.md @@ -1,6 +1,6 @@ # Spec: Export ZIP import (device-to-device, zero-server) -**Status:** **Complete (v1) — ready for implementation.** All product locks resolved 2026-08-16 (see [§ Product decisions](#product-decisions-locked-2026-08-16)). Mode is **replace**. +**Status:** **Complete (v1) — implemented, automated tests outstanding.** All product locks resolved 2026-08-16 (see [§ Product decisions](#product-decisions-locked-2026-08-16)). Mode is **replace**. Code lives in `client/lib/import/` + Settings → **Import data**; the 17 cases in [§ Test expectations](#test-expectations) are **not written**. CES-71 stays blocked until this is on `main` with a working round-trip. **Linear:** [CES-70](https://linear.app/personal-interests-llc/issue/CES-70) **Depends on:** [CES-41](https://linear.app/personal-interests-llc/issue/CES-41) export ([PR #21](https://github.com/JMNofziger/cestovni/pull/21), `client/lib/export/`) **Blocks:** [CES-71](https://linear.app/personal-interests-llc/issue/CES-71) (`cadence_km` → `cadence_m` rename) @@ -19,11 +19,11 @@ Let a user move their structured history between two devices with **zero server* ## Current state vs expected outcome -| | Current | Expected after CES-70 | +| | Current | Done when CES-70 is 🟩 | |---|---|---| | Export | Settings → **Export data** writes a STORE ZIP: `manifest.json`, `README_export.txt`, five CSVs | unchanged | -| Import | none — a ZIP is a dead end inside the app | Settings → **Import data** replaces local history from a ZIP the app produced | -| Mode | undecided | **Replace** — destructive restore, typed confirmation when there is data to lose | +| Import | Settings → **Import data** (`client/lib/import/` + `import_data_section.dart`) replaces local history from a ZIP the app produced. **Not on `main` yet.** Automated tests in `client/test/import/` do not exist. | Same code on `main`; [§ Test expectations](#test-expectations) green | +| Mode | **Replace** — destructive restore, typed `REPLACE` when there is data to lose. Merge is not built. | unchanged | | Photos | never exported (`photos_in_export: false`) | never imported; a photo-shaped ZIP **fails closed** | | `row_version` | client never writes it; CSV cells empty, manifest `null` | imported rows keep `row_version = NULL` (never synced) | | Identity | `user_key_hash` = SHA-256(`settings.id`)[:8]; `settings.id` is **not** in the ZIP | local `settings.id` never overwritten; source hash advisory only | diff --git a/docs/specs/export-v1.md b/docs/specs/export-v1.md index 553e09b..59b5450 100644 --- a/docs/specs/export-v1.md +++ b/docs/specs/export-v1.md @@ -216,7 +216,7 @@ Both targets are validated by `tests/export/` fixtures. ## Non-goals (v1) - **No incremental / diff exports.** Every export is the full structured state. -- **No re-import tool in-app (v1).** The canonical columns + manifest make third-party re-import possible today; we do not build UX for it in v1. **This is deferred, not permanent** — a device-to-device ZIP import (zero-server sync path for cost-conscious/self-host-only users) is a named v1.x+ roadmap item; see [`sync-protocol.md` — v1.x roadmap](sync-protocol.md#v1x-roadmap-pointer-only--tbd-in-this-pass) and [CES-70](https://linear.app/personal-interests-llc/issue/CES-70). +- **No re-import tool in-app (v1) — historical.** This non-goal was deferred, not permanent. In-app restore is [CES-70](https://linear.app/personal-interests-llc/issue/CES-70): Settings → **Import data**, specified in [`export-import.md`](export-import.md) (replace mode). Implementation is in `client/lib/import/`; automated tests are still outstanding and the code is not on `main` yet. Merge / live multi-device sync remains a v1.x item in [`sync-protocol.md`](sync-protocol.md#v1x-roadmap-pointer-only--tbd-in-this-pass). - **No photos.** Period. - **No signed / encrypted ZIP.** User-managed; they can encrypt after export if they want. diff --git a/docs/specs/sync-protocol.md b/docs/specs/sync-protocol.md index 52f156d..df8ee41 100644 --- a/docs/specs/sync-protocol.md +++ b/docs/specs/sync-protocol.md @@ -211,7 +211,7 @@ Dead-letter is a **signal** that something is wrong — either a client bug or a - **Push channels.** Options: long-poll on `/changes`, server-sent events, WebSocket. Decision deferred to when real-time UX requirements are set. - **Conflict UX.** How to surface (if ever) a rejected amendment to the user. Deferred with the above. - **Re-evaluation of managed sync runtime.** Gate defined in ADR 002 revisit gates. -- **Device-to-device import from a self-produced export ZIP** ([CES-70](https://linear.app/personal-interests-llc/issue/CES-70)) — a real, named roadmap item, not a permanent non-goal: parse the canonical CSV columns from an export ZIP (see [`export-v1.md`](export-v1.md)) and upsert into local SQLite on a second device, so cost-conscious or self-host-only users can move data between devices with **zero server**. Needs its own spec pass before implementation — at minimum: `id` collision handling across devices, how imported rows interact with `row_version`/outbox state for never-synced data, and duplicate-detection UX. Not specified or built in this pass. +- **Device-to-device import from a self-produced export ZIP** ([CES-70](https://linear.app/personal-interests-llc/issue/CES-70)) — specified in [`export-import.md`](export-import.md) (replace, not merge; no outbox enqueue; `row_version` stays `NULL`). Implementation is in `client/lib/import/` + Settings → **Import data**. Automated tests and merge to `main` are still outstanding. This is **not** live multi-device sync: merge rules, tombstones, and conflict UX remain unspecified and must not be inferred from replace-mode restore. ## References diff --git a/tests/export/README.md b/tests/export/README.md index fb3f1fb..2a12c0d 100644 --- a/tests/export/README.md +++ b/tests/export/README.md @@ -16,7 +16,7 @@ them up without a second runner: **Not in CI (per `export-v1.md` § A4):** 10 000-row / 30 s / 10 MB device timing. That pass moves to [CES-68](https://linear.app/personal-interests-llc/issue/CES-68). -**Not in this folder:** ZIP import ([CES-70](https://linear.app/personal-interests-llc/issue/CES-70)). +**Import (CES-70):** code is in `client/lib/import/`; tests are **not written**. Pointer: [`../import/README.md`](../import/README.md). Run them with: diff --git a/tests/import/README.md b/tests/import/README.md new file mode 100644 index 0000000..c128c2a --- /dev/null +++ b/tests/import/README.md @@ -0,0 +1,26 @@ +# tests/import — pointer + +`docs/specs/export-import.md` § Test expectations places import tests +in `tests/import/` (or a pointer from here). They will live in the +Flutter client so `flutter test` picks them up without a second runner: + +| Spec expectation | Implementation | +|------------------|----------------| +| Golden round-trip | **Not written** — expected `client/test/import/` | +| Idempotency, replace, settings in-place, outbox, drafts | **Not written** | +| Header / photo / duplicate-id / FK / value rejects | **Not written** | +| Never-synced state, atomicity, confirmation gate | **Not written** | +| Module purity + streaming | **Not written** | +| Header constants == export | **Not written** — must import `client/lib/export/headers.dart`, never copy | + +**Code is implemented** in `client/lib/import/` + Settings → **Import data** +(`client/lib/app/pages/import_data_section.dart`). The 17 cases are the +remaining CES-70 work before this goes 🟩. Do not mark [CES-70](https://linear.app/personal-interests-llc/issue/CES-70) Done, and do not unblock [CES-71](https://linear.app/personal-interests-llc/issue/CES-71), until those tests exist and the code is on `main`. + +**Not in this folder:** ZIP export ([CES-41](https://linear.app/personal-interests-llc/issue/CES-41)) — see [`../export/README.md`](../export/README.md). + +When tests land, run: + +```bash +cd client && flutter test --no-pub test/import/ test/app/settings_page_test.dart +``` From 68b6ee6141253c6847f498c8721647039999a168 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 21 Aug 2026 11:39:57 +0000 Subject: [PATCH 2/3] docs(CES-70): record PR #25 and Linear In Progress in the execution prompt Co-authored-by: JMNofziger --- docs/product/prompts/ces-70-import.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/product/prompts/ces-70-import.md b/docs/product/prompts/ces-70-import.md index bd26edd..3af9c1e 100644 --- a/docs/product/prompts/ces-70-import.md +++ b/docs/product/prompts/ces-70-import.md @@ -135,6 +135,6 @@ Validate → `DELETE` children before parents (`maintenance_events` → `fill_up 5. **Errors.** 17 `ImportErrorCode` values + 6 `ImportWarningCode` values. Validation happens before the txn; apply is one Drift transaction (`E_TXN_FAILED` on failure). 6. **Tests.** None. `client/test/import/` does not exist. Pointer: [`tests/import/README.md`](../../../tests/import/README.md). 7. **Limits.** Device timing deferred to CES-68. Keyword is English-only. Pre-M3 every user with fill-ups has a non-empty outbox — keep "queued changes discarded" quiet. Confirm dialog currently returns the keyword even on empty DB (service skips the check). -8. **PR / Linear.** Filled in on the PR once opened. CES-70 stays **In Progress**. CES-71 stays **Backlog**. +8. **PR / Linear.** Draft [PR #25](https://github.com/JMNofziger/cestovni/pull/25). CES-70 **In Progress**. CES-71 **Backlog** (automation had wrongly marked both Done). Tag: `CES-70 — ZIP import`. From db26f3644f455bb3465330d8faf99ace8a39aae3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 21 Aug 2026 12:13:16 +0000 Subject: [PATCH 3/3] =?UTF-8?q?test(import):=20CES-70=20spec=20=C2=A7=20Te?= =?UTF-8?q?st=20expectations=20(no=20production=20changes)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Land client/test/import/ covering the 17 spec cases. flutter analyze clean; flutter test --no-pub 303 passed / 1 skipped (E2E); telemetry-gate PASS. CES-70 stays In Progress until this is on main. Co-authored-by: JMNofziger --- client/README.md | 6 +- client/test/import/_zip.dart | 190 +++++++++++++ client/test/import/headers_drift_test.dart | 28 ++ client/test/import/module_purity_test.dart | 84 ++++++ client/test/import/rejects_test.dart | 165 +++++++++++ client/test/import/replace_test.dart | 311 +++++++++++++++++++++ client/test/import/round_trip_test.dart | 162 +++++++++++ client/test/import/streaming_test.dart | 155 ++++++++++ docs/product/README.md | 2 +- docs/product/delivery-plan-v1.md | 12 +- docs/product/prompts/ces-70-import.md | 64 ++--- docs/product/ux/UX_IMPLEMENTATION_GAPS.md | 4 +- docs/product/ux/cestovni-views.md | 2 +- docs/specs/ARCHITECTURE.md | 2 +- docs/specs/README.md | 2 +- docs/specs/export-import.md | 4 +- docs/specs/export-v1.md | 2 +- docs/specs/sync-protocol.md | 2 +- tests/import/README.md | 33 ++- 19 files changed, 1166 insertions(+), 64 deletions(-) create mode 100644 client/test/import/_zip.dart create mode 100644 client/test/import/headers_drift_test.dart create mode 100644 client/test/import/module_purity_test.dart create mode 100644 client/test/import/rejects_test.dart create mode 100644 client/test/import/replace_test.dart create mode 100644 client/test/import/round_trip_test.dart create mode 100644 client/test/import/streaming_test.dart diff --git a/client/README.md b/client/README.md index 69d2caf..bfd3048 100644 --- a/client/README.md +++ b/client/README.md @@ -1,7 +1,7 @@ # Cestovni — mobile client **Stack:** Flutter + Drift (ADR 003). -**Milestones:** M0 closed (CES-36, CES-37). **Android M1 closed** on `main` (`bb1d5d5`, 2026-08-16) — CES-38 math, CES-39 Log/History/vehicles, CES-57/65 prefs, CES-66 Metrics, CES-67 Maint, CES-40 photos. **CES-41 export** on `main`. **CES-70 import** implemented (Settings → Import data, `client/lib/import/`); automated tests still outstanding. See [`docs/product/delivery-plan-v1.md`](../docs/product/delivery-plan-v1.md). +**Milestones:** M0 closed (CES-36, CES-37). **Android M1 closed** on `main` (`bb1d5d5`, 2026-08-16) — CES-38 math, CES-39 Log/History/vehicles, CES-57/65 prefs, CES-66 Metrics, CES-67 Maint, CES-40 photos. **CES-41 export** on `main`. **CES-70 import** implemented (Settings → Import data, `client/lib/import/`); spec tests green on PR #25, not on `main`. See [`docs/product/delivery-plan-v1.md`](../docs/product/delivery-plan-v1.md). ## Quick start @@ -37,7 +37,7 @@ client/ consumption/ # CES-38 math + validation photos/ # CES-40 receipt photo pipeline export/ # CES-41 ZIP export (CSV + STORE zip + share) - import/ # CES-70 ZIP import (replace; tests outstanding) + import/ # CES-70 ZIP import (replace; spec tests in test/import/) metrics/ # CES-66 aggregation maintenance/ # CES-67 date-only + history ledger db/ @@ -50,7 +50,7 @@ client/ consumption/ # golden fixtures + module purity photos/ # EXIF strip, TTL, cleanup, no-upload invariant export/ # ZIP golden, streaming, photos excluded - import/ # CES-70 — not written yet (see spec § Test expectations) + import/ # CES-70 spec § Test expectations db/ shell_smoke_test.dart ``` diff --git a/client/test/import/_zip.dart b/client/test/import/_zip.dart new file mode 100644 index 0000000..dcb6521 --- /dev/null +++ b/client/test/import/_zip.dart @@ -0,0 +1,190 @@ +import 'dart:convert'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:cestovni/db/app_database.dart'; +import 'package:cestovni/export/app_version.dart'; +import 'package:cestovni/export/csv.dart'; +import 'package:cestovni/export/snapshot.dart'; +import 'package:cestovni/export/store_zip_sink.dart'; +import 'package:cestovni/import/csv_parse.dart'; +import 'package:cestovni/import/import_errors.dart'; +import 'package:cestovni/import/import_service.dart'; +import 'package:cestovni/import/zip_read.dart'; +import 'package:cestovni/photos/photo_store.dart'; +import 'package:flutter_test/flutter_test.dart'; + +/// STORE-ZIP a live database the same way CES-41 export does. +Future exportDbToZip(AppDatabase db) async { + final dir = Directory.systemTemp.createTempSync('cestovni-import-export-'); + final file = File('${dir.path}/export.zip'); + final sink = FileZipSink(file) + ..stamp = DateTime.utc(2026, 8, 16, 12, 0, 0); + writeSnapshotToSink( + sink: sink, + snapshot: await takeExportSnapshot(db), + appVersion: kAppVersion, + exportedAt: DateTime.utc(2026, 8, 16, 12, 0, 0), + ); + sink.close(); + final bytes = file.readAsBytesSync(); + dir.deleteSync(recursive: true); + return Uint8List.fromList(bytes); +} + +/// Pack already-decoded ZIP entries back into a STORE ZIP. +Uint8List packZip(Map entries) { + final dir = Directory.systemTemp.createTempSync('cestovni-import-pack-'); + final file = File('${dir.path}/pack.zip'); + final sink = FileZipSink(file) + ..stamp = DateTime.utc(2026, 8, 16, 12, 0, 0); + for (final entry in entries.entries) { + sink.startFile(entry.key); + sink.add(entry.value); + sink.closeFile(); + } + sink.close(); + final bytes = file.readAsBytesSync(); + dir.deleteSync(recursive: true); + return Uint8List.fromList(bytes); +} + +Map unpackZip(Uint8List bytes) => readZipEntries(bytes); + +int csvColumn(String header, String name) { + final index = header.split(',').indexOf(name); + if (index < 0) { + throw StateError('$name is not in $header'); + } + return index; +} + +/// Replace one data-row cell. [dataRow] is 0-based among data records +/// (header is row 0 of the file, skipped here). +Uint8List mutateCsvCell({ + required Uint8List csv, + required String file, + required int dataRow, + required String column, + required String value, +}) { + final records = parseCsv(utf8.decode(csv), file: file); + if (records.isEmpty) { + throw StateError('$file has no header'); + } + final header = records.first.fields.join(','); + final col = csvColumn(header, column); + final target = records[dataRow + 1]; + final fields = List.from(target.fields); + fields[col] = value; + final rebuilt = [ + for (var i = 0; i < records.length; i++) + i == dataRow + 1 + ? CsvRecord(fields: fields, line: target.line) + : records[i], + ]; + return _csvBytes(rebuilt); +} + +Uint8List duplicateCsvDataRow({ + required Uint8List csv, + required String file, + int dataRow = 0, +}) { + final records = parseCsv(utf8.decode(csv), file: file); + final copy = records[dataRow + 1]; + return _csvBytes([...records, copy]); +} + +Uint8List _csvBytes(List records) { + final builder = BytesBuilder(); + builder.add(utf8Bom); + for (final record in records) { + builder.add(csvRowBytes(record.fields)); + } + return builder.takeBytes(); +} + +Uint8List mutateManifest( + Uint8List jsonBytes, + void Function(Map manifest) edit, +) { + final decoded = jsonDecode(utf8.decode(jsonBytes)); + if (decoded is! Map) { + throw StateError('manifest.json is not an object'); + } + final map = Map.from(decoded); + edit(map); + return Uint8List.fromList(utf8.encode(jsonEncode(map))); +} + +Future importZip( + AppDatabase dest, + Uint8List bytes, { + String? confirmation, + PhotoStore? photoStore, +}) async { + final service = ImportService(db: dest, photoStore: photoStore); + final preview = await service.preview(bytes); + await service.commit(preview, typedConfirmation: confirmation); +} + +Future expectImportRejected( + AppDatabase dest, + Uint8List bytes, { + required ImportErrorCode code, + String? confirmation, +}) async { + final before = await historyFingerprint(dest); + final service = ImportService(db: dest); + try { + final preview = await service.preview(bytes); + await service.commit(preview, typedConfirmation: confirmation); + fail('expected ${code.wire}, import succeeded'); + } on ImportException catch (error) { + expect(error.code, code, reason: error.display); + } + expect( + await historyFingerprint(dest), + before, + reason: '${code.wire} must leave the database untouched', + ); +} + +Future historyFingerprint(AppDatabase db) async { + final vehicles = (await db.select(db.vehicles).get()) + ..sort((a, b) => a.id.compareTo(b.id)); + final fills = (await db.select(db.fillUps).get()) + ..sort((a, b) => a.id.compareTo(b.id)); + final rules = (await db.select(db.maintenanceRules).get()) + ..sort((a, b) => a.id.compareTo(b.id)); + final events = (await db.select(db.maintenanceEvents).get()) + ..sort((a, b) => a.id.compareTo(b.id)); + final outbox = await db.select(db.outbox).get(); + return jsonEncode({ + 'vehicles': [ + for (final row in vehicles) + [row.id, row.name, row.fuelType, row.tankCapacityUL, row.updatedAt], + ], + 'fill_ups': [ + for (final row in fills) + [ + row.id, + row.vehicleId, + row.odometerM, + row.volumeUL, + row.totalPriceCents, + row.notes, + row.rowVersion, + ], + ], + 'rules': [ + for (final row in rules) + [row.id, row.cadenceKm, row.cadenceDays, row.notes], + ], + 'events': [ + for (final row in events) [row.id, row.category, row.shop, row.costCents], + ], + 'outbox': outbox.length, + }); +} diff --git a/client/test/import/headers_drift_test.dart b/client/test/import/headers_drift_test.dart new file mode 100644 index 0000000..6d6b5b0 --- /dev/null +++ b/client/test/import/headers_drift_test.dart @@ -0,0 +1,28 @@ +import 'package:cestovni/export/headers.dart'; +import 'package:cestovni/import/validate.dart'; +import 'package:flutter_test/flutter_test.dart'; + +/// Spec: import expected headers *are* the export constants — never a +/// parallel copy that can drift. +void main() { + test('importCsvHeaders is the export constant set', () { + expect(importCsvHeaders, { + 'vehicles.csv': vehiclesCsvHeader, + 'fill_ups.csv': fillUpsCsvHeader, + 'maintenance_rules.csv': maintenanceRulesCsvHeader, + 'maintenance_events.csv': maintenanceEventsCsvHeader, + 'settings.csv': settingsCsvHeader, + }); + expect(importCsvHeaders['vehicles.csv'], same(vehiclesCsvHeader)); + expect(importCsvHeaders['fill_ups.csv'], same(fillUpsCsvHeader)); + expect( + importCsvHeaders['maintenance_rules.csv'], + same(maintenanceRulesCsvHeader), + ); + expect( + importCsvHeaders['maintenance_events.csv'], + same(maintenanceEventsCsvHeader), + ); + expect(importCsvHeaders['settings.csv'], same(settingsCsvHeader)); + }); +} diff --git a/client/test/import/module_purity_test.dart b/client/test/import/module_purity_test.dart new file mode 100644 index 0000000..3d0ea38 --- /dev/null +++ b/client/test/import/module_purity_test.dart @@ -0,0 +1,84 @@ +/// Static guard on `client/lib/import/`: parser / validator / planner +/// stay pure Dart. Mirrors `client/test/export/module_purity_test.dart`. +library; + +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; + +const _bridgeFiles = { + // Drift transaction + settings update + draft reconcile. + 'apply.dart', + // Platform picker, dart:io inflater, photo-file cleanup. + 'import_service.dart', +}; + +const _forbiddenForPureFiles = [ + 'dart:io', + 'package:drift/', + 'package:flutter/', + 'package:file_picker/', + 'package:cestovni/db/', +]; + +void main() { + final importDir = _resolveImportDir(); + + test('import module directory is discoverable', () { + expect( + importDir.existsSync(), + isTrue, + reason: 'client/lib/import/ must exist (looked at ${importDir.path}).', + ); + }); + + final dartFiles = importDir + .listSync(recursive: true) + .whereType() + .where((f) => f.path.endsWith('.dart')) + .toList() + ..sort((a, b) => a.path.compareTo(b.path)); + + test('csv_parse, validate, plan, zip_read stay pure', () { + final violations = []; + + for (final file in dartFiles) { + final basename = _basename(file.path); + if (_bridgeFiles.contains(basename)) continue; + + final contents = file.readAsStringSync(); + for (final forbidden in _forbiddenForPureFiles) { + if (contents.contains("import '$forbidden") || + contents.contains('import "$forbidden')) { + violations.add('$basename imports $forbidden'); + } + } + } + + expect( + violations, + isEmpty, + reason: 'Pure files in client/lib/import/ must not import the file ' + 'system, platform channels, Flutter, or the app DB. Violations:\n - ' + '${violations.join("\n - ")}', + ); + }); + + test('every declared bridge file exists', () { + final present = dartFiles.map((f) => _basename(f.path)).toSet(); + expect(_bridgeFiles.difference(present), isEmpty); + }); +} + +Directory _resolveImportDir() { + for (final candidate in const ['lib/import', 'client/lib/import']) { + final dir = Directory(candidate); + if (dir.existsSync()) return dir.absolute; + } + return Directory('lib/import').absolute; +} + +String _basename(String path) { + final idx = path.lastIndexOf('/'); + return idx < 0 ? path : path.substring(idx + 1); +} diff --git a/client/test/import/rejects_test.dart b/client/test/import/rejects_test.dart new file mode 100644 index 0000000..e4e9185 --- /dev/null +++ b/client/test/import/rejects_test.dart @@ -0,0 +1,165 @@ +import 'dart:typed_data'; + +import 'package:cestovni/db/repositories/settings_repository.dart'; +import 'package:cestovni/import/import_errors.dart'; +import 'package:flutter_test/flutter_test.dart'; + +import '../db/_harness.dart'; +import '../export/_seed.dart'; +import '_zip.dart'; + +void main() { + late Uint8List golden; + + setUp(() async { + final source = openInMemoryDb(); + await seedGoldenExport(source); + golden = await exportDbToZip(source); + await source.close(); + }); + + Future expectRejects( + Uint8List bytes, + ImportErrorCode code, + ) async { + final dest = openInMemoryDb(); + addTearDown(dest.close); + await SettingsRepository(dest).getOrBootstrap(); + await expectImportRejected(dest, bytes, code: code); + } + + test('3 header mutation → E_HEADER_MISMATCH, zero writes', () async { + final entries = unpackZip(golden); + entries['fill_ups.csv'] = mutateCsvCell( + csv: entries['fill_ups.csv']!, + file: 'fill_ups.csv', + dataRow: -1, // header record + column: 'odometer_m', + value: 'odometer_meters', + ); + await expectRejects(packZip(entries), ImportErrorCode.headerMismatch); + }); + + test('4 photo path in ZIP → E_PHOTOS_PRESENT', () async { + final entries = unpackZip(golden); + entries['photos/x.jpg'] = Uint8List.fromList(const [0xFF, 0xD8, 0xFF, 0x00]); + await expectRejects(packZip(entries), ImportErrorCode.photosPresent); + }); + + test('4 photos_in_export true → E_PHOTOS_PRESENT', () async { + final entries = unpackZip(golden); + entries['manifest.json'] = mutateManifest( + entries['manifest.json']!, + (m) => m['photos_in_export'] = true, + ); + await expectRejects(packZip(entries), ImportErrorCode.photosPresent); + }); + + test('7 negative volume_uL → E_VALUE_INVALID', () async { + final entries = unpackZip(golden); + entries['fill_ups.csv'] = mutateCsvCell( + csv: entries['fill_ups.csv']!, + file: 'fill_ups.csv', + dataRow: 0, + column: 'volume_uL', + value: '-1', + ); + await expectRejects(packZip(entries), ImportErrorCode.valueInvalid); + }); + + test('7 1.0 in an INT column → E_VALUE_INVALID', () async { + final entries = unpackZip(golden); + entries['fill_ups.csv'] = mutateCsvCell( + csv: entries['fill_ups.csv']!, + file: 'fill_ups.csv', + dataRow: 0, + column: 'odometer_m', + value: '1.0', + ); + await expectRejects(packZip(entries), ImportErrorCode.valueInvalid); + }); + + test('7 bad currency_code → E_VALUE_INVALID', () async { + final entries = unpackZip(golden); + entries['fill_ups.csv'] = mutateCsvCell( + csv: entries['fill_ups.csv']!, + file: 'fill_ups.csv', + dataRow: 0, + column: 'currency_code', + value: 'eu', + ); + await expectRejects(packZip(entries), ImportErrorCode.valueInvalid); + }); + + test('7 unknown fuel_type → E_VALUE_INVALID', () async { + final entries = unpackZip(golden); + entries['vehicles.csv'] = mutateCsvCell( + csv: entries['vehicles.csv']!, + file: 'vehicles.csv', + dataRow: 0, + column: 'fuel_type', + value: 'petrol', + ); + await expectRejects(packZip(entries), ImportErrorCode.valueInvalid); + }); + + test('7 unknown category → E_VALUE_INVALID', () async { + final entries = unpackZip(golden); + entries['maintenance_events.csv'] = mutateCsvCell( + csv: entries['maintenance_events.csv']!, + file: 'maintenance_events.csv', + dataRow: 0, + column: 'category', + value: 'wax', + ); + await expectRejects(packZip(entries), ImportErrorCode.valueInvalid); + }); + + test('7 duplicate id → E_DUPLICATE_ID', () async { + final entries = unpackZip(golden); + entries['fill_ups.csv'] = duplicateCsvDataRow( + csv: entries['fill_ups.csv']!, + file: 'fill_ups.csv', + ); + entries['manifest.json'] = mutateManifest( + entries['manifest.json']!, + (m) { + final counts = Map.from(m['row_counts'] as Map); + counts['fill_ups'] = 2; + m['row_counts'] = counts; + }, + ); + await expectRejects(packZip(entries), ImportErrorCode.duplicateId); + }); + + test('7 missing cadence → E_CADENCE_MISSING', () async { + final entries = unpackZip(golden); + entries['maintenance_rules.csv'] = mutateCsvCell( + csv: entries['maintenance_rules.csv']!, + file: 'maintenance_rules.csv', + dataRow: 0, + column: 'cadence_km', + value: '', + ); + entries['maintenance_rules.csv'] = mutateCsvCell( + csv: entries['maintenance_rules.csv']!, + file: 'maintenance_rules.csv', + dataRow: 0, + column: 'cadence_days', + value: '', + ); + await expectRejects(packZip(entries), ImportErrorCode.cadenceMissing); + }); + + test('10 fill_ups.vehicle_id orphan → E_FK_ORPHAN', () async { + final entries = unpackZip(golden); + entries['fill_ups.csv'] = mutateCsvCell( + csv: entries['fill_ups.csv']!, + file: 'fill_ups.csv', + dataRow: 0, + column: 'vehicle_id', + value: '00000000-0000-4000-8000-000000000099', + ); + await expectRejects(packZip(entries), ImportErrorCode.fkOrphan); + }); +} diff --git a/client/test/import/replace_test.dart b/client/test/import/replace_test.dart new file mode 100644 index 0000000..cc46b5c --- /dev/null +++ b/client/test/import/replace_test.dart @@ -0,0 +1,311 @@ +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:cestovni/db/repositories/drafts_repository.dart'; +import 'package:cestovni/db/repositories/fill_ups_repository.dart'; +import 'package:cestovni/db/repositories/outbox_repository.dart'; +import 'package:cestovni/db/repositories/photo_refs_repository.dart'; +import 'package:cestovni/db/repositories/settings_repository.dart'; +import 'package:cestovni/db/repositories/vehicles_repository.dart'; +import 'package:cestovni/import/apply.dart'; +import 'package:cestovni/import/import_errors.dart'; +import 'package:cestovni/import/import_service.dart'; +import 'package:cestovni/import/plan.dart'; +import 'package:cestovni/photos/photo_store.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; + +import '../db/_harness.dart'; +import '../export/_seed.dart'; +import '_zip.dart'; + +void main() { + test('8 atomicity: failure while writing events rolls back prior history', + () async { + final dest = openInMemoryDb(); + addTearDown(dest.close); + await seedGoldenExport(dest); + final before = await historyFingerprint(dest); + + try { + await ImportApplier(dest).apply(_planThatFailsOnEvents()); + fail('expected E_TXN_FAILED'); + } on ImportException catch (error) { + expect(error.code, ImportErrorCode.txnFailed); + } + + expect(await historyFingerprint(dest), before); + final names = + (await dest.select(dest.vehicles).get()).map((v) => v.name).toSet(); + expect(names, contains('Octavia')); + expect(names, isNot(contains('Incoming'))); + }); + + test('13 replace clears prior history; disjoint ZIP wins', () async { + final source = openInMemoryDb(); + addTearDown(source.close); + final seed = await seedGoldenExport(source); + final zip = await exportDbToZip(source); + + final dest = openInMemoryDb(); + addTearDown(dest.close); + await SettingsRepository(dest).getOrBootstrap(); + final localId = await VehiclesRepository(dest).create( + const VehicleDraft(name: 'LocalOnly', fuelType: VehicleFuelType.diesel), + ); + + await importZip(dest, zip, confirmation: importConfirmationKeyword); + + final vehicles = await dest.select(dest.vehicles).get(); + expect(vehicles.map((v) => v.id).toSet(), {seed.vehicleId}); + expect(vehicles.single.name, 'Octavia'); + expect(vehicles.any((v) => v.id == localId), isFalse); + expect(await dest.select(dest.fillUps).get(), hasLength(1)); + }); + + test('14 settings updated in place; id unchanged; default vehicle validated', + () async { + final source = openInMemoryDb(); + addTearDown(source.close); + final seed = await seedGoldenExport(source); + final zip = await exportDbToZip(source); + + final dest = openInMemoryDb(); + addTearDown(dest.close); + final original = await SettingsRepository(dest).update( + preferredDistanceUnit: 'mi', + preferredVolumeUnit: 'gal', + currencyCode: 'USD', + timezone: 'Europe/Prague', + ); + + await importZip(dest, zip); + final adopted = await SettingsRepository(dest).getOrBootstrap(); + expect(adopted.id, original.id); + expect(adopted.preferredDistanceUnit, 'km'); + expect(adopted.preferredVolumeUnit, 'L'); + expect(adopted.currencyCode, 'EUR'); + expect(adopted.timezone, 'UTC'); + expect(adopted.defaultVehicleId, seed.vehicleId); + + final dangling = unpackZip(zip); + dangling['settings.csv'] = mutateCsvCell( + csv: dangling['settings.csv']!, + file: 'settings.csv', + dataRow: 0, + column: 'default_vehicle_id', + value: '00000000-0000-4000-8000-000000000099', + ); + await importZip( + dest, + packZip(dangling), + confirmation: importConfirmationKeyword, + ); + final cleared = await SettingsRepository(dest).getOrBootstrap(); + expect(cleared.id, original.id); + expect(cleared.defaultVehicleId, isNull); + }); + + test('15 outbox is cleared and the discarded count is reported', () async { + final source = openInMemoryDb(); + addTearDown(source.close); + await seedGoldenExport(source); + final zip = await exportDbToZip(source); + + final dest = openInMemoryDb(); + addTearDown(dest.close); + await SettingsRepository(dest).getOrBootstrap(); + final vehicleId = await VehiclesRepository(dest).create( + const VehicleDraft(name: 'Queued', fuelType: VehicleFuelType.gasoline), + ); + await FillUpsRepository(dest).create( + FillUpDraft( + vehicleId: vehicleId, + filledAt: DateTime.utc(2026, 8, 2, 9), + odometerM: 1000, + volumeUL: 10000000, + totalPriceCents: 100, + currencyCode: 'EUR', + isFull: true, + ), + ); + expect(await OutboxRepository(dest).pendingMutationIds(), isNotEmpty); + final pending = (await OutboxRepository(dest).pendingMutationIds()).length; + + final service = ImportService(db: dest); + final preview = await service.preview(zip); + expect(preview.footprint.queuedChanges, pending); + final outcome = await service.commit( + preview, + typedConfirmation: importConfirmationKeyword, + ); + + expect(outcome.queueDiscarded, pending); + expect(await dest.select(dest.outbox).get(), isEmpty); + }); + + test('16 drafts: surviving vehicle keeps draft+photo; destroyed purges both', + () async { + final source = openInMemoryDb(); + addTearDown(source.close); + await seedGoldenExport(source); + final zip = await exportDbToZip(source); + + final dest = openInMemoryDb(); + addTearDown(dest.close); + await SettingsRepository(dest).getOrBootstrap(); + await importZip(dest, zip); + + final keptVehicle = + (await dest.select(dest.vehicles).get()).single.id; + final keptDraftId = await DraftsRepository(dest).save( + DraftSnapshot(vehicleId: keptVehicle, notes: 'keep me'), + ); + final doomedVehicle = await VehiclesRepository(dest).create( + const VehicleDraft(name: 'Doomed', fuelType: VehicleFuelType.diesel), + ); + final doomedDraftId = await DraftsRepository(dest).save( + DraftSnapshot(vehicleId: doomedVehicle, notes: 'drop me'), + ); + + final sandbox = Directory.systemTemp.createTempSync('cestovni-import-photos-'); + addTearDown(() { + if (sandbox.existsSync()) sandbox.deleteSync(recursive: true); + }); + final store = PhotoStore.inDirectory(Directory(p.join(sandbox.path, 'photos'))); + const jpeg = [0xFF, 0xD8, 0xFF, 0xD9]; + final keptPhoto = await PhotoRefsRepository(dest).insert( + draftId: keptDraftId, + capturedAt: DateTime.utc(2026, 8, 16), + byteSize: jpeg.length, + sha256Hex: 'aa' * 32, + ttlExpiresAt: DateTime.utc(2026, 9, 16), + ); + final doomedPhoto = await PhotoRefsRepository(dest).insert( + draftId: doomedDraftId, + capturedAt: DateTime.utc(2026, 8, 16), + byteSize: jpeg.length, + sha256Hex: 'bb' * 32, + ttlExpiresAt: DateTime.utc(2026, 9, 16), + ); + await store.write(keptPhoto.id, Uint8List.fromList(jpeg)); + await store.write(doomedPhoto.id, Uint8List.fromList(jpeg)); + + final service = ImportService(db: dest, photoStore: store); + final preview = await service.preview(zip); + expect(preview.footprint.draftsAtRisk, 1); + final outcome = await service.commit( + preview, + typedConfirmation: importConfirmationKeyword, + ); + + expect(outcome.draftsDiscarded, 1); + expect(outcome.photoIdsToDelete, [doomedPhoto.id]); + expect( + await DraftsRepository(dest).openDraftForVehicle(keptVehicle), + isNotNull, + ); + expect( + await DraftsRepository(dest).openDraftForVehicle(doomedVehicle), + isNull, + ); + expect( + await PhotoRefsRepository(dest).findById(keptPhoto.id), + isNotNull, + ); + expect( + await PhotoRefsRepository(dest).findById(doomedPhoto.id), + isNull, + ); + expect(await store.exists(keptPhoto.id), isTrue); + expect(await store.exists(doomedPhoto.id), isFalse); + expect( + (await dest.select(dest.vehicles).get()).map((v) => v.name), + ['Octavia'], + ); + }); + + test('17 confirmation: keyword required iff local history is non-empty', + () async { + final source = openInMemoryDb(); + addTearDown(source.close); + await seedGoldenExport(source); + final zip = await exportDbToZip(source); + + final empty = openInMemoryDb(); + addTearDown(empty.close); + await SettingsRepository(empty).getOrBootstrap(); + final emptyService = ImportService(db: empty); + final emptyPreview = await emptyService.preview(zip); + expect(emptyPreview.requiresTypedConfirmation, isFalse); + await emptyService.commit(emptyPreview); + expect(await empty.select(empty.vehicles).get(), hasLength(1)); + + final populated = openInMemoryDb(); + addTearDown(populated.close); + await seedGoldenExport(populated); + await expectImportRejected( + populated, + zip, + code: ImportErrorCode.notConfirmed, + ); + await importZip( + populated, + zip, + confirmation: importConfirmationKeyword, + ); + expect(await populated.select(populated.vehicles).get(), hasLength(1)); + }); +} + +ImportPlan _planThatFailsOnEvents() { + const incomingVehicle = '11111111-1111-4111-8111-111111111111'; + const missingVehicle = '33333333-3333-4333-8333-333333333333'; + return ImportPlan( + manifest: const ImportedManifest( + schemaVersion: 1, + exportedAtUtc: '2026-08-16T12:00:00Z', + appVersion: '0.0.1', + appPlatform: 'android', + timezone: 'UTC', + userKeyHash: 'deadbeef', + outboxPendingCount: 0, + rowCounts: { + 'vehicles': 1, + 'fill_ups': 0, + 'maintenance_rules': 0, + 'maintenance_events': 1, + 'settings': 1, + }, + ), + vehicles: const [ + ImportedVehicle( + id: incomingVehicle, + name: 'Incoming', + fuelType: 'gasoline', + updatedAt: '2026-08-16T12:00:00.000Z', + ), + ], + fillUps: const [], + maintenanceRules: const [], + maintenanceEvents: const [ + ImportedMaintenanceEvent( + id: '22222222-2222-4222-8222-222222222222', + vehicleId: missingVehicle, + performedAt: '2026-08-16T12:00:00.000Z', + costCents: 0, + currencyCode: 'EUR', + category: 'oil', + updatedAt: '2026-08-16T12:00:00.000Z', + ), + ], + settings: const ImportedSettings( + preferredDistanceUnit: 'km', + preferredVolumeUnit: 'L', + currencyCode: 'EUR', + timezone: 'UTC', + updatedAt: '2026-08-16T12:00:00.000Z', + ), + warnings: const [], + ); +} diff --git a/client/test/import/round_trip_test.dart b/client/test/import/round_trip_test.dart new file mode 100644 index 0000000..04f447a --- /dev/null +++ b/client/test/import/round_trip_test.dart @@ -0,0 +1,162 @@ +import 'package:cestovni/db/repositories/settings_repository.dart'; +import 'package:cestovni/import/import_service.dart'; +import 'package:flutter_test/flutter_test.dart'; + +import '../db/_harness.dart'; +import '../export/_seed.dart'; +import '_zip.dart'; + +void main() { + test('1 golden round-trip: canonical columns equal row for row', () async { + final source = openInMemoryDb(); + addTearDown(source.close); + final seed = await seedGoldenExport(source); + final zip = await exportDbToZip(source); + + final dest = openInMemoryDb(); + addTearDown(dest.close); + final destSettings = await SettingsRepository(dest).getOrBootstrap(); + + await importZip(dest, zip); + + final vehicles = await dest.select(dest.vehicles).get(); + expect(vehicles, hasLength(1)); + final vehicle = vehicles.single; + expect(vehicle.id, seed.vehicleId); + expect(vehicle.name, 'Octavia'); + expect(vehicle.make, 'Skoda'); + expect(vehicle.model, 'Mk3'); + expect(vehicle.year, 2018); + expect(vehicle.fuelType, 'gasoline'); + expect(vehicle.tankCapacityUL, 55000000); + expect(vehicle.rowVersion, isNull); + + final fills = await dest.select(dest.fillUps).get(); + expect(fills, hasLength(1)); + final fill = fills.single; + expect(fill.id, seed.fillUpId); + expect(fill.vehicleId, seed.vehicleId); + expect(fill.odometerM, 120000000); + expect(fill.volumeUL, 42000000); + expect(fill.totalPriceCents, 6100); + expect(fill.currencyCode, 'EUR'); + expect(fill.isFull, isTrue); + expect(fill.notes, 'hello, "world"'); + expect(fill.rowVersion, isNull); + + final rules = await dest.select(dest.maintenanceRules).get(); + expect(rules, hasLength(1)); + final rule = rules.single; + expect(rule.id, seed.ruleId); + expect(rule.cadenceKm, 10000000, reason: 'cadence_km is meters, verbatim'); + expect(rule.cadenceDays, 365); + expect(rule.rowVersion, isNull); + + final events = await dest.select(dest.maintenanceEvents).get(); + expect(events, hasLength(1)); + final event = events.single; + expect(event.id, seed.eventId); + expect(event.category, 'oil'); + expect(event.shop, 'Bosch, Praha'); + expect(event.costCents, 8900); + expect(event.rowVersion, isNull); + + final settings = await SettingsRepository(dest).getOrBootstrap(); + expect(settings.id, destSettings.id, reason: 'local identity preserved'); + expect(settings.timezone, 'UTC'); + expect(settings.defaultVehicleId, seed.vehicleId); + }); + + test('2 idempotency: importing the same ZIP twice is a no-op', () async { + final source = openInMemoryDb(); + addTearDown(source.close); + await seedGoldenExport(source); + final zip = await exportDbToZip(source); + + final dest = openInMemoryDb(); + addTearDown(dest.close); + await SettingsRepository(dest).getOrBootstrap(); + + await importZip(dest, zip); + final once = await historyFingerprint(dest); + await importZip(dest, zip, confirmation: importConfirmationKeyword); + expect(await historyFingerprint(dest), once); + + final ids = (await dest.select(dest.fillUps).get()).map((r) => r.id); + expect(ids.toSet(), hasLength(ids.length)); + }); + + test('5 derived columns are unread: corrupt odometer_km keeps odometer_m', + () async { + final source = openInMemoryDb(); + addTearDown(source.close); + await seedGoldenExport(source); + final entries = unpackZip(await exportDbToZip(source)); + entries['fill_ups.csv'] = mutateCsvCell( + csv: entries['fill_ups.csv']!, + file: 'fill_ups.csv', + dataRow: 0, + column: 'odometer_km', + value: '999999', + ); + + final dest = openInMemoryDb(); + addTearDown(dest.close); + await SettingsRepository(dest).getOrBootstrap(); + await importZip(dest, packZip(entries)); + + final fill = (await dest.select(dest.fillUps).get()).single; + expect(fill.odometerM, 120000000); + }); + + test('6 cadence_km = 10000 stores 10000 meters, not 10 or 1e7', () async { + final source = openInMemoryDb(); + addTearDown(source.close); + await seedGoldenExport(source); + final entries = unpackZip(await exportDbToZip(source)); + entries['maintenance_rules.csv'] = mutateCsvCell( + csv: entries['maintenance_rules.csv']!, + file: 'maintenance_rules.csv', + dataRow: 0, + column: 'cadence_km', + value: '10000', + ); + + final dest = openInMemoryDb(); + addTearDown(dest.close); + await SettingsRepository(dest).getOrBootstrap(); + await importZip(dest, packZip(entries)); + + final rule = (await dest.select(dest.maintenanceRules).get()).single; + expect(rule.cadenceKm, 10000); + expect(rule.cadenceKm, isNot(10)); + expect(rule.cadenceKm, isNot(10000000)); + }); + + test('9 imported rows have row_version IS NULL and nothing is enqueued', + () async { + final source = openInMemoryDb(); + addTearDown(source.close); + await seedGoldenExport(source); + final zip = await exportDbToZip(source); + + final dest = openInMemoryDb(); + addTearDown(dest.close); + await SettingsRepository(dest).getOrBootstrap(); + await importZip(dest, zip); + + for (final row in await dest.select(dest.vehicles).get()) { + expect(row.rowVersion, isNull); + } + for (final row in await dest.select(dest.fillUps).get()) { + expect(row.rowVersion, isNull); + } + for (final row in await dest.select(dest.maintenanceRules).get()) { + expect(row.rowVersion, isNull); + } + for (final row in await dest.select(dest.maintenanceEvents).get()) { + expect(row.rowVersion, isNull); + } + expect(await dest.select(dest.outbox).get(), isEmpty); + }); +} diff --git a/client/test/import/streaming_test.dart b/client/test/import/streaming_test.dart new file mode 100644 index 0000000..eef1b08 --- /dev/null +++ b/client/test/import/streaming_test.dart @@ -0,0 +1,155 @@ +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:cestovni/db/repositories/settings_repository.dart'; +import 'package:cestovni/export/assembler.dart'; +import 'package:cestovni/export/headers.dart'; +import 'package:cestovni/export/manifest.dart'; +import 'package:cestovni/export/readme.dart'; +import 'package:cestovni/export/store_zip_sink.dart'; +import 'package:flutter_test/flutter_test.dart'; + +import '../db/_harness.dart'; +import '_zip.dart'; + +/// Spec item 12 / export A4: 1 000 fill-up rows is a CI-sized archive, +/// not a device-timing gate. Device 10k/30s stays on CES-68. +void main() { + test('12 1000-row fill_ups ZIP imports without buffering as one table', + () async { + const vehicleId = 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'; + final zip = _thousandFillUpsZip(vehicleId); + + final dest = openInMemoryDb(); + addTearDown(dest.close); + await SettingsRepository(dest).getOrBootstrap(); + await importZip(dest, zip); + + expect(await dest.select(dest.fillUps).get(), hasLength(1000)); + expect( + (await dest.select(dest.vehicles).get()).single.id, + vehicleId, + ); + }); +} + +Uint8List _thousandFillUpsZip(String vehicleId) { + final dir = Directory.systemTemp.createTempSync('cestovni-import-stream-'); + final file = File('${dir.path}/thousand.zip'); + final sink = FileZipSink(file) + ..stamp = DateTime.utc(2026, 8, 16, 12); + assembleExportZip( + sink: sink, + manifestJson: encodeManifest( + exportManifest( + exportedAtUtc: '2026-08-16T12:00:00Z', + appVersion: '0.0.1', + appPlatform: 'android', + timezone: 'UTC', + userKeyHash: 'aabbccdd', + preferredDistanceUnit: 'km', + preferredVolumeUnit: 'L', + currencyCode: 'EUR', + vehiclesCount: 1, + fillUpsCount: 1000, + maintenanceRulesCount: 0, + maintenanceEventsCount: 0, + settingsCount: 1, + outboxPendingCount: 0, + outboxPendingHash: null, + ), + ), + readmeText: buildReadmeExport( + exportedAtUtc: '2026-08-16T12:00:00Z', + preferredDistanceUnit: 'km', + preferredVolumeUnit: 'L', + currencyCode: 'EUR', + timezone: 'UTC', + outboxPendingCount: 0, + ), + tables: [ + ExportCsvTable( + filename: 'vehicles.csv', + header: vehiclesCsvHeader, + rows: [ + [ + vehicleId, + 'aabbccdd', + 'Fleet', + null, + null, + null, + null, + 'gasoline', + null, + null, + null, + null, + '2026-08-16T12:00:00.000Z', + ], + ], + ), + ExportCsvTable( + filename: 'fill_ups.csv', + header: fillUpsCsvHeader, + rows: _lazyFillUps(vehicleId, 1000), + ), + const ExportCsvTable( + filename: 'maintenance_rules.csv', + header: maintenanceRulesCsvHeader, + rows: [], + ), + const ExportCsvTable( + filename: 'maintenance_events.csv', + header: maintenanceEventsCsvHeader, + rows: [], + ), + ExportCsvTable( + filename: 'settings.csv', + header: settingsCsvHeader, + rows: [ + [ + 'aabbccdd', + 'km', + 'L', + 'EUR', + 'UTC', + vehicleId, + null, + '2026-08-16T12:00:00.000Z', + ], + ], + ), + ], + ); + final bytes = file.readAsBytesSync(); + dir.deleteSync(recursive: true); + return Uint8List.fromList(bytes); +} + +Iterable> _lazyFillUps(String vehicleId, int count) sync* { + for (var i = 0; i < count; i++) { + yield [ + '00000000-0000-4000-8000-${i.toString().padLeft(12, '0')}', + 'aabbccdd', + vehicleId, + '2026-08-01T10:00:00.000Z', + '2026-08-01 10:00:00', + 1000000 + i, + '1', + '1', + 40000000, + '40.00', + '10.57', + 5000, + '50.00', + 'EUR', + true, + false, + false, + null, + null, + '2026-08-16T12:00:00.000Z', + ]; + } +} diff --git a/docs/product/README.md b/docs/product/README.md index 2eb42b2..54662ed 100644 --- a/docs/product/README.md +++ b/docs/product/README.md @@ -24,7 +24,7 @@ Store listing skeleton, privacy policy outline, in-app Data & privacy bullets: ` ## Delivery (Phase 3 / Stage 5) -Active engineering breakdown, milestone spine, per-vertical backlog with `Spec:` paths, test strategy, and **repo progress**: `[delivery-plan-v1.md](delivery-plan-v1.md)` (RYG checklist is source of truth). Linear epic **[CES-35](https://linear.app/personal-interests-llc/issue/CES-35)**. **M0 closed** (**CES-36** / **CES-37**). **Android M1 closed** on `main` (`bb1d5d5`, 2026-08-16): **CES-38** / **CES-39** / **CES-57** / **CES-65** / **CES-66** / **CES-67** / **CES-40** **Done**. **CES-41** export shipped. In flight: **[CES-70](https://linear.app/personal-interests-llc/issue/CES-70)** ZIP import — prompt [`prompts/ces-70-import.md`](prompts/ces-70-import.md); tests outstanding. Prerequisites **CES-53**–**CES-56** **Done**. Linear MCP/API: [`../linear/mcp-setup.md`](../linear/mcp-setup.md). +Active engineering breakdown, milestone spine, per-vertical backlog with `Spec:` paths, test strategy, and **repo progress**: `[delivery-plan-v1.md](delivery-plan-v1.md)` (RYG checklist is source of truth). Linear epic **[CES-35](https://linear.app/personal-interests-llc/issue/CES-35)**. **M0 closed** (**CES-36** / **CES-37**). **Android M1 closed** on `main` (`bb1d5d5`, 2026-08-16): **CES-38** / **CES-39** / **CES-57** / **CES-65** / **CES-66** / **CES-67** / **CES-40** **Done**. **CES-41** export shipped. In flight: **[CES-70](https://linear.app/personal-interests-llc/issue/CES-70)** ZIP import — prompt [`prompts/ces-70-import.md`](prompts/ces-70-import.md); spec tests green on PR #25, not on `main`. Prerequisites **CES-53**–**CES-56** **Done**. Linear MCP/API: [`../linear/mcp-setup.md`](../linear/mcp-setup.md). ## Specs diff --git a/docs/product/delivery-plan-v1.md b/docs/product/delivery-plan-v1.md index e46712f..be29678 100644 --- a/docs/product/delivery-plan-v1.md +++ b/docs/product/delivery-plan-v1.md @@ -12,7 +12,7 @@ Stage 5 exit (copied from workflow): **running build with test strategy tied to ## Current focus -**In flight:** **[CES-70](https://linear.app/personal-interests-llc/issue/CES-70) ZIP import** — implementation on `main`-bound branch; **automated tests still outstanding** (see M2 rollup). Product locks resolved 2026-08-16: mode is **replace**, cross-account imports stay a warning. **Do not start M3 (CES-42–45)** unless product redirects. Parallel ops: **[CES-63](https://linear.app/personal-interests-llc/issue/CES-63)** iPhone install-doc + T1; **[CES-68](https://linear.app/personal-interests-llc/issue/CES-68)** Android APK anytime for demo distribution. +**In flight:** **[CES-70](https://linear.app/personal-interests-llc/issue/CES-70) ZIP import** — implementation + spec tests on draft [PR #25](https://github.com/JMNofziger/cestovni/pull/25); **not on `main` yet**. Product locks resolved 2026-08-16: mode is **replace**, cross-account imports stay a warning. **Do not start M3 (CES-42–45)** unless product redirects. Parallel ops: **[CES-63](https://linear.app/personal-interests-llc/issue/CES-63)** iPhone install-doc + T1; **[CES-68](https://linear.app/personal-interests-llc/issue/CES-68)** Android APK anytime for demo distribution. | Track | Issue | Why now | Done when | | ----- | ----- | ------- | --------- | @@ -96,7 +96,7 @@ Rollup mirrors milestones **M0→M5** and verticals **CES-36..CES-47** ([epic CE ### M1 — Local logging + math -- 🟩 **M1 rollup (closed 2026-08-16)** — offline logging usable end-to-end without a server. Log / History / vehicles / settings / metrics / Maint / photos on `main` (`bb1d5d5`). **CES-41 export shipped** (M2). **CES-70 import** is the remaining M2 work (implemented, tests outstanding) — not M3. +- 🟩 **M1 rollup (closed 2026-08-16)** — offline logging usable end-to-end without a server. Log / History / vehicles / settings / metrics / Maint / photos on `main` (`bb1d5d5`). **CES-41 export shipped** (M2). **CES-70 import** is the remaining M2 work (tests green on PR #25, not on `main`) — not M3. - 🟩 **CES-38 — Consumption math + golden tests** — **Done in repo on `main`** (2026-05): `client/lib/consumption/`, auto-discovery runner over 20 `tests/math/fixtures/`, module-purity test, validation wired in Log/History save paths. Follow-ups: **CES-51** / **CES-52** (non-blocking). - 🟩 **CES-39 — Fill-up + vehicle UI (core)** — **Done** (repo 2026-05, Linear 2026-07-17): repos + vehicle CRUD + Log/History UI; 121+ tests. Out of scope → **CES-65** / **CES-66** / **CES-67** / **CES-40**. - 🟩 **CES-57 — Settings prefs + default vehicle** — **Done** (PR #9, Linear Done). Display wiring closed as **CES-65**. @@ -108,9 +108,9 @@ Rollup mirrors milestones **M0→M5** and verticals **CES-36..CES-47** ([epic CE ### M2 — Export + import -- 🟨 **M2 rollup** — on-device ZIP export shipped; import implemented, tests outstanding. +- 🟨 **M2 rollup** — on-device ZIP export shipped; import implemented with spec tests green on PR #25 (not on `main`). - 🟩 **CES-41 — Export ZIP** — **Done in repo** (`client/lib/export/`, Settings → Export data). STORE ZIP (no `archive` write path), A1 headers, `photos_in_export: false`, streaming test over 1 000 lazy rows. Device 10k timing deferred to CES-68. Tests: `client/test/export/` + pointer [`tests/export/README.md`](../../tests/export/README.md). - - 🟨 **CES-70 — ZIP import** — **implemented, not yet tested.** `client/lib/import/` (`zip_read` · `csv_parse` · `validate` · `plan` · `apply` · `import_service`) + Settings → **Import data** (`client/lib/app/pages/import_data_section.dart`). **Replace** semantics per [`export-import.md`](../specs/export-import.md) § Replace semantics: hard-delete the four history tables and re-insert, `settings` updated in place, outbox cleared, drafts kept only when their vehicle survives, photo files deleted post-commit. Header constants shared from `client/lib/export/headers.dart`. **Outstanding before this goes 🟩:** the 17 cases in spec § Test expectations — none are written yet (development-only pass, testing deferred to a manual run by product). + - 🟨 **CES-70 — ZIP import** — **implemented; spec tests green on PR #25, not on `main`.** `client/lib/import/` (`zip_read` · `csv_parse` · `validate` · `plan` · `apply` · `import_service`) + Settings → **Import data**. **Replace** semantics per [`export-import.md`](../specs/export-import.md). Tests: `client/test/import/` mapped in [`tests/import/README.md`](../../tests/import/README.md). **Outstanding before this goes 🟩:** merge to `main`. ### M3 — Backup + restore @@ -211,7 +211,7 @@ Leading emoji tracks **exit** state (independent of per-vertical RYG above, but - 🟩 Every vertical above has a Linear issue with a `Spec:` line. *(CES-35 epic + CES-36..CES-47; M0 follow-ups CES-48/49/50 created 2026-04-22, linked to their downstream verticals via `blocks`.)* - 🟨 M0 + M1 land: offline app runs, fill-up works end-to-end, golden math tests green. *(**M0 closed**. **M1 closed** — CES-38/39/57 + CES-65/66/67 + CES-40 all Done. **CES-41 export shipped.** **Next:** CES-70 import — see [Current focus](#current-focus).)* -- 🟨 M2 lands: ZIP export round-trips for a representative fixture. *(CES-41 **Done**. CES-70 import implemented; the round-trip proof is one of its outstanding tests.)* +- 🟨 M2 lands: ZIP export round-trips for a representative fixture. *(CES-41 **Done**. CES-70 import + spec tests green on PR #25; still 🟨 until merge.)* - 🟥 M3 lands: backup/restore passes `tests/contract/` + integration fixtures; RLS regression green. - 🟥 M4 lands: `ci/telemetry-gate.`* green; client emits only allow-listed events. - 🟥 M5 lands: migration rollback spec real (not stub); rollback tooling proven against fixture. @@ -234,5 +234,5 @@ When every exit bullet above is 🟩, Stage 5 exit is met — flip workflow perc - `[launch-copy-v1.md](launch-copy-v1.md)` — Stage 4 copy; feeds Stage 6. - `[../specs/platform-compliance-v1.md](../specs/platform-compliance-v1.md)` — compliance posture already signed off. -*Last updated: 2026-08-21 — CES-70 ZIP import implemented (replace mode); its automated tests are the remaining M2 work.* +*Last updated: 2026-08-21 — CES-70 ZIP import implemented; spec § Test expectations green on PR #25, not on `main`.* diff --git a/docs/product/prompts/ces-70-import.md b/docs/product/prompts/ces-70-import.md index 3af9c1e..10167a8 100644 --- a/docs/product/prompts/ces-70-import.md +++ b/docs/product/prompts/ces-70-import.md @@ -1,13 +1,12 @@ # Cursor execution prompt — CES-70 ZIP import -> **Status: IMPLEMENTED, tests outstanding** (2026-08-21). Mode is `replace`. Code is on `cursor/ces-70-zip-import-40e4` in `client/lib/import/` + Settings → **Import data**. -> **Do not re-implement.** The remaining work is the 17 cases in spec § Test expectations (`client/test/import/` does not exist yet). -> Linear **[CES-70](https://linear.app/personal-interests-llc/issue/CES-70)** — **In Progress** until those tests land on `main`. GitHub PR automation will flip it Done; that is wrong until tests exist. -> Do **not** pick up M3 (CES-42–45), CES-51, CES-71, or PWA-lite unless the user explicitly redirects. **Do not unblock CES-71** until import is on `main` and round-trips `cadence_km`. +> **Status: IMPLEMENTED + TESTS GREEN** (2026-08-21). Mode is `replace`. Code and `client/test/import/` are on draft [PR #25](https://github.com/JMNofziger/cestovni/pull/25). +> **Do not re-implement.** Remaining work is merge to `main`. Device timing stays CES-68. +> Linear **[CES-70](https://linear.app/personal-interests-llc/issue/CES-70)** — **In Progress** until this is on `main`. GitHub automation will flip it Done on merge; that is correct only after merge. +> Do **not** pick up M3 (CES-42–45), CES-51, CES-71, or PWA-lite unless the user explicitly redirects. **Do not unblock CES-71** until import is on `main`. -**Branch:** `cursor/ces-70-zip-import-40e4` (cut from `main` at `7e7ae1b`, which includes CES-41 `client/lib/export/`) +**Branch:** `cursor/ces-70-zip-import-40e4` **Spec (normative):** [`docs/specs/export-import.md`](../../specs/export-import.md) -**Also read:** [`docs/specs/export-v1.md`](../../specs/export-v1.md) § v1 amendments · `client/lib/export/` · [`client/lib/photos/photo_export_guard.dart`](../../../client/lib/photos/photo_export_guard.dart) --- @@ -15,13 +14,12 @@ | Item | State | |------|-------| -| Last coding | **CES-70** import — implemented, **not on `main`**, **no tests** | -| M1 | **Closed.** Log / History / Metrics / Maint / photos ship on Android | -| M2 | **CES-41 done** on `main`. CES-70 import implemented; 17 spec tests remaining | -| Prerequisite | Header constants imported from `client/lib/export/headers.dart` (never copied) | -| Parallel (do not do here) | CES-63 iPhone install-doc · CES-68 APK · M3 CES-42–45 · CES-71 cadence rename | +| Last coding | **CES-70** import + spec tests — **not on `main`** | +| M1 | **Closed.** | +| M2 | CES-41 on `main`. CES-70 tests green on PR #25 | +| Parallel (do not do here) | CES-63 · CES-68 · M3 · CES-71 | -**Next coding:** land `client/test/import/` covering spec § Test expectations. Do **not** cut a second implementation branch off stale spec history — that would delete `client/lib/export/`. +**Next:** merge PR #25. Do **not** cut a second implementation branch off stale spec history. --- ## Goal @@ -88,7 +86,7 @@ Validate → `DELETE` children before parents (`maintenance_events` → `fill_up 3. ✅ Strict CSV coercion. 4. ✅ 17 error codes + 6 warning codes. 5. ✅ Settings UI: **Import data** under **Export data**. -6. ❌ Tests per spec § Test expectations (all 17) — **this is the remaining work.** +6. ✅ Tests per spec § Test expectations (all 17) — `client/test/import/`. ## Scope (out) @@ -109,32 +107,32 @@ Validate → `DELETE` children before parents (`maintenance_events` → `fill_up ## Acceptance - [x] Implementation in `client/lib/import/` + Settings → Import data (replace) -- [x] `delivery-plan-v1.md` M2 row + Current focus updated (honest: tests outstanding) -- [ ] Golden round-trip: export fixture → import into empty DB → canonical columns equal row for row -- [ ] Importing the same ZIP twice yields identical state (idempotent, no duplicate `id`s) -- [ ] Replace clears prior history: populated DB + disjoint ZIP → exactly the ZIP's rows remain -- [ ] `settings` updated in place, `settings.id` unchanged, prefs adopted, `default_vehicle_id` validated -- [ ] Outbox cleared with the discarded count reported -- [ ] Drafts reconciled: surviving vehicle keeps draft + photos; destroyed vehicle purges both, files deleted post-commit -- [ ] Typed keyword enforced when local history is non-empty, skipped when empty -- [ ] Header mutation, photo-shaped content, duplicate id, FK orphan, and each value violation reject with the DB untouched -- [ ] Imported rows have `row_version IS NULL`; nothing enqueued -- [ ] Atomicity: an induced mid-write failure leaves pre-existing rows intact -- [ ] Module-purity + streaming tests present (device timing deferred to CES-68 per export A4) -- [ ] Header-constant drift test (import expected set *is* the export constant set) -- [ ] `flutter analyze` + `flutter test --no-pub` + `python3 ci/telemetry-gate.py` green -- [ ] CES-71 unblocked — **only after this is on `main` with a working round-trip** -- [ ] Linear CES-70 Done + closeout comment — **not before tests** +- [x] `delivery-plan-v1.md` M2 row + Current focus updated +- [x] Golden round-trip: export fixture → import into empty DB → canonical columns equal row for row +- [x] Importing the same ZIP twice yields identical state (idempotent, no duplicate `id`s) +- [x] Replace clears prior history: populated DB + disjoint ZIP → exactly the ZIP's rows remain +- [x] `settings` updated in place, `settings.id` unchanged, prefs adopted, `default_vehicle_id` validated +- [x] Outbox cleared with the discarded count reported +- [x] Drafts reconciled: surviving vehicle keeps draft + photos; destroyed vehicle purges both, files deleted post-commit +- [x] Typed keyword enforced when local history is non-empty, skipped when empty +- [x] Header mutation, photo-shaped content, duplicate id, FK orphan, and each value violation reject with the DB untouched +- [x] Imported rows have `row_version IS NULL`; nothing enqueued +- [x] Atomicity: an induced mid-write failure leaves pre-existing rows intact +- [x] Module-purity + streaming tests present (device timing deferred to CES-68 per export A4) +- [x] Header-constant drift test (import expected set *is* the export constant set) +- [x] `flutter analyze` + `flutter test --no-pub` + `python3 ci/telemetry-gate.py` green +- [ ] CES-71 unblocked — **only after this is on `main`** +- [ ] Linear CES-70 Done + closeout comment — **not before merge to `main`** ## Implementation status (2026-08-21) 1. **ZIP reader.** Central-directory sizes (`client/lib/import/zip_read.dart`). STORE is native; DEFLATE via injected `Inflate`. Production inflater is `ZLibDecoder(raw: true)` in `import_service.dart` (`dart:io`), so the parser stays pure. 2. **Confirm dialog.** Incoming vs replaced counts, both `user_key_hash` values, export-first button. Typed keyword `REPLACE` (`importConfirmationKeyword`). Service enforces the keyword only when `requiresTypedConfirmation` (local history non-empty). Empty DB still shows the dialog; the keyword is not required. -3. **Headers.** `validate.dart` imports `client/lib/export/headers.dart`. **Drift test not written.** +3. **Headers.** `validate.dart` imports `client/lib/export/headers.dart`. Drift test: `client/test/import/headers_drift_test.dart` (`same()` identity). 4. **Drafts/photos.** Apply deletes `photo_refs` then drafts inside the txn; returns `photoIdsToDelete`. `ImportService.commit` deletes files **after** commit. Failures are swallowed — `PhotoService.sweep` collects orphans. 5. **Errors.** 17 `ImportErrorCode` values + 6 `ImportWarningCode` values. Validation happens before the txn; apply is one Drift transaction (`E_TXN_FAILED` on failure). -6. **Tests.** None. `client/test/import/` does not exist. Pointer: [`tests/import/README.md`](../../../tests/import/README.md). -7. **Limits.** Device timing deferred to CES-68. Keyword is English-only. Pre-M3 every user with fill-ups has a non-empty outbox — keep "queued changes discarded" quiet. Confirm dialog currently returns the keyword even on empty DB (service skips the check). -8. **PR / Linear.** Draft [PR #25](https://github.com/JMNofziger/cestovni/pull/25). CES-70 **In Progress**. CES-71 **Backlog** (automation had wrongly marked both Done). +6. **Tests.** `client/test/import/` — 27 cases covering spec items 1–17. Pointer: [`tests/import/README.md`](../../../tests/import/README.md). Full suite: `flutter analyze` clean, `flutter test --no-pub` **303 passed / 1 skipped** (E2E), telemetry-gate PASS. +7. **Limits.** Device timing deferred to CES-68. Keyword is English-only. Pre-M3 every user with fill-ups has a non-empty outbox — keep "queued changes discarded" quiet. +8. **PR / Linear.** Draft [PR #25](https://github.com/JMNofziger/cestovni/pull/25). CES-70 **In Progress** until merge. CES-71 **Backlog**. Tag: `CES-70 — ZIP import`. diff --git a/docs/product/ux/UX_IMPLEMENTATION_GAPS.md b/docs/product/ux/UX_IMPLEMENTATION_GAPS.md index 631c814..b6c041d 100644 --- a/docs/product/ux/UX_IMPLEMENTATION_GAPS.md +++ b/docs/product/ux/UX_IMPLEMENTATION_GAPS.md @@ -2,9 +2,9 @@ **Purpose:** Track documentation and product gaps discovered before M1 UI execution so they do not leak into implementation as silent contradictions. -**Gate (closed):** Critical-gap rows **Done** (repo + Linear). **CES-39 Done** (2026-07-17). **CES-65 + CES-66 Done** (repo + Linear 2026-07-22). **CES-67 Done** (2026-08-15). **CES-40 Done** (receipt photos, 2026-08-15) — **M1 verticals all closed.** **CES-41** export shipped. Next spine item: import (**CES-70**, M2) — implemented, tests outstanding. +**Gate (closed):** Critical-gap rows **Done** (repo + Linear). **CES-39 Done** (2026-07-17). **CES-65 + CES-66 Done** (repo + Linear 2026-07-22). **CES-67 Done** (2026-08-15). **CES-40 Done** (receipt photos, 2026-08-15) — **M1 verticals all closed.** **CES-41** export shipped. Next spine item: import (**CES-70**, M2) — tests green on PR #25, not on `main`. -**Last reviewed:** 2026-08-21 (hygiene: CES-70 import implemented, tests outstanding) +**Last reviewed:** 2026-08-21 (CES-70 spec tests green on PR #25) --- diff --git a/docs/product/ux/cestovni-views.md b/docs/product/ux/cestovni-views.md index a72873b..a0d524f 100644 --- a/docs/product/ux/cestovni-views.md +++ b/docs/product/ux/cestovni-views.md @@ -159,7 +159,7 @@ Screenshot: `screenshots/dark-midnight/settings.png` - Preferences currently implemented in `settings_page.dart` (distance/volume/currency/timezone). - Vehicle CRUD is implemented across `vehicle_list_page.dart`, `vehicle_detail_page.dart`, and `vehicle_form_page.dart`. - Export is **shipped (CES-41)** — Settings → Backup → **Export data** (`client/lib/export/`, photos excluded). -- Import is **implemented (CES-70)** — Settings → Backup → **Import data** (`client/lib/import/` + `import_data_section.dart`), replace mode, typed `REPLACE` when local history is non-empty. Automated tests outstanding; not on `main` yet. Destructive reset remains Later. +- Import is **implemented (CES-70)** — Settings → Backup → **Import data** (`client/lib/import/` + `import_data_section.dart`), replace mode, typed `REPLACE` when local history is non-empty. Spec tests green on PR #25; not on `main` yet. Destructive reset remains Later. **Scope gate** diff --git a/docs/specs/ARCHITECTURE.md b/docs/specs/ARCHITECTURE.md index efe428d..2767538 100644 --- a/docs/specs/ARCHITECTURE.md +++ b/docs/specs/ARCHITECTURE.md @@ -77,7 +77,7 @@ Parent **CES-22**; children **CES-26–CES-32** (see [`README.md`](README.md)). ## Implementation status (Stage 5) -- **M0 (2026-04-18):** Mobile shell + local Drift database on `main` — repository root [`client/`](../../client/). Matches **ADR 003** (Flutter + Drift) and **`data-model.md`** client tables (`schema_version` **3** as of 2026-06-30: `0001_init` + `0002_add_maintenance_events_category_shop` + `0003_settings_default_vehicle_id`; fresh `onCreate` uses `m.createAll()`). **CES-41 export** is on `main` (`client/lib/export/`). **CES-70 import** is implemented (`client/lib/import/`) with tests outstanding. Not yet: production server backup (M3), Sentry wiring (M4). CI: [`ci/client-build.yml`](../../ci/client-build.yml); telemetry allow-list gate includes a **Dart source scan** for literal `Telemetry.emit` event names ([`ci/telemetry-gate.py`](../../ci/telemetry-gate.py)). +- **M0 (2026-04-18):** Mobile shell + local Drift database on `main` — repository root [`client/`](../../client/). Matches **ADR 003** (Flutter + Drift) and **`data-model.md`** client tables (`schema_version` **3** as of 2026-06-30: `0001_init` + `0002_add_maintenance_events_category_shop` + `0003_settings_default_vehicle_id`; fresh `onCreate` uses `m.createAll()`). **CES-41 export** is on `main` (`client/lib/export/`). **CES-70 import** is implemented (`client/lib/import/`) with spec tests green on PR #25. Not yet: production server backup (M3), Sentry wiring (M4). CI: [`ci/client-build.yml`](../../ci/client-build.yml); telemetry allow-list gate includes a **Dart source scan** for literal `Telemetry.emit` event names ([`ci/telemetry-gate.py`](../../ci/telemetry-gate.py)). ## Related compliance / ops diff --git a/docs/specs/README.md b/docs/specs/README.md index ec39f11..df30d2c 100644 --- a/docs/specs/README.md +++ b/docs/specs/README.md @@ -19,7 +19,7 @@ Each Linear issue should include a **`Spec:`** line pointing to the relevant fil | [`consumption-math.md`](consumption-math.md) | Math, segments, fill-up flags | CES-26 | Complete (v1) | | [`si-units.md`](si-units.md) | Canonical INT storage + conversions | CES-27 | Complete (v1) | | [`export-v1.md`](export-v1.md) | ZIP / CSV export contract | CES-28 | Complete (v1) | -| [`export-import.md`](export-import.md) | Device-to-device ZIP import (zero-server) | CES-70 | Complete (v1) — implemented; tests outstanding | +| [`export-import.md`](export-import.md) | Device-to-device ZIP import (zero-server) | CES-70 | Complete (v1) — tests green on PR #25; not on `main` | | [`telemetry-allowlist.md`](telemetry-allowlist.md) | Crash + product event allow-list | CES-29 | Complete (v1) | | [`telemetry-events.v1.yaml`](telemetry-events.v1.yaml) | Machine-readable event catalogue | CES-29 | Complete (v1) | | [`photo-pipeline.md`](photo-pipeline.md) | Ephemeral photos / deferred entry | CES-30 | Complete (v1) | diff --git a/docs/specs/export-import.md b/docs/specs/export-import.md index 9010733..ae6ecb7 100644 --- a/docs/specs/export-import.md +++ b/docs/specs/export-import.md @@ -1,6 +1,6 @@ # Spec: Export ZIP import (device-to-device, zero-server) -**Status:** **Complete (v1) — implemented, automated tests outstanding.** All product locks resolved 2026-08-16 (see [§ Product decisions](#product-decisions-locked-2026-08-16)). Mode is **replace**. Code lives in `client/lib/import/` + Settings → **Import data**; the 17 cases in [§ Test expectations](#test-expectations) are **not written**. CES-71 stays blocked until this is on `main` with a working round-trip. +**Status:** **Complete (v1) — implemented, tests green on PR #25, not on `main` yet.** All product locks resolved 2026-08-16 (see [§ Product decisions](#product-decisions-locked-2026-08-16)). Mode is **replace**. Code lives in `client/lib/import/` + Settings → **Import data**; the 17 cases in [§ Test expectations](#test-expectations) are in `client/test/import/`. CES-71 stays blocked until this is on `main`. **Linear:** [CES-70](https://linear.app/personal-interests-llc/issue/CES-70) **Depends on:** [CES-41](https://linear.app/personal-interests-llc/issue/CES-41) export ([PR #21](https://github.com/JMNofziger/cestovni/pull/21), `client/lib/export/`) **Blocks:** [CES-71](https://linear.app/personal-interests-llc/issue/CES-71) (`cadence_km` → `cadence_m` rename) @@ -22,7 +22,7 @@ Let a user move their structured history between two devices with **zero server* | | Current | Done when CES-70 is 🟩 | |---|---|---| | Export | Settings → **Export data** writes a STORE ZIP: `manifest.json`, `README_export.txt`, five CSVs | unchanged | -| Import | Settings → **Import data** (`client/lib/import/` + `import_data_section.dart`) replaces local history from a ZIP the app produced. **Not on `main` yet.** Automated tests in `client/test/import/` do not exist. | Same code on `main`; [§ Test expectations](#test-expectations) green | +| Import | Settings → **Import data** (`client/lib/import/` + `import_data_section.dart`) replaces local history from a ZIP the app produced. **Not on `main` yet.** Tests live in `client/test/import/` (spec § Test expectations; green on PR #25). | Same code on `main` | | Mode | **Replace** — destructive restore, typed `REPLACE` when there is data to lose. Merge is not built. | unchanged | | Photos | never exported (`photos_in_export: false`) | never imported; a photo-shaped ZIP **fails closed** | | `row_version` | client never writes it; CSV cells empty, manifest `null` | imported rows keep `row_version = NULL` (never synced) | diff --git a/docs/specs/export-v1.md b/docs/specs/export-v1.md index 59b5450..77b00a3 100644 --- a/docs/specs/export-v1.md +++ b/docs/specs/export-v1.md @@ -216,7 +216,7 @@ Both targets are validated by `tests/export/` fixtures. ## Non-goals (v1) - **No incremental / diff exports.** Every export is the full structured state. -- **No re-import tool in-app (v1) — historical.** This non-goal was deferred, not permanent. In-app restore is [CES-70](https://linear.app/personal-interests-llc/issue/CES-70): Settings → **Import data**, specified in [`export-import.md`](export-import.md) (replace mode). Implementation is in `client/lib/import/`; automated tests are still outstanding and the code is not on `main` yet. Merge / live multi-device sync remains a v1.x item in [`sync-protocol.md`](sync-protocol.md#v1x-roadmap-pointer-only--tbd-in-this-pass). +- **No re-import tool in-app (v1) — historical.** This non-goal was deferred, not permanent. In-app restore is [CES-70](https://linear.app/personal-interests-llc/issue/CES-70): Settings → **Import data**, specified in [`export-import.md`](export-import.md) (replace mode). Implementation + spec tests are on [PR #25](https://github.com/JMNofziger/cestovni/pull/25); not on `main` yet. Merge / live multi-device sync remains a v1.x item in [`sync-protocol.md`](sync-protocol.md#v1x-roadmap-pointer-only--tbd-in-this-pass). - **No photos.** Period. - **No signed / encrypted ZIP.** User-managed; they can encrypt after export if they want. diff --git a/docs/specs/sync-protocol.md b/docs/specs/sync-protocol.md index df8ee41..1e15102 100644 --- a/docs/specs/sync-protocol.md +++ b/docs/specs/sync-protocol.md @@ -211,7 +211,7 @@ Dead-letter is a **signal** that something is wrong — either a client bug or a - **Push channels.** Options: long-poll on `/changes`, server-sent events, WebSocket. Decision deferred to when real-time UX requirements are set. - **Conflict UX.** How to surface (if ever) a rejected amendment to the user. Deferred with the above. - **Re-evaluation of managed sync runtime.** Gate defined in ADR 002 revisit gates. -- **Device-to-device import from a self-produced export ZIP** ([CES-70](https://linear.app/personal-interests-llc/issue/CES-70)) — specified in [`export-import.md`](export-import.md) (replace, not merge; no outbox enqueue; `row_version` stays `NULL`). Implementation is in `client/lib/import/` + Settings → **Import data**. Automated tests and merge to `main` are still outstanding. This is **not** live multi-device sync: merge rules, tombstones, and conflict UX remain unspecified and must not be inferred from replace-mode restore. +- **Device-to-device import from a self-produced export ZIP** ([CES-70](https://linear.app/personal-interests-llc/issue/CES-70)) — specified in [`export-import.md`](export-import.md) (replace, not merge; no outbox enqueue; `row_version` stays `NULL`). Implementation is in `client/lib/import/` + Settings → **Import data**. Spec tests green on [PR #25](https://github.com/JMNofziger/cestovni/pull/25); merge to `main` outstanding. This is **not** live multi-device sync: merge rules, tombstones, and conflict UX remain unspecified and must not be inferred from replace-mode restore. ## References diff --git a/tests/import/README.md b/tests/import/README.md index c128c2a..ed0beb2 100644 --- a/tests/import/README.md +++ b/tests/import/README.md @@ -1,25 +1,34 @@ # tests/import — pointer -`docs/specs/export-import.md` § Test expectations places import tests -in `tests/import/` (or a pointer from here). They will live in the +`docs/specs/export-import.md` § Test expectations. Cases live in the Flutter client so `flutter test` picks them up without a second runner: | Spec expectation | Implementation | |------------------|----------------| -| Golden round-trip | **Not written** — expected `client/test/import/` | -| Idempotency, replace, settings in-place, outbox, drafts | **Not written** | -| Header / photo / duplicate-id / FK / value rejects | **Not written** | -| Never-synced state, atomicity, confirmation gate | **Not written** | -| Module purity + streaming | **Not written** | -| Header constants == export | **Not written** — must import `client/lib/export/headers.dart`, never copy | +| 1 Golden round-trip | [`client/test/import/round_trip_test.dart`](../../client/test/import/round_trip_test.dart) | +| 2 Idempotency | `round_trip_test.dart` | +| 3 Header mutation | [`client/test/import/rejects_test.dart`](../../client/test/import/rejects_test.dart) | +| 4 Photo fail-closed | `rejects_test.dart` | +| 5 Derived columns ignored | `round_trip_test.dart` | +| 6 Cadence meters verbatim | `round_trip_test.dart` | +| 7 Value validation + duplicate id | `rejects_test.dart` | +| 8 Atomicity | [`client/test/import/replace_test.dart`](../../client/test/import/replace_test.dart) | +| 9 Never-synced state | `round_trip_test.dart` | +| 10 FK orphan | `rejects_test.dart` | +| 11 Module purity | [`client/test/import/module_purity_test.dart`](../../client/test/import/module_purity_test.dart) | +| 12 Streaming (1 000 rows; device timing is CES-68) | [`client/test/import/streaming_test.dart`](../../client/test/import/streaming_test.dart) | +| 13 Replace clears prior history | `replace_test.dart` | +| 14 `settings` in place | `replace_test.dart` | +| 15 Outbox cleared | `replace_test.dart` | +| 16 Drafts / photos reconcile | `replace_test.dart` | +| 17 Confirmation gate | `replace_test.dart` | +| Header constants == export | [`client/test/import/headers_drift_test.dart`](../../client/test/import/headers_drift_test.dart) | -**Code is implemented** in `client/lib/import/` + Settings → **Import data** -(`client/lib/app/pages/import_data_section.dart`). The 17 cases are the -remaining CES-70 work before this goes 🟩. Do not mark [CES-70](https://linear.app/personal-interests-llc/issue/CES-70) Done, and do not unblock [CES-71](https://linear.app/personal-interests-llc/issue/CES-71), until those tests exist and the code is on `main`. +**Not in CI (per `export-v1.md` § A4):** 10 000-row device timing. That pass moves to [CES-68](https://linear.app/personal-interests-llc/issue/CES-68). **Not in this folder:** ZIP export ([CES-41](https://linear.app/personal-interests-llc/issue/CES-41)) — see [`../export/README.md`](../export/README.md). -When tests land, run: +Run them with: ```bash cd client && flutter test --no-pub test/import/ test/app/settings_page_test.dart