Skip to content

fix: improve diff conversion for CSV, XML, and file parsing (#77, #78) #121

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Exotic209093 merged 1 commit into from
Sep 9, 2026
Merged

fix: improve diff conversion for CSV, XML, and file parsing (#77, #78) #121

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion src/ui/utils/csv.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@ function neutralizeFormulaInjection(s: string): string {

function escapeCsvValue(value: unknown): string {
if (value === null || value === undefined) return '';
const s = neutralizeFormulaInjection(String(value));
// Serialize objects/arrays to JSON to avoid "[object Object]" in output,
// then neutralize formula-injection payloads on the resulting string.
const raw = typeof value === 'object' ? JSON.stringify(value) : String(value);
const s = neutralizeFormulaInjection(raw);
if (/[",\n\r]/.test(s)) {
return `"${s.replace(/"/g, '""')}"`;
}
Expand Down
32 changes: 26 additions & 6 deletions src/ui/utils/dataDiff.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,18 @@ export interface DataDiffResult {
changed: RecordDiff[];
unchanged: RecordDiff[];
summary: { total: number; added: number; removed: number; changed: number };
/** Number of duplicate match-key values silently collapsed during indexing. */
duplicateKeyCount: number;
}

/**
* Stringify a value for comparison. Objects and arrays are JSON-stringified
* so that nested structures compare by content rather than as "[object Object]".
*/
function stringifyForCompare(value: unknown): string {
if (value === null || value === undefined) return '';
if (typeof value === 'object') return JSON.stringify(value);

@chatgpt-codex-connector chatgpt-codex-connector Bot Sep 9, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Canonicalize nested objects before comparing them

When two JSON files contain semantically identical nested objects with different property order, direct JSON.stringify calls produce different strings鈥攆or example, {a: 1, b: 2} versus {b: 2, a: 1}. The local Diff flow preserves JSON insertion order and passes these values here, so it incorrectly reports the field and record as changed. Use an order-independent deep comparison or canonical key ordering for objects.

Useful? React with 馃憤聽/ 馃憥.

return String(value);
}

export function diffRecords(
Expand All @@ -38,15 +50,20 @@ export function diffRecords(
objectName: string,
): DataDiffResult {
const sourceMap = new Map<string, Record<string, unknown>>();
let duplicateKeyCount = 0;
for (const rec of sourceRecords) {
const key = String(rec[matchField] ?? '');
if (key) sourceMap.set(key, rec);
if (!key) continue;
if (sourceMap.has(key)) duplicateKeyCount++;
sourceMap.set(key, rec);
}

const targetMap = new Map<string, Record<string, unknown>>();
for (const rec of targetRecords) {
const key = String(rec[matchField] ?? '');
if (key) targetMap.set(key, rec);
if (!key) continue;
if (targetMap.has(key)) duplicateKeyCount++;
targetMap.set(key, rec);
}

const allKeys = new Set([...sourceMap.keys(), ...targetMap.keys()]);
Expand All @@ -70,7 +87,9 @@ export function diffRecords(
for (const field of compareFields) {
const sv = source[field];
const tv = target[field];
if (String(sv ?? '') !== String(tv ?? '')) {
const sStr = stringifyForCompare(sv);
const tStr = stringifyForCompare(tv);
if (sStr !== tStr) {
changedFields.push(field);
fieldDiffs[field] = { source: sv, target: tv };
}
Expand All @@ -95,6 +114,7 @@ export function diffRecords(
changed,
unchanged,
summary: { total: allKeys.size, added: added.length, removed: removed.length, changed: changed.length },
duplicateKeyCount,
};
}

Expand All @@ -103,9 +123,9 @@ export function diffToCsv(diff: DataDiffResult): string {
const all = [...diff.added, ...diff.removed, ...diff.changed];
for (const d of all) {
const fieldCols = diff.fields.map(f => {
const sv = d.sourceRecord?.[f] ?? '';
const tv = d.targetRecord?.[f] ?? '';
return `"${String(sv).replace(/"/g, '""')}","${String(tv).replace(/"/g, '""')}"`;
const sv = stringifyForCompare(d.sourceRecord?.[f]);
const tv = stringifyForCompare(d.targetRecord?.[f]);
return `"${sv.replace(/"/g, '""')}","${tv.replace(/"/g, '""')}"`;
}).join(',');
rows.push(`"${d.keyValue}","${d.status}","${d.changedFields.join('; ')}",${fieldCols}`);
}
Expand Down
34 changes: 30 additions & 4 deletions src/ui/utils/fileParse.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,23 @@ export function inferHeaders(records: Array<Record<string, unknown>>): string[]
export async function parseJsonFile(file: File): Promise<ParsedDataset> {
const text = await file.text();
const parsed = JSON.parse(text) as unknown;
const records = Array.isArray(parsed) ? parsed : [parsed];

// Unwrap metadata-wrapped exports (e.g. { exportedAt, records: [...] })
// so that re-importing our own JSON output yields the original records.
let records: unknown[];
if (Array.isArray(parsed)) {
records = parsed;
} else if (
parsed &&
typeof parsed === 'object' &&
!Array.isArray(parsed) &&
Array.isArray((parsed as Record<string, unknown>).records)
) {
records = (parsed as Record<string, unknown>).records as unknown[];
Comment on lines +72 to +74

@chatgpt-codex-connector chatgpt-codex-connector Bot Sep 9, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Restrict unwrapping to actual metadata export envelopes

When an uploaded JSON document is intended to be one record but has an array-valued field named records, this branch discards the outer record and treats that field's elements as the dataset. For example, {"Id":"1","records":[{"value":"x"}]} loses both Id and the enclosing structure. Since WaveLink's own metadata export includes identifying fields such as exportedAt, recordCount, columns, and records, verify that envelope shape before unwrapping instead of treating every object with a records array as metadata.

Useful? React with 馃憤聽/ 馃憥.

} else {
records = [parsed];
}

const objects = records
.filter(r => r && typeof r === 'object')
.map(r => r as Record<string, unknown>);
Expand Down Expand Up @@ -123,14 +139,24 @@ export async function parseExcelFile(file: File): Promise<ParsedDataset> {
}
const XLSX = await import(/* webpackChunkName: "xlsx" */ 'xlsx/dist/xlsx.mini.min.js');
const buf = await file.arrayBuffer();
const wb = XLSX.read(buf, { type: 'array', sheetRows: MAX_EXCEL_ROWS + 1 });
// cellDates converts Excel date serial numbers to JS Date objects at read time;
// raw: true on sheet_to_json preserves numeric precision (avoids scientific notation on long IDs).
const wb = XLSX.read(buf, { type: 'array', sheetRows: MAX_EXCEL_ROWS + 1, cellDates: true });
const sheetName = wb.SheetNames[0];
if (!sheetName) throw new Error('Excel file contains no sheets');
const sheet = wb.Sheets[sheetName];
const rows = XLSX.utils.sheet_to_json<Record<string, unknown>>(sheet, { defval: null, raw: false });
if (rows.length > MAX_EXCEL_ROWS) {
const rawRows = XLSX.utils.sheet_to_json<Record<string, unknown>>(sheet, { defval: null, raw: true });

@chatgpt-codex-connector chatgpt-codex-connector Bot Sep 9, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Preserve formatted identifier text when parsing Excel

When an XLSX column contains numeric identifiers with formatting such as 000000, raw: true returns the underlying number (for example, 123) instead of the displayed value (000123); it can likewise expose rounded numeric values for long identifiers. The parsed value flows through parseAnyFile into conversion and data-push mappings, so an upsert can use a different external ID and create or update the wrong record. The previous formatted-string parsing should be retained for identifier fidelity rather than relying on raw JavaScript numbers.

Useful? React with 馃憤聽/ 馃憥.

if (rawRows.length > MAX_EXCEL_ROWS) {
throw new Error(`Excel worksheets must contain ${MAX_EXCEL_ROWS.toLocaleString()} rows or fewer`);
}
// Convert Date objects to ISO strings for consistent round-trip fidelity.
const rows = rawRows.map(row => {
const out: Record<string, unknown> = {};
for (const [k, v] of Object.entries(row)) {
out[k] = v instanceof Date ? v.toISOString() : v;
}
return out;
});
return { records: rows, headers: inferHeaders(rows) };
}

Expand Down
4 changes: 3 additions & 1 deletion src/ui/utils/xml.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@
*/
function escapeXml(value: unknown): string {
if (value === null || value === undefined) return '';
return String(value)
// Serialize objects/arrays to JSON to avoid "[object Object]" in output.
const s = typeof value === 'object' ? JSON.stringify(value) : String(value);
return s
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
Expand Down
Loading