Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions features/upload/utils/column-detection.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,52 @@ describe("detectColumns", () => {
expect(result?.dateColumn).toBe("timestamp");
});

it("matches Xquik CSV export headers after normalization", () => {
const result = detectColumns(["tweet_text", "retweet_count", "user_name", "tweet_created_at"]);
expect(result).toEqual({
reviewColumn: "tweet_text",
ratingColumn: undefined,
authorColumn: "user_name",
dateColumn: "tweet_created_at",
});
});

it("matches social post export aliases", () => {
const result = detectColumns(["post_text", "x_handle", "published_at"]);
expect(result).toEqual({
reviewColumn: "post_text",
ratingColumn: undefined,
authorColumn: "x_handle",
dateColumn: "published_at",
});
});

it("matches via substring when no exact alias is present", () => {
const result = detectColumns(["product_review_text", "overall_rating"]);
expect(result?.reviewColumn).toBe("product_review_text");
expect(result?.ratingColumn).toBe("overall_rating");
});

it("does not treat short header fragments as aliases", () => {
expect(detectColumns(["at", "id"])).toBeNull();
expect(detectColumns(["review", "at", "id"])).toEqual({
reviewColumn: "review",
ratingColumn: undefined,
authorColumn: undefined,
dateColumn: undefined,
});
});

it("does not match aliases inside unrelated normalized words", () => {
expect(detectColumns(["postal_code", "candidate", "contest"])).toBeNull();
expect(detectColumns(["review", "postal_code", "candidate"])).toEqual({
reviewColumn: "review",
ratingColumn: undefined,
authorColumn: undefined,
dateColumn: undefined,
});
});

it("leaves optional columns undefined when absent", () => {
const result = detectColumns(["feedback"]);
expect(result?.reviewColumn).toBe("feedback");
Expand Down
81 changes: 60 additions & 21 deletions features/upload/utils/column-detection.ts
Original file line number Diff line number Diff line change
@@ -1,40 +1,79 @@
import type { DetectedColumns } from "../types";

const REVIEW_ALIASES = [
"review", "review_text", "review_body", "reviewtext",
"text", "body", "content", "comment", "comments",
"feedback", "message", "description", "opinion",
"review",
"review_text",
"review_body",
"reviewtext",
"text",
"body",
"content",
"comment",
"comments",
"feedback",
"message",
"description",
"opinion",
"tweet",
"tweet_text",
"post",
"post_text",
"caption",
];

const RATING_ALIASES = [
"rating", "ratings", "stars", "star", "score",
"rate", "star_rating", "overall", "overall_rating",
"rating",
"ratings",
"stars",
"star",
"score",
"rate",
"star_rating",
"overall",
"overall_rating",
];

const AUTHOR_ALIASES = [
"author", "reviewer", "name", "username", "user",
"customer", "customer_name", "reviewer_name",
"author",
"reviewer",
"name",
"username",
"user",
"customer",
"customer_name",
"reviewer_name",
"user_name",
"screen_name",
"handle",
"x_handle",
];

const DATE_ALIASES = [
"date", "created_at", "createdat", "timestamp",
"time", "posted", "posted_at", "review_date",
"date",
"created_at",
"createdat",
"timestamp",
"time",
"posted",
"posted_at",
"review_date",
"created",
"created_time",
"posted_date",
"published_at",
"tweet_created_at",
];

function matchColumn(
headers: string[],
aliases: string[]
): string | undefined {
// Exact match first — most reliable
function matchColumn(headers: string[], aliases: string[]): string | undefined {
// Exact match first - most reliable
for (const alias of aliases) {
const found = headers.find((h) => h === alias);
const found = headers.find((header) => header === alias);
if (found) return found;
}
// Substring match — handles "product_review_text" etc.
// Token-sequence match - handles "product_review_text" without matching
// unrelated words such as "postal_code" for the "post" alias.
for (const alias of aliases) {
const found = headers.find(
(h) => h.includes(alias) || alias.includes(h)
);
const found = headers.find((header) => `_${header}_`.includes(`_${alias}_`));
if (found) return found;
}
return undefined;
Expand All @@ -43,7 +82,7 @@ function matchColumn(
export function detectColumns(normalizedHeaders: string[]): DetectedColumns | null {
const reviewColumn = matchColumn(normalizedHeaders, REVIEW_ALIASES);

// Review column is mandatory without it we can't proceed
// Review column is mandatory - without it we can't proceed
if (!reviewColumn) return null;

return {
Expand All @@ -52,4 +91,4 @@ export function detectColumns(normalizedHeaders: string[]): DetectedColumns | nu
authorColumn: matchColumn(normalizedHeaders, AUTHOR_ALIASES),
dateColumn: matchColumn(normalizedHeaders, DATE_ALIASES),
};
}
}