From d5cfd7587b0fb673ed474421b305239ba0b0bda5 Mon Sep 17 00:00:00 2001 From: carlh171112 Date: Sat, 11 Jul 2026 14:39:26 -0700 Subject: [PATCH 1/2] feat(copycat): implement plagiarism detection engine with containment scoring and direction assessment This commit introduces the `copycat` module, which provides a deterministic mechanism for detecting code plagiarism by measuring the containment of added code in prior art. Key features include: - Calculation of containment scores based on shingling of code lines. - Determination of copy direction based on submission timestamps. - Public-safe findings that report potential overlaps without revealing sensitive information. Additionally, unit tests for the `copycat` functionality have been added to ensure accuracy and reliability of the detection mechanisms. --- src/signals/copycat.ts | 168 ++++++++++++++++++++++++++++++++++ test/unit/copycat.test.ts | 186 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 354 insertions(+) create mode 100644 src/signals/copycat.ts create mode 100644 test/unit/copycat.test.ts diff --git a/src/signals/copycat.ts b/src/signals/copycat.ts new file mode 100644 index 000000000..f63495462 --- /dev/null +++ b/src/signals/copycat.ts @@ -0,0 +1,168 @@ +// Copycat / plagiarism detection engine (#1969) — the deterministic containment/similarity primitive the +// (currently inert) `gate.copycat.mode` / `gate.copycat.minScore` config would act on. A natural sibling of the +// deterministic anti-slop signal (src/signals/slop.ts) and duplicate-cluster adjudication +// (src/signals/duplicate-winner.ts): given a PR's ADDED code and one piece of prior art, it measures how much of +// the PR's added code is CONTAINED in that prior art, resolves copy DIRECTION by submission timestamp (the +// earlier submission is the original, never the copier), and maps the result through the configured tier +// (warn → label → block) into a public-safe finding. +// +// PURE / PRECISION-FIRST: no IO, no Date.now(), no randomness — identical inputs always yield the identical +// verdict. It is deliberately false-accusation-averse: it only ever `wouldAct` when the score clears the +// threshold AND the candidate is unambiguously the LATER (copying) submission AND a non-`off` mode is set. Any +// missing/ambiguous timestamp — or the candidate being the earlier work — resolves to "do not act". +// +// DEFERRED (a later slice of #1969): the gate call-site that feeds a PR's real added-code + prior-art into this +// engine and turns a `wouldAct` verdict into an actual label/block plus the cross-repo "strikes" escalation. +// This slice is the engine + direction + tier mapping only, mirroring how the miner-side self-plagiarism +// throttle (packages/gittensory-engine/src/governor/self-plagiarism.ts) "gates nothing on its own". + +import type { AdvisorySeverity, CopycatGateMode } from "../types"; +import type { SignalFinding } from "./engine"; + +/** Precision-first default: only a HIGH containment (>= 85% of the PR's added code found in the prior art) trips + * the check when `gate.copycat.minScore` is unset. Mirrors the conservative 0.85 spirit of the miner-side + * self-plagiarism throttle (governor/self-plagiarism.ts's DEFAULT_SELF_PLAGIARISM_SIMILARITY_THRESHOLD). */ +export const DEFAULT_COPYCAT_MIN_SCORE = 85; + +/** Shingle width: consecutive normalized lines folded into one token, so containment reflects COPIED PASSAGES + * (multi-line runs) rather than incidental single-line coincidences (a lone `}` / `return null;`) that would + * inflate a naive line-set overlap. */ +const SHINGLE_SIZE = 3; + +/** Copy direction between the candidate PR and one prior-art submission, decided purely by submission time. */ +export type CopycatDirection = "candidate_copied" | "candidate_is_prior" | "ambiguous"; + +/** Normalize one source line for structural comparison: collapse internal whitespace runs, trim, lowercase — so + * pure reformatting/indentation churn never reads as copied content. */ +function normalizeLine(line: string): string { + return line.replace(/\s+/g, " ").trim().toLowerCase(); +} + +/** Drop blank/whitespace-only lines and normalize the rest, preserving order. */ +function normalizedLines(lines: readonly string[]): string[] { + return lines.map(normalizeLine).filter((line) => line.length > 0); +} + +/** Fold normalized lines into a set of SHINGLE_SIZE-line shingles. Fewer than SHINGLE_SIZE non-trivial lines + * collapse to a single whole-block shingle so tiny snippets still compare (never silently score 0). */ +export function codeShingles(lines: readonly string[]): Set { + const normalized = normalizedLines(lines); + if (normalized.length === 0) return new Set(); + if (normalized.length < SHINGLE_SIZE) return new Set([normalized.join("\n")]); + const shingles = new Set(); + for (let i = 0; i + SHINGLE_SIZE <= normalized.length; i += 1) { + shingles.add(normalized.slice(i, i + SHINGLE_SIZE).join("\n")); + } + return shingles; +} + +/** Asymmetric containment (0-100): the percentage of the CANDIDATE's added-code shingles that also appear in the + * PRIOR ART. Unlike symmetric Jaccard, this answers "how much of THIS PR is copied FROM prior art" without being + * diluted by a large prior-art corpus. 0 when either side has no comparable content. */ +export function containmentScore(candidateLines: readonly string[], priorArtLines: readonly string[]): number { + const candidate = codeShingles(candidateLines); + if (candidate.size === 0) return 0; + const prior = codeShingles(priorArtLines); + if (prior.size === 0) return 0; + let contained = 0; + for (const shingle of candidate) { + if (prior.has(shingle)) contained += 1; + } + return Math.round((contained / candidate.size) * 100); +} + +/** Parse an ISO-8601 submission time to epoch ms; null for a missing/empty/unparseable value. */ +function submissionTimeMs(value: string | null | undefined): number | null { + if (!value) return null; + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : null; +} + +/** Copy direction by submission time: the EARLIER submission is the original, so the LATER one is the potential + * copier. Any missing/unparseable timestamp — or an exact tie — is "ambiguous" (fail-safe: never accuse). */ +export function copycatDirection( + candidateAt: string | null | undefined, + priorAt: string | null | undefined, +): CopycatDirection { + const candidateMs = submissionTimeMs(candidateAt); + const priorMs = submissionTimeMs(priorAt); + if (candidateMs === null || priorMs === null) return "ambiguous"; + if (candidateMs > priorMs) return "candidate_copied"; + if (candidateMs < priorMs) return "candidate_is_prior"; + return "ambiguous"; +} + +/** Clamp `gate.copycat.minScore` into 0-100; a non-numeric/non-finite value falls back to the engine default. */ +function normalizeMinScore(value: number | null | undefined): number { + if (typeof value !== "number" || !Number.isFinite(value)) return DEFAULT_COPYCAT_MIN_SCORE; + return Math.min(100, Math.max(0, Math.round(value))); +} + +/** Per-tier finding severity. `off` never produces a finding (see {@link assessCopycat}); it maps to `info` only + * so the lookup is total over {@link CopycatGateMode} without an unreachable branch. */ +const MODE_SEVERITY: Record = { + off: "info", + warn: "info", + label: "warning", + block: "critical", +}; + +/** Public-safe finding — reports the containment score + threshold only, never raw code, filenames, or any + * contributor identity (the caller redacts further; this text is already accusation-neutral). */ +function buildFinding(mode: CopycatGateMode, score: number, minScore: number): SignalFinding { + return { + code: "copycat_overlap", + title: "Potential copied code detected", + severity: MODE_SEVERITY[mode], + detail: `This pull request's added code reaches ${score}% containment against earlier prior art (threshold ${minScore}%).`, + action: "Confirm the overlapping code is original or properly attributed before merging.", + publicText: `High overlap (${score}%) with earlier prior art — please confirm originality or attribution.`, + }; +} + +export type CopycatAssessmentInput = { + /** The PR's ADDED source lines (the candidate). */ + candidateLines: readonly string[]; + /** One piece of prior art's source lines to compare against. */ + priorArtLines: readonly string[]; + /** ISO-8601 submission time of the candidate PR; absent/unparseable ⇒ ambiguous direction (never acts). */ + candidateSubmittedAt?: string | null | undefined; + /** ISO-8601 submission time of the prior art; absent/unparseable ⇒ ambiguous direction (never acts). */ + priorSubmittedAt?: string | null | undefined; + /** `gate.copycat.mode`; `off`/absent ⇒ never acts (score is still computed for observability). */ + mode?: CopycatGateMode | null | undefined; + /** `gate.copycat.minScore` (0-100); absent/out-of-range ⇒ {@link DEFAULT_COPYCAT_MIN_SCORE}. */ + minScore?: number | null | undefined; +}; + +export type CopycatAssessment = { + /** Asymmetric containment of the candidate in the prior art, 0-100. */ + score: number; + direction: CopycatDirection; + /** The resolved threshold the score was tested against. */ + minScore: number; + /** True ONLY when a non-`off` mode is set AND score >= threshold AND the candidate is the later (copying) work. */ + wouldAct: boolean; + findings: SignalFinding[]; +}; + +/** + * Assess one PR's added code against one piece of prior art (#1969). Pure and precision-first: the containment + * score is always computed for observability, but a finding is emitted ONLY when the configured mode is non-`off`, + * the score clears the (resolved) threshold, and the candidate is unambiguously the later submission — so an + * earlier-submitted victim, or any ambiguous/missing timing, is never flagged. + */ +export function assessCopycat(input: CopycatAssessmentInput): CopycatAssessment { + const minScore = normalizeMinScore(input.minScore); + const score = containmentScore(input.candidateLines, input.priorArtLines); + const direction = copycatDirection(input.candidateSubmittedAt, input.priorSubmittedAt); + const mode = input.mode ?? "off"; + const wouldAct = mode !== "off" && score >= minScore && direction === "candidate_copied"; + return { + score, + direction, + minScore, + wouldAct, + findings: wouldAct ? [buildFinding(mode, score, minScore)] : [], + }; +} diff --git a/test/unit/copycat.test.ts b/test/unit/copycat.test.ts new file mode 100644 index 000000000..726716743 --- /dev/null +++ b/test/unit/copycat.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, it } from "vitest"; +import { + assessCopycat, + codeShingles, + containmentScore, + copycatDirection, + DEFAULT_COPYCAT_MIN_SCORE, +} from "../../src/signals/copycat"; + +// Six distinct normalized lines → four 3-line shingles; used as a reusable "prior art" corpus. +const BLOCK = [ + "function add(a, b) {", + "const total = a + b;", + "logger.debug(total);", + "return total;", + "}", + "export default add;", +]; + +// First 4 BLOCK lines (2 contained shingles) + 2 novel lines (2 non-matching shingles) → 50% containment. +const HALF_COPIED = [ + "function add(a, b) {", + "const total = a + b;", + "logger.debug(total);", + "return total;", + "noveltyOne();", + "noveltyTwo();", +]; + +describe("codeShingles", () => { + it("returns an empty set when every line is blank/whitespace-only", () => { + expect(codeShingles(["", " ", "\t"]).size).toBe(0); + }); + + it("collapses a sub-shingle-width snippet into a single whole-block shingle", () => { + const shingles = codeShingles(["const x = 1;", "const y = 2;"]); + expect(shingles.size).toBe(1); + expect([...shingles][0]).toBe("const x = 1;\nconst y = 2;"); + }); + + it("produces sliding 3-line shingles for a longer block", () => { + // 6 non-trivial lines → 6 - 3 + 1 = 4 shingles. + expect(codeShingles(BLOCK).size).toBe(4); + }); + + it("ignores blank lines and normalizes whitespace/case before shingling", () => { + const a = codeShingles(["Const X = 1;", "", " const y = 2; ", "const z = 3;"]); + const b = codeShingles(["const x = 1;", "const y = 2;", "const z = 3;"]); + expect([...a]).toEqual([...b]); + }); +}); + +describe("containmentScore", () => { + it("is 0 when the candidate has no comparable content", () => { + expect(containmentScore(["", " "], BLOCK)).toBe(0); + }); + + it("is 0 when the prior art has no comparable content", () => { + expect(containmentScore(BLOCK, [])).toBe(0); + }); + + it("is 100 when every candidate shingle appears in the prior art", () => { + expect(containmentScore(BLOCK, BLOCK)).toBe(100); + }); + + it("is 0 when nothing overlaps", () => { + expect(containmentScore(["alpha();", "beta();", "gamma();"], BLOCK)).toBe(0); + }); + + it("reports the partial percentage of candidate shingles found in the prior art", () => { + // Candidate = the first 4 BLOCK lines (2 contained shingles) plus 2 novel lines (2 non-matching shingles) + // → 4 shingles total, 2 contained → 50%. + expect(containmentScore(HALF_COPIED, BLOCK)).toBe(50); + }); +}); + +describe("copycatDirection", () => { + it("is candidate_copied when the candidate is submitted AFTER the prior art", () => { + expect(copycatDirection("2026-06-02T00:00:00Z", "2026-06-01T00:00:00Z")).toBe("candidate_copied"); + }); + + it("is candidate_is_prior when the candidate is submitted BEFORE the prior art", () => { + expect(copycatDirection("2026-06-01T00:00:00Z", "2026-06-02T00:00:00Z")).toBe("candidate_is_prior"); + }); + + it("is ambiguous on an exact timestamp tie", () => { + expect(copycatDirection("2026-06-01T00:00:00Z", "2026-06-01T00:00:00Z")).toBe("ambiguous"); + }); + + it("is ambiguous when either timestamp is missing", () => { + expect(copycatDirection(null, "2026-06-01T00:00:00Z")).toBe("ambiguous"); + expect(copycatDirection("2026-06-01T00:00:00Z", undefined)).toBe("ambiguous"); + expect(copycatDirection("", "2026-06-01T00:00:00Z")).toBe("ambiguous"); + }); + + it("is ambiguous when either timestamp is unparseable", () => { + expect(copycatDirection("not-a-date", "2026-06-01T00:00:00Z")).toBe("ambiguous"); + expect(copycatDirection("2026-06-01T00:00:00Z", "nonsense")).toBe("ambiguous"); + }); +}); + +describe("assessCopycat", () => { + const laterCopier = { + candidateLines: BLOCK, + priorArtLines: BLOCK, + candidateSubmittedAt: "2026-06-02T00:00:00Z", + priorSubmittedAt: "2026-06-01T00:00:00Z", + }; + + it("acts (emits a finding) on a high-containment later submission with a non-off mode", () => { + const result = assessCopycat({ ...laterCopier, mode: "block" }); + expect(result.score).toBe(100); + expect(result.direction).toBe("candidate_copied"); + expect(result.minScore).toBe(DEFAULT_COPYCAT_MIN_SCORE); + expect(result.wouldAct).toBe(true); + expect(result.findings).toHaveLength(1); + expect(result.findings[0]).toMatchObject({ code: "copycat_overlap", severity: "critical" }); + expect(result.findings[0]?.publicText).toContain("100%"); + }); + + it("maps each non-off tier to its severity", () => { + expect(assessCopycat({ ...laterCopier, mode: "warn" }).findings[0]?.severity).toBe("info"); + expect(assessCopycat({ ...laterCopier, mode: "label" }).findings[0]?.severity).toBe("warning"); + expect(assessCopycat({ ...laterCopier, mode: "block" }).findings[0]?.severity).toBe("critical"); + }); + + it("never acts when the mode is off (or absent), but still reports the score", () => { + const off = assessCopycat({ ...laterCopier, mode: "off" }); + expect(off.score).toBe(100); + expect(off.wouldAct).toBe(false); + expect(off.findings).toEqual([]); + const absent = assessCopycat(laterCopier); + expect(absent.wouldAct).toBe(false); + expect(absent.findings).toEqual([]); + }); + + it("never acts when the score is below the threshold", () => { + const result = assessCopycat({ + candidateLines: ["alpha();", "beta();", "gamma();"], + priorArtLines: BLOCK, + candidateSubmittedAt: "2026-06-02T00:00:00Z", + priorSubmittedAt: "2026-06-01T00:00:00Z", + mode: "block", + }); + expect(result.score).toBe(0); + expect(result.wouldAct).toBe(false); + expect(result.findings).toEqual([]); + }); + + it("never acts when the candidate is the EARLIER (victim) submission, even at 100% containment", () => { + const result = assessCopycat({ + candidateLines: BLOCK, + priorArtLines: BLOCK, + candidateSubmittedAt: "2026-06-01T00:00:00Z", + priorSubmittedAt: "2026-06-02T00:00:00Z", + mode: "block", + }); + expect(result.score).toBe(100); + expect(result.direction).toBe("candidate_is_prior"); + expect(result.wouldAct).toBe(false); + expect(result.findings).toEqual([]); + }); + + it("honors a custom in-range minScore and reports it back", () => { + // 50% containment with a 40 threshold → acts; the same score with the default 85 would not. + const acting = assessCopycat({ + candidateLines: HALF_COPIED, + priorArtLines: BLOCK, + candidateSubmittedAt: "2026-06-02T00:00:00Z", + priorSubmittedAt: "2026-06-01T00:00:00Z", + mode: "label", + minScore: 40, + }); + expect(acting.score).toBe(50); + expect(acting.minScore).toBe(40); + expect(acting.wouldAct).toBe(true); + }); + + it("clamps and rounds an out-of-range or non-numeric minScore", () => { + expect(assessCopycat({ ...laterCopier, mode: "off", minScore: -5 }).minScore).toBe(0); + expect(assessCopycat({ ...laterCopier, mode: "off", minScore: 150 }).minScore).toBe(100); + expect(assessCopycat({ ...laterCopier, mode: "off", minScore: 82.6 }).minScore).toBe(83); + expect(assessCopycat({ ...laterCopier, mode: "off", minScore: Number.NaN }).minScore).toBe(DEFAULT_COPYCAT_MIN_SCORE); + expect(assessCopycat({ ...laterCopier, mode: "off", minScore: null }).minScore).toBe(DEFAULT_COPYCAT_MIN_SCORE); + }); +}); From 0f1ffa267de15155bd5f61e2f5812d489887caeb Mon Sep 17 00:00:00 2001 From: carlh171112 Date: Sat, 11 Jul 2026 14:54:48 -0700 Subject: [PATCH 2/2] refactor(copycat): enhance shingle generation and containment scoring This commit refactors the `copycat` module to improve the shingle generation process by introducing a new function, `codeShingleList`, which preserves duplicates in the shingles. The existing `codeShingles` function is updated to return a distinct set of shingles. Additionally, the containment scoring logic is adjusted to work with the new shingle list format, ensuring accurate scoring based on the total shingle count. These changes enhance the module's ability to detect code plagiarism more effectively. --- src/signals/copycat.ts | 32 +++++++++++++++++++++----------- test/unit/copycat.test.ts | 16 ++++++++++++++++ 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/signals/copycat.ts b/src/signals/copycat.ts index f63495462..40aa88717 100644 --- a/src/signals/copycat.ts +++ b/src/signals/copycat.ts @@ -43,32 +43,42 @@ function normalizedLines(lines: readonly string[]): string[] { return lines.map(normalizeLine).filter((line) => line.length > 0); } -/** Fold normalized lines into a set of SHINGLE_SIZE-line shingles. Fewer than SHINGLE_SIZE non-trivial lines - * collapse to a single whole-block shingle so tiny snippets still compare (never silently score 0). */ -export function codeShingles(lines: readonly string[]): Set { +/** Fold normalized lines into the ORDERED MULTISET of SHINGLE_SIZE-line shingles — duplicates preserved, so a + * passage copied twice counts twice toward containment (see {@link containmentScore}, which must divide by the + * candidate's TOTAL shingle count, not its distinct count). Fewer than SHINGLE_SIZE non-trivial lines collapse + * to a single whole-block shingle so tiny snippets still compare (never silently score 0). */ +export function codeShingleList(lines: readonly string[]): string[] { const normalized = normalizedLines(lines); - if (normalized.length === 0) return new Set(); - if (normalized.length < SHINGLE_SIZE) return new Set([normalized.join("\n")]); - const shingles = new Set(); + if (normalized.length === 0) return []; + if (normalized.length < SHINGLE_SIZE) return [normalized.join("\n")]; + const shingles: string[] = []; for (let i = 0; i + SHINGLE_SIZE <= normalized.length; i += 1) { - shingles.add(normalized.slice(i, i + SHINGLE_SIZE).join("\n")); + shingles.push(normalized.slice(i, i + SHINGLE_SIZE).join("\n")); } return shingles; } +/** The DISTINCT SHINGLE_SIZE-line shingles of `lines` — a de-duplicated view of {@link codeShingleList}, used as + * the prior-art lookup set (membership only, so duplicates there are irrelevant). */ +export function codeShingles(lines: readonly string[]): Set { + return new Set(codeShingleList(lines)); +} + /** Asymmetric containment (0-100): the percentage of the CANDIDATE's added-code shingles that also appear in the * PRIOR ART. Unlike symmetric Jaccard, this answers "how much of THIS PR is copied FROM prior art" without being - * diluted by a large prior-art corpus. 0 when either side has no comparable content. */ + * diluted by a large prior-art corpus. The candidate is a MULTISET (its total shingle count is the denominator, + * so a repeated copied passage is not undercounted), while the prior art is a lookup Set (membership only). 0 + * when either side has no comparable content. */ export function containmentScore(candidateLines: readonly string[], priorArtLines: readonly string[]): number { - const candidate = codeShingles(candidateLines); - if (candidate.size === 0) return 0; + const candidate = codeShingleList(candidateLines); + if (candidate.length === 0) return 0; const prior = codeShingles(priorArtLines); if (prior.size === 0) return 0; let contained = 0; for (const shingle of candidate) { if (prior.has(shingle)) contained += 1; } - return Math.round((contained / candidate.size) * 100); + return Math.round((contained / candidate.length) * 100); } /** Parse an ISO-8601 submission time to epoch ms; null for a missing/empty/unparseable value. */ diff --git a/test/unit/copycat.test.ts b/test/unit/copycat.test.ts index 726716743..66a1bc657 100644 --- a/test/unit/copycat.test.ts +++ b/test/unit/copycat.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { assessCopycat, + codeShingleList, codeShingles, containmentScore, copycatDirection, @@ -43,6 +44,13 @@ describe("codeShingles", () => { expect(codeShingles(BLOCK).size).toBe(4); }); + it("codeShingleList keeps duplicate shingles that codeShingles (the distinct set) collapses", () => { + // Two identical 3-line blocks back-to-back → shingles [ABC, BCA, CAB, ABC]: 4 in the list, 3 distinct. + const repeated = ["alpha();", "beta();", "gamma();", "alpha();", "beta();", "gamma();"]; + expect(codeShingleList(repeated)).toHaveLength(4); + expect(codeShingles(repeated).size).toBe(3); + }); + it("ignores blank lines and normalizes whitespace/case before shingling", () => { const a = codeShingles(["Const X = 1;", "", " const y = 2; ", "const z = 3;"]); const b = codeShingles(["const x = 1;", "const y = 2;", "const z = 3;"]); @@ -72,6 +80,14 @@ describe("containmentScore", () => { // → 4 shingles total, 2 contained → 50%. expect(containmentScore(HALF_COPIED, BLOCK)).toBe(50); }); + + it("counts a repeated copied shingle as a MULTISET, not a distinct set (regression)", () => { + // Candidate shingles = [ABC, BCA, CAB, ABC]; prior art = {ABC}. Multiset: 2 of 4 contained → 50%. + // The earlier distinct-Set denominator undercounted this as 1 of 3 → 33%. + const repeatedCopier = ["alpha();", "beta();", "gamma();", "alpha();", "beta();", "gamma();"]; + const priorArt = ["alpha();", "beta();", "gamma();"]; + expect(containmentScore(repeatedCopier, priorArt)).toBe(50); + }); }); describe("copycatDirection", () => {