diff --git a/app/core/safe-download.ts b/app/core/safe-download.ts index 817d98d..afd3718 100644 --- a/app/core/safe-download.ts +++ b/app/core/safe-download.ts @@ -3,7 +3,7 @@ import { lookup } from "node:dns/promises"; import { open, mkdir, link, rm } from "node:fs/promises"; import http from "node:http"; import https from "node:https"; -import { isIP } from "node:net"; +import { isIP, type LookupFunction } from "node:net"; import { basename, dirname, join } from "node:path"; export type SafeDownloadCode = "UNSUPPORTED_MEDIA_URL" | "MEDIA_TIMEOUT" | "MEDIA_TOO_LARGE" | "MEDIA_TYPE_REJECTED" | "MEDIA_HTTP_ERROR" | "MEDIA_WRITE_FAILED"; @@ -51,6 +51,16 @@ function inV6(value: bigint, base: string, bits: number): boolean { return (value >> shift) === (b >> shift); } +const PROXY_FAKE_IP_MEDIA_HOSTS = new Set([ + "pbs.twimg.com", + "video.twimg.com", +]); + +function isProxyFakeIp(address: string): boolean { + if (isIP(address) !== 4) return false; + return inV4(ipv4Number(address)!, "198.18.0.0", 15); +} + export function isPublicIp(address: string): boolean { if (address.toLowerCase().startsWith("::ffff:")) { const mapped = address.slice(7); @@ -80,17 +90,30 @@ async function resolvePublic(hostname: string, resolver: NonNullable !isPublicIp(item.address))) throw new SafeDownloadError("UNSUPPORTED_MEDIA_URL", `media hostname is not public: ${hostname}`); + const nonPublicAddresses = addresses.filter((item) => !isPublicIp(item.address)); + const proxyFakeIpsAllowed = PROXY_FAKE_IP_MEDIA_HOSTS.has(hostname.toLowerCase()) + && nonPublicAddresses.length > 0 + && nonPublicAddresses.every((item) => isProxyFakeIp(item.address)); + if (!addresses.length || (nonPublicAddresses.length > 0 && !proxyFakeIpsAllowed)) { + throw new SafeDownloadError("UNSUPPORTED_MEDIA_URL", `media hostname is not public: ${hostname}`); + } return addresses[0]; } +export function createPinnedLookup(pinned: Address): LookupFunction { + return (_hostname, options, callback) => { + if (options.all) callback(null, [pinned]); + else callback(null, pinned.address, pinned.family); + }; +} + function productionOpen(url: URL, pinned: Address, signal: AbortSignal): Promise { return new Promise((resolve, reject) => { const client = url.protocol === "https:" ? https : http; const request = client.request(url, { method: "GET", headers: { "User-Agent": "Mozilla/5.0", Host: url.host }, agent: false, ...(url.protocol === "https:" ? { servername: url.hostname } : {}), - lookup: (_hostname, _options, callback) => callback(null, pinned.address, pinned.family), + lookup: createPinnedLookup(pinned), }, (response) => resolve({ status: response.statusCode || 0, headers: response.headers as any, body: response })); const abort = () => request.destroy(new SafeDownloadError("MEDIA_TIMEOUT", "media download timed out", true)); signal.addEventListener("abort", abort, { once: true }); diff --git a/app/core/save.ts b/app/core/save.ts index 5a8bc6a..1dfd986 100644 --- a/app/core/save.ts +++ b/app/core/save.ts @@ -103,13 +103,16 @@ export async function savePayload(data: Record, cfg: X2MDConfig | R return withCaptureLock(dir, key, async () => { const existing = await timeSaveStage(metrics, "dedupe", async () => (await readSaveIndex(dir)).entries[key]); const latest = existing?.revisions.find((item) => item.revision === existing.latest_revision); - if (existing && policy === "skip" && latest?.files.length && latest.files.every(existsSync)) { - const saved = latest?.files || []; + const currentSaveDirectories = new Set(savePaths.map((savePath) => resolve(savePath))); + const intactFiles = existing && policy === "skip" + ? (latest?.files || []).filter((file) => existsSync(file) && currentSaveDirectories.has(resolve(dirname(file)))) + : []; + const intactDirectories = new Set(intactFiles.map((file) => resolve(dirname(file)))); + if (existing && policy === "skip" && savePaths.every((savePath) => intactDirectories.has(resolve(savePath)))) { + const saved = intactFiles; const historyId = readSaveHistory(dir).find((item) => saved.includes(item.path))?.id; return finish({ success: true, outcome: "skipped", capture_key: key, saved, files: saved.map((path, index) => ({ path, ...(historyId && index === 0 ? { history_id: historyId } : {}) })), errors: [], warnings: [], media: { completed: 0, failed: 0, pending: 0 } }); } - const intactFiles = existing && policy === "skip" ? (latest?.files || []).filter(existsSync) : []; - const intactDirectories = new Set(intactFiles.map((file) => resolve(dirname(file)))); const transactionSavePaths = policy === "skip" ? savePaths.filter((savePath) => !intactDirectories.has(resolve(savePath))) : savePaths; diff --git a/app/tests/api.test.ts b/app/tests/api.test.ts index 2c72277..b1eeb7b 100644 --- a/app/tests/api.test.ts +++ b/app/tests/api.test.ts @@ -1,6 +1,6 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -344,7 +344,7 @@ test("POST /save 开启后拒绝私网图片并回退远程链接", async () => }); -test("POST /save 开启图片下载时 Twitter 仍保持远程原图链接", async () => { +test("POST /save 开启图片下载时 Twitter 使用本地附件引用", async () => { const appDir = tempApp(); const mdDir = join(appDir, "md"); await handleApiRequest(new Request("http://127.0.0.1:9527/config", { @@ -357,33 +357,25 @@ test("POST /save 开启图片下载时 Twitter 仍保持远程原图链接", asy }), }), { appDir, testBypassAuth: true }); - let fetchCalled = false; - const originalFetch = globalThis.fetch; - globalThis.fetch = (async () => { - fetchCalled = true; - return new Response(new Uint8Array([1, 2, 3]), { headers: { "content-type": "image/jpeg" } }); - }) as unknown as typeof fetch; - try { - const res = await handleApiRequest(new Request("http://127.0.0.1:9527/save", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - type: "tweet", - platform: "Twitter/X", - text: "remote twitter image", - url: "https://x.com/a/status/192", - images: ["https://pbs.twimg.com/media/abc.jpg?format=jpg&name=small"], - }), - }), { appDir, testBypassAuth: true }); - const body = await json(res); - const md = readFileSync(body.saved[0], "utf8"); - assert.equal(res.status, 200); - assert.equal(fetchCalled, false); - assert.equal(existsSync(join(mdDir, "attachments", "192", "image_1.jpg")), false); - assert.match(md, /!\[\]\(https:\/\/pbs\.twimg\.com\/media\/abc\.jpg\?format=jpg&name=orig\)/); - } finally { - globalThis.fetch = originalFetch; - } + const attachment = join(mdDir, "attachments", "192", "image_1.jpg"); + mkdirSync(join(attachment, ".."), { recursive: true }); + writeFileSync(attachment, "existing image"); + const res = await handleApiRequest(new Request("http://127.0.0.1:9527/save", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + type: "tweet", + platform: "Twitter/X", + text: "local twitter image", + url: "https://x.com/a/status/192", + images: ["https://pbs.twimg.com/media/abc.jpg?format=jpg&name=small"], + }), + }), { appDir, testBypassAuth: true }); + const body = await json(res); + const md = readFileSync(body.saved[0], "utf8"); + assert.equal(res.status, 200); + assert.equal(existsSync(attachment), true); + assert.match(md, /!\[\]\(attachments\/192\/image_1\.jpg\)/); }); test("POST /save 图片下载失败时回退远程 URL 且不污染笔记正文", async () => { diff --git a/app/tests/safe-download.test.ts b/app/tests/safe-download.test.ts index 4ff2081..c1480a1 100644 --- a/app/tests/safe-download.test.ts +++ b/app/tests/safe-download.test.ts @@ -4,7 +4,7 @@ import { existsSync, mkdtempSync, readFileSync, readdirSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { isPublicIp, safeDownload, SafeDownloadError } from "../core/safe-download.ts"; +import { createPinnedLookup, isPublicIp, safeDownload, SafeDownloadError } from "../core/safe-download.ts"; const dir = () => mkdtempSync(join(tmpdir(), "x2md-download-")); const resolver = async () => [{ address: "93.184.216.34", family: 4 }]; @@ -17,6 +17,44 @@ test("private, loopback, link-local and reserved addresses are rejected", () => assert.equal(isPublicIp("2606:4700:4700::1111"), true); }); +test("trusted platform media hosts may use proxy fake IPs without weakening arbitrary hosts", async () => { + const root = dir(); + const opened: string[] = []; + const fakeIpResolver = async () => [{ address: "198.18.0.52", family: 4 }]; + const openResponse = async (url: URL, pinned: { address: string }) => { + opened.push(`${url.hostname}=${pinned.address}`); + return response(chunks("image")); + }; + + await safeDownload("https://pbs.twimg.com/media/example.jpg", join(root, "trusted.jpg"), { + allowedContentTypes: ["image/"], resolver: fakeIpResolver, openResponse, + }); + await assert.rejects(() => safeDownload("https://attacker.example/image.jpg", join(root, "blocked.jpg"), { + allowedContentTypes: ["image/"], resolver: fakeIpResolver, openResponse, + }), (error: any) => error?.code === "UNSUPPORTED_MEDIA_URL"); + await assert.rejects(() => safeDownload("https://pbs.twimg.com/media/private.jpg", join(root, "private.jpg"), { + allowedContentTypes: ["image/"], resolver: async () => [{ address: "127.0.0.1", family: 4 }], openResponse, + }), (error: any) => error?.code === "UNSUPPORTED_MEDIA_URL"); + + assert.deepEqual(opened, ["pbs.twimg.com=198.18.0.52"]); + assert.equal(existsSync(join(root, "trusted.jpg")), true); + assert.equal(existsSync(join(root, "blocked.jpg")), false); + assert.equal(existsSync(join(root, "private.jpg")), false); +}); + +test("pinned lookup supports single-address and all-address callback contracts", async () => { + const lookup = createPinnedLookup({ address: "198.18.0.52", family: 4 }); + const single = await new Promise<{ address: string; family: number }>((resolve, reject) => { + lookup("pbs.twimg.com", {}, (error: Error | null, address: string, family: number) => error ? reject(error) : resolve({ address, family })); + }); + const all = await new Promise>((resolve, reject) => { + lookup("pbs.twimg.com", { all: true }, (error: Error | null, addresses: Array<{ address: string; family: number }>) => error ? reject(error) : resolve(addresses)); + }); + + assert.deepEqual(single, { address: "198.18.0.52", family: 4 }); + assert.deepEqual(all, [{ address: "198.18.0.52", family: 4 }]); +}); + test("pins validated DNS and validates every redirect", async () => { const target = join(dir(), "image.jpg"); const resolved: string[] = []; diff --git a/app/tests/save-index.test.ts b/app/tests/save-index.test.ts index 615e52a..922b0fc 100644 --- a/app/tests/save-index.test.ts +++ b/app/tests/save-index.test.ts @@ -67,6 +67,25 @@ test("skip restores only a missing target without duplicating intact targets", a assert.deepEqual(new Set(entry.revisions.at(-1)?.files), new Set(first.saved)); }); +test("skip saves an existing capture into a newly configured directory", async () => { + const appDir = mkdtempSync(join(tmpdir(), "x2md-index-moved-")); + const oldDir = join(appDir, "old-location"); + const newDir = join(appDir, "new-location"); + const item = capture("moved"); + const first = await savePayload(legacy("moved"), normalizeConfig({ save_paths: [oldDir] }), appDir, item); + + const second = await savePayload(legacy("moved"), normalizeConfig({ save_paths: [newDir] }), appDir, item); + + assert.equal(first.outcome, "saved"); + assert.equal(second.outcome, "saved"); + assert.equal(first.saved.length, 1); + assert.equal(second.saved.length, 1); + assert.equal(first.saved[0].startsWith(oldDir), true); + assert.equal(second.saved[0].startsWith(newDir), true); + assert.equal(existsSync(first.saved[0]), true); + assert.equal(existsSync(second.saved[0]), true); +}); + test("20 different keys with the same title produce unique files", async () => { const { appDir, cfg } = setup(); const results = await Promise.all(Array.from({ length: 20 }, (_, index) => savePayload(legacy(String(index + 10)), cfg, appDir, capture(String(index + 10)))));