Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 26 additions & 3 deletions app/core/safe-download.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import { lookup } from "node:dns/promises";
import { open, mkdir, link, rm } from "node:fs/promises";
import http from "node:http";
import https from "node:https";
import { isIP } from "node:net";
import { isIP, type LookupFunction } from "node:net";
import { basename, dirname, join } from "node:path";

export type SafeDownloadCode = "UNSUPPORTED_MEDIA_URL" | "MEDIA_TIMEOUT" | "MEDIA_TOO_LARGE" | "MEDIA_TYPE_REJECTED" | "MEDIA_HTTP_ERROR" | "MEDIA_WRITE_FAILED";
Expand Down Expand Up @@ -51,6 +51,16 @@ function inV6(value: bigint, base: string, bits: number): boolean {
return (value >> shift) === (b >> shift);
}

const PROXY_FAKE_IP_MEDIA_HOSTS = new Set([
"pbs.twimg.com",
"video.twimg.com",
]);

function isProxyFakeIp(address: string): boolean {
if (isIP(address) !== 4) return false;
return inV4(ipv4Number(address)!, "198.18.0.0", 15);
}

export function isPublicIp(address: string): boolean {
if (address.toLowerCase().startsWith("::ffff:")) {
const mapped = address.slice(7);
Expand Down Expand Up @@ -80,17 +90,30 @@ async function resolvePublic(hostname: string, resolver: NonNullable<Options["re
return { address: hostname, family: isIP(hostname) };
}
const addresses = await resolver(hostname);
if (!addresses.length || addresses.some((item) => !isPublicIp(item.address))) throw new SafeDownloadError("UNSUPPORTED_MEDIA_URL", `media hostname is not public: ${hostname}`);
const nonPublicAddresses = addresses.filter((item) => !isPublicIp(item.address));
const proxyFakeIpsAllowed = PROXY_FAKE_IP_MEDIA_HOSTS.has(hostname.toLowerCase())
&& nonPublicAddresses.length > 0
&& nonPublicAddresses.every((item) => isProxyFakeIp(item.address));
if (!addresses.length || (nonPublicAddresses.length > 0 && !proxyFakeIpsAllowed)) {
throw new SafeDownloadError("UNSUPPORTED_MEDIA_URL", `media hostname is not public: ${hostname}`);
}
return addresses[0];
}

export function createPinnedLookup(pinned: Address): LookupFunction {
return (_hostname, options, callback) => {
if (options.all) callback(null, [pinned]);
else callback(null, pinned.address, pinned.family);
};
}

function productionOpen(url: URL, pinned: Address, signal: AbortSignal): Promise<DownloadResponse> {
return new Promise((resolve, reject) => {
const client = url.protocol === "https:" ? https : http;
const request = client.request(url, {
method: "GET", headers: { "User-Agent": "Mozilla/5.0", Host: url.host }, agent: false,
...(url.protocol === "https:" ? { servername: url.hostname } : {}),
lookup: (_hostname, _options, callback) => callback(null, pinned.address, pinned.family),
lookup: createPinnedLookup(pinned),
}, (response) => resolve({ status: response.statusCode || 0, headers: response.headers as any, body: response }));
const abort = () => request.destroy(new SafeDownloadError("MEDIA_TIMEOUT", "media download timed out", true));
signal.addEventListener("abort", abort, { once: true });
Expand Down
11 changes: 7 additions & 4 deletions app/core/save.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,13 +103,16 @@ export async function savePayload(data: Record<string, any>, cfg: X2MDConfig | R
return withCaptureLock(dir, key, async () => {
const existing = await timeSaveStage(metrics, "dedupe", async () => (await readSaveIndex(dir)).entries[key]);
const latest = existing?.revisions.find((item) => item.revision === existing.latest_revision);
if (existing && policy === "skip" && latest?.files.length && latest.files.every(existsSync)) {
const saved = latest?.files || [];
const currentSaveDirectories = new Set(savePaths.map((savePath) => resolve(savePath)));
const intactFiles = existing && policy === "skip"
? (latest?.files || []).filter((file) => existsSync(file) && currentSaveDirectories.has(resolve(dirname(file))))
: [];
const intactDirectories = new Set(intactFiles.map((file) => resolve(dirname(file))));
if (existing && policy === "skip" && savePaths.every((savePath) => intactDirectories.has(resolve(savePath)))) {
const saved = intactFiles;
const historyId = readSaveHistory(dir).find((item) => saved.includes(item.path))?.id;
return finish({ success: true, outcome: "skipped", capture_key: key, saved, files: saved.map((path, index) => ({ path, ...(historyId && index === 0 ? { history_id: historyId } : {}) })), errors: [], warnings: [], media: { completed: 0, failed: 0, pending: 0 } });
}
const intactFiles = existing && policy === "skip" ? (latest?.files || []).filter(existsSync) : [];
const intactDirectories = new Set(intactFiles.map((file) => resolve(dirname(file))));
const transactionSavePaths = policy === "skip"
? savePaths.filter((savePath) => !intactDirectories.has(resolve(savePath)))
: savePaths;
Expand Down
50 changes: 21 additions & 29 deletions app/tests/api.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import test from "node:test";
import assert from "node:assert/strict";
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

Expand Down Expand Up @@ -344,7 +344,7 @@ test("POST /save 开启后拒绝私网图片并回退远程链接", async () =>
});


test("POST /save 开启图片下载时 Twitter 仍保持远程原图链接", async () => {
test("POST /save 开启图片下载时 Twitter 使用本地附件引用", async () => {
const appDir = tempApp();
const mdDir = join(appDir, "md");
await handleApiRequest(new Request("http://127.0.0.1:9527/config", {
Expand All @@ -357,33 +357,25 @@ test("POST /save 开启图片下载时 Twitter 仍保持远程原图链接", asy
}),
}), { appDir, testBypassAuth: true });

let fetchCalled = false;
const originalFetch = globalThis.fetch;
globalThis.fetch = (async () => {
fetchCalled = true;
return new Response(new Uint8Array([1, 2, 3]), { headers: { "content-type": "image/jpeg" } });
}) as unknown as typeof fetch;
try {
const res = await handleApiRequest(new Request("http://127.0.0.1:9527/save", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
type: "tweet",
platform: "Twitter/X",
text: "remote twitter image",
url: "https://x.com/a/status/192",
images: ["https://pbs.twimg.com/media/abc.jpg?format=jpg&name=small"],
}),
}), { appDir, testBypassAuth: true });
const body = await json(res);
const md = readFileSync(body.saved[0], "utf8");
assert.equal(res.status, 200);
assert.equal(fetchCalled, false);
assert.equal(existsSync(join(mdDir, "attachments", "192", "image_1.jpg")), false);
assert.match(md, /!\[\]\(https:\/\/pbs\.twimg\.com\/media\/abc\.jpg\?format=jpg&name=orig\)/);
} finally {
globalThis.fetch = originalFetch;
}
const attachment = join(mdDir, "attachments", "192", "image_1.jpg");
mkdirSync(join(attachment, ".."), { recursive: true });
writeFileSync(attachment, "existing image");
const res = await handleApiRequest(new Request("http://127.0.0.1:9527/save", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
type: "tweet",
platform: "Twitter/X",
text: "local twitter image",
url: "https://x.com/a/status/192",
images: ["https://pbs.twimg.com/media/abc.jpg?format=jpg&name=small"],
}),
}), { appDir, testBypassAuth: true });
const body = await json(res);
const md = readFileSync(body.saved[0], "utf8");
assert.equal(res.status, 200);
assert.equal(existsSync(attachment), true);
assert.match(md, /!\[\]\(attachments\/192\/image_1\.jpg\)/);
});

test("POST /save 图片下载失败时回退远程 URL 且不污染笔记正文", async () => {
Expand Down
40 changes: 39 additions & 1 deletion app/tests/safe-download.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { existsSync, mkdtempSync, readFileSync, readdirSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";

import { isPublicIp, safeDownload, SafeDownloadError } from "../core/safe-download.ts";
import { createPinnedLookup, isPublicIp, safeDownload, SafeDownloadError } from "../core/safe-download.ts";

const dir = () => mkdtempSync(join(tmpdir(), "x2md-download-"));
const resolver = async () => [{ address: "93.184.216.34", family: 4 }];
Expand All @@ -17,6 +17,44 @@ test("private, loopback, link-local and reserved addresses are rejected", () =>
assert.equal(isPublicIp("2606:4700:4700::1111"), true);
});

test("trusted platform media hosts may use proxy fake IPs without weakening arbitrary hosts", async () => {
const root = dir();
const opened: string[] = [];
const fakeIpResolver = async () => [{ address: "198.18.0.52", family: 4 }];
const openResponse = async (url: URL, pinned: { address: string }) => {
opened.push(`${url.hostname}=${pinned.address}`);
return response(chunks("image"));
};

await safeDownload("https://pbs.twimg.com/media/example.jpg", join(root, "trusted.jpg"), {
allowedContentTypes: ["image/"], resolver: fakeIpResolver, openResponse,
});
await assert.rejects(() => safeDownload("https://attacker.example/image.jpg", join(root, "blocked.jpg"), {
allowedContentTypes: ["image/"], resolver: fakeIpResolver, openResponse,
}), (error: any) => error?.code === "UNSUPPORTED_MEDIA_URL");
await assert.rejects(() => safeDownload("https://pbs.twimg.com/media/private.jpg", join(root, "private.jpg"), {
allowedContentTypes: ["image/"], resolver: async () => [{ address: "127.0.0.1", family: 4 }], openResponse,
}), (error: any) => error?.code === "UNSUPPORTED_MEDIA_URL");

assert.deepEqual(opened, ["pbs.twimg.com=198.18.0.52"]);
assert.equal(existsSync(join(root, "trusted.jpg")), true);
assert.equal(existsSync(join(root, "blocked.jpg")), false);
assert.equal(existsSync(join(root, "private.jpg")), false);
});

test("pinned lookup supports single-address and all-address callback contracts", async () => {
const lookup = createPinnedLookup({ address: "198.18.0.52", family: 4 });
const single = await new Promise<{ address: string; family: number }>((resolve, reject) => {
lookup("pbs.twimg.com", {}, (error: Error | null, address: string, family: number) => error ? reject(error) : resolve({ address, family }));
});
const all = await new Promise<Array<{ address: string; family: number }>>((resolve, reject) => {
lookup("pbs.twimg.com", { all: true }, (error: Error | null, addresses: Array<{ address: string; family: number }>) => error ? reject(error) : resolve(addresses));
});

assert.deepEqual(single, { address: "198.18.0.52", family: 4 });
assert.deepEqual(all, [{ address: "198.18.0.52", family: 4 }]);
});

test("pins validated DNS and validates every redirect", async () => {
const target = join(dir(), "image.jpg");
const resolved: string[] = [];
Expand Down
19 changes: 19 additions & 0 deletions app/tests/save-index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,25 @@ test("skip restores only a missing target without duplicating intact targets", a
assert.deepEqual(new Set(entry.revisions.at(-1)?.files), new Set(first.saved));
});

test("skip saves an existing capture into a newly configured directory", async () => {
const appDir = mkdtempSync(join(tmpdir(), "x2md-index-moved-"));
const oldDir = join(appDir, "old-location");
const newDir = join(appDir, "new-location");
const item = capture("moved");
const first = await savePayload(legacy("moved"), normalizeConfig({ save_paths: [oldDir] }), appDir, item);

const second = await savePayload(legacy("moved"), normalizeConfig({ save_paths: [newDir] }), appDir, item);

assert.equal(first.outcome, "saved");
assert.equal(second.outcome, "saved");
assert.equal(first.saved.length, 1);
assert.equal(second.saved.length, 1);
assert.equal(first.saved[0].startsWith(oldDir), true);
assert.equal(second.saved[0].startsWith(newDir), true);
assert.equal(existsSync(first.saved[0]), true);
assert.equal(existsSync(second.saved[0]), true);
});

test("20 different keys with the same title produce unique files", async () => {
const { appDir, cfg } = setup();
const results = await Promise.all(Array.from({ length: 20 }, (_, index) => savePayload(legacy(String(index + 10)), cfg, appDir, capture(String(index + 10)))));
Expand Down