From 85e0a04221ecf451da27d96e47d04163920ffa63 Mon Sep 17 00:00:00 2001 From: Peter Kirkham Date: Mon, 27 Jul 2026 10:58:44 +0100 Subject: [PATCH] feat(canvas): add authoritative cloud compilation Generated-By: PostHog Code Task-Id: 3d87598e-5366-4937-8f6e-30808221241e --- .depot/workflows/ci-backend.yml | 4 + .dockerignore | 1 + .github/workflows/ci-backend.yml | 5 + Dockerfile | 5 +- common/canvas-builder/build.mjs | 150 ++++++ common/canvas-builder/package-lock.json | 453 ++++++++++++++++++ common/canvas-builder/package.json | 8 + .../api/file_system/canvas_build_service.py | 166 +++++-- .../test/test_canvas_cloud_builder.py | 60 +++ 9 files changed, 813 insertions(+), 39 deletions(-) create mode 100644 common/canvas-builder/build.mjs create mode 100644 common/canvas-builder/package-lock.json create mode 100644 common/canvas-builder/package.json create mode 100644 posthog/api/file_system/test/test_canvas_cloud_builder.py diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.yml index df9781c6be02..eaa4245582a6 100644 --- a/.depot/workflows/ci-backend.yml +++ b/.depot/workflows/ci-backend.yml @@ -1645,6 +1645,10 @@ jobs: shell: bash run: | UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev + - name: Install canvas builder dependencies + if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' }} + shell: bash + run: npm ci --ignore-scripts --omit=dev --prefix common/canvas-builder - name: Install the working version of hogql-parser if: ${{ needs.changes.outputs.backend == 'true' && steps.hogql-parser-diff.outputs.changed == 'true' }} shell: bash diff --git a/.dockerignore b/.dockerignore index 8fb9cdba8b6d..2058dfb9dbcc 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,6 +4,7 @@ !.kearc !bin !common/alerting +!common/canvas-builder !common/hogvm !common/esbuilder !common/migration_utils diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index 60279ee3c3e0..d83a575fc4e7 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -2539,6 +2539,11 @@ jobs: run: | UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev + - name: Install canvas builder dependencies + if: ${{ needs.changes.outputs.backend == 'true' && matrix.segment == 'Core' }} + shell: bash + run: npm ci --ignore-scripts --omit=dev --prefix common/canvas-builder + - name: Install the working version of hogql-parser if: ${{ needs.changes.outputs.backend == 'true' && steps.hogql-parser-diff.outputs.changed == 'true' }} shell: bash diff --git a/Dockerfile b/Dockerfile index 2cfe99494aa4..9626021bb468 100644 --- a/Dockerfile +++ b/Dockerfile @@ -112,12 +112,14 @@ SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] COPY turbo.json package.json pnpm-lock.yaml pnpm-workspace.yaml tsconfig.json ./ COPY bin/turbo bin/turbo COPY patches/ patches/ +COPY common/canvas-builder/ common/canvas-builder/ COPY common/esbuilder/ common/esbuilder/ COPY common/plugin_transpiler/ common/plugin_transpiler/ RUN --mount=type=cache,id=pnpm,target=/tmp/pnpm-store-v24 \ corepack enable && \ NODE_OPTIONS="--max-old-space-size=4096" CI=1 pnpm --filter=@posthog/plugin-transpiler... install --frozen-lockfile --store-dir /tmp/pnpm-store-v24 && \ - NODE_OPTIONS="--max-old-space-size=4096" bin/turbo --filter=@posthog/plugin-transpiler build + NODE_OPTIONS="--max-old-space-size=4096" bin/turbo --filter=@posthog/plugin-transpiler build && \ + cd common/canvas-builder && npm ci --ignore-scripts --omit=dev # The transpiler bundle externalizes @babel/standalone (its only external runtime require — a # self-contained 24MB package with no deps). Materialize it as real files inside the transpiler's @@ -397,6 +399,7 @@ ENV TIKTOKEN_CACHE_DIR=/code/.tiktoken_cache COPY --from=node-scripts-build --chown=posthog:posthog /code/common/plugin_transpiler/dist /code/common/plugin_transpiler/dist COPY --from=node-scripts-build --chown=posthog:posthog /code/common/plugin_transpiler/node_modules /code/common/plugin_transpiler/node_modules COPY --from=node-scripts-build --chown=posthog:posthog /code/common/plugin_transpiler/package.json /code/common/plugin_transpiler/package.json +COPY --from=node-scripts-build --chown=posthog:posthog /code/common/canvas-builder /code/common/canvas-builder # Add in custom bin files and Django deps. COPY --chown=posthog:posthog ./bin ./bin/ diff --git a/common/canvas-builder/build.mjs b/common/canvas-builder/build.mjs new file mode 100644 index 000000000000..030f303d5473 --- /dev/null +++ b/common/canvas-builder/build.mjs @@ -0,0 +1,150 @@ +import { build } from "esbuild" +import { createHash } from "node:crypto" +import path from "node:path" + +const admitted = { + react: ["19.0.0", "https://esm.sh/react@19.0.0"], + "react-dom": ["19.0.0", "https://esm.sh/react-dom@19.0.0?external=react"], + "@posthog/quill": ["0.3.0-beta.18", "https://esm.sh/@posthog/quill@0.3.0-beta.18?external=react,react-dom"], + recharts: ["2.15.0", "https://esm.sh/recharts@2.15.0?external=react,react-dom"], + "lucide-react": ["1.21.0", "https://esm.sh/lucide-react@1.21.0?external=react"], + dayjs: ["1.11.13", "https://esm.sh/dayjs@1.11.13"], +} +const runtimeImports = { + "react/jsx-runtime": "https://esm.sh/react@19.0.0/jsx-runtime", + "react-dom/client": "https://esm.sh/react-dom@19.0.0/client?external=react", +} +const moduleScript = /]*type\s*=\s*["']module["'][^>]*\ssrc\s*=\s*["']([^"']+)["'][^>]*>\s*<\/script>/gi +const stylesheet = /]*rel\s*=\s*["']stylesheet["'][^>]*\shref\s*=\s*["']([^"']+)["'][^>]*\/?>/gi +const forbiddenHtml = /(?:src|href)\s*=\s*["']\s*(javascript|data:text\/html|vbscript)/i +const extensions = ["", ".ts", ".tsx", ".js", ".jsx", ".css", ".json", ".svg", ".txt"] + +function diagnostic(code, message, file, line) { + return { severity: "error", code, message: String(message).slice(0, 10000), ...(file ? { path: file } : {}), ...(line ? { line } : {}) } +} + +function sha256(content) { + return createHash("sha256").update(content, "utf8").digest("hex") +} + +function normalize(value) { + return value.replace(/^\.?\//, "") +} + +function packageName(specifier) { + return specifier.startsWith("@") ? specifier.split("/").slice(0, 2).join("/") : specifier.split("/")[0] +} + +function resolveFile(files, importer, specifier) { + const base = path.posix.normalize(path.posix.join(path.posix.dirname(importer), specifier)) + if (base.startsWith("..")) return null + return extensions.map((extension) => base + extension).find((candidate) => candidate in files) ?? null +} + +function loader(file) { + const extension = path.posix.extname(file) + return ({ ".ts": "ts", ".tsx": "tsx", ".jsx": "jsx", ".css": "css", ".json": "json", ".svg": "dataurl", ".txt": "text" })[extension] ?? "js" +} + +function validate(project) { + const diagnostics = [] + if (project.canvasSdkVersion !== "0.1.0") diagnostics.push(diagnostic("unsupported_sdk", "Canvas SDK version is unavailable")) + for (const [name, version] of Object.entries(project.dependencies ?? {})) { + if (!admitted[name]) diagnostics.push(diagnostic("dependency_not_admitted", `dependency "${name}" is not platform-supported`)) + else if (admitted[name][0] !== version) diagnostics.push(diagnostic("dependency_version_mismatch", `dependency "${name}" must use ${admitted[name][0]}`)) + } + const html = project.files?.[project.entryHtml] + if (typeof html !== "string") diagnostics.push(diagnostic("entry_not_found", "Canvas entry HTML does not exist", project.entryHtml)) + else if (forbiddenHtml.test(html)) diagnostics.push(diagnostic("forbidden_url_scheme", "Canvas HTML contains a forbidden URL scheme", project.entryHtml)) + return diagnostics +} + +async function bundleEntry(project, entry, externalImports) { + const files = project.files + const plugin = { + name: "canvas-virtual-fs", + setup(pluginBuild) { + pluginBuild.onResolve({ filter: /.*/ }, (args) => { + if (args.kind === "entry-point") return { path: normalize(args.path), namespace: "canvas" } + if (args.path.startsWith(".") || args.path.startsWith("/")) { + const specifier = args.path.startsWith("/") ? `./${normalize(args.path)}` : args.path + const resolved = resolveFile(files, args.importer, specifier) + return resolved ? { path: resolved, namespace: "canvas" } : { errors: [{ text: `cannot resolve "${args.path}"` }] } + } + const name = packageName(args.path) + if (!(name in project.dependencies) || !admitted[name]) return { errors: [{ text: `import_not_declared: "${args.path}"` }] } + if (args.path !== name && !(args.path in runtimeImports)) return { errors: [{ text: `import_not_declared: "${args.path}"` }] } + externalImports.add(args.path) + return { path: args.path, external: true } + }) + pluginBuild.onLoad({ filter: /.*/, namespace: "canvas" }, (args) => ({ contents: files[args.path], loader: loader(args.path), resolveDir: "/" })) + }, + } + return build({ + entryPoints: [entry], bundle: true, write: false, format: "esm", platform: "browser", target: "es2022", + jsx: "automatic", minify: true, sourcemap: false, logLevel: "silent", outdir: "out", plugins: [plugin], + }) +} + +function artifact(pathname, content) { + return { path: pathname, content, contentHash: sha256(content), sizeBytes: Buffer.byteLength(content, "utf8") } +} + +async function buildCanvas(project) { + const diagnostics = validate(project) + if (diagnostics.length) return { contractVersion: 1, status: "failed", diagnostics } + let html = project.files[project.entryHtml] + const refs = [ + ...[...html.matchAll(moduleScript)].map((match) => [match[1], "js"]), + ...[...html.matchAll(stylesheet)].map((match) => [match[1], "css"]), + ] + if (!refs.length) return { contractVersion: 1, status: "failed", diagnostics: [diagnostic("no_entry_module", "Canvas HTML references no module scripts or stylesheets", project.entryHtml)] } + const files = [] + const externalImports = new Set() + try { + for (const [reference, kind] of refs) { + const entry = normalize(reference) + if (!(entry in project.files)) return { contractVersion: 1, status: "failed", diagnostics: [diagnostic("entry_not_found", `Canvas entry ${entry} does not exist`, project.entryHtml)] } + const result = await bundleEntry(project, entry, externalImports) + let javascript = "" + let css = "" + for (const output of result.outputFiles ?? []) output.path.endsWith(".css") ? (css = output.text) : (javascript = output.text) + const content = kind === "css" ? css : javascript + const emitted = `assets/${path.posix.basename(entry).replace(/\.[^.]+$/, "")}-${sha256(content).slice(0, 10)}.${kind}` + files.push(artifact(emitted, content)) + html = html.split(`"${reference}"`).join(`"./${emitted}"`).split(`'${reference}'`).join(`'./${emitted}'`) + if (kind === "js" && css) { + const cssPath = `assets/${path.posix.basename(entry).replace(/\.[^.]+$/, "")}-${sha256(css).slice(0, 10)}.css` + files.push(artifact(cssPath, css)) + html = html.replace("", ``) + } + } + } catch (error) { + const errors = error?.errors ?? [] + return { contractVersion: 1, status: "failed", diagnostics: errors.length ? errors.slice(0, 500).map((entry) => diagnostic(entry.text.startsWith("import_not_declared:") ? "import_not_declared" : "bundle_error", entry.text, entry.location?.file?.replace(/^canvas:/, ""), entry.location?.line)) : [diagnostic("bundle_error", error instanceof Error ? error.message : String(error))] } + } + if (externalImports.size) { + const imports = {} + for (const specifier of externalImports) imports[specifier] = runtimeImports[specifier] ?? admitted[packageName(specifier)][1] + const importMap = `` + html = html.includes("") ? html.replace("", `${importMap}`) : `${importMap}${html}` + } + files.unshift(artifact(project.entryHtml, html)) + const manifest = { + entryHtml: project.entryHtml, + assets: files.map(({ path, contentHash, sizeBytes }) => ({ path, contentHash, sizeBytes })), + dependencies: project.dependencies, + canvasSdkVersion: project.canvasSdkVersion, + capabilities: { posthog: { insights: [], inlineQueries: true, captureEvents: [] }, network: { origins: [] } }, + } + return { contractVersion: 1, status: "ready", diagnostics: [], manifest, files } +} + +let input = "" +for await (const chunk of process.stdin) input += chunk +try { + const request = JSON.parse(input) + process.stdout.write(JSON.stringify(await buildCanvas(request.project))) +} catch (error) { + process.stdout.write(JSON.stringify({ contractVersion: 1, status: "failed", diagnostics: [diagnostic("invalid_build_request", error instanceof Error ? error.message : String(error))] })) +} diff --git a/common/canvas-builder/package-lock.json b/common/canvas-builder/package-lock.json new file mode 100644 index 000000000000..ae6c23848f21 --- /dev/null +++ b/common/canvas-builder/package-lock.json @@ -0,0 +1,453 @@ +{ + "name": "@posthog/canvas-builder", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@posthog/canvas-builder", + "dependencies": { + "esbuild": "0.25.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.0.tgz", + "integrity": "sha512-O7vun9Sf8DFjH2UtqK8Ku3LkquL9SZL8OLY1T5NZkA34+wG3OQF7cl4Ql8vdNzM6fzBbYfLaiRLIOZ+2FOCgBQ==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.0.tgz", + "integrity": "sha512-PTyWCYYiU0+1eJKmw21lWtC+d08JDZPQ5g+kFyxP0V+es6VPPSUhM6zk8iImp2jbV6GwjX4pap0JFbUQN65X1g==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.0.tgz", + "integrity": "sha512-grvv8WncGjDSyUBjN9yHXNt+cq0snxXbDxy5pJtzMKGmmpPxeAmAhWxXI+01lU5rwZomDgD3kJwulEnhTRUd6g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.0.tgz", + "integrity": "sha512-m/ix7SfKG5buCnxasr52+LI78SQ+wgdENi9CqyCXwjVR2X4Jkz+BpC3le3AoBPYTC9NHklwngVXvbJ9/Akhrfg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.0.tgz", + "integrity": "sha512-mVwdUb5SRkPayVadIOI78K7aAnPamoeFR2bT5nszFUZ9P8UpK4ratOdYbZZXYSqPKMHfS1wdHCJk1P1EZpRdvw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.0.tgz", + "integrity": "sha512-DgDaYsPWFTS4S3nWpFcMn/33ZZwAAeAFKNHNa1QN0rI4pUjgqf0f7ONmXf6d22tqTY+H9FNdgeaAa+YIFUn2Rg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.0.tgz", + "integrity": "sha512-VN4ocxy6dxefN1MepBx/iD1dH5K8qNtNe227I0mnTRjry8tj5MRk4zprLEdG8WPyAPb93/e4pSgi1SoHdgOa4w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.0.tgz", + "integrity": "sha512-mrSgt7lCh07FY+hDD1TxiTyIHyttn6vnjesnPoVDNmDfOmggTLXRv8Id5fNZey1gl/V2dyVK1VXXqVsQIiAk+A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.0.tgz", + "integrity": "sha512-vkB3IYj2IDo3g9xX7HqhPYxVkNQe8qTK55fraQyTzTX/fxaDtXiEnavv9geOsonh2Fd2RMB+i5cbhu2zMNWJwg==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.0.tgz", + "integrity": "sha512-9QAQjTWNDM/Vk2bgBl17yWuZxZNQIF0OUUuPZRKoDtqF2k4EtYbpyiG5/Dk7nqeK6kIJWPYldkOcBqjXjrUlmg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.0.tgz", + "integrity": "sha512-43ET5bHbphBegyeqLb7I1eYn2P/JYGNmzzdidq/w0T8E2SsYL1U6un2NFROFRg1JZLTzdCoRomg8Rvf9M6W6Gg==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.0.tgz", + "integrity": "sha512-fC95c/xyNFueMhClxJmeRIj2yrSMdDfmqJnyOY4ZqsALkDrrKJfIg5NTMSzVBr5YW1jf+l7/cndBfP3MSDpoHw==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.0.tgz", + "integrity": "sha512-nkAMFju7KDW73T1DdH7glcyIptm95a7Le8irTQNO/qtkoyypZAnjchQgooFUDQhNAy4iu08N79W4T4pMBwhPwQ==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.0.tgz", + "integrity": "sha512-NhyOejdhRGS8Iwv+KKR2zTq2PpysF9XqY+Zk77vQHqNbo/PwZCzB5/h7VGuREZm1fixhs4Q/qWRSi5zmAiO4Fw==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.0.tgz", + "integrity": "sha512-5S/rbP5OY+GHLC5qXp1y/Mx//e92L1YDqkiBbO9TQOvuFXM+iDqUNG5XopAnXoRH3FjIUDkeGcY1cgNvnXp/kA==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.0.tgz", + "integrity": "sha512-XM2BFsEBz0Fw37V0zU4CXfcfuACMrppsMFKdYY2WuTS3yi8O1nFOhil/xhKTmE1nPmVyvQJjJivgDT+xh8pXJA==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.0.tgz", + "integrity": "sha512-9yl91rHw/cpwMCNytUDxwj2XjFpxML0y9HAOH9pNVQDpQrBxHy01Dx+vaMu0N1CKa/RzBD2hB4u//nfc+Sd3Cw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.0.tgz", + "integrity": "sha512-RuG4PSMPFfrkH6UwCAqBzauBWTygTvb1nxWasEJooGSJ/NwRw7b2HOwyRTQIU97Hq37l3npXoZGYMy3b3xYvPw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.0.tgz", + "integrity": "sha512-jl+qisSB5jk01N5f7sPCsBENCOlPiS/xptD5yxOx2oqQfyourJwIKLRA2yqWdifj3owQZCL2sn6o08dBzZGQzA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.0.tgz", + "integrity": "sha512-21sUNbq2r84YE+SJDfaQRvdgznTD8Xc0oc3p3iW/a1EVWeNj/SdUCbm5U0itZPQYRuRTW20fPMWMpcrciH2EJw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.0.tgz", + "integrity": "sha512-2gwwriSMPcCFRlPlKx3zLQhfN/2WjJ2NSlg5TKLQOJdV0mSxIcYNTMhk3H3ulL/cak+Xj0lY1Ym9ysDV1igceg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.0.tgz", + "integrity": "sha512-bxI7ThgLzPrPz484/S9jLlvUAHYMzy6I0XiU1ZMeAEOBcS0VePBFxh1JjTQt3Xiat5b6Oh4x7UC7IwKQKIJRIg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.0.tgz", + "integrity": "sha512-ZUAc2YK6JW89xTbXvftxdnYy3m4iHIkDtK3CLce8wg8M2L+YZhIvO1DKpxrd0Yr59AeNNkTiic9YLf6FTtXWMw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.0.tgz", + "integrity": "sha512-eSNxISBu8XweVEWG31/JzjkIGbGIJN/TrRoiSVZwZ6pkC6VX4Im/WV2cz559/TXLcYbcrDN8JtKgd9DJVIo8GA==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.0.tgz", + "integrity": "sha512-ZENoHJBxA20C2zFzh6AI4fT6RraMzjYw4xKWemRTRmRVtN9c5DcH9r/f2ihEkMjOW5eGgrwCslG/+Y/3bL+DHQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/esbuild": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.0.tgz", + "integrity": "sha512-BXq5mqc8ltbaN34cDqWuYKyNhX8D/Z0J1xdtdQ8UcIIIyJyz+ZMKUt58tF3SrZ85jcfN/PZYhjR5uDQAYNVbuw==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.0", + "@esbuild/android-arm": "0.25.0", + "@esbuild/android-arm64": "0.25.0", + "@esbuild/android-x64": "0.25.0", + "@esbuild/darwin-arm64": "0.25.0", + "@esbuild/darwin-x64": "0.25.0", + "@esbuild/freebsd-arm64": "0.25.0", + "@esbuild/freebsd-x64": "0.25.0", + "@esbuild/linux-arm": "0.25.0", + "@esbuild/linux-arm64": "0.25.0", + "@esbuild/linux-ia32": "0.25.0", + "@esbuild/linux-loong64": "0.25.0", + "@esbuild/linux-mips64el": "0.25.0", + "@esbuild/linux-ppc64": "0.25.0", + "@esbuild/linux-riscv64": "0.25.0", + "@esbuild/linux-s390x": "0.25.0", + "@esbuild/linux-x64": "0.25.0", + "@esbuild/netbsd-arm64": "0.25.0", + "@esbuild/netbsd-x64": "0.25.0", + "@esbuild/openbsd-arm64": "0.25.0", + "@esbuild/openbsd-x64": "0.25.0", + "@esbuild/sunos-x64": "0.25.0", + "@esbuild/win32-arm64": "0.25.0", + "@esbuild/win32-ia32": "0.25.0", + "@esbuild/win32-x64": "0.25.0" + } + } + } +} diff --git a/common/canvas-builder/package.json b/common/canvas-builder/package.json new file mode 100644 index 000000000000..8a67aae1160c --- /dev/null +++ b/common/canvas-builder/package.json @@ -0,0 +1,8 @@ +{ + "name": "@posthog/canvas-builder", + "private": true, + "type": "module", + "dependencies": { + "esbuild": "0.25.0" + } +} diff --git a/posthog/api/file_system/canvas_build_service.py b/posthog/api/file_system/canvas_build_service.py index 50a2bab95229..94100d0e2799 100644 --- a/posthog/api/file_system/canvas_build_service.py +++ b/posthog/api/file_system/canvas_build_service.py @@ -21,7 +21,9 @@ import json import time import hashlib +import subprocess from datetime import timedelta +from pathlib import Path from typing import Any from uuid import UUID @@ -31,19 +33,96 @@ import structlog -from posthog.api.file_system.canvas_source import ( - CANVAS_COMPONENT_PATH, - SYNTHETIC_INDEX_HTML, - extract_legacy_code, - has_errors, - validate_source_project, -) +from posthog.api.file_system.canvas_source import SYNTHETIC_INDEX_HTML, has_errors, validate_source_project from posthog.models.file_system.canvas_build import CanvasBuild, CanvasSourceVersion from posthog.models.file_system.file_system import FileSystem from posthog.storage import object_storage logger = structlog.get_logger(__name__) +CANVAS_BUILDER_PATH = Path(__file__).resolve().parents[3] / "common" / "canvas-builder" / "build.mjs" +MAX_ARTIFACT_FILES = 256 +MAX_ARTIFACT_FILE_BYTES = 1024 * 1024 +MAX_ARTIFACT_TOTAL_BYTES = 8 * 1024 * 1024 + + +def run_cloud_builder(project: dict[str, Any]) -> dict[str, Any]: + process = subprocess.run( + ["node", "--max-old-space-size=256", str(CANVAS_BUILDER_PATH)], + input=json.dumps({"project": project}, separators=(",", ":")), + capture_output=True, + text=True, + timeout=45, + check=False, + cwd=CANVAS_BUILDER_PATH.parent, + env={"PATH": "/usr/local/bin:/usr/bin:/bin", "NODE_ENV": "production"}, + ) + if process.returncode != 0: + raise RuntimeError("canvas builder process failed") + result = json.loads(process.stdout) + if not isinstance(result, dict): + raise ValueError("canvas builder returned an invalid response") + return result + + +def _valid_artifact_path(value: str) -> bool: + segments = value.split("/") + return ( + bool(value) + and not value.startswith("/") + and "\\" not in value + and all(segment not in {"", ".", ".."} for segment in segments) + and not any(character in value for character in "\r\n\0") + ) + + +def validate_builder_output(result: dict[str, Any]) -> tuple[list[dict[str, Any]], dict[str, Any], list[dict[str, Any]]]: + if result.get("contractVersion") != 1 or result.get("status") != "ready": + raise ValueError("canvas builder did not return a ready contract") + files = result.get("files") + manifest = result.get("manifest") + diagnostics = result.get("diagnostics") + if not isinstance(files, list) or not isinstance(manifest, dict) or not isinstance(diagnostics, list): + raise ValueError("canvas builder omitted artifacts, manifest, or diagnostics") + if len(files) > MAX_ARTIFACT_FILES: + raise ValueError("canvas artifact manifest has too many files") + seen: set[str] = set() + total = 0 + for artifact in files: + if not isinstance(artifact, dict): + raise ValueError("canvas builder emitted an invalid artifact") + path = artifact.get("path") + content = artifact.get("content") + digest = artifact.get("contentHash") + size = artifact.get("sizeBytes") + if ( + not isinstance(path, str) + or not _valid_artifact_path(path) + or path in seen + or not isinstance(content, str) + or not isinstance(digest, str) + or len(digest) != 64 + or not isinstance(size, int) + or isinstance(size, bool) + ): + raise ValueError("canvas builder emitted an invalid artifact") + encoded = content.encode("utf-8") + if hashlib.sha256(encoded).hexdigest() != digest or len(encoded) != size: + raise ValueError("canvas artifact integrity does not match its manifest") + if size > MAX_ARTIFACT_FILE_BYTES: + raise ValueError("canvas artifact exceeds the per-file size limit") + seen.add(path) + total += size + if total > MAX_ARTIFACT_TOTAL_BYTES: + raise ValueError("canvas build exceeds the total artifact size limit") + assets = manifest.get("assets") + if not isinstance(assets, list) or {asset.get("path") for asset in assets if isinstance(asset, dict)} != seen: + raise ValueError("canvas artifact manifest does not match emitted files") + entry = manifest.get("entryHtml") + if not isinstance(entry, str) or entry not in seen: + raise ValueError("canvas build does not contain its entry HTML") + return files, manifest, diagnostics[:500] + # Retention policy (see the canvas build pipeline plan): every referenced # source version is kept for the canvas's lifetime; artifacts are bounded. FAILED_BUILD_RETENTION = timedelta(hours=24) @@ -149,10 +228,9 @@ def run_canvas_build(team_id: int, build_id: str) -> None: failed build records diagnostics and leaves the last-known-good build untouched. Idempotent: a re-delivered task for a finished build is a no-op. - Until the isolated build image (node + esbuild, the shared build recipe) - ships, the artifact for legacy-compatible projects freezes the project - files themselves — the runtime keeps compiling the component exactly as it - does today, now from an immutable, content-addressed snapshot. + The isolated Node process runs the same versioned contract as local + previews. Only its validated manifest and files may cross back into the + control plane. """ build = CanvasBuild.objects.for_team(team_id).filter(id=build_id).select_related("source_version", "canvas").first() if build is None: @@ -184,34 +262,34 @@ def run_canvas_build(team_id: int, build_id: str) -> None: _finish_failed(build, diagnostics) return + project_files = dict(project["files"]) + project_files.setdefault(project.get("entryHtml", "index.html"), SYNTHETIC_INDEX_HTML) + project = {**project, "files": project_files} + try: + result = run_cloud_builder(project) + if result.get("status") != "ready": + builder_diagnostics = result.get("diagnostics") + _finish_failed(build, builder_diagnostics[:500] if isinstance(builder_diagnostics, list) else []) + return + files, manifest, diagnostics = validate_builder_output(result) + except (subprocess.TimeoutExpired, OSError, json.JSONDecodeError, RuntimeError, ValueError) as error: + logger.warning("canvas_build_process_failed", build_id=str(build.id), error_type=type(error).__name__) + _finish_failed( + build, + [{"severity": "error", "code": "build_unavailable", "message": "The canvas build service is unavailable."}], + ) + return + prefix = artifact_object_prefix(build.team_id, build.canvas_id, build.id) - entry_html = project.get("entryHtml", "index.html") - files = dict(project["files"]) - # A legacy-compatible project may omit its entry shell (the read path - # synthesizes it); the frozen artifact must be self-consistent, so - # materialize the same shell the runtime would. - files.setdefault(entry_html, SYNTHETIC_INDEX_HTML) - assets = [] - for path, content in sorted(files.items()): - content_bytes = content.encode("utf-8") - content_hash = hashlib.sha256(content_bytes).hexdigest() - object_storage.write(f"{prefix}/{path}", content_bytes, extras={"ContentType": "text/plain; charset=utf-8"}) - assets.append({"path": path, "contentHash": content_hash, "sizeBytes": len(content_bytes)}) - - manifest = { - "entryHtml": entry_html, - "assets": assets, - "dependencies": project.get("dependencies", {}), - "canvasSdkVersion": project.get("canvasSdkVersion", "0.1.0"), - "capabilities": { - "posthog": {"insights": [], "inlineQueries": True, "captureEvents": []}, - "network": {"origins": []}, - }, - # The legacy runtime mounts this component; the isolated build image - # will replace this with compiled chunks under the same manifest shape. - "legacyComponentPath": CANVAS_COMPONENT_PATH, - "legacyCode": extract_legacy_code(project) if CANVAS_COMPONENT_PATH in project["files"] else None, - } + manifest_assets = {asset["path"]: asset for asset in manifest["assets"]} + for artifact in files: + content_type = _artifact_content_type(artifact["path"]) + object_storage.write( + f"{prefix}/{artifact['path']}", + artifact["content"].encode("utf-8"), + extras={"ContentType": content_type, "CacheControl": "private, max-age=31536000, immutable"}, + ) + manifest_assets[artifact["path"]]["contentType"] = content_type integrity = hashlib.sha256(json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest() # Second transaction: mark ready and advance the live pointer only while @@ -236,6 +314,18 @@ def run_canvas_build(team_id: int, build_id: str) -> None: dashboard.save(update_fields=["meta"]) +def _artifact_content_type(path: str) -> str: + if path.endswith(".html"): + return "text/html; charset=utf-8" + if path.endswith(".js"): + return "text/javascript; charset=utf-8" + if path.endswith(".css"): + return "text/css; charset=utf-8" + if path.endswith(".json"): + return "application/json; charset=utf-8" + return "application/octet-stream" + + def _finish_failed(build: CanvasBuild, diagnostics: list[dict[str, Any]]) -> None: build.status = CanvasBuild.STATUS_FAILED build.diagnostics = diagnostics diff --git a/posthog/api/file_system/test/test_canvas_cloud_builder.py b/posthog/api/file_system/test/test_canvas_cloud_builder.py new file mode 100644 index 000000000000..977ca5454907 --- /dev/null +++ b/posthog/api/file_system/test/test_canvas_cloud_builder.py @@ -0,0 +1,60 @@ +import hashlib +from typing import Any + +from unittest.mock import patch + +from django.test import SimpleTestCase + +from posthog.api.file_system.canvas_build_service import run_cloud_builder, validate_builder_output + + +class TestCanvasCloudBuilder(SimpleTestCase): + def _project(self, source: str) -> dict[str, Any]: + return { + "schemaVersion": 1, + "files": { + "index.html": '
', + "src/main.ts": source, + }, + "entryHtml": "index.html", + "dependencies": {}, + "canvasSdkVersion": "0.1.0", + } + + def test_builds_vanilla_typescript_with_the_shared_contract(self) -> None: + result = run_cloud_builder(self._project('document.querySelector("#root")!.textContent = "Hello"')) + + files, manifest, diagnostics = validate_builder_output(result) + self.assertEqual(diagnostics, []) + self.assertEqual(manifest["entryHtml"], "index.html") + self.assertTrue(any(file["path"].endswith(".js") for file in files)) + + def test_rejects_undeclared_package_imports(self) -> None: + result = run_cloud_builder(self._project('import React from "react"; void React')) + + self.assertEqual(result["status"], "failed") + self.assertEqual(result["diagnostics"][0]["code"], "import_not_declared") + + def test_rejects_artifact_content_that_does_not_match_manifest(self) -> None: + result = { + "contractVersion": 1, + "status": "ready", + "diagnostics": [], + "files": [{"path": "index.html", "content": "tampered", "contentHash": hashlib.sha256(b"safe").hexdigest(), "sizeBytes": 4}], + "manifest": {"entryHtml": "index.html", "assets": []}, + } + + with self.assertRaisesMessage(ValueError, "integrity"): + validate_builder_output(result) + + @patch("posthog.api.file_system.canvas_build_service.subprocess.run") + def test_builder_has_bounded_process_resources(self, run: Any) -> None: + run.return_value.returncode = 0 + run.return_value.stdout = '{"contractVersion":1,"status":"failed","diagnostics":[]}' + + run_cloud_builder({"files": {}}) + + args, kwargs = run.call_args + self.assertEqual(args[0][:2], ["node", "--max-old-space-size=256"]) + self.assertEqual(kwargs["timeout"], 45) + self.assertEqual(kwargs["env"], {"PATH": "/usr/local/bin:/usr/bin:/bin", "NODE_ENV": "production"})