From 4e48a38c9ebe0578919e261c8e5695370f3a48ef Mon Sep 17 00:00:00 2001 From: Prajwal Aradhya Date: Mon, 17 Aug 2026 19:47:12 +0100 Subject: [PATCH 1/2] feat(portal,server): custom blocks authoring, canvas integration and queued execution Closes #258. Portal - Input params editor and icon picker, wired into a 4-step create page and a settings modal, so a custom block can finally declare its input contract. - List page reworked: grid of the real canvas node, search, empty state, and a click straight into the block's canvas. - Custom blocks now load from the DB into the block picker under their own category (plugin-sourced ones get "Built-in", ready for the shipped set), and a placed block renders its own label, description and icon on the canvas and in the settings panel. - A custom block's `integration_selector` params are injected into the integration picker on its own canvas: picking one writes `param:` and the real integration is chosen by the caller, so no test/open action is shown. Engine - `params` is bound alongside `input` in a custom block's JS: configuration is `params.` at any depth, `input` is always the previous block's output. - Nine db/log emitters take their connection through `node.value`, so a `param:` integration resolves at call time. - New `queued` invoke mode: durable background execution over a single shared JetStream work queue (`fluxify.jobs..`), kind-routed so crons, workflows and schedules can reuse the same stream and worker. Fixes - The compiler never registered custom blocks into its own library, so any route calling one failed with "No codegen for block type". It now registers its own output and resolves dependencies before compiling, and unregisters on rename or delete. - Recursion is refused: a block cannot call itself, directly or through a chain, checked on every canvas save and reflected in the picker. - Handle shape rules no longer depend on stylesheet order. Co-Authored-By: Claude Opus 5 --- .../components/canvas/BlockPickerSidebar.tsx | 93 +++- .../components/canvas/blocks/BlockNode.tsx | 23 +- .../src/components/canvas/blocks/blocks.css | 17 +- .../canvas/blocks/defaultBlockData.ts | 5 +- .../src/components/canvas/blocks/index.ts | 1 + .../canvas/blocks/useCustomBlockDefs.ts | 46 ++ .../components/canvas/panel/BlockPanel.tsx | 28 +- .../panel/blocks/CustomBlockSettings.tsx | 5 +- .../src/components/canvas/panel/fields.tsx | 52 +- .../customBlocks/CustomBlockSettingsModal.tsx | 311 +++++++++++ .../customBlocks/CustomBlockSwitcher.tsx | 119 ++++ .../components/customBlocks/IconPicker.tsx | 164 ++++++ .../customBlocks/InputParamsEditor.tsx | 307 +++++++++++ .../components/customBlocks/premadeIcons.tsx | 517 ++++++++++++++++++ .../components/routes/RouteSettingsModal.tsx | 59 ++ apps/portal/src/query/customBlocksQuery.ts | 10 + .../_authed/$projectId/custom-blocks.tsx | 243 ++++---- .../_authed/$projectId/custom-blocks_.new.tsx | 347 ++++++++++++ ...rojectId_.custom-block-canvas.$blockId.tsx | 38 +- apps/portal/src/services/customBlocks.ts | 12 + apps/server/deployments/compiledWorker.ts | 54 ++ .../src/api/v1/custom-blocks/create/dto.ts | 5 + .../api/v1/custom-blocks/create/repository.ts | 4 +- .../api/v1/custom-blocks/delete/service.ts | 9 +- .../custom-blocks/delete/tests/delete.test.ts | 4 + .../server/src/api/v1/custom-blocks/shared.ts | 253 ++++++++- .../src/api/v1/routes/create/repository.ts | 6 +- .../src/api/v1/routes/delete/service.ts | 7 + apps/server/src/lib/env.ts | 22 + apps/server/src/modules/canvas/repository.ts | 37 +- apps/server/src/modules/canvas/service.ts | 68 +++ .../src/modules/canvas/tests/service.spec.ts | 61 +++ apps/server/src/modules/compiler/service.ts | 86 ++- apps/server/src/modules/jobs/consumer.ts | 167 ++++++ .../server/src/modules/jobs/customBlockJob.ts | 35 ++ apps/server/src/modules/jobs/publisher.ts | 37 ++ apps/server/src/modules/jobs/registry.ts | 34 ++ apps/server/src/modules/jobs/subjects.ts | 41 ++ apps/server/src/modules/jobs/types.ts | 22 + .../modules/requestRouter/executionProcess.ts | 31 ++ .../src/modules/requestRouter/service.ts | 55 ++ .../src/modules/requestRouter/threadTypes.ts | 8 +- .../testRunner/testExecutionProcess.ts | 15 +- packages/blocks/builtin/customBlock.ts | 56 +- packages/blocks/builtin/db/delete.ts | 2 +- packages/blocks/builtin/db/getAll.ts | 2 +- packages/blocks/builtin/db/getSingle.ts | 2 +- packages/blocks/builtin/db/insert.ts | 2 +- packages/blocks/builtin/db/insertBulk.ts | 2 +- packages/blocks/builtin/db/native.ts | 2 +- packages/blocks/builtin/db/transaction.ts | 2 +- packages/blocks/builtin/db/update.ts | 2 +- packages/blocks/builtin/log/cloudLogs.ts | 2 +- .../builtin/tests/compilerCustomBlock.spec.ts | 129 ++++- packages/blocks/compiler.ts | 19 +- packages/blocks/index.ts | 1 + packages/blocks/jobs.ts | 51 ++ .../IntegrationSelector.tsx | 41 +- .../src/IntegrationSelector/types.ts | 14 + 59 files changed, 3574 insertions(+), 213 deletions(-) create mode 100644 apps/portal/src/components/canvas/blocks/useCustomBlockDefs.ts create mode 100644 apps/portal/src/components/customBlocks/CustomBlockSettingsModal.tsx create mode 100644 apps/portal/src/components/customBlocks/CustomBlockSwitcher.tsx create mode 100644 apps/portal/src/components/customBlocks/IconPicker.tsx create mode 100644 apps/portal/src/components/customBlocks/InputParamsEditor.tsx create mode 100644 apps/portal/src/components/customBlocks/premadeIcons.tsx create mode 100644 apps/portal/src/routes/_authed/$projectId/custom-blocks_.new.tsx create mode 100644 apps/server/src/modules/jobs/consumer.ts create mode 100644 apps/server/src/modules/jobs/customBlockJob.ts create mode 100644 apps/server/src/modules/jobs/publisher.ts create mode 100644 apps/server/src/modules/jobs/registry.ts create mode 100644 apps/server/src/modules/jobs/subjects.ts create mode 100644 apps/server/src/modules/jobs/types.ts create mode 100644 packages/blocks/jobs.ts diff --git a/apps/portal/src/components/canvas/BlockPickerSidebar.tsx b/apps/portal/src/components/canvas/BlockPickerSidebar.tsx index 359bee90..fe177e9d 100644 --- a/apps/portal/src/components/canvas/BlockPickerSidebar.tsx +++ b/apps/portal/src/components/canvas/BlockPickerSidebar.tsx @@ -1,7 +1,8 @@ import { useEffect, useMemo, useState, type ReactNode } from "react"; -import { Button, Input, Label, Sidebar, TextField } from "@fluxify/components"; +import { Button, cn, Input, Label, Sidebar, TextField } from "@fluxify/components"; import { TbArrowLeft, + TbBoxMultiple, TbChevronRight, TbCode, TbDatabase, @@ -12,15 +13,18 @@ import { TbWorld, TbX, } from "react-icons/tb"; +import { CustomBlockIcon } from "@/components/customBlocks/IconPicker"; import { pickerBlockCatalogEntries, blockIcon, type BlockDefinition, type BlockType, } from "./blocks"; +import { useCustomBlockDefs } from "./blocks/useCustomBlockDefs"; import "./blockPickerSidebar.css"; -type BlockCategory = BlockDefinition["category"]; +/** Catalog categories plus the ones only custom blocks land in. */ +type BlockCategory = BlockDefinition["category"] | "Custom" | "Built-in"; type CategoryDetails = { description: string; @@ -52,9 +56,27 @@ const CATEGORY_DETAILS: Record = { description: "Transform data and annotate the canvas", icon: , }, + Custom: { + description: "Blocks built in this project", + icon: , + }, + "Built-in": { + description: "Blocks shipped with plugins", + icon: , + }, }; -const categories = Object.keys(CATEGORY_DETAILS) as BlockCategory[]; +const allCategories = Object.keys(CATEGORY_DETAILS) as BlockCategory[]; + +type PickerItem = { + type: string; + name: string; + description: string; + category: BlockCategory; + icon: ReactNode; + /** Set when the block cannot be added — the reason is shown in its place. */ + disabledReason?: string; +}; export type BlockPickerSidebarProps = { isOpen: boolean; @@ -62,7 +84,7 @@ export type BlockPickerSidebarProps = { onAdd: (type: BlockType) => void; }; -/** Core-block picker. Custom blocks join this catalog after their API lands. */ +/** Core blocks from the catalog plus the project's own custom blocks. */ export function BlockPickerSidebar({ isOpen, onOpenChange, @@ -80,15 +102,42 @@ export function BlockPickerSidebar({ } }, [isOpen]); - const blocks = useMemo(() => pickerBlockCatalogEntries(), []); + const customDefs = useCustomBlockDefs(); + const blocks = useMemo(() => { + const core = pickerBlockCatalogEntries().map(([type, definition]) => ({ + type: type as string, + name: definition.name, + description: definition.description, + category: definition.category as BlockCategory, + icon: blockIcon(type), + })); + const custom = customDefs.map((def) => ({ + type: def.name, + name: def.label, + description: def.description ?? "Custom block", + category: (def.sourceType === "plugin" ? "Built-in" : "Custom") as BlockCategory, + icon: , + disabledReason: def.isSelf + ? "A block can't call itself — that would recurse forever." + : undefined, + })); + return [...custom, ...core]; + }, [customDefs]); + + // An empty category is a dead end — only offer the ones holding something. + const categories = useMemo( + () => allCategories.filter((category) => blocks.some((b) => b.category === category)), + [blocks], + ); + const visibleBlocks = useMemo(() => { const normalizedQuery = query.trim().toLowerCase(); - return blocks.filter(([type, definition]) => { - if (!normalizedQuery && selectedCategory && definition.category !== selectedCategory) { + return blocks.filter((block) => { + if (!normalizedQuery && selectedCategory && block.category !== selectedCategory) { return false; } if (!normalizedQuery) return true; - return [type, definition.name, definition.description, definition.category] + return [block.type, block.name, block.description, block.category] .join(" ") .toLowerCase() .includes(normalizedQuery); @@ -177,26 +226,36 @@ export function BlockPickerSidebar({ ) : (
- {visibleBlocks.map(([type, definition]) => ( + {visibleBlocks.map((block) => ( ))} {visibleBlocks.length === 0 && ( -

No core blocks match.

+

No blocks match.

)}
)} diff --git a/apps/portal/src/components/canvas/blocks/BlockNode.tsx b/apps/portal/src/components/canvas/blocks/BlockNode.tsx index 1ae006b6..337a990e 100644 --- a/apps/portal/src/components/canvas/blocks/BlockNode.tsx +++ b/apps/portal/src/components/canvas/blocks/BlockNode.tsx @@ -1,4 +1,5 @@ import type { NodeProps, NodeTypes } from "@xyflow/react"; +import { CustomBlockIcon } from "@/components/customBlocks/IconPicker"; import { BaseBlock } from "./BaseBlock"; import { blockCatalogEntries } from "./blockCatalog"; import { blockIcon } from "./blockIconMap"; @@ -6,6 +7,7 @@ import { blockLabels } from "./blockLabels"; import { BLOCK_TYPES } from "./blockTypes"; import { BlockHandle } from "./handles/BlockHandle"; import { StickyNoteBlock } from "./StickyNoteBlock"; +import { useCustomBlockDefs } from "./useCustomBlockDefs"; function status(value: unknown): boolean | null { return typeof value === "boolean" ? value : null; @@ -23,7 +25,10 @@ export function BlockNode({ positionAbsoluteX, positionAbsoluteY, }: NodeProps) { - const { name, description, definition } = blockLabels(type, data); + const { name, description, definition, custom } = blockLabels(type, data); + // A custom block has no catalog entry: its name, blurb and icon live in the DB. + // Anything the user typed on this node still wins. + const customDef = useCustomBlockDefs().find((def) => def.name === type); const isRouteOwned = type === BLOCK_TYPES.entrypoint || type === BLOCK_TYPES.errorHandler; @@ -32,9 +37,19 @@ export function BlockNode({ blockId={id} blockType={type} position={{ x: positionAbsoluteX, y: positionAbsoluteY }} - name={name} - description={description} - icon={blockIcon(type)} + name={custom ? name : (customDef?.label ?? name)} + description={ + description === definition.description + ? (customDef?.description ?? description) + : description + } + icon={ + customDef ? ( + + ) : ( + blockIcon(type) + ) + } color={definition.tint} selected={selected} status={status(data?.status)} diff --git a/apps/portal/src/components/canvas/blocks/blocks.css b/apps/portal/src/components/canvas/blocks/blocks.css index 70a164ce..8c232ea0 100644 --- a/apps/portal/src/components/canvas/blocks/blocks.css +++ b/apps/portal/src/components/canvas/blocks/blocks.css @@ -436,25 +436,30 @@ html[data-theme="dark"] .fx-block, min-height: 0; } -.fx-handle--circle { +/* Every shape rule carries .react-flow__handle: the library's own + `.react-flow__handle { width: 6px; height: 6px; border-radius: 100% }` is a + single class too, so on a tie the sheet that loads last wins. Importing this + file from a non-canvas module (a block preview, say) is enough to flip that + order and turn the inbound bar into an oval. */ +.fx-handle--circle.react-flow__handle { width: 10px; height: 10px; border-radius: 9999px; } /* The inbound socket: a bar on the block's edge. */ -.fx-handle--rect { +.fx-handle--rect.react-flow__handle { border-radius: 2px; } -.fx-handle--rect.fx-handle--left, -.fx-handle--rect.fx-handle--right { +.fx-handle--rect.fx-handle--left.react-flow__handle, +.fx-handle--rect.fx-handle--right.react-flow__handle { width: 6px; height: 16px; } -.fx-handle--rect.fx-handle--top, -.fx-handle--rect.fx-handle--bottom { +.fx-handle--rect.fx-handle--top.react-flow__handle, +.fx-handle--rect.fx-handle--bottom.react-flow__handle { width: 16px; height: 6px; } diff --git a/apps/portal/src/components/canvas/blocks/defaultBlockData.ts b/apps/portal/src/components/canvas/blocks/defaultBlockData.ts index 61f93eb1..256b9a0f 100644 --- a/apps/portal/src/components/canvas/blocks/defaultBlockData.ts +++ b/apps/portal/src/components/canvas/blocks/defaultBlockData.ts @@ -1,4 +1,4 @@ -import { BLOCK_TYPES, type BlockType } from "./blockTypes"; +import { BLOCK_TYPES, BLOCK_TYPE_LIST, type BlockType } from "./blockTypes"; /** * Complete, schema-valid starting data for every core block created on canvas. @@ -6,6 +6,9 @@ import { BLOCK_TYPES, type BlockType } from "./blockTypes"; * insertion affordances always create the same valid block payload. */ export function defaultBlockData(type: BlockType): Record { + // A custom block: its input params are filled in the panel, but the engine + // always reads an invocation mode. + if (!BLOCK_TYPE_LIST.includes(type)) return { invoke: "sync" }; if (type === BLOCK_TYPES.response) return { httpCode: "200" }; if (type === BLOCK_TYPES.if) return { conditions: [] }; if (type === BLOCK_TYPES.forloop) return { start: 0, end: 1, step: 1 }; diff --git a/apps/portal/src/components/canvas/blocks/index.ts b/apps/portal/src/components/canvas/blocks/index.ts index 3b25bda7..23bd4ead 100644 --- a/apps/portal/src/components/canvas/blocks/index.ts +++ b/apps/portal/src/components/canvas/blocks/index.ts @@ -18,6 +18,7 @@ export { type BlockDefinition, } from "./blockCatalog"; export { blockLabels, type BlockLabels } from "./blockLabels"; +export { useCustomBlockDefs, type CustomBlockDef } from "./useCustomBlockDefs"; export { BLOCK_TYPES, BLOCK_TYPE_LIST, type BlockType } from "./blockTypes"; export { StickyNoteBlock } from "./StickyNoteBlock"; export { diff --git a/apps/portal/src/components/canvas/blocks/useCustomBlockDefs.ts b/apps/portal/src/components/canvas/blocks/useCustomBlockDefs.ts new file mode 100644 index 00000000..8ba6ad09 --- /dev/null +++ b/apps/portal/src/components/canvas/blocks/useCustomBlockDefs.ts @@ -0,0 +1,46 @@ +import { useMemo } from "react"; +import { useParams } from "@tanstack/react-router"; +import { customBlocksQuery } from "@/query/customBlocksQuery"; +import type { IconValue } from "@/components/customBlocks/IconPicker"; + +export type CustomBlockDef = { + id: string; + /** The block type on canvas — what the engine looks up. */ + name: string; + label: string; + description?: string; + icon?: IconValue["icon"]; + iconUrl?: string; + sourceType?: string | null; + /** This is the block whose canvas is open: adding it would be recursion. */ + isSelf?: boolean; +}; + +/** + * The project's custom blocks, shaped like catalog entries so the picker and the + * node can render them. On a custom block's own canvas (`blockId` in the route) + * that block is flagged `isSelf` — the picker offers it disabled rather than + * letting a block call itself. + */ +export function useCustomBlockDefs(): CustomBlockDef[] { + const params = useParams({ strict: false }) as { + projectId?: string; + blockId?: string; + }; + const projectId = params?.projectId ?? ""; + const { data } = customBlocksQuery.getAll.useQuery(projectId); + + return useMemo(() => { + if (!data) return []; + return data.map((block) => ({ + id: block.id, + name: block.name, + label: block.label || block.name, + description: block.description ?? undefined, + icon: (block.icon as IconValue["icon"]) ?? undefined, + iconUrl: block.iconUrl ?? undefined, + sourceType: block.sourceType, + isSelf: block.id === params?.blockId, + })); + }, [data, params?.blockId]); +} diff --git a/apps/portal/src/components/canvas/panel/BlockPanel.tsx b/apps/portal/src/components/canvas/panel/BlockPanel.tsx index 3f39b1fc..f76fe978 100644 --- a/apps/portal/src/components/canvas/panel/BlockPanel.tsx +++ b/apps/portal/src/components/canvas/panel/BlockPanel.tsx @@ -4,8 +4,10 @@ import "./panel.css"; import { BlockSettings } from "./BlockSettings"; import { blockSettingsTabs } from "./blockSettingsRegistry"; import { useBlockPanelResize } from "./useBlockPanelResize"; +import { CustomBlockIcon } from "@/components/customBlocks/IconPicker"; import { blockIcon } from "../blocks/blockIconMap"; import { blockLabels } from "../blocks/blockLabels"; +import { useCustomBlockDefs } from "../blocks/useCustomBlockDefs"; import type { BlockNode } from "../types"; export type BlockPanelProps = { @@ -47,7 +49,12 @@ export function BlockPanel({ const current = block ?? shown.current; const type = current?.type ?? "unknown"; - const { name, description, definition } = blockLabels(type, current?.data); + const { name, description, definition, custom } = blockLabels(type, current?.data); + // A custom block: the label titles the panel and the identifier sits under it, + // since that is what flows and `param:` references are written against. + const customDef = useCustomBlockDefs().find((def) => def.name === type); + const title = custom ? name : (customDef?.label ?? name); + const subtitle = customDef ? customDef.name : description; const tabs = blockSettingsTabs(current?.type); const { @@ -110,16 +117,23 @@ export function BlockPanel({ className="fx-panel__icon" style={definition.tint ? { color: definition.tint } : undefined} > - {blockIcon(type)} + {customDef ? ( + + ) : ( + blockIcon(type) + )} {/* Renaming lives in the General tab; the header only shows it. */} - - {name} + + {title} - {/* What the block does, under whatever it was named. */} - - {description} + {/* What the block does — or, for a custom block, what it is called. */} + + {subtitle} {current.id && ( diff --git a/apps/portal/src/components/canvas/panel/blocks/CustomBlockSettings.tsx b/apps/portal/src/components/canvas/panel/blocks/CustomBlockSettings.tsx index 55b9c975..eb271cb3 100644 --- a/apps/portal/src/components/canvas/panel/blocks/CustomBlockSettings.tsx +++ b/apps/portal/src/components/canvas/panel/blocks/CustomBlockSettings.tsx @@ -58,7 +58,7 @@ export function CustomBlockSettingsPanel({ block }: { block: BlockNode }) { onPress={() => window.open( withBasePath( - `/${projectId}/custom-blocks/${customBlock.id}`, + `/${projectId}/custom-block-canvas/${customBlock.id}`, ), "_blank", "noopener,noreferrer", @@ -76,11 +76,12 @@ export function CustomBlockSettingsPanel({ block }: { block: BlockNode }) { data={block.data} name="invoke" label="Execution Mode" - hint="Sync waits for execution and passes output; Async fires in background." + hint="Sync waits for the output. Async fires on this worker and is lost if it restarts. Queued is durable — another worker picks it up, and it may be retried." placeholder="Select execution mode" options={[ { value: "sync", label: "Synchronous (Wait for output)" }, { value: "async", label: "Asynchronous (Fire & forget)" }, + { value: "queued", label: "Queued (Durable background job)" }, ]} /> diff --git a/apps/portal/src/components/canvas/panel/fields.tsx b/apps/portal/src/components/canvas/panel/fields.tsx index eca4496a..a63d6293 100644 --- a/apps/portal/src/components/canvas/panel/fields.tsx +++ b/apps/portal/src/components/canvas/panel/fields.tsx @@ -14,10 +14,12 @@ import { } from "@fluxify/components"; import { useParams } from "@tanstack/react-router"; import { useReactFlow } from "@xyflow/react"; -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import type { ReactNode } from "react"; import { withBasePath } from "@/constants/routes"; import { integrationService } from "@/services/integrations"; +import { customBlocksQuery } from "@/query/customBlocksQuery"; +import type { CustomBlockInputParam } from "./blocks/CustomBlockSettings"; import { useCanvasChanges } from "../changes/ChangesContext"; import type { BlockData } from "../types"; @@ -224,6 +226,43 @@ export function BlockCheckboxField({ ); } +/** + * On a custom block's canvas, the block's own `integration_selector` input + * parameters are offered alongside the project's integrations. Picking one + * writes `param:`, which the engine substitutes with whatever the calling + * block was configured with — so the concrete integration is chosen on the + * caller's side, not here. `blockId` is only present on the custom block route, + * which is what keeps this out of route canvases. + */ +function useCustomBlockParamIntegrations( + projectId: string, + customBlockId: string | undefined, + group: string, +) { + const { data: blocks } = customBlocksQuery.getAll.useQuery(projectId); + + return useMemo(() => { + if (!customBlockId) return undefined; + const block = blocks?.find((b) => b.id === customBlockId); + const params = Array.isArray(block?.inputParams) + ? (block.inputParams as CustomBlockInputParam[]) + : []; + const matching = params.filter( + (param) => param.type === "integration_selector" && param.group === group, + ); + if (matching.length === 0) return undefined; + return matching.map((param) => ({ + id: `param:${param.name}`, + name: param.label || param.name, + group, + variant: "Input parameter", + config: {}, + external: true, + hint: `Set by whoever places this block — the “${param.label || param.name}” input.`, + })); + }, [blocks, customBlockId, group]); +} + export type BlockIntegrationFieldProps = { blockId: string; data: BlockData; @@ -246,9 +285,17 @@ export function BlockIntegrationField({ }: BlockIntegrationFieldProps) { const { updateNodeData } = useReactFlow(); const { enabled: editable } = useCanvasChanges(); - const params = useParams({ strict: false }) as { projectId?: string }; + const params = useParams({ strict: false }) as { + projectId?: string; + blockId?: string; + }; const projectId = params?.projectId ?? ""; const selectedId = typeof data[name] === "string" ? (data[name] as string) : ""; + const injectedIntegrations = useCustomBlockParamIntegrations( + projectId, + params?.blockId, + group, + ); const loadIntegrations = useCallback(async () => { if (!projectId) return []; @@ -283,6 +330,7 @@ export function BlockIntegrationField({ group={group} selectedId={selectedId} loadIntegrations={loadIntegrations} + injectedIntegrations={injectedIntegrations} onSelect={(id) => { if (!editable) return; updateNodeData(blockId, { [name]: id }); diff --git a/apps/portal/src/components/customBlocks/CustomBlockSettingsModal.tsx b/apps/portal/src/components/customBlocks/CustomBlockSettingsModal.tsx new file mode 100644 index 00000000..4730910a --- /dev/null +++ b/apps/portal/src/components/customBlocks/CustomBlockSettingsModal.tsx @@ -0,0 +1,311 @@ +import { useMemo, useState } from "react"; +import { useNavigate } from "@tanstack/react-router"; +import { + Button, + CloseButton, + DeleteButton, + Input, + Label, + Modal, + Spinner, + Tabs, + TextField, + toast, +} from "@fluxify/components"; +import type { inputParamSchema } from "@fluxify/server/src/api/v1/custom-blocks/create/dto"; +import type { z } from "zod"; +import { customBlocksQuery } from "@/query/customBlocksQuery"; +import { showErrorNotification } from "@/lib/errorNotifier"; +import { ConfirmDialog } from "@/components/common/ConfirmDialog"; +import type { CustomBlockInputParam } from "@/components/canvas/panel/blocks/CustomBlockSettings"; +import { InputParamsEditor, validateInputParams } from "./InputParamsEditor"; +import { ICON_URL_MAX, IconPicker, type IconValue } from "./IconPicker"; + +/** + * Same shape as the route canvas' settings modal: everything editable about the + * block, reachable from the canvas it belongs to, one save for the lot. + */ +export function CustomBlockSettingsModal({ + projectId, + blockId, + isOpen, + onOpenChange, +}: { + projectId: string; + blockId: string; + isOpen: boolean; + onOpenChange: (open: boolean) => void; +}) { + const { data: blocks, isLoading } = customBlocksQuery.getAll.useQuery(projectId); + const block = blocks?.find((b) => b.id === blockId); + + return ( + + + + + {isLoading || !block ? ( +
+ {isLoading ? :

Custom block not found.

} +
+ ) : ( + onOpenChange(false)} + onClose={() => onOpenChange(false)} + /> + )} +
+
+
+
+ ); +} + +type BlockData = NonNullable< + ReturnType["data"] +>[number]; + +function CustomBlockSettingsForm({ + projectId, + block, + onSaved, + onClose, +}: { + projectId: string; + block: BlockData; + onSaved: () => void; + onClose: () => void; +}) { + const update = customBlocksQuery.update.mutation(projectId, block.id); + const readOnly = Boolean(block.sourceType && block.sourceType !== "user-defined"); + + const [label, setLabel] = useState(block.label); + const [description, setDescription] = useState(block.description ?? ""); + const [params, setParams] = useState( + Array.isArray(block.inputParams) ? (block.inputParams as CustomBlockInputParam[]) : [], + ); + const [iconValue, setIconValue] = useState({ + icon: (block.icon as IconValue["icon"]) ?? undefined, + iconUrl: block.iconUrl ?? undefined, + }); + const [tab, setTab] = useState("general"); + const [confirmDelete, setConfirmDelete] = useState(false); + const remove = customBlocksQuery.remove.mutation(projectId); + const navigate = useNavigate(); + // a plugin block is owned by its plugin; the API refuses to delete it + const canDelete = block.sourceType !== "plugin"; + + function deleteBlock() { + remove.mutate(block.id, { + onSuccess: () => { + toast.success("Custom block deleted"); + setConfirmDelete(false); + onClose(); + navigate({ to: "/$projectId/custom-blocks", params: { projectId } }); + }, + onError: (error) => showErrorNotification(error as Error), + }); + } + + const payload = useMemo( + () => ({ + label: label.trim(), + description, + icon: iconValue.icon, + iconUrl: iconValue.iconUrl, + inputParams: params as unknown as z.infer[], + }), + [label, description, params, iconValue], + ); + const [baseline] = useState(() => JSON.stringify(payload)); + const isDirty = JSON.stringify(payload) !== baseline; + const labelIsValid = label.trim().length > 0; + + function save() { + if ((iconValue.iconUrl?.length ?? 0) > ICON_URL_MAX) { + toast.danger("Icon image is too large"); + setTab("general"); + return; + } + const error = validateInputParams(params); + if (error) { + toast.danger(error); + setTab("inputs"); + return; + } + update.mutate(payload, { + onSuccess: () => { + toast.success("Custom block settings saved"); + onSaved(); + }, + onError: (error) => showErrorNotification(error as Error), + }); + } + + return ( + <> + +
+ + Custom block settings + +

{block.name}

+
+ +
+ + + setTab(String(key))} + className="flex h-full min-h-0 flex-row" + > + + General + Input parameters + {canDelete && Danger zone} + + + {/* the icon grid inside scrolls, the panel itself never does */} + + +
+

Identity

+

+ How this block shows up in the block picker and on a route canvas. + The name is fixed once created. +

+
+ + + + + + + + + + } + /> +
+ + +
+ +
+
+ + {canDelete && ( + +
+
+
+

{block.label}

+

{block.name}

+
+ setConfirmDelete(true)}> + Delete block + +
+
+
+ )} +
+
+ + + Delete {block.label}? This can't be undone. + + + + + {readOnly + ? `This block comes from a ${block.sourceType} source and isn't editable here.` + : isDirty + ? "Unsaved changes" + : "All changes saved"} + +
+ + {!readOnly && ( + + )} +
+
+ + ); +} + +function Section({ + title, + description, + children, +}: { + title: string; + description: string; + children: React.ReactNode; +}) { + return ( +
+
+

{title}

+

{description}

+
+ {children} +
+ ); +} diff --git a/apps/portal/src/components/customBlocks/CustomBlockSwitcher.tsx b/apps/portal/src/components/customBlocks/CustomBlockSwitcher.tsx new file mode 100644 index 00000000..7e959201 --- /dev/null +++ b/apps/portal/src/components/customBlocks/CustomBlockSwitcher.tsx @@ -0,0 +1,119 @@ +import { useNavigate } from "@tanstack/react-router"; +import { Button, ListBox, Select, Spinner } from "@fluxify/components"; +import { TbArrowLeft, TbChevronLeft, TbChevronRight } from "react-icons/tb"; +import { customBlocksQuery } from "@/query/customBlocksQuery"; + +/** + * Canvas header nav, same shape as the route canvas' switcher: back to the + * project's custom block list plus a stepper over the sibling blocks. + */ +export function CustomBlockSwitcher({ + projectId, + blockId, +}: { + projectId: string; + blockId: string; +}) { + const navigate = useNavigate(); + const { data, isLoading } = customBlocksQuery.getAll.useQuery(projectId); + const blocks = data ?? []; + const index = blocks.findIndex((block) => block.id === blockId); + const current = index >= 0 ? blocks[index] : undefined; + + function go(to: string) { + navigate({ + to: "/$projectId/custom-block-canvas/$blockId", + params: { projectId, blockId: to }, + }); + } + + function step(delta: number) { + const next = blocks[index + delta]; + if (next) go(next.id); + } + + return ( +
+ + + + + {isLoading ? ( + + ) : ( +
+ + + + + + + {index >= 0 && ( + + {index + 1} / {blocks.length} + + )} +
+ )} +
+ ); +} diff --git a/apps/portal/src/components/customBlocks/IconPicker.tsx b/apps/portal/src/components/customBlocks/IconPicker.tsx new file mode 100644 index 00000000..0236001b --- /dev/null +++ b/apps/portal/src/components/customBlocks/IconPicker.tsx @@ -0,0 +1,164 @@ +import { useMemo, useState } from "react"; +import { Button, Input, Label, Tabs, TextField } from "@fluxify/components"; +import { TbBox, TbSearch } from "react-icons/tb"; +import { BaseBlock } from "@/components/canvas/blocks/BaseBlock"; +import { PREMADE_ICON_NAMES, premadeIcon } from "./premadeIcons"; + +/** `iconUrl` is `text` in the DB but the API caps it — see custom-blocks/create/dto.ts. */ +export const ICON_URL_MAX = 68266; + +export type IconKind = "premade-list" | "custom"; + +export type IconValue = { + icon?: IconKind; + /** Premade icon name, or a URL / data URI when `icon` is `custom`. */ + iconUrl?: string; +}; + +/** The icon a block shows, for both the preview here and (later) the canvas. */ +export function CustomBlockIcon({ icon, iconUrl, size = 18 }: IconValue & { size?: number }) { + if (icon === "custom" && iconUrl) { + return ; + } + const Premade = icon === "premade-list" ? premadeIcon(iconUrl) : undefined; + return Premade ? : ; +} + +export function IconPicker({ + value, + onChange, + isDisabled, + previewName, + previewDescription, + header, +}: { + value: IconValue; + onChange: (next: IconValue) => void; + isDisabled?: boolean; + previewName: string; + previewDescription?: string; + /** Rendered in the left column, above the preview. */ + header?: React.ReactNode; +}) { + const [search, setSearch] = useState(""); + const custom = value.icon === "custom"; + + const results = useMemo(() => { + const q = search.trim().toLowerCase(); + if (!q) return PREMADE_ICON_NAMES; + return PREMADE_ICON_NAMES.filter((name) => name.includes(q)); + }, [search]); + + const customUrl = custom ? (value.iconUrl ?? "") : ""; + const tooLong = customUrl.length > ICON_URL_MAX; + + return ( +
+ {/* left 30%: what the block will actually look like */} +
+ {header} +
+
+ } + showToolbar={false} + /> +
+

+ {custom + ? customUrl + ? "Custom image" + : "No image yet — default icon" + : (value.iconUrl ?? "No icon selected — default icon")} +

+
+ {value.icon && ( + + )} +
+ + {/* right 70%: the picker itself, the only thing allowed to scroll */} + + onChange( + key === "custom" + ? { icon: "custom", iconUrl: "" } + : { icon: "premade-list", iconUrl: undefined }, + ) + } + className="flex min-h-0 w-[70%] flex-1 flex-col" + > + + Premade + URL / base64 + + + + + + + + {results.length === 0 ? ( +

+ No icon matches “{search}”. +

+ ) : ( +
+ {results.map((name) => { + const Icon = premadeIcon(name)!; + const selected = value.icon === "premade-list" && value.iconUrl === name; + return ( + + ); + })} +
+ )} +
+ + + onChange({ icon: "custom", iconUrl: next })} + isDisabled={isDisabled} + isInvalid={tooLong} + > + + +

+ {customUrl.length.toLocaleString()} / {ICON_URL_MAX.toLocaleString()} characters + {tooLong ? " — too large, use a smaller image or a URL" : ""} +

+
+
+
+
+ ); +} diff --git a/apps/portal/src/components/customBlocks/InputParamsEditor.tsx b/apps/portal/src/components/customBlocks/InputParamsEditor.tsx new file mode 100644 index 00000000..0bd62b00 --- /dev/null +++ b/apps/portal/src/components/customBlocks/InputParamsEditor.tsx @@ -0,0 +1,307 @@ +import { useCallback, useId } from "react"; +import { + ArrayEditor, + Button, + CustomSelect, + DeleteIconButton, + Input, + Label, + TextField, +} from "@fluxify/components"; +import { TbPlus } from "react-icons/tb"; +import type { CustomBlockInputParam } from "@/components/canvas/panel/blocks/CustomBlockSettings"; + +const PARAM_TYPE_OPTIONS = [ + { value: "text_input", label: "Text input" }, + { value: "checkbox", label: "Checkbox" }, + { value: "dropdown", label: "Dropdown" }, + { value: "array_editor", label: "Array editor" }, + { value: "integration_selector", label: "Integration selector" }, +]; + +const NAME_REGEX = /^[a-z0-9_]+$/; + +function emptyParam( + type: CustomBlockInputParam["type"], +): CustomBlockInputParam { + const base = { name: "", label: "", type, description: "" }; + if (type === "dropdown") return { ...base, options: [] }; + if (type === "integration_selector") return { ...base, group: "", tags: [] }; + return base; +} + +export function validateInputParams( + params: CustomBlockInputParam[], +): string | null { + const seen = new Set(); + for (const p of params) { + if (!p.name.trim() || !NAME_REGEX.test(p.name)) { + return `"${p.label || p.name || "unnamed"}": name must be lowercase letters, digits, underscores.`; + } + if (seen.has(p.name)) return `Duplicate parameter name "${p.name}".`; + seen.add(p.name); + if (!p.label.trim()) return `"${p.name}": label is required.`; + if (p.type === "dropdown") { + const options = p.options ?? []; + if (options.length === 0) return `"${p.name}": add at least one option.`; + for (const opt of options) { + const value = typeof opt === "string" ? opt : opt.value; + if (!value?.trim()) return `"${p.name}": option values can't be empty.`; + } + } + if (p.type === "integration_selector" && !p.group?.trim()) { + return `"${p.name}": integration group is required.`; + } + } + return null; +} + +export function InputParamsEditor({ + params, + onChange, + isDisabled, +}: { + params: CustomBlockInputParam[]; + onChange: (next: CustomBlockInputParam[]) => void; + isDisabled?: boolean; +}) { + const update = useCallback( + (index: number, patch: Partial) => { + onChange(params.map((p, i) => (i === index ? { ...p, ...patch } : p))); + }, + [params, onChange], + ); + + const remove = useCallback( + (index: number) => onChange(params.filter((_, i) => i !== index)), + [params, onChange], + ); + + const sampleName = + params.find((p) => p.name.trim())?.name.trim() ?? "channel"; + + const add = useCallback( + () => onChange([...params, emptyParam("text_input")]), + [params, onChange], + ); + + return ( +
+ {/* `params` is bound alongside `input` in every JS expression of a + custom block's graph — see compiler.ts `js()` */} +

+ In this block's JavaScript, read these as{" "} + params.{sampleName} — + available in every block of the canvas, not just the first.{" "} + input stays what it is + everywhere else: the previous block's output, or whatever the caller + passed in at the entrypoint. +

+ {params.length === 0 ? ( +
+ No input parameters yet. Add one so this block can be configured when + placed on a route. +
+ ) : ( + params.map((param, index) => ( + update(index, patch)} + onRemove={() => remove(index)} + /> + )) + )} + + {!isDisabled && ( + + )} +
+ ); +} + +function ParamRow({ + param, + isDisabled, + onChange, + onRemove, +}: { + param: CustomBlockInputParam; + isDisabled?: boolean; + onChange: (patch: Partial) => void; + onRemove: () => void; +}) { + const id = useId(); + + function changeType(type: string) { + const next = emptyParam(type as CustomBlockInputParam["type"]); + onChange({ + ...next, + name: param.name, + label: param.label, + description: param.description, + }); + } + + return ( +
+
+ onChange({ name })} + > + + + + onChange({ label })} + > + + + +
+ +
+ +
+ + onChange({ description })} + > + + + + + {param.type === "dropdown" && ( + onChange({ options })} + /> + )} + + {param.type === "integration_selector" && ( +
+ onChange({ group })} + > + + + + onChange({ tags })} + addButtonLabel="Add tag" + placeholder="e.g. postgres" + /> +
+ )} +
+ ); +} + +function DropdownOptionsEditor({ + id, + options, + isDisabled, + onChange, +}: { + id: string; + options: (string | { label: string; value: string })[]; + isDisabled?: boolean; + onChange: (next: { label: string; value: string }[]) => void; +}) { + const normalized = options.map((o) => + typeof o === "string" ? { label: o, value: o } : o, + ); + + function update( + index: number, + patch: Partial<{ label: string; value: string }>, + ) { + onChange(normalized.map((o, i) => (i === index ? { ...o, ...patch } : o))); + } + + function remove(index: number) { + onChange(normalized.filter((_, i) => i !== index)); + } + + return ( +
+ + {normalized.length === 0 && ( +

No options added yet.

+ )} + {normalized.map((opt, index) => ( +
+ update(index, { label })} + > + + + update(index, { value })} + > + + + remove(index)} + /> +
+ ))} + {!isDisabled && ( + + )} +
+ ); +} diff --git a/apps/portal/src/components/customBlocks/premadeIcons.tsx b/apps/portal/src/components/customBlocks/premadeIcons.tsx new file mode 100644 index 00000000..7277016c --- /dev/null +++ b/apps/portal/src/components/customBlocks/premadeIcons.tsx @@ -0,0 +1,517 @@ +import type { IconType } from "react-icons"; +import { + TbActivity, + TbAdjustments, + TbAlertTriangle, + TbAntenna, + TbApi, + TbArchive, + TbAtom, + TbBarcode, + TbBattery, + TbBell, + TbBellRinging, + TbBinary, + TbBinaryTree, + TbBluetooth, + TbBolt, + TbBook, + TbBookmark, + TbBox, + TbBoxMultiple, + TbBraces, + TbBrackets, + TbBrain, + TbBrandApple, + TbBrandAws, + TbBrandAzure, + TbBrandBitbucket, + TbBrandCloudflare, + TbBrandDebian, + TbBrandDiscord, + TbBrandDjango, + TbBrandDocker, + TbBrandFacebook, + TbBrandFigma, + TbBrandFirebase, + TbBrandGithub, + TbBrandGitlab, + TbBrandGmail, + TbBrandGolang, + TbBrandGoogle, + TbBrandInstagram, + TbBrandJavascript, + TbBrandJira, + TbBrandLaravel, + TbBrandLinkedin, + TbBrandMongodb, + TbBrandMysql, + TbBrandNextjs, + TbBrandNodejs, + TbBrandNotion, + TbBrandNpm, + TbBrandOpenai, + TbBrandPaypal, + TbBrandPhp, + TbBrandPython, + TbBrandReact, + TbBrandRust, + TbBrandSlack, + TbBrandStripe, + TbBrandSupabase, + TbBrandTelegram, + TbBrandTrello, + TbBrandTypescript, + TbBrandUbuntu, + TbBrandVercel, + TbBrandVue, + TbBrandWhatsapp, + TbBrandWindows, + TbBrandX, + TbBrandYoutube, + TbBriefcase, + TbBroadcast, + TbBug, + TbBuildingSkyscraper, + TbBuildingStore, + TbCalendar, + TbCalendarEvent, + TbCamera, + TbCertificate, + TbChartBar, + TbChartDots, + TbChartLine, + TbChartPie, + TbCheck, + TbClipboard, + TbClipboardCheck, + TbClock, + TbClockHour4, + TbCloud, + TbCloudComputing, + TbCloudDataConnection, + TbCloudDownload, + TbCloudLock, + TbCloudOff, + TbCloudUpload, + TbCode, + TbCompass, + TbCopy, + TbCpu, + TbCpu2, + TbCreditCard, + TbCurrencyDollar, + TbDatabase, + TbDatabaseCog, + TbDatabaseExport, + TbDatabaseImport, + TbDatabasePlus, + TbDatabaseSearch, + TbDeviceDesktop, + TbDeviceLaptop, + TbDeviceMobile, + TbDeviceTablet, + TbDeviceWatch, + TbDownload, + TbDroplet, + TbEdit, + TbEye, + TbFile, + TbFileCode, + TbFileExport, + TbFileImport, + TbFileText, + TbFileZip, + TbFiles, + TbFilter, + TbFingerprint, + TbFlag, + TbFlame, + TbFlask, + TbFolder, + TbFolderOpen, + TbFolderPlus, + TbGauge, + TbGift, + TbGitBranch, + TbGitCommit, + TbGitCompare, + TbGitFork, + TbGitMerge, + TbGitPullRequest, + TbGlobe, + TbHeadphones, + TbHeart, + TbHelp, + TbHierarchy, + TbHistory, + TbHourglass, + TbId, + TbInbox, + TbInfoCircle, + TbJson, + TbKey, + TbKeyboard, + TbLanguage, + TbLayersIntersect, + TbLeaf, + TbLifebuoy, + TbLink, + TbLoader, + TbLock, + TbLockOpen, + TbLogin, + TbLogout, + TbMail, + TbMailForward, + TbMap2, + TbMapPin, + TbMathFunction, + TbMessage, + TbMessages, + TbMicrophone, + TbMicroscope, + TbMoodSmile, + TbMouse, + TbNetwork, + TbNotebook, + TbPackage, + TbPackageExport, + TbPackages, + TbPaperclip, + TbPhoto, + TbPin, + TbPlayerPause, + TbPlayerPlay, + TbPlayerStop, + TbPlug, + TbPlugConnected, + TbPrinter, + TbPuzzle, + TbQrcode, + TbRefresh, + TbRegex, + TbRepeat, + TbReportAnalytics, + TbRobot, + TbRocket, + TbRouter, + TbSatellite, + TbScan, + TbSchool, + TbSearch, + TbSend, + TbServer, + TbServer2, + TbServerBolt, + TbServerCog, + TbSettings, + TbShare, + TbShield, + TbShieldCheck, + TbShieldLock, + TbShoppingCart, + TbSitemap, + TbSparkles, + TbSpeakerphone, + TbSql, + TbStack2, + TbStar, + TbTable, + TbTableExport, + TbTag, + TbTags, + TbTargetArrow, + TbTemperature, + TbTerminal2, + TbThumbUp, + TbTicket, + TbTool, + TbTools, + TbTopologyStar, + TbTrash, + TbTrendingUp, + TbTruckDelivery, + TbUnlink, + TbUpload, + TbUsb, + TbUser, + TbUserCheck, + TbUserPlus, + TbUserShield, + TbUsers, + TbVariable, + TbVideo, + TbVolume, + TbWand, + TbWebhook, + TbWifi, + TbWorld, + TbWorldWww, + TbX,} from "react-icons/tb"; +import type { PremadeIconType } from "@fluxify/server/src/api/v1/custom-blocks/shared"; + +/** + * The premade icon set offered in the custom block settings. Typed against the + * server enum, so adding one here without adding it there (or vice versa) is a + * typecheck error rather than a runtime "Invalid premade icon name" 400. + */ +export const PREMADE_ICONS: Record = { + "code": TbCode, + "terminal-2": TbTerminal2, + "bug": TbBug, + "braces": TbBraces, + "brackets": TbBrackets, + "binary": TbBinary, + "api": TbApi, + "webhook": TbWebhook, + "math-function": TbMathFunction, + "variable": TbVariable, + "regex": TbRegex, + "json": TbJson, + "sql": TbSql, + "file-code": TbFileCode, + "git-branch": TbGitBranch, + "git-commit": TbGitCommit, + "git-merge": TbGitMerge, + "git-pull-request": TbGitPullRequest, + "git-fork": TbGitFork, + "brand-github": TbBrandGithub, + "brand-gitlab": TbBrandGitlab, + "brand-bitbucket": TbBrandBitbucket, + "package": TbPackage, + "packages": TbPackages, + "brand-npm": TbBrandNpm, + "brand-docker": TbBrandDocker, + "brand-aws": TbBrandAws, + "brand-azure": TbBrandAzure, + "brand-google": TbBrandGoogle, + "brand-vercel": TbBrandVercel, + "brand-cloudflare": TbBrandCloudflare, + "brand-firebase": TbBrandFirebase, + "brand-supabase": TbBrandSupabase, + "brand-stripe": TbBrandStripe, + "brand-slack": TbBrandSlack, + "brand-discord": TbBrandDiscord, + "brand-telegram": TbBrandTelegram, + "brand-whatsapp": TbBrandWhatsapp, + "brand-x": TbBrandX, + "brand-facebook": TbBrandFacebook, + "brand-instagram": TbBrandInstagram, + "brand-linkedin": TbBrandLinkedin, + "brand-youtube": TbBrandYoutube, + "brand-gmail": TbBrandGmail, + "brand-openai": TbBrandOpenai, + "brand-python": TbBrandPython, + "brand-javascript": TbBrandJavascript, + "brand-typescript": TbBrandTypescript, + "brand-react": TbBrandReact, + "brand-nextjs": TbBrandNextjs, + "brand-vue": TbBrandVue, + "brand-nodejs": TbBrandNodejs, + "brand-golang": TbBrandGolang, + "brand-rust": TbBrandRust, + "brand-php": TbBrandPhp, + "brand-laravel": TbBrandLaravel, + "brand-django": TbBrandDjango, + "brand-mysql": TbBrandMysql, + "brand-mongodb": TbBrandMongodb, + "brand-figma": TbBrandFigma, + "brand-notion": TbBrandNotion, + "brand-jira": TbBrandJira, + "brand-trello": TbBrandTrello, + "brand-paypal": TbBrandPaypal, + "brand-apple": TbBrandApple, + "brand-windows": TbBrandWindows, + "brand-ubuntu": TbBrandUbuntu, + "brand-debian": TbBrandDebian, + "cloud": TbCloud, + "cloud-upload": TbCloudUpload, + "cloud-download": TbCloudDownload, + "cloud-computing": TbCloudComputing, + "cloud-lock": TbCloudLock, + "cloud-data-connection": TbCloudDataConnection, + "server": TbServer, + "server-2": TbServer2, + "server-bolt": TbServerBolt, + "server-cog": TbServerCog, + "database": TbDatabase, + "database-export": TbDatabaseExport, + "database-import": TbDatabaseImport, + "database-search": TbDatabaseSearch, + "database-plus": TbDatabasePlus, + "database-cog": TbDatabaseCog, + "table": TbTable, + "table-export": TbTableExport, + "topology-star": TbTopologyStar, + "network": TbNetwork, + "router": TbRouter, + "wifi": TbWifi, + "world": TbWorld, + "world-www": TbWorldWww, + "globe": TbGlobe, + "link": TbLink, + "unlink": TbUnlink, + "lock": TbLock, + "lock-open": TbLockOpen, + "key": TbKey, + "shield": TbShield, + "shield-lock": TbShieldLock, + "shield-check": TbShieldCheck, + "fingerprint": TbFingerprint, + "certificate": TbCertificate, + "user": TbUser, + "users": TbUsers, + "user-plus": TbUserPlus, + "user-check": TbUserCheck, + "user-shield": TbUserShield, + "id": TbId, + "login": TbLogin, + "logout": TbLogout, + "mail": TbMail, + "mail-forward": TbMailForward, + "message": TbMessage, + "messages": TbMessages, + "bell": TbBell, + "bell-ringing": TbBellRinging, + "send": TbSend, + "inbox": TbInbox, + "calendar": TbCalendar, + "calendar-event": TbCalendarEvent, + "clock": TbClock, + "clock-hour-4": TbClockHour4, + "history": TbHistory, + "hourglass": TbHourglass, + "refresh": TbRefresh, + "repeat": TbRepeat, + "player-play": TbPlayerPlay, + "player-pause": TbPlayerPause, + "player-stop": TbPlayerStop, + "rocket": TbRocket, + "bolt": TbBolt, + "flame": TbFlame, + "sparkles": TbSparkles, + "robot": TbRobot, + "brain": TbBrain, + "cpu": TbCpu, + "cpu-2": TbCpu2, + "device-desktop": TbDeviceDesktop, + "device-mobile": TbDeviceMobile, + "device-laptop": TbDeviceLaptop, + "device-tablet": TbDeviceTablet, + "device-watch": TbDeviceWatch, + "printer": TbPrinter, + "camera": TbCamera, + "video": TbVideo, + "microphone": TbMicrophone, + "headphones": TbHeadphones, + "volume": TbVolume, + "photo": TbPhoto, + "file": TbFile, + "file-text": TbFileText, + "file-import": TbFileImport, + "file-export": TbFileExport, + "file-zip": TbFileZip, + "files": TbFiles, + "folder": TbFolder, + "folder-open": TbFolderOpen, + "folder-plus": TbFolderPlus, + "clipboard": TbClipboard, + "clipboard-check": TbClipboardCheck, + "notebook": TbNotebook, + "book": TbBook, + "bookmark": TbBookmark, + "tag": TbTag, + "tags": TbTags, + "chart-bar": TbChartBar, + "chart-line": TbChartLine, + "chart-pie": TbChartPie, + "chart-dots": TbChartDots, + "report-analytics": TbReportAnalytics, + "activity": TbActivity, + "gauge": TbGauge, + "target-arrow": TbTargetArrow, + "trending-up": TbTrendingUp, + "currency-dollar": TbCurrencyDollar, + "credit-card": TbCreditCard, + "shopping-cart": TbShoppingCart, + "building-store": TbBuildingStore, + "truck-delivery": TbTruckDelivery, + "package-export": TbPackageExport, + "box": TbBox, + "box-multiple": TbBoxMultiple, + "stack-2": TbStack2, + "layers-intersect": TbLayersIntersect, + "sitemap": TbSitemap, + "hierarchy": TbHierarchy, + "binary-tree": TbBinaryTree, + "git-compare": TbGitCompare, + "filter": TbFilter, + "search": TbSearch, + "adjustments": TbAdjustments, + "settings": TbSettings, + "tool": TbTool, + "tools": TbTools, + "wand": TbWand, + "puzzle": TbPuzzle, + "plug": TbPlug, + "plug-connected": TbPlugConnected, + "battery": TbBattery, + "trash": TbTrash, + "archive": TbArchive, + "copy": TbCopy, + "edit": TbEdit, + "check": TbCheck, + "x": TbX, + "alert-triangle": TbAlertTriangle, + "info-circle": TbInfoCircle, + "help": TbHelp, + "eye": TbEye, + "map-pin": TbMapPin, + "map-2": TbMap2, + "compass": TbCompass, + "flag": TbFlag, + "star": TbStar, + "heart": TbHeart, + "thumb-up": TbThumbUp, + "mood-smile": TbMoodSmile, + "language": TbLanguage, + "qrcode": TbQrcode, + "barcode": TbBarcode, + "scan": TbScan, + "temperature": TbTemperature, + "droplet": TbDroplet, + "leaf": TbLeaf, + "building-skyscraper": TbBuildingSkyscraper, + "briefcase": TbBriefcase, + "school": TbSchool, + "microscope": TbMicroscope, + "flask": TbFlask, + "atom": TbAtom, + "ticket": TbTicket, + "gift": TbGift, + "paperclip": TbPaperclip, + "pin": TbPin, + "lifebuoy": TbLifebuoy, + "speakerphone": TbSpeakerphone, + "broadcast": TbBroadcast, + "antenna": TbAntenna, + "satellite": TbSatellite, + "keyboard": TbKeyboard, + "mouse": TbMouse, + "usb": TbUsb, + "bluetooth": TbBluetooth, + "share": TbShare, + "download": TbDownload, + "upload": TbUpload, + "cloud-off": TbCloudOff, + "loader": TbLoader, + "chart": TbChartBar, + "image": TbPhoto, +}; + +export const PREMADE_ICON_NAMES = Object.keys(PREMADE_ICONS) as PremadeIconType[]; + +export function premadeIcon(name: string | null | undefined): IconType | undefined { + return name ? PREMADE_ICONS[name as PremadeIconType] : undefined; +} diff --git a/apps/portal/src/components/routes/RouteSettingsModal.tsx b/apps/portal/src/components/routes/RouteSettingsModal.tsx index 334f1ed4..e72c2527 100644 --- a/apps/portal/src/components/routes/RouteSettingsModal.tsx +++ b/apps/portal/src/components/routes/RouteSettingsModal.tsx @@ -24,6 +24,9 @@ import type { HttpMethod } from "@fluxify/server/src/db/schema"; import { DEFAULT_CONTENT_TYPES } from "@fluxify/server/src/lib/routeConfig"; import { ROUTE_REGEX } from "@fluxify/server/src/api/v1/routes/constants"; import { TbAlertTriangle } from "react-icons/tb"; +import { useNavigate } from "@tanstack/react-router"; +import { DeleteButton } from "@fluxify/components"; +import { ConfirmDialog } from "@/components/common/ConfirmDialog"; import { routesQuery } from "@/query/routesQuery"; import { projectSettingsKeysQuery } from "@/query/projectSettingsKeysQuery"; import { showErrorNotification } from "@/lib/errorNotifier"; @@ -141,6 +144,24 @@ function RouteSettingsForm({ () => paramConfigFrom(route.paramsSchema), ); const [tab, setTab] = useState("general"); + const [confirmDelete, setConfirmDelete] = useState(false); + const remove = routesQuery.remove.mutation(); + const navigate = useNavigate(); + + function deleteRoute() { + remove.mutate(route.id, { + onSuccess: () => { + toast.success("Route deleted"); + setConfirmDelete(false); + onClose(); + navigate({ + to: "/$projectId/routes", + params: { projectId: route.projectId }, + }); + }, + onError: (error) => showErrorNotification(error as Error), + }); + } const pathParams = useMemo(() => extractPathParams(path), [path]); const hasBody = METHODS_WITH_BODY.includes(method); @@ -198,6 +219,7 @@ function RouteSettingsForm({ { id: "query", label: "Query" }, hasBody && { id: "body", label: "Body" }, { id: "advanced", label: "Advanced" }, + { id: "danger", label: "Danger zone" }, ].filter(Boolean) as { id: string; label: string }[], [pathParams.length, hasBody], ); @@ -412,9 +434,46 @@ function RouteSettingsForm({ )} + + +
+
+
+

+ {route.method} {route.path} +

+

+ Any client calling it will start getting a 404. +

+
+ setConfirmDelete(true)}> + Delete route + +
+
+
+ + Delete{" "} + + {route.method} {route.path} + + ? This can't be undone. + + {isDirty ? "Unsaved changes" : "All changes saved"} diff --git a/apps/portal/src/query/customBlocksQuery.ts b/apps/portal/src/query/customBlocksQuery.ts index 5c378cb3..ff829fd7 100644 --- a/apps/portal/src/query/customBlocksQuery.ts +++ b/apps/portal/src/query/customBlocksQuery.ts @@ -54,4 +54,14 @@ export const customBlocksQuery = { }); }, }, + update: { + mutation(projectId: string, id: string) { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (body: z.infer) => + customBlocksService.update(id, body), + onSuccess: () => qc.invalidateQueries({ queryKey: key(projectId) }), + }); + }, + }, }; diff --git a/apps/portal/src/routes/_authed/$projectId/custom-blocks.tsx b/apps/portal/src/routes/_authed/$projectId/custom-blocks.tsx index 1f7e6152..de27e213 100644 --- a/apps/portal/src/routes/_authed/$projectId/custom-blocks.tsx +++ b/apps/portal/src/routes/_authed/$projectId/custom-blocks.tsx @@ -1,22 +1,21 @@ -import { useState } from "react"; +import { useMemo, useState } from "react"; import { createFileRoute, useNavigate } from "@tanstack/react-router"; import { Button, - Card, - CloseButton, DeleteIconButton, Input, Label, - Modal, Spinner, TextField, toast, } from "@fluxify/components"; -import { TbPlus } from "react-icons/tb"; +import { TbBoxMultiple, TbPlus, TbSearch } from "react-icons/tb"; import { customBlocksQuery } from "@/query/customBlocksQuery"; import { showErrorNotification } from "@/lib/errorNotifier"; import { ConfirmDialog } from "@/components/common/ConfirmDialog"; import { createRouteHead } from "@/lib/seo"; +import { BaseBlock } from "@/components/canvas/blocks/BaseBlock"; +import { CustomBlockIcon, type IconValue } from "@/components/customBlocks/IconPicker"; export const Route = createFileRoute("/_authed/$projectId/custom-blocks")({ head: createRouteHead( @@ -26,7 +25,9 @@ export const Route = createFileRoute("/_authed/$projectId/custom-blocks")({ component: CustomBlocksPage, }); -type Block = { id: string; label: string; name: string; description?: string | null }; +type Block = NonNullable< + ReturnType["data"] +>[number]; function CustomBlocksPage() { const { projectId } = Route.useParams(); @@ -34,15 +35,49 @@ function CustomBlocksPage() { const remove = customBlocksQuery.remove.mutation(projectId); const navigate = useNavigate(); const [pendingDelete, setPendingDelete] = useState(null); + const [search, setSearch] = useState(""); + + const blocks = useMemo(() => { + const q = search.trim().toLowerCase(); + if (!data) return []; + if (!q) return data; + return data.filter((b) => + [b.label, b.name, b.description ?? ""].some((v) => v.toLowerCase().includes(q)), + ); + }, [data, search]); + + function openCanvas(blockId: string) { + navigate({ + to: "/$projectId/custom-block-canvas/$blockId", + params: { projectId, blockId }, + }); + } return ( -
-
+
+

Custom Blocks

-

Reusable blocks for your flows.

+

+ Reusable blocks for your flows. Click one to open its canvas. +

+
+
+ {data && data.length > 0 && ( + + + + + )} +
-
{isLoading ? ( @@ -52,38 +87,77 @@ function CustomBlocksPage() { ) : isError ? (

Couldn't load custom blocks.

) : !data || data.length === 0 ? ( -

No custom blocks yet.

+ } + title="No custom blocks yet" + description="A custom block wraps a piece of flow you want to reuse across routes." + /> + ) : blocks.length === 0 ? ( + } + title={`No block matches “${search}”`} + description="Try a different name or description." + /> ) : ( -
- {data.map((block) => ( - - - {block.label} - {block.name} - - -

- {block.description || "No description"} -

-
- - - setPendingDelete(block as Block)} + showToolbar={false} /> - -
+
+ +
+
+

{block.name}

+

+ {Array.isArray(block.inputParams) ? block.inputParams.length : 0} input + {(Array.isArray(block.inputParams) ? block.inputParams.length : 0) === 1 + ? "" + : "s"} + {block.sourceType && block.sourceType !== "user-defined" + ? ` · ${block.sourceType}` + : ""} +

+
+ {block.sourceType !== "plugin" && ( +
e.stopPropagation()} + onKeyDown={(e) => e.stopPropagation()} + > + setPendingDelete(block)} + /> +
+ )} +
+
))}
)} @@ -110,82 +184,21 @@ function CustomBlocksPage() { ); } -function CreateBlockButton({ projectId }: { projectId: string }) { - const create = customBlocksQuery.create.mutation(projectId); - const [open, setOpen] = useState(false); - const [name, setName] = useState(""); - const [label, setLabel] = useState(""); - const [description, setDescription] = useState(""); - - function reset() { - setName(""); - setLabel(""); - setDescription(""); - } - - function submit(e: React.FormEvent) { - e.preventDefault(); - create.mutate( - { name, label, description, projectId }, - { - onSuccess: () => { - toast.success("Custom block created"); - reset(); - setOpen(false); - }, - onError: (err) => showErrorNotification(err as Error), - }, - ); - } - +function EmptyState({ + icon, + title, + description, +}: { + icon: React.ReactNode; + title: string; + description: string; +}) { return ( - - - - - - - - -
- Create a custom block -

- You can build its logic on the canvas afterwards. -

-
- -
-
- -
- - - - - - - - - - - - -
-
- - - - -
-
-
-
-
+
+ {icon} +

{title}

+

{description}

+
); } + diff --git a/apps/portal/src/routes/_authed/$projectId/custom-blocks_.new.tsx b/apps/portal/src/routes/_authed/$projectId/custom-blocks_.new.tsx new file mode 100644 index 00000000..8d7d9bfb --- /dev/null +++ b/apps/portal/src/routes/_authed/$projectId/custom-blocks_.new.tsx @@ -0,0 +1,347 @@ +import { useMemo, useState } from "react"; +import { createFileRoute, useNavigate } from "@tanstack/react-router"; +import { Button, cn, Input, Label, TextField, toast } from "@fluxify/components"; +import { TbArrowLeft, TbArrowRight, TbCheck } from "react-icons/tb"; +import type { inputParamSchema } from "@fluxify/server/src/api/v1/custom-blocks/create/dto"; +import type { z } from "zod"; +import { customBlocksQuery } from "@/query/customBlocksQuery"; +import { showErrorNotification } from "@/lib/errorNotifier"; +import { createRouteHead } from "@/lib/seo"; +import type { CustomBlockInputParam } from "@/components/canvas/panel/blocks/CustomBlockSettings"; +import { + InputParamsEditor, + validateInputParams, +} from "@/components/customBlocks/InputParamsEditor"; +import { + ICON_URL_MAX, + IconPicker, + type IconValue, +} from "@/components/customBlocks/IconPicker"; + +export const Route = createFileRoute("/_authed/$projectId/custom-blocks_/new")({ + head: createRouteHead( + "New Custom Block", + "Create a reusable custom block: identity, icon and input parameters.", + ), + component: CreateCustomBlockPage, +}); + +const NAME_REGEX = /^[a-z0-9_]+$/; + +/** `Send Slack message` → `send_slack_message`, the shape the API accepts. */ +function slugify(value: string) { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_+|_+$/g, ""); +} + +const STEPS = [ + { key: "basics", label: "Basics" }, + { key: "icon", label: "Icon" }, + { key: "inputs", label: "Inputs" }, + { key: "review", label: "Review" }, +] as const; + +function CreateCustomBlockPage() { + const { projectId } = Route.useParams(); + const navigate = useNavigate(); + const create = customBlocksQuery.create.mutation(projectId); + + const [label, setLabel] = useState(""); + const [name, setName] = useState(""); + // until the user edits `name` by hand it tracks the label + const [nameTouched, setNameTouched] = useState(false); + const [description, setDescription] = useState(""); + const [iconValue, setIconValue] = useState({}); + const [params, setParams] = useState([]); + const [step, setStep] = useState(0); + + const nameIsValid = NAME_REGEX.test(name); + const basicsValid = label.trim().length >= 2 && nameIsValid; + const iconTooLong = (iconValue.iconUrl?.length ?? 0) > ICON_URL_MAX; + const paramsError = useMemo(() => validateInputParams(params), [params]); + + const current = Math.min(step, STEPS.length - 1); + const currentKey = STEPS[current].key; + const isLast = current === STEPS.length - 1; + + function submit() { + if (iconTooLong) { + toast.danger("Icon image is too large"); + setStep(1); + return; + } + if (paramsError) { + toast.danger(paramsError); + setStep(2); + return; + } + create.mutate( + { + projectId, + name, + label: label.trim(), + description, + icon: iconValue.icon, + iconUrl: iconValue.iconUrl, + inputParams: params as unknown as z.infer[], + }, + { + onSuccess: (created) => { + toast.success("Custom block created"); + navigate({ + to: "/$projectId/custom-block-canvas/$blockId", + params: { projectId, blockId: created.id }, + }); + }, + onError: (error) => showErrorNotification(error as Error), + }, + ); + } + + return ( + // full height of the scroll container so only the step body scrolls and the + // Back/Next bar stays pinned +
+
+ +
+

+ Create a custom block +

+

+ A reusable piece of flow you can drop onto any route canvas. +

+
+
+ + + +
+ + +
+ {currentKey === "basics" && ( +
+ { + setLabel(next); + if (!nameTouched) setName(slugify(next)); + }} + > + + + + + { + setNameTouched(true); + setName(next); + }} + isInvalid={name.length > 0 && !nameIsValid} + > + + +

+ How flows refer to this block. Lowercase letters, digits and + underscores. Can't be changed later. +

+
+ + + + + +
+ )} + + {currentKey === "icon" && ( +
+ +
+ )} + + {currentKey === "inputs" && ( +
+ +
+ )} + + {currentKey === "review" && ( +
+ + + + + +
+ )} +
+
+ +
+ + {isLast ? ( + + ) : ( + + )} +
+
+ ); +} + +function StepHeading({ + current, + total, + title, + description, +}: { + current: number; + total: number; + title: string; + description: string; +}) { + return ( +
+

+ Step {current + 1} of {total} +

+

+ {title} +

+

{description}

+
+ ); +} + +function SummaryItem({ + label, + value, + mono, +}: { + label: string; + value: string; + mono?: boolean; +}) { + return ( +
+
{label}
+
+ {value || "—"} +
+
+ ); +} diff --git a/apps/portal/src/routes/_authed/$projectId_.custom-block-canvas.$blockId.tsx b/apps/portal/src/routes/_authed/$projectId_.custom-block-canvas.$blockId.tsx index b055da69..dc381648 100644 --- a/apps/portal/src/routes/_authed/$projectId_.custom-block-canvas.$blockId.tsx +++ b/apps/portal/src/routes/_authed/$projectId_.custom-block-canvas.$blockId.tsx @@ -1,7 +1,12 @@ +import { useState } from "react"; import { createFileRoute } from "@tanstack/react-router"; +import { Button } from "@fluxify/components"; +import { TbSettings } from "react-icons/tb"; import { customBlocksQuery } from "@/query/customBlocksQuery"; import { customBlocksService } from "@/services/customBlocks"; import { CanvasWorkbench } from "@/components/canvas"; +import { CustomBlockSettingsModal } from "@/components/customBlocks/CustomBlockSettingsModal"; +import { CustomBlockSwitcher } from "@/components/customBlocks/CustomBlockSwitcher"; import { createRouteHead } from "@/lib/seo"; export const Route = createFileRoute( @@ -15,15 +20,34 @@ export const Route = createFileRoute( }); function CustomBlockCanvasPage() { - const { blockId } = Route.useParams(); + const { projectId, blockId } = Route.useParams(); const save = customBlocksQuery.saveCanvas.mutation(blockId); + const [settingsOpen, setSettingsOpen] = useState(false); return ( - customBlocksService.getCanvasItems(blockId)} - save={(payload) => save.mutateAsync(payload)} - /> + <> + customBlocksService.getCanvasItems(blockId)} + save={(payload) => save.mutateAsync(payload)} + headerLeft={} + headerActions={ + + } + /> + {/* mounted only while open: the form seeds its state from the loaded block */} + {settingsOpen && ( + + )} + ); } diff --git a/apps/portal/src/services/customBlocks.ts b/apps/portal/src/services/customBlocks.ts index d8165334..60be6073 100644 --- a/apps/portal/src/services/customBlocks.ts +++ b/apps/portal/src/services/customBlocks.ts @@ -4,6 +4,10 @@ import { requestBodySchema as createRequestSchema, responseSchema as createResponseSchema, } from "@fluxify/server/src/api/v1/custom-blocks/create/dto"; +import { + requestBodySchema as updateRequestSchema, + responseSchema as updateResponseSchema, +} from "@fluxify/server/src/api/v1/custom-blocks/update/dto"; import { httpClient } from "@/lib/http"; import { canvasEndpoints } from "./canvas"; @@ -23,7 +27,15 @@ export const customBlocksService = { async delete(id: string) { await httpClient.delete(`${baseUrl}/${id}`); }, + async update( + id: string, + data: z.infer, + ): Promise> { + const result = await httpClient.put(`${baseUrl}/${id}`, data); + return result.data; + }, // a custom block's canvas is stored and served exactly like a route's ...canvasEndpoints(baseUrl), createRequestSchema, + updateRequestSchema, }; diff --git a/apps/server/deployments/compiledWorker.ts b/apps/server/deployments/compiledWorker.ts index ea3643bf..00225638 100644 --- a/apps/server/deployments/compiledWorker.ts +++ b/apps/server/deployments/compiledWorker.ts @@ -22,6 +22,9 @@ import { asyncExecutorLimitsFromEnv } from "../src/modules/requestRouter/asyncEx import { executionRuntimeEnvironment } from "../src/modules/requestRouter/executionEnvironment"; import type { ArtifactEntry } from "../src/modules/requestRouter/compiledRuntime"; import { closeNats } from "../src/db/nats"; +import { startJobWorker } from "../src/modules/jobs/consumer"; +import { enqueueJob } from "../src/modules/jobs/publisher"; +import type { JobEnvelope } from "../src/modules/jobs/types"; import { OTLP_AUTH_HEADER_NAME, OTLP_AUTH_HEADER_VALUE, @@ -145,6 +148,7 @@ function spawnExecution() { if (execution !== process) return; execution = undefined; markNotReady(); + failPendingJobs("execution process exited mid-job"); watchdog.setEnabled(timeoutPolicyEnabled()); if (shuttingDown) return; logger.error( @@ -159,12 +163,49 @@ function spawnExecution() { child.send({ type: "bootstrap", bootstrap } satisfies ExecutionMessage); } +/** + * Jobs handed to the execution process, waiting on its reply. The broker's ack + * is driven by that reply, so a child that dies mid-job must reject its pending + * work — otherwise the consumer sits on the message until the ack wait elapses. + */ +const pendingJobs = new Map< + string, + { resolve: () => void; reject: (error: Error) => void } +>(); + +function runJobInExecution(job: JobEnvelope) { + return new Promise((resolve, reject) => { + if (!execution) return reject(new Error("execution process is not running")); + pendingJobs.set(job.id, { resolve, reject }); + execution.send({ type: "job", job } satisfies ExecutionMessage); + }); +} + +function failPendingJobs(reason: string) { + for (const [, pending] of pendingJobs) pending.reject(new Error(reason)); + pendingJobs.clear(); +} + function onExecutionEvent(event: ExecutionEvent) { switch (event.type) { case "ready": markReady(); logger.info("execution process ready", "WORKER.execution"); return; + case "job-finished": { + const pending = pendingJobs.get(event.id); + pendingJobs.delete(event.id); + if (!pending) return; + return event.error ? pending.reject(new Error(event.error)) : pending.resolve(); + } + case "enqueue-job": + // user code queued work; failures are logged, the graph moved on already + return void enqueueJob(event.job).catch((error) => + logger.error( + `failed to queue ${event.job.kind}/${event.job.target}: ${String(error)}`, + "WORKER.jobs", + ), + ); case "heartbeat": return watchdog.heartbeat(); case "execution-started": @@ -192,6 +233,19 @@ await artifactWatch.initialized; spawnExecution(); synchronizeMonitoring(); +// Background work for this project. Separate from the request path on purpose: +// a queued job must not compete with traffic for the same acceptance. +await startJobWorker({ + projectId: WORKER_PROJECT_ID, + handle: runJobInExecution, + concurrency: Number(getEnv("JOBS_CONCURRENCY")) || undefined, + ackWaitMs: Number(getEnv("JOBS_ACK_WAIT_MS")) || undefined, + maxDeliver: Number(getEnv("JOBS_MAX_DELIVER")) || undefined, + retryDelayMs: Number(getEnv("JOBS_RETRY_DELAY_MS")) || undefined, +}).catch((error) => + logger.error(`job worker failed to start: ${String(error)}`, "WORKER.jobs"), +); + function evaluateTimeouts() { const timedOut = watchdog.findTimedOut(); if (!timedOut || !execution || terminatingForTimeout) return; diff --git a/apps/server/src/api/v1/custom-blocks/create/dto.ts b/apps/server/src/api/v1/custom-blocks/create/dto.ts index ebe8e127..d75d0cd3 100644 --- a/apps/server/src/api/v1/custom-blocks/create/dto.ts +++ b/apps/server/src/api/v1/custom-blocks/create/dto.ts @@ -5,16 +5,19 @@ export const inputParamSchema = z.discriminatedUnion("type", [ type: z.literal("text_input"), name: z.string().regex(/^[a-z0-9_]+$/), label: z.string(), + description: z.string().optional(), }), z.object({ type: z.literal("checkbox"), name: z.string().regex(/^[a-z0-9_]+$/), label: z.string(), + description: z.string().optional(), }), z.object({ type: z.literal("array_editor"), name: z.string().regex(/^[a-z0-9_]+$/), label: z.string(), + description: z.string().optional(), }), z.object({ type: z.literal("integration_selector"), @@ -23,6 +26,7 @@ export const inputParamSchema = z.discriminatedUnion("type", [ group: z.string(), variant: z.string().optional(), tags: z.array(z.string()).default([]), + description: z.string().optional(), }), z.object({ type: z.literal("dropdown"), @@ -34,6 +38,7 @@ export const inputParamSchema = z.discriminatedUnion("type", [ value: z.string(), }) ), + description: z.string().optional(), }), ]); diff --git a/apps/server/src/api/v1/custom-blocks/create/repository.ts b/apps/server/src/api/v1/custom-blocks/create/repository.ts index 9f8b3a85..d910c747 100644 --- a/apps/server/src/api/v1/custom-blocks/create/repository.ts +++ b/apps/server/src/api/v1/custom-blocks/create/repository.ts @@ -38,7 +38,9 @@ export async function createDependencies( id: id3, customBlockId, type: BlockTypes.errorHandler, - position: { x: 25, y: 0 }, + // a block is ~168px wide; anything less than that sits on top of the + // entrypoint on the current node design + position: { x: -240, y: 0 }, data: { next: "", retryAfterFail: false, diff --git a/apps/server/src/api/v1/custom-blocks/delete/service.ts b/apps/server/src/api/v1/custom-blocks/delete/service.ts index 6243be34..2b11507d 100644 --- a/apps/server/src/api/v1/custom-blocks/delete/service.ts +++ b/apps/server/src/api/v1/custom-blocks/delete/service.ts @@ -8,12 +8,14 @@ import { ForbiddenError } from "../../../../errors/forbidError"; import { hasProjectAccess } from "../../../auth/common"; import { AuthACL } from "../../../../db/schema"; import { User } from "better-auth"; +import { dropCustomBlock } from "../../../../modules/compiler/service"; export default async function handleRequest( id: string, user: User & { isSystemAdmin: boolean }, acl: AuthACL[] ): Promise> { + let projectId: string | undefined; await db.transaction(async (tx) => { const existingBlock = await getCustomBlockById(id, tx); if (!existingBlock) { @@ -28,9 +30,14 @@ export default async function handleRequest( throw new ForbiddenError("Cannot delete a custom block originating from a plugin"); } + projectId = existingBlock.projectId!; await deleteCustomBlock(id, tx); - await publishMessage(CHAN_ON_CUSTOM_BLOCK_CHANGE, id); }); + // same as routes: the compiler can't resolve the project of a row that no + // longer exists, so the stale artifact would keep being served from KV + if (projectId) await dropCustomBlock(projectId, id); + await publishMessage(CHAN_ON_CUSTOM_BLOCK_CHANGE, id); + return { id }; } diff --git a/apps/server/src/api/v1/custom-blocks/delete/tests/delete.test.ts b/apps/server/src/api/v1/custom-blocks/delete/tests/delete.test.ts index 5206e2d0..afad30de 100644 --- a/apps/server/src/api/v1/custom-blocks/delete/tests/delete.test.ts +++ b/apps/server/src/api/v1/custom-blocks/delete/tests/delete.test.ts @@ -10,6 +10,10 @@ mock.module("../../../../../db/redis", () => ({ CHAN_ON_CUSTOM_BLOCK_CHANGE: "chan:on-custom-block-change", })); +mock.module("../../../../../modules/compiler/service", () => ({ + dropCustomBlock: mock(), +})); + import handleRequest from "../service"; import * as repo from "../repository"; import { NotFoundError } from "../../../../../errors/notFoundError"; diff --git a/apps/server/src/api/v1/custom-blocks/shared.ts b/apps/server/src/api/v1/custom-blocks/shared.ts index 03f6d644..e2daf637 100644 --- a/apps/server/src/api/v1/custom-blocks/shared.ts +++ b/apps/server/src/api/v1/custom-blocks/shared.ts @@ -1,19 +1,262 @@ import { z } from "zod"; +/** + * Names of the premade icons the portal ships. Kept in sync with + * `apps/portal/src/components/customBlocks/premadeIcons.tsx` — the portal map is + * typed as `Record`, so a mismatch fails the typecheck. + */ export const premadeIconEnum = z.enum([ - "database", + "code", + "terminal-2", + "bug", + "braces", + "brackets", + "binary", "api", - "robot", - "tool", - "user", + "webhook", + "math-function", + "variable", + "regex", + "json", + "sql", + "file-code", + "git-branch", + "git-commit", + "git-merge", + "git-pull-request", + "git-fork", + "brand-github", + "brand-gitlab", + "brand-bitbucket", + "package", + "packages", + "brand-npm", + "brand-docker", + "brand-aws", + "brand-azure", + "brand-google", + "brand-vercel", + "brand-cloudflare", + "brand-firebase", + "brand-supabase", + "brand-stripe", + "brand-slack", + "brand-discord", + "brand-telegram", + "brand-whatsapp", + "brand-x", + "brand-facebook", + "brand-instagram", + "brand-linkedin", + "brand-youtube", + "brand-gmail", + "brand-openai", + "brand-python", + "brand-javascript", + "brand-typescript", + "brand-react", + "brand-nextjs", + "brand-vue", + "brand-nodejs", + "brand-golang", + "brand-rust", + "brand-php", + "brand-laravel", + "brand-django", + "brand-mysql", + "brand-mongodb", + "brand-figma", + "brand-notion", + "brand-jira", + "brand-trello", + "brand-paypal", + "brand-apple", + "brand-windows", + "brand-ubuntu", + "brand-debian", + "cloud", + "cloud-upload", + "cloud-download", + "cloud-computing", + "cloud-lock", + "cloud-data-connection", + "server", + "server-2", + "server-bolt", + "server-cog", + "database", + "database-export", + "database-import", + "database-search", + "database-plus", + "database-cog", + "table", + "table-export", + "topology-star", + "network", + "router", + "wifi", + "world", + "world-www", + "globe", + "link", + "unlink", "lock", + "lock-open", + "key", + "shield", + "shield-lock", + "shield-check", + "fingerprint", + "certificate", + "user", + "users", + "user-plus", + "user-check", + "user-shield", + "id", + "login", + "logout", "mail", + "mail-forward", + "message", + "messages", + "bell", + "bell-ringing", + "send", + "inbox", "calendar", + "calendar-event", + "clock", + "clock-hour-4", + "history", + "hourglass", + "refresh", + "repeat", + "player-play", + "player-pause", + "player-stop", + "rocket", + "bolt", + "flame", + "sparkles", + "robot", + "brain", + "cpu", + "cpu-2", + "device-desktop", + "device-mobile", + "device-laptop", + "device-tablet", + "device-watch", + "printer", + "camera", + "video", + "microphone", + "headphones", + "volume", + "photo", "file", + "file-text", + "file-import", + "file-export", + "file-zip", + "files", "folder", + "folder-open", + "folder-plus", + "clipboard", + "clipboard-check", + "notebook", + "book", + "bookmark", + "tag", + "tags", + "chart-bar", + "chart-line", + "chart-pie", + "chart-dots", + "report-analytics", + "activity", + "gauge", + "target-arrow", + "trending-up", + "currency-dollar", + "credit-card", + "shopping-cart", + "building-store", + "truck-delivery", + "package-export", + "box", + "box-multiple", + "stack-2", + "layers-intersect", + "sitemap", + "hierarchy", + "binary-tree", + "git-compare", + "filter", + "search", + "adjustments", + "settings", + "tool", + "tools", + "wand", + "puzzle", + "plug", + "plug-connected", + "battery", + "trash", + "archive", + "copy", + "edit", + "check", + "x", + "alert-triangle", + "info-circle", + "help", + "eye", + "map-pin", + "map-2", + "compass", + "flag", + "star", + "heart", + "thumb-up", + "mood-smile", + "language", + "qrcode", + "barcode", + "scan", + "temperature", + "droplet", + "leaf", + "building-skyscraper", + "briefcase", + "school", + "microscope", + "flask", + "atom", + "ticket", + "gift", + "paperclip", + "pin", + "lifebuoy", + "speakerphone", + "broadcast", + "antenna", + "satellite", + "keyboard", + "mouse", + "usb", + "bluetooth", + "share", + "download", + "upload", + "cloud-off", + "loader", "chart", "image", - "code", ]); export type PremadeIconType = z.infer; diff --git a/apps/server/src/api/v1/routes/create/repository.ts b/apps/server/src/api/v1/routes/create/repository.ts index 801738c6..ced53f06 100644 --- a/apps/server/src/api/v1/routes/create/repository.ts +++ b/apps/server/src/api/v1/routes/create/repository.ts @@ -47,7 +47,7 @@ export async function createDependency( type: "response", position: { x: 0, - y: 100, + y: 160, }, data: { httpCode: "200", @@ -57,8 +57,10 @@ export async function createDependency( id: id3, routeId, type: BlockTypes.errorHandler, + // a block is ~168px wide; anything less than that sits on top of the + // entrypoint on the current node design position: { - x: -100, + x: -240, y: 0, }, data: { diff --git a/apps/server/src/api/v1/routes/delete/service.ts b/apps/server/src/api/v1/routes/delete/service.ts index 0e415ab6..2dc720fb 100644 --- a/apps/server/src/api/v1/routes/delete/service.ts +++ b/apps/server/src/api/v1/routes/delete/service.ts @@ -7,11 +7,13 @@ import { AuthACL } from "../../../../db/schema"; import { canAccessProject } from "../../../../lib/acl"; import { NotFoundError } from "../../../../errors/notFoundError"; import { ForbiddenError } from "../../../../errors/forbidError"; +import { dropRoute } from "../../../../modules/compiler/service"; export default async function handleRequest( id: string, acl: AuthACL[] = [] ): Promise> { + let projectId: string | undefined; await db.transaction(async (tx) => { const existingRoute = await findRouteById(id, tx); if (!existingRoute) { @@ -25,8 +27,13 @@ export default async function handleRequest( if (!canAccess) { throw new ForbiddenError(); } + projectId = existingRoute.projectId!; await deleteRoute(id, tx); }); + // the compiler resolves a route's project from the database, so once the row + // is gone it can no longer work out which artifact key to drop — workers would + // keep serving the deleted route from KV. Drop it here, where we still know. + if (projectId) await dropRoute(projectId, id); await publishMessage(CHAN_ON_ROUTE_CHANGE, id); return ""; } diff --git a/apps/server/src/lib/env.ts b/apps/server/src/lib/env.ts index a83efd12..a4fc5087 100644 --- a/apps/server/src/lib/env.ts +++ b/apps/server/src/lib/env.ts @@ -149,6 +149,28 @@ export const serverEnvSchema = baseEnvSchema.extend({ .enum(["true", "false"]) .optional() .describe("Enable admin control-plane API endpoints ('true' | 'false')"), + + // Background job queue. One stream serves every kind of job, so these tune + // the worker rather than any one feature. + JOBS_CONCURRENCY: z + .string() + .optional() + .describe("Background jobs this worker runs at once (default 5)"), + + JOBS_ACK_WAIT_MS: z + .string() + .optional() + .describe("How long a job may run before it is assumed lost (default 300000)"), + + JOBS_MAX_DELIVER: z + .string() + .optional() + .describe("Attempts before a failing job is dropped (default 5)"), + + JOBS_RETRY_DELAY_MS: z + .string() + .optional() + .describe("Delay before a failed job is redelivered (default 10000)"), }); // Extract keys using keyof diff --git a/apps/server/src/modules/canvas/repository.ts b/apps/server/src/modules/canvas/repository.ts index cde7356e..6fb0095d 100644 --- a/apps/server/src/modules/canvas/repository.ts +++ b/apps/server/src/modules/canvas/repository.ts @@ -232,9 +232,17 @@ export async function getCustomBlockNames( parent: CanvasParent, tx?: DbTransactionType, ): Promise { + return (await getProjectCustomBlocks(parent, tx)).map((row) => row.name); +} + +/** Same set as `getCustomBlockNames`, with the id needed to read each canvas. */ +export async function getProjectCustomBlocks( + parent: CanvasParent, + tx?: DbTransactionType, +): Promise<{ id: string; name: string }[]> { const table = parent.type === "route" ? routesEntity : customBlocksListEntity; - const rows = await (tx ?? db) - .select({ name: customBlocksListEntity.name }) + return await (tx ?? db) + .select({ id: customBlocksListEntity.id, name: customBlocksListEntity.name }) .from(customBlocksListEntity) .where( eq( @@ -245,7 +253,30 @@ export async function getCustomBlockNames( .where(eq(table.id, parent.id)), ), ); - return rows.map((row) => row.name); +} + +/** + * Which custom block each of these canvases holds a block of — the call graph a + * recursion check walks. Only the type matters, so this stays one narrow read + * however big the canvases are. + */ +export async function getCustomBlockCalls( + customBlockIds: string[], + tx?: DbTransactionType, +): Promise<{ parentId: string; type: string }[]> { + if (!customBlockIds.length) return []; + const rows = await (tx ?? db) + .selectDistinct({ parentId: blocksEntity.parentId, type: blocksEntity.type }) + .from(blocksEntity) + .where( + and( + eq(blocksEntity.parentType, "custom_block"), + inArray(blocksEntity.parentId, customBlockIds), + ), + ); + return rows.flatMap((row) => + row.parentId && row.type ? [{ parentId: row.parentId, type: row.type }] : [], + ); } export async function touchParent( diff --git a/apps/server/src/modules/canvas/service.ts b/apps/server/src/modules/canvas/service.ts index e834fdd6..b1cb66a1 100644 --- a/apps/server/src/modules/canvas/service.ts +++ b/apps/server/src/modules/canvas/service.ts @@ -14,7 +14,9 @@ import { deleteStructuralBlocks, getBlocks, getBlocksCountByType, + getCustomBlockCalls, getCustomBlockNames, + getProjectCustomBlocks, getEdges, parentExists, parentKeys, @@ -143,6 +145,71 @@ async function assertBlockTypesExist( ); } +/** + * A custom block that reaches itself — directly or through a chain of other + * custom blocks — never terminates: `lib.invoke` would call into the block it is + * already inside. The editor hides the block from its own picker, but a cycle + * can still be closed from the other end (B gains a call to A while A already + * calls B), so the save that closes it is refused here. + * + * Only runs for a custom block's own canvas; a route can call anything. + */ +async function assertNoCustomBlockRecursion( + parent: CanvasParent, + data: CanvasChanges, + deleteBlockIds: string[], + tx: DbTransactionType, +) { + if (parent.type !== "custom_block") return; + + const blocks = await getProjectCustomBlocks(parent, tx); + const self = blocks.find((b) => b.id === parent.id); + if (!self) return; + const nameById = new Map(blocks.map((b) => [b.id, b.name])); + const names = new Set(blocks.map((b) => b.name)); + + // this canvas as it will be once the delta lands; the rest as stored + const deleted = new Set(deleteBlockIds); + const incoming = new Set(data.changes.blocks.map((b) => b.id)); + const selfCalls = new Set( + [ + ...data.changes.blocks.map((b) => b.type), + ...(await getBlocks(parent, tx)) + .filter((b) => !deleted.has(b.id) && !incoming.has(b.id)) + .map((b) => b.type ?? ""), + ].filter((type) => names.has(type)), + ); + + const calls = new Map>([[self.name, selfCalls]]); + for (const row of await getCustomBlockCalls( + blocks.map((b) => b.id), + tx, + )) { + const caller = nameById.get(row.parentId); + if (!caller || caller === self.name || !names.has(row.type)) continue; + const set = calls.get(caller) ?? new Set(); + set.add(row.type); + calls.set(caller, set); + } + + // shortest path back to self, so the message names the actual chain + const queue: string[][] = [[self.name]]; + const seen = new Set(); + while (queue.length > 0) { + const path = queue.shift()!; + for (const callee of calls.get(path[path.length - 1]) ?? []) { + if (callee === self.name) { + throw new ConflictError( + `Custom block "${self.name}" cannot call itself — recursion is not allowed (${[...path, callee].join(" → ")}).`, + ); + } + if (seen.has(callee)) continue; + seen.add(callee); + queue.push([...path, callee]); + } + } +} + /** * A verified agent run's canvas can arrive after the entrypoint/error handler * it means to replace already exist under different ids (e.g. the route was @@ -217,6 +284,7 @@ export async function saveCanvas( // a tx nests as a savepoint, so the outer transaction still decides the outcome await (outer ?? db).transaction(async (tx) => { await assertBlockTypesExist(parent, data, tx); + await assertNoCustomBlockRecursion(parent, data, deleteBlockIds, tx); await assertEdgeTargetsExist(parent, data, deleteBlockIds, tx); await assertCanvasHasNoCycles( parent, diff --git a/apps/server/src/modules/canvas/tests/service.spec.ts b/apps/server/src/modules/canvas/tests/service.spec.ts index 48c70b72..b4911317 100644 --- a/apps/server/src/modules/canvas/tests/service.spec.ts +++ b/apps/server/src/modules/canvas/tests/service.spec.ts @@ -42,6 +42,8 @@ const delEdges = spyOn(repository, "deleteEdges"); const delStructural = spyOn(repository, "deleteStructuralBlocks"); const getEdges = spyOn(repository, "getEdges"); const customBlockNames = spyOn(repository, "getCustomBlockNames"); +const projectCustomBlocks = spyOn(repository, "getProjectCustomBlocks"); +const customBlockCalls = spyOn(repository, "getCustomBlockCalls"); /** `changes` with its single block retyped */ const withBlockType = (type: string) => ({ @@ -66,6 +68,8 @@ describe("canvas saveCanvas", () => { } parentExists.mockResolvedValue(true); customBlockNames.mockResolvedValue([]); + projectCustomBlocks.mockResolvedValue([]); + customBlockCalls.mockResolvedValue([]); }); it("rejects a block type that is neither built-in nor a custom block", async () => { @@ -185,6 +189,63 @@ describe("canvas saveCanvas", () => { expect(upsertEdges).not.toHaveBeenCalled(); }); + describe("custom block recursion", () => { + const project = [ + { id: "cb-1", name: "audit" }, + { id: "cb-2", name: "logger" }, + ]; + /** `changes` carrying one instance of the named custom block */ + const calling = (type: string) => ({ + ...changes, + actionsToPerform: { blocks: [], edges: [] }, + changes: { + blocks: [{ id: "b1", type, data: {}, position: { x: 0, y: 0 } }], + edges: [], + }, + }); + + beforeEach(() => { + projectCustomBlocks.mockResolvedValue(project as any); + customBlockNames.mockResolvedValue(project.map((b) => b.name)); + }); + + it("rejects a custom block placed on its own canvas", async () => { + await expect( + saveCanvas({ type: "custom_block", id: "cb-1" }, calling("audit"), ["p1"]), + ).rejects.toThrow(/cannot call itself/); + expect(upsertBlocks).not.toHaveBeenCalled(); + }); + + it("rejects a chain that comes back around — logger already calls audit", async () => { + customBlockCalls.mockResolvedValue([ + { parentId: "cb-2", type: "audit" }, + ] as any); + + await expect( + saveCanvas({ type: "custom_block", id: "cb-1" }, calling("logger"), ["p1"]), + ).rejects.toThrow(/audit → logger → audit/); + }); + + it("allows a one-way call between two custom blocks", async () => { + await saveCanvas({ type: "custom_block", id: "cb-1" }, calling("logger"), [ + "p1", + ]); + + expect(upsertBlocks).toHaveBeenCalled(); + }); + + it("leaves route canvases alone — a route is never re-entered", async () => { + customBlockCalls.mockResolvedValue([ + { parentId: "cb-2", type: "audit" }, + { parentId: "cb-1", type: "logger" }, + ] as any); + + await saveCanvas({ type: "route", id: "r-1" }, calling("audit"), ["p1"]); + + expect(upsertBlocks).toHaveBeenCalled(); + }); + }); + it("rejects a duplicate structural block when the caller is not the AI harness", async () => { spyOn(repository, "getBlocksCountByType").mockResolvedValue([ { count: 2, type: "entrypoint" }, diff --git a/apps/server/src/modules/compiler/service.ts b/apps/server/src/modules/compiler/service.ts index 19dd418c..2cfa475a 100644 --- a/apps/server/src/modules/compiler/service.ts +++ b/apps/server/src/modules/compiler/service.ts @@ -2,6 +2,9 @@ import { logger } from "@fluxify/common"; import { BlockTypes, compileGraph, + hasCustomBlock, + registerCompiledCustomBlock, + unregisterCustomBlock, type BlockDTOType, type EdgeDTOSchemaType, } from "@fluxify/blocks"; @@ -120,6 +123,10 @@ export async function compileRoute(routeId: string) { return; } + // a route that calls a custom block only emits if that block is in this + // process's library — the artifact in KV is for workers, not for us + await ensureCustomBlocksRegistered(route.projectId!); + const { blocks, edges } = await loadGraph({ type: "route", id: routeId }); const { source } = compileGraph(blocks, edges); @@ -150,6 +157,66 @@ export async function dropRoute(projectId: string, routeId: string) { await deleteArtifact(routeKey(projectId, routeId)); } +/** custom blocks being compiled right now — see `ensureCustomBlocksRegistered` */ +const inFlight = new Set(); +/** what each custom block id is registered as, so a rename or delete can undo it */ +const registeredNames = new Map(); + +function registerLocally(id: string, name: string, source: string) { + const previous = registeredNames.get(id); + // a rename would otherwise leave the old name resolving to this block forever + if (previous && previous !== name) unregisterCustomBlock(previous); + registerCompiledCustomBlock(name, source); + registeredNames.set(id, name); +} + +function unregisterLocally(id: string) { + const name = registeredNames.get(id); + if (!name) return; + unregisterCustomBlock(name); + registeredNames.delete(id); +} + +/** + * Makes sure every custom block of a project is in this process's library + * before something that may call one is compiled. + * + * Compiling publishes an artifact for the workers; it does not make the block + * callable here, and `compileGraph` resolves a non-builtin type by asking the + * library. Without this, a route compile that happens before the block's own + * compile — a route saved after a restart, a fresh consumer — fails with + * "No codegen for block type". + * + * Order is discovered rather than declared: a block that calls another is + * retried once its callee lands. Cycles are impossible (the canvas save + * refuses them), so the fixpoint always terminates. + */ +async function ensureCustomBlocksRegistered(projectId: string) { + const rows = await db + .select({ id: customBlocksListEntity.id, name: customBlocksListEntity.name }) + .from(customBlocksListEntity) + .where(eq(customBlocksListEntity.projectId, projectId)); + + let pending = rows.filter( + (row) => !hasCustomBlock(row.name) && !inFlight.has(row.id), + ); + while (pending.length > 0) { + const failed: typeof pending = []; + let lastError: unknown; + for (const row of pending) { + try { + await compileCustomBlock(row.id); + } catch (error) { + lastError = error; + failed.push(row); + } + } + // nothing compiled this pass: the failures are real, not ordering + if (failed.length === pending.length) throw lastError; + pending = failed; + } +} + /** compile one custom block; a deleted one is dropped from the library */ export async function compileCustomBlock(id: string) { const [block] = await db @@ -163,12 +230,24 @@ export async function compileCustomBlock(id: string) { if (!block) { logger.info(`[compiler] dropping custom block ${id}`, "COMPILER"); + unregisterLocally(id); return; } - const { blocks, edges } = await loadGraph({ type: "custom_block", id }); - // `param:` placeholders resolve from the invocation, not from a caller's data - const { source } = compileGraph(blocks, edges, { asCustomBlock: true }); + // a custom block may call another one; same library requirement as a route + inFlight.add(id); + let source: string; + try { + await ensureCustomBlocksRegistered(block.projectId!); + const { blocks, edges } = await loadGraph({ type: "custom_block", id }); + // `param:` placeholders resolve from the invocation, not from a caller's data + ({ source } = compileGraph(blocks, edges, { asCustomBlock: true })); + } finally { + inFlight.delete(id); + } + // the compiler is also a consumer of its own output: the next route to call + // this block resolves it from here + registerLocally(block.id, block.name, source); const artifact: CustomBlockArtifact = { id: block.id, @@ -182,6 +261,7 @@ export async function compileCustomBlock(id: string) { } export async function dropCustomBlock(projectId: string, id: string) { + unregisterLocally(id); await deleteArtifact(customBlockKey(projectId, id)); } diff --git a/apps/server/src/modules/jobs/consumer.ts b/apps/server/src/modules/jobs/consumer.ts new file mode 100644 index 00000000..404779d5 --- /dev/null +++ b/apps/server/src/modules/jobs/consumer.ts @@ -0,0 +1,167 @@ +import { logger } from "@fluxify/common"; +import { AckPolicy, RetentionPolicy, StringCodec, type JsMsg } from "nats"; +import { natsConnection } from "../../db/nats"; +import { + JOBS_STREAM, + JOBS_SUBJECTS, + jobConsumerName, + projectJobFilter, +} from "./subjects"; +import type { JobEnvelope } from "./types"; + +/** + * The job worker's transport half: one durable pull consumer on the shared + * stream, filtered to this deployment's project. Running the work is the + * caller's `handle` — this file only decides what gets acked, retried or + * dropped. + * + * A work-queue stream removes a message once it is acked, so a restart never + * replays yesterday's jobs. Everything below is tunable because a queued custom + * block and a nightly cron want very different ack waits. + */ + +export type JobWorkerOptions = { + /** Project this deployment serves, or "*" for every project. */ + projectId: string; + /** Runs the job. Resolve to ack, reject to retry. */ + handle: (job: JobEnvelope) => Promise; + /** Jobs in flight at once. */ + concurrency?: number; + /** How long a job may run before the broker assumes the worker died. */ + ackWaitMs?: number; + /** Attempts before the job is dropped and logged as dead. */ + maxDeliver?: number; + /** Wait before a failed job is redelivered. */ + retryDelayMs?: number; + /** How long an unclaimed job stays on the stream. */ + maxAgeMs?: number; +}; + +const sc = StringCodec(); +const DEFAULTS = { + concurrency: 5, + ackWaitMs: 5 * 60_000, + maxDeliver: 5, + retryDelayMs: 10_000, + maxAgeMs: 7 * 24 * 60 * 60_000, +}; + +let running = false; + +export async function startJobWorker(options: JobWorkerOptions) { + if (running) return; + const config: Required = { + ...DEFAULTS, + ...stripUndefined(options), + projectId: options.projectId, + handle: options.handle, + }; + const nc = natsConnection(); + const jsm = await nc.jetstreamManager(); + + await ensureStream(jsm, config.maxAgeMs); + await ensureConsumer(jsm, config); + + const consumer = await nc + .jetstream() + .consumers.get(JOBS_STREAM, jobConsumerName(config.projectId)); + const messages = await consumer.consume({ max_messages: config.concurrency }); + running = true; + logger.info( + `[jobs] worker listening on ${projectJobFilter(config.projectId)}`, + "JOBS", + ); + + void (async () => { + const inFlight = new Set>(); + for await (const message of messages) { + // Bound the concurrency ourselves: `max_messages` limits what the server + // pushes, not what we start. + if (inFlight.size >= config.concurrency) await Promise.race(inFlight); + const task = settle(message, config).finally(() => inFlight.delete(task)); + inFlight.add(task); + } + await Promise.allSettled(inFlight); + })(); +} + +async function settle(message: JsMsg, config: Required) { + let job: JobEnvelope | undefined; + try { + job = JSON.parse(sc.decode(message.data)) as JobEnvelope; + job.attempt = message.info.redeliveryCount; + await config.handle(job); + message.ack(); + } catch (error) { + const label = job ? `${job.kind}/${job.target}` : message.subject; + // Unparseable, or a kind nobody handles: retrying changes nothing. + if (!job || isPermanent(error)) { + logger.error(`[jobs] dropping ${label}: ${String(error)}`, "JOBS"); + return message.term(); + } + if (message.info.redeliveryCount >= config.maxDeliver) { + logger.error( + `[jobs] ${label} failed ${message.info.redeliveryCount} times, giving up: ${String(error)}`, + "JOBS", + ); + return message.term(); + } + logger.warn( + `[jobs] ${label} failed (attempt ${message.info.redeliveryCount}), retrying: ${String(error)}`, + "JOBS", + ); + message.nak(config.retryDelayMs); + } +} + +/** A handler can opt a failure out of retries by naming it. */ +function isPermanent(error: unknown) { + return (error as Error)?.name === "UnknownJobKindError"; +} + +function stripUndefined(value: T): Partial { + return Object.fromEntries( + Object.entries(value).filter(([, v]) => v !== undefined), + ) as Partial; +} + +async function ensureStream(jsm: any, maxAgeMs: number) { + const spec = { + name: JOBS_STREAM, + subjects: [JOBS_SUBJECTS], + // a job is work, not history: once acked it leaves the stream + retention: RetentionPolicy.Workqueue, + max_age: maxAgeMs * 1_000_000, // ns + // publisher dedupe window for `msgID` + duplicate_window: 2 * 60 * 1_000_000_000, + }; + try { + await jsm.streams.add(spec); + } catch { + await jsm.streams.update(JOBS_STREAM, { + subjects: spec.subjects, + max_age: spec.max_age, + }); + } +} + +async function ensureConsumer(jsm: any, config: Required) { + const spec = { + durable_name: jobConsumerName(config.projectId), + ack_policy: AckPolicy.Explicit, + ack_wait: config.ackWaitMs * 1_000_000, // ns + max_deliver: config.maxDeliver, + filter_subject: projectJobFilter(config.projectId), + max_ack_pending: config.concurrency, + }; + try { + await jsm.consumers.add(JOBS_STREAM, spec); + } catch { + // already exists — pick up changed limits without losing pending work + await jsm.consumers.update(JOBS_STREAM, spec.durable_name, { + ack_wait: spec.ack_wait, + max_deliver: spec.max_deliver, + max_ack_pending: spec.max_ack_pending, + }); + } +} diff --git a/apps/server/src/modules/jobs/customBlockJob.ts b/apps/server/src/modules/jobs/customBlockJob.ts new file mode 100644 index 00000000..ebdc15d4 --- /dev/null +++ b/apps/server/src/modules/jobs/customBlockJob.ts @@ -0,0 +1,35 @@ +import { + CUSTOM_BLOCK_JOB, + invokeCustomBlock, + type CustomBlockArgs, +} from "@fluxify/blocks"; +import { logger } from "@fluxify/common"; +import { createJobContext } from "../requestRouter/service"; +import { registerJobHandler } from "./registry"; + +/** + * Runs a custom block that its caller queued instead of awaiting. + * + * Registered on the execution process — the one that holds the compiled block + * library and can run user code. It throws on failure so the consumer retries; + * the caller is long gone, so the queue is the only thing that can. + */ +export function registerCustomBlockJobHandler() { + registerJobHandler(CUSTOM_BLOCK_JOB, async (job) => { + const context = createJobContext({ + id: job.id, + projectId: job.projectId, + target: job.target, + }); + try { + await invokeCustomBlock( + context, + job.target, + (job.payload as CustomBlockArgs) ?? { params: {} }, + ); + logger.info(`[jobs] ran custom block ${job.target}`, "JOBS.custom-block"); + } finally { + context.dbFactory?.dispose(); + } + }); +} diff --git a/apps/server/src/modules/jobs/publisher.ts b/apps/server/src/modules/jobs/publisher.ts new file mode 100644 index 00000000..63397ea5 --- /dev/null +++ b/apps/server/src/modules/jobs/publisher.ts @@ -0,0 +1,37 @@ +import { logger } from "@fluxify/common"; +import { StringCodec } from "nats"; +import { natsConnection } from "../../db/nats"; +import { jobSubject } from "./subjects"; +import type { JobEnvelope } from "./types"; + +const sc = StringCodec(); + +export type JobInput = Omit & { + /** Supply one to make a retry of the same logical work collapse. */ + id?: string; +}; + +/** + * Publishes to JetStream, so the job outlives the process that queued it. The + * message id is the broker's dedupe key within its duplicate window: publishing + * the same id twice (a retried request, a redelivered upstream message) enqueues + * the work once. + * + * Throws rather than logging: the caller asked for durable work, and a queue + * that swallows failures is worse than one that refuses them. + */ +export async function enqueueJob(input: JobInput): Promise { + const job: JobEnvelope = { + ...input, + id: input.id ?? crypto.randomUUID(), + enqueuedAt: new Date().toISOString(), + }; + const subject = jobSubject(job.projectId, job.kind); + + await natsConnection() + .jetstream() + .publish(subject, sc.encode(JSON.stringify(job)), { msgID: job.id }); + + logger.debug(`[jobs] queued ${subject} (${job.target})`, "JOBS.publish"); + return job; +} diff --git a/apps/server/src/modules/jobs/registry.ts b/apps/server/src/modules/jobs/registry.ts new file mode 100644 index 00000000..8da69a4e --- /dev/null +++ b/apps/server/src/modules/jobs/registry.ts @@ -0,0 +1,34 @@ +import type { JobEnvelope, JobHandler } from "./types"; + +/** + * What each job kind means, on whichever process actually runs the work. + * + * Kept apart from the consumer on purpose: the transport (stream, acks, + * redelivery) belongs to the process that owns the NATS connection, while the + * handlers belong to the process that owns user code. Adding a cron or workflow + * kind later is one `registerJobHandler` call and no transport changes. + */ + +const handlers = new Map(); + +export function registerJobHandler(kind: string, handler: JobHandler) { + handlers.set(kind, handler); +} + +export function jobKinds() { + return [...handlers.keys()]; +} + +/** Thrown for a kind nobody handles — retrying that can never succeed. */ +export class UnknownJobKindError extends Error { + constructor(kind: string) { + super(`No handler registered for job kind "${kind}"`); + this.name = "UnknownJobKindError"; + } +} + +export async function runJob(job: JobEnvelope) { + const handler = handlers.get(job.kind); + if (!handler) throw new UnknownJobKindError(job.kind); + await handler(job); +} diff --git a/apps/server/src/modules/jobs/subjects.ts b/apps/server/src/modules/jobs/subjects.ts new file mode 100644 index 00000000..66729345 --- /dev/null +++ b/apps/server/src/modules/jobs/subjects.ts @@ -0,0 +1,41 @@ +/** + * Naming for the job queue. + * + * ONE stream carries every kind of background work — queued custom blocks + * today, crons/workflows/scheduled jobs later. A stream per feature would mean + * a pile of mostly-idle streams and consumers to operate, so kinds are a subject + * token instead: adding one costs a handler registration and nothing else. + * + * Subjects are `fluxify.jobs..`. Project comes first so a + * worker can filter on its own tenant with a single wildcard, which is also what + * keeps consumers non-overlapping — a work-queue stream requires that. + */ + +export const JOBS_STREAM = "FLUXIFY_JOBS"; + +const SUBJECT_ROOT = "fluxify.jobs"; +/** everything the job worker can consume */ +export const JOBS_SUBJECTS = `${SUBJECT_ROOT}.>`; +/** serves every project — the catch-all worker deployment */ +export const ALL_PROJECTS = "*"; + +export const jobSubject = (projectId: string, kind: string) => + `${SUBJECT_ROOT}.${projectId}.${kind}`; + +/** What one worker deployment subscribes to. */ +export function projectJobFilter(projectId: string) { + return projectId === ALL_PROJECTS + ? JOBS_SUBJECTS + : `${SUBJECT_ROOT}.${projectId}.>`; +} + +/** + * Durable name per deployment, so replicas of the same worker compete for the + * same messages while different projects never see each other's. + * Consumer names allow no dots or wildcards. + */ +export function jobConsumerName(projectId: string) { + return projectId === ALL_PROJECTS + ? "fluxify_jobs_all" + : `fluxify_jobs_${projectId.replace(/[^a-zA-Z0-9_-]/g, "_")}`; +} diff --git a/apps/server/src/modules/jobs/types.ts b/apps/server/src/modules/jobs/types.ts new file mode 100644 index 00000000..a754befc --- /dev/null +++ b/apps/server/src/modules/jobs/types.ts @@ -0,0 +1,22 @@ +/** + * One queued unit of work, as it travels on the wire. + * + * Deliberately kind-agnostic: a queued custom block, a cron firing, a workflow + * step and a scheduled job are all this shape, so they all share one stream and + * one worker. `kind` is the only thing that decides how `target` and `payload` + * are read — see the handler registry. + */ +export type JobEnvelope = { + /** Dedupe key on the broker and correlation id in logs. */ + id: string; + kind: string; + projectId: string; + target: string; + payload?: unknown; + origin?: Record; + enqueuedAt: string; + /** Delivery attempt, filled in by the consumer, not the publisher. */ + attempt?: number; +}; + +export type JobHandler = (job: JobEnvelope) => Promise; diff --git a/apps/server/src/modules/requestRouter/executionProcess.ts b/apps/server/src/modules/requestRouter/executionProcess.ts index 531ba707..c9b0dc38 100644 --- a/apps/server/src/modules/requestRouter/executionProcess.ts +++ b/apps/server/src/modules/requestRouter/executionProcess.ts @@ -1,5 +1,9 @@ import { initializeLogger, logger } from "@fluxify/common"; +import { setJobEnqueuer } from "@fluxify/blocks"; import { createHttpContext } from "./httpContext"; +import { registerCustomBlockJobHandler } from "../jobs/customBlockJob"; +import { runJob } from "../jobs/registry"; +import type { JobEnvelope } from "../jobs/types"; import { applyArtifactUpdate, compiledRouteValidators, @@ -37,9 +41,23 @@ process.on("message", (message: ExecutionMessage) => { applyArtifactUpdate(message.entry.key, message.entry.value); return; } + if (message.type === "job") return void executeJob(message.job); setMonitoring(message.enabled); }); +/** + * The supervisor acks the message on the reply, so every path must send one — + * a swallowed error here stalls the job until its ack wait elapses. + */ +async function executeJob(job: JobEnvelope) { + try { + await runJob(job); + send({ type: "job-finished", id: job.id }); + } catch (error) { + send({ type: "job-finished", id: job.id, error: String(error) }); + } +} + function bootstrap(nextBoot: ExecutionBootstrap) { if (boot) throw new Error("execution process was bootstrapped twice"); boot = nextBoot; @@ -55,6 +73,19 @@ function bootstrap(nextBoot: ExecutionBootstrap) { logger.error(`async dispatch failed: ${String(error)}`, "WORKER.execution"), ); setMonitoring(boot.workerTimeoutsEnabled); + registerCustomBlockJobHandler(); + // This process holds no broker connection: queueing is a message to the + // supervisor, which owns NATS. + setJobEnqueuer((request) => + send({ + type: "enqueue-job", + job: { + ...request, + id: crypto.randomUUID(), + enqueuedAt: new Date().toISOString(), + }, + }), + ); server = Bun.serve({ port: boot.port, diff --git a/apps/server/src/modules/requestRouter/service.ts b/apps/server/src/modules/requestRouter/service.ts index 71d85d27..65534f73 100644 --- a/apps/server/src/modules/requestRouter/service.ts +++ b/apps/server/src/modules/requestRouter/service.ts @@ -322,6 +322,61 @@ export async function executeRouteInternal( } } +/** + * A context for work that arrived off the queue rather than off a request. + * + * Same integrations, app config and logger a route gets — the difference is + * that there is no HTTP exchange, so cookies and headers are no-ops and the + * request accessors return empty. Nothing from the enqueuing request survives + * except the payload it queued. + * + * ponytail: fixed timeout. Give it a per-job setting when a job legitimately + * needs to outlive it. + */ +export const DEFAULT_JOB_TIMEOUT_SECONDS = 300; + +export function createJobContext(job: { + id: string; + projectId: string; + target: string; + timeoutSeconds?: number; +}): BlockContext { + const trigger: TriggerContext = { + kind: "job", + source: "nats", + reply: "async", + id: job.id, + }; + const requestData = { + method: "", + path: job.target, + headers: {}, + query: {}, + body: undefined, + params: {}, + }; + const vars = setupContextVars( + undefined, + requestData, + job.target, + httpClient, + job.projectId, + undefined, + trigger, + ); + const vm = createJsVM(vars); + return createContext( + { id: job.target, projectId: job.projectId }, + requestData, + vm, + vars, + new DbFactory(vm, dbIntegrationsCache, dbConnectionManager), + httpClient, + trigger, + job.timeoutSeconds ?? DEFAULT_JOB_TIMEOUT_SECONDS, + ); +} + function validateSchema( compiled: CompiledRequestSchema | undefined, schema: unknown, diff --git a/apps/server/src/modules/requestRouter/threadTypes.ts b/apps/server/src/modules/requestRouter/threadTypes.ts index 8c1aada3..ea98fe2d 100644 --- a/apps/server/src/modules/requestRouter/threadTypes.ts +++ b/apps/server/src/modules/requestRouter/threadTypes.ts @@ -1,5 +1,6 @@ import type { ArtifactEntry } from "./compiledRuntime"; import type { AsyncExecutorLimits } from "./asyncExecutor"; +import type { JobEnvelope } from "../jobs/types"; /** handed to the isolated execution process over Bun IPC at spawn */ export type ExecutionBootstrap = { @@ -28,12 +29,17 @@ export type ExecutionBootstrap = { export type ExecutionMessage = | { type: "bootstrap"; bootstrap: ExecutionBootstrap } | { type: "artifact"; entry: ArtifactEntry } - | { type: "monitoring"; enabled: boolean }; + | { type: "monitoring"; enabled: boolean } + // the supervisor owns NATS, the child owns user code: jobs cross here + | { type: "job"; job: JobEnvelope }; /** isolated execution process -> supervisor */ export type ExecutionEvent = | { type: "ready" } | { type: "heartbeat" } + | { type: "job-finished"; id: string; error?: string } + /** user code asked to queue work; only the supervisor can publish it */ + | { type: "enqueue-job"; job: JobEnvelope } | { type: "execution-started"; requestId: string; diff --git a/apps/server/src/modules/testRunner/testExecutionProcess.ts b/apps/server/src/modules/testRunner/testExecutionProcess.ts index 29ebad1d..4cfd7942 100644 --- a/apps/server/src/modules/testRunner/testExecutionProcess.ts +++ b/apps/server/src/modules/testRunner/testExecutionProcess.ts @@ -1,4 +1,8 @@ -import { instantiateCompiled, registerCompiledCustomBlock } from "@fluxify/blocks"; +import { + instantiateCompiled, + registerCompiledCustomBlock, + setJobEnqueuer, +} from "@fluxify/blocks"; import { hydrateAppConfig } from "../../loaders/appconfigLoader"; import { hydrateIntegrations } from "../../loaders/integrationsLoader"; import { hydrateProjectSettings } from "../../loaders/projectSettingsLoader"; @@ -33,6 +37,9 @@ process.on("message", (message: TestBootstrapMessage) => { void runSuite(message.bootstrap); }); +/** What the suite would have queued, kept for debugging a run. */ +const queuedJobs: string[] = []; + async function runSuite(boot: TestBootstrap) { const startedAt = Date.now(); try { @@ -52,6 +59,12 @@ async function runSuite(boot: TestBootstrap) { registerCompiledCustomBlock(block.name, block.source); } + // A test run holds no broker connection, and firing real background work + // from an assertion is not something a suite should be able to do. + setJobEnqueuer((job) => + queuedJobs.push(`${job.kind}/${job.target}`), + ); + const run = instantiateCompiled(boot.source); setBlocksExecutor((_target, context) => run(context, context.requestBody)); diff --git a/packages/blocks/builtin/customBlock.ts b/packages/blocks/builtin/customBlock.ts index 4c9c3dff..470daa6d 100644 --- a/packages/blocks/builtin/customBlock.ts +++ b/packages/blocks/builtin/customBlock.ts @@ -3,6 +3,7 @@ import { BaseBlock, BlockOptions, BlockOutput, Context } from "../baseBlock"; import { Engine } from "../engine"; import { logger } from "@fluxify/common"; import type { BlockDTOType, EdgeDTOSchemaType } from "../builderTypes"; +import { enqueueJob } from "../jobs"; import { compileGraph, emitJsObject, @@ -11,12 +12,15 @@ import { } from "../compiler"; export const customBlockInvokeSchema = z - .enum(["sync", "async"]) + .enum(["sync", "async", "queued"]) .default("sync") .describe( - "sync waits for the custom block and takes its output; async fires it and moves on", + "sync waits for the custom block and takes its output; async fires it on this worker and moves on; queued hands it to the job queue for another worker to run", ); +/** The job kind a queued custom block travels under. */ +export const CUSTOM_BLOCK_JOB = "custom-block"; + export type CompiledCustomBlock = (ctx: Context, input?: any) => Promise; /** @@ -86,16 +90,23 @@ function traced( return { context: child, close: () => scope.close() }; } +/** + * How a custom block is called: what it was configured with, and the value + * flowing into the calling block. They stay apart inside the callee — `params` + * is its configuration at any depth, `input` is the previous block's output. + */ +export type CustomBlockArgs = { params: Record; input?: any }; + /** sync: wait for the custom block and hand back its output */ export async function invokeCustomBlock( context: Context, name: string, - params: any, + args: CustomBlockArgs, blockId?: string, ) { const scope = traced(context, name, blockId, false); try { - const result = await lookup(name)(scope.context, params); + const result = await lookup(name)(scope.context, args); return result?.output; } finally { scope.close(); @@ -110,11 +121,11 @@ export async function invokeCustomBlock( export function invokeCustomBlockAsync( context: Context, name: string, - params: any, + args: CustomBlockArgs, blockId?: string, ) { const scope = traced(context, name, blockId, true); - lookup(name)(scope.context, params) + lookup(name)(scope.context, args) .catch((error) => { logger.error(`Async custom block '${name}' failed`, "BLOCKS.customBlock", { error, @@ -123,15 +134,44 @@ export function invokeCustomBlockAsync( .finally(() => scope.close()); } +/** + * queued: hand the work to the job queue and move on. Unlike `async` this + * survives the worker — the job is persisted by the broker and picked up by + * whichever worker serves this project, so a shutdown mid-flight redelivers + * instead of losing the work. + * + * The job carries the evaluated arguments only: the callee gets a fresh context + * on the other side, so nothing request-scoped (cookies, headers, `ctx.vars`) + * crosses. They must therefore be JSON-serializable. + */ +export function enqueueCustomBlock( + context: Context, + name: string, + args: CustomBlockArgs, + blockId?: string, +) { + enqueueJob({ + kind: CUSTOM_BLOCK_JOB, + projectId: context.projectId, + target: name, + payload: args, + origin: { blockId, route: context.route, apiId: context.apiId }, + }); +} + export function emitCustomBlock(node: EmitNode) { const { blockName, blockDescription, invoke, ...params } = (node.block.data ?? {}) as Record; const mode = customBlockInvokeSchema.parse(invoke); const name = JSON.stringify(node.block.type); - // same shape the interpreted block passes: evaluated params plus the input - const args = `{ ...${emitJsObject(params, node)}, input: ${node.in} }`; + // the callee reads its configuration as `params` and the value flowing into + // this block as `input` — one meaning each, at every depth of its graph + const args = `{ params: ${emitJsObject(params, node)}, input: ${node.in} }`; const id = JSON.stringify(node.block.id); + if (mode === "queued") { + return `lib.enqueue(ctx, ${name}, ${args}, ${id});\n${node.next()}`; + } if (mode === "async") { return `lib.invokeAsync(ctx, ${name}, ${args}, ${id});\n${node.next()}`; } diff --git a/packages/blocks/builtin/db/delete.ts b/packages/blocks/builtin/db/delete.ts index 32a1aa60..4a61c8a4 100644 --- a/packages/blocks/builtin/db/delete.ts +++ b/packages/blocks/builtin/db/delete.ts @@ -47,7 +47,7 @@ export async function runDeleteDb( export function emitDeleteDb(node: EmitNode) { const input = deleteDbBlockSchema.parse(node.block.data); - return `${node.in} = await lib.dbDelete(ctx, ${JSON.stringify(input.connection)}, ${node.value(input.tableName)}, ${emitWhereConditions(input.conditions, node)}); + return `${node.in} = await lib.dbDelete(ctx, ${node.value(input.connection)}, ${node.value(input.tableName)}, ${emitWhereConditions(input.conditions, node)}); ${node.next()}`; } diff --git a/packages/blocks/builtin/db/getAll.ts b/packages/blocks/builtin/db/getAll.ts index 43af0e61..9493d399 100644 --- a/packages/blocks/builtin/db/getAll.ts +++ b/packages/blocks/builtin/db/getAll.ts @@ -86,7 +86,7 @@ export async function runGetAllDb( export function emitGetAllDb(node: EmitNode) { const input = getAllDbBlockSchema.parse(node.block.data); const sort = `{ attribute: ${node.value(input.sort.attribute)}, direction: ${JSON.stringify(input.sort.direction)} }`; - return `${node.in} = await lib.dbGetAll(ctx, ${JSON.stringify(input.connection)}, ${node.value(input.tableName)}, ${emitWhereConditions(input.conditions, node)}, lib.num(${node.value(input.limit)}, 1000), lib.num(${node.value(input.offset)}, 0), ${sort}, { joins: ${JSON.stringify(input.joins ?? [])}, columns: ${JSON.stringify(input.columns ?? ["*"])} }); + return `${node.in} = await lib.dbGetAll(ctx, ${node.value(input.connection)}, ${node.value(input.tableName)}, ${emitWhereConditions(input.conditions, node)}, lib.num(${node.value(input.limit)}, 1000), lib.num(${node.value(input.offset)}, 0), ${sort}, { joins: ${JSON.stringify(input.joins ?? [])}, columns: ${JSON.stringify(input.columns ?? ["*"])} }); ${node.next()}`; } diff --git a/packages/blocks/builtin/db/getSingle.ts b/packages/blocks/builtin/db/getSingle.ts index ecef0a11..4919cebf 100644 --- a/packages/blocks/builtin/db/getSingle.ts +++ b/packages/blocks/builtin/db/getSingle.ts @@ -60,7 +60,7 @@ export async function runGetSingleDb( export function emitGetSingleDb(node: EmitNode) { const input = getSingleDbBlockSchema.parse(node.block.data); - return `${node.in} = await lib.dbGetSingle(ctx, ${JSON.stringify(input.connection)}, ${node.value(input.tableName)}, ${emitWhereConditions(input.conditions, node)}, { joins: ${JSON.stringify(input.joins ?? [])}, columns: ${JSON.stringify(input.columns ?? ["*"])} }); + return `${node.in} = await lib.dbGetSingle(ctx, ${node.value(input.connection)}, ${node.value(input.tableName)}, ${emitWhereConditions(input.conditions, node)}, { joins: ${JSON.stringify(input.joins ?? [])}, columns: ${JSON.stringify(input.columns ?? ["*"])} }); ${node.next()}`; } diff --git a/packages/blocks/builtin/db/insert.ts b/packages/blocks/builtin/db/insert.ts index e08a2990..7f85d6bd 100644 --- a/packages/blocks/builtin/db/insert.ts +++ b/packages/blocks/builtin/db/insert.ts @@ -70,7 +70,7 @@ export function emitInsertDb(node: EmitNode) { return `const ${data} = ${payload}; if (typeof ${data} !== "object") throw new Error("error in insert: data to insert is not an object"); -${node.in} = await lib.dbInsert(ctx, ${JSON.stringify(input.connection)}, ${node.value(input.tableName)}, ${data}); +${node.in} = await lib.dbInsert(ctx, ${node.value(input.connection)}, ${node.value(input.tableName)}, ${data}); ${node.next()}`; } diff --git a/packages/blocks/builtin/db/insertBulk.ts b/packages/blocks/builtin/db/insertBulk.ts index da0d6b56..8b2897d6 100644 --- a/packages/blocks/builtin/db/insertBulk.ts +++ b/packages/blocks/builtin/db/insertBulk.ts @@ -61,7 +61,7 @@ export function emitInsertBulkDb(node: EmitNode) { return `const ${data} = ${payload}; if (!Array.isArray(${data})) throw new Error("error in insert bulk: data to insert is not an array"); -${node.in} = await lib.dbInsertBulk(ctx, ${JSON.stringify(input.connection)}, ${node.value(input.tableName)}, ${data}); +${node.in} = await lib.dbInsertBulk(ctx, ${node.value(input.connection)}, ${node.value(input.tableName)}, ${data}); ${node.next()}`; } diff --git a/packages/blocks/builtin/db/native.ts b/packages/blocks/builtin/db/native.ts index dcdcdad2..7d578ea9 100644 --- a/packages/blocks/builtin/db/native.ts +++ b/packages/blocks/builtin/db/native.ts @@ -52,7 +52,7 @@ export async function runNativeDb( export function emitNativeDb(node: EmitNode) { const input = nativeDbBlockSchema.parse(node.block.data); const code = input.js.startsWith("js:") ? input.js.slice(3) : input.js; - return `${node.in} = await lib.dbNative(ctx, ${JSON.stringify(input.connection)}, async () => ${node.js(code, node.in)}); + return `${node.in} = await lib.dbNative(ctx, ${node.value(input.connection)}, async () => ${node.js(code, node.in)}); ${node.next()}`; } diff --git a/packages/blocks/builtin/db/transaction.ts b/packages/blocks/builtin/db/transaction.ts index 25b401a0..24ea1d69 100644 --- a/packages/blocks/builtin/db/transaction.ts +++ b/packages/blocks/builtin/db/transaction.ts @@ -57,7 +57,7 @@ export async function runTransactionDb( export function emitTransactionDb(node: EmitNode) { const input = transactionDbBlockSchema.parse(node.block.data); const result = node.v("tx"); - return `const ${result} = await lib.dbTransaction(ctx, ${JSON.stringify(input.connection)}, async () => { + return `const ${result} = await lib.dbTransaction(ctx, ${node.value(input.connection)}, async () => { ${node.body("executor", "undefined")} }); if (${result} !== undefined) return ${result}; diff --git a/packages/blocks/builtin/db/update.ts b/packages/blocks/builtin/db/update.ts index 1348ad7f..d168f85c 100644 --- a/packages/blocks/builtin/db/update.ts +++ b/packages/blocks/builtin/db/update.ts @@ -81,7 +81,7 @@ export function emitUpdateDb(node: EmitNode) { return `const ${data} = ${payload}; if (typeof ${data} !== "object") throw new Error("error in update: data to update is not an object"); -${node.in} = await lib.dbUpdate(ctx, ${JSON.stringify(input.connection)}, ${node.value(input.tableName)}, ${data}, ${emitWhereConditions(input.conditions, node)}); +${node.in} = await lib.dbUpdate(ctx, ${node.value(input.connection)}, ${node.value(input.tableName)}, ${data}, ${emitWhereConditions(input.conditions, node)}); ${node.next()}`; } diff --git a/packages/blocks/builtin/log/cloudLogs.ts b/packages/blocks/builtin/log/cloudLogs.ts index 1b83684d..51a0a594 100644 --- a/packages/blocks/builtin/log/cloudLogs.ts +++ b/packages/blocks/builtin/log/cloudLogs.ts @@ -48,7 +48,7 @@ export function emitCloudLogs(node: EmitNode) { const { connection, level, message } = cloudLogsBlockSchema.parse( node.block.data, ); - return `await lib.cloudLog(ctx, ${JSON.stringify(connection)}, ${JSON.stringify(level)}, ${emitLogMessage(message, node)}, ${node.in});\n${node.next()}`; + return `await lib.cloudLog(ctx, ${node.value(connection)}, ${JSON.stringify(level)}, ${emitLogMessage(message, node)}, ${node.in});\n${node.next()}`; } export class CloudLogsBlock extends BaseBlock { diff --git a/packages/blocks/builtin/tests/compilerCustomBlock.spec.ts b/packages/blocks/builtin/tests/compilerCustomBlock.spec.ts index aa52bc09..33efd06a 100644 --- a/packages/blocks/builtin/tests/compilerCustomBlock.spec.ts +++ b/packages/blocks/builtin/tests/compilerCustomBlock.spec.ts @@ -9,6 +9,7 @@ import { unregisterCustomBlock, } from "../customBlock"; import { BlockTypes } from "../../blockTypes"; +import { setJobEnqueuer } from "../../jobs"; import type { BlockDTOType, EdgeDTOSchemaType } from "../../builderTypes"; const block = (id: string, type: string, data: any = {}): BlockDTOType => ({ @@ -49,7 +50,8 @@ function registerDoubler(name = "double_it") { [ block("c1", BlockTypes.entrypoint), block("c2", BlockTypes.jsrunner, { - value: "calls = (typeof calls === 'number' ? calls : 0) + 1; return input.value * 2;", + value: + "calls = (typeof calls === 'number' ? calls : 0) + 1; return params.value * 2;", }), ], [edge("c1", "c2")], @@ -58,6 +60,7 @@ function registerDoubler(name = "double_it") { afterEach(() => { for (const name of customBlockNames()) unregisterCustomBlock(name); + setJobEnqueuer(); }); describe("compiled custom blocks", () => { @@ -106,7 +109,7 @@ describe("compiled custom blocks", () => { block("c1", BlockTypes.entrypoint), block("c2", BlockTypes.jsrunner, { value: - "await new Promise((r) => setTimeout(r, 5)); sideEffect = input.value; return sideEffect;", + "await new Promise((r) => setTimeout(r, 5)); sideEffect = params.value; return sideEffect;", }), ], [edge("c1", "c2")], @@ -157,12 +160,12 @@ describe("compiled custom blocks", () => { expect(result.output.body).toBe("unharmed"); }); - it("passes evaluated params plus the flowing value as input", async () => { + it("separates the configured params from the flowing input", async () => { registerCustomBlock( "echo_params", [ block("c1", BlockTypes.entrypoint), - block("c2", BlockTypes.jsrunner, { value: "return input;" }), + block("c2", BlockTypes.jsrunner, { value: "return { params, input };" }), ], [edge("c1", "c2")], ); @@ -181,12 +184,38 @@ describe("compiled custom blocks", () => { const result = await run(createContext(), { n: 41 }); expect(result.output.body).toEqual({ - literal: "kept", - computed: 42, + params: { literal: "kept", computed: 42 }, input: { n: 41 }, }); }); + it("keeps params visible further down the callee's graph", async () => { + // `input` becomes the previous block's output; `params` does not move + registerCustomBlock( + "two_steps", + [ + block("c1", BlockTypes.entrypoint), + block("c2", BlockTypes.jsrunner, { value: "return 'step-one';" }), + block("c3", BlockTypes.jsrunner, { + value: "return { seen: input, still: params.label };", + }), + ], + [edge("c1", "c2"), edge("c2", "c3")], + ); + + const { run } = compileGraph( + [ + block("1", BlockTypes.entrypoint), + block("2", "two_steps", { label: "kept" }), + block("3", BlockTypes.response, { httpCode: "200" }), + ], + [edge("1", "2"), edge("2", "3")], + ); + + const result = await run(createContext(), null); + expect(result.output.body).toEqual({ seen: "step-one", still: "kept" }); + }); + it("resolves param: placeholders from the invocation, not the caller", async () => { // compiled once for the whole worker, so a placeholder cannot be baked in registerCustomBlock( @@ -218,6 +247,56 @@ describe("compiled custom blocks", () => { expect(second.vars.greeting).toBe("hola"); }); + it("queued invoke hands the work to the job queue and does not run it", async () => { + registerDoubler(); + const queued: any[] = []; + setJobEnqueuer((job) => queued.push(job)); + + const { run, source } = compileGraph( + [ + block("1", BlockTypes.entrypoint), + block("2", "double_it", { value: 21, invoke: "queued" }), + block("3", BlockTypes.response, { httpCode: "200" }), + ], + [edge("1", "2"), edge("2", "3")], + ); + expect(source).toContain("lib.enqueue"); + + const ctx = createContext(); + const result = await run(ctx, { keep: "me" }); + + // the caller kept its own value and the block never executed here + expect(result.output.body).toEqual({ keep: "me" }); + expect(ctx.vars.calls).toBeUndefined(); + expect(queued).toEqual([ + { + kind: "custom-block", + projectId: "proj-1", + target: "double_it", + payload: { params: { value: 21 }, input: { keep: "me" } }, + origin: { blockId: "2", route: "/custom", apiId: "api-1" }, + }, + ]); + }); + + it("a queued invoke fails loudly when no queue is wired", async () => { + registerDoubler(); + setJobEnqueuer(); + + const { run } = compileGraph( + [ + block("1", BlockTypes.entrypoint), + block("2", "double_it", { value: 1, invoke: "queued" }), + block("3", BlockTypes.response, { httpCode: "200" }), + ], + [edge("1", "2"), edge("2", "3")], + ); + + const result = await run(createContext(), null); + expect(result.successful).toBe(false); + expect(String(result.error)).toContain("No job queue"); + }); + it("registers already-compiled source without running the compiler", async () => { // what a worker does: it receives JS from the artifact store, never a graph const source = registerDoubler("shipped"); @@ -302,4 +381,42 @@ describe("compiled cloud logs", () => { // the block passes its input through untouched expect(result.output.body).toEqual({ id: 9 }); }); + + it("takes a custom block's integration from the caller's param", async () => { + // the integration selector input param: the block picks `param:obs`, the + // caller picks the concrete integration id + registerCustomBlock( + "audit", + [ + block("c1", BlockTypes.entrypoint), + block("c2", BlockTypes.cloudLogs, { + connection: "param:obs", + level: "info", + message: "audited", + }), + ], + [edge("c1", "c2")], + ); + + const { run } = compileGraph( + [ + block("1", BlockTypes.entrypoint), + block("2", "audit", { obs: "obs-42" }), + block("3", BlockTypes.response, { httpCode: "200" }), + ], + [edge("1", "2"), edge("2", "3")], + ); + + const asked: any[] = []; + const ctx = createContext(); + ctx.integrationFactory = { + create: (options: any) => { + asked.push(options); + return { logInfo() {}, logWarn() {}, logError() {} }; + }, + }; + await run(ctx, null); + + expect(asked).toEqual([{ integrationId: "obs-42", type: "observability" }]); + }); }); diff --git a/packages/blocks/compiler.ts b/packages/blocks/compiler.ts index 7d3cf058..9dffe59d 100644 --- a/packages/blocks/compiler.ts +++ b/packages/blocks/compiler.ts @@ -16,6 +16,7 @@ import { emitConsoleLog, runConsoleLog } from "./builtin/log/console"; import { emitCloudLogs, runCloudLog } from "./builtin/log/cloudLogs"; import { emitCustomBlock, + enqueueCustomBlock, hasCustomBlock, invokeCustomBlock, invokeCustomBlockAsync, @@ -124,6 +125,7 @@ export const compilerLib = { cloudLog: runCloudLog, invoke: invokeCustomBlock, invokeAsync: invokeCustomBlockAsync, + enqueue: enqueueCustomBlock, }; /** @@ -171,7 +173,11 @@ export function scopeFor(vars: Record, skip?: Set) { function makeScope(vars: Record, skip?: Set) { return new Proxy(vars, { - has: (target, key: any) => key !== "input" && !skip?.has(key), + // `input` and `params` are function parameters of the emitted JS wrapper; + // letting `with` resolve them off vars would shadow them with a variable + // that merely shares the name. + has: (target, key: any) => + key !== "input" && key !== "params" && !skip?.has(key), get: (target, key: any) => key === Symbol.unscopables ? undefined @@ -351,7 +357,10 @@ export function compileGraph( const method = sync ? "run" : "runAsync"; return `(await ctx.vm.${method}(${JSON.stringify(code)}${extras ? `, ${extras}` : ""}))`; } - return `(await (async function (input) { with ($scope) {\n${code}\n} })(${extras ?? "undefined"}))`; + // `params` is the custom block's invocation arguments — undefined in a + // route graph, so `input` keeps meaning exactly one thing everywhere: the + // previous block's output. + return `(await (async function (input, params) { with ($scope) {\n${code}\n} })(${extras ?? "undefined"}, $state.params))`; } /** @@ -530,10 +539,12 @@ throw $error; "vars,", `scope: lib.scope(vars${imports.scopeSkip}),`, "trace: ctx.trace,", - `params: ${asCustomBlock ? "input ?? {}" : "undefined"},`, + // a custom block is called with { params, input }: its configuration and + // the caller's flowing value, kept apart all the way down the graph + `params: ${asCustomBlock ? "input?.params ?? {}" : "undefined"},`, "};", "try {", - `return await ${blockFunctionName(entry.id)}($state, input, $endSuccess);`, + `return await ${blockFunctionName(entry.id)}($state, ${asCustomBlock ? "input?.input" : "input"}, $endSuccess);`, "} catch ($error) {", errorHandlerBody, "}", diff --git a/packages/blocks/index.ts b/packages/blocks/index.ts index 4895a873..ec8a0401 100644 --- a/packages/blocks/index.ts +++ b/packages/blocks/index.ts @@ -11,6 +11,7 @@ export * from "./builtin/loops/foreach"; export * from "./engine"; export * from "./compiler"; export * from "./builtin/customBlock"; +export * from "./jobs"; export * from "./blockTypes"; export * from "./baseBlock"; export * from "./categories"; diff --git a/packages/blocks/jobs.ts b/packages/blocks/jobs.ts new file mode 100644 index 00000000..abdc6b1a --- /dev/null +++ b/packages/blocks/jobs.ts @@ -0,0 +1,51 @@ +/** + * The seam between a graph and whatever queue the host runs. + * + * `packages/blocks` must not know about NATS — it runs inside the isolated + * execution process, which owns no connections. The host registers an enqueuer + * at startup and the generated code calls `lib.enqueue`, exactly the way + * `setBlocksExecutor` inverts route execution. + */ + +/** One unit of queued work. `kind` decides how a consumer reads `target`. */ +export type JobRequest = { + /** "custom-block" today; crons, workflows and schedules share this queue. */ + kind: string; + projectId: string; + /** What to run — a custom block's name, a route id, a workflow key. */ + target: string; + /** Must be JSON-serializable: it crosses a process and a broker. */ + payload?: unknown; + /** Where it was queued from, for correlation in traces and logs. */ + origin?: { + blockId?: string; + route?: string; + apiId?: string; + }; +}; + +export type JobEnqueuer = (job: JobRequest) => void; + +let enqueuer: JobEnqueuer | undefined; + +/** Called once by the host process. Pass nothing to detach (tests, shutdown). */ +export function setJobEnqueuer(next?: JobEnqueuer) { + enqueuer = next; +} + +export function jobQueueAvailable() { + return enqueuer !== undefined; +} + +/** + * Throws when no queue is wired. Dropping the job silently would be worse: the + * caller has already moved on believing the work is durable. + */ +export function enqueueJob(job: JobRequest) { + if (!enqueuer) { + throw new Error( + `No job queue is configured — cannot queue ${job.kind} "${job.target}"`, + ); + } + enqueuer(job); +} diff --git a/packages/components/src/IntegrationSelector/IntegrationSelector.tsx b/packages/components/src/IntegrationSelector/IntegrationSelector.tsx index 5dd7ba17..a77f1ba7 100644 --- a/packages/components/src/IntegrationSelector/IntegrationSelector.tsx +++ b/packages/components/src/IntegrationSelector/IntegrationSelector.tsx @@ -230,9 +230,21 @@ function PickerModal({ - - {integration.name} - +
+ + {integration.name} + {integration.external && ( + + From caller + + )} + + {integration.external && integration.hint && ( + + {integration.hint} + + )} +
@@ -280,6 +292,7 @@ function PickerModal({ export function IntegrationSelector({ selectedId, loadIntegrations, + injectedIntegrations, onSelect, onTestConnection, openInNewTabUrl, @@ -292,7 +305,12 @@ export function IntegrationSelector({ description, className, }: IntegrationSelectorProps) { - const [integrations, setIntegrations] = useState([]); + const [loaded, setLoaded] = useState([]); + // injected entries need no fetch and always come first + const integrations = useMemo( + () => [...(injectedIntegrations ?? []), ...loaded], + [injectedIntegrations, loaded], + ); const [loadStatus, setLoadStatus] = useState("idle"); const [loadError, setLoadError] = useState(null); @@ -316,7 +334,7 @@ export function IntegrationSelector({ setLoadError(null); try { const data = await loadIntegrations(); - setIntegrations(data); + setLoaded(data); setLoadStatus("success"); } catch (err) { const msg = @@ -443,8 +461,9 @@ export function IntegrationSelector({
{loadStatus === "success" && ( <> - {/* Test Connection */} - {selectedIntegration && onTestConnection && ( + {/* Test Connection — an external entry has nothing to test + here; the real integration lives on the caller's side */} + {selectedIntegration && !selectedIntegration.external && onTestConnection && (
+ + {selectedIntegration?.external && selectedIntegration.hint && ( +

+ {selectedIntegration.hint} +

+ )} {/* Picker Modal */} diff --git a/packages/components/src/IntegrationSelector/types.ts b/packages/components/src/IntegrationSelector/types.ts index 7bb19f34..fa3a4094 100644 --- a/packages/components/src/IntegrationSelector/types.ts +++ b/packages/components/src/IntegrationSelector/types.ts @@ -11,6 +11,15 @@ export interface Integration { variant: string; config: Record; tags?: string[]; + /** + * Not a real integration row: a placeholder whose concrete integration is + * chosen elsewhere (a custom block's input parameter, resolved by the caller). + * These sort first, carry a badge, and expose no test/open actions — there is + * nothing on this side to test or open. + */ + external?: boolean; + /** Explains an `external` entry in the picker and under the field. */ + hint?: string; } /** Internal async-load state. */ @@ -31,6 +40,11 @@ export interface IntegrationSelectorProps { * identity changes. */ loadIntegrations: () => Promise; + /** + * Entries injected ahead of the loaded ones without a fetch — used for + * `external` placeholders. Listed first in the picker. + */ + injectedIntegrations?: Integration[]; /** Called with the selected integration id, or empty string to clear. */ onSelect?: (id: string) => void; /** From 15fb52d6f8d6b7b67bc6dfefefc889fa8639c920 Mon Sep 17 00:00:00 2001 From: Prajwal Aradhya Date: Mon, 17 Aug 2026 19:57:13 +0100 Subject: [PATCH 2/2] test(e2e): custom block coverage through a jwt_ops fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One custom block, two input params, three routes that configure it differently — signing, lenient verification, and strict verification that ends on the block's own response block. The harness gained what custom blocks need: fixtures declare the blocks they call in `uses`, and the runner registers them before compiling the route, the same order the real compiler works in. Custom blocks live in `blocks/` in the shape the portal saves them, input contract included. Also adds `engine: "none"` for graphs that touch no database, so a pure-JS fixture no longer starts a Postgres container. The block's secret is hardcoded — custom block params cannot reference app config yet. Co-Authored-By: Claude Opus 5 --- testing/e2e/README.md | 28 +++- testing/e2e/blocks/jwt-ops.json | 125 +++++++++++++++++ .../e2e/graphs/custom-blocks/jwt-sign.json | 73 ++++++++++ .../custom-blocks/jwt-verify-strict.json | 126 ++++++++++++++++++ .../e2e/graphs/custom-blocks/jwt-verify.json | 73 ++++++++++ testing/e2e/src/customBlocks.ts | 22 +++ testing/e2e/src/engines.ts | 8 +- testing/e2e/src/graph.ts | 37 ++++- testing/e2e/src/runner.ts | 18 ++- testing/e2e/tests/fixtures.test.ts | 28 +++- testing/e2e/tests/jwtCustomBlock.test.ts | 120 +++++++++++++++++ 11 files changed, 651 insertions(+), 7 deletions(-) create mode 100644 testing/e2e/blocks/jwt-ops.json create mode 100644 testing/e2e/graphs/custom-blocks/jwt-sign.json create mode 100644 testing/e2e/graphs/custom-blocks/jwt-verify-strict.json create mode 100644 testing/e2e/graphs/custom-blocks/jwt-verify.json create mode 100644 testing/e2e/src/customBlocks.ts create mode 100644 testing/e2e/tests/jwtCustomBlock.test.ts diff --git a/testing/e2e/README.md b/testing/e2e/README.md index 5fbb0d66..1b996a8b 100644 --- a/testing/e2e/README.md +++ b/testing/e2e/README.md @@ -36,6 +36,7 @@ A fixture declares its engine; Postgres is the default. |---|---|---| | `pg` | `postgres:bullseye` | `users`, `orders`, `auth_users` | | `mongo` | `mongo:7.0`, single-node replica set | `todos` | +| `none` | — | graphs that touch no database | Graphs are written **per engine**, not run across all of them. The adapters do not agree on what a result looks like — Mongo ids are hex strings off `_id`, @@ -61,6 +62,30 @@ route-level schema the portal stores, and it runs before any block does, so a to the tests: a login test posts one, and the storage test hashes one and compares it against the column read straight out of Postgres. +## Custom blocks + +`blocks/` holds custom blocks in the shape the portal saves them — a graph plus +the `inputParams` its callers configure. A route fixture names the ones it calls +in `uses`, and the harness registers them before compiling the route, which is +the same order the real compiler works in: a caller only emits once the library +knows the name. + +`blocks/jwt-ops.json` is the worked example. One block, two input params +(`operation`, `failOnInvalid`), and three routes under `graphs/custom-blocks/` +that configure it differently — signing, lenient verification, and strict +verification that ends on the block's *own* response block. Worth copying: + +- Read config as `params.` inside the block, at any depth. `input` is the + previous block's output, exactly as in a route. +- The callee's spans land in the caller's trace, so `executed` shows the inner + block ids inline. Assert on them — that is what distinguishes "the block ran" + from "the caller returned something". +- A response block inside a custom block ends *the block*, handing + `{ httpCode, body }` back to the caller. The caller still chooses the status. + +The secret is hardcoded in the block. It should come from app config, which +custom block params cannot reference yet. + ## Adding a graph 1. Drop a JSON file in `graphs/`, or in a subfolder for a multi-endpoint @@ -72,7 +97,8 @@ compares it against the column read straight out of Postgres. the field the compiler reads. An `if` whose two edges both say `"toHandle": "source"` fails to compile with a fan-out error. 2. Point any db block's `connection` at `"primary"` — the harness wires that id - to the right container for the fixture's engine. + to the right container for the fixture's engine. A graph with no db block + should set `"engine": "none"` so no container starts for it. 3. Add a test file in `tests/` with `beforeEach(() => resetDatabase(engine))` and assertions on `runGraph(fixture)`. diff --git a/testing/e2e/blocks/jwt-ops.json b/testing/e2e/blocks/jwt-ops.json new file mode 100644 index 00000000..107ec40d --- /dev/null +++ b/testing/e2e/blocks/jwt-ops.json @@ -0,0 +1,125 @@ +{ + "name": "jwt_ops", + "description": "Signs or verifies a JWT, chosen by the `operation` input param. With `failOnInvalid` set, a bad token ends the block on a 401 response instead of returning a result. The secret is baked in until custom blocks can read app config.", + "inputParams": [ + { + "name": "operation", + "label": "Operation", + "type": "dropdown", + "options": ["sign", "verify"], + "description": "Sign a new token from the incoming payload, or verify the one it carries." + }, + { + "name": "failOnInvalid", + "label": "Reject invalid tokens", + "type": "checkbox", + "description": "End the block with a 401 instead of returning { valid: false }." + } + ], + "blocks": [ + { + "id": "cb-entry", + "type": "entrypoint", + "position": { "x": 0, "y": 0 }, + "data": { "blockName": "Called" } + }, + { + "id": "cb-is-sign", + "type": "if", + "position": { "x": 240, "y": 0 }, + "data": { + "blockName": "Signing?", + "conditions": [ + { + "lhs": "", + "rhs": "", + "operator": "js", + "chain": "and", + "js": "return params.operation === \"sign\";" + } + ] + } + }, + { + "id": "cb-sign", + "type": "jsrunner", + "position": { "x": 480, "y": -120 }, + "data": { + "blockName": "Sign a token", + "value": "return {\n\ttoken: jwt.sign(input ?? {}, \"e2e-custom-block-secret\", {\n\t\texpiresIn: \"1h\",\n\t\tissuer: \"fluxify-e2e-block\",\n\t}),\n};" + } + }, + { + "id": "cb-verify", + "type": "jsrunner", + "position": { "x": 480, "y": 120 }, + "data": { + "blockName": "Verify the token", + "value": "const result = jwt.verify(input?.token ?? \"\", \"e2e-custom-block-secret\");\nreturn result.success\n\t? { valid: true, payload: result.payload }\n\t: { valid: false, error: \"invalid_token\" };" + } + }, + { + "id": "cb-guard", + "type": "if", + "position": { "x": 720, "y": 120 }, + "data": { + "blockName": "Let it through?", + "conditions": [ + { + "lhs": "", + "rhs": "", + "operator": "js", + "chain": "and", + "js": "return input.valid || !params.failOnInvalid;" + } + ] + } + }, + { + "id": "cb-reject", + "type": "response", + "position": { "x": 960, "y": 240 }, + "data": { + "blockName": "401 Unauthorized", + "httpCode": "401" + } + } + ], + "edges": [ + { + "id": "cb-e-entry-branch", + "from": "cb-entry", + "to": "cb-is-sign", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "cb-e-branch-sign", + "from": "cb-is-sign", + "to": "cb-sign", + "fromHandle": "success", + "toHandle": "success" + }, + { + "id": "cb-e-branch-verify", + "from": "cb-is-sign", + "to": "cb-verify", + "fromHandle": "failure", + "toHandle": "failure" + }, + { + "id": "cb-e-verify-guard", + "from": "cb-verify", + "to": "cb-guard", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "cb-e-guard-reject", + "from": "cb-guard", + "to": "cb-reject", + "fromHandle": "failure", + "toHandle": "failure" + } + ] +} diff --git a/testing/e2e/graphs/custom-blocks/jwt-sign.json b/testing/e2e/graphs/custom-blocks/jwt-sign.json new file mode 100644 index 00000000..a13b6a7e --- /dev/null +++ b/testing/e2e/graphs/custom-blocks/jwt-sign.json @@ -0,0 +1,73 @@ +{ + "name": "custom-blocks/jwt-sign", + "description": "POST /jwt/sign — builds a claims object and hands it to the jwt_ops custom block with operation=sign. The token comes back from the block's own graph.", + "engine": "none", + "route": { + "method": "POST", + "path": "/jwt/sign" + }, + "uses": ["jwt-ops"], + "schemas": { + "body": { + "dataType": "object", + "properties": [{ "key": "sub", "dataType": "str", "required": true }] + } + }, + "blocks": [ + { + "id": "entry", + "type": "entrypoint", + "position": { "x": 0, "y": 0 }, + "data": { "blockName": "Request" } + }, + { + "id": "claims", + "type": "jsrunner", + "position": { "x": 240, "y": 0 }, + "data": { + "blockName": "Build the claims", + "value": "return { sub: getRequestBody().sub, role: \"admin\" };" + } + }, + { + "id": "sign", + "type": "jwt_ops", + "position": { "x": 480, "y": 0 }, + "data": { + "blockName": "Sign it", + "invoke": "sync", + "operation": "sign", + "failOnInvalid": false + } + }, + { + "id": "reply", + "type": "response", + "position": { "x": 720, "y": 0 }, + "data": { "blockName": "200 OK", "httpCode": "200" } + } + ], + "edges": [ + { + "id": "e-entry-claims", + "from": "entry", + "to": "claims", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-claims-sign", + "from": "claims", + "to": "sign", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-sign-reply", + "from": "sign", + "to": "reply", + "fromHandle": "source", + "toHandle": "source" + } + ] +} diff --git a/testing/e2e/graphs/custom-blocks/jwt-verify-strict.json b/testing/e2e/graphs/custom-blocks/jwt-verify-strict.json new file mode 100644 index 00000000..c8619a34 --- /dev/null +++ b/testing/e2e/graphs/custom-blocks/jwt-verify-strict.json @@ -0,0 +1,126 @@ +{ + "name": "custom-blocks/jwt-verify-strict", + "description": "POST /jwt/verify-strict — the same custom block with failOnInvalid on, so a bad token ends the block on its own response block. The route branches on what came back and answers 401.", + "engine": "none", + "route": { + "method": "POST", + "path": "/jwt/verify-strict" + }, + "uses": ["jwt-ops"], + "schemas": { + "body": { + "dataType": "object", + "properties": [{ "key": "token", "dataType": "str", "required": true }] + } + }, + "blocks": [ + { + "id": "entry", + "type": "entrypoint", + "position": { "x": 0, "y": 0 }, + "data": { "blockName": "Request" } + }, + { + "id": "carry-token", + "type": "jsrunner", + "position": { "x": 240, "y": 0 }, + "data": { + "blockName": "Take the token", + "value": "return { token: getRequestBody().token };" + } + }, + { + "id": "verify", + "type": "jwt_ops", + "position": { "x": 480, "y": 0 }, + "data": { + "blockName": "Verify it", + "invoke": "sync", + "operation": "verify", + "failOnInvalid": true + } + }, + { + "id": "block-refused", + "type": "if", + "position": { "x": 720, "y": 0 }, + "data": { + "blockName": "Did the block refuse it?", + "conditions": [ + { + "lhs": "", + "rhs": "", + "operator": "js", + "chain": "and", + "js": "return input?.httpCode === \"401\";" + } + ] + } + }, + { + "id": "refusal-body", + "type": "jsrunner", + "position": { "x": 960, "y": 120 }, + "data": { + "blockName": "Unwrap the block's body", + "value": "return input.body;" + } + }, + { + "id": "rejected", + "type": "response", + "position": { "x": 1200, "y": 120 }, + "data": { "blockName": "401 Unauthorized", "httpCode": "401" } + }, + { + "id": "accepted", + "type": "response", + "position": { "x": 960, "y": -120 }, + "data": { "blockName": "200 OK", "httpCode": "200" } + } + ], + "edges": [ + { + "id": "e-entry-carry", + "from": "entry", + "to": "carry-token", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-carry-verify", + "from": "carry-token", + "to": "verify", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-verify-check", + "from": "verify", + "to": "block-refused", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-check-refusal", + "from": "block-refused", + "to": "refusal-body", + "fromHandle": "success", + "toHandle": "success" + }, + { + "id": "e-refusal-rejected", + "from": "refusal-body", + "to": "rejected", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-check-accepted", + "from": "block-refused", + "to": "accepted", + "fromHandle": "failure", + "toHandle": "failure" + } + ] +} diff --git a/testing/e2e/graphs/custom-blocks/jwt-verify.json b/testing/e2e/graphs/custom-blocks/jwt-verify.json new file mode 100644 index 00000000..99ff8680 --- /dev/null +++ b/testing/e2e/graphs/custom-blocks/jwt-verify.json @@ -0,0 +1,73 @@ +{ + "name": "custom-blocks/jwt-verify", + "description": "POST /jwt/verify — the same custom block configured with operation=verify. `failOnInvalid` is off, so a bad token returns { valid: false } and the route decides what to do with it.", + "engine": "none", + "route": { + "method": "POST", + "path": "/jwt/verify" + }, + "uses": ["jwt-ops"], + "schemas": { + "body": { + "dataType": "object", + "properties": [{ "key": "token", "dataType": "str", "required": true }] + } + }, + "blocks": [ + { + "id": "entry", + "type": "entrypoint", + "position": { "x": 0, "y": 0 }, + "data": { "blockName": "Request" } + }, + { + "id": "carry-token", + "type": "jsrunner", + "position": { "x": 240, "y": 0 }, + "data": { + "blockName": "Take the token", + "value": "return { token: getRequestBody().token };" + } + }, + { + "id": "verify", + "type": "jwt_ops", + "position": { "x": 480, "y": 0 }, + "data": { + "blockName": "Verify it", + "invoke": "sync", + "operation": "verify", + "failOnInvalid": false + } + }, + { + "id": "reply", + "type": "response", + "position": { "x": 720, "y": 0 }, + "data": { "blockName": "200 OK", "httpCode": "200" } + } + ], + "edges": [ + { + "id": "e-entry-carry", + "from": "entry", + "to": "carry-token", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-carry-verify", + "from": "carry-token", + "to": "verify", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-verify-reply", + "from": "verify", + "to": "reply", + "fromHandle": "source", + "toHandle": "source" + } + ] +} diff --git a/testing/e2e/src/customBlocks.ts b/testing/e2e/src/customBlocks.ts new file mode 100644 index 00000000..572525ce --- /dev/null +++ b/testing/e2e/src/customBlocks.ts @@ -0,0 +1,22 @@ +import { registerCustomBlock, unregisterCustomBlock } from "@fluxify/blocks"; +import { loadCustomBlock, type GraphFixture } from "./graph"; + +/** + * Puts a fixture's custom blocks in the library before its route is compiled. + * + * This is the same two-step the real compiler does — compile the blocks, then + * the routes — because a route that calls a custom block only emits if the + * library already knows the name. Registration is worker-global, so the + * returned dispose keeps one fixture's blocks out of another's run. + */ +export async function registerFixtureBlocks(fixture: GraphFixture) { + const registered: string[] = []; + for (const file of fixture.uses ?? []) { + const block = await loadCustomBlock(file); + registerCustomBlock(block.name, block.blocks, block.edges); + registered.push(block.name); + } + return () => { + for (const name of registered) unregisterCustomBlock(name); + }; +} diff --git a/testing/e2e/src/engines.ts b/testing/e2e/src/engines.ts index 1e89c068..b119c9de 100644 --- a/testing/e2e/src/engines.ts +++ b/testing/e2e/src/engines.ts @@ -12,15 +12,19 @@ import { seedPostgres, seedMongo } from "./seed"; * weakened until it stopped proving much. Each engine gets graphs that exercise * what is actually distinctive about it. */ -export type Engine = "pg" | "mongo"; +/** `none` is for graphs that touch no database — no container starts for them. */ +export type Engine = "none" | "pg" | "mongo"; -export async function connectionFor(engine: Engine): Promise { +export async function connectionFor( + engine: Exclude, +): Promise { if (engine === "mongo") return (await mongo()).connection; return (await database()).connection; } /** Drops and re-seeds the engine's fixtures. Call from `beforeEach`. */ export async function resetDatabase(engine: Engine) { + if (engine === "none") return; if (engine === "mongo") return seedMongo((await mongo()).db); return seedPostgres((await database()).sql); } diff --git a/testing/e2e/src/graph.ts b/testing/e2e/src/graph.ts index 93501fc8..3fc08e8c 100644 --- a/testing/e2e/src/graph.ts +++ b/testing/e2e/src/graph.ts @@ -20,11 +20,31 @@ export type GraphFixture = { * validation block on the canvas. */ schemas?: { body?: unknown; query?: unknown; params?: unknown }; + /** + * Custom blocks this graph calls, by file name in `blocks/`. They are + * compiled and registered into the block library before the route is + * compiled — the same order the real compiler works in, and the reason a + * route calling one resolves at all. + */ + uses?: string[]; + blocks: BlockDTOType[]; + edges: EdgeDTOSchemaType; +}; + +/** + * A custom block as the portal saves it: a graph plus the input contract its + * callers configure. `name` is what a calling block carries as its type. + */ +export type CustomBlockFixture = { + name: string; + description: string; + inputParams?: unknown[]; blocks: BlockDTOType[]; edges: EdgeDTOSchemaType; }; const graphsDir = join(import.meta.dir, "..", "graphs"); +const blocksDir = join(import.meta.dir, "..", "blocks"); export async function loadGraph(name: string): Promise { const fixture = (await Bun.file( @@ -44,7 +64,22 @@ export async function loadGraph(name: string): Promise { * named by path (`auth/login`). */ export function graphNames(): string[] { - return readdirSync(graphsDir, { recursive: true }) + return jsonNames(graphsDir); +} + +/** Loads one custom block by file name — `blocks/jwt-ops.json` is `jwt-ops`. */ +export async function loadCustomBlock(file: string): Promise { + return (await Bun.file( + join(blocksDir, `${file}.json`), + ).json()) as CustomBlockFixture; +} + +export function customBlockFiles(): string[] { + return jsonNames(blocksDir); +} + +function jsonNames(dir: string): string[] { + return readdirSync(dir, { recursive: true }) .map((file) => String(file).replaceAll("\\", "/")) .filter((file) => file.endsWith(".json")) .map((file) => file.slice(0, -".json".length)); diff --git a/testing/e2e/src/runner.ts b/testing/e2e/src/runner.ts index 2c119366..a3656471 100644 --- a/testing/e2e/src/runner.ts +++ b/testing/e2e/src/runner.ts @@ -6,6 +6,7 @@ import { import { setBlocksExecutor } from "@fluxify/server/src/modules/requestRouter/executor"; import { executeRouteInternal } from "@fluxify/server/src/modules/requestRouter/service"; import { connectionFor } from "./engines"; +import { registerFixtureBlocks } from "./customBlocks"; import type { GraphFixture } from "./graph"; /** @@ -49,7 +50,10 @@ export type GraphRun = { /** Points the project's `primary` db integration at the fixture's engine. */ async function hydrateDatabase(fixture: GraphFixture) { - const connection = await connectionFor(fixture.engine ?? "pg"); + const engine = fixture.engine ?? "pg"; + // a graph that touches no database should not start a container for it + if (engine === "none") return; + const connection = await connectionFor(engine); hydrateIntegrations(PROJECT_ID, { db: { [DB_CONNECTION]: { ...connection, [OWNER_KEY]: PROJECT_ID } }, }); @@ -60,7 +64,19 @@ export async function runGraph( request: GraphRequest = {}, ): Promise { await hydrateDatabase(fixture); + const disposeBlocks = await registerFixtureBlocks(fixture); + try { + return await execute(fixture, request); + } finally { + disposeBlocks(); + } +} + +async function execute( + fixture: GraphFixture, + request: GraphRequest, +): Promise { const { run, source } = compileGraph(fixture.blocks, fixture.edges); const spans: BlockTraceSpan[] = []; diff --git a/testing/e2e/tests/fixtures.test.ts b/testing/e2e/tests/fixtures.test.ts index 17bf7348..8b8dd98c 100644 --- a/testing/e2e/tests/fixtures.test.ts +++ b/testing/e2e/tests/fixtures.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from "bun:test"; import { compileGraph } from "@fluxify/blocks"; -import { graphNames, loadGraph } from "../src/graph"; +import { registerFixtureBlocks } from "../src/customBlocks"; +import { + customBlockFiles, + graphNames, + loadCustomBlock, + loadGraph, +} from "../src/graph"; /** * Guards the fixtures themselves. A graph that no longer compiles is a broken @@ -16,7 +22,25 @@ describe("graph fixtures", () => { it(`${name} compiles`, async () => { const fixture = await loadGraph(name); expect(fixture.route.path.startsWith("/")).toBe(true); - expect(compileGraph(fixture.blocks, fixture.edges).source).toBeString(); + // a caller only emits once its custom blocks are in the library + const dispose = await registerFixtureBlocks(fixture); + try { + expect(compileGraph(fixture.blocks, fixture.edges).source).toBeString(); + } finally { + dispose(); + } + }); + } +}); + +describe("custom block fixtures", () => { + for (const file of customBlockFiles()) { + it(`${file} compiles as a custom block`, async () => { + const block = await loadCustomBlock(file); + const { source } = compileGraph(block.blocks, block.edges, { + asCustomBlock: true, + }); + expect(source).toBeString(); }); } }); diff --git a/testing/e2e/tests/jwtCustomBlock.test.ts b/testing/e2e/tests/jwtCustomBlock.test.ts new file mode 100644 index 00000000..dcc807bc --- /dev/null +++ b/testing/e2e/tests/jwtCustomBlock.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "bun:test"; +import { loadGraph } from "../src/graph"; +import { runGraph } from "../src/runner"; + +/** + * One custom block, three routes, three configurations. What this covers that a + * block unit spec cannot: the params a caller configures actually reach the + * callee's graph, the callee's own branching runs, and a response block inside a + * custom block ends the block rather than the request. + * + * No database — the graphs are pure JS, so the fixtures declare `engine: none` + * and no container starts. + */ + +const sign = await loadGraph("custom-blocks/jwt-sign"); +const verify = await loadGraph("custom-blocks/jwt-verify"); +const strict = await loadGraph("custom-blocks/jwt-verify-strict"); + +async function issueToken(sub = "user-42") { + const run = await runGraph(sign, { body: { sub } }); + return run.body.token as string; +} + +describe("jwt_ops custom block", () => { + it("signs a token through the block's sign branch", async () => { + const run = await runGraph(sign, { body: { sub: "user-42" } }); + + expect(run.status).toBe(200); + expect(run.body.token).toBeString(); + // three segments: the block really produced a JWT, not a stringified object + expect(run.body.token.split(".")).toHaveLength(3); + }); + + it("runs the callee's blocks inside the caller's trace", async () => { + const run = await runGraph(sign, { body: { sub: "user-42" } }); + + // the custom block's own ids appear between the caller's, which is the + // only way to tell the block ran from the caller merely returning + expect(run.executed).toEqual([ + "entry", + "claims", + "cb-entry", + "cb-is-sign", + "cb-sign", + "sign", + "reply", + ]); + expect(run.spans.every((span) => span.outcome === "success")).toBe(true); + }); + + it("verifies a token it signed, taking the other branch of the same block", async () => { + const token = await issueToken("user-42"); + const run = await runGraph(verify, { body: { token } }); + + expect(run.status).toBe(200); + expect(run.body.valid).toBe(true); + expect(run.body.payload).toMatchObject({ sub: "user-42", role: "admin" }); + // the `operation` param picked the verify path, not the sign path + expect(run.executed).toContain("cb-verify"); + expect(run.executed).not.toContain("cb-sign"); + }); + + it("returns valid:false for a bad token when the block is not strict", async () => { + const run = await runGraph(verify, { body: { token: "not.a.token" } }); + + expect(run.status).toBe(200); + expect(run.body).toEqual({ valid: false, error: "invalid_token" }); + // the guard let it through: no response block ran inside the callee + expect(run.executed).not.toContain("cb-reject"); + }); + + it("ends on the block's own response block when failOnInvalid is set", async () => { + const run = await runGraph(strict, { body: { token: "not.a.token" } }); + + expect(run.status).toBe(401); + expect(run.body).toEqual({ valid: false, error: "invalid_token" }); + expect(run.executed).toContain("cb-reject"); + // the callee's response ended the callee, not the request — the caller + // still got to branch on it + expect(run.executed).toContain("block-refused"); + }); + + it("passes a good token straight through the strict route", async () => { + const token = await issueToken("user-7"); + const run = await runGraph(strict, { body: { token } }); + + expect(run.status).toBe(200); + expect(run.body.payload).toMatchObject({ sub: "user-7" }); + expect(run.executed).not.toContain("cb-reject"); + }); + + it("rejects a token signed with a different secret", async () => { + // the secret lives inside the block; nothing the caller passes can change it + const foreign = [ + Buffer.from(JSON.stringify({ alg: "HS256", typ: "JWT" })).toString( + "base64url", + ), + Buffer.from(JSON.stringify({ sub: "intruder" })).toString("base64url"), + "forged", + ].join("."); + + const run = await runGraph(strict, { body: { token: foreign } }); + expect(run.status).toBe(401); + }); + + it("validates the request before any block runs", async () => { + const run = await runGraph(verify, { body: {} }); + + expect(run.status).toBe(400); + expect(run.executed).toEqual([]); + }); + + it("compiles the caller to an invoke rather than inlining the block", async () => { + const run = await runGraph(sign, { body: { sub: "user-42" } }); + + expect(run.source).toContain('lib.invoke(ctx, "jwt_ops"'); + // the callee's body is compiled once, into the library — not here + expect(run.source).not.toContain("e2e-custom-block-secret"); + }); +});