diff --git a/apps/portal/src/components/canvas/BlockPickerSidebar.tsx b/apps/portal/src/components/canvas/BlockPickerSidebar.tsx index 359bee90..fe177e9d 100644 --- a/apps/portal/src/components/canvas/BlockPickerSidebar.tsx +++ b/apps/portal/src/components/canvas/BlockPickerSidebar.tsx @@ -1,7 +1,8 @@ import { useEffect, useMemo, useState, type ReactNode } from "react"; -import { Button, Input, Label, Sidebar, TextField } from "@fluxify/components"; +import { Button, cn, Input, Label, Sidebar, TextField } from "@fluxify/components"; import { TbArrowLeft, + TbBoxMultiple, TbChevronRight, TbCode, TbDatabase, @@ -12,15 +13,18 @@ import { TbWorld, TbX, } from "react-icons/tb"; +import { CustomBlockIcon } from "@/components/customBlocks/IconPicker"; import { pickerBlockCatalogEntries, blockIcon, type BlockDefinition, type BlockType, } from "./blocks"; +import { useCustomBlockDefs } from "./blocks/useCustomBlockDefs"; import "./blockPickerSidebar.css"; -type BlockCategory = BlockDefinition["category"]; +/** Catalog categories plus the ones only custom blocks land in. */ +type BlockCategory = BlockDefinition["category"] | "Custom" | "Built-in"; type CategoryDetails = { description: string; @@ -52,9 +56,27 @@ const CATEGORY_DETAILS: Record = { description: "Transform data and annotate the canvas", icon: , }, + Custom: { + description: "Blocks built in this project", + icon: , + }, + "Built-in": { + description: "Blocks shipped with plugins", + icon: , + }, }; -const categories = Object.keys(CATEGORY_DETAILS) as BlockCategory[]; +const allCategories = Object.keys(CATEGORY_DETAILS) as BlockCategory[]; + +type PickerItem = { + type: string; + name: string; + description: string; + category: BlockCategory; + icon: ReactNode; + /** Set when the block cannot be added — the reason is shown in its place. */ + disabledReason?: string; +}; export type BlockPickerSidebarProps = { isOpen: boolean; @@ -62,7 +84,7 @@ export type BlockPickerSidebarProps = { onAdd: (type: BlockType) => void; }; -/** Core-block picker. Custom blocks join this catalog after their API lands. */ +/** Core blocks from the catalog plus the project's own custom blocks. */ export function BlockPickerSidebar({ isOpen, onOpenChange, @@ -80,15 +102,42 @@ export function BlockPickerSidebar({ } }, [isOpen]); - const blocks = useMemo(() => pickerBlockCatalogEntries(), []); + const customDefs = useCustomBlockDefs(); + const blocks = useMemo(() => { + const core = pickerBlockCatalogEntries().map(([type, definition]) => ({ + type: type as string, + name: definition.name, + description: definition.description, + category: definition.category as BlockCategory, + icon: blockIcon(type), + })); + const custom = customDefs.map((def) => ({ + type: def.name, + name: def.label, + description: def.description ?? "Custom block", + category: (def.sourceType === "plugin" ? "Built-in" : "Custom") as BlockCategory, + icon: , + disabledReason: def.isSelf + ? "A block can't call itself — that would recurse forever." + : undefined, + })); + return [...custom, ...core]; + }, [customDefs]); + + // An empty category is a dead end — only offer the ones holding something. + const categories = useMemo( + () => allCategories.filter((category) => blocks.some((b) => b.category === category)), + [blocks], + ); + const visibleBlocks = useMemo(() => { const normalizedQuery = query.trim().toLowerCase(); - return blocks.filter(([type, definition]) => { - if (!normalizedQuery && selectedCategory && definition.category !== selectedCategory) { + return blocks.filter((block) => { + if (!normalizedQuery && selectedCategory && block.category !== selectedCategory) { return false; } if (!normalizedQuery) return true; - return [type, definition.name, definition.description, definition.category] + return [block.type, block.name, block.description, block.category] .join(" ") .toLowerCase() .includes(normalizedQuery); @@ -177,26 +226,36 @@ export function BlockPickerSidebar({ ) : (
- {visibleBlocks.map(([type, definition]) => ( + {visibleBlocks.map((block) => ( ))} {visibleBlocks.length === 0 && ( -

No core blocks match.

+

No blocks match.

)}
)} diff --git a/apps/portal/src/components/canvas/blocks/BlockNode.tsx b/apps/portal/src/components/canvas/blocks/BlockNode.tsx index 1ae006b6..337a990e 100644 --- a/apps/portal/src/components/canvas/blocks/BlockNode.tsx +++ b/apps/portal/src/components/canvas/blocks/BlockNode.tsx @@ -1,4 +1,5 @@ import type { NodeProps, NodeTypes } from "@xyflow/react"; +import { CustomBlockIcon } from "@/components/customBlocks/IconPicker"; import { BaseBlock } from "./BaseBlock"; import { blockCatalogEntries } from "./blockCatalog"; import { blockIcon } from "./blockIconMap"; @@ -6,6 +7,7 @@ import { blockLabels } from "./blockLabels"; import { BLOCK_TYPES } from "./blockTypes"; import { BlockHandle } from "./handles/BlockHandle"; import { StickyNoteBlock } from "./StickyNoteBlock"; +import { useCustomBlockDefs } from "./useCustomBlockDefs"; function status(value: unknown): boolean | null { return typeof value === "boolean" ? value : null; @@ -23,7 +25,10 @@ export function BlockNode({ positionAbsoluteX, positionAbsoluteY, }: NodeProps) { - const { name, description, definition } = blockLabels(type, data); + const { name, description, definition, custom } = blockLabels(type, data); + // A custom block has no catalog entry: its name, blurb and icon live in the DB. + // Anything the user typed on this node still wins. + const customDef = useCustomBlockDefs().find((def) => def.name === type); const isRouteOwned = type === BLOCK_TYPES.entrypoint || type === BLOCK_TYPES.errorHandler; @@ -32,9 +37,19 @@ export function BlockNode({ blockId={id} blockType={type} position={{ x: positionAbsoluteX, y: positionAbsoluteY }} - name={name} - description={description} - icon={blockIcon(type)} + name={custom ? name : (customDef?.label ?? name)} + description={ + description === definition.description + ? (customDef?.description ?? description) + : description + } + icon={ + customDef ? ( + + ) : ( + blockIcon(type) + ) + } color={definition.tint} selected={selected} status={status(data?.status)} diff --git a/apps/portal/src/components/canvas/blocks/blocks.css b/apps/portal/src/components/canvas/blocks/blocks.css index 70a164ce..8c232ea0 100644 --- a/apps/portal/src/components/canvas/blocks/blocks.css +++ b/apps/portal/src/components/canvas/blocks/blocks.css @@ -436,25 +436,30 @@ html[data-theme="dark"] .fx-block, min-height: 0; } -.fx-handle--circle { +/* Every shape rule carries .react-flow__handle: the library's own + `.react-flow__handle { width: 6px; height: 6px; border-radius: 100% }` is a + single class too, so on a tie the sheet that loads last wins. Importing this + file from a non-canvas module (a block preview, say) is enough to flip that + order and turn the inbound bar into an oval. */ +.fx-handle--circle.react-flow__handle { width: 10px; height: 10px; border-radius: 9999px; } /* The inbound socket: a bar on the block's edge. */ -.fx-handle--rect { +.fx-handle--rect.react-flow__handle { border-radius: 2px; } -.fx-handle--rect.fx-handle--left, -.fx-handle--rect.fx-handle--right { +.fx-handle--rect.fx-handle--left.react-flow__handle, +.fx-handle--rect.fx-handle--right.react-flow__handle { width: 6px; height: 16px; } -.fx-handle--rect.fx-handle--top, -.fx-handle--rect.fx-handle--bottom { +.fx-handle--rect.fx-handle--top.react-flow__handle, +.fx-handle--rect.fx-handle--bottom.react-flow__handle { width: 16px; height: 6px; } diff --git a/apps/portal/src/components/canvas/blocks/defaultBlockData.ts b/apps/portal/src/components/canvas/blocks/defaultBlockData.ts index 61f93eb1..256b9a0f 100644 --- a/apps/portal/src/components/canvas/blocks/defaultBlockData.ts +++ b/apps/portal/src/components/canvas/blocks/defaultBlockData.ts @@ -1,4 +1,4 @@ -import { BLOCK_TYPES, type BlockType } from "./blockTypes"; +import { BLOCK_TYPES, BLOCK_TYPE_LIST, type BlockType } from "./blockTypes"; /** * Complete, schema-valid starting data for every core block created on canvas. @@ -6,6 +6,9 @@ import { BLOCK_TYPES, type BlockType } from "./blockTypes"; * insertion affordances always create the same valid block payload. */ export function defaultBlockData(type: BlockType): Record { + // A custom block: its input params are filled in the panel, but the engine + // always reads an invocation mode. + if (!BLOCK_TYPE_LIST.includes(type)) return { invoke: "sync" }; if (type === BLOCK_TYPES.response) return { httpCode: "200" }; if (type === BLOCK_TYPES.if) return { conditions: [] }; if (type === BLOCK_TYPES.forloop) return { start: 0, end: 1, step: 1 }; diff --git a/apps/portal/src/components/canvas/blocks/index.ts b/apps/portal/src/components/canvas/blocks/index.ts index 3b25bda7..23bd4ead 100644 --- a/apps/portal/src/components/canvas/blocks/index.ts +++ b/apps/portal/src/components/canvas/blocks/index.ts @@ -18,6 +18,7 @@ export { type BlockDefinition, } from "./blockCatalog"; export { blockLabels, type BlockLabels } from "./blockLabels"; +export { useCustomBlockDefs, type CustomBlockDef } from "./useCustomBlockDefs"; export { BLOCK_TYPES, BLOCK_TYPE_LIST, type BlockType } from "./blockTypes"; export { StickyNoteBlock } from "./StickyNoteBlock"; export { diff --git a/apps/portal/src/components/canvas/blocks/useCustomBlockDefs.ts b/apps/portal/src/components/canvas/blocks/useCustomBlockDefs.ts new file mode 100644 index 00000000..8ba6ad09 --- /dev/null +++ b/apps/portal/src/components/canvas/blocks/useCustomBlockDefs.ts @@ -0,0 +1,46 @@ +import { useMemo } from "react"; +import { useParams } from "@tanstack/react-router"; +import { customBlocksQuery } from "@/query/customBlocksQuery"; +import type { IconValue } from "@/components/customBlocks/IconPicker"; + +export type CustomBlockDef = { + id: string; + /** The block type on canvas — what the engine looks up. */ + name: string; + label: string; + description?: string; + icon?: IconValue["icon"]; + iconUrl?: string; + sourceType?: string | null; + /** This is the block whose canvas is open: adding it would be recursion. */ + isSelf?: boolean; +}; + +/** + * The project's custom blocks, shaped like catalog entries so the picker and the + * node can render them. On a custom block's own canvas (`blockId` in the route) + * that block is flagged `isSelf` — the picker offers it disabled rather than + * letting a block call itself. + */ +export function useCustomBlockDefs(): CustomBlockDef[] { + const params = useParams({ strict: false }) as { + projectId?: string; + blockId?: string; + }; + const projectId = params?.projectId ?? ""; + const { data } = customBlocksQuery.getAll.useQuery(projectId); + + return useMemo(() => { + if (!data) return []; + return data.map((block) => ({ + id: block.id, + name: block.name, + label: block.label || block.name, + description: block.description ?? undefined, + icon: (block.icon as IconValue["icon"]) ?? undefined, + iconUrl: block.iconUrl ?? undefined, + sourceType: block.sourceType, + isSelf: block.id === params?.blockId, + })); + }, [data, params?.blockId]); +} diff --git a/apps/portal/src/components/canvas/panel/BlockPanel.tsx b/apps/portal/src/components/canvas/panel/BlockPanel.tsx index 3f39b1fc..f76fe978 100644 --- a/apps/portal/src/components/canvas/panel/BlockPanel.tsx +++ b/apps/portal/src/components/canvas/panel/BlockPanel.tsx @@ -4,8 +4,10 @@ import "./panel.css"; import { BlockSettings } from "./BlockSettings"; import { blockSettingsTabs } from "./blockSettingsRegistry"; import { useBlockPanelResize } from "./useBlockPanelResize"; +import { CustomBlockIcon } from "@/components/customBlocks/IconPicker"; import { blockIcon } from "../blocks/blockIconMap"; import { blockLabels } from "../blocks/blockLabels"; +import { useCustomBlockDefs } from "../blocks/useCustomBlockDefs"; import type { BlockNode } from "../types"; export type BlockPanelProps = { @@ -47,7 +49,12 @@ export function BlockPanel({ const current = block ?? shown.current; const type = current?.type ?? "unknown"; - const { name, description, definition } = blockLabels(type, current?.data); + const { name, description, definition, custom } = blockLabels(type, current?.data); + // A custom block: the label titles the panel and the identifier sits under it, + // since that is what flows and `param:` references are written against. + const customDef = useCustomBlockDefs().find((def) => def.name === type); + const title = custom ? name : (customDef?.label ?? name); + const subtitle = customDef ? customDef.name : description; const tabs = blockSettingsTabs(current?.type); const { @@ -110,16 +117,23 @@ export function BlockPanel({ className="fx-panel__icon" style={definition.tint ? { color: definition.tint } : undefined} > - {blockIcon(type)} + {customDef ? ( + + ) : ( + blockIcon(type) + )} {/* Renaming lives in the General tab; the header only shows it. */} - - {name} + + {title} - {/* What the block does, under whatever it was named. */} - - {description} + {/* What the block does — or, for a custom block, what it is called. */} + + {subtitle} {current.id && ( diff --git a/apps/portal/src/components/canvas/panel/blocks/CustomBlockSettings.tsx b/apps/portal/src/components/canvas/panel/blocks/CustomBlockSettings.tsx index 55b9c975..eb271cb3 100644 --- a/apps/portal/src/components/canvas/panel/blocks/CustomBlockSettings.tsx +++ b/apps/portal/src/components/canvas/panel/blocks/CustomBlockSettings.tsx @@ -58,7 +58,7 @@ export function CustomBlockSettingsPanel({ block }: { block: BlockNode }) { onPress={() => window.open( withBasePath( - `/${projectId}/custom-blocks/${customBlock.id}`, + `/${projectId}/custom-block-canvas/${customBlock.id}`, ), "_blank", "noopener,noreferrer", @@ -76,11 +76,12 @@ export function CustomBlockSettingsPanel({ block }: { block: BlockNode }) { data={block.data} name="invoke" label="Execution Mode" - hint="Sync waits for execution and passes output; Async fires in background." + hint="Sync waits for the output. Async fires on this worker and is lost if it restarts. Queued is durable — another worker picks it up, and it may be retried." placeholder="Select execution mode" options={[ { value: "sync", label: "Synchronous (Wait for output)" }, { value: "async", label: "Asynchronous (Fire & forget)" }, + { value: "queued", label: "Queued (Durable background job)" }, ]} /> diff --git a/apps/portal/src/components/canvas/panel/fields.tsx b/apps/portal/src/components/canvas/panel/fields.tsx index eca4496a..a63d6293 100644 --- a/apps/portal/src/components/canvas/panel/fields.tsx +++ b/apps/portal/src/components/canvas/panel/fields.tsx @@ -14,10 +14,12 @@ import { } from "@fluxify/components"; import { useParams } from "@tanstack/react-router"; import { useReactFlow } from "@xyflow/react"; -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import type { ReactNode } from "react"; import { withBasePath } from "@/constants/routes"; import { integrationService } from "@/services/integrations"; +import { customBlocksQuery } from "@/query/customBlocksQuery"; +import type { CustomBlockInputParam } from "./blocks/CustomBlockSettings"; import { useCanvasChanges } from "../changes/ChangesContext"; import type { BlockData } from "../types"; @@ -224,6 +226,43 @@ export function BlockCheckboxField({ ); } +/** + * On a custom block's canvas, the block's own `integration_selector` input + * parameters are offered alongside the project's integrations. Picking one + * writes `param:`, which the engine substitutes with whatever the calling + * block was configured with — so the concrete integration is chosen on the + * caller's side, not here. `blockId` is only present on the custom block route, + * which is what keeps this out of route canvases. + */ +function useCustomBlockParamIntegrations( + projectId: string, + customBlockId: string | undefined, + group: string, +) { + const { data: blocks } = customBlocksQuery.getAll.useQuery(projectId); + + return useMemo(() => { + if (!customBlockId) return undefined; + const block = blocks?.find((b) => b.id === customBlockId); + const params = Array.isArray(block?.inputParams) + ? (block.inputParams as CustomBlockInputParam[]) + : []; + const matching = params.filter( + (param) => param.type === "integration_selector" && param.group === group, + ); + if (matching.length === 0) return undefined; + return matching.map((param) => ({ + id: `param:${param.name}`, + name: param.label || param.name, + group, + variant: "Input parameter", + config: {}, + external: true, + hint: `Set by whoever places this block — the “${param.label || param.name}” input.`, + })); + }, [blocks, customBlockId, group]); +} + export type BlockIntegrationFieldProps = { blockId: string; data: BlockData; @@ -246,9 +285,17 @@ export function BlockIntegrationField({ }: BlockIntegrationFieldProps) { const { updateNodeData } = useReactFlow(); const { enabled: editable } = useCanvasChanges(); - const params = useParams({ strict: false }) as { projectId?: string }; + const params = useParams({ strict: false }) as { + projectId?: string; + blockId?: string; + }; const projectId = params?.projectId ?? ""; const selectedId = typeof data[name] === "string" ? (data[name] as string) : ""; + const injectedIntegrations = useCustomBlockParamIntegrations( + projectId, + params?.blockId, + group, + ); const loadIntegrations = useCallback(async () => { if (!projectId) return []; @@ -283,6 +330,7 @@ export function BlockIntegrationField({ group={group} selectedId={selectedId} loadIntegrations={loadIntegrations} + injectedIntegrations={injectedIntegrations} onSelect={(id) => { if (!editable) return; updateNodeData(blockId, { [name]: id }); diff --git a/apps/portal/src/components/customBlocks/CustomBlockSettingsModal.tsx b/apps/portal/src/components/customBlocks/CustomBlockSettingsModal.tsx new file mode 100644 index 00000000..4730910a --- /dev/null +++ b/apps/portal/src/components/customBlocks/CustomBlockSettingsModal.tsx @@ -0,0 +1,311 @@ +import { useMemo, useState } from "react"; +import { useNavigate } from "@tanstack/react-router"; +import { + Button, + CloseButton, + DeleteButton, + Input, + Label, + Modal, + Spinner, + Tabs, + TextField, + toast, +} from "@fluxify/components"; +import type { inputParamSchema } from "@fluxify/server/src/api/v1/custom-blocks/create/dto"; +import type { z } from "zod"; +import { customBlocksQuery } from "@/query/customBlocksQuery"; +import { showErrorNotification } from "@/lib/errorNotifier"; +import { ConfirmDialog } from "@/components/common/ConfirmDialog"; +import type { CustomBlockInputParam } from "@/components/canvas/panel/blocks/CustomBlockSettings"; +import { InputParamsEditor, validateInputParams } from "./InputParamsEditor"; +import { ICON_URL_MAX, IconPicker, type IconValue } from "./IconPicker"; + +/** + * Same shape as the route canvas' settings modal: everything editable about the + * block, reachable from the canvas it belongs to, one save for the lot. + */ +export function CustomBlockSettingsModal({ + projectId, + blockId, + isOpen, + onOpenChange, +}: { + projectId: string; + blockId: string; + isOpen: boolean; + onOpenChange: (open: boolean) => void; +}) { + const { data: blocks, isLoading } = customBlocksQuery.getAll.useQuery(projectId); + const block = blocks?.find((b) => b.id === blockId); + + return ( + + + + + {isLoading || !block ? ( +
+ {isLoading ? :

Custom block not found.

} +
+ ) : ( + onOpenChange(false)} + onClose={() => onOpenChange(false)} + /> + )} + + + + + ); +} + +type BlockData = NonNullable< + ReturnType["data"] +>[number]; + +function CustomBlockSettingsForm({ + projectId, + block, + onSaved, + onClose, +}: { + projectId: string; + block: BlockData; + onSaved: () => void; + onClose: () => void; +}) { + const update = customBlocksQuery.update.mutation(projectId, block.id); + const readOnly = Boolean(block.sourceType && block.sourceType !== "user-defined"); + + const [label, setLabel] = useState(block.label); + const [description, setDescription] = useState(block.description ?? ""); + const [params, setParams] = useState( + Array.isArray(block.inputParams) ? (block.inputParams as CustomBlockInputParam[]) : [], + ); + const [iconValue, setIconValue] = useState({ + icon: (block.icon as IconValue["icon"]) ?? undefined, + iconUrl: block.iconUrl ?? undefined, + }); + const [tab, setTab] = useState("general"); + const [confirmDelete, setConfirmDelete] = useState(false); + const remove = customBlocksQuery.remove.mutation(projectId); + const navigate = useNavigate(); + // a plugin block is owned by its plugin; the API refuses to delete it + const canDelete = block.sourceType !== "plugin"; + + function deleteBlock() { + remove.mutate(block.id, { + onSuccess: () => { + toast.success("Custom block deleted"); + setConfirmDelete(false); + onClose(); + navigate({ to: "/$projectId/custom-blocks", params: { projectId } }); + }, + onError: (error) => showErrorNotification(error as Error), + }); + } + + const payload = useMemo( + () => ({ + label: label.trim(), + description, + icon: iconValue.icon, + iconUrl: iconValue.iconUrl, + inputParams: params as unknown as z.infer[], + }), + [label, description, params, iconValue], + ); + const [baseline] = useState(() => JSON.stringify(payload)); + const isDirty = JSON.stringify(payload) !== baseline; + const labelIsValid = label.trim().length > 0; + + function save() { + if ((iconValue.iconUrl?.length ?? 0) > ICON_URL_MAX) { + toast.danger("Icon image is too large"); + setTab("general"); + return; + } + const error = validateInputParams(params); + if (error) { + toast.danger(error); + setTab("inputs"); + return; + } + update.mutate(payload, { + onSuccess: () => { + toast.success("Custom block settings saved"); + onSaved(); + }, + onError: (error) => showErrorNotification(error as Error), + }); + } + + return ( + <> + +
+ + Custom block settings + +

{block.name}

+
+ + + + + setTab(String(key))} + className="flex h-full min-h-0 flex-row" + > + + General + Input parameters + {canDelete && Danger zone} + + + {/* the icon grid inside scrolls, the panel itself never does */} + + +
+

Identity

+

+ How this block shows up in the block picker and on a route canvas. + The name is fixed once created. +

+
+ + + + + + + + + + } + /> + + + +
+ +
+ + + {canDelete && ( + +
+
+
+

{block.label}

+

{block.name}

+
+ setConfirmDelete(true)}> + Delete block + +
+
+ + )} + + + + + Delete {block.label}? This can't be undone. + + + + + {readOnly + ? `This block comes from a ${block.sourceType} source and isn't editable here.` + : isDirty + ? "Unsaved changes" + : "All changes saved"} + +
+ + {!readOnly && ( + + )} +
+ + + ); +} + +function Section({ + title, + description, + children, +}: { + title: string; + description: string; + children: React.ReactNode; +}) { + return ( +
+
+

{title}

+

{description}

+
+ {children} +
+ ); +} diff --git a/apps/portal/src/components/customBlocks/CustomBlockSwitcher.tsx b/apps/portal/src/components/customBlocks/CustomBlockSwitcher.tsx new file mode 100644 index 00000000..7e959201 --- /dev/null +++ b/apps/portal/src/components/customBlocks/CustomBlockSwitcher.tsx @@ -0,0 +1,119 @@ +import { useNavigate } from "@tanstack/react-router"; +import { Button, ListBox, Select, Spinner } from "@fluxify/components"; +import { TbArrowLeft, TbChevronLeft, TbChevronRight } from "react-icons/tb"; +import { customBlocksQuery } from "@/query/customBlocksQuery"; + +/** + * Canvas header nav, same shape as the route canvas' switcher: back to the + * project's custom block list plus a stepper over the sibling blocks. + */ +export function CustomBlockSwitcher({ + projectId, + blockId, +}: { + projectId: string; + blockId: string; +}) { + const navigate = useNavigate(); + const { data, isLoading } = customBlocksQuery.getAll.useQuery(projectId); + const blocks = data ?? []; + const index = blocks.findIndex((block) => block.id === blockId); + const current = index >= 0 ? blocks[index] : undefined; + + function go(to: string) { + navigate({ + to: "/$projectId/custom-block-canvas/$blockId", + params: { projectId, blockId: to }, + }); + } + + function step(delta: number) { + const next = blocks[index + delta]; + if (next) go(next.id); + } + + return ( +
+ + + + + {isLoading ? ( + + ) : ( +
+ + + + + + + {index >= 0 && ( + + {index + 1} / {blocks.length} + + )} +
+ )} +
+ ); +} diff --git a/apps/portal/src/components/customBlocks/IconPicker.tsx b/apps/portal/src/components/customBlocks/IconPicker.tsx new file mode 100644 index 00000000..0236001b --- /dev/null +++ b/apps/portal/src/components/customBlocks/IconPicker.tsx @@ -0,0 +1,164 @@ +import { useMemo, useState } from "react"; +import { Button, Input, Label, Tabs, TextField } from "@fluxify/components"; +import { TbBox, TbSearch } from "react-icons/tb"; +import { BaseBlock } from "@/components/canvas/blocks/BaseBlock"; +import { PREMADE_ICON_NAMES, premadeIcon } from "./premadeIcons"; + +/** `iconUrl` is `text` in the DB but the API caps it — see custom-blocks/create/dto.ts. */ +export const ICON_URL_MAX = 68266; + +export type IconKind = "premade-list" | "custom"; + +export type IconValue = { + icon?: IconKind; + /** Premade icon name, or a URL / data URI when `icon` is `custom`. */ + iconUrl?: string; +}; + +/** The icon a block shows, for both the preview here and (later) the canvas. */ +export function CustomBlockIcon({ icon, iconUrl, size = 18 }: IconValue & { size?: number }) { + if (icon === "custom" && iconUrl) { + return ; + } + const Premade = icon === "premade-list" ? premadeIcon(iconUrl) : undefined; + return Premade ? : ; +} + +export function IconPicker({ + value, + onChange, + isDisabled, + previewName, + previewDescription, + header, +}: { + value: IconValue; + onChange: (next: IconValue) => void; + isDisabled?: boolean; + previewName: string; + previewDescription?: string; + /** Rendered in the left column, above the preview. */ + header?: React.ReactNode; +}) { + const [search, setSearch] = useState(""); + const custom = value.icon === "custom"; + + const results = useMemo(() => { + const q = search.trim().toLowerCase(); + if (!q) return PREMADE_ICON_NAMES; + return PREMADE_ICON_NAMES.filter((name) => name.includes(q)); + }, [search]); + + const customUrl = custom ? (value.iconUrl ?? "") : ""; + const tooLong = customUrl.length > ICON_URL_MAX; + + return ( +
+ {/* left 30%: what the block will actually look like */} +
+ {header} +
+
+ } + showToolbar={false} + /> +
+

+ {custom + ? customUrl + ? "Custom image" + : "No image yet — default icon" + : (value.iconUrl ?? "No icon selected — default icon")} +

+
+ {value.icon && ( + + )} +
+ + {/* right 70%: the picker itself, the only thing allowed to scroll */} + + onChange( + key === "custom" + ? { icon: "custom", iconUrl: "" } + : { icon: "premade-list", iconUrl: undefined }, + ) + } + className="flex min-h-0 w-[70%] flex-1 flex-col" + > + + Premade + URL / base64 + + + + + + + + {results.length === 0 ? ( +

+ No icon matches “{search}”. +

+ ) : ( +
+ {results.map((name) => { + const Icon = premadeIcon(name)!; + const selected = value.icon === "premade-list" && value.iconUrl === name; + return ( + + ); + })} +
+ )} + + + + onChange({ icon: "custom", iconUrl: next })} + isDisabled={isDisabled} + isInvalid={tooLong} + > + + +

+ {customUrl.length.toLocaleString()} / {ICON_URL_MAX.toLocaleString()} characters + {tooLong ? " — too large, use a smaller image or a URL" : ""} +

+ + + +
+ ); +} diff --git a/apps/portal/src/components/customBlocks/InputParamsEditor.tsx b/apps/portal/src/components/customBlocks/InputParamsEditor.tsx new file mode 100644 index 00000000..0bd62b00 --- /dev/null +++ b/apps/portal/src/components/customBlocks/InputParamsEditor.tsx @@ -0,0 +1,307 @@ +import { useCallback, useId } from "react"; +import { + ArrayEditor, + Button, + CustomSelect, + DeleteIconButton, + Input, + Label, + TextField, +} from "@fluxify/components"; +import { TbPlus } from "react-icons/tb"; +import type { CustomBlockInputParam } from "@/components/canvas/panel/blocks/CustomBlockSettings"; + +const PARAM_TYPE_OPTIONS = [ + { value: "text_input", label: "Text input" }, + { value: "checkbox", label: "Checkbox" }, + { value: "dropdown", label: "Dropdown" }, + { value: "array_editor", label: "Array editor" }, + { value: "integration_selector", label: "Integration selector" }, +]; + +const NAME_REGEX = /^[a-z0-9_]+$/; + +function emptyParam( + type: CustomBlockInputParam["type"], +): CustomBlockInputParam { + const base = { name: "", label: "", type, description: "" }; + if (type === "dropdown") return { ...base, options: [] }; + if (type === "integration_selector") return { ...base, group: "", tags: [] }; + return base; +} + +export function validateInputParams( + params: CustomBlockInputParam[], +): string | null { + const seen = new Set(); + for (const p of params) { + if (!p.name.trim() || !NAME_REGEX.test(p.name)) { + return `"${p.label || p.name || "unnamed"}": name must be lowercase letters, digits, underscores.`; + } + if (seen.has(p.name)) return `Duplicate parameter name "${p.name}".`; + seen.add(p.name); + if (!p.label.trim()) return `"${p.name}": label is required.`; + if (p.type === "dropdown") { + const options = p.options ?? []; + if (options.length === 0) return `"${p.name}": add at least one option.`; + for (const opt of options) { + const value = typeof opt === "string" ? opt : opt.value; + if (!value?.trim()) return `"${p.name}": option values can't be empty.`; + } + } + if (p.type === "integration_selector" && !p.group?.trim()) { + return `"${p.name}": integration group is required.`; + } + } + return null; +} + +export function InputParamsEditor({ + params, + onChange, + isDisabled, +}: { + params: CustomBlockInputParam[]; + onChange: (next: CustomBlockInputParam[]) => void; + isDisabled?: boolean; +}) { + const update = useCallback( + (index: number, patch: Partial) => { + onChange(params.map((p, i) => (i === index ? { ...p, ...patch } : p))); + }, + [params, onChange], + ); + + const remove = useCallback( + (index: number) => onChange(params.filter((_, i) => i !== index)), + [params, onChange], + ); + + const sampleName = + params.find((p) => p.name.trim())?.name.trim() ?? "channel"; + + const add = useCallback( + () => onChange([...params, emptyParam("text_input")]), + [params, onChange], + ); + + return ( +
+ {/* `params` is bound alongside `input` in every JS expression of a + custom block's graph — see compiler.ts `js()` */} +

+ In this block's JavaScript, read these as{" "} + params.{sampleName} — + available in every block of the canvas, not just the first.{" "} + input stays what it is + everywhere else: the previous block's output, or whatever the caller + passed in at the entrypoint. +

+ {params.length === 0 ? ( +
+ No input parameters yet. Add one so this block can be configured when + placed on a route. +
+ ) : ( + params.map((param, index) => ( + update(index, patch)} + onRemove={() => remove(index)} + /> + )) + )} + + {!isDisabled && ( + + )} +
+ ); +} + +function ParamRow({ + param, + isDisabled, + onChange, + onRemove, +}: { + param: CustomBlockInputParam; + isDisabled?: boolean; + onChange: (patch: Partial) => void; + onRemove: () => void; +}) { + const id = useId(); + + function changeType(type: string) { + const next = emptyParam(type as CustomBlockInputParam["type"]); + onChange({ + ...next, + name: param.name, + label: param.label, + description: param.description, + }); + } + + return ( +
+
+ onChange({ name })} + > + + + + onChange({ label })} + > + + + +
+ +
+ +
+ + onChange({ description })} + > + + + + + {param.type === "dropdown" && ( + onChange({ options })} + /> + )} + + {param.type === "integration_selector" && ( +
+ onChange({ group })} + > + + + + onChange({ tags })} + addButtonLabel="Add tag" + placeholder="e.g. postgres" + /> +
+ )} +
+ ); +} + +function DropdownOptionsEditor({ + id, + options, + isDisabled, + onChange, +}: { + id: string; + options: (string | { label: string; value: string })[]; + isDisabled?: boolean; + onChange: (next: { label: string; value: string }[]) => void; +}) { + const normalized = options.map((o) => + typeof o === "string" ? { label: o, value: o } : o, + ); + + function update( + index: number, + patch: Partial<{ label: string; value: string }>, + ) { + onChange(normalized.map((o, i) => (i === index ? { ...o, ...patch } : o))); + } + + function remove(index: number) { + onChange(normalized.filter((_, i) => i !== index)); + } + + return ( +
+ + {normalized.length === 0 && ( +

No options added yet.

+ )} + {normalized.map((opt, index) => ( +
+ update(index, { label })} + > + + + update(index, { value })} + > + + + remove(index)} + /> +
+ ))} + {!isDisabled && ( + + )} +
+ ); +} diff --git a/apps/portal/src/components/customBlocks/premadeIcons.tsx b/apps/portal/src/components/customBlocks/premadeIcons.tsx new file mode 100644 index 00000000..7277016c --- /dev/null +++ b/apps/portal/src/components/customBlocks/premadeIcons.tsx @@ -0,0 +1,517 @@ +import type { IconType } from "react-icons"; +import { + TbActivity, + TbAdjustments, + TbAlertTriangle, + TbAntenna, + TbApi, + TbArchive, + TbAtom, + TbBarcode, + TbBattery, + TbBell, + TbBellRinging, + TbBinary, + TbBinaryTree, + TbBluetooth, + TbBolt, + TbBook, + TbBookmark, + TbBox, + TbBoxMultiple, + TbBraces, + TbBrackets, + TbBrain, + TbBrandApple, + TbBrandAws, + TbBrandAzure, + TbBrandBitbucket, + TbBrandCloudflare, + TbBrandDebian, + TbBrandDiscord, + TbBrandDjango, + TbBrandDocker, + TbBrandFacebook, + TbBrandFigma, + TbBrandFirebase, + TbBrandGithub, + TbBrandGitlab, + TbBrandGmail, + TbBrandGolang, + TbBrandGoogle, + TbBrandInstagram, + TbBrandJavascript, + TbBrandJira, + TbBrandLaravel, + TbBrandLinkedin, + TbBrandMongodb, + TbBrandMysql, + TbBrandNextjs, + TbBrandNodejs, + TbBrandNotion, + TbBrandNpm, + TbBrandOpenai, + TbBrandPaypal, + TbBrandPhp, + TbBrandPython, + TbBrandReact, + TbBrandRust, + TbBrandSlack, + TbBrandStripe, + TbBrandSupabase, + TbBrandTelegram, + TbBrandTrello, + TbBrandTypescript, + TbBrandUbuntu, + TbBrandVercel, + TbBrandVue, + TbBrandWhatsapp, + TbBrandWindows, + TbBrandX, + TbBrandYoutube, + TbBriefcase, + TbBroadcast, + TbBug, + TbBuildingSkyscraper, + TbBuildingStore, + TbCalendar, + TbCalendarEvent, + TbCamera, + TbCertificate, + TbChartBar, + TbChartDots, + TbChartLine, + TbChartPie, + TbCheck, + TbClipboard, + TbClipboardCheck, + TbClock, + TbClockHour4, + TbCloud, + TbCloudComputing, + TbCloudDataConnection, + TbCloudDownload, + TbCloudLock, + TbCloudOff, + TbCloudUpload, + TbCode, + TbCompass, + TbCopy, + TbCpu, + TbCpu2, + TbCreditCard, + TbCurrencyDollar, + TbDatabase, + TbDatabaseCog, + TbDatabaseExport, + TbDatabaseImport, + TbDatabasePlus, + TbDatabaseSearch, + TbDeviceDesktop, + TbDeviceLaptop, + TbDeviceMobile, + TbDeviceTablet, + TbDeviceWatch, + TbDownload, + TbDroplet, + TbEdit, + TbEye, + TbFile, + TbFileCode, + TbFileExport, + TbFileImport, + TbFileText, + TbFileZip, + TbFiles, + TbFilter, + TbFingerprint, + TbFlag, + TbFlame, + TbFlask, + TbFolder, + TbFolderOpen, + TbFolderPlus, + TbGauge, + TbGift, + TbGitBranch, + TbGitCommit, + TbGitCompare, + TbGitFork, + TbGitMerge, + TbGitPullRequest, + TbGlobe, + TbHeadphones, + TbHeart, + TbHelp, + TbHierarchy, + TbHistory, + TbHourglass, + TbId, + TbInbox, + TbInfoCircle, + TbJson, + TbKey, + TbKeyboard, + TbLanguage, + TbLayersIntersect, + TbLeaf, + TbLifebuoy, + TbLink, + TbLoader, + TbLock, + TbLockOpen, + TbLogin, + TbLogout, + TbMail, + TbMailForward, + TbMap2, + TbMapPin, + TbMathFunction, + TbMessage, + TbMessages, + TbMicrophone, + TbMicroscope, + TbMoodSmile, + TbMouse, + TbNetwork, + TbNotebook, + TbPackage, + TbPackageExport, + TbPackages, + TbPaperclip, + TbPhoto, + TbPin, + TbPlayerPause, + TbPlayerPlay, + TbPlayerStop, + TbPlug, + TbPlugConnected, + TbPrinter, + TbPuzzle, + TbQrcode, + TbRefresh, + TbRegex, + TbRepeat, + TbReportAnalytics, + TbRobot, + TbRocket, + TbRouter, + TbSatellite, + TbScan, + TbSchool, + TbSearch, + TbSend, + TbServer, + TbServer2, + TbServerBolt, + TbServerCog, + TbSettings, + TbShare, + TbShield, + TbShieldCheck, + TbShieldLock, + TbShoppingCart, + TbSitemap, + TbSparkles, + TbSpeakerphone, + TbSql, + TbStack2, + TbStar, + TbTable, + TbTableExport, + TbTag, + TbTags, + TbTargetArrow, + TbTemperature, + TbTerminal2, + TbThumbUp, + TbTicket, + TbTool, + TbTools, + TbTopologyStar, + TbTrash, + TbTrendingUp, + TbTruckDelivery, + TbUnlink, + TbUpload, + TbUsb, + TbUser, + TbUserCheck, + TbUserPlus, + TbUserShield, + TbUsers, + TbVariable, + TbVideo, + TbVolume, + TbWand, + TbWebhook, + TbWifi, + TbWorld, + TbWorldWww, + TbX,} from "react-icons/tb"; +import type { PremadeIconType } from "@fluxify/server/src/api/v1/custom-blocks/shared"; + +/** + * The premade icon set offered in the custom block settings. Typed against the + * server enum, so adding one here without adding it there (or vice versa) is a + * typecheck error rather than a runtime "Invalid premade icon name" 400. + */ +export const PREMADE_ICONS: Record = { + "code": TbCode, + "terminal-2": TbTerminal2, + "bug": TbBug, + "braces": TbBraces, + "brackets": TbBrackets, + "binary": TbBinary, + "api": TbApi, + "webhook": TbWebhook, + "math-function": TbMathFunction, + "variable": TbVariable, + "regex": TbRegex, + "json": TbJson, + "sql": TbSql, + "file-code": TbFileCode, + "git-branch": TbGitBranch, + "git-commit": TbGitCommit, + "git-merge": TbGitMerge, + "git-pull-request": TbGitPullRequest, + "git-fork": TbGitFork, + "brand-github": TbBrandGithub, + "brand-gitlab": TbBrandGitlab, + "brand-bitbucket": TbBrandBitbucket, + "package": TbPackage, + "packages": TbPackages, + "brand-npm": TbBrandNpm, + "brand-docker": TbBrandDocker, + "brand-aws": TbBrandAws, + "brand-azure": TbBrandAzure, + "brand-google": TbBrandGoogle, + "brand-vercel": TbBrandVercel, + "brand-cloudflare": TbBrandCloudflare, + "brand-firebase": TbBrandFirebase, + "brand-supabase": TbBrandSupabase, + "brand-stripe": TbBrandStripe, + "brand-slack": TbBrandSlack, + "brand-discord": TbBrandDiscord, + "brand-telegram": TbBrandTelegram, + "brand-whatsapp": TbBrandWhatsapp, + "brand-x": TbBrandX, + "brand-facebook": TbBrandFacebook, + "brand-instagram": TbBrandInstagram, + "brand-linkedin": TbBrandLinkedin, + "brand-youtube": TbBrandYoutube, + "brand-gmail": TbBrandGmail, + "brand-openai": TbBrandOpenai, + "brand-python": TbBrandPython, + "brand-javascript": TbBrandJavascript, + "brand-typescript": TbBrandTypescript, + "brand-react": TbBrandReact, + "brand-nextjs": TbBrandNextjs, + "brand-vue": TbBrandVue, + "brand-nodejs": TbBrandNodejs, + "brand-golang": TbBrandGolang, + "brand-rust": TbBrandRust, + "brand-php": TbBrandPhp, + "brand-laravel": TbBrandLaravel, + "brand-django": TbBrandDjango, + "brand-mysql": TbBrandMysql, + "brand-mongodb": TbBrandMongodb, + "brand-figma": TbBrandFigma, + "brand-notion": TbBrandNotion, + "brand-jira": TbBrandJira, + "brand-trello": TbBrandTrello, + "brand-paypal": TbBrandPaypal, + "brand-apple": TbBrandApple, + "brand-windows": TbBrandWindows, + "brand-ubuntu": TbBrandUbuntu, + "brand-debian": TbBrandDebian, + "cloud": TbCloud, + "cloud-upload": TbCloudUpload, + "cloud-download": TbCloudDownload, + "cloud-computing": TbCloudComputing, + "cloud-lock": TbCloudLock, + "cloud-data-connection": TbCloudDataConnection, + "server": TbServer, + "server-2": TbServer2, + "server-bolt": TbServerBolt, + "server-cog": TbServerCog, + "database": TbDatabase, + "database-export": TbDatabaseExport, + "database-import": TbDatabaseImport, + "database-search": TbDatabaseSearch, + "database-plus": TbDatabasePlus, + "database-cog": TbDatabaseCog, + "table": TbTable, + "table-export": TbTableExport, + "topology-star": TbTopologyStar, + "network": TbNetwork, + "router": TbRouter, + "wifi": TbWifi, + "world": TbWorld, + "world-www": TbWorldWww, + "globe": TbGlobe, + "link": TbLink, + "unlink": TbUnlink, + "lock": TbLock, + "lock-open": TbLockOpen, + "key": TbKey, + "shield": TbShield, + "shield-lock": TbShieldLock, + "shield-check": TbShieldCheck, + "fingerprint": TbFingerprint, + "certificate": TbCertificate, + "user": TbUser, + "users": TbUsers, + "user-plus": TbUserPlus, + "user-check": TbUserCheck, + "user-shield": TbUserShield, + "id": TbId, + "login": TbLogin, + "logout": TbLogout, + "mail": TbMail, + "mail-forward": TbMailForward, + "message": TbMessage, + "messages": TbMessages, + "bell": TbBell, + "bell-ringing": TbBellRinging, + "send": TbSend, + "inbox": TbInbox, + "calendar": TbCalendar, + "calendar-event": TbCalendarEvent, + "clock": TbClock, + "clock-hour-4": TbClockHour4, + "history": TbHistory, + "hourglass": TbHourglass, + "refresh": TbRefresh, + "repeat": TbRepeat, + "player-play": TbPlayerPlay, + "player-pause": TbPlayerPause, + "player-stop": TbPlayerStop, + "rocket": TbRocket, + "bolt": TbBolt, + "flame": TbFlame, + "sparkles": TbSparkles, + "robot": TbRobot, + "brain": TbBrain, + "cpu": TbCpu, + "cpu-2": TbCpu2, + "device-desktop": TbDeviceDesktop, + "device-mobile": TbDeviceMobile, + "device-laptop": TbDeviceLaptop, + "device-tablet": TbDeviceTablet, + "device-watch": TbDeviceWatch, + "printer": TbPrinter, + "camera": TbCamera, + "video": TbVideo, + "microphone": TbMicrophone, + "headphones": TbHeadphones, + "volume": TbVolume, + "photo": TbPhoto, + "file": TbFile, + "file-text": TbFileText, + "file-import": TbFileImport, + "file-export": TbFileExport, + "file-zip": TbFileZip, + "files": TbFiles, + "folder": TbFolder, + "folder-open": TbFolderOpen, + "folder-plus": TbFolderPlus, + "clipboard": TbClipboard, + "clipboard-check": TbClipboardCheck, + "notebook": TbNotebook, + "book": TbBook, + "bookmark": TbBookmark, + "tag": TbTag, + "tags": TbTags, + "chart-bar": TbChartBar, + "chart-line": TbChartLine, + "chart-pie": TbChartPie, + "chart-dots": TbChartDots, + "report-analytics": TbReportAnalytics, + "activity": TbActivity, + "gauge": TbGauge, + "target-arrow": TbTargetArrow, + "trending-up": TbTrendingUp, + "currency-dollar": TbCurrencyDollar, + "credit-card": TbCreditCard, + "shopping-cart": TbShoppingCart, + "building-store": TbBuildingStore, + "truck-delivery": TbTruckDelivery, + "package-export": TbPackageExport, + "box": TbBox, + "box-multiple": TbBoxMultiple, + "stack-2": TbStack2, + "layers-intersect": TbLayersIntersect, + "sitemap": TbSitemap, + "hierarchy": TbHierarchy, + "binary-tree": TbBinaryTree, + "git-compare": TbGitCompare, + "filter": TbFilter, + "search": TbSearch, + "adjustments": TbAdjustments, + "settings": TbSettings, + "tool": TbTool, + "tools": TbTools, + "wand": TbWand, + "puzzle": TbPuzzle, + "plug": TbPlug, + "plug-connected": TbPlugConnected, + "battery": TbBattery, + "trash": TbTrash, + "archive": TbArchive, + "copy": TbCopy, + "edit": TbEdit, + "check": TbCheck, + "x": TbX, + "alert-triangle": TbAlertTriangle, + "info-circle": TbInfoCircle, + "help": TbHelp, + "eye": TbEye, + "map-pin": TbMapPin, + "map-2": TbMap2, + "compass": TbCompass, + "flag": TbFlag, + "star": TbStar, + "heart": TbHeart, + "thumb-up": TbThumbUp, + "mood-smile": TbMoodSmile, + "language": TbLanguage, + "qrcode": TbQrcode, + "barcode": TbBarcode, + "scan": TbScan, + "temperature": TbTemperature, + "droplet": TbDroplet, + "leaf": TbLeaf, + "building-skyscraper": TbBuildingSkyscraper, + "briefcase": TbBriefcase, + "school": TbSchool, + "microscope": TbMicroscope, + "flask": TbFlask, + "atom": TbAtom, + "ticket": TbTicket, + "gift": TbGift, + "paperclip": TbPaperclip, + "pin": TbPin, + "lifebuoy": TbLifebuoy, + "speakerphone": TbSpeakerphone, + "broadcast": TbBroadcast, + "antenna": TbAntenna, + "satellite": TbSatellite, + "keyboard": TbKeyboard, + "mouse": TbMouse, + "usb": TbUsb, + "bluetooth": TbBluetooth, + "share": TbShare, + "download": TbDownload, + "upload": TbUpload, + "cloud-off": TbCloudOff, + "loader": TbLoader, + "chart": TbChartBar, + "image": TbPhoto, +}; + +export const PREMADE_ICON_NAMES = Object.keys(PREMADE_ICONS) as PremadeIconType[]; + +export function premadeIcon(name: string | null | undefined): IconType | undefined { + return name ? PREMADE_ICONS[name as PremadeIconType] : undefined; +} diff --git a/apps/portal/src/components/routes/RouteSettingsModal.tsx b/apps/portal/src/components/routes/RouteSettingsModal.tsx index 334f1ed4..e72c2527 100644 --- a/apps/portal/src/components/routes/RouteSettingsModal.tsx +++ b/apps/portal/src/components/routes/RouteSettingsModal.tsx @@ -24,6 +24,9 @@ import type { HttpMethod } from "@fluxify/server/src/db/schema"; import { DEFAULT_CONTENT_TYPES } from "@fluxify/server/src/lib/routeConfig"; import { ROUTE_REGEX } from "@fluxify/server/src/api/v1/routes/constants"; import { TbAlertTriangle } from "react-icons/tb"; +import { useNavigate } from "@tanstack/react-router"; +import { DeleteButton } from "@fluxify/components"; +import { ConfirmDialog } from "@/components/common/ConfirmDialog"; import { routesQuery } from "@/query/routesQuery"; import { projectSettingsKeysQuery } from "@/query/projectSettingsKeysQuery"; import { showErrorNotification } from "@/lib/errorNotifier"; @@ -141,6 +144,24 @@ function RouteSettingsForm({ () => paramConfigFrom(route.paramsSchema), ); const [tab, setTab] = useState("general"); + const [confirmDelete, setConfirmDelete] = useState(false); + const remove = routesQuery.remove.mutation(); + const navigate = useNavigate(); + + function deleteRoute() { + remove.mutate(route.id, { + onSuccess: () => { + toast.success("Route deleted"); + setConfirmDelete(false); + onClose(); + navigate({ + to: "/$projectId/routes", + params: { projectId: route.projectId }, + }); + }, + onError: (error) => showErrorNotification(error as Error), + }); + } const pathParams = useMemo(() => extractPathParams(path), [path]); const hasBody = METHODS_WITH_BODY.includes(method); @@ -198,6 +219,7 @@ function RouteSettingsForm({ { id: "query", label: "Query" }, hasBody && { id: "body", label: "Body" }, { id: "advanced", label: "Advanced" }, + { id: "danger", label: "Danger zone" }, ].filter(Boolean) as { id: string; label: string }[], [pathParams.length, hasBody], ); @@ -412,9 +434,46 @@ function RouteSettingsForm({ )} + + +
+
+
+

+ {route.method} {route.path} +

+

+ Any client calling it will start getting a 404. +

+
+ setConfirmDelete(true)}> + Delete route + +
+
+ + + Delete{" "} + + {route.method} {route.path} + + ? This can't be undone. + + {isDirty ? "Unsaved changes" : "All changes saved"} diff --git a/apps/portal/src/query/customBlocksQuery.ts b/apps/portal/src/query/customBlocksQuery.ts index 5c378cb3..ff829fd7 100644 --- a/apps/portal/src/query/customBlocksQuery.ts +++ b/apps/portal/src/query/customBlocksQuery.ts @@ -54,4 +54,14 @@ export const customBlocksQuery = { }); }, }, + update: { + mutation(projectId: string, id: string) { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (body: z.infer) => + customBlocksService.update(id, body), + onSuccess: () => qc.invalidateQueries({ queryKey: key(projectId) }), + }); + }, + }, }; diff --git a/apps/portal/src/routes/_authed/$projectId/custom-blocks.tsx b/apps/portal/src/routes/_authed/$projectId/custom-blocks.tsx index 1f7e6152..de27e213 100644 --- a/apps/portal/src/routes/_authed/$projectId/custom-blocks.tsx +++ b/apps/portal/src/routes/_authed/$projectId/custom-blocks.tsx @@ -1,22 +1,21 @@ -import { useState } from "react"; +import { useMemo, useState } from "react"; import { createFileRoute, useNavigate } from "@tanstack/react-router"; import { Button, - Card, - CloseButton, DeleteIconButton, Input, Label, - Modal, Spinner, TextField, toast, } from "@fluxify/components"; -import { TbPlus } from "react-icons/tb"; +import { TbBoxMultiple, TbPlus, TbSearch } from "react-icons/tb"; import { customBlocksQuery } from "@/query/customBlocksQuery"; import { showErrorNotification } from "@/lib/errorNotifier"; import { ConfirmDialog } from "@/components/common/ConfirmDialog"; import { createRouteHead } from "@/lib/seo"; +import { BaseBlock } from "@/components/canvas/blocks/BaseBlock"; +import { CustomBlockIcon, type IconValue } from "@/components/customBlocks/IconPicker"; export const Route = createFileRoute("/_authed/$projectId/custom-blocks")({ head: createRouteHead( @@ -26,7 +25,9 @@ export const Route = createFileRoute("/_authed/$projectId/custom-blocks")({ component: CustomBlocksPage, }); -type Block = { id: string; label: string; name: string; description?: string | null }; +type Block = NonNullable< + ReturnType["data"] +>[number]; function CustomBlocksPage() { const { projectId } = Route.useParams(); @@ -34,15 +35,49 @@ function CustomBlocksPage() { const remove = customBlocksQuery.remove.mutation(projectId); const navigate = useNavigate(); const [pendingDelete, setPendingDelete] = useState(null); + const [search, setSearch] = useState(""); + + const blocks = useMemo(() => { + const q = search.trim().toLowerCase(); + if (!data) return []; + if (!q) return data; + return data.filter((b) => + [b.label, b.name, b.description ?? ""].some((v) => v.toLowerCase().includes(q)), + ); + }, [data, search]); + + function openCanvas(blockId: string) { + navigate({ + to: "/$projectId/custom-block-canvas/$blockId", + params: { projectId, blockId }, + }); + } return ( -
-
+
+

Custom Blocks

-

Reusable blocks for your flows.

+

+ Reusable blocks for your flows. Click one to open its canvas. +

+
+
+ {data && data.length > 0 && ( + + + + + )} +
-
{isLoading ? ( @@ -52,38 +87,77 @@ function CustomBlocksPage() { ) : isError ? (

Couldn't load custom blocks.

) : !data || data.length === 0 ? ( -

No custom blocks yet.

+ } + title="No custom blocks yet" + description="A custom block wraps a piece of flow you want to reuse across routes." + /> + ) : blocks.length === 0 ? ( + } + title={`No block matches “${search}”`} + description="Try a different name or description." + /> ) : ( -
- {data.map((block) => ( - - - {block.label} - {block.name} - - -

- {block.description || "No description"} -

- - - - setPendingDelete(block as Block)} + showToolbar={false} /> - - +
+ +
+
+

{block.name}

+

+ {Array.isArray(block.inputParams) ? block.inputParams.length : 0} input + {(Array.isArray(block.inputParams) ? block.inputParams.length : 0) === 1 + ? "" + : "s"} + {block.sourceType && block.sourceType !== "user-defined" + ? ` · ${block.sourceType}` + : ""} +

+
+ {block.sourceType !== "plugin" && ( +
e.stopPropagation()} + onKeyDown={(e) => e.stopPropagation()} + > + setPendingDelete(block)} + /> +
+ )} +
+
))}
)} @@ -110,82 +184,21 @@ function CustomBlocksPage() { ); } -function CreateBlockButton({ projectId }: { projectId: string }) { - const create = customBlocksQuery.create.mutation(projectId); - const [open, setOpen] = useState(false); - const [name, setName] = useState(""); - const [label, setLabel] = useState(""); - const [description, setDescription] = useState(""); - - function reset() { - setName(""); - setLabel(""); - setDescription(""); - } - - function submit(e: React.FormEvent) { - e.preventDefault(); - create.mutate( - { name, label, description, projectId }, - { - onSuccess: () => { - toast.success("Custom block created"); - reset(); - setOpen(false); - }, - onError: (err) => showErrorNotification(err as Error), - }, - ); - } - +function EmptyState({ + icon, + title, + description, +}: { + icon: React.ReactNode; + title: string; + description: string; +}) { return ( - - - - - - - - -
- Create a custom block -

- You can build its logic on the canvas afterwards. -

-
- - -
- -
- - - - - - - - - - - - -
- - - - - -
- - - - +
+ {icon} +

{title}

+

{description}

+
); } + diff --git a/apps/portal/src/routes/_authed/$projectId/custom-blocks_.new.tsx b/apps/portal/src/routes/_authed/$projectId/custom-blocks_.new.tsx new file mode 100644 index 00000000..8d7d9bfb --- /dev/null +++ b/apps/portal/src/routes/_authed/$projectId/custom-blocks_.new.tsx @@ -0,0 +1,347 @@ +import { useMemo, useState } from "react"; +import { createFileRoute, useNavigate } from "@tanstack/react-router"; +import { Button, cn, Input, Label, TextField, toast } from "@fluxify/components"; +import { TbArrowLeft, TbArrowRight, TbCheck } from "react-icons/tb"; +import type { inputParamSchema } from "@fluxify/server/src/api/v1/custom-blocks/create/dto"; +import type { z } from "zod"; +import { customBlocksQuery } from "@/query/customBlocksQuery"; +import { showErrorNotification } from "@/lib/errorNotifier"; +import { createRouteHead } from "@/lib/seo"; +import type { CustomBlockInputParam } from "@/components/canvas/panel/blocks/CustomBlockSettings"; +import { + InputParamsEditor, + validateInputParams, +} from "@/components/customBlocks/InputParamsEditor"; +import { + ICON_URL_MAX, + IconPicker, + type IconValue, +} from "@/components/customBlocks/IconPicker"; + +export const Route = createFileRoute("/_authed/$projectId/custom-blocks_/new")({ + head: createRouteHead( + "New Custom Block", + "Create a reusable custom block: identity, icon and input parameters.", + ), + component: CreateCustomBlockPage, +}); + +const NAME_REGEX = /^[a-z0-9_]+$/; + +/** `Send Slack message` → `send_slack_message`, the shape the API accepts. */ +function slugify(value: string) { + return value + .toLowerCase() + .replace(/[^a-z0-9]+/g, "_") + .replace(/^_+|_+$/g, ""); +} + +const STEPS = [ + { key: "basics", label: "Basics" }, + { key: "icon", label: "Icon" }, + { key: "inputs", label: "Inputs" }, + { key: "review", label: "Review" }, +] as const; + +function CreateCustomBlockPage() { + const { projectId } = Route.useParams(); + const navigate = useNavigate(); + const create = customBlocksQuery.create.mutation(projectId); + + const [label, setLabel] = useState(""); + const [name, setName] = useState(""); + // until the user edits `name` by hand it tracks the label + const [nameTouched, setNameTouched] = useState(false); + const [description, setDescription] = useState(""); + const [iconValue, setIconValue] = useState({}); + const [params, setParams] = useState([]); + const [step, setStep] = useState(0); + + const nameIsValid = NAME_REGEX.test(name); + const basicsValid = label.trim().length >= 2 && nameIsValid; + const iconTooLong = (iconValue.iconUrl?.length ?? 0) > ICON_URL_MAX; + const paramsError = useMemo(() => validateInputParams(params), [params]); + + const current = Math.min(step, STEPS.length - 1); + const currentKey = STEPS[current].key; + const isLast = current === STEPS.length - 1; + + function submit() { + if (iconTooLong) { + toast.danger("Icon image is too large"); + setStep(1); + return; + } + if (paramsError) { + toast.danger(paramsError); + setStep(2); + return; + } + create.mutate( + { + projectId, + name, + label: label.trim(), + description, + icon: iconValue.icon, + iconUrl: iconValue.iconUrl, + inputParams: params as unknown as z.infer[], + }, + { + onSuccess: (created) => { + toast.success("Custom block created"); + navigate({ + to: "/$projectId/custom-block-canvas/$blockId", + params: { projectId, blockId: created.id }, + }); + }, + onError: (error) => showErrorNotification(error as Error), + }, + ); + } + + return ( + // full height of the scroll container so only the step body scrolls and the + // Back/Next bar stays pinned +
+
+ +
+

+ Create a custom block +

+

+ A reusable piece of flow you can drop onto any route canvas. +

+
+
+ + + +
+ + +
+ {currentKey === "basics" && ( +
+ { + setLabel(next); + if (!nameTouched) setName(slugify(next)); + }} + > + + + + + { + setNameTouched(true); + setName(next); + }} + isInvalid={name.length > 0 && !nameIsValid} + > + + +

+ How flows refer to this block. Lowercase letters, digits and + underscores. Can't be changed later. +

+ + + + + + +
+ )} + + {currentKey === "icon" && ( +
+ +
+ )} + + {currentKey === "inputs" && ( +
+ +
+ )} + + {currentKey === "review" && ( +
+ + + + + +
+ )} +
+
+ +
+ + {isLast ? ( + + ) : ( + + )} +
+
+ ); +} + +function StepHeading({ + current, + total, + title, + description, +}: { + current: number; + total: number; + title: string; + description: string; +}) { + return ( +
+

+ Step {current + 1} of {total} +

+

+ {title} +

+

{description}

+
+ ); +} + +function SummaryItem({ + label, + value, + mono, +}: { + label: string; + value: string; + mono?: boolean; +}) { + return ( +
+
{label}
+
+ {value || "—"} +
+
+ ); +} diff --git a/apps/portal/src/routes/_authed/$projectId_.custom-block-canvas.$blockId.tsx b/apps/portal/src/routes/_authed/$projectId_.custom-block-canvas.$blockId.tsx index b055da69..dc381648 100644 --- a/apps/portal/src/routes/_authed/$projectId_.custom-block-canvas.$blockId.tsx +++ b/apps/portal/src/routes/_authed/$projectId_.custom-block-canvas.$blockId.tsx @@ -1,7 +1,12 @@ +import { useState } from "react"; import { createFileRoute } from "@tanstack/react-router"; +import { Button } from "@fluxify/components"; +import { TbSettings } from "react-icons/tb"; import { customBlocksQuery } from "@/query/customBlocksQuery"; import { customBlocksService } from "@/services/customBlocks"; import { CanvasWorkbench } from "@/components/canvas"; +import { CustomBlockSettingsModal } from "@/components/customBlocks/CustomBlockSettingsModal"; +import { CustomBlockSwitcher } from "@/components/customBlocks/CustomBlockSwitcher"; import { createRouteHead } from "@/lib/seo"; export const Route = createFileRoute( @@ -15,15 +20,34 @@ export const Route = createFileRoute( }); function CustomBlockCanvasPage() { - const { blockId } = Route.useParams(); + const { projectId, blockId } = Route.useParams(); const save = customBlocksQuery.saveCanvas.mutation(blockId); + const [settingsOpen, setSettingsOpen] = useState(false); return ( - customBlocksService.getCanvasItems(blockId)} - save={(payload) => save.mutateAsync(payload)} - /> + <> + customBlocksService.getCanvasItems(blockId)} + save={(payload) => save.mutateAsync(payload)} + headerLeft={} + headerActions={ + + } + /> + {/* mounted only while open: the form seeds its state from the loaded block */} + {settingsOpen && ( + + )} + ); } diff --git a/apps/portal/src/services/customBlocks.ts b/apps/portal/src/services/customBlocks.ts index d8165334..60be6073 100644 --- a/apps/portal/src/services/customBlocks.ts +++ b/apps/portal/src/services/customBlocks.ts @@ -4,6 +4,10 @@ import { requestBodySchema as createRequestSchema, responseSchema as createResponseSchema, } from "@fluxify/server/src/api/v1/custom-blocks/create/dto"; +import { + requestBodySchema as updateRequestSchema, + responseSchema as updateResponseSchema, +} from "@fluxify/server/src/api/v1/custom-blocks/update/dto"; import { httpClient } from "@/lib/http"; import { canvasEndpoints } from "./canvas"; @@ -23,7 +27,15 @@ export const customBlocksService = { async delete(id: string) { await httpClient.delete(`${baseUrl}/${id}`); }, + async update( + id: string, + data: z.infer, + ): Promise> { + const result = await httpClient.put(`${baseUrl}/${id}`, data); + return result.data; + }, // a custom block's canvas is stored and served exactly like a route's ...canvasEndpoints(baseUrl), createRequestSchema, + updateRequestSchema, }; diff --git a/apps/server/deployments/compiledWorker.ts b/apps/server/deployments/compiledWorker.ts index ea3643bf..00225638 100644 --- a/apps/server/deployments/compiledWorker.ts +++ b/apps/server/deployments/compiledWorker.ts @@ -22,6 +22,9 @@ import { asyncExecutorLimitsFromEnv } from "../src/modules/requestRouter/asyncEx import { executionRuntimeEnvironment } from "../src/modules/requestRouter/executionEnvironment"; import type { ArtifactEntry } from "../src/modules/requestRouter/compiledRuntime"; import { closeNats } from "../src/db/nats"; +import { startJobWorker } from "../src/modules/jobs/consumer"; +import { enqueueJob } from "../src/modules/jobs/publisher"; +import type { JobEnvelope } from "../src/modules/jobs/types"; import { OTLP_AUTH_HEADER_NAME, OTLP_AUTH_HEADER_VALUE, @@ -145,6 +148,7 @@ function spawnExecution() { if (execution !== process) return; execution = undefined; markNotReady(); + failPendingJobs("execution process exited mid-job"); watchdog.setEnabled(timeoutPolicyEnabled()); if (shuttingDown) return; logger.error( @@ -159,12 +163,49 @@ function spawnExecution() { child.send({ type: "bootstrap", bootstrap } satisfies ExecutionMessage); } +/** + * Jobs handed to the execution process, waiting on its reply. The broker's ack + * is driven by that reply, so a child that dies mid-job must reject its pending + * work — otherwise the consumer sits on the message until the ack wait elapses. + */ +const pendingJobs = new Map< + string, + { resolve: () => void; reject: (error: Error) => void } +>(); + +function runJobInExecution(job: JobEnvelope) { + return new Promise((resolve, reject) => { + if (!execution) return reject(new Error("execution process is not running")); + pendingJobs.set(job.id, { resolve, reject }); + execution.send({ type: "job", job } satisfies ExecutionMessage); + }); +} + +function failPendingJobs(reason: string) { + for (const [, pending] of pendingJobs) pending.reject(new Error(reason)); + pendingJobs.clear(); +} + function onExecutionEvent(event: ExecutionEvent) { switch (event.type) { case "ready": markReady(); logger.info("execution process ready", "WORKER.execution"); return; + case "job-finished": { + const pending = pendingJobs.get(event.id); + pendingJobs.delete(event.id); + if (!pending) return; + return event.error ? pending.reject(new Error(event.error)) : pending.resolve(); + } + case "enqueue-job": + // user code queued work; failures are logged, the graph moved on already + return void enqueueJob(event.job).catch((error) => + logger.error( + `failed to queue ${event.job.kind}/${event.job.target}: ${String(error)}`, + "WORKER.jobs", + ), + ); case "heartbeat": return watchdog.heartbeat(); case "execution-started": @@ -192,6 +233,19 @@ await artifactWatch.initialized; spawnExecution(); synchronizeMonitoring(); +// Background work for this project. Separate from the request path on purpose: +// a queued job must not compete with traffic for the same acceptance. +await startJobWorker({ + projectId: WORKER_PROJECT_ID, + handle: runJobInExecution, + concurrency: Number(getEnv("JOBS_CONCURRENCY")) || undefined, + ackWaitMs: Number(getEnv("JOBS_ACK_WAIT_MS")) || undefined, + maxDeliver: Number(getEnv("JOBS_MAX_DELIVER")) || undefined, + retryDelayMs: Number(getEnv("JOBS_RETRY_DELAY_MS")) || undefined, +}).catch((error) => + logger.error(`job worker failed to start: ${String(error)}`, "WORKER.jobs"), +); + function evaluateTimeouts() { const timedOut = watchdog.findTimedOut(); if (!timedOut || !execution || terminatingForTimeout) return; diff --git a/apps/server/src/api/v1/custom-blocks/create/dto.ts b/apps/server/src/api/v1/custom-blocks/create/dto.ts index ebe8e127..d75d0cd3 100644 --- a/apps/server/src/api/v1/custom-blocks/create/dto.ts +++ b/apps/server/src/api/v1/custom-blocks/create/dto.ts @@ -5,16 +5,19 @@ export const inputParamSchema = z.discriminatedUnion("type", [ type: z.literal("text_input"), name: z.string().regex(/^[a-z0-9_]+$/), label: z.string(), + description: z.string().optional(), }), z.object({ type: z.literal("checkbox"), name: z.string().regex(/^[a-z0-9_]+$/), label: z.string(), + description: z.string().optional(), }), z.object({ type: z.literal("array_editor"), name: z.string().regex(/^[a-z0-9_]+$/), label: z.string(), + description: z.string().optional(), }), z.object({ type: z.literal("integration_selector"), @@ -23,6 +26,7 @@ export const inputParamSchema = z.discriminatedUnion("type", [ group: z.string(), variant: z.string().optional(), tags: z.array(z.string()).default([]), + description: z.string().optional(), }), z.object({ type: z.literal("dropdown"), @@ -34,6 +38,7 @@ export const inputParamSchema = z.discriminatedUnion("type", [ value: z.string(), }) ), + description: z.string().optional(), }), ]); diff --git a/apps/server/src/api/v1/custom-blocks/create/repository.ts b/apps/server/src/api/v1/custom-blocks/create/repository.ts index 9f8b3a85..d910c747 100644 --- a/apps/server/src/api/v1/custom-blocks/create/repository.ts +++ b/apps/server/src/api/v1/custom-blocks/create/repository.ts @@ -38,7 +38,9 @@ export async function createDependencies( id: id3, customBlockId, type: BlockTypes.errorHandler, - position: { x: 25, y: 0 }, + // a block is ~168px wide; anything less than that sits on top of the + // entrypoint on the current node design + position: { x: -240, y: 0 }, data: { next: "", retryAfterFail: false, diff --git a/apps/server/src/api/v1/custom-blocks/delete/service.ts b/apps/server/src/api/v1/custom-blocks/delete/service.ts index 6243be34..2b11507d 100644 --- a/apps/server/src/api/v1/custom-blocks/delete/service.ts +++ b/apps/server/src/api/v1/custom-blocks/delete/service.ts @@ -8,12 +8,14 @@ import { ForbiddenError } from "../../../../errors/forbidError"; import { hasProjectAccess } from "../../../auth/common"; import { AuthACL } from "../../../../db/schema"; import { User } from "better-auth"; +import { dropCustomBlock } from "../../../../modules/compiler/service"; export default async function handleRequest( id: string, user: User & { isSystemAdmin: boolean }, acl: AuthACL[] ): Promise> { + let projectId: string | undefined; await db.transaction(async (tx) => { const existingBlock = await getCustomBlockById(id, tx); if (!existingBlock) { @@ -28,9 +30,14 @@ export default async function handleRequest( throw new ForbiddenError("Cannot delete a custom block originating from a plugin"); } + projectId = existingBlock.projectId!; await deleteCustomBlock(id, tx); - await publishMessage(CHAN_ON_CUSTOM_BLOCK_CHANGE, id); }); + // same as routes: the compiler can't resolve the project of a row that no + // longer exists, so the stale artifact would keep being served from KV + if (projectId) await dropCustomBlock(projectId, id); + await publishMessage(CHAN_ON_CUSTOM_BLOCK_CHANGE, id); + return { id }; } diff --git a/apps/server/src/api/v1/custom-blocks/delete/tests/delete.test.ts b/apps/server/src/api/v1/custom-blocks/delete/tests/delete.test.ts index 5206e2d0..afad30de 100644 --- a/apps/server/src/api/v1/custom-blocks/delete/tests/delete.test.ts +++ b/apps/server/src/api/v1/custom-blocks/delete/tests/delete.test.ts @@ -10,6 +10,10 @@ mock.module("../../../../../db/redis", () => ({ CHAN_ON_CUSTOM_BLOCK_CHANGE: "chan:on-custom-block-change", })); +mock.module("../../../../../modules/compiler/service", () => ({ + dropCustomBlock: mock(), +})); + import handleRequest from "../service"; import * as repo from "../repository"; import { NotFoundError } from "../../../../../errors/notFoundError"; diff --git a/apps/server/src/api/v1/custom-blocks/shared.ts b/apps/server/src/api/v1/custom-blocks/shared.ts index 03f6d644..e2daf637 100644 --- a/apps/server/src/api/v1/custom-blocks/shared.ts +++ b/apps/server/src/api/v1/custom-blocks/shared.ts @@ -1,19 +1,262 @@ import { z } from "zod"; +/** + * Names of the premade icons the portal ships. Kept in sync with + * `apps/portal/src/components/customBlocks/premadeIcons.tsx` — the portal map is + * typed as `Record`, so a mismatch fails the typecheck. + */ export const premadeIconEnum = z.enum([ - "database", + "code", + "terminal-2", + "bug", + "braces", + "brackets", + "binary", "api", - "robot", - "tool", - "user", + "webhook", + "math-function", + "variable", + "regex", + "json", + "sql", + "file-code", + "git-branch", + "git-commit", + "git-merge", + "git-pull-request", + "git-fork", + "brand-github", + "brand-gitlab", + "brand-bitbucket", + "package", + "packages", + "brand-npm", + "brand-docker", + "brand-aws", + "brand-azure", + "brand-google", + "brand-vercel", + "brand-cloudflare", + "brand-firebase", + "brand-supabase", + "brand-stripe", + "brand-slack", + "brand-discord", + "brand-telegram", + "brand-whatsapp", + "brand-x", + "brand-facebook", + "brand-instagram", + "brand-linkedin", + "brand-youtube", + "brand-gmail", + "brand-openai", + "brand-python", + "brand-javascript", + "brand-typescript", + "brand-react", + "brand-nextjs", + "brand-vue", + "brand-nodejs", + "brand-golang", + "brand-rust", + "brand-php", + "brand-laravel", + "brand-django", + "brand-mysql", + "brand-mongodb", + "brand-figma", + "brand-notion", + "brand-jira", + "brand-trello", + "brand-paypal", + "brand-apple", + "brand-windows", + "brand-ubuntu", + "brand-debian", + "cloud", + "cloud-upload", + "cloud-download", + "cloud-computing", + "cloud-lock", + "cloud-data-connection", + "server", + "server-2", + "server-bolt", + "server-cog", + "database", + "database-export", + "database-import", + "database-search", + "database-plus", + "database-cog", + "table", + "table-export", + "topology-star", + "network", + "router", + "wifi", + "world", + "world-www", + "globe", + "link", + "unlink", "lock", + "lock-open", + "key", + "shield", + "shield-lock", + "shield-check", + "fingerprint", + "certificate", + "user", + "users", + "user-plus", + "user-check", + "user-shield", + "id", + "login", + "logout", "mail", + "mail-forward", + "message", + "messages", + "bell", + "bell-ringing", + "send", + "inbox", "calendar", + "calendar-event", + "clock", + "clock-hour-4", + "history", + "hourglass", + "refresh", + "repeat", + "player-play", + "player-pause", + "player-stop", + "rocket", + "bolt", + "flame", + "sparkles", + "robot", + "brain", + "cpu", + "cpu-2", + "device-desktop", + "device-mobile", + "device-laptop", + "device-tablet", + "device-watch", + "printer", + "camera", + "video", + "microphone", + "headphones", + "volume", + "photo", "file", + "file-text", + "file-import", + "file-export", + "file-zip", + "files", "folder", + "folder-open", + "folder-plus", + "clipboard", + "clipboard-check", + "notebook", + "book", + "bookmark", + "tag", + "tags", + "chart-bar", + "chart-line", + "chart-pie", + "chart-dots", + "report-analytics", + "activity", + "gauge", + "target-arrow", + "trending-up", + "currency-dollar", + "credit-card", + "shopping-cart", + "building-store", + "truck-delivery", + "package-export", + "box", + "box-multiple", + "stack-2", + "layers-intersect", + "sitemap", + "hierarchy", + "binary-tree", + "git-compare", + "filter", + "search", + "adjustments", + "settings", + "tool", + "tools", + "wand", + "puzzle", + "plug", + "plug-connected", + "battery", + "trash", + "archive", + "copy", + "edit", + "check", + "x", + "alert-triangle", + "info-circle", + "help", + "eye", + "map-pin", + "map-2", + "compass", + "flag", + "star", + "heart", + "thumb-up", + "mood-smile", + "language", + "qrcode", + "barcode", + "scan", + "temperature", + "droplet", + "leaf", + "building-skyscraper", + "briefcase", + "school", + "microscope", + "flask", + "atom", + "ticket", + "gift", + "paperclip", + "pin", + "lifebuoy", + "speakerphone", + "broadcast", + "antenna", + "satellite", + "keyboard", + "mouse", + "usb", + "bluetooth", + "share", + "download", + "upload", + "cloud-off", + "loader", "chart", "image", - "code", ]); export type PremadeIconType = z.infer; diff --git a/apps/server/src/api/v1/routes/create/repository.ts b/apps/server/src/api/v1/routes/create/repository.ts index 801738c6..ced53f06 100644 --- a/apps/server/src/api/v1/routes/create/repository.ts +++ b/apps/server/src/api/v1/routes/create/repository.ts @@ -47,7 +47,7 @@ export async function createDependency( type: "response", position: { x: 0, - y: 100, + y: 160, }, data: { httpCode: "200", @@ -57,8 +57,10 @@ export async function createDependency( id: id3, routeId, type: BlockTypes.errorHandler, + // a block is ~168px wide; anything less than that sits on top of the + // entrypoint on the current node design position: { - x: -100, + x: -240, y: 0, }, data: { diff --git a/apps/server/src/api/v1/routes/delete/service.ts b/apps/server/src/api/v1/routes/delete/service.ts index 0e415ab6..2dc720fb 100644 --- a/apps/server/src/api/v1/routes/delete/service.ts +++ b/apps/server/src/api/v1/routes/delete/service.ts @@ -7,11 +7,13 @@ import { AuthACL } from "../../../../db/schema"; import { canAccessProject } from "../../../../lib/acl"; import { NotFoundError } from "../../../../errors/notFoundError"; import { ForbiddenError } from "../../../../errors/forbidError"; +import { dropRoute } from "../../../../modules/compiler/service"; export default async function handleRequest( id: string, acl: AuthACL[] = [] ): Promise> { + let projectId: string | undefined; await db.transaction(async (tx) => { const existingRoute = await findRouteById(id, tx); if (!existingRoute) { @@ -25,8 +27,13 @@ export default async function handleRequest( if (!canAccess) { throw new ForbiddenError(); } + projectId = existingRoute.projectId!; await deleteRoute(id, tx); }); + // the compiler resolves a route's project from the database, so once the row + // is gone it can no longer work out which artifact key to drop — workers would + // keep serving the deleted route from KV. Drop it here, where we still know. + if (projectId) await dropRoute(projectId, id); await publishMessage(CHAN_ON_ROUTE_CHANGE, id); return ""; } diff --git a/apps/server/src/lib/env.ts b/apps/server/src/lib/env.ts index a83efd12..a4fc5087 100644 --- a/apps/server/src/lib/env.ts +++ b/apps/server/src/lib/env.ts @@ -149,6 +149,28 @@ export const serverEnvSchema = baseEnvSchema.extend({ .enum(["true", "false"]) .optional() .describe("Enable admin control-plane API endpoints ('true' | 'false')"), + + // Background job queue. One stream serves every kind of job, so these tune + // the worker rather than any one feature. + JOBS_CONCURRENCY: z + .string() + .optional() + .describe("Background jobs this worker runs at once (default 5)"), + + JOBS_ACK_WAIT_MS: z + .string() + .optional() + .describe("How long a job may run before it is assumed lost (default 300000)"), + + JOBS_MAX_DELIVER: z + .string() + .optional() + .describe("Attempts before a failing job is dropped (default 5)"), + + JOBS_RETRY_DELAY_MS: z + .string() + .optional() + .describe("Delay before a failed job is redelivered (default 10000)"), }); // Extract keys using keyof diff --git a/apps/server/src/modules/canvas/repository.ts b/apps/server/src/modules/canvas/repository.ts index cde7356e..6fb0095d 100644 --- a/apps/server/src/modules/canvas/repository.ts +++ b/apps/server/src/modules/canvas/repository.ts @@ -232,9 +232,17 @@ export async function getCustomBlockNames( parent: CanvasParent, tx?: DbTransactionType, ): Promise { + return (await getProjectCustomBlocks(parent, tx)).map((row) => row.name); +} + +/** Same set as `getCustomBlockNames`, with the id needed to read each canvas. */ +export async function getProjectCustomBlocks( + parent: CanvasParent, + tx?: DbTransactionType, +): Promise<{ id: string; name: string }[]> { const table = parent.type === "route" ? routesEntity : customBlocksListEntity; - const rows = await (tx ?? db) - .select({ name: customBlocksListEntity.name }) + return await (tx ?? db) + .select({ id: customBlocksListEntity.id, name: customBlocksListEntity.name }) .from(customBlocksListEntity) .where( eq( @@ -245,7 +253,30 @@ export async function getCustomBlockNames( .where(eq(table.id, parent.id)), ), ); - return rows.map((row) => row.name); +} + +/** + * Which custom block each of these canvases holds a block of — the call graph a + * recursion check walks. Only the type matters, so this stays one narrow read + * however big the canvases are. + */ +export async function getCustomBlockCalls( + customBlockIds: string[], + tx?: DbTransactionType, +): Promise<{ parentId: string; type: string }[]> { + if (!customBlockIds.length) return []; + const rows = await (tx ?? db) + .selectDistinct({ parentId: blocksEntity.parentId, type: blocksEntity.type }) + .from(blocksEntity) + .where( + and( + eq(blocksEntity.parentType, "custom_block"), + inArray(blocksEntity.parentId, customBlockIds), + ), + ); + return rows.flatMap((row) => + row.parentId && row.type ? [{ parentId: row.parentId, type: row.type }] : [], + ); } export async function touchParent( diff --git a/apps/server/src/modules/canvas/service.ts b/apps/server/src/modules/canvas/service.ts index e834fdd6..b1cb66a1 100644 --- a/apps/server/src/modules/canvas/service.ts +++ b/apps/server/src/modules/canvas/service.ts @@ -14,7 +14,9 @@ import { deleteStructuralBlocks, getBlocks, getBlocksCountByType, + getCustomBlockCalls, getCustomBlockNames, + getProjectCustomBlocks, getEdges, parentExists, parentKeys, @@ -143,6 +145,71 @@ async function assertBlockTypesExist( ); } +/** + * A custom block that reaches itself — directly or through a chain of other + * custom blocks — never terminates: `lib.invoke` would call into the block it is + * already inside. The editor hides the block from its own picker, but a cycle + * can still be closed from the other end (B gains a call to A while A already + * calls B), so the save that closes it is refused here. + * + * Only runs for a custom block's own canvas; a route can call anything. + */ +async function assertNoCustomBlockRecursion( + parent: CanvasParent, + data: CanvasChanges, + deleteBlockIds: string[], + tx: DbTransactionType, +) { + if (parent.type !== "custom_block") return; + + const blocks = await getProjectCustomBlocks(parent, tx); + const self = blocks.find((b) => b.id === parent.id); + if (!self) return; + const nameById = new Map(blocks.map((b) => [b.id, b.name])); + const names = new Set(blocks.map((b) => b.name)); + + // this canvas as it will be once the delta lands; the rest as stored + const deleted = new Set(deleteBlockIds); + const incoming = new Set(data.changes.blocks.map((b) => b.id)); + const selfCalls = new Set( + [ + ...data.changes.blocks.map((b) => b.type), + ...(await getBlocks(parent, tx)) + .filter((b) => !deleted.has(b.id) && !incoming.has(b.id)) + .map((b) => b.type ?? ""), + ].filter((type) => names.has(type)), + ); + + const calls = new Map>([[self.name, selfCalls]]); + for (const row of await getCustomBlockCalls( + blocks.map((b) => b.id), + tx, + )) { + const caller = nameById.get(row.parentId); + if (!caller || caller === self.name || !names.has(row.type)) continue; + const set = calls.get(caller) ?? new Set(); + set.add(row.type); + calls.set(caller, set); + } + + // shortest path back to self, so the message names the actual chain + const queue: string[][] = [[self.name]]; + const seen = new Set(); + while (queue.length > 0) { + const path = queue.shift()!; + for (const callee of calls.get(path[path.length - 1]) ?? []) { + if (callee === self.name) { + throw new ConflictError( + `Custom block "${self.name}" cannot call itself — recursion is not allowed (${[...path, callee].join(" → ")}).`, + ); + } + if (seen.has(callee)) continue; + seen.add(callee); + queue.push([...path, callee]); + } + } +} + /** * A verified agent run's canvas can arrive after the entrypoint/error handler * it means to replace already exist under different ids (e.g. the route was @@ -217,6 +284,7 @@ export async function saveCanvas( // a tx nests as a savepoint, so the outer transaction still decides the outcome await (outer ?? db).transaction(async (tx) => { await assertBlockTypesExist(parent, data, tx); + await assertNoCustomBlockRecursion(parent, data, deleteBlockIds, tx); await assertEdgeTargetsExist(parent, data, deleteBlockIds, tx); await assertCanvasHasNoCycles( parent, diff --git a/apps/server/src/modules/canvas/tests/service.spec.ts b/apps/server/src/modules/canvas/tests/service.spec.ts index 48c70b72..b4911317 100644 --- a/apps/server/src/modules/canvas/tests/service.spec.ts +++ b/apps/server/src/modules/canvas/tests/service.spec.ts @@ -42,6 +42,8 @@ const delEdges = spyOn(repository, "deleteEdges"); const delStructural = spyOn(repository, "deleteStructuralBlocks"); const getEdges = spyOn(repository, "getEdges"); const customBlockNames = spyOn(repository, "getCustomBlockNames"); +const projectCustomBlocks = spyOn(repository, "getProjectCustomBlocks"); +const customBlockCalls = spyOn(repository, "getCustomBlockCalls"); /** `changes` with its single block retyped */ const withBlockType = (type: string) => ({ @@ -66,6 +68,8 @@ describe("canvas saveCanvas", () => { } parentExists.mockResolvedValue(true); customBlockNames.mockResolvedValue([]); + projectCustomBlocks.mockResolvedValue([]); + customBlockCalls.mockResolvedValue([]); }); it("rejects a block type that is neither built-in nor a custom block", async () => { @@ -185,6 +189,63 @@ describe("canvas saveCanvas", () => { expect(upsertEdges).not.toHaveBeenCalled(); }); + describe("custom block recursion", () => { + const project = [ + { id: "cb-1", name: "audit" }, + { id: "cb-2", name: "logger" }, + ]; + /** `changes` carrying one instance of the named custom block */ + const calling = (type: string) => ({ + ...changes, + actionsToPerform: { blocks: [], edges: [] }, + changes: { + blocks: [{ id: "b1", type, data: {}, position: { x: 0, y: 0 } }], + edges: [], + }, + }); + + beforeEach(() => { + projectCustomBlocks.mockResolvedValue(project as any); + customBlockNames.mockResolvedValue(project.map((b) => b.name)); + }); + + it("rejects a custom block placed on its own canvas", async () => { + await expect( + saveCanvas({ type: "custom_block", id: "cb-1" }, calling("audit"), ["p1"]), + ).rejects.toThrow(/cannot call itself/); + expect(upsertBlocks).not.toHaveBeenCalled(); + }); + + it("rejects a chain that comes back around — logger already calls audit", async () => { + customBlockCalls.mockResolvedValue([ + { parentId: "cb-2", type: "audit" }, + ] as any); + + await expect( + saveCanvas({ type: "custom_block", id: "cb-1" }, calling("logger"), ["p1"]), + ).rejects.toThrow(/audit → logger → audit/); + }); + + it("allows a one-way call between two custom blocks", async () => { + await saveCanvas({ type: "custom_block", id: "cb-1" }, calling("logger"), [ + "p1", + ]); + + expect(upsertBlocks).toHaveBeenCalled(); + }); + + it("leaves route canvases alone — a route is never re-entered", async () => { + customBlockCalls.mockResolvedValue([ + { parentId: "cb-2", type: "audit" }, + { parentId: "cb-1", type: "logger" }, + ] as any); + + await saveCanvas({ type: "route", id: "r-1" }, calling("audit"), ["p1"]); + + expect(upsertBlocks).toHaveBeenCalled(); + }); + }); + it("rejects a duplicate structural block when the caller is not the AI harness", async () => { spyOn(repository, "getBlocksCountByType").mockResolvedValue([ { count: 2, type: "entrypoint" }, diff --git a/apps/server/src/modules/compiler/service.ts b/apps/server/src/modules/compiler/service.ts index 19dd418c..2cfa475a 100644 --- a/apps/server/src/modules/compiler/service.ts +++ b/apps/server/src/modules/compiler/service.ts @@ -2,6 +2,9 @@ import { logger } from "@fluxify/common"; import { BlockTypes, compileGraph, + hasCustomBlock, + registerCompiledCustomBlock, + unregisterCustomBlock, type BlockDTOType, type EdgeDTOSchemaType, } from "@fluxify/blocks"; @@ -120,6 +123,10 @@ export async function compileRoute(routeId: string) { return; } + // a route that calls a custom block only emits if that block is in this + // process's library — the artifact in KV is for workers, not for us + await ensureCustomBlocksRegistered(route.projectId!); + const { blocks, edges } = await loadGraph({ type: "route", id: routeId }); const { source } = compileGraph(blocks, edges); @@ -150,6 +157,66 @@ export async function dropRoute(projectId: string, routeId: string) { await deleteArtifact(routeKey(projectId, routeId)); } +/** custom blocks being compiled right now — see `ensureCustomBlocksRegistered` */ +const inFlight = new Set(); +/** what each custom block id is registered as, so a rename or delete can undo it */ +const registeredNames = new Map(); + +function registerLocally(id: string, name: string, source: string) { + const previous = registeredNames.get(id); + // a rename would otherwise leave the old name resolving to this block forever + if (previous && previous !== name) unregisterCustomBlock(previous); + registerCompiledCustomBlock(name, source); + registeredNames.set(id, name); +} + +function unregisterLocally(id: string) { + const name = registeredNames.get(id); + if (!name) return; + unregisterCustomBlock(name); + registeredNames.delete(id); +} + +/** + * Makes sure every custom block of a project is in this process's library + * before something that may call one is compiled. + * + * Compiling publishes an artifact for the workers; it does not make the block + * callable here, and `compileGraph` resolves a non-builtin type by asking the + * library. Without this, a route compile that happens before the block's own + * compile — a route saved after a restart, a fresh consumer — fails with + * "No codegen for block type". + * + * Order is discovered rather than declared: a block that calls another is + * retried once its callee lands. Cycles are impossible (the canvas save + * refuses them), so the fixpoint always terminates. + */ +async function ensureCustomBlocksRegistered(projectId: string) { + const rows = await db + .select({ id: customBlocksListEntity.id, name: customBlocksListEntity.name }) + .from(customBlocksListEntity) + .where(eq(customBlocksListEntity.projectId, projectId)); + + let pending = rows.filter( + (row) => !hasCustomBlock(row.name) && !inFlight.has(row.id), + ); + while (pending.length > 0) { + const failed: typeof pending = []; + let lastError: unknown; + for (const row of pending) { + try { + await compileCustomBlock(row.id); + } catch (error) { + lastError = error; + failed.push(row); + } + } + // nothing compiled this pass: the failures are real, not ordering + if (failed.length === pending.length) throw lastError; + pending = failed; + } +} + /** compile one custom block; a deleted one is dropped from the library */ export async function compileCustomBlock(id: string) { const [block] = await db @@ -163,12 +230,24 @@ export async function compileCustomBlock(id: string) { if (!block) { logger.info(`[compiler] dropping custom block ${id}`, "COMPILER"); + unregisterLocally(id); return; } - const { blocks, edges } = await loadGraph({ type: "custom_block", id }); - // `param:` placeholders resolve from the invocation, not from a caller's data - const { source } = compileGraph(blocks, edges, { asCustomBlock: true }); + // a custom block may call another one; same library requirement as a route + inFlight.add(id); + let source: string; + try { + await ensureCustomBlocksRegistered(block.projectId!); + const { blocks, edges } = await loadGraph({ type: "custom_block", id }); + // `param:` placeholders resolve from the invocation, not from a caller's data + ({ source } = compileGraph(blocks, edges, { asCustomBlock: true })); + } finally { + inFlight.delete(id); + } + // the compiler is also a consumer of its own output: the next route to call + // this block resolves it from here + registerLocally(block.id, block.name, source); const artifact: CustomBlockArtifact = { id: block.id, @@ -182,6 +261,7 @@ export async function compileCustomBlock(id: string) { } export async function dropCustomBlock(projectId: string, id: string) { + unregisterLocally(id); await deleteArtifact(customBlockKey(projectId, id)); } diff --git a/apps/server/src/modules/jobs/consumer.ts b/apps/server/src/modules/jobs/consumer.ts new file mode 100644 index 00000000..404779d5 --- /dev/null +++ b/apps/server/src/modules/jobs/consumer.ts @@ -0,0 +1,167 @@ +import { logger } from "@fluxify/common"; +import { AckPolicy, RetentionPolicy, StringCodec, type JsMsg } from "nats"; +import { natsConnection } from "../../db/nats"; +import { + JOBS_STREAM, + JOBS_SUBJECTS, + jobConsumerName, + projectJobFilter, +} from "./subjects"; +import type { JobEnvelope } from "./types"; + +/** + * The job worker's transport half: one durable pull consumer on the shared + * stream, filtered to this deployment's project. Running the work is the + * caller's `handle` — this file only decides what gets acked, retried or + * dropped. + * + * A work-queue stream removes a message once it is acked, so a restart never + * replays yesterday's jobs. Everything below is tunable because a queued custom + * block and a nightly cron want very different ack waits. + */ + +export type JobWorkerOptions = { + /** Project this deployment serves, or "*" for every project. */ + projectId: string; + /** Runs the job. Resolve to ack, reject to retry. */ + handle: (job: JobEnvelope) => Promise; + /** Jobs in flight at once. */ + concurrency?: number; + /** How long a job may run before the broker assumes the worker died. */ + ackWaitMs?: number; + /** Attempts before the job is dropped and logged as dead. */ + maxDeliver?: number; + /** Wait before a failed job is redelivered. */ + retryDelayMs?: number; + /** How long an unclaimed job stays on the stream. */ + maxAgeMs?: number; +}; + +const sc = StringCodec(); +const DEFAULTS = { + concurrency: 5, + ackWaitMs: 5 * 60_000, + maxDeliver: 5, + retryDelayMs: 10_000, + maxAgeMs: 7 * 24 * 60 * 60_000, +}; + +let running = false; + +export async function startJobWorker(options: JobWorkerOptions) { + if (running) return; + const config: Required = { + ...DEFAULTS, + ...stripUndefined(options), + projectId: options.projectId, + handle: options.handle, + }; + const nc = natsConnection(); + const jsm = await nc.jetstreamManager(); + + await ensureStream(jsm, config.maxAgeMs); + await ensureConsumer(jsm, config); + + const consumer = await nc + .jetstream() + .consumers.get(JOBS_STREAM, jobConsumerName(config.projectId)); + const messages = await consumer.consume({ max_messages: config.concurrency }); + running = true; + logger.info( + `[jobs] worker listening on ${projectJobFilter(config.projectId)}`, + "JOBS", + ); + + void (async () => { + const inFlight = new Set>(); + for await (const message of messages) { + // Bound the concurrency ourselves: `max_messages` limits what the server + // pushes, not what we start. + if (inFlight.size >= config.concurrency) await Promise.race(inFlight); + const task = settle(message, config).finally(() => inFlight.delete(task)); + inFlight.add(task); + } + await Promise.allSettled(inFlight); + })(); +} + +async function settle(message: JsMsg, config: Required) { + let job: JobEnvelope | undefined; + try { + job = JSON.parse(sc.decode(message.data)) as JobEnvelope; + job.attempt = message.info.redeliveryCount; + await config.handle(job); + message.ack(); + } catch (error) { + const label = job ? `${job.kind}/${job.target}` : message.subject; + // Unparseable, or a kind nobody handles: retrying changes nothing. + if (!job || isPermanent(error)) { + logger.error(`[jobs] dropping ${label}: ${String(error)}`, "JOBS"); + return message.term(); + } + if (message.info.redeliveryCount >= config.maxDeliver) { + logger.error( + `[jobs] ${label} failed ${message.info.redeliveryCount} times, giving up: ${String(error)}`, + "JOBS", + ); + return message.term(); + } + logger.warn( + `[jobs] ${label} failed (attempt ${message.info.redeliveryCount}), retrying: ${String(error)}`, + "JOBS", + ); + message.nak(config.retryDelayMs); + } +} + +/** A handler can opt a failure out of retries by naming it. */ +function isPermanent(error: unknown) { + return (error as Error)?.name === "UnknownJobKindError"; +} + +function stripUndefined(value: T): Partial { + return Object.fromEntries( + Object.entries(value).filter(([, v]) => v !== undefined), + ) as Partial; +} + +async function ensureStream(jsm: any, maxAgeMs: number) { + const spec = { + name: JOBS_STREAM, + subjects: [JOBS_SUBJECTS], + // a job is work, not history: once acked it leaves the stream + retention: RetentionPolicy.Workqueue, + max_age: maxAgeMs * 1_000_000, // ns + // publisher dedupe window for `msgID` + duplicate_window: 2 * 60 * 1_000_000_000, + }; + try { + await jsm.streams.add(spec); + } catch { + await jsm.streams.update(JOBS_STREAM, { + subjects: spec.subjects, + max_age: spec.max_age, + }); + } +} + +async function ensureConsumer(jsm: any, config: Required) { + const spec = { + durable_name: jobConsumerName(config.projectId), + ack_policy: AckPolicy.Explicit, + ack_wait: config.ackWaitMs * 1_000_000, // ns + max_deliver: config.maxDeliver, + filter_subject: projectJobFilter(config.projectId), + max_ack_pending: config.concurrency, + }; + try { + await jsm.consumers.add(JOBS_STREAM, spec); + } catch { + // already exists — pick up changed limits without losing pending work + await jsm.consumers.update(JOBS_STREAM, spec.durable_name, { + ack_wait: spec.ack_wait, + max_deliver: spec.max_deliver, + max_ack_pending: spec.max_ack_pending, + }); + } +} diff --git a/apps/server/src/modules/jobs/customBlockJob.ts b/apps/server/src/modules/jobs/customBlockJob.ts new file mode 100644 index 00000000..ebdc15d4 --- /dev/null +++ b/apps/server/src/modules/jobs/customBlockJob.ts @@ -0,0 +1,35 @@ +import { + CUSTOM_BLOCK_JOB, + invokeCustomBlock, + type CustomBlockArgs, +} from "@fluxify/blocks"; +import { logger } from "@fluxify/common"; +import { createJobContext } from "../requestRouter/service"; +import { registerJobHandler } from "./registry"; + +/** + * Runs a custom block that its caller queued instead of awaiting. + * + * Registered on the execution process — the one that holds the compiled block + * library and can run user code. It throws on failure so the consumer retries; + * the caller is long gone, so the queue is the only thing that can. + */ +export function registerCustomBlockJobHandler() { + registerJobHandler(CUSTOM_BLOCK_JOB, async (job) => { + const context = createJobContext({ + id: job.id, + projectId: job.projectId, + target: job.target, + }); + try { + await invokeCustomBlock( + context, + job.target, + (job.payload as CustomBlockArgs) ?? { params: {} }, + ); + logger.info(`[jobs] ran custom block ${job.target}`, "JOBS.custom-block"); + } finally { + context.dbFactory?.dispose(); + } + }); +} diff --git a/apps/server/src/modules/jobs/publisher.ts b/apps/server/src/modules/jobs/publisher.ts new file mode 100644 index 00000000..63397ea5 --- /dev/null +++ b/apps/server/src/modules/jobs/publisher.ts @@ -0,0 +1,37 @@ +import { logger } from "@fluxify/common"; +import { StringCodec } from "nats"; +import { natsConnection } from "../../db/nats"; +import { jobSubject } from "./subjects"; +import type { JobEnvelope } from "./types"; + +const sc = StringCodec(); + +export type JobInput = Omit & { + /** Supply one to make a retry of the same logical work collapse. */ + id?: string; +}; + +/** + * Publishes to JetStream, so the job outlives the process that queued it. The + * message id is the broker's dedupe key within its duplicate window: publishing + * the same id twice (a retried request, a redelivered upstream message) enqueues + * the work once. + * + * Throws rather than logging: the caller asked for durable work, and a queue + * that swallows failures is worse than one that refuses them. + */ +export async function enqueueJob(input: JobInput): Promise { + const job: JobEnvelope = { + ...input, + id: input.id ?? crypto.randomUUID(), + enqueuedAt: new Date().toISOString(), + }; + const subject = jobSubject(job.projectId, job.kind); + + await natsConnection() + .jetstream() + .publish(subject, sc.encode(JSON.stringify(job)), { msgID: job.id }); + + logger.debug(`[jobs] queued ${subject} (${job.target})`, "JOBS.publish"); + return job; +} diff --git a/apps/server/src/modules/jobs/registry.ts b/apps/server/src/modules/jobs/registry.ts new file mode 100644 index 00000000..8da69a4e --- /dev/null +++ b/apps/server/src/modules/jobs/registry.ts @@ -0,0 +1,34 @@ +import type { JobEnvelope, JobHandler } from "./types"; + +/** + * What each job kind means, on whichever process actually runs the work. + * + * Kept apart from the consumer on purpose: the transport (stream, acks, + * redelivery) belongs to the process that owns the NATS connection, while the + * handlers belong to the process that owns user code. Adding a cron or workflow + * kind later is one `registerJobHandler` call and no transport changes. + */ + +const handlers = new Map(); + +export function registerJobHandler(kind: string, handler: JobHandler) { + handlers.set(kind, handler); +} + +export function jobKinds() { + return [...handlers.keys()]; +} + +/** Thrown for a kind nobody handles — retrying that can never succeed. */ +export class UnknownJobKindError extends Error { + constructor(kind: string) { + super(`No handler registered for job kind "${kind}"`); + this.name = "UnknownJobKindError"; + } +} + +export async function runJob(job: JobEnvelope) { + const handler = handlers.get(job.kind); + if (!handler) throw new UnknownJobKindError(job.kind); + await handler(job); +} diff --git a/apps/server/src/modules/jobs/subjects.ts b/apps/server/src/modules/jobs/subjects.ts new file mode 100644 index 00000000..66729345 --- /dev/null +++ b/apps/server/src/modules/jobs/subjects.ts @@ -0,0 +1,41 @@ +/** + * Naming for the job queue. + * + * ONE stream carries every kind of background work — queued custom blocks + * today, crons/workflows/scheduled jobs later. A stream per feature would mean + * a pile of mostly-idle streams and consumers to operate, so kinds are a subject + * token instead: adding one costs a handler registration and nothing else. + * + * Subjects are `fluxify.jobs..`. Project comes first so a + * worker can filter on its own tenant with a single wildcard, which is also what + * keeps consumers non-overlapping — a work-queue stream requires that. + */ + +export const JOBS_STREAM = "FLUXIFY_JOBS"; + +const SUBJECT_ROOT = "fluxify.jobs"; +/** everything the job worker can consume */ +export const JOBS_SUBJECTS = `${SUBJECT_ROOT}.>`; +/** serves every project — the catch-all worker deployment */ +export const ALL_PROJECTS = "*"; + +export const jobSubject = (projectId: string, kind: string) => + `${SUBJECT_ROOT}.${projectId}.${kind}`; + +/** What one worker deployment subscribes to. */ +export function projectJobFilter(projectId: string) { + return projectId === ALL_PROJECTS + ? JOBS_SUBJECTS + : `${SUBJECT_ROOT}.${projectId}.>`; +} + +/** + * Durable name per deployment, so replicas of the same worker compete for the + * same messages while different projects never see each other's. + * Consumer names allow no dots or wildcards. + */ +export function jobConsumerName(projectId: string) { + return projectId === ALL_PROJECTS + ? "fluxify_jobs_all" + : `fluxify_jobs_${projectId.replace(/[^a-zA-Z0-9_-]/g, "_")}`; +} diff --git a/apps/server/src/modules/jobs/types.ts b/apps/server/src/modules/jobs/types.ts new file mode 100644 index 00000000..a754befc --- /dev/null +++ b/apps/server/src/modules/jobs/types.ts @@ -0,0 +1,22 @@ +/** + * One queued unit of work, as it travels on the wire. + * + * Deliberately kind-agnostic: a queued custom block, a cron firing, a workflow + * step and a scheduled job are all this shape, so they all share one stream and + * one worker. `kind` is the only thing that decides how `target` and `payload` + * are read — see the handler registry. + */ +export type JobEnvelope = { + /** Dedupe key on the broker and correlation id in logs. */ + id: string; + kind: string; + projectId: string; + target: string; + payload?: unknown; + origin?: Record; + enqueuedAt: string; + /** Delivery attempt, filled in by the consumer, not the publisher. */ + attempt?: number; +}; + +export type JobHandler = (job: JobEnvelope) => Promise; diff --git a/apps/server/src/modules/requestRouter/executionProcess.ts b/apps/server/src/modules/requestRouter/executionProcess.ts index 531ba707..c9b0dc38 100644 --- a/apps/server/src/modules/requestRouter/executionProcess.ts +++ b/apps/server/src/modules/requestRouter/executionProcess.ts @@ -1,5 +1,9 @@ import { initializeLogger, logger } from "@fluxify/common"; +import { setJobEnqueuer } from "@fluxify/blocks"; import { createHttpContext } from "./httpContext"; +import { registerCustomBlockJobHandler } from "../jobs/customBlockJob"; +import { runJob } from "../jobs/registry"; +import type { JobEnvelope } from "../jobs/types"; import { applyArtifactUpdate, compiledRouteValidators, @@ -37,9 +41,23 @@ process.on("message", (message: ExecutionMessage) => { applyArtifactUpdate(message.entry.key, message.entry.value); return; } + if (message.type === "job") return void executeJob(message.job); setMonitoring(message.enabled); }); +/** + * The supervisor acks the message on the reply, so every path must send one — + * a swallowed error here stalls the job until its ack wait elapses. + */ +async function executeJob(job: JobEnvelope) { + try { + await runJob(job); + send({ type: "job-finished", id: job.id }); + } catch (error) { + send({ type: "job-finished", id: job.id, error: String(error) }); + } +} + function bootstrap(nextBoot: ExecutionBootstrap) { if (boot) throw new Error("execution process was bootstrapped twice"); boot = nextBoot; @@ -55,6 +73,19 @@ function bootstrap(nextBoot: ExecutionBootstrap) { logger.error(`async dispatch failed: ${String(error)}`, "WORKER.execution"), ); setMonitoring(boot.workerTimeoutsEnabled); + registerCustomBlockJobHandler(); + // This process holds no broker connection: queueing is a message to the + // supervisor, which owns NATS. + setJobEnqueuer((request) => + send({ + type: "enqueue-job", + job: { + ...request, + id: crypto.randomUUID(), + enqueuedAt: new Date().toISOString(), + }, + }), + ); server = Bun.serve({ port: boot.port, diff --git a/apps/server/src/modules/requestRouter/service.ts b/apps/server/src/modules/requestRouter/service.ts index 71d85d27..65534f73 100644 --- a/apps/server/src/modules/requestRouter/service.ts +++ b/apps/server/src/modules/requestRouter/service.ts @@ -322,6 +322,61 @@ export async function executeRouteInternal( } } +/** + * A context for work that arrived off the queue rather than off a request. + * + * Same integrations, app config and logger a route gets — the difference is + * that there is no HTTP exchange, so cookies and headers are no-ops and the + * request accessors return empty. Nothing from the enqueuing request survives + * except the payload it queued. + * + * ponytail: fixed timeout. Give it a per-job setting when a job legitimately + * needs to outlive it. + */ +export const DEFAULT_JOB_TIMEOUT_SECONDS = 300; + +export function createJobContext(job: { + id: string; + projectId: string; + target: string; + timeoutSeconds?: number; +}): BlockContext { + const trigger: TriggerContext = { + kind: "job", + source: "nats", + reply: "async", + id: job.id, + }; + const requestData = { + method: "", + path: job.target, + headers: {}, + query: {}, + body: undefined, + params: {}, + }; + const vars = setupContextVars( + undefined, + requestData, + job.target, + httpClient, + job.projectId, + undefined, + trigger, + ); + const vm = createJsVM(vars); + return createContext( + { id: job.target, projectId: job.projectId }, + requestData, + vm, + vars, + new DbFactory(vm, dbIntegrationsCache, dbConnectionManager), + httpClient, + trigger, + job.timeoutSeconds ?? DEFAULT_JOB_TIMEOUT_SECONDS, + ); +} + function validateSchema( compiled: CompiledRequestSchema | undefined, schema: unknown, diff --git a/apps/server/src/modules/requestRouter/threadTypes.ts b/apps/server/src/modules/requestRouter/threadTypes.ts index 8c1aada3..ea98fe2d 100644 --- a/apps/server/src/modules/requestRouter/threadTypes.ts +++ b/apps/server/src/modules/requestRouter/threadTypes.ts @@ -1,5 +1,6 @@ import type { ArtifactEntry } from "./compiledRuntime"; import type { AsyncExecutorLimits } from "./asyncExecutor"; +import type { JobEnvelope } from "../jobs/types"; /** handed to the isolated execution process over Bun IPC at spawn */ export type ExecutionBootstrap = { @@ -28,12 +29,17 @@ export type ExecutionBootstrap = { export type ExecutionMessage = | { type: "bootstrap"; bootstrap: ExecutionBootstrap } | { type: "artifact"; entry: ArtifactEntry } - | { type: "monitoring"; enabled: boolean }; + | { type: "monitoring"; enabled: boolean } + // the supervisor owns NATS, the child owns user code: jobs cross here + | { type: "job"; job: JobEnvelope }; /** isolated execution process -> supervisor */ export type ExecutionEvent = | { type: "ready" } | { type: "heartbeat" } + | { type: "job-finished"; id: string; error?: string } + /** user code asked to queue work; only the supervisor can publish it */ + | { type: "enqueue-job"; job: JobEnvelope } | { type: "execution-started"; requestId: string; diff --git a/apps/server/src/modules/testRunner/testExecutionProcess.ts b/apps/server/src/modules/testRunner/testExecutionProcess.ts index 29ebad1d..4cfd7942 100644 --- a/apps/server/src/modules/testRunner/testExecutionProcess.ts +++ b/apps/server/src/modules/testRunner/testExecutionProcess.ts @@ -1,4 +1,8 @@ -import { instantiateCompiled, registerCompiledCustomBlock } from "@fluxify/blocks"; +import { + instantiateCompiled, + registerCompiledCustomBlock, + setJobEnqueuer, +} from "@fluxify/blocks"; import { hydrateAppConfig } from "../../loaders/appconfigLoader"; import { hydrateIntegrations } from "../../loaders/integrationsLoader"; import { hydrateProjectSettings } from "../../loaders/projectSettingsLoader"; @@ -33,6 +37,9 @@ process.on("message", (message: TestBootstrapMessage) => { void runSuite(message.bootstrap); }); +/** What the suite would have queued, kept for debugging a run. */ +const queuedJobs: string[] = []; + async function runSuite(boot: TestBootstrap) { const startedAt = Date.now(); try { @@ -52,6 +59,12 @@ async function runSuite(boot: TestBootstrap) { registerCompiledCustomBlock(block.name, block.source); } + // A test run holds no broker connection, and firing real background work + // from an assertion is not something a suite should be able to do. + setJobEnqueuer((job) => + queuedJobs.push(`${job.kind}/${job.target}`), + ); + const run = instantiateCompiled(boot.source); setBlocksExecutor((_target, context) => run(context, context.requestBody)); diff --git a/packages/blocks/builtin/customBlock.ts b/packages/blocks/builtin/customBlock.ts index 4c9c3dff..470daa6d 100644 --- a/packages/blocks/builtin/customBlock.ts +++ b/packages/blocks/builtin/customBlock.ts @@ -3,6 +3,7 @@ import { BaseBlock, BlockOptions, BlockOutput, Context } from "../baseBlock"; import { Engine } from "../engine"; import { logger } from "@fluxify/common"; import type { BlockDTOType, EdgeDTOSchemaType } from "../builderTypes"; +import { enqueueJob } from "../jobs"; import { compileGraph, emitJsObject, @@ -11,12 +12,15 @@ import { } from "../compiler"; export const customBlockInvokeSchema = z - .enum(["sync", "async"]) + .enum(["sync", "async", "queued"]) .default("sync") .describe( - "sync waits for the custom block and takes its output; async fires it and moves on", + "sync waits for the custom block and takes its output; async fires it on this worker and moves on; queued hands it to the job queue for another worker to run", ); +/** The job kind a queued custom block travels under. */ +export const CUSTOM_BLOCK_JOB = "custom-block"; + export type CompiledCustomBlock = (ctx: Context, input?: any) => Promise; /** @@ -86,16 +90,23 @@ function traced( return { context: child, close: () => scope.close() }; } +/** + * How a custom block is called: what it was configured with, and the value + * flowing into the calling block. They stay apart inside the callee — `params` + * is its configuration at any depth, `input` is the previous block's output. + */ +export type CustomBlockArgs = { params: Record; input?: any }; + /** sync: wait for the custom block and hand back its output */ export async function invokeCustomBlock( context: Context, name: string, - params: any, + args: CustomBlockArgs, blockId?: string, ) { const scope = traced(context, name, blockId, false); try { - const result = await lookup(name)(scope.context, params); + const result = await lookup(name)(scope.context, args); return result?.output; } finally { scope.close(); @@ -110,11 +121,11 @@ export async function invokeCustomBlock( export function invokeCustomBlockAsync( context: Context, name: string, - params: any, + args: CustomBlockArgs, blockId?: string, ) { const scope = traced(context, name, blockId, true); - lookup(name)(scope.context, params) + lookup(name)(scope.context, args) .catch((error) => { logger.error(`Async custom block '${name}' failed`, "BLOCKS.customBlock", { error, @@ -123,15 +134,44 @@ export function invokeCustomBlockAsync( .finally(() => scope.close()); } +/** + * queued: hand the work to the job queue and move on. Unlike `async` this + * survives the worker — the job is persisted by the broker and picked up by + * whichever worker serves this project, so a shutdown mid-flight redelivers + * instead of losing the work. + * + * The job carries the evaluated arguments only: the callee gets a fresh context + * on the other side, so nothing request-scoped (cookies, headers, `ctx.vars`) + * crosses. They must therefore be JSON-serializable. + */ +export function enqueueCustomBlock( + context: Context, + name: string, + args: CustomBlockArgs, + blockId?: string, +) { + enqueueJob({ + kind: CUSTOM_BLOCK_JOB, + projectId: context.projectId, + target: name, + payload: args, + origin: { blockId, route: context.route, apiId: context.apiId }, + }); +} + export function emitCustomBlock(node: EmitNode) { const { blockName, blockDescription, invoke, ...params } = (node.block.data ?? {}) as Record; const mode = customBlockInvokeSchema.parse(invoke); const name = JSON.stringify(node.block.type); - // same shape the interpreted block passes: evaluated params plus the input - const args = `{ ...${emitJsObject(params, node)}, input: ${node.in} }`; + // the callee reads its configuration as `params` and the value flowing into + // this block as `input` — one meaning each, at every depth of its graph + const args = `{ params: ${emitJsObject(params, node)}, input: ${node.in} }`; const id = JSON.stringify(node.block.id); + if (mode === "queued") { + return `lib.enqueue(ctx, ${name}, ${args}, ${id});\n${node.next()}`; + } if (mode === "async") { return `lib.invokeAsync(ctx, ${name}, ${args}, ${id});\n${node.next()}`; } diff --git a/packages/blocks/builtin/db/delete.ts b/packages/blocks/builtin/db/delete.ts index 32a1aa60..4a61c8a4 100644 --- a/packages/blocks/builtin/db/delete.ts +++ b/packages/blocks/builtin/db/delete.ts @@ -47,7 +47,7 @@ export async function runDeleteDb( export function emitDeleteDb(node: EmitNode) { const input = deleteDbBlockSchema.parse(node.block.data); - return `${node.in} = await lib.dbDelete(ctx, ${JSON.stringify(input.connection)}, ${node.value(input.tableName)}, ${emitWhereConditions(input.conditions, node)}); + return `${node.in} = await lib.dbDelete(ctx, ${node.value(input.connection)}, ${node.value(input.tableName)}, ${emitWhereConditions(input.conditions, node)}); ${node.next()}`; } diff --git a/packages/blocks/builtin/db/getAll.ts b/packages/blocks/builtin/db/getAll.ts index 43af0e61..9493d399 100644 --- a/packages/blocks/builtin/db/getAll.ts +++ b/packages/blocks/builtin/db/getAll.ts @@ -86,7 +86,7 @@ export async function runGetAllDb( export function emitGetAllDb(node: EmitNode) { const input = getAllDbBlockSchema.parse(node.block.data); const sort = `{ attribute: ${node.value(input.sort.attribute)}, direction: ${JSON.stringify(input.sort.direction)} }`; - return `${node.in} = await lib.dbGetAll(ctx, ${JSON.stringify(input.connection)}, ${node.value(input.tableName)}, ${emitWhereConditions(input.conditions, node)}, lib.num(${node.value(input.limit)}, 1000), lib.num(${node.value(input.offset)}, 0), ${sort}, { joins: ${JSON.stringify(input.joins ?? [])}, columns: ${JSON.stringify(input.columns ?? ["*"])} }); + return `${node.in} = await lib.dbGetAll(ctx, ${node.value(input.connection)}, ${node.value(input.tableName)}, ${emitWhereConditions(input.conditions, node)}, lib.num(${node.value(input.limit)}, 1000), lib.num(${node.value(input.offset)}, 0), ${sort}, { joins: ${JSON.stringify(input.joins ?? [])}, columns: ${JSON.stringify(input.columns ?? ["*"])} }); ${node.next()}`; } diff --git a/packages/blocks/builtin/db/getSingle.ts b/packages/blocks/builtin/db/getSingle.ts index ecef0a11..4919cebf 100644 --- a/packages/blocks/builtin/db/getSingle.ts +++ b/packages/blocks/builtin/db/getSingle.ts @@ -60,7 +60,7 @@ export async function runGetSingleDb( export function emitGetSingleDb(node: EmitNode) { const input = getSingleDbBlockSchema.parse(node.block.data); - return `${node.in} = await lib.dbGetSingle(ctx, ${JSON.stringify(input.connection)}, ${node.value(input.tableName)}, ${emitWhereConditions(input.conditions, node)}, { joins: ${JSON.stringify(input.joins ?? [])}, columns: ${JSON.stringify(input.columns ?? ["*"])} }); + return `${node.in} = await lib.dbGetSingle(ctx, ${node.value(input.connection)}, ${node.value(input.tableName)}, ${emitWhereConditions(input.conditions, node)}, { joins: ${JSON.stringify(input.joins ?? [])}, columns: ${JSON.stringify(input.columns ?? ["*"])} }); ${node.next()}`; } diff --git a/packages/blocks/builtin/db/insert.ts b/packages/blocks/builtin/db/insert.ts index e08a2990..7f85d6bd 100644 --- a/packages/blocks/builtin/db/insert.ts +++ b/packages/blocks/builtin/db/insert.ts @@ -70,7 +70,7 @@ export function emitInsertDb(node: EmitNode) { return `const ${data} = ${payload}; if (typeof ${data} !== "object") throw new Error("error in insert: data to insert is not an object"); -${node.in} = await lib.dbInsert(ctx, ${JSON.stringify(input.connection)}, ${node.value(input.tableName)}, ${data}); +${node.in} = await lib.dbInsert(ctx, ${node.value(input.connection)}, ${node.value(input.tableName)}, ${data}); ${node.next()}`; } diff --git a/packages/blocks/builtin/db/insertBulk.ts b/packages/blocks/builtin/db/insertBulk.ts index da0d6b56..8b2897d6 100644 --- a/packages/blocks/builtin/db/insertBulk.ts +++ b/packages/blocks/builtin/db/insertBulk.ts @@ -61,7 +61,7 @@ export function emitInsertBulkDb(node: EmitNode) { return `const ${data} = ${payload}; if (!Array.isArray(${data})) throw new Error("error in insert bulk: data to insert is not an array"); -${node.in} = await lib.dbInsertBulk(ctx, ${JSON.stringify(input.connection)}, ${node.value(input.tableName)}, ${data}); +${node.in} = await lib.dbInsertBulk(ctx, ${node.value(input.connection)}, ${node.value(input.tableName)}, ${data}); ${node.next()}`; } diff --git a/packages/blocks/builtin/db/native.ts b/packages/blocks/builtin/db/native.ts index dcdcdad2..7d578ea9 100644 --- a/packages/blocks/builtin/db/native.ts +++ b/packages/blocks/builtin/db/native.ts @@ -52,7 +52,7 @@ export async function runNativeDb( export function emitNativeDb(node: EmitNode) { const input = nativeDbBlockSchema.parse(node.block.data); const code = input.js.startsWith("js:") ? input.js.slice(3) : input.js; - return `${node.in} = await lib.dbNative(ctx, ${JSON.stringify(input.connection)}, async () => ${node.js(code, node.in)}); + return `${node.in} = await lib.dbNative(ctx, ${node.value(input.connection)}, async () => ${node.js(code, node.in)}); ${node.next()}`; } diff --git a/packages/blocks/builtin/db/transaction.ts b/packages/blocks/builtin/db/transaction.ts index 25b401a0..24ea1d69 100644 --- a/packages/blocks/builtin/db/transaction.ts +++ b/packages/blocks/builtin/db/transaction.ts @@ -57,7 +57,7 @@ export async function runTransactionDb( export function emitTransactionDb(node: EmitNode) { const input = transactionDbBlockSchema.parse(node.block.data); const result = node.v("tx"); - return `const ${result} = await lib.dbTransaction(ctx, ${JSON.stringify(input.connection)}, async () => { + return `const ${result} = await lib.dbTransaction(ctx, ${node.value(input.connection)}, async () => { ${node.body("executor", "undefined")} }); if (${result} !== undefined) return ${result}; diff --git a/packages/blocks/builtin/db/update.ts b/packages/blocks/builtin/db/update.ts index 1348ad7f..d168f85c 100644 --- a/packages/blocks/builtin/db/update.ts +++ b/packages/blocks/builtin/db/update.ts @@ -81,7 +81,7 @@ export function emitUpdateDb(node: EmitNode) { return `const ${data} = ${payload}; if (typeof ${data} !== "object") throw new Error("error in update: data to update is not an object"); -${node.in} = await lib.dbUpdate(ctx, ${JSON.stringify(input.connection)}, ${node.value(input.tableName)}, ${data}, ${emitWhereConditions(input.conditions, node)}); +${node.in} = await lib.dbUpdate(ctx, ${node.value(input.connection)}, ${node.value(input.tableName)}, ${data}, ${emitWhereConditions(input.conditions, node)}); ${node.next()}`; } diff --git a/packages/blocks/builtin/log/cloudLogs.ts b/packages/blocks/builtin/log/cloudLogs.ts index 1b83684d..51a0a594 100644 --- a/packages/blocks/builtin/log/cloudLogs.ts +++ b/packages/blocks/builtin/log/cloudLogs.ts @@ -48,7 +48,7 @@ export function emitCloudLogs(node: EmitNode) { const { connection, level, message } = cloudLogsBlockSchema.parse( node.block.data, ); - return `await lib.cloudLog(ctx, ${JSON.stringify(connection)}, ${JSON.stringify(level)}, ${emitLogMessage(message, node)}, ${node.in});\n${node.next()}`; + return `await lib.cloudLog(ctx, ${node.value(connection)}, ${JSON.stringify(level)}, ${emitLogMessage(message, node)}, ${node.in});\n${node.next()}`; } export class CloudLogsBlock extends BaseBlock { diff --git a/packages/blocks/builtin/tests/compilerCustomBlock.spec.ts b/packages/blocks/builtin/tests/compilerCustomBlock.spec.ts index aa52bc09..33efd06a 100644 --- a/packages/blocks/builtin/tests/compilerCustomBlock.spec.ts +++ b/packages/blocks/builtin/tests/compilerCustomBlock.spec.ts @@ -9,6 +9,7 @@ import { unregisterCustomBlock, } from "../customBlock"; import { BlockTypes } from "../../blockTypes"; +import { setJobEnqueuer } from "../../jobs"; import type { BlockDTOType, EdgeDTOSchemaType } from "../../builderTypes"; const block = (id: string, type: string, data: any = {}): BlockDTOType => ({ @@ -49,7 +50,8 @@ function registerDoubler(name = "double_it") { [ block("c1", BlockTypes.entrypoint), block("c2", BlockTypes.jsrunner, { - value: "calls = (typeof calls === 'number' ? calls : 0) + 1; return input.value * 2;", + value: + "calls = (typeof calls === 'number' ? calls : 0) + 1; return params.value * 2;", }), ], [edge("c1", "c2")], @@ -58,6 +60,7 @@ function registerDoubler(name = "double_it") { afterEach(() => { for (const name of customBlockNames()) unregisterCustomBlock(name); + setJobEnqueuer(); }); describe("compiled custom blocks", () => { @@ -106,7 +109,7 @@ describe("compiled custom blocks", () => { block("c1", BlockTypes.entrypoint), block("c2", BlockTypes.jsrunner, { value: - "await new Promise((r) => setTimeout(r, 5)); sideEffect = input.value; return sideEffect;", + "await new Promise((r) => setTimeout(r, 5)); sideEffect = params.value; return sideEffect;", }), ], [edge("c1", "c2")], @@ -157,12 +160,12 @@ describe("compiled custom blocks", () => { expect(result.output.body).toBe("unharmed"); }); - it("passes evaluated params plus the flowing value as input", async () => { + it("separates the configured params from the flowing input", async () => { registerCustomBlock( "echo_params", [ block("c1", BlockTypes.entrypoint), - block("c2", BlockTypes.jsrunner, { value: "return input;" }), + block("c2", BlockTypes.jsrunner, { value: "return { params, input };" }), ], [edge("c1", "c2")], ); @@ -181,12 +184,38 @@ describe("compiled custom blocks", () => { const result = await run(createContext(), { n: 41 }); expect(result.output.body).toEqual({ - literal: "kept", - computed: 42, + params: { literal: "kept", computed: 42 }, input: { n: 41 }, }); }); + it("keeps params visible further down the callee's graph", async () => { + // `input` becomes the previous block's output; `params` does not move + registerCustomBlock( + "two_steps", + [ + block("c1", BlockTypes.entrypoint), + block("c2", BlockTypes.jsrunner, { value: "return 'step-one';" }), + block("c3", BlockTypes.jsrunner, { + value: "return { seen: input, still: params.label };", + }), + ], + [edge("c1", "c2"), edge("c2", "c3")], + ); + + const { run } = compileGraph( + [ + block("1", BlockTypes.entrypoint), + block("2", "two_steps", { label: "kept" }), + block("3", BlockTypes.response, { httpCode: "200" }), + ], + [edge("1", "2"), edge("2", "3")], + ); + + const result = await run(createContext(), null); + expect(result.output.body).toEqual({ seen: "step-one", still: "kept" }); + }); + it("resolves param: placeholders from the invocation, not the caller", async () => { // compiled once for the whole worker, so a placeholder cannot be baked in registerCustomBlock( @@ -218,6 +247,56 @@ describe("compiled custom blocks", () => { expect(second.vars.greeting).toBe("hola"); }); + it("queued invoke hands the work to the job queue and does not run it", async () => { + registerDoubler(); + const queued: any[] = []; + setJobEnqueuer((job) => queued.push(job)); + + const { run, source } = compileGraph( + [ + block("1", BlockTypes.entrypoint), + block("2", "double_it", { value: 21, invoke: "queued" }), + block("3", BlockTypes.response, { httpCode: "200" }), + ], + [edge("1", "2"), edge("2", "3")], + ); + expect(source).toContain("lib.enqueue"); + + const ctx = createContext(); + const result = await run(ctx, { keep: "me" }); + + // the caller kept its own value and the block never executed here + expect(result.output.body).toEqual({ keep: "me" }); + expect(ctx.vars.calls).toBeUndefined(); + expect(queued).toEqual([ + { + kind: "custom-block", + projectId: "proj-1", + target: "double_it", + payload: { params: { value: 21 }, input: { keep: "me" } }, + origin: { blockId: "2", route: "/custom", apiId: "api-1" }, + }, + ]); + }); + + it("a queued invoke fails loudly when no queue is wired", async () => { + registerDoubler(); + setJobEnqueuer(); + + const { run } = compileGraph( + [ + block("1", BlockTypes.entrypoint), + block("2", "double_it", { value: 1, invoke: "queued" }), + block("3", BlockTypes.response, { httpCode: "200" }), + ], + [edge("1", "2"), edge("2", "3")], + ); + + const result = await run(createContext(), null); + expect(result.successful).toBe(false); + expect(String(result.error)).toContain("No job queue"); + }); + it("registers already-compiled source without running the compiler", async () => { // what a worker does: it receives JS from the artifact store, never a graph const source = registerDoubler("shipped"); @@ -302,4 +381,42 @@ describe("compiled cloud logs", () => { // the block passes its input through untouched expect(result.output.body).toEqual({ id: 9 }); }); + + it("takes a custom block's integration from the caller's param", async () => { + // the integration selector input param: the block picks `param:obs`, the + // caller picks the concrete integration id + registerCustomBlock( + "audit", + [ + block("c1", BlockTypes.entrypoint), + block("c2", BlockTypes.cloudLogs, { + connection: "param:obs", + level: "info", + message: "audited", + }), + ], + [edge("c1", "c2")], + ); + + const { run } = compileGraph( + [ + block("1", BlockTypes.entrypoint), + block("2", "audit", { obs: "obs-42" }), + block("3", BlockTypes.response, { httpCode: "200" }), + ], + [edge("1", "2"), edge("2", "3")], + ); + + const asked: any[] = []; + const ctx = createContext(); + ctx.integrationFactory = { + create: (options: any) => { + asked.push(options); + return { logInfo() {}, logWarn() {}, logError() {} }; + }, + }; + await run(ctx, null); + + expect(asked).toEqual([{ integrationId: "obs-42", type: "observability" }]); + }); }); diff --git a/packages/blocks/compiler.ts b/packages/blocks/compiler.ts index 7d3cf058..9dffe59d 100644 --- a/packages/blocks/compiler.ts +++ b/packages/blocks/compiler.ts @@ -16,6 +16,7 @@ import { emitConsoleLog, runConsoleLog } from "./builtin/log/console"; import { emitCloudLogs, runCloudLog } from "./builtin/log/cloudLogs"; import { emitCustomBlock, + enqueueCustomBlock, hasCustomBlock, invokeCustomBlock, invokeCustomBlockAsync, @@ -124,6 +125,7 @@ export const compilerLib = { cloudLog: runCloudLog, invoke: invokeCustomBlock, invokeAsync: invokeCustomBlockAsync, + enqueue: enqueueCustomBlock, }; /** @@ -171,7 +173,11 @@ export function scopeFor(vars: Record, skip?: Set) { function makeScope(vars: Record, skip?: Set) { return new Proxy(vars, { - has: (target, key: any) => key !== "input" && !skip?.has(key), + // `input` and `params` are function parameters of the emitted JS wrapper; + // letting `with` resolve them off vars would shadow them with a variable + // that merely shares the name. + has: (target, key: any) => + key !== "input" && key !== "params" && !skip?.has(key), get: (target, key: any) => key === Symbol.unscopables ? undefined @@ -351,7 +357,10 @@ export function compileGraph( const method = sync ? "run" : "runAsync"; return `(await ctx.vm.${method}(${JSON.stringify(code)}${extras ? `, ${extras}` : ""}))`; } - return `(await (async function (input) { with ($scope) {\n${code}\n} })(${extras ?? "undefined"}))`; + // `params` is the custom block's invocation arguments — undefined in a + // route graph, so `input` keeps meaning exactly one thing everywhere: the + // previous block's output. + return `(await (async function (input, params) { with ($scope) {\n${code}\n} })(${extras ?? "undefined"}, $state.params))`; } /** @@ -530,10 +539,12 @@ throw $error; "vars,", `scope: lib.scope(vars${imports.scopeSkip}),`, "trace: ctx.trace,", - `params: ${asCustomBlock ? "input ?? {}" : "undefined"},`, + // a custom block is called with { params, input }: its configuration and + // the caller's flowing value, kept apart all the way down the graph + `params: ${asCustomBlock ? "input?.params ?? {}" : "undefined"},`, "};", "try {", - `return await ${blockFunctionName(entry.id)}($state, input, $endSuccess);`, + `return await ${blockFunctionName(entry.id)}($state, ${asCustomBlock ? "input?.input" : "input"}, $endSuccess);`, "} catch ($error) {", errorHandlerBody, "}", diff --git a/packages/blocks/index.ts b/packages/blocks/index.ts index 4895a873..ec8a0401 100644 --- a/packages/blocks/index.ts +++ b/packages/blocks/index.ts @@ -11,6 +11,7 @@ export * from "./builtin/loops/foreach"; export * from "./engine"; export * from "./compiler"; export * from "./builtin/customBlock"; +export * from "./jobs"; export * from "./blockTypes"; export * from "./baseBlock"; export * from "./categories"; diff --git a/packages/blocks/jobs.ts b/packages/blocks/jobs.ts new file mode 100644 index 00000000..abdc6b1a --- /dev/null +++ b/packages/blocks/jobs.ts @@ -0,0 +1,51 @@ +/** + * The seam between a graph and whatever queue the host runs. + * + * `packages/blocks` must not know about NATS — it runs inside the isolated + * execution process, which owns no connections. The host registers an enqueuer + * at startup and the generated code calls `lib.enqueue`, exactly the way + * `setBlocksExecutor` inverts route execution. + */ + +/** One unit of queued work. `kind` decides how a consumer reads `target`. */ +export type JobRequest = { + /** "custom-block" today; crons, workflows and schedules share this queue. */ + kind: string; + projectId: string; + /** What to run — a custom block's name, a route id, a workflow key. */ + target: string; + /** Must be JSON-serializable: it crosses a process and a broker. */ + payload?: unknown; + /** Where it was queued from, for correlation in traces and logs. */ + origin?: { + blockId?: string; + route?: string; + apiId?: string; + }; +}; + +export type JobEnqueuer = (job: JobRequest) => void; + +let enqueuer: JobEnqueuer | undefined; + +/** Called once by the host process. Pass nothing to detach (tests, shutdown). */ +export function setJobEnqueuer(next?: JobEnqueuer) { + enqueuer = next; +} + +export function jobQueueAvailable() { + return enqueuer !== undefined; +} + +/** + * Throws when no queue is wired. Dropping the job silently would be worse: the + * caller has already moved on believing the work is durable. + */ +export function enqueueJob(job: JobRequest) { + if (!enqueuer) { + throw new Error( + `No job queue is configured — cannot queue ${job.kind} "${job.target}"`, + ); + } + enqueuer(job); +} diff --git a/packages/components/src/IntegrationSelector/IntegrationSelector.tsx b/packages/components/src/IntegrationSelector/IntegrationSelector.tsx index 5dd7ba17..a77f1ba7 100644 --- a/packages/components/src/IntegrationSelector/IntegrationSelector.tsx +++ b/packages/components/src/IntegrationSelector/IntegrationSelector.tsx @@ -230,9 +230,21 @@ function PickerModal({ - - {integration.name} - +
+ + {integration.name} + {integration.external && ( + + From caller + + )} + + {integration.external && integration.hint && ( + + {integration.hint} + + )} +
@@ -280,6 +292,7 @@ function PickerModal({ export function IntegrationSelector({ selectedId, loadIntegrations, + injectedIntegrations, onSelect, onTestConnection, openInNewTabUrl, @@ -292,7 +305,12 @@ export function IntegrationSelector({ description, className, }: IntegrationSelectorProps) { - const [integrations, setIntegrations] = useState([]); + const [loaded, setLoaded] = useState([]); + // injected entries need no fetch and always come first + const integrations = useMemo( + () => [...(injectedIntegrations ?? []), ...loaded], + [injectedIntegrations, loaded], + ); const [loadStatus, setLoadStatus] = useState("idle"); const [loadError, setLoadError] = useState(null); @@ -316,7 +334,7 @@ export function IntegrationSelector({ setLoadError(null); try { const data = await loadIntegrations(); - setIntegrations(data); + setLoaded(data); setLoadStatus("success"); } catch (err) { const msg = @@ -443,8 +461,9 @@ export function IntegrationSelector({
{loadStatus === "success" && ( <> - {/* Test Connection */} - {selectedIntegration && onTestConnection && ( + {/* Test Connection — an external entry has nothing to test + here; the real integration lives on the caller's side */} + {selectedIntegration && !selectedIntegration.external && onTestConnection && (
+ + {selectedIntegration?.external && selectedIntegration.hint && ( +

+ {selectedIntegration.hint} +

+ )} {/* Picker Modal */} diff --git a/packages/components/src/IntegrationSelector/types.ts b/packages/components/src/IntegrationSelector/types.ts index 7bb19f34..fa3a4094 100644 --- a/packages/components/src/IntegrationSelector/types.ts +++ b/packages/components/src/IntegrationSelector/types.ts @@ -11,6 +11,15 @@ export interface Integration { variant: string; config: Record; tags?: string[]; + /** + * Not a real integration row: a placeholder whose concrete integration is + * chosen elsewhere (a custom block's input parameter, resolved by the caller). + * These sort first, carry a badge, and expose no test/open actions — there is + * nothing on this side to test or open. + */ + external?: boolean; + /** Explains an `external` entry in the picker and under the field. */ + hint?: string; } /** Internal async-load state. */ @@ -31,6 +40,11 @@ export interface IntegrationSelectorProps { * identity changes. */ loadIntegrations: () => Promise; + /** + * Entries injected ahead of the loaded ones without a fetch — used for + * `external` placeholders. Listed first in the picker. + */ + injectedIntegrations?: Integration[]; /** Called with the selected integration id, or empty string to clear. */ onSelect?: (id: string) => void; /** diff --git a/testing/e2e/README.md b/testing/e2e/README.md index 5fbb0d66..1b996a8b 100644 --- a/testing/e2e/README.md +++ b/testing/e2e/README.md @@ -36,6 +36,7 @@ A fixture declares its engine; Postgres is the default. |---|---|---| | `pg` | `postgres:bullseye` | `users`, `orders`, `auth_users` | | `mongo` | `mongo:7.0`, single-node replica set | `todos` | +| `none` | — | graphs that touch no database | Graphs are written **per engine**, not run across all of them. The adapters do not agree on what a result looks like — Mongo ids are hex strings off `_id`, @@ -61,6 +62,30 @@ route-level schema the portal stores, and it runs before any block does, so a to the tests: a login test posts one, and the storage test hashes one and compares it against the column read straight out of Postgres. +## Custom blocks + +`blocks/` holds custom blocks in the shape the portal saves them — a graph plus +the `inputParams` its callers configure. A route fixture names the ones it calls +in `uses`, and the harness registers them before compiling the route, which is +the same order the real compiler works in: a caller only emits once the library +knows the name. + +`blocks/jwt-ops.json` is the worked example. One block, two input params +(`operation`, `failOnInvalid`), and three routes under `graphs/custom-blocks/` +that configure it differently — signing, lenient verification, and strict +verification that ends on the block's *own* response block. Worth copying: + +- Read config as `params.` inside the block, at any depth. `input` is the + previous block's output, exactly as in a route. +- The callee's spans land in the caller's trace, so `executed` shows the inner + block ids inline. Assert on them — that is what distinguishes "the block ran" + from "the caller returned something". +- A response block inside a custom block ends *the block*, handing + `{ httpCode, body }` back to the caller. The caller still chooses the status. + +The secret is hardcoded in the block. It should come from app config, which +custom block params cannot reference yet. + ## Adding a graph 1. Drop a JSON file in `graphs/`, or in a subfolder for a multi-endpoint @@ -72,7 +97,8 @@ compares it against the column read straight out of Postgres. the field the compiler reads. An `if` whose two edges both say `"toHandle": "source"` fails to compile with a fan-out error. 2. Point any db block's `connection` at `"primary"` — the harness wires that id - to the right container for the fixture's engine. + to the right container for the fixture's engine. A graph with no db block + should set `"engine": "none"` so no container starts for it. 3. Add a test file in `tests/` with `beforeEach(() => resetDatabase(engine))` and assertions on `runGraph(fixture)`. diff --git a/testing/e2e/blocks/jwt-ops.json b/testing/e2e/blocks/jwt-ops.json new file mode 100644 index 00000000..107ec40d --- /dev/null +++ b/testing/e2e/blocks/jwt-ops.json @@ -0,0 +1,125 @@ +{ + "name": "jwt_ops", + "description": "Signs or verifies a JWT, chosen by the `operation` input param. With `failOnInvalid` set, a bad token ends the block on a 401 response instead of returning a result. The secret is baked in until custom blocks can read app config.", + "inputParams": [ + { + "name": "operation", + "label": "Operation", + "type": "dropdown", + "options": ["sign", "verify"], + "description": "Sign a new token from the incoming payload, or verify the one it carries." + }, + { + "name": "failOnInvalid", + "label": "Reject invalid tokens", + "type": "checkbox", + "description": "End the block with a 401 instead of returning { valid: false }." + } + ], + "blocks": [ + { + "id": "cb-entry", + "type": "entrypoint", + "position": { "x": 0, "y": 0 }, + "data": { "blockName": "Called" } + }, + { + "id": "cb-is-sign", + "type": "if", + "position": { "x": 240, "y": 0 }, + "data": { + "blockName": "Signing?", + "conditions": [ + { + "lhs": "", + "rhs": "", + "operator": "js", + "chain": "and", + "js": "return params.operation === \"sign\";" + } + ] + } + }, + { + "id": "cb-sign", + "type": "jsrunner", + "position": { "x": 480, "y": -120 }, + "data": { + "blockName": "Sign a token", + "value": "return {\n\ttoken: jwt.sign(input ?? {}, \"e2e-custom-block-secret\", {\n\t\texpiresIn: \"1h\",\n\t\tissuer: \"fluxify-e2e-block\",\n\t}),\n};" + } + }, + { + "id": "cb-verify", + "type": "jsrunner", + "position": { "x": 480, "y": 120 }, + "data": { + "blockName": "Verify the token", + "value": "const result = jwt.verify(input?.token ?? \"\", \"e2e-custom-block-secret\");\nreturn result.success\n\t? { valid: true, payload: result.payload }\n\t: { valid: false, error: \"invalid_token\" };" + } + }, + { + "id": "cb-guard", + "type": "if", + "position": { "x": 720, "y": 120 }, + "data": { + "blockName": "Let it through?", + "conditions": [ + { + "lhs": "", + "rhs": "", + "operator": "js", + "chain": "and", + "js": "return input.valid || !params.failOnInvalid;" + } + ] + } + }, + { + "id": "cb-reject", + "type": "response", + "position": { "x": 960, "y": 240 }, + "data": { + "blockName": "401 Unauthorized", + "httpCode": "401" + } + } + ], + "edges": [ + { + "id": "cb-e-entry-branch", + "from": "cb-entry", + "to": "cb-is-sign", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "cb-e-branch-sign", + "from": "cb-is-sign", + "to": "cb-sign", + "fromHandle": "success", + "toHandle": "success" + }, + { + "id": "cb-e-branch-verify", + "from": "cb-is-sign", + "to": "cb-verify", + "fromHandle": "failure", + "toHandle": "failure" + }, + { + "id": "cb-e-verify-guard", + "from": "cb-verify", + "to": "cb-guard", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "cb-e-guard-reject", + "from": "cb-guard", + "to": "cb-reject", + "fromHandle": "failure", + "toHandle": "failure" + } + ] +} diff --git a/testing/e2e/graphs/custom-blocks/jwt-sign.json b/testing/e2e/graphs/custom-blocks/jwt-sign.json new file mode 100644 index 00000000..a13b6a7e --- /dev/null +++ b/testing/e2e/graphs/custom-blocks/jwt-sign.json @@ -0,0 +1,73 @@ +{ + "name": "custom-blocks/jwt-sign", + "description": "POST /jwt/sign — builds a claims object and hands it to the jwt_ops custom block with operation=sign. The token comes back from the block's own graph.", + "engine": "none", + "route": { + "method": "POST", + "path": "/jwt/sign" + }, + "uses": ["jwt-ops"], + "schemas": { + "body": { + "dataType": "object", + "properties": [{ "key": "sub", "dataType": "str", "required": true }] + } + }, + "blocks": [ + { + "id": "entry", + "type": "entrypoint", + "position": { "x": 0, "y": 0 }, + "data": { "blockName": "Request" } + }, + { + "id": "claims", + "type": "jsrunner", + "position": { "x": 240, "y": 0 }, + "data": { + "blockName": "Build the claims", + "value": "return { sub: getRequestBody().sub, role: \"admin\" };" + } + }, + { + "id": "sign", + "type": "jwt_ops", + "position": { "x": 480, "y": 0 }, + "data": { + "blockName": "Sign it", + "invoke": "sync", + "operation": "sign", + "failOnInvalid": false + } + }, + { + "id": "reply", + "type": "response", + "position": { "x": 720, "y": 0 }, + "data": { "blockName": "200 OK", "httpCode": "200" } + } + ], + "edges": [ + { + "id": "e-entry-claims", + "from": "entry", + "to": "claims", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-claims-sign", + "from": "claims", + "to": "sign", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-sign-reply", + "from": "sign", + "to": "reply", + "fromHandle": "source", + "toHandle": "source" + } + ] +} diff --git a/testing/e2e/graphs/custom-blocks/jwt-verify-strict.json b/testing/e2e/graphs/custom-blocks/jwt-verify-strict.json new file mode 100644 index 00000000..c8619a34 --- /dev/null +++ b/testing/e2e/graphs/custom-blocks/jwt-verify-strict.json @@ -0,0 +1,126 @@ +{ + "name": "custom-blocks/jwt-verify-strict", + "description": "POST /jwt/verify-strict — the same custom block with failOnInvalid on, so a bad token ends the block on its own response block. The route branches on what came back and answers 401.", + "engine": "none", + "route": { + "method": "POST", + "path": "/jwt/verify-strict" + }, + "uses": ["jwt-ops"], + "schemas": { + "body": { + "dataType": "object", + "properties": [{ "key": "token", "dataType": "str", "required": true }] + } + }, + "blocks": [ + { + "id": "entry", + "type": "entrypoint", + "position": { "x": 0, "y": 0 }, + "data": { "blockName": "Request" } + }, + { + "id": "carry-token", + "type": "jsrunner", + "position": { "x": 240, "y": 0 }, + "data": { + "blockName": "Take the token", + "value": "return { token: getRequestBody().token };" + } + }, + { + "id": "verify", + "type": "jwt_ops", + "position": { "x": 480, "y": 0 }, + "data": { + "blockName": "Verify it", + "invoke": "sync", + "operation": "verify", + "failOnInvalid": true + } + }, + { + "id": "block-refused", + "type": "if", + "position": { "x": 720, "y": 0 }, + "data": { + "blockName": "Did the block refuse it?", + "conditions": [ + { + "lhs": "", + "rhs": "", + "operator": "js", + "chain": "and", + "js": "return input?.httpCode === \"401\";" + } + ] + } + }, + { + "id": "refusal-body", + "type": "jsrunner", + "position": { "x": 960, "y": 120 }, + "data": { + "blockName": "Unwrap the block's body", + "value": "return input.body;" + } + }, + { + "id": "rejected", + "type": "response", + "position": { "x": 1200, "y": 120 }, + "data": { "blockName": "401 Unauthorized", "httpCode": "401" } + }, + { + "id": "accepted", + "type": "response", + "position": { "x": 960, "y": -120 }, + "data": { "blockName": "200 OK", "httpCode": "200" } + } + ], + "edges": [ + { + "id": "e-entry-carry", + "from": "entry", + "to": "carry-token", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-carry-verify", + "from": "carry-token", + "to": "verify", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-verify-check", + "from": "verify", + "to": "block-refused", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-check-refusal", + "from": "block-refused", + "to": "refusal-body", + "fromHandle": "success", + "toHandle": "success" + }, + { + "id": "e-refusal-rejected", + "from": "refusal-body", + "to": "rejected", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-check-accepted", + "from": "block-refused", + "to": "accepted", + "fromHandle": "failure", + "toHandle": "failure" + } + ] +} diff --git a/testing/e2e/graphs/custom-blocks/jwt-verify.json b/testing/e2e/graphs/custom-blocks/jwt-verify.json new file mode 100644 index 00000000..99ff8680 --- /dev/null +++ b/testing/e2e/graphs/custom-blocks/jwt-verify.json @@ -0,0 +1,73 @@ +{ + "name": "custom-blocks/jwt-verify", + "description": "POST /jwt/verify — the same custom block configured with operation=verify. `failOnInvalid` is off, so a bad token returns { valid: false } and the route decides what to do with it.", + "engine": "none", + "route": { + "method": "POST", + "path": "/jwt/verify" + }, + "uses": ["jwt-ops"], + "schemas": { + "body": { + "dataType": "object", + "properties": [{ "key": "token", "dataType": "str", "required": true }] + } + }, + "blocks": [ + { + "id": "entry", + "type": "entrypoint", + "position": { "x": 0, "y": 0 }, + "data": { "blockName": "Request" } + }, + { + "id": "carry-token", + "type": "jsrunner", + "position": { "x": 240, "y": 0 }, + "data": { + "blockName": "Take the token", + "value": "return { token: getRequestBody().token };" + } + }, + { + "id": "verify", + "type": "jwt_ops", + "position": { "x": 480, "y": 0 }, + "data": { + "blockName": "Verify it", + "invoke": "sync", + "operation": "verify", + "failOnInvalid": false + } + }, + { + "id": "reply", + "type": "response", + "position": { "x": 720, "y": 0 }, + "data": { "blockName": "200 OK", "httpCode": "200" } + } + ], + "edges": [ + { + "id": "e-entry-carry", + "from": "entry", + "to": "carry-token", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-carry-verify", + "from": "carry-token", + "to": "verify", + "fromHandle": "source", + "toHandle": "source" + }, + { + "id": "e-verify-reply", + "from": "verify", + "to": "reply", + "fromHandle": "source", + "toHandle": "source" + } + ] +} diff --git a/testing/e2e/src/customBlocks.ts b/testing/e2e/src/customBlocks.ts new file mode 100644 index 00000000..572525ce --- /dev/null +++ b/testing/e2e/src/customBlocks.ts @@ -0,0 +1,22 @@ +import { registerCustomBlock, unregisterCustomBlock } from "@fluxify/blocks"; +import { loadCustomBlock, type GraphFixture } from "./graph"; + +/** + * Puts a fixture's custom blocks in the library before its route is compiled. + * + * This is the same two-step the real compiler does — compile the blocks, then + * the routes — because a route that calls a custom block only emits if the + * library already knows the name. Registration is worker-global, so the + * returned dispose keeps one fixture's blocks out of another's run. + */ +export async function registerFixtureBlocks(fixture: GraphFixture) { + const registered: string[] = []; + for (const file of fixture.uses ?? []) { + const block = await loadCustomBlock(file); + registerCustomBlock(block.name, block.blocks, block.edges); + registered.push(block.name); + } + return () => { + for (const name of registered) unregisterCustomBlock(name); + }; +} diff --git a/testing/e2e/src/engines.ts b/testing/e2e/src/engines.ts index 1e89c068..b119c9de 100644 --- a/testing/e2e/src/engines.ts +++ b/testing/e2e/src/engines.ts @@ -12,15 +12,19 @@ import { seedPostgres, seedMongo } from "./seed"; * weakened until it stopped proving much. Each engine gets graphs that exercise * what is actually distinctive about it. */ -export type Engine = "pg" | "mongo"; +/** `none` is for graphs that touch no database — no container starts for them. */ +export type Engine = "none" | "pg" | "mongo"; -export async function connectionFor(engine: Engine): Promise { +export async function connectionFor( + engine: Exclude, +): Promise { if (engine === "mongo") return (await mongo()).connection; return (await database()).connection; } /** Drops and re-seeds the engine's fixtures. Call from `beforeEach`. */ export async function resetDatabase(engine: Engine) { + if (engine === "none") return; if (engine === "mongo") return seedMongo((await mongo()).db); return seedPostgres((await database()).sql); } diff --git a/testing/e2e/src/graph.ts b/testing/e2e/src/graph.ts index 93501fc8..3fc08e8c 100644 --- a/testing/e2e/src/graph.ts +++ b/testing/e2e/src/graph.ts @@ -20,11 +20,31 @@ export type GraphFixture = { * validation block on the canvas. */ schemas?: { body?: unknown; query?: unknown; params?: unknown }; + /** + * Custom blocks this graph calls, by file name in `blocks/`. They are + * compiled and registered into the block library before the route is + * compiled — the same order the real compiler works in, and the reason a + * route calling one resolves at all. + */ + uses?: string[]; + blocks: BlockDTOType[]; + edges: EdgeDTOSchemaType; +}; + +/** + * A custom block as the portal saves it: a graph plus the input contract its + * callers configure. `name` is what a calling block carries as its type. + */ +export type CustomBlockFixture = { + name: string; + description: string; + inputParams?: unknown[]; blocks: BlockDTOType[]; edges: EdgeDTOSchemaType; }; const graphsDir = join(import.meta.dir, "..", "graphs"); +const blocksDir = join(import.meta.dir, "..", "blocks"); export async function loadGraph(name: string): Promise { const fixture = (await Bun.file( @@ -44,7 +64,22 @@ export async function loadGraph(name: string): Promise { * named by path (`auth/login`). */ export function graphNames(): string[] { - return readdirSync(graphsDir, { recursive: true }) + return jsonNames(graphsDir); +} + +/** Loads one custom block by file name — `blocks/jwt-ops.json` is `jwt-ops`. */ +export async function loadCustomBlock(file: string): Promise { + return (await Bun.file( + join(blocksDir, `${file}.json`), + ).json()) as CustomBlockFixture; +} + +export function customBlockFiles(): string[] { + return jsonNames(blocksDir); +} + +function jsonNames(dir: string): string[] { + return readdirSync(dir, { recursive: true }) .map((file) => String(file).replaceAll("\\", "/")) .filter((file) => file.endsWith(".json")) .map((file) => file.slice(0, -".json".length)); diff --git a/testing/e2e/src/runner.ts b/testing/e2e/src/runner.ts index 2c119366..a3656471 100644 --- a/testing/e2e/src/runner.ts +++ b/testing/e2e/src/runner.ts @@ -6,6 +6,7 @@ import { import { setBlocksExecutor } from "@fluxify/server/src/modules/requestRouter/executor"; import { executeRouteInternal } from "@fluxify/server/src/modules/requestRouter/service"; import { connectionFor } from "./engines"; +import { registerFixtureBlocks } from "./customBlocks"; import type { GraphFixture } from "./graph"; /** @@ -49,7 +50,10 @@ export type GraphRun = { /** Points the project's `primary` db integration at the fixture's engine. */ async function hydrateDatabase(fixture: GraphFixture) { - const connection = await connectionFor(fixture.engine ?? "pg"); + const engine = fixture.engine ?? "pg"; + // a graph that touches no database should not start a container for it + if (engine === "none") return; + const connection = await connectionFor(engine); hydrateIntegrations(PROJECT_ID, { db: { [DB_CONNECTION]: { ...connection, [OWNER_KEY]: PROJECT_ID } }, }); @@ -60,7 +64,19 @@ export async function runGraph( request: GraphRequest = {}, ): Promise { await hydrateDatabase(fixture); + const disposeBlocks = await registerFixtureBlocks(fixture); + try { + return await execute(fixture, request); + } finally { + disposeBlocks(); + } +} + +async function execute( + fixture: GraphFixture, + request: GraphRequest, +): Promise { const { run, source } = compileGraph(fixture.blocks, fixture.edges); const spans: BlockTraceSpan[] = []; diff --git a/testing/e2e/tests/fixtures.test.ts b/testing/e2e/tests/fixtures.test.ts index 17bf7348..8b8dd98c 100644 --- a/testing/e2e/tests/fixtures.test.ts +++ b/testing/e2e/tests/fixtures.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from "bun:test"; import { compileGraph } from "@fluxify/blocks"; -import { graphNames, loadGraph } from "../src/graph"; +import { registerFixtureBlocks } from "../src/customBlocks"; +import { + customBlockFiles, + graphNames, + loadCustomBlock, + loadGraph, +} from "../src/graph"; /** * Guards the fixtures themselves. A graph that no longer compiles is a broken @@ -16,7 +22,25 @@ describe("graph fixtures", () => { it(`${name} compiles`, async () => { const fixture = await loadGraph(name); expect(fixture.route.path.startsWith("/")).toBe(true); - expect(compileGraph(fixture.blocks, fixture.edges).source).toBeString(); + // a caller only emits once its custom blocks are in the library + const dispose = await registerFixtureBlocks(fixture); + try { + expect(compileGraph(fixture.blocks, fixture.edges).source).toBeString(); + } finally { + dispose(); + } + }); + } +}); + +describe("custom block fixtures", () => { + for (const file of customBlockFiles()) { + it(`${file} compiles as a custom block`, async () => { + const block = await loadCustomBlock(file); + const { source } = compileGraph(block.blocks, block.edges, { + asCustomBlock: true, + }); + expect(source).toBeString(); }); } }); diff --git a/testing/e2e/tests/jwtCustomBlock.test.ts b/testing/e2e/tests/jwtCustomBlock.test.ts new file mode 100644 index 00000000..dcc807bc --- /dev/null +++ b/testing/e2e/tests/jwtCustomBlock.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "bun:test"; +import { loadGraph } from "../src/graph"; +import { runGraph } from "../src/runner"; + +/** + * One custom block, three routes, three configurations. What this covers that a + * block unit spec cannot: the params a caller configures actually reach the + * callee's graph, the callee's own branching runs, and a response block inside a + * custom block ends the block rather than the request. + * + * No database — the graphs are pure JS, so the fixtures declare `engine: none` + * and no container starts. + */ + +const sign = await loadGraph("custom-blocks/jwt-sign"); +const verify = await loadGraph("custom-blocks/jwt-verify"); +const strict = await loadGraph("custom-blocks/jwt-verify-strict"); + +async function issueToken(sub = "user-42") { + const run = await runGraph(sign, { body: { sub } }); + return run.body.token as string; +} + +describe("jwt_ops custom block", () => { + it("signs a token through the block's sign branch", async () => { + const run = await runGraph(sign, { body: { sub: "user-42" } }); + + expect(run.status).toBe(200); + expect(run.body.token).toBeString(); + // three segments: the block really produced a JWT, not a stringified object + expect(run.body.token.split(".")).toHaveLength(3); + }); + + it("runs the callee's blocks inside the caller's trace", async () => { + const run = await runGraph(sign, { body: { sub: "user-42" } }); + + // the custom block's own ids appear between the caller's, which is the + // only way to tell the block ran from the caller merely returning + expect(run.executed).toEqual([ + "entry", + "claims", + "cb-entry", + "cb-is-sign", + "cb-sign", + "sign", + "reply", + ]); + expect(run.spans.every((span) => span.outcome === "success")).toBe(true); + }); + + it("verifies a token it signed, taking the other branch of the same block", async () => { + const token = await issueToken("user-42"); + const run = await runGraph(verify, { body: { token } }); + + expect(run.status).toBe(200); + expect(run.body.valid).toBe(true); + expect(run.body.payload).toMatchObject({ sub: "user-42", role: "admin" }); + // the `operation` param picked the verify path, not the sign path + expect(run.executed).toContain("cb-verify"); + expect(run.executed).not.toContain("cb-sign"); + }); + + it("returns valid:false for a bad token when the block is not strict", async () => { + const run = await runGraph(verify, { body: { token: "not.a.token" } }); + + expect(run.status).toBe(200); + expect(run.body).toEqual({ valid: false, error: "invalid_token" }); + // the guard let it through: no response block ran inside the callee + expect(run.executed).not.toContain("cb-reject"); + }); + + it("ends on the block's own response block when failOnInvalid is set", async () => { + const run = await runGraph(strict, { body: { token: "not.a.token" } }); + + expect(run.status).toBe(401); + expect(run.body).toEqual({ valid: false, error: "invalid_token" }); + expect(run.executed).toContain("cb-reject"); + // the callee's response ended the callee, not the request — the caller + // still got to branch on it + expect(run.executed).toContain("block-refused"); + }); + + it("passes a good token straight through the strict route", async () => { + const token = await issueToken("user-7"); + const run = await runGraph(strict, { body: { token } }); + + expect(run.status).toBe(200); + expect(run.body.payload).toMatchObject({ sub: "user-7" }); + expect(run.executed).not.toContain("cb-reject"); + }); + + it("rejects a token signed with a different secret", async () => { + // the secret lives inside the block; nothing the caller passes can change it + const foreign = [ + Buffer.from(JSON.stringify({ alg: "HS256", typ: "JWT" })).toString( + "base64url", + ), + Buffer.from(JSON.stringify({ sub: "intruder" })).toString("base64url"), + "forged", + ].join("."); + + const run = await runGraph(strict, { body: { token: foreign } }); + expect(run.status).toBe(401); + }); + + it("validates the request before any block runs", async () => { + const run = await runGraph(verify, { body: {} }); + + expect(run.status).toBe(400); + expect(run.executed).toEqual([]); + }); + + it("compiles the caller to an invoke rather than inlining the block", async () => { + const run = await runGraph(sign, { body: { sub: "user-42" } }); + + expect(run.source).toContain('lib.invoke(ctx, "jwt_ops"'); + // the callee's body is compiled once, into the library — not here + expect(run.source).not.toContain("e2e-custom-block-secret"); + }); +});