diff --git a/README.md b/README.md index 6dd549e7f..646e0a34e 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ Visit [http://localhost:3000](http://localhost:3000) to get started. - **Firecrawl**: Scrape URL, Search Web - **GitHub**: Create Issue, List Issues, Get Issue, Update Issue - **Linear**: Create Ticket, Find Issues +- **Olostep**: Scrape URL, Search Web, Map Website, AI Answer - **Perplexity**: Search Web, Ask Question, Research Topic - **Resend**: Send Email - **Slack**: Send Slack Message diff --git a/plugins/index.ts b/plugins/index.ts index 495c6e339..a16c0cc7d 100644 --- a/plugins/index.ts +++ b/plugins/index.ts @@ -12,6 +12,8 @@ * To remove an integration: * 1. Delete the plugin directory * 2. Run: pnpm discover-plugins (or it runs automatically on build) + * + * Discovered plugins: ai-gateway, firecrawl, linear, olostep, perplexity, resend, slack, v0 */ import "./ai-gateway"; @@ -21,6 +23,7 @@ import "./fal"; import "./firecrawl"; import "./github"; import "./linear"; +import "./olostep"; import "./perplexity"; import "./resend"; import "./slack"; @@ -28,7 +31,36 @@ import "./stripe"; import "./superagent"; import "./v0"; import "./webflow"; + ActionConfigField, + ActionConfigFieldBase, + ActionConfigFieldGroup, + ActionWithFullId, + IntegrationPlugin, + PluginAction, +} from "./registry"; +// Export the registry utilities +export { + computeActionId, + findActionById, + flattenConfigFields, + generateAIActionPrompts, + getActionsByCategory, + getAllActions, + getAllDependencies, + getAllEnvVars, + getAllIntegrations, + getCredentialMapping, + getDependenciesForActions, + getIntegration, + getIntegrationLabels, + getIntegrationTypes, + getPluginEnvVars, + getSortedIntegrationTypes, + isFieldGroup, + parseActionId, + registerIntegration, +} from "./registry"; export type { ActionConfigField, ActionConfigFieldBase, diff --git a/plugins/olostep/codegen/answer.ts b/plugins/olostep/codegen/answer.ts new file mode 100644 index 000000000..708c13118 --- /dev/null +++ b/plugins/olostep/codegen/answer.ts @@ -0,0 +1,50 @@ +/** + * Code generation template for Olostep Answer action + * This template is used when exporting workflows to standalone Next.js projects + * It uses environment variables instead of integrationId + */ +export const answerCodegenTemplate = `export async function olostepAnswerStep(input: { + question: string; + urls?: string[]; + searchQuery?: string; +}) { + "use step"; + + const requestBody: Record = { + question: input.question, + }; + + if (input.urls && input.urls.length > 0) { + requestBody.urls = input.urls; + } + + if (input.searchQuery) { + requestBody.search_query = input.searchQuery; + } + + const response = await fetch('https://api.olostep.com/v1/answer', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': \`Bearer \${process.env.OLOSTEP_API_KEY}\`, + }, + body: JSON.stringify(requestBody), + }); + + if (!response.ok) { + throw new Error(\`Olostep API error: \${await response.text()}\`); + } + + const result = await response.json(); + + return { + answer: result.answer || result.response || '', + sources: (result.sources || result.references || []).map((source: any) => ({ + url: source.url || source.link, + title: source.title, + })), + }; +}`; + + + diff --git a/plugins/olostep/codegen/map.ts b/plugins/olostep/codegen/map.ts new file mode 100644 index 000000000..757264193 --- /dev/null +++ b/plugins/olostep/codegen/map.ts @@ -0,0 +1,40 @@ +/** + * Code generation template for Olostep Map action + * This template is used when exporting workflows to standalone Next.js projects + * It uses environment variables instead of integrationId + */ +export const mapCodegenTemplate = `export async function olostepMapStep(input: { + url: string; + limit?: number; + includeSubdomains?: boolean; +}) { + "use step"; + + const response = await fetch('https://api.olostep.com/v1/map', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': \`Bearer \${process.env.OLOSTEP_API_KEY}\`, + }, + body: JSON.stringify({ + url: input.url, + limit: input.limit || 100, + include_subdomains: input.includeSubdomains || false, + }), + }); + + if (!response.ok) { + throw new Error(\`Olostep API error: \${await response.text()}\`); + } + + const result = await response.json(); + const urls = result.urls || result.links || []; + + return { + urls: urls.slice(0, input.limit || 100), + totalUrls: urls.length, + }; +}`; + + + diff --git a/plugins/olostep/codegen/scrape.ts b/plugins/olostep/codegen/scrape.ts new file mode 100644 index 000000000..8a33ffa1d --- /dev/null +++ b/plugins/olostep/codegen/scrape.ts @@ -0,0 +1,46 @@ +/** + * Code generation template for Olostep Scrape action + * This template is used when exporting workflows to standalone Next.js projects + * It uses environment variables instead of integrationId + */ +export const scrapeCodegenTemplate = `export async function olostepScrapeStep(input: { + url: string; + formats?: ('markdown' | 'html' | 'text')[]; + waitForSelector?: string; +}) { + "use step"; + + const response = await fetch('https://api.olostep.com/v1/scrapes', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': \`Bearer \${process.env.OLOSTEP_API_KEY}\`, + }, + body: JSON.stringify({ + url_to_scrape: input.url, + formats: input.formats || ['markdown'], + wait_for_selector: input.waitForSelector, + }), + }); + + if (!response.ok) { + throw new Error(\`Olostep API error: \${await response.text()}\`); + } + + const result = await response.json(); + + return { + markdown: result.markdown_content || result.markdown, + html: result.html_content || result.html, + metadata: { + title: result.title, + url: result.url, + statusCode: result.status_code, + }, + }; +}`; + + + + + diff --git a/plugins/olostep/codegen/search.ts b/plugins/olostep/codegen/search.ts new file mode 100644 index 000000000..875ce2f7e --- /dev/null +++ b/plugins/olostep/codegen/search.ts @@ -0,0 +1,52 @@ +/** + * Code generation template for Olostep Search action + * This template is used when exporting workflows to standalone Next.js projects + * It uses environment variables instead of integrationId + */ +export const searchCodegenTemplate = `export async function olostepSearchStep(input: { + query: string; + limit?: number; + country?: string; +}) { + "use step"; + + const params = new URLSearchParams({ + query: input.query, + limit: String(input.limit || 10), + }); + + if (input.country) { + params.append('country', input.country); + } + + const response = await fetch( + \`https://api.olostep.com/v1/google-search?\${params.toString()}\`, + { + method: 'GET', + headers: { + 'Authorization': \`Bearer \${process.env.OLOSTEP_API_KEY}\`, + }, + } + ); + + if (!response.ok) { + throw new Error(\`Olostep API error: \${await response.text()}\`); + } + + const result = await response.json(); + + return { + results: (result.results || result.items || []).slice(0, input.limit || 10).map((item: any) => ({ + url: item.url || item.link, + title: item.title, + description: item.description || item.snippet, + markdown: item.markdown, + })), + totalResults: result.total_results, + }; +}`; + + + + + diff --git a/plugins/olostep/icon.tsx b/plugins/olostep/icon.tsx new file mode 100644 index 000000000..b77bca016 --- /dev/null +++ b/plugins/olostep/icon.tsx @@ -0,0 +1,24 @@ +export function OlostepIcon({ className }: { className?: string }) { + return ( + + Olostep + + + O + + + ); +} diff --git a/plugins/olostep/index.ts b/plugins/olostep/index.ts new file mode 100644 index 000000000..2653403b4 --- /dev/null +++ b/plugins/olostep/index.ts @@ -0,0 +1,161 @@ +import type { IntegrationPlugin } from "../registry"; +import { registerIntegration } from "../registry"; +import { answerCodegenTemplate } from "./codegen/answer"; +import { mapCodegenTemplate } from "./codegen/map"; +import { scrapeCodegenTemplate } from "./codegen/scrape"; +import { searchCodegenTemplate } from "./codegen/search"; +import { OlostepIcon } from "./icon"; + +const olostepPlugin: IntegrationPlugin = { + type: "olostep", + label: "Olostep", + description: "Web Data API for AI - Search, extract, and structure web data", + + icon: OlostepIcon, + + formFields: [ + { + id: "olostepApiKey", + label: "API Key", + type: "password", + placeholder: "ols_...", + configKey: "olostepApiKey", + envVar: "OLOSTEP_API_KEY", + helpText: "Get your API key from ", + helpLink: { + text: "olostep.com", + url: "https://olostep.com/dashboard", + }, + }, + ], + + testConfig: { + getTestFunction: async () => { + const { testOlostep } = await import("./test"); + return testOlostep; + }, + }, + + dependencies: {}, + + actions: [ + { + slug: "scrape", + label: "Scrape URL", + description: "Extract content from any URL with full JavaScript rendering", + category: "Olostep", + stepFunction: "olostepScrapeStep", + stepImportPath: "scrape", + configFields: [ + { + key: "url", + label: "URL", + type: "template-input", + placeholder: "https://example.com or {{NodeName.url}}", + example: "https://example.com", + required: true, + }, + { + key: "waitForSelector", + label: "Wait for Selector", + type: "text", + placeholder: "CSS selector to wait for (optional)", + example: ".main-content", + }, + ], + codegenTemplate: scrapeCodegenTemplate, + }, + { + slug: "search", + label: "Search Web", + description: "Search the web using Google Search via Olostep", + category: "Olostep", + stepFunction: "olostepSearchStep", + stepImportPath: "search", + configFields: [ + { + key: "query", + label: "Search Query", + type: "template-input", + placeholder: "Search query or {{NodeName.query}}", + example: "latest AI news", + required: true, + }, + { + key: "limit", + label: "Result Limit", + type: "number", + placeholder: "10", + min: 1, + example: "10", + }, + { + key: "country", + label: "Country", + type: "text", + placeholder: "Country code (e.g., us, uk)", + example: "us", + }, + ], + codegenTemplate: searchCodegenTemplate, + }, + { + slug: "map", + label: "Map Website", + description: "Discover all URLs from a website (sitemap discovery)", + category: "Olostep", + stepFunction: "olostepMapStep", + stepImportPath: "map", + configFields: [ + { + key: "url", + label: "Website URL", + type: "template-input", + placeholder: "https://example.com or {{NodeName.url}}", + example: "https://example.com", + required: true, + }, + { + key: "limit", + label: "Max URLs", + type: "number", + placeholder: "100", + min: 1, + example: "100", + }, + ], + codegenTemplate: mapCodegenTemplate, + }, + { + slug: "answer", + label: "AI Answer", + description: "Get AI-powered answers from web content", + category: "Olostep", + stepFunction: "olostepAnswerStep", + stepImportPath: "answer", + configFields: [ + { + key: "question", + label: "Question", + type: "template-input", + placeholder: "What is the latest news about...?", + example: "What are the key features of this product?", + required: true, + }, + { + key: "searchQuery", + label: "Search Query (optional)", + type: "template-input", + placeholder: "Search the web first for context", + example: "product features comparison 2024", + }, + ], + codegenTemplate: answerCodegenTemplate, + }, + ], +}; + +// Auto-register on import +registerIntegration(olostepPlugin); + +export default olostepPlugin; diff --git a/plugins/olostep/steps/answer.ts b/plugins/olostep/steps/answer.ts new file mode 100644 index 000000000..21e564169 --- /dev/null +++ b/plugins/olostep/steps/answer.ts @@ -0,0 +1,95 @@ +import "server-only"; + +import { fetchCredentials } from "@/lib/credential-fetcher"; +import { type StepInput, withStepLogging } from "@/lib/steps/step-handler"; +import { getErrorMessage } from "@/lib/utils"; + +type AnswerResult = { + answer: string; + sources: Array<{ + url: string; + title?: string; + }>; +}; + +export type OlostepAnswerInput = StepInput & { + integrationId?: string; + question: string; + urls?: string[]; + searchQuery?: string; +}; + +/** + * Answer logic using Olostep API + * Get AI-powered answers from web content + */ +async function getAnswer(input: OlostepAnswerInput): Promise { + const credentials = input.integrationId + ? await fetchCredentials(input.integrationId) + : {}; + + const apiKey = credentials.OLOSTEP_API_KEY; + + if (!apiKey) { + throw new Error("Olostep API Key is not configured."); + } + + try { + const requestBody: Record = { + question: input.question, + }; + + // If URLs are provided, use them as context + if (input.urls && input.urls.length > 0) { + requestBody.urls = input.urls; + } + + // If search query is provided, search first + if (input.searchQuery) { + requestBody.search_query = input.searchQuery; + } + + const response = await fetch("https://api.olostep.com/v1/answer", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify(requestBody), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Olostep API error: ${errorText}`); + } + + const result = await response.json(); + + return { + answer: result.answer || result.response || "", + sources: (result.sources || result.references || []).map( + (source: { url?: string; link?: string; title?: string }) => ({ + url: source.url || source.link, + title: source.title, + }) + ), + }; + } catch (error) { + throw new Error(`Failed to get answer: ${getErrorMessage(error)}`); + } +} + +/** + * Olostep Answer Step + * Gets AI-powered answers from web content using Olostep + */ +export async function olostepAnswerStep( + input: OlostepAnswerInput +): Promise { + "use step"; + return withStepLogging(input, () => getAnswer(input)); +} +olostepAnswerStep.maxRetries = 0; + + + diff --git a/plugins/olostep/steps/map.ts b/plugins/olostep/steps/map.ts new file mode 100644 index 000000000..995b06dc1 --- /dev/null +++ b/plugins/olostep/steps/map.ts @@ -0,0 +1,79 @@ +import "server-only"; + +import { fetchCredentials } from "@/lib/credential-fetcher"; +import { type StepInput, withStepLogging } from "@/lib/steps/step-handler"; +import { getErrorMessage } from "@/lib/utils"; + +type MapResult = { + urls: string[]; + totalUrls: number; +}; + +export type OlostepMapInput = StepInput & { + integrationId?: string; + url: string; + limit?: number; + includeSubdomains?: boolean; +}; + +/** + * Map logic using Olostep API + * Gets all URLs from a website (sitemap discovery) + */ +async function mapUrls(input: OlostepMapInput): Promise { + const credentials = input.integrationId + ? await fetchCredentials(input.integrationId) + : {}; + + const apiKey = credentials.OLOSTEP_API_KEY; + + if (!apiKey) { + throw new Error("Olostep API Key is not configured."); + } + + try { + const response = await fetch("https://api.olostep.com/v1/map", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + url: input.url, + limit: input.limit || 100, + include_subdomains: input.includeSubdomains || false, + }), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Olostep API error: ${errorText}`); + } + + const result = await response.json(); + + const urls = result.urls || result.links || []; + + return { + urls: urls.slice(0, input.limit || 100), + totalUrls: urls.length, + }; + } catch (error) { + throw new Error(`Failed to map URLs: ${getErrorMessage(error)}`); + } +} + +/** + * Olostep Map Step + * Discovers all URLs from a website using Olostep + */ +export async function olostepMapStep( + input: OlostepMapInput +): Promise { + "use step"; + return withStepLogging(input, () => mapUrls(input)); +} +olostepMapStep.maxRetries = 0; + + + diff --git a/plugins/olostep/steps/scrape.ts b/plugins/olostep/steps/scrape.ts new file mode 100644 index 000000000..427498964 --- /dev/null +++ b/plugins/olostep/steps/scrape.ts @@ -0,0 +1,84 @@ +import "server-only"; + +import { fetchCredentials } from "@/lib/credential-fetcher"; +import { type StepInput, withStepLogging } from "@/lib/steps/step-handler"; +import { getErrorMessage } from "@/lib/utils"; + +type ScrapeResult = { + markdown?: string; + html?: string; + metadata?: Record; +}; + +export type OlostepScrapeInput = StepInput & { + integrationId?: string; + url: string; + formats?: ("markdown" | "html" | "text")[]; + waitForSelector?: string; +}; + +/** + * Scrape logic using Olostep API + */ +async function scrape(input: OlostepScrapeInput): Promise { + const credentials = input.integrationId + ? await fetchCredentials(input.integrationId) + : {}; + + const apiKey = credentials.OLOSTEP_API_KEY; + + if (!apiKey) { + throw new Error("Olostep API Key is not configured."); + } + + try { + const response = await fetch("https://api.olostep.com/v1/scrapes", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + url_to_scrape: input.url, + formats: input.formats || ["markdown"], + wait_for_selector: input.waitForSelector, + }), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Olostep API error: ${errorText}`); + } + + const result = await response.json(); + + return { + markdown: result.markdown_content || result.markdown, + html: result.html_content || result.html, + metadata: { + title: result.title, + url: result.url, + statusCode: result.status_code, + }, + }; + } catch (error) { + throw new Error(`Failed to scrape: ${getErrorMessage(error)}`); + } +} + +/** + * Olostep Scrape Step + * Scrapes content from a URL using Olostep + */ +export async function olostepScrapeStep( + input: OlostepScrapeInput +): Promise { + "use step"; + return withStepLogging(input, () => scrape(input)); +} +olostepScrapeStep.maxRetries = 0; + + + + + diff --git a/plugins/olostep/steps/search.ts b/plugins/olostep/steps/search.ts new file mode 100644 index 000000000..035fcbbac --- /dev/null +++ b/plugins/olostep/steps/search.ts @@ -0,0 +1,113 @@ +import "server-only"; + +import { fetchCredentials } from "@/lib/credential-fetcher"; +import { type StepInput, withStepLogging } from "@/lib/steps/step-handler"; +import { getErrorMessage } from "@/lib/utils"; + +type SearchResultItem = { + url: string; + title?: string; + description?: string; + markdown?: string; +}; + +type SearchResult = { + results: SearchResultItem[]; + totalResults?: number; +}; + +export type OlostepSearchInput = StepInput & { + integrationId?: string; + query: string; + limit?: number; + country?: string; +}; + +/** + * Search logic using Olostep Google Search API + */ +async function search(input: OlostepSearchInput): Promise { + const credentials = input.integrationId + ? await fetchCredentials(input.integrationId) + : {}; + + const apiKey = credentials.OLOSTEP_API_KEY; + + if (!apiKey) { + throw new Error("Olostep API Key is not configured."); + } + + try { + // Use the map endpoint with google search for web search functionality + const params = new URLSearchParams({ + query: input.query, + limit: String(input.limit || 10), + }); + + if (input.country) { + params.append("country", input.country); + } + + const response = await fetch( + `https://api.olostep.com/v1/google-search?${params.toString()}`, + { + method: "GET", + headers: { + Authorization: `Bearer ${apiKey}`, + }, + } + ); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Olostep API error: ${errorText}`); + } + + const result = await response.json(); + + // Transform the response to a consistent format + // Filter before slicing to ensure we return the requested number of valid results + const results: SearchResultItem[] = (result.results || result.items || []) + .map( + (item: { + url?: string; + link?: string; + title?: string; + description?: string; + snippet?: string; + markdown?: string; + }) => ({ + url: item.url || item.link, + title: item.title, + description: item.description || item.snippet, + markdown: item.markdown, + }) + ) + .filter((item: SearchResultItem) => item.url) + .slice(0, input.limit || 10); + + return { + results, + totalResults: result.total_results || results.length, + }; + } catch (error) { + throw new Error(`Failed to search: ${getErrorMessage(error)}`); + } +} + +/** + * Olostep Search Step + * Searches the web using Olostep + */ +export async function olostepSearchStep( + input: OlostepSearchInput +): Promise { + "use step"; + return withStepLogging(input, () => search(input)); +} +olostepSearchStep.maxRetries = 0; + + + + + diff --git a/plugins/olostep/test.ts b/plugins/olostep/test.ts new file mode 100644 index 000000000..05497f897 --- /dev/null +++ b/plugins/olostep/test.ts @@ -0,0 +1,40 @@ +export async function testOlostep(credentials: Record) { + try { + const apiKey = credentials.OLOSTEP_API_KEY; + + if (!apiKey) { + return { + success: false, + error: "OLOSTEP_API_KEY is required", + }; + } + + const response = await fetch("https://api.olostep.com/v1/scrapes", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + url_to_scrape: "https://example.com", + formats: ["markdown"], + }), + }); + + if (response.ok) { + return { success: true }; + } + const error = await response.text(); + return { success: false, error }; + } catch (error) { + return { + success: false, + error: error instanceof Error ? error.message : String(error), + }; + } +} + + + + +