From cb2eb000779c5d5acf89bfc44427f9960d25a51a Mon Sep 17 00:00:00 2001 From: meirk-brd Date: Tue, 13 Jan 2026 18:06:30 +0200 Subject: [PATCH 1/4] add bright data block and tools --- .../app/api/tools/brightdata/dataset/route.ts | 167 ++++++++++++++++++ .../tools/brightdata/scrape-markdown/route.ts | 80 +++++++++ .../tools/brightdata/search-engine/route.ts | 98 ++++++++++ apps/sim/blocks/blocks/brightdata.ts | 119 +++++++++++++ apps/sim/blocks/registry.ts | 2 + apps/sim/components/icons.tsx | 61 +++++++ .../brightdata/dataset_amazon_product.ts | 65 +++++++ apps/sim/tools/brightdata/index.ts | 4 + apps/sim/tools/brightdata/scrape_markdown.ts | 75 ++++++++ apps/sim/tools/brightdata/search_engine.ts | 73 ++++++++ apps/sim/tools/brightdata/types.ts | 107 +++++++++++ apps/sim/tools/registry.ts | 8 + 12 files changed, 859 insertions(+) create mode 100644 apps/sim/app/api/tools/brightdata/dataset/route.ts create mode 100644 apps/sim/app/api/tools/brightdata/scrape-markdown/route.ts create mode 100644 apps/sim/app/api/tools/brightdata/search-engine/route.ts create mode 100644 apps/sim/blocks/blocks/brightdata.ts create mode 100644 apps/sim/tools/brightdata/dataset_amazon_product.ts create mode 100644 apps/sim/tools/brightdata/index.ts create mode 100644 apps/sim/tools/brightdata/scrape_markdown.ts create mode 100644 apps/sim/tools/brightdata/search_engine.ts create mode 100644 apps/sim/tools/brightdata/types.ts diff --git a/apps/sim/app/api/tools/brightdata/dataset/route.ts b/apps/sim/app/api/tools/brightdata/dataset/route.ts new file mode 100644 index 0000000000..7e0c0a0c1a --- /dev/null +++ b/apps/sim/app/api/tools/brightdata/dataset/route.ts @@ -0,0 +1,167 @@ +import { randomUUID } from 'crypto' +import { createLogger } from '@sim/logger' +import { NextResponse } from 'next/server' + +const logger = createLogger('BrightDataDatasetAPI') + +export const maxDuration = 600 + +export async function POST(request: Request) { + const requestId = randomUUID().slice(0, 8) + + try { + const body = await request.json() + const datasetId = typeof body?.datasetId === 'string' ? body.datasetId : undefined + const apiToken = typeof body?.apiToken === 'string' ? body.apiToken : undefined + + if (!datasetId || !apiToken) { + return NextResponse.json({ error: 'Missing required parameters' }, { status: 400 }) + } + + const params: Record = { ...body } + delete params.datasetId + delete params.apiToken + + logger.info(`[${requestId}] Triggering dataset`, { datasetId }) + + const triggerResponse = await fetch( + `https://api.brightdata.com/datasets/v3/trigger?dataset_id=${encodeURIComponent( + datasetId + )}&include_errors=true`, + { + method: 'POST', + headers: { + Authorization: `Bearer ${apiToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify([params]), + } + ) + + const triggerText = await triggerResponse.text() + let triggerPayload: unknown = triggerText + + try { + triggerPayload = JSON.parse(triggerText) + } catch { + triggerPayload = triggerText + } + + if (!triggerResponse.ok) { + const errorMessage = + typeof triggerPayload === 'object' && triggerPayload !== null && 'error' in triggerPayload + ? String((triggerPayload as { error?: unknown }).error) + : triggerResponse.statusText + + logger.error(`[${requestId}] Dataset trigger failed`, { + datasetId, + status: triggerResponse.status, + error: errorMessage, + }) + + return NextResponse.json( + { error: errorMessage || 'Dataset trigger failed' }, + { status: triggerResponse.status } + ) + } + + const snapshotId = + typeof triggerPayload === 'object' && triggerPayload !== null && 'snapshot_id' in triggerPayload + ? String((triggerPayload as { snapshot_id?: unknown }).snapshot_id ?? '') + : '' + + if (!snapshotId) { + logger.error(`[${requestId}] Dataset trigger missing snapshot ID`, { datasetId }) + return NextResponse.json({ error: 'No snapshot ID returned from request' }, { status: 500 }) + } + + logger.info(`[${requestId}] Dataset triggered`, { datasetId, snapshotId }) + + const maxAttempts = 600 + let attempts = 0 + + while (attempts < maxAttempts) { + const snapshotResponse = await fetch( + `https://api.brightdata.com/datasets/v3/snapshot/${snapshotId}?format=json`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${apiToken}`, + 'Content-Type': 'application/json', + }, + } + ) + + const snapshotText = await snapshotResponse.text() + let snapshotPayload: unknown = snapshotText + + try { + snapshotPayload = JSON.parse(snapshotText) + } catch { + snapshotPayload = snapshotText + } + + if (!snapshotResponse.ok) { + if (snapshotResponse.status === 400) { + const errorMessage = + typeof snapshotPayload === 'object' && + snapshotPayload !== null && + 'error' in snapshotPayload + ? String((snapshotPayload as { error?: unknown }).error) + : snapshotResponse.statusText + + logger.error(`[${requestId}] Dataset snapshot fetch failed`, { + datasetId, + snapshotId, + status: snapshotResponse.status, + error: errorMessage, + }) + + return NextResponse.json( + { error: errorMessage || 'Dataset snapshot fetch failed' }, + { status: snapshotResponse.status } + ) + } + + attempts += 1 + await new Promise((resolve) => setTimeout(resolve, 1000)) + continue + } + + const status = + typeof snapshotPayload === 'object' && snapshotPayload !== null && 'status' in snapshotPayload + ? String((snapshotPayload as { status?: unknown }).status ?? '') + : '' + + if (['running', 'building', 'starting'].includes(status)) { + attempts += 1 + await new Promise((resolve) => setTimeout(resolve, 1000)) + continue + } + + const snapshotAt = + typeof snapshotPayload === 'object' && + snapshotPayload !== null && + 'snapshot_at' in snapshotPayload + ? String((snapshotPayload as { snapshot_at?: unknown }).snapshot_at ?? '') + : undefined + + logger.info(`[${requestId}] Dataset snapshot received`, { datasetId, snapshotId }) + + return NextResponse.json({ + data: snapshotPayload, + snapshot_at: snapshotAt || undefined, + }) + } + + logger.error(`[${requestId}] Dataset snapshot timed out`, { datasetId, snapshotId }) + return NextResponse.json( + { error: 'Timeout waiting for dataset snapshot' }, + { status: 504 } + ) + } catch (error) { + const message = error instanceof Error ? error.message : 'Dataset fetch failed' + logger.error(`[${requestId}] Dataset fetch failed`, { error: message }) + return NextResponse.json({ error: message }, { status: 500 }) + } +} diff --git a/apps/sim/app/api/tools/brightdata/scrape-markdown/route.ts b/apps/sim/app/api/tools/brightdata/scrape-markdown/route.ts new file mode 100644 index 0000000000..6874db85f0 --- /dev/null +++ b/apps/sim/app/api/tools/brightdata/scrape-markdown/route.ts @@ -0,0 +1,80 @@ +import { randomUUID } from 'crypto' +import { createLogger } from '@sim/logger' +import { NextResponse } from 'next/server' + +const logger = createLogger('BrightDataScrapeMarkdownAPI') + +export async function POST(request: Request) { + const requestId = randomUUID().slice(0, 8) + + try { + const body = await request.json() + const url = typeof body?.url === 'string' ? body.url : undefined + const apiToken = typeof body?.apiToken === 'string' ? body.apiToken : undefined + const unlockerZone = typeof body?.unlockerZone === 'string' ? body.unlockerZone : undefined + + if (!url || !apiToken) { + return NextResponse.json({ error: 'Missing required parameters' }, { status: 400 }) + } + + logger.info(`[${requestId}] Scraping URL as markdown`, { url }) + + const response = await fetch('https://api.brightdata.com/request', { + method: 'POST', + headers: { + Authorization: `Bearer ${apiToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + zone: unlockerZone || 'mcp_unlocker', + url, + format: 'raw', + data_format: 'markdown', + }), + }) + + const responseText = await response.text() + let payload: unknown = responseText + + try { + payload = JSON.parse(responseText) + } catch { + payload = responseText + } + + if (!response.ok) { + const errorMessage = + typeof payload === 'object' && payload !== null && 'error' in payload + ? String((payload as { error?: unknown }).error) + : response.statusText + + logger.error(`[${requestId}] Scraping failed`, { url, status: response.status, error: errorMessage }) + + return NextResponse.json({ error: errorMessage || 'Scraping failed' }, { status: response.status }) + } + + const markdown = + typeof payload === 'object' && payload !== null && 'markdown' in payload + ? String((payload as { markdown?: unknown }).markdown ?? '') + : typeof payload === 'string' + ? payload + : JSON.stringify(payload) + + const title = + typeof payload === 'object' && payload !== null && 'title' in payload + ? String((payload as { title?: unknown }).title ?? '') + : undefined + + logger.info(`[${requestId}] Scraping completed`, { url }) + + return NextResponse.json({ + markdown, + url, + title: title || undefined, + }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Scraping failed' + logger.error(`[${requestId}] Scraping failed`, { error: message }) + return NextResponse.json({ error: message }, { status: 500 }) + } +} diff --git a/apps/sim/app/api/tools/brightdata/search-engine/route.ts b/apps/sim/app/api/tools/brightdata/search-engine/route.ts new file mode 100644 index 0000000000..c6c14834fc --- /dev/null +++ b/apps/sim/app/api/tools/brightdata/search-engine/route.ts @@ -0,0 +1,98 @@ +import { randomUUID } from 'crypto' +import { createLogger } from '@sim/logger' +import { NextResponse } from 'next/server' + +const logger = createLogger('BrightDataSearchEngineAPI') + +export async function POST(request: Request) { + const requestId = randomUUID().slice(0, 8) + + try { + const body = await request.json() + const query = typeof body?.query === 'string' ? body.query : undefined + const apiToken = typeof body?.apiToken === 'string' ? body.apiToken : undefined + const unlockerZone = typeof body?.unlockerZone === 'string' ? body.unlockerZone : undefined + const maxResults = + typeof body?.maxResults === 'number' + ? body.maxResults + : typeof body?.maxResults === 'string' + ? Number(body.maxResults) + : undefined + + if (!query || !apiToken) { + return NextResponse.json({ error: 'Missing required parameters' }, { status: 400 }) + } + + logger.info(`[${requestId}] Searching`, { query, maxResults }) + + const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(query)}&start=0&brd_json=1` + + const response = await fetch('https://api.brightdata.com/request', { + method: 'POST', + headers: { + Authorization: `Bearer ${apiToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + zone: unlockerZone || 'mcp_unlocker', + url: searchUrl, + format: 'raw', + data_format: 'parsed_light', + }), + }) + + const responseText = await response.text() + let payload: unknown = responseText + + try { + payload = JSON.parse(responseText) + } catch { + payload = responseText + } + + if (!response.ok) { + const errorMessage = + typeof payload === 'object' && payload !== null && 'error' in payload + ? String((payload as { error?: unknown }).error) + : response.statusText + + logger.error(`[${requestId}] Search failed`, { query, status: response.status, error: errorMessage }) + + return NextResponse.json({ error: errorMessage || 'Search failed' }, { status: response.status }) + } + + let normalizedResults: Array<{ title: string; url: string; snippet: string }> = [] + + if (typeof payload === 'object' && payload !== null) { + const organic = (payload as { organic?: unknown }).organic + if (Array.isArray(organic)) { + normalizedResults = organic + .map((entry) => { + if (!entry || typeof entry !== 'object') return null + const rawTitle = (entry as { title?: unknown }).title + const rawLink = (entry as { link?: unknown }).link + const rawDescription = (entry as { description?: unknown }).description + const title = typeof rawTitle === 'string' ? rawTitle : '' + const url = typeof rawLink === 'string' ? rawLink : '' + const snippet = typeof rawDescription === 'string' ? rawDescription : '' + if (!title || !url) return null + return { title, url, snippet } + }) + .filter(Boolean) as Array<{ title: string; url: string; snippet: string }> + } + } + + const maxCount = Number.isFinite(maxResults) ? Number(maxResults) : undefined + const results = maxCount ? normalizedResults.slice(0, maxCount) : normalizedResults + + logger.info(`[${requestId}] Search completed`, { resultCount: results.length }) + + return NextResponse.json({ + results, + }) + } catch (error) { + const message = error instanceof Error ? error.message : 'Search failed' + logger.error(`[${requestId}] Search failed`, { error: message }) + return NextResponse.json({ error: message }, { status: 500 }) + } +} diff --git a/apps/sim/blocks/blocks/brightdata.ts b/apps/sim/blocks/blocks/brightdata.ts new file mode 100644 index 0000000000..e77d134bee --- /dev/null +++ b/apps/sim/blocks/blocks/brightdata.ts @@ -0,0 +1,119 @@ +import { BrightDataIcon } from '@/components/icons' +import type { BlockConfig } from '@/blocks/types' +import { AuthMode } from '@/blocks/types' +import type { BrightDataResponse } from '@/tools/brightdata/types' + +export const BrightDataBlock: BlockConfig = { + type: 'brightdata', + name: 'Bright Data', + description: 'Web scraping, search, and dataset access', + authMode: AuthMode.ApiKey, + longDescription: + "Access Bright Data's web data collection tools including web scraping, search, and datasets.", + docsLink: 'https://docs.sim.ai/tools/brightdata', + category: 'tools', + bgColor: '#3D7FFC', + icon: BrightDataIcon, + + subBlocks: [ + { + id: 'operation', + title: 'Operation', + type: 'dropdown', + required: true, + options: [ + { label: 'Scrape as Markdown', id: 'scrape_markdown' }, + { label: 'Search Engine', id: 'search_engine' }, + { label: 'Amazon Product Dataset', id: 'dataset_amazon_product' }, + ], + value: () => 'scrape_markdown', + }, + { + id: 'url', + title: 'URL', + type: 'short-input', + placeholder: 'https://example.com', + condition: { field: 'operation', value: 'scrape_markdown' }, + required: true, + }, + { + id: 'query', + title: 'Search Query', + type: 'short-input', + placeholder: 'Enter search query', + condition: { field: 'operation', value: 'search_engine' }, + required: true, + }, + { + id: 'maxResults', + title: 'Max Results', + type: 'short-input', + placeholder: '10', + condition: { field: 'operation', value: 'search_engine' }, + }, + { + id: 'url', + title: 'Amazon Product URL', + type: 'short-input', + placeholder: 'https://www.amazon.com/dp/...', + condition: { field: 'operation', value: 'dataset_amazon_product' }, + required: true, + }, + { + id: 'apiToken', + title: 'API Token', + type: 'short-input', + placeholder: 'Your Bright Data API token', + password: true, + required: true, + }, + { + id: 'unlockerZone', + title: 'Unlocker Zone', + type: 'short-input', + placeholder: 'mcp_unlocker', + mode: 'advanced', + }, + ], + + tools: { + access: [ + 'brightdata_scrape_markdown', + 'brightdata_search_engine', + 'brightdata_dataset_amazon_product', + ], + config: { + tool: (params: Record) => { + switch (params.operation) { + case 'scrape_markdown': + return 'brightdata_scrape_markdown' + case 'search_engine': + return 'brightdata_search_engine' + case 'dataset_amazon_product': + return 'brightdata_dataset_amazon_product' + default: + throw new Error('Invalid operation selected') + } + }, + }, + }, + + inputs: { + operation: { type: 'string', description: 'Operation to perform' }, + url: { type: 'string', description: 'URL to scrape or dataset input' }, + query: { type: 'string', description: 'Search query' }, + maxResults: { type: 'number', description: 'Maximum search results' }, + apiToken: { type: 'string', description: 'Bright Data API token' }, + unlockerZone: { type: 'string', description: 'Unlocker zone name' }, + }, + + outputs: { + markdown: { type: 'string', description: 'Scraped markdown content' }, + results: { type: 'array', description: 'Search results' }, + data: { type: 'object', description: 'Dataset response' }, + url: { type: 'string', description: 'Current or scraped URL' }, + title: { type: 'string', description: 'Page title' }, + success: { type: 'boolean', description: 'Operation success status' }, + snapshot_at: { type: 'string', description: 'Dataset snapshot timestamp' }, + }, +} diff --git a/apps/sim/blocks/registry.ts b/apps/sim/blocks/registry.ts index 85a5b7ac2f..e4e366b73f 100644 --- a/apps/sim/blocks/registry.ts +++ b/apps/sim/blocks/registry.ts @@ -7,6 +7,7 @@ import { ApifyBlock } from '@/blocks/blocks/apify' import { ApolloBlock } from '@/blocks/blocks/apollo' import { ArxivBlock } from '@/blocks/blocks/arxiv' import { AsanaBlock } from '@/blocks/blocks/asana' +import { BrightDataBlock } from '@/blocks/blocks/brightdata' // import { BoxBlock } from '@/blocks/blocks/box' // TODO: Box OAuth integration import { BrowserUseBlock } from '@/blocks/blocks/browser_use' import { CalendlyBlock } from '@/blocks/blocks/calendly' @@ -158,6 +159,7 @@ export const registry: Record = { arxiv: ArxivBlock, asana: AsanaBlock, // box: BoxBlock, // TODO: Box OAuth integration + brightdata: BrightDataBlock, browser_use: BrowserUseBlock, calendly: CalendlyBlock, chat_trigger: ChatTriggerBlock, diff --git a/apps/sim/components/icons.tsx b/apps/sim/components/icons.tsx index 91803c3316..370c365f95 100644 --- a/apps/sim/components/icons.tsx +++ b/apps/sim/components/icons.tsx @@ -425,6 +425,67 @@ export function FirecrawlIcon(props: SVGProps) { ) } +export function BrightDataIcon(props: SVGProps) { + return ( + + + + + + + + + + + + + + ) +} + export function JinaAIIcon(props: SVGProps) { return ( = { + id: 'brightdata_dataset_amazon_product', + name: 'Bright Data Amazon Product Dataset', + description: 'Get structured Amazon product data from Bright Data dataset', + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Amazon product URL (must contain /dp/)', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => ({ + datasetId: 'gd_l7q7dkf244hwjntr0', + url: params.url, + apiToken: params.apiToken, + }), + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured Amazon product data', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/index.ts b/apps/sim/tools/brightdata/index.ts new file mode 100644 index 0000000000..b3f50ffc32 --- /dev/null +++ b/apps/sim/tools/brightdata/index.ts @@ -0,0 +1,4 @@ +export { scrapeMarkdownTool } from './scrape_markdown' +export { searchEngineTool } from './search_engine' +export { datasetAmazonProductTool } from './dataset_amazon_product' +export * from './types' diff --git a/apps/sim/tools/brightdata/scrape_markdown.ts b/apps/sim/tools/brightdata/scrape_markdown.ts new file mode 100644 index 0000000000..bb1e20cea7 --- /dev/null +++ b/apps/sim/tools/brightdata/scrape_markdown.ts @@ -0,0 +1,75 @@ +import type { ScrapeMarkdownParams, ScrapeMarkdownResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data tool for scraping a URL into markdown. + */ +export const scrapeMarkdownTool: ToolConfig = { + id: 'brightdata_scrape_markdown', + name: 'Bright Data Scrape as Markdown', + description: 'Scrape any website and convert it to clean markdown format using Bright Data', + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'The URL to scrape', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + unlockerZone: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Bright Data unlocker zone name (default: mcp_unlocker)', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/scrape-markdown', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => ({ + url: params.url, + apiToken: params.apiToken, + unlockerZone: params.unlockerZone || 'mcp_unlocker', + }), + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data scrape failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + markdown: { + type: 'string', + description: 'The scraped content in markdown format', + }, + url: { + type: 'string', + description: 'The URL that was scraped', + }, + title: { + type: 'string', + description: 'The page title', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/search_engine.ts b/apps/sim/tools/brightdata/search_engine.ts new file mode 100644 index 0000000000..ff235907e1 --- /dev/null +++ b/apps/sim/tools/brightdata/search_engine.ts @@ -0,0 +1,73 @@ +import type { SearchEngineParams, SearchEngineResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data search engine tool. + */ +export const searchEngineTool: ToolConfig = { + id: 'brightdata_search_engine', + name: 'Bright Data Search Engine', + description: 'Search the web using Bright Data search engine', + version: '1.0.0', + + params: { + query: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Search query', + }, + maxResults: { + type: 'number', + required: false, + visibility: 'user-only', + description: 'Maximum number of results to return (default: 10)', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + unlockerZone: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Bright Data unlocker zone name', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/search-engine', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => ({ + query: params.query, + maxResults: params.maxResults || 10, + apiToken: params.apiToken, + unlockerZone: params.unlockerZone || 'mcp_unlocker', + }), + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data search failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + results: { + type: 'array', + description: 'Search results with title, URL, and snippet', + }, + }, +} diff --git a/apps/sim/tools/brightdata/types.ts b/apps/sim/tools/brightdata/types.ts new file mode 100644 index 0000000000..35d06bc18e --- /dev/null +++ b/apps/sim/tools/brightdata/types.ts @@ -0,0 +1,107 @@ +import type { ToolResponse } from '@/tools/types' + +export interface BrightDataAuthParams { + apiToken: string + unlockerZone?: string + browserZone?: string +} + + +export interface ScrapeMarkdownParams extends BrightDataAuthParams { + url: string +} + + +export interface ScrapeMarkdownResponse extends ToolResponse { + output: { + markdown: string + url: string + title?: string + } +} + + +export interface SearchEngineParams extends BrightDataAuthParams { + query: string + maxResults?: number +} + + +export interface SearchEngineResponse extends ToolResponse { + output: { + results: Array<{ + title: string + url: string + snippet: string + }> + } +} + + +export interface BrowserNavigateParams extends BrightDataAuthParams { + url: string + sessionKey?: string + country?: string +} + + +export interface BrowserNavigateResponse extends ToolResponse { + output: { + success: boolean + url: string + title?: string + sessionKey?: string + } +} + + +export interface BrowserSnapshotParams extends BrightDataAuthParams { + sessionKey?: string +} + + +export interface BrowserSnapshotResponse extends ToolResponse { + output: { + html: string + url: string + title?: string + } +} + + +export interface BrowserScreenshotParams extends BrightDataAuthParams { + sessionKey?: string +} + + +export interface BrowserScreenshotResponse extends ToolResponse { + output: { + image: string + url: string + } +} + + +export interface DatasetParams extends BrightDataAuthParams { + datasetId: string + url?: string + keyword?: string + [key: string]: unknown +} + + +export interface DatasetResponse extends ToolResponse { + output: { + data: unknown + snapshot_at?: string + } +} + + +export type BrightDataResponse = + | ScrapeMarkdownResponse + | SearchEngineResponse + | BrowserNavigateResponse + | BrowserSnapshotResponse + | BrowserScreenshotResponse + | DatasetResponse diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index 05bcb9f5a3..744e22f1d5 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -51,6 +51,11 @@ import { asanaSearchTasksTool, asanaUpdateTaskTool, } from '@/tools/asana' +import { + datasetAmazonProductTool, + scrapeMarkdownTool, + searchEngineTool, +} from '@/tools/brightdata' import { browserUseRunTaskTool } from '@/tools/browser_use' import { calendlyCancelEventTool, @@ -1532,6 +1537,9 @@ export const tools: Record = { asana_get_projects: asanaGetProjectsTool, asana_search_tasks: asanaSearchTasksTool, asana_add_comment: asanaAddCommentTool, + brightdata_scrape_markdown: scrapeMarkdownTool, + brightdata_search_engine: searchEngineTool, + brightdata_dataset_amazon_product: datasetAmazonProductTool, browser_use_run_task: browserUseRunTaskTool, openai_embeddings: openAIEmbeddingsTool, http_request: httpRequestTool, From a55b2248fff9a53597bbb1556937ba19175b0cd3 Mon Sep 17 00:00:00 2001 From: meirk-brd Date: Wed, 14 Jan 2026 09:34:25 +0200 Subject: [PATCH 2/4] add datasets --- apps/sim/blocks/blocks/brightdata.ts | 209 +++++++++++++++++- .../brightdata/dataset_amazon_product.ts | 22 +- .../dataset_amazon_product_reviews.ts | 69 ++++++ .../dataset_amazon_product_search.ts | 78 +++++++ .../brightdata/dataset_apple_app_store.ts | 69 ++++++ .../brightdata/dataset_bestbuy_products.ts | 69 ++++++ .../dataset_booking_hotel_listings.ts | 69 ++++++ .../brightdata/dataset_crunchbase_company.ts | 69 ++++++ .../tools/brightdata/dataset_ebay_product.ts | 69 ++++++ .../tools/brightdata/dataset_etsy_products.ts | 69 ++++++ .../dataset_facebook_company_reviews.ts | 76 +++++++ .../brightdata/dataset_facebook_events.ts | 69 ++++++ .../dataset_facebook_marketplace_listings.ts | 69 ++++++ .../brightdata/dataset_facebook_posts.ts | 69 ++++++ .../dataset_github_repository_file.ts | 69 ++++++ .../brightdata/dataset_google_maps_reviews.ts | 80 +++++++ .../brightdata/dataset_google_play_store.ts | 69 ++++++ .../brightdata/dataset_google_shopping.ts | 69 ++++++ .../brightdata/dataset_homedepot_products.ts | 69 ++++++ .../brightdata/dataset_instagram_comments.ts | 69 ++++++ .../brightdata/dataset_instagram_posts.ts | 69 ++++++ .../brightdata/dataset_instagram_profiles.ts | 69 ++++++ .../brightdata/dataset_instagram_reels.ts | 69 ++++++ .../dataset_linkedin_company_profile.ts | 69 ++++++ .../dataset_linkedin_job_listings.ts | 69 ++++++ .../dataset_linkedin_people_search.ts | 83 +++++++ .../dataset_linkedin_person_profile.ts | 69 ++++++ .../brightdata/dataset_linkedin_posts.ts | 69 ++++++ .../tools/brightdata/dataset_npm_package.ts | 69 ++++++ .../tools/brightdata/dataset_pypi_package.ts | 69 ++++++ .../tools/brightdata/dataset_reddit_posts.ts | 69 ++++++ .../tools/brightdata/dataset_reuter_news.ts | 69 ++++++ .../brightdata/dataset_tiktok_comments.ts | 69 ++++++ .../tools/brightdata/dataset_tiktok_posts.ts | 69 ++++++ .../brightdata/dataset_tiktok_profiles.ts | 69 ++++++ .../tools/brightdata/dataset_tiktok_shop.ts | 69 ++++++ .../brightdata/dataset_walmart_product.ts | 69 ++++++ .../brightdata/dataset_walmart_seller.ts | 69 ++++++ apps/sim/tools/brightdata/dataset_x_posts.ts | 69 ++++++ .../dataset_yahoo_finance_business.ts | 69 ++++++ .../brightdata/dataset_youtube_comments.ts | 80 +++++++ .../brightdata/dataset_youtube_profiles.ts | 69 ++++++ .../brightdata/dataset_youtube_videos.ts | 69 ++++++ .../tools/brightdata/dataset_zara_products.ts | 69 ++++++ .../dataset_zillow_properties_listing.ts | 69 ++++++ .../dataset_zoominfo_company_profile.ts | 69 ++++++ apps/sim/tools/brightdata/index.ts | 44 ++++ apps/sim/tools/registry.ts | 92 +++++++- 48 files changed, 3438 insertions(+), 17 deletions(-) create mode 100644 apps/sim/tools/brightdata/dataset_amazon_product_reviews.ts create mode 100644 apps/sim/tools/brightdata/dataset_amazon_product_search.ts create mode 100644 apps/sim/tools/brightdata/dataset_apple_app_store.ts create mode 100644 apps/sim/tools/brightdata/dataset_bestbuy_products.ts create mode 100644 apps/sim/tools/brightdata/dataset_booking_hotel_listings.ts create mode 100644 apps/sim/tools/brightdata/dataset_crunchbase_company.ts create mode 100644 apps/sim/tools/brightdata/dataset_ebay_product.ts create mode 100644 apps/sim/tools/brightdata/dataset_etsy_products.ts create mode 100644 apps/sim/tools/brightdata/dataset_facebook_company_reviews.ts create mode 100644 apps/sim/tools/brightdata/dataset_facebook_events.ts create mode 100644 apps/sim/tools/brightdata/dataset_facebook_marketplace_listings.ts create mode 100644 apps/sim/tools/brightdata/dataset_facebook_posts.ts create mode 100644 apps/sim/tools/brightdata/dataset_github_repository_file.ts create mode 100644 apps/sim/tools/brightdata/dataset_google_maps_reviews.ts create mode 100644 apps/sim/tools/brightdata/dataset_google_play_store.ts create mode 100644 apps/sim/tools/brightdata/dataset_google_shopping.ts create mode 100644 apps/sim/tools/brightdata/dataset_homedepot_products.ts create mode 100644 apps/sim/tools/brightdata/dataset_instagram_comments.ts create mode 100644 apps/sim/tools/brightdata/dataset_instagram_posts.ts create mode 100644 apps/sim/tools/brightdata/dataset_instagram_profiles.ts create mode 100644 apps/sim/tools/brightdata/dataset_instagram_reels.ts create mode 100644 apps/sim/tools/brightdata/dataset_linkedin_company_profile.ts create mode 100644 apps/sim/tools/brightdata/dataset_linkedin_job_listings.ts create mode 100644 apps/sim/tools/brightdata/dataset_linkedin_people_search.ts create mode 100644 apps/sim/tools/brightdata/dataset_linkedin_person_profile.ts create mode 100644 apps/sim/tools/brightdata/dataset_linkedin_posts.ts create mode 100644 apps/sim/tools/brightdata/dataset_npm_package.ts create mode 100644 apps/sim/tools/brightdata/dataset_pypi_package.ts create mode 100644 apps/sim/tools/brightdata/dataset_reddit_posts.ts create mode 100644 apps/sim/tools/brightdata/dataset_reuter_news.ts create mode 100644 apps/sim/tools/brightdata/dataset_tiktok_comments.ts create mode 100644 apps/sim/tools/brightdata/dataset_tiktok_posts.ts create mode 100644 apps/sim/tools/brightdata/dataset_tiktok_profiles.ts create mode 100644 apps/sim/tools/brightdata/dataset_tiktok_shop.ts create mode 100644 apps/sim/tools/brightdata/dataset_walmart_product.ts create mode 100644 apps/sim/tools/brightdata/dataset_walmart_seller.ts create mode 100644 apps/sim/tools/brightdata/dataset_x_posts.ts create mode 100644 apps/sim/tools/brightdata/dataset_yahoo_finance_business.ts create mode 100644 apps/sim/tools/brightdata/dataset_youtube_comments.ts create mode 100644 apps/sim/tools/brightdata/dataset_youtube_profiles.ts create mode 100644 apps/sim/tools/brightdata/dataset_youtube_videos.ts create mode 100644 apps/sim/tools/brightdata/dataset_zara_products.ts create mode 100644 apps/sim/tools/brightdata/dataset_zillow_properties_listing.ts create mode 100644 apps/sim/tools/brightdata/dataset_zoominfo_company_profile.ts diff --git a/apps/sim/blocks/blocks/brightdata.ts b/apps/sim/blocks/blocks/brightdata.ts index e77d134bee..2a73d5b42b 100644 --- a/apps/sim/blocks/blocks/brightdata.ts +++ b/apps/sim/blocks/blocks/brightdata.ts @@ -3,6 +3,54 @@ import type { BlockConfig } from '@/blocks/types' import { AuthMode } from '@/blocks/types' import type { BrightDataResponse } from '@/tools/brightdata/types' +const DATASET_TOOL_MAP: Record = { + dataset_amazon_product: 'brightdata_dataset_amazon_product', + dataset_amazon_product_reviews: 'brightdata_dataset_amazon_product_reviews', + dataset_amazon_product_search: 'brightdata_dataset_amazon_product_search', + dataset_walmart_product: 'brightdata_dataset_walmart_product', + dataset_walmart_seller: 'brightdata_dataset_walmart_seller', + dataset_ebay_product: 'brightdata_dataset_ebay_product', + dataset_homedepot_products: 'brightdata_dataset_homedepot_products', + dataset_zara_products: 'brightdata_dataset_zara_products', + dataset_etsy_products: 'brightdata_dataset_etsy_products', + dataset_bestbuy_products: 'brightdata_dataset_bestbuy_products', + dataset_linkedin_person_profile: 'brightdata_dataset_linkedin_person_profile', + dataset_linkedin_company_profile: 'brightdata_dataset_linkedin_company_profile', + dataset_linkedin_job_listings: 'brightdata_dataset_linkedin_job_listings', + dataset_linkedin_posts: 'brightdata_dataset_linkedin_posts', + dataset_linkedin_people_search: 'brightdata_dataset_linkedin_people_search', + dataset_crunchbase_company: 'brightdata_dataset_crunchbase_company', + dataset_zoominfo_company_profile: 'brightdata_dataset_zoominfo_company_profile', + dataset_instagram_profiles: 'brightdata_dataset_instagram_profiles', + dataset_instagram_posts: 'brightdata_dataset_instagram_posts', + dataset_instagram_reels: 'brightdata_dataset_instagram_reels', + dataset_instagram_comments: 'brightdata_dataset_instagram_comments', + dataset_facebook_posts: 'brightdata_dataset_facebook_posts', + dataset_facebook_marketplace_listings: 'brightdata_dataset_facebook_marketplace_listings', + dataset_facebook_company_reviews: 'brightdata_dataset_facebook_company_reviews', + dataset_facebook_events: 'brightdata_dataset_facebook_events', + dataset_tiktok_profiles: 'brightdata_dataset_tiktok_profiles', + dataset_tiktok_posts: 'brightdata_dataset_tiktok_posts', + dataset_tiktok_shop: 'brightdata_dataset_tiktok_shop', + dataset_tiktok_comments: 'brightdata_dataset_tiktok_comments', + dataset_google_maps_reviews: 'brightdata_dataset_google_maps_reviews', + dataset_google_shopping: 'brightdata_dataset_google_shopping', + dataset_google_play_store: 'brightdata_dataset_google_play_store', + dataset_apple_app_store: 'brightdata_dataset_apple_app_store', + dataset_reuter_news: 'brightdata_dataset_reuter_news', + dataset_github_repository_file: 'brightdata_dataset_github_repository_file', + dataset_yahoo_finance_business: 'brightdata_dataset_yahoo_finance_business', + dataset_x_posts: 'brightdata_dataset_x_posts', + dataset_zillow_properties_listing: 'brightdata_dataset_zillow_properties_listing', + dataset_booking_hotel_listings: 'brightdata_dataset_booking_hotel_listings', + dataset_youtube_profiles: 'brightdata_dataset_youtube_profiles', + dataset_youtube_comments: 'brightdata_dataset_youtube_comments', + dataset_reddit_posts: 'brightdata_dataset_reddit_posts', + dataset_youtube_videos: 'brightdata_dataset_youtube_videos', + dataset_npm_package: 'brightdata_dataset_npm_package', + dataset_pypi_package: 'brightdata_dataset_pypi_package', +} + export const BrightDataBlock: BlockConfig = { type: 'brightdata', name: 'Bright Data', @@ -25,6 +73,50 @@ export const BrightDataBlock: BlockConfig = { { label: 'Scrape as Markdown', id: 'scrape_markdown' }, { label: 'Search Engine', id: 'search_engine' }, { label: 'Amazon Product Dataset', id: 'dataset_amazon_product' }, + { label: 'Amazon Product Reviews Dataset', id: 'dataset_amazon_product_reviews' }, + { label: 'Amazon Product Search Dataset', id: 'dataset_amazon_product_search' }, + { label: 'Walmart Product Dataset', id: 'dataset_walmart_product' }, + { label: 'Walmart Seller Dataset', id: 'dataset_walmart_seller' }, + { label: 'Ebay Product Dataset', id: 'dataset_ebay_product' }, + { label: 'Homedepot Products Dataset', id: 'dataset_homedepot_products' }, + { label: 'Zara Products Dataset', id: 'dataset_zara_products' }, + { label: 'Etsy Products Dataset', id: 'dataset_etsy_products' }, + { label: 'Bestbuy Products Dataset', id: 'dataset_bestbuy_products' }, + { label: 'Linkedin Person Profile Dataset', id: 'dataset_linkedin_person_profile' }, + { label: 'Linkedin Company Profile Dataset', id: 'dataset_linkedin_company_profile' }, + { label: 'Linkedin Job Listings Dataset', id: 'dataset_linkedin_job_listings' }, + { label: 'Linkedin Posts Dataset', id: 'dataset_linkedin_posts' }, + { label: 'Linkedin People Search Dataset', id: 'dataset_linkedin_people_search' }, + { label: 'Crunchbase Company Dataset', id: 'dataset_crunchbase_company' }, + { label: 'Zoominfo Company Profile Dataset', id: 'dataset_zoominfo_company_profile' }, + { label: 'Instagram Profiles Dataset', id: 'dataset_instagram_profiles' }, + { label: 'Instagram Posts Dataset', id: 'dataset_instagram_posts' }, + { label: 'Instagram Reels Dataset', id: 'dataset_instagram_reels' }, + { label: 'Instagram Comments Dataset', id: 'dataset_instagram_comments' }, + { label: 'Facebook Posts Dataset', id: 'dataset_facebook_posts' }, + { label: 'Facebook Marketplace Listings Dataset', id: 'dataset_facebook_marketplace_listings' }, + { label: 'Facebook Company Reviews Dataset', id: 'dataset_facebook_company_reviews' }, + { label: 'Facebook Events Dataset', id: 'dataset_facebook_events' }, + { label: 'Tiktok Profiles Dataset', id: 'dataset_tiktok_profiles' }, + { label: 'Tiktok Posts Dataset', id: 'dataset_tiktok_posts' }, + { label: 'Tiktok Shop Dataset', id: 'dataset_tiktok_shop' }, + { label: 'Tiktok Comments Dataset', id: 'dataset_tiktok_comments' }, + { label: 'Google Maps Reviews Dataset', id: 'dataset_google_maps_reviews' }, + { label: 'Google Shopping Dataset', id: 'dataset_google_shopping' }, + { label: 'Google Play Store Dataset', id: 'dataset_google_play_store' }, + { label: 'Apple App Store Dataset', id: 'dataset_apple_app_store' }, + { label: 'Reuter News Dataset', id: 'dataset_reuter_news' }, + { label: 'Github Repository File Dataset', id: 'dataset_github_repository_file' }, + { label: 'Yahoo Finance Business Dataset', id: 'dataset_yahoo_finance_business' }, + { label: 'X Posts Dataset', id: 'dataset_x_posts' }, + { label: 'Zillow Properties Listing Dataset', id: 'dataset_zillow_properties_listing' }, + { label: 'Booking Hotel Listings Dataset', id: 'dataset_booking_hotel_listings' }, + { label: 'Youtube Profiles Dataset', id: 'dataset_youtube_profiles' }, + { label: 'Youtube Comments Dataset', id: 'dataset_youtube_comments' }, + { label: 'Reddit Posts Dataset', id: 'dataset_reddit_posts' }, + { label: 'Youtube Videos Dataset', id: 'dataset_youtube_videos' }, + { label: 'Npm Package Dataset', id: 'dataset_npm_package' }, + { label: 'Pypi Package Dataset', id: 'dataset_pypi_package' }, ], value: () => 'scrape_markdown', }, @@ -53,10 +145,64 @@ export const BrightDataBlock: BlockConfig = { }, { id: 'url', - title: 'Amazon Product URL', + title: 'Dataset URL', + type: 'short-input', + placeholder: 'https://example.com', + condition: { field: 'operation', value: ['dataset_amazon_product', 'dataset_amazon_product_reviews', 'dataset_amazon_product_search', 'dataset_walmart_product', 'dataset_walmart_seller', 'dataset_ebay_product', 'dataset_homedepot_products', 'dataset_zara_products', 'dataset_etsy_products', 'dataset_bestbuy_products', 'dataset_linkedin_person_profile', 'dataset_linkedin_company_profile', 'dataset_linkedin_job_listings', 'dataset_linkedin_posts', 'dataset_linkedin_people_search', 'dataset_crunchbase_company', 'dataset_zoominfo_company_profile', 'dataset_instagram_profiles', 'dataset_instagram_posts', 'dataset_instagram_reels', 'dataset_instagram_comments', 'dataset_facebook_posts', 'dataset_facebook_marketplace_listings', 'dataset_facebook_company_reviews', 'dataset_facebook_events', 'dataset_tiktok_profiles', 'dataset_tiktok_posts', 'dataset_tiktok_shop', 'dataset_tiktok_comments', 'dataset_google_maps_reviews', 'dataset_google_shopping', 'dataset_google_play_store', 'dataset_apple_app_store', 'dataset_reuter_news', 'dataset_github_repository_file', 'dataset_yahoo_finance_business', 'dataset_x_posts', 'dataset_zillow_properties_listing', 'dataset_booking_hotel_listings', 'dataset_youtube_profiles', 'dataset_youtube_comments', 'dataset_reddit_posts', 'dataset_youtube_videos'] }, + required: true, + }, + { + id: 'keyword', + title: 'Keyword', + type: 'short-input', + placeholder: 'Enter keyword', + condition: { field: 'operation', value: ['dataset_amazon_product_search'] }, + required: true, + }, + { + id: 'first_name', + title: 'First Name', + type: 'short-input', + placeholder: 'First name', + condition: { field: 'operation', value: ['dataset_linkedin_people_search'] }, + required: true, + }, + { + id: 'last_name', + title: 'Last Name', + type: 'short-input', + placeholder: 'Last name', + condition: { field: 'operation', value: ['dataset_linkedin_people_search'] }, + required: true, + }, + { + id: 'num_of_reviews', + title: 'Number of Reviews', + type: 'short-input', + placeholder: '10', + condition: { field: 'operation', value: ['dataset_facebook_company_reviews'] }, + required: true, + }, + { + id: 'days_limit', + title: 'Days Limit', + type: 'short-input', + placeholder: '3', + condition: { field: 'operation', value: ['dataset_google_maps_reviews'] }, + }, + { + id: 'num_of_comments', + title: 'Number of Comments', + type: 'short-input', + placeholder: '10', + condition: { field: 'operation', value: ['dataset_youtube_comments'] }, + }, + { + id: 'package_name', + title: 'Package Name', type: 'short-input', - placeholder: 'https://www.amazon.com/dp/...', - condition: { field: 'operation', value: 'dataset_amazon_product' }, + placeholder: '@brightdata/sdk', + condition: { field: 'operation', value: ['dataset_npm_package', 'dataset_pypi_package'] }, required: true, }, { @@ -81,16 +227,60 @@ export const BrightDataBlock: BlockConfig = { 'brightdata_scrape_markdown', 'brightdata_search_engine', 'brightdata_dataset_amazon_product', + 'brightdata_dataset_amazon_product_reviews', + 'brightdata_dataset_amazon_product_search', + 'brightdata_dataset_walmart_product', + 'brightdata_dataset_walmart_seller', + 'brightdata_dataset_ebay_product', + 'brightdata_dataset_homedepot_products', + 'brightdata_dataset_zara_products', + 'brightdata_dataset_etsy_products', + 'brightdata_dataset_bestbuy_products', + 'brightdata_dataset_linkedin_person_profile', + 'brightdata_dataset_linkedin_company_profile', + 'brightdata_dataset_linkedin_job_listings', + 'brightdata_dataset_linkedin_posts', + 'brightdata_dataset_linkedin_people_search', + 'brightdata_dataset_crunchbase_company', + 'brightdata_dataset_zoominfo_company_profile', + 'brightdata_dataset_instagram_profiles', + 'brightdata_dataset_instagram_posts', + 'brightdata_dataset_instagram_reels', + 'brightdata_dataset_instagram_comments', + 'brightdata_dataset_facebook_posts', + 'brightdata_dataset_facebook_marketplace_listings', + 'brightdata_dataset_facebook_company_reviews', + 'brightdata_dataset_facebook_events', + 'brightdata_dataset_tiktok_profiles', + 'brightdata_dataset_tiktok_posts', + 'brightdata_dataset_tiktok_shop', + 'brightdata_dataset_tiktok_comments', + 'brightdata_dataset_google_maps_reviews', + 'brightdata_dataset_google_shopping', + 'brightdata_dataset_google_play_store', + 'brightdata_dataset_apple_app_store', + 'brightdata_dataset_reuter_news', + 'brightdata_dataset_github_repository_file', + 'brightdata_dataset_yahoo_finance_business', + 'brightdata_dataset_x_posts', + 'brightdata_dataset_zillow_properties_listing', + 'brightdata_dataset_booking_hotel_listings', + 'brightdata_dataset_youtube_profiles', + 'brightdata_dataset_youtube_comments', + 'brightdata_dataset_reddit_posts', + 'brightdata_dataset_youtube_videos', + 'brightdata_dataset_npm_package', + 'brightdata_dataset_pypi_package', ], config: { tool: (params: Record) => { + const datasetTool = DATASET_TOOL_MAP[String(params.operation)] + if (datasetTool) return datasetTool switch (params.operation) { case 'scrape_markdown': return 'brightdata_scrape_markdown' case 'search_engine': return 'brightdata_search_engine' - case 'dataset_amazon_product': - return 'brightdata_dataset_amazon_product' default: throw new Error('Invalid operation selected') } @@ -103,6 +293,13 @@ export const BrightDataBlock: BlockConfig = { url: { type: 'string', description: 'URL to scrape or dataset input' }, query: { type: 'string', description: 'Search query' }, maxResults: { type: 'number', description: 'Maximum search results' }, + keyword: { type: 'string', description: 'Dataset keyword input' }, + first_name: { type: 'string', description: 'Dataset first name input' }, + last_name: { type: 'string', description: 'Dataset last name input' }, + num_of_reviews: { type: 'string', description: 'Dataset number of reviews input' }, + days_limit: { type: 'string', description: 'Dataset days limit input' }, + num_of_comments: { type: 'string', description: 'Dataset number of comments input' }, + package_name: { type: 'string', description: 'Dataset package name input' }, apiToken: { type: 'string', description: 'Bright Data API token' }, unlockerZone: { type: 'string', description: 'Unlocker zone name' }, }, @@ -110,7 +307,7 @@ export const BrightDataBlock: BlockConfig = { outputs: { markdown: { type: 'string', description: 'Scraped markdown content' }, results: { type: 'array', description: 'Search results' }, - data: { type: 'object', description: 'Dataset response' }, + data: { type: 'json', description: 'Dataset response' }, url: { type: 'string', description: 'Current or scraped URL' }, title: { type: 'string', description: 'Page title' }, success: { type: 'boolean', description: 'Operation success status' }, diff --git a/apps/sim/tools/brightdata/dataset_amazon_product.ts b/apps/sim/tools/brightdata/dataset_amazon_product.ts index afe380a44c..cbfd808e47 100644 --- a/apps/sim/tools/brightdata/dataset_amazon_product.ts +++ b/apps/sim/tools/brightdata/dataset_amazon_product.ts @@ -2,12 +2,12 @@ import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' import type { ToolConfig } from '@/tools/types' /** - * Bright Data Amazon product dataset tool. + * Bright Data Amazon Product dataset tool. */ export const datasetAmazonProductTool: ToolConfig = { id: 'brightdata_dataset_amazon_product', name: 'Bright Data Amazon Product Dataset', - description: 'Get structured Amazon product data from Bright Data dataset', + description: "Quickly read structured amazon product data.\nRequires a valid product URL with /dp/ in it.\nThis can be a cache lookup, so it can be more reliable than scraping", version: '1.0.0', params: { @@ -15,7 +15,7 @@ export const datasetAmazonProductTool: ToolConfig ({ 'Content-Type': 'application/json', }), - body: (params) => ({ - datasetId: 'gd_l7q7dkf244hwjntr0', - url: params.url, - apiToken: params.apiToken, - }), + body: (params) => { + const body: Record = { + datasetId: 'gd_l7q7dkf244hwjntr0', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, }, transformResponse: async (response: Response) => { @@ -54,7 +58,7 @@ export const datasetAmazonProductTool: ToolConfig = { + id: 'brightdata_dataset_amazon_product_reviews', + name: 'Bright Data Amazon Product Reviews Dataset', + description: "Quickly read structured amazon product review data.\nRequires a valid product URL with /dp/ in it.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_le8e811kzy4ggddlq', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_amazon_product_search.ts b/apps/sim/tools/brightdata/dataset_amazon_product_search.ts new file mode 100644 index 0000000000..0ba8102c98 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_amazon_product_search.ts @@ -0,0 +1,78 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Amazon Product Search dataset tool. + */ +export const datasetAmazonProductSearchTool: ToolConfig = { + id: 'brightdata_dataset_amazon_product_search', + name: 'Bright Data Amazon Product Search Dataset', + description: "Quickly read structured amazon product search data.\nRequires a valid search keyword and amazon domain URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + keyword: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Search keyword', + }, + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lwdb4vjm1ehb499uxs', + apiToken: params.apiToken, + keyword: params.keyword, + url: params.url, + } + + body.pages_to_search = '1' + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_apple_app_store.ts b/apps/sim/tools/brightdata/dataset_apple_app_store.ts new file mode 100644 index 0000000000..04c836d44d --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_apple_app_store.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Apple App Store dataset tool. + */ +export const datasetAppleAppStoreTool: ToolConfig = { + id: 'brightdata_dataset_apple_app_store', + name: 'Bright Data Apple App Store Dataset', + description: "Quickly read structured apple app store data.\nRequires a valid apple app store app URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lsk9ki3u2iishmwrui', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_bestbuy_products.ts b/apps/sim/tools/brightdata/dataset_bestbuy_products.ts new file mode 100644 index 0000000000..acae97cbe3 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_bestbuy_products.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Bestbuy Products dataset tool. + */ +export const datasetBestbuyProductsTool: ToolConfig = { + id: 'brightdata_dataset_bestbuy_products', + name: 'Bright Data Bestbuy Products Dataset', + description: "Quickly read structured bestbuy product data.\nRequires a valid bestbuy product URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_ltre1jqe1jfr7cccf', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_booking_hotel_listings.ts b/apps/sim/tools/brightdata/dataset_booking_hotel_listings.ts new file mode 100644 index 0000000000..ec806a4d18 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_booking_hotel_listings.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Booking Hotel Listings dataset tool. + */ +export const datasetBookingHotelListingsTool: ToolConfig = { + id: 'brightdata_dataset_booking_hotel_listings', + name: 'Bright Data Booking Hotel Listings Dataset', + description: "Quickly read structured booking hotel listings data.\nRequires a valid booking hotel listing URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_m5mbdl081229ln6t4a', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_crunchbase_company.ts b/apps/sim/tools/brightdata/dataset_crunchbase_company.ts new file mode 100644 index 0000000000..8a02aaf593 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_crunchbase_company.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Crunchbase Company dataset tool. + */ +export const datasetCrunchbaseCompanyTool: ToolConfig = { + id: 'brightdata_dataset_crunchbase_company', + name: 'Bright Data Crunchbase Company Dataset', + description: "Quickly read structured crunchbase company data\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_l1vijqt9jfj7olije', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_ebay_product.ts b/apps/sim/tools/brightdata/dataset_ebay_product.ts new file mode 100644 index 0000000000..0e9e891c0a --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_ebay_product.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Ebay Product dataset tool. + */ +export const datasetEbayProductTool: ToolConfig = { + id: 'brightdata_dataset_ebay_product', + name: 'Bright Data Ebay Product Dataset', + description: "Quickly read structured ebay product data.\nRequires a valid ebay product URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_ltr9mjt81n0zzdk1fb', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_etsy_products.ts b/apps/sim/tools/brightdata/dataset_etsy_products.ts new file mode 100644 index 0000000000..c484c757d0 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_etsy_products.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Etsy Products dataset tool. + */ +export const datasetEtsyProductsTool: ToolConfig = { + id: 'brightdata_dataset_etsy_products', + name: 'Bright Data Etsy Products Dataset', + description: "Quickly read structured etsy product data.\nRequires a valid etsy product URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_ltppk0jdv1jqz25mz', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_facebook_company_reviews.ts b/apps/sim/tools/brightdata/dataset_facebook_company_reviews.ts new file mode 100644 index 0000000000..671f933fe1 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_facebook_company_reviews.ts @@ -0,0 +1,76 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Facebook Company Reviews dataset tool. + */ +export const datasetFacebookCompanyReviewsTool: ToolConfig = { + id: 'brightdata_dataset_facebook_company_reviews', + name: 'Bright Data Facebook Company Reviews Dataset', + description: "Quickly read structured Facebook company reviews data.\nRequires a valid Facebook company URL and number of reviews.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + num_of_reviews: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Number of reviews to fetch', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_m0dtqpiu1mbcyc2g86', + apiToken: params.apiToken, + url: params.url, + num_of_reviews: params.num_of_reviews, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_facebook_events.ts b/apps/sim/tools/brightdata/dataset_facebook_events.ts new file mode 100644 index 0000000000..073bc45e05 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_facebook_events.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Facebook Events dataset tool. + */ +export const datasetFacebookEventsTool: ToolConfig = { + id: 'brightdata_dataset_facebook_events', + name: 'Bright Data Facebook Events Dataset', + description: "Quickly read structured Facebook events data.\nRequires a valid Facebook event URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_m14sd0to1jz48ppm51', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_facebook_marketplace_listings.ts b/apps/sim/tools/brightdata/dataset_facebook_marketplace_listings.ts new file mode 100644 index 0000000000..0e6f2be719 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_facebook_marketplace_listings.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Facebook Marketplace Listings dataset tool. + */ +export const datasetFacebookMarketplaceListingsTool: ToolConfig = { + id: 'brightdata_dataset_facebook_marketplace_listings', + name: 'Bright Data Facebook Marketplace Listings Dataset', + description: "Quickly read structured Facebook marketplace listing data.\nRequires a valid Facebook marketplace listing URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lvt9iwuh6fbcwmx1a', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_facebook_posts.ts b/apps/sim/tools/brightdata/dataset_facebook_posts.ts new file mode 100644 index 0000000000..933e1e8264 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_facebook_posts.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Facebook Posts dataset tool. + */ +export const datasetFacebookPostsTool: ToolConfig = { + id: 'brightdata_dataset_facebook_posts', + name: 'Bright Data Facebook Posts Dataset', + description: "Quickly read structured Facebook post data.\nRequires a valid Facebook post URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lyclm1571iy3mv57zw', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_github_repository_file.ts b/apps/sim/tools/brightdata/dataset_github_repository_file.ts new file mode 100644 index 0000000000..1797ef76ac --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_github_repository_file.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Github Repository File dataset tool. + */ +export const datasetGithubRepositoryFileTool: ToolConfig = { + id: 'brightdata_dataset_github_repository_file', + name: 'Bright Data Github Repository File Dataset', + description: "Quickly read structured github repository data.\nRequires a valid github repository file URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lyrexgxc24b3d4imjt', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_google_maps_reviews.ts b/apps/sim/tools/brightdata/dataset_google_maps_reviews.ts new file mode 100644 index 0000000000..bca3e9ae83 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_google_maps_reviews.ts @@ -0,0 +1,80 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Google Maps Reviews dataset tool. + */ +export const datasetGoogleMapsReviewsTool: ToolConfig = { + id: 'brightdata_dataset_google_maps_reviews', + name: 'Bright Data Google Maps Reviews Dataset', + description: "Quickly read structured Google maps reviews data.\nRequires a valid Google maps URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + days_limit: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Days limit (default: 3)', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_luzfs1dn2oa0teb81', + apiToken: params.apiToken, + url: params.url, + days_limit: params.days_limit, + } + + if (body.days_limit === undefined) { + body.days_limit = '3' + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_google_play_store.ts b/apps/sim/tools/brightdata/dataset_google_play_store.ts new file mode 100644 index 0000000000..01c257471f --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_google_play_store.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Google Play Store dataset tool. + */ +export const datasetGooglePlayStoreTool: ToolConfig = { + id: 'brightdata_dataset_google_play_store', + name: 'Bright Data Google Play Store Dataset', + description: "Quickly read structured Google play store data.\nRequires a valid Google play store app URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lsk382l8xei8vzm4u', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_google_shopping.ts b/apps/sim/tools/brightdata/dataset_google_shopping.ts new file mode 100644 index 0000000000..4aa86fa6d9 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_google_shopping.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Google Shopping dataset tool. + */ +export const datasetGoogleShoppingTool: ToolConfig = { + id: 'brightdata_dataset_google_shopping', + name: 'Bright Data Google Shopping Dataset', + description: "Quickly read structured Google shopping data.\nRequires a valid Google shopping product URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_ltppk50q18kdw67omz', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_homedepot_products.ts b/apps/sim/tools/brightdata/dataset_homedepot_products.ts new file mode 100644 index 0000000000..2831aebc87 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_homedepot_products.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Homedepot Products dataset tool. + */ +export const datasetHomedepotProductsTool: ToolConfig = { + id: 'brightdata_dataset_homedepot_products', + name: 'Bright Data Homedepot Products Dataset', + description: "Quickly read structured homedepot product data.\nRequires a valid homedepot product URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lmusivh019i7g97q2n', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_instagram_comments.ts b/apps/sim/tools/brightdata/dataset_instagram_comments.ts new file mode 100644 index 0000000000..61892c6aa8 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_instagram_comments.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Instagram Comments dataset tool. + */ +export const datasetInstagramCommentsTool: ToolConfig = { + id: 'brightdata_dataset_instagram_comments', + name: 'Bright Data Instagram Comments Dataset', + description: "Quickly read structured Instagram comments data.\nRequires a valid Instagram URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_ltppn085pokosxh13', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_instagram_posts.ts b/apps/sim/tools/brightdata/dataset_instagram_posts.ts new file mode 100644 index 0000000000..67af5930ac --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_instagram_posts.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Instagram Posts dataset tool. + */ +export const datasetInstagramPostsTool: ToolConfig = { + id: 'brightdata_dataset_instagram_posts', + name: 'Bright Data Instagram Posts Dataset', + description: "Quickly read structured Instagram post data.\nRequires a valid Instagram URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lk5ns7kz21pck8jpis', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_instagram_profiles.ts b/apps/sim/tools/brightdata/dataset_instagram_profiles.ts new file mode 100644 index 0000000000..5d5ea2d316 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_instagram_profiles.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Instagram Profiles dataset tool. + */ +export const datasetInstagramProfilesTool: ToolConfig = { + id: 'brightdata_dataset_instagram_profiles', + name: 'Bright Data Instagram Profiles Dataset', + description: "Quickly read structured Instagram profile data.\nRequires a valid Instagram URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_l1vikfch901nx3by4', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_instagram_reels.ts b/apps/sim/tools/brightdata/dataset_instagram_reels.ts new file mode 100644 index 0000000000..bd4e03ea28 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_instagram_reels.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Instagram Reels dataset tool. + */ +export const datasetInstagramReelsTool: ToolConfig = { + id: 'brightdata_dataset_instagram_reels', + name: 'Bright Data Instagram Reels Dataset', + description: "Quickly read structured Instagram reel data.\nRequires a valid Instagram URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lyclm20il4r5helnj', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_linkedin_company_profile.ts b/apps/sim/tools/brightdata/dataset_linkedin_company_profile.ts new file mode 100644 index 0000000000..7274454061 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_linkedin_company_profile.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Linkedin Company Profile dataset tool. + */ +export const datasetLinkedinCompanyProfileTool: ToolConfig = { + id: 'brightdata_dataset_linkedin_company_profile', + name: 'Bright Data Linkedin Company Profile Dataset', + description: "Quickly read structured linkedin company profile data\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_l1vikfnt1wgvvqz95w', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_linkedin_job_listings.ts b/apps/sim/tools/brightdata/dataset_linkedin_job_listings.ts new file mode 100644 index 0000000000..f5964174e5 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_linkedin_job_listings.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Linkedin Job Listings dataset tool. + */ +export const datasetLinkedinJobListingsTool: ToolConfig = { + id: 'brightdata_dataset_linkedin_job_listings', + name: 'Bright Data Linkedin Job Listings Dataset', + description: "Quickly read structured linkedin job listings data\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lpfll7v5hcqtkxl6l', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_linkedin_people_search.ts b/apps/sim/tools/brightdata/dataset_linkedin_people_search.ts new file mode 100644 index 0000000000..8c514f671c --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_linkedin_people_search.ts @@ -0,0 +1,83 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Linkedin People Search dataset tool. + */ +export const datasetLinkedinPeopleSearchTool: ToolConfig = { + id: 'brightdata_dataset_linkedin_people_search', + name: 'Bright Data Linkedin People Search Dataset', + description: "Quickly read structured linkedin people search data\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + first_name: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'First name', + }, + last_name: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Last name', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_m8d03he47z8nwb5xc', + apiToken: params.apiToken, + url: params.url, + first_name: params.first_name, + last_name: params.last_name, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_linkedin_person_profile.ts b/apps/sim/tools/brightdata/dataset_linkedin_person_profile.ts new file mode 100644 index 0000000000..a8b3fa4823 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_linkedin_person_profile.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Linkedin Person Profile dataset tool. + */ +export const datasetLinkedinPersonProfileTool: ToolConfig = { + id: 'brightdata_dataset_linkedin_person_profile', + name: 'Bright Data Linkedin Person Profile Dataset', + description: "Quickly read structured linkedin people profile data.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_l1viktl72bvl7bjuj0', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_linkedin_posts.ts b/apps/sim/tools/brightdata/dataset_linkedin_posts.ts new file mode 100644 index 0000000000..d15b199cde --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_linkedin_posts.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Linkedin Posts dataset tool. + */ +export const datasetLinkedinPostsTool: ToolConfig = { + id: 'brightdata_dataset_linkedin_posts', + name: 'Bright Data Linkedin Posts Dataset', + description: "Quickly read structured linkedin posts data\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lyy3tktm25m4avu764', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_npm_package.ts b/apps/sim/tools/brightdata/dataset_npm_package.ts new file mode 100644 index 0000000000..9fbb58cdfb --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_npm_package.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Npm Package dataset tool. + */ +export const datasetNpmPackageTool: ToolConfig = { + id: 'brightdata_dataset_npm_package', + name: 'Bright Data Npm Package Dataset', + description: "Quickly read structured npm package data.\nRequires a valid npm package name (e.g., @brightdata/sdk).\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + package_name: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Package name', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_mk57m0301khq4jmsul', + apiToken: params.apiToken, + package_name: params.package_name, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_pypi_package.ts b/apps/sim/tools/brightdata/dataset_pypi_package.ts new file mode 100644 index 0000000000..b511e32961 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_pypi_package.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Pypi Package dataset tool. + */ +export const datasetPypiPackageTool: ToolConfig = { + id: 'brightdata_dataset_pypi_package', + name: 'Bright Data Pypi Package Dataset', + description: "Quickly read structured PyPI package data.\nRequires a valid PyPI package name (e.g., langchain-brightdata).\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + package_name: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Package name', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_mk57kc3t1wwgmnepp9', + apiToken: params.apiToken, + package_name: params.package_name, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_reddit_posts.ts b/apps/sim/tools/brightdata/dataset_reddit_posts.ts new file mode 100644 index 0000000000..1fc3f26813 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_reddit_posts.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Reddit Posts dataset tool. + */ +export const datasetRedditPostsTool: ToolConfig = { + id: 'brightdata_dataset_reddit_posts', + name: 'Bright Data Reddit Posts Dataset', + description: "Quickly read structured reddit posts data.\nRequires a valid reddit post URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lvz8ah06191smkebj4', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_reuter_news.ts b/apps/sim/tools/brightdata/dataset_reuter_news.ts new file mode 100644 index 0000000000..4a6f52a110 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_reuter_news.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Reuter News dataset tool. + */ +export const datasetReuterNewsTool: ToolConfig = { + id: 'brightdata_dataset_reuter_news', + name: 'Bright Data Reuter News Dataset', + description: "Quickly read structured reuter news data.\nRequires a valid reuter news report URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lyptx9h74wtlvpnfu', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_tiktok_comments.ts b/apps/sim/tools/brightdata/dataset_tiktok_comments.ts new file mode 100644 index 0000000000..99704f8fa6 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_tiktok_comments.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Tiktok Comments dataset tool. + */ +export const datasetTiktokCommentsTool: ToolConfig = { + id: 'brightdata_dataset_tiktok_comments', + name: 'Bright Data Tiktok Comments Dataset', + description: "Quickly read structured Tiktok comments data.\nRequires a valid Tiktok video URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lkf2st302ap89utw5k', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_tiktok_posts.ts b/apps/sim/tools/brightdata/dataset_tiktok_posts.ts new file mode 100644 index 0000000000..73f57ae184 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_tiktok_posts.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Tiktok Posts dataset tool. + */ +export const datasetTiktokPostsTool: ToolConfig = { + id: 'brightdata_dataset_tiktok_posts', + name: 'Bright Data Tiktok Posts Dataset', + description: "Quickly read structured Tiktok post data.\nRequires a valid Tiktok post URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lu702nij2f790tmv9h', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_tiktok_profiles.ts b/apps/sim/tools/brightdata/dataset_tiktok_profiles.ts new file mode 100644 index 0000000000..e44a13c8b4 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_tiktok_profiles.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Tiktok Profiles dataset tool. + */ +export const datasetTiktokProfilesTool: ToolConfig = { + id: 'brightdata_dataset_tiktok_profiles', + name: 'Bright Data Tiktok Profiles Dataset', + description: "Quickly read structured Tiktok profiles data.\nRequires a valid Tiktok profile URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_l1villgoiiidt09ci', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_tiktok_shop.ts b/apps/sim/tools/brightdata/dataset_tiktok_shop.ts new file mode 100644 index 0000000000..0b07724fa2 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_tiktok_shop.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Tiktok Shop dataset tool. + */ +export const datasetTiktokShopTool: ToolConfig = { + id: 'brightdata_dataset_tiktok_shop', + name: 'Bright Data Tiktok Shop Dataset', + description: "Quickly read structured Tiktok shop data.\nRequires a valid Tiktok shop product URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_m45m1u911dsa4274pi', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_walmart_product.ts b/apps/sim/tools/brightdata/dataset_walmart_product.ts new file mode 100644 index 0000000000..d5de7b00cc --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_walmart_product.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Walmart Product dataset tool. + */ +export const datasetWalmartProductTool: ToolConfig = { + id: 'brightdata_dataset_walmart_product', + name: 'Bright Data Walmart Product Dataset', + description: "Quickly read structured walmart product data.\nRequires a valid product URL with /ip/ in it.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_l95fol7l1ru6rlo116', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_walmart_seller.ts b/apps/sim/tools/brightdata/dataset_walmart_seller.ts new file mode 100644 index 0000000000..53ae740d16 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_walmart_seller.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Walmart Seller dataset tool. + */ +export const datasetWalmartSellerTool: ToolConfig = { + id: 'brightdata_dataset_walmart_seller', + name: 'Bright Data Walmart Seller Dataset', + description: "Quickly read structured walmart seller data.\nRequires a valid walmart seller URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_m7ke48w81ocyu4hhz0', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_x_posts.ts b/apps/sim/tools/brightdata/dataset_x_posts.ts new file mode 100644 index 0000000000..ac88fd9f56 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_x_posts.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data X Posts dataset tool. + */ +export const datasetXPostsTool: ToolConfig = { + id: 'brightdata_dataset_x_posts', + name: 'Bright Data X Posts Dataset', + description: "Quickly read structured X post data.\nRequires a valid X post URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lwxkxvnf1cynvib9co', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_yahoo_finance_business.ts b/apps/sim/tools/brightdata/dataset_yahoo_finance_business.ts new file mode 100644 index 0000000000..0a8bfc143b --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_yahoo_finance_business.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Yahoo Finance Business dataset tool. + */ +export const datasetYahooFinanceBusinessTool: ToolConfig = { + id: 'brightdata_dataset_yahoo_finance_business', + name: 'Bright Data Yahoo Finance Business Dataset', + description: "Quickly read structured yahoo finance business data.\nRequires a valid yahoo finance business URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lmrpz3vxmz972ghd7', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_youtube_comments.ts b/apps/sim/tools/brightdata/dataset_youtube_comments.ts new file mode 100644 index 0000000000..99ffa39aa7 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_youtube_comments.ts @@ -0,0 +1,80 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Youtube Comments dataset tool. + */ +export const datasetYoutubeCommentsTool: ToolConfig = { + id: 'brightdata_dataset_youtube_comments', + name: 'Bright Data Youtube Comments Dataset', + description: "Quickly read structured youtube comments data.\nRequires a valid youtube video URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + num_of_comments: { + type: 'string', + required: false, + visibility: 'user-or-llm', + description: 'Number of comments to fetch (default: 10)', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lk9q0ew71spt1mxywf', + apiToken: params.apiToken, + url: params.url, + num_of_comments: params.num_of_comments, + } + + if (body.num_of_comments === undefined) { + body.num_of_comments = '10' + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_youtube_profiles.ts b/apps/sim/tools/brightdata/dataset_youtube_profiles.ts new file mode 100644 index 0000000000..df0c988e1e --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_youtube_profiles.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Youtube Profiles dataset tool. + */ +export const datasetYoutubeProfilesTool: ToolConfig = { + id: 'brightdata_dataset_youtube_profiles', + name: 'Bright Data Youtube Profiles Dataset', + description: "Quickly read structured youtube profiles data.\nRequires a valid youtube profile URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lk538t2k2p1k3oos71', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_youtube_videos.ts b/apps/sim/tools/brightdata/dataset_youtube_videos.ts new file mode 100644 index 0000000000..0e11bfbe9c --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_youtube_videos.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Youtube Videos dataset tool. + */ +export const datasetYoutubeVideosTool: ToolConfig = { + id: 'brightdata_dataset_youtube_videos', + name: 'Bright Data Youtube Videos Dataset', + description: "Quickly read structured YouTube videos data.\nRequires a valid YouTube video URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lk56epmy2i5g7lzu0k', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_zara_products.ts b/apps/sim/tools/brightdata/dataset_zara_products.ts new file mode 100644 index 0000000000..80267437a9 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_zara_products.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Zara Products dataset tool. + */ +export const datasetZaraProductsTool: ToolConfig = { + id: 'brightdata_dataset_zara_products', + name: 'Bright Data Zara Products Dataset', + description: "Quickly read structured zara product data.\nRequires a valid zara product URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lct4vafw1tgx27d4o0', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_zillow_properties_listing.ts b/apps/sim/tools/brightdata/dataset_zillow_properties_listing.ts new file mode 100644 index 0000000000..3e31b309bb --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_zillow_properties_listing.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Zillow Properties Listing dataset tool. + */ +export const datasetZillowPropertiesListingTool: ToolConfig = { + id: 'brightdata_dataset_zillow_properties_listing', + name: 'Bright Data Zillow Properties Listing Dataset', + description: "Quickly read structured zillow properties listing data.\nRequires a valid zillow properties listing URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_lfqkr8wm13ixtbd8f5', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/dataset_zoominfo_company_profile.ts b/apps/sim/tools/brightdata/dataset_zoominfo_company_profile.ts new file mode 100644 index 0000000000..e931b3d810 --- /dev/null +++ b/apps/sim/tools/brightdata/dataset_zoominfo_company_profile.ts @@ -0,0 +1,69 @@ +import type { DatasetParams, DatasetResponse } from '@/tools/brightdata/types' +import type { ToolConfig } from '@/tools/types' + +/** + * Bright Data Zoominfo Company Profile dataset tool. + */ +export const datasetZoominfoCompanyProfileTool: ToolConfig = { + id: 'brightdata_dataset_zoominfo_company_profile', + name: 'Bright Data Zoominfo Company Profile Dataset', + description: "Quickly read structured ZoomInfo company profile data.\nRequires a valid ZoomInfo company URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + version: '1.0.0', + + params: { + url: { + type: 'string', + required: true, + visibility: 'user-or-llm', + description: 'Dataset input URL', + }, + apiToken: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'Bright Data API token', + }, + }, + + request: { + method: 'POST', + url: '/api/tools/brightdata/dataset', + headers: () => ({ + 'Content-Type': 'application/json', + }), + body: (params) => { + const body: Record = { + datasetId: 'gd_m0ci4a4ivx3j5l6nx', + apiToken: params.apiToken, + url: params.url, + } + + return body + }, + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + + if (!response.ok) { + throw new Error(data.error || 'Bright Data dataset fetch failed') + } + + return { + success: true, + output: data, + } + }, + + outputs: { + data: { + type: 'object', + description: 'Structured dataset response', + }, + snapshot_at: { + type: 'string', + description: 'Timestamp of data snapshot', + optional: true, + }, + }, +} diff --git a/apps/sim/tools/brightdata/index.ts b/apps/sim/tools/brightdata/index.ts index b3f50ffc32..f18238e47b 100644 --- a/apps/sim/tools/brightdata/index.ts +++ b/apps/sim/tools/brightdata/index.ts @@ -1,4 +1,48 @@ export { scrapeMarkdownTool } from './scrape_markdown' export { searchEngineTool } from './search_engine' export { datasetAmazonProductTool } from './dataset_amazon_product' +export { datasetAmazonProductReviewsTool } from './dataset_amazon_product_reviews' +export { datasetAmazonProductSearchTool } from './dataset_amazon_product_search' +export { datasetWalmartProductTool } from './dataset_walmart_product' +export { datasetWalmartSellerTool } from './dataset_walmart_seller' +export { datasetEbayProductTool } from './dataset_ebay_product' +export { datasetHomedepotProductsTool } from './dataset_homedepot_products' +export { datasetZaraProductsTool } from './dataset_zara_products' +export { datasetEtsyProductsTool } from './dataset_etsy_products' +export { datasetBestbuyProductsTool } from './dataset_bestbuy_products' +export { datasetLinkedinPersonProfileTool } from './dataset_linkedin_person_profile' +export { datasetLinkedinCompanyProfileTool } from './dataset_linkedin_company_profile' +export { datasetLinkedinJobListingsTool } from './dataset_linkedin_job_listings' +export { datasetLinkedinPostsTool } from './dataset_linkedin_posts' +export { datasetLinkedinPeopleSearchTool } from './dataset_linkedin_people_search' +export { datasetCrunchbaseCompanyTool } from './dataset_crunchbase_company' +export { datasetZoominfoCompanyProfileTool } from './dataset_zoominfo_company_profile' +export { datasetInstagramProfilesTool } from './dataset_instagram_profiles' +export { datasetInstagramPostsTool } from './dataset_instagram_posts' +export { datasetInstagramReelsTool } from './dataset_instagram_reels' +export { datasetInstagramCommentsTool } from './dataset_instagram_comments' +export { datasetFacebookPostsTool } from './dataset_facebook_posts' +export { datasetFacebookMarketplaceListingsTool } from './dataset_facebook_marketplace_listings' +export { datasetFacebookCompanyReviewsTool } from './dataset_facebook_company_reviews' +export { datasetFacebookEventsTool } from './dataset_facebook_events' +export { datasetTiktokProfilesTool } from './dataset_tiktok_profiles' +export { datasetTiktokPostsTool } from './dataset_tiktok_posts' +export { datasetTiktokShopTool } from './dataset_tiktok_shop' +export { datasetTiktokCommentsTool } from './dataset_tiktok_comments' +export { datasetGoogleMapsReviewsTool } from './dataset_google_maps_reviews' +export { datasetGoogleShoppingTool } from './dataset_google_shopping' +export { datasetGooglePlayStoreTool } from './dataset_google_play_store' +export { datasetAppleAppStoreTool } from './dataset_apple_app_store' +export { datasetReuterNewsTool } from './dataset_reuter_news' +export { datasetGithubRepositoryFileTool } from './dataset_github_repository_file' +export { datasetYahooFinanceBusinessTool } from './dataset_yahoo_finance_business' +export { datasetXPostsTool } from './dataset_x_posts' +export { datasetZillowPropertiesListingTool } from './dataset_zillow_properties_listing' +export { datasetBookingHotelListingsTool } from './dataset_booking_hotel_listings' +export { datasetYoutubeProfilesTool } from './dataset_youtube_profiles' +export { datasetYoutubeCommentsTool } from './dataset_youtube_comments' +export { datasetRedditPostsTool } from './dataset_reddit_posts' +export { datasetYoutubeVideosTool } from './dataset_youtube_videos' +export { datasetNpmPackageTool } from './dataset_npm_package' +export { datasetPypiPackageTool } from './dataset_pypi_package' export * from './types' diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index 744e22f1d5..27f2d7c5cc 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -52,9 +52,53 @@ import { asanaUpdateTaskTool, } from '@/tools/asana' import { - datasetAmazonProductTool, scrapeMarkdownTool, searchEngineTool, + datasetAmazonProductTool, + datasetAmazonProductReviewsTool, + datasetAmazonProductSearchTool, + datasetWalmartProductTool, + datasetWalmartSellerTool, + datasetEbayProductTool, + datasetHomedepotProductsTool, + datasetZaraProductsTool, + datasetEtsyProductsTool, + datasetBestbuyProductsTool, + datasetLinkedinPersonProfileTool, + datasetLinkedinCompanyProfileTool, + datasetLinkedinJobListingsTool, + datasetLinkedinPostsTool, + datasetLinkedinPeopleSearchTool, + datasetCrunchbaseCompanyTool, + datasetZoominfoCompanyProfileTool, + datasetInstagramProfilesTool, + datasetInstagramPostsTool, + datasetInstagramReelsTool, + datasetInstagramCommentsTool, + datasetFacebookPostsTool, + datasetFacebookMarketplaceListingsTool, + datasetFacebookCompanyReviewsTool, + datasetFacebookEventsTool, + datasetTiktokProfilesTool, + datasetTiktokPostsTool, + datasetTiktokShopTool, + datasetTiktokCommentsTool, + datasetGoogleMapsReviewsTool, + datasetGoogleShoppingTool, + datasetGooglePlayStoreTool, + datasetAppleAppStoreTool, + datasetReuterNewsTool, + datasetGithubRepositoryFileTool, + datasetYahooFinanceBusinessTool, + datasetXPostsTool, + datasetZillowPropertiesListingTool, + datasetBookingHotelListingsTool, + datasetYoutubeProfilesTool, + datasetYoutubeCommentsTool, + datasetRedditPostsTool, + datasetYoutubeVideosTool, + datasetNpmPackageTool, + datasetPypiPackageTool, } from '@/tools/brightdata' import { browserUseRunTaskTool } from '@/tools/browser_use' import { @@ -1537,9 +1581,53 @@ export const tools: Record = { asana_get_projects: asanaGetProjectsTool, asana_search_tasks: asanaSearchTasksTool, asana_add_comment: asanaAddCommentTool, + brightdata_dataset_amazon_product: datasetAmazonProductTool, + brightdata_dataset_amazon_product_reviews: datasetAmazonProductReviewsTool, + brightdata_dataset_amazon_product_search: datasetAmazonProductSearchTool, + brightdata_dataset_walmart_product: datasetWalmartProductTool, + brightdata_dataset_walmart_seller: datasetWalmartSellerTool, + brightdata_dataset_ebay_product: datasetEbayProductTool, + brightdata_dataset_homedepot_products: datasetHomedepotProductsTool, + brightdata_dataset_zara_products: datasetZaraProductsTool, + brightdata_dataset_etsy_products: datasetEtsyProductsTool, + brightdata_dataset_bestbuy_products: datasetBestbuyProductsTool, + brightdata_dataset_linkedin_person_profile: datasetLinkedinPersonProfileTool, + brightdata_dataset_linkedin_company_profile: datasetLinkedinCompanyProfileTool, + brightdata_dataset_linkedin_job_listings: datasetLinkedinJobListingsTool, + brightdata_dataset_linkedin_posts: datasetLinkedinPostsTool, + brightdata_dataset_linkedin_people_search: datasetLinkedinPeopleSearchTool, + brightdata_dataset_crunchbase_company: datasetCrunchbaseCompanyTool, + brightdata_dataset_zoominfo_company_profile: datasetZoominfoCompanyProfileTool, + brightdata_dataset_instagram_profiles: datasetInstagramProfilesTool, + brightdata_dataset_instagram_posts: datasetInstagramPostsTool, + brightdata_dataset_instagram_reels: datasetInstagramReelsTool, + brightdata_dataset_instagram_comments: datasetInstagramCommentsTool, + brightdata_dataset_facebook_posts: datasetFacebookPostsTool, + brightdata_dataset_facebook_marketplace_listings: datasetFacebookMarketplaceListingsTool, + brightdata_dataset_facebook_company_reviews: datasetFacebookCompanyReviewsTool, + brightdata_dataset_facebook_events: datasetFacebookEventsTool, + brightdata_dataset_tiktok_profiles: datasetTiktokProfilesTool, + brightdata_dataset_tiktok_posts: datasetTiktokPostsTool, + brightdata_dataset_tiktok_shop: datasetTiktokShopTool, + brightdata_dataset_tiktok_comments: datasetTiktokCommentsTool, + brightdata_dataset_google_maps_reviews: datasetGoogleMapsReviewsTool, + brightdata_dataset_google_shopping: datasetGoogleShoppingTool, + brightdata_dataset_google_play_store: datasetGooglePlayStoreTool, + brightdata_dataset_apple_app_store: datasetAppleAppStoreTool, + brightdata_dataset_reuter_news: datasetReuterNewsTool, + brightdata_dataset_github_repository_file: datasetGithubRepositoryFileTool, + brightdata_dataset_yahoo_finance_business: datasetYahooFinanceBusinessTool, + brightdata_dataset_x_posts: datasetXPostsTool, + brightdata_dataset_zillow_properties_listing: datasetZillowPropertiesListingTool, + brightdata_dataset_booking_hotel_listings: datasetBookingHotelListingsTool, + brightdata_dataset_youtube_profiles: datasetYoutubeProfilesTool, + brightdata_dataset_youtube_comments: datasetYoutubeCommentsTool, + brightdata_dataset_reddit_posts: datasetRedditPostsTool, + brightdata_dataset_youtube_videos: datasetYoutubeVideosTool, + brightdata_dataset_npm_package: datasetNpmPackageTool, + brightdata_dataset_pypi_package: datasetPypiPackageTool, brightdata_scrape_markdown: scrapeMarkdownTool, brightdata_search_engine: searchEngineTool, - brightdata_dataset_amazon_product: datasetAmazonProductTool, browser_use_run_task: browserUseRunTaskTool, openai_embeddings: openAIEmbeddingsTool, http_request: httpRequestTool, From b3f0c9f26cf6d7bb37320facbf9eb8eec8311d65 Mon Sep 17 00:00:00 2001 From: meirk-brd Date: Wed, 14 Jan 2026 09:56:17 +0200 Subject: [PATCH 3/4] lint --- .../app/api/tools/brightdata/dataset/route.ts | 17 ++--- .../tools/brightdata/scrape-markdown/route.ts | 11 ++- .../tools/brightdata/search-engine/route.ts | 13 +++- apps/sim/blocks/blocks/brightdata.ts | 54 +++++++++++++- .../brightdata/dataset_amazon_product.ts | 3 +- .../dataset_amazon_product_reviews.ts | 3 +- .../dataset_amazon_product_search.ts | 3 +- .../brightdata/dataset_apple_app_store.ts | 3 +- .../brightdata/dataset_bestbuy_products.ts | 3 +- .../dataset_booking_hotel_listings.ts | 3 +- .../brightdata/dataset_crunchbase_company.ts | 3 +- .../tools/brightdata/dataset_ebay_product.ts | 3 +- .../tools/brightdata/dataset_etsy_products.ts | 3 +- .../dataset_facebook_company_reviews.ts | 3 +- .../brightdata/dataset_facebook_events.ts | 3 +- .../dataset_facebook_marketplace_listings.ts | 3 +- .../brightdata/dataset_facebook_posts.ts | 3 +- .../dataset_github_repository_file.ts | 3 +- .../brightdata/dataset_google_maps_reviews.ts | 3 +- .../brightdata/dataset_google_play_store.ts | 3 +- .../brightdata/dataset_google_shopping.ts | 3 +- .../brightdata/dataset_homedepot_products.ts | 3 +- .../brightdata/dataset_instagram_comments.ts | 3 +- .../brightdata/dataset_instagram_posts.ts | 3 +- .../brightdata/dataset_instagram_profiles.ts | 3 +- .../brightdata/dataset_instagram_reels.ts | 3 +- .../dataset_linkedin_company_profile.ts | 3 +- .../dataset_linkedin_job_listings.ts | 3 +- .../dataset_linkedin_people_search.ts | 3 +- .../dataset_linkedin_person_profile.ts | 3 +- .../brightdata/dataset_linkedin_posts.ts | 3 +- .../tools/brightdata/dataset_npm_package.ts | 3 +- .../tools/brightdata/dataset_pypi_package.ts | 3 +- .../tools/brightdata/dataset_reddit_posts.ts | 3 +- .../tools/brightdata/dataset_reuter_news.ts | 3 +- .../brightdata/dataset_tiktok_comments.ts | 3 +- .../tools/brightdata/dataset_tiktok_posts.ts | 3 +- .../brightdata/dataset_tiktok_profiles.ts | 3 +- .../tools/brightdata/dataset_tiktok_shop.ts | 3 +- .../brightdata/dataset_walmart_product.ts | 3 +- .../brightdata/dataset_walmart_seller.ts | 3 +- apps/sim/tools/brightdata/dataset_x_posts.ts | 3 +- .../dataset_yahoo_finance_business.ts | 3 +- .../brightdata/dataset_youtube_comments.ts | 3 +- .../brightdata/dataset_youtube_profiles.ts | 3 +- .../brightdata/dataset_youtube_videos.ts | 3 +- .../tools/brightdata/dataset_zara_products.ts | 3 +- .../dataset_zillow_properties_listing.ts | 3 +- .../dataset_zoominfo_company_profile.ts | 3 +- apps/sim/tools/brightdata/index.ts | 68 +++++++++--------- apps/sim/tools/brightdata/types.ts | 13 ---- apps/sim/tools/registry.ts | 70 +++++++++---------- apps/sim/tsconfig.brightdata.json | 10 +++ 53 files changed, 249 insertions(+), 142 deletions(-) create mode 100644 apps/sim/tsconfig.brightdata.json diff --git a/apps/sim/app/api/tools/brightdata/dataset/route.ts b/apps/sim/app/api/tools/brightdata/dataset/route.ts index 7e0c0a0c1a..13a83e3e81 100644 --- a/apps/sim/app/api/tools/brightdata/dataset/route.ts +++ b/apps/sim/app/api/tools/brightdata/dataset/route.ts @@ -19,8 +19,8 @@ export async function POST(request: Request) { } const params: Record = { ...body } - delete params.datasetId - delete params.apiToken + params.datasetId = undefined + params.apiToken = undefined logger.info(`[${requestId}] Triggering dataset`, { datasetId }) @@ -66,7 +66,9 @@ export async function POST(request: Request) { } const snapshotId = - typeof triggerPayload === 'object' && triggerPayload !== null && 'snapshot_id' in triggerPayload + typeof triggerPayload === 'object' && + triggerPayload !== null && + 'snapshot_id' in triggerPayload ? String((triggerPayload as { snapshot_id?: unknown }).snapshot_id ?? '') : '' @@ -129,7 +131,9 @@ export async function POST(request: Request) { } const status = - typeof snapshotPayload === 'object' && snapshotPayload !== null && 'status' in snapshotPayload + typeof snapshotPayload === 'object' && + snapshotPayload !== null && + 'status' in snapshotPayload ? String((snapshotPayload as { status?: unknown }).status ?? '') : '' @@ -155,10 +159,7 @@ export async function POST(request: Request) { } logger.error(`[${requestId}] Dataset snapshot timed out`, { datasetId, snapshotId }) - return NextResponse.json( - { error: 'Timeout waiting for dataset snapshot' }, - { status: 504 } - ) + return NextResponse.json({ error: 'Timeout waiting for dataset snapshot' }, { status: 504 }) } catch (error) { const message = error instanceof Error ? error.message : 'Dataset fetch failed' logger.error(`[${requestId}] Dataset fetch failed`, { error: message }) diff --git a/apps/sim/app/api/tools/brightdata/scrape-markdown/route.ts b/apps/sim/app/api/tools/brightdata/scrape-markdown/route.ts index 6874db85f0..bd7a61b688 100644 --- a/apps/sim/app/api/tools/brightdata/scrape-markdown/route.ts +++ b/apps/sim/app/api/tools/brightdata/scrape-markdown/route.ts @@ -48,9 +48,16 @@ export async function POST(request: Request) { ? String((payload as { error?: unknown }).error) : response.statusText - logger.error(`[${requestId}] Scraping failed`, { url, status: response.status, error: errorMessage }) + logger.error(`[${requestId}] Scraping failed`, { + url, + status: response.status, + error: errorMessage, + }) - return NextResponse.json({ error: errorMessage || 'Scraping failed' }, { status: response.status }) + return NextResponse.json( + { error: errorMessage || 'Scraping failed' }, + { status: response.status } + ) } const markdown = diff --git a/apps/sim/app/api/tools/brightdata/search-engine/route.ts b/apps/sim/app/api/tools/brightdata/search-engine/route.ts index c6c14834fc..f6b39551f0 100644 --- a/apps/sim/app/api/tools/brightdata/search-engine/route.ts +++ b/apps/sim/app/api/tools/brightdata/search-engine/route.ts @@ -56,9 +56,16 @@ export async function POST(request: Request) { ? String((payload as { error?: unknown }).error) : response.statusText - logger.error(`[${requestId}] Search failed`, { query, status: response.status, error: errorMessage }) - - return NextResponse.json({ error: errorMessage || 'Search failed' }, { status: response.status }) + logger.error(`[${requestId}] Search failed`, { + query, + status: response.status, + error: errorMessage, + }) + + return NextResponse.json( + { error: errorMessage || 'Search failed' }, + { status: response.status } + ) } let normalizedResults: Array<{ title: string; url: string; snippet: string }> = [] diff --git a/apps/sim/blocks/blocks/brightdata.ts b/apps/sim/blocks/blocks/brightdata.ts index 2a73d5b42b..568071bcb9 100644 --- a/apps/sim/blocks/blocks/brightdata.ts +++ b/apps/sim/blocks/blocks/brightdata.ts @@ -94,7 +94,10 @@ export const BrightDataBlock: BlockConfig = { { label: 'Instagram Reels Dataset', id: 'dataset_instagram_reels' }, { label: 'Instagram Comments Dataset', id: 'dataset_instagram_comments' }, { label: 'Facebook Posts Dataset', id: 'dataset_facebook_posts' }, - { label: 'Facebook Marketplace Listings Dataset', id: 'dataset_facebook_marketplace_listings' }, + { + label: 'Facebook Marketplace Listings Dataset', + id: 'dataset_facebook_marketplace_listings', + }, { label: 'Facebook Company Reviews Dataset', id: 'dataset_facebook_company_reviews' }, { label: 'Facebook Events Dataset', id: 'dataset_facebook_events' }, { label: 'Tiktok Profiles Dataset', id: 'dataset_tiktok_profiles' }, @@ -148,7 +151,54 @@ export const BrightDataBlock: BlockConfig = { title: 'Dataset URL', type: 'short-input', placeholder: 'https://example.com', - condition: { field: 'operation', value: ['dataset_amazon_product', 'dataset_amazon_product_reviews', 'dataset_amazon_product_search', 'dataset_walmart_product', 'dataset_walmart_seller', 'dataset_ebay_product', 'dataset_homedepot_products', 'dataset_zara_products', 'dataset_etsy_products', 'dataset_bestbuy_products', 'dataset_linkedin_person_profile', 'dataset_linkedin_company_profile', 'dataset_linkedin_job_listings', 'dataset_linkedin_posts', 'dataset_linkedin_people_search', 'dataset_crunchbase_company', 'dataset_zoominfo_company_profile', 'dataset_instagram_profiles', 'dataset_instagram_posts', 'dataset_instagram_reels', 'dataset_instagram_comments', 'dataset_facebook_posts', 'dataset_facebook_marketplace_listings', 'dataset_facebook_company_reviews', 'dataset_facebook_events', 'dataset_tiktok_profiles', 'dataset_tiktok_posts', 'dataset_tiktok_shop', 'dataset_tiktok_comments', 'dataset_google_maps_reviews', 'dataset_google_shopping', 'dataset_google_play_store', 'dataset_apple_app_store', 'dataset_reuter_news', 'dataset_github_repository_file', 'dataset_yahoo_finance_business', 'dataset_x_posts', 'dataset_zillow_properties_listing', 'dataset_booking_hotel_listings', 'dataset_youtube_profiles', 'dataset_youtube_comments', 'dataset_reddit_posts', 'dataset_youtube_videos'] }, + condition: { + field: 'operation', + value: [ + 'dataset_amazon_product', + 'dataset_amazon_product_reviews', + 'dataset_amazon_product_search', + 'dataset_walmart_product', + 'dataset_walmart_seller', + 'dataset_ebay_product', + 'dataset_homedepot_products', + 'dataset_zara_products', + 'dataset_etsy_products', + 'dataset_bestbuy_products', + 'dataset_linkedin_person_profile', + 'dataset_linkedin_company_profile', + 'dataset_linkedin_job_listings', + 'dataset_linkedin_posts', + 'dataset_linkedin_people_search', + 'dataset_crunchbase_company', + 'dataset_zoominfo_company_profile', + 'dataset_instagram_profiles', + 'dataset_instagram_posts', + 'dataset_instagram_reels', + 'dataset_instagram_comments', + 'dataset_facebook_posts', + 'dataset_facebook_marketplace_listings', + 'dataset_facebook_company_reviews', + 'dataset_facebook_events', + 'dataset_tiktok_profiles', + 'dataset_tiktok_posts', + 'dataset_tiktok_shop', + 'dataset_tiktok_comments', + 'dataset_google_maps_reviews', + 'dataset_google_shopping', + 'dataset_google_play_store', + 'dataset_apple_app_store', + 'dataset_reuter_news', + 'dataset_github_repository_file', + 'dataset_yahoo_finance_business', + 'dataset_x_posts', + 'dataset_zillow_properties_listing', + 'dataset_booking_hotel_listings', + 'dataset_youtube_profiles', + 'dataset_youtube_comments', + 'dataset_reddit_posts', + 'dataset_youtube_videos', + ], + }, required: true, }, { diff --git a/apps/sim/tools/brightdata/dataset_amazon_product.ts b/apps/sim/tools/brightdata/dataset_amazon_product.ts index cbfd808e47..b086df75b9 100644 --- a/apps/sim/tools/brightdata/dataset_amazon_product.ts +++ b/apps/sim/tools/brightdata/dataset_amazon_product.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetAmazonProductTool: ToolConfig = { id: 'brightdata_dataset_amazon_product', name: 'Bright Data Amazon Product Dataset', - description: "Quickly read structured amazon product data.\nRequires a valid product URL with /dp/ in it.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured amazon product data.\nRequires a valid product URL with /dp/ in it.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_amazon_product_reviews.ts b/apps/sim/tools/brightdata/dataset_amazon_product_reviews.ts index d20ab224dd..fca9c6a631 100644 --- a/apps/sim/tools/brightdata/dataset_amazon_product_reviews.ts +++ b/apps/sim/tools/brightdata/dataset_amazon_product_reviews.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetAmazonProductReviewsTool: ToolConfig = { id: 'brightdata_dataset_amazon_product_reviews', name: 'Bright Data Amazon Product Reviews Dataset', - description: "Quickly read structured amazon product review data.\nRequires a valid product URL with /dp/ in it.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured amazon product review data.\nRequires a valid product URL with /dp/ in it.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_amazon_product_search.ts b/apps/sim/tools/brightdata/dataset_amazon_product_search.ts index 0ba8102c98..b8b56a4d09 100644 --- a/apps/sim/tools/brightdata/dataset_amazon_product_search.ts +++ b/apps/sim/tools/brightdata/dataset_amazon_product_search.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetAmazonProductSearchTool: ToolConfig = { id: 'brightdata_dataset_amazon_product_search', name: 'Bright Data Amazon Product Search Dataset', - description: "Quickly read structured amazon product search data.\nRequires a valid search keyword and amazon domain URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured amazon product search data.\nRequires a valid search keyword and amazon domain URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_apple_app_store.ts b/apps/sim/tools/brightdata/dataset_apple_app_store.ts index 04c836d44d..9319dc0737 100644 --- a/apps/sim/tools/brightdata/dataset_apple_app_store.ts +++ b/apps/sim/tools/brightdata/dataset_apple_app_store.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetAppleAppStoreTool: ToolConfig = { id: 'brightdata_dataset_apple_app_store', name: 'Bright Data Apple App Store Dataset', - description: "Quickly read structured apple app store data.\nRequires a valid apple app store app URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured apple app store data.\nRequires a valid apple app store app URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_bestbuy_products.ts b/apps/sim/tools/brightdata/dataset_bestbuy_products.ts index acae97cbe3..8008d9828a 100644 --- a/apps/sim/tools/brightdata/dataset_bestbuy_products.ts +++ b/apps/sim/tools/brightdata/dataset_bestbuy_products.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetBestbuyProductsTool: ToolConfig = { id: 'brightdata_dataset_bestbuy_products', name: 'Bright Data Bestbuy Products Dataset', - description: "Quickly read structured bestbuy product data.\nRequires a valid bestbuy product URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured bestbuy product data.\nRequires a valid bestbuy product URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_booking_hotel_listings.ts b/apps/sim/tools/brightdata/dataset_booking_hotel_listings.ts index ec806a4d18..5dd6bb4fb9 100644 --- a/apps/sim/tools/brightdata/dataset_booking_hotel_listings.ts +++ b/apps/sim/tools/brightdata/dataset_booking_hotel_listings.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetBookingHotelListingsTool: ToolConfig = { id: 'brightdata_dataset_booking_hotel_listings', name: 'Bright Data Booking Hotel Listings Dataset', - description: "Quickly read structured booking hotel listings data.\nRequires a valid booking hotel listing URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured booking hotel listings data.\nRequires a valid booking hotel listing URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_crunchbase_company.ts b/apps/sim/tools/brightdata/dataset_crunchbase_company.ts index 8a02aaf593..f5494defdb 100644 --- a/apps/sim/tools/brightdata/dataset_crunchbase_company.ts +++ b/apps/sim/tools/brightdata/dataset_crunchbase_company.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetCrunchbaseCompanyTool: ToolConfig = { id: 'brightdata_dataset_crunchbase_company', name: 'Bright Data Crunchbase Company Dataset', - description: "Quickly read structured crunchbase company data\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured crunchbase company data\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_ebay_product.ts b/apps/sim/tools/brightdata/dataset_ebay_product.ts index 0e9e891c0a..9031c4f92b 100644 --- a/apps/sim/tools/brightdata/dataset_ebay_product.ts +++ b/apps/sim/tools/brightdata/dataset_ebay_product.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetEbayProductTool: ToolConfig = { id: 'brightdata_dataset_ebay_product', name: 'Bright Data Ebay Product Dataset', - description: "Quickly read structured ebay product data.\nRequires a valid ebay product URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured ebay product data.\nRequires a valid ebay product URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_etsy_products.ts b/apps/sim/tools/brightdata/dataset_etsy_products.ts index c484c757d0..6dbfadcb1b 100644 --- a/apps/sim/tools/brightdata/dataset_etsy_products.ts +++ b/apps/sim/tools/brightdata/dataset_etsy_products.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetEtsyProductsTool: ToolConfig = { id: 'brightdata_dataset_etsy_products', name: 'Bright Data Etsy Products Dataset', - description: "Quickly read structured etsy product data.\nRequires a valid etsy product URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured etsy product data.\nRequires a valid etsy product URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_facebook_company_reviews.ts b/apps/sim/tools/brightdata/dataset_facebook_company_reviews.ts index 671f933fe1..bb463d414f 100644 --- a/apps/sim/tools/brightdata/dataset_facebook_company_reviews.ts +++ b/apps/sim/tools/brightdata/dataset_facebook_company_reviews.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetFacebookCompanyReviewsTool: ToolConfig = { id: 'brightdata_dataset_facebook_company_reviews', name: 'Bright Data Facebook Company Reviews Dataset', - description: "Quickly read structured Facebook company reviews data.\nRequires a valid Facebook company URL and number of reviews.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured Facebook company reviews data.\nRequires a valid Facebook company URL and number of reviews.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_facebook_events.ts b/apps/sim/tools/brightdata/dataset_facebook_events.ts index 073bc45e05..b6fa6fa7a2 100644 --- a/apps/sim/tools/brightdata/dataset_facebook_events.ts +++ b/apps/sim/tools/brightdata/dataset_facebook_events.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetFacebookEventsTool: ToolConfig = { id: 'brightdata_dataset_facebook_events', name: 'Bright Data Facebook Events Dataset', - description: "Quickly read structured Facebook events data.\nRequires a valid Facebook event URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured Facebook events data.\nRequires a valid Facebook event URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_facebook_marketplace_listings.ts b/apps/sim/tools/brightdata/dataset_facebook_marketplace_listings.ts index 0e6f2be719..d680e37334 100644 --- a/apps/sim/tools/brightdata/dataset_facebook_marketplace_listings.ts +++ b/apps/sim/tools/brightdata/dataset_facebook_marketplace_listings.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetFacebookMarketplaceListingsTool: ToolConfig = { id: 'brightdata_dataset_facebook_marketplace_listings', name: 'Bright Data Facebook Marketplace Listings Dataset', - description: "Quickly read structured Facebook marketplace listing data.\nRequires a valid Facebook marketplace listing URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured Facebook marketplace listing data.\nRequires a valid Facebook marketplace listing URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_facebook_posts.ts b/apps/sim/tools/brightdata/dataset_facebook_posts.ts index 933e1e8264..8dc8fcadb7 100644 --- a/apps/sim/tools/brightdata/dataset_facebook_posts.ts +++ b/apps/sim/tools/brightdata/dataset_facebook_posts.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetFacebookPostsTool: ToolConfig = { id: 'brightdata_dataset_facebook_posts', name: 'Bright Data Facebook Posts Dataset', - description: "Quickly read structured Facebook post data.\nRequires a valid Facebook post URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured Facebook post data.\nRequires a valid Facebook post URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_github_repository_file.ts b/apps/sim/tools/brightdata/dataset_github_repository_file.ts index 1797ef76ac..6e8d86fff1 100644 --- a/apps/sim/tools/brightdata/dataset_github_repository_file.ts +++ b/apps/sim/tools/brightdata/dataset_github_repository_file.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetGithubRepositoryFileTool: ToolConfig = { id: 'brightdata_dataset_github_repository_file', name: 'Bright Data Github Repository File Dataset', - description: "Quickly read structured github repository data.\nRequires a valid github repository file URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured github repository data.\nRequires a valid github repository file URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_google_maps_reviews.ts b/apps/sim/tools/brightdata/dataset_google_maps_reviews.ts index bca3e9ae83..84b9a10b1f 100644 --- a/apps/sim/tools/brightdata/dataset_google_maps_reviews.ts +++ b/apps/sim/tools/brightdata/dataset_google_maps_reviews.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetGoogleMapsReviewsTool: ToolConfig = { id: 'brightdata_dataset_google_maps_reviews', name: 'Bright Data Google Maps Reviews Dataset', - description: "Quickly read structured Google maps reviews data.\nRequires a valid Google maps URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured Google maps reviews data.\nRequires a valid Google maps URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_google_play_store.ts b/apps/sim/tools/brightdata/dataset_google_play_store.ts index 01c257471f..c86600a60d 100644 --- a/apps/sim/tools/brightdata/dataset_google_play_store.ts +++ b/apps/sim/tools/brightdata/dataset_google_play_store.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetGooglePlayStoreTool: ToolConfig = { id: 'brightdata_dataset_google_play_store', name: 'Bright Data Google Play Store Dataset', - description: "Quickly read structured Google play store data.\nRequires a valid Google play store app URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured Google play store data.\nRequires a valid Google play store app URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_google_shopping.ts b/apps/sim/tools/brightdata/dataset_google_shopping.ts index 4aa86fa6d9..c0c4a11c9d 100644 --- a/apps/sim/tools/brightdata/dataset_google_shopping.ts +++ b/apps/sim/tools/brightdata/dataset_google_shopping.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetGoogleShoppingTool: ToolConfig = { id: 'brightdata_dataset_google_shopping', name: 'Bright Data Google Shopping Dataset', - description: "Quickly read structured Google shopping data.\nRequires a valid Google shopping product URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured Google shopping data.\nRequires a valid Google shopping product URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_homedepot_products.ts b/apps/sim/tools/brightdata/dataset_homedepot_products.ts index 2831aebc87..cc94739ee1 100644 --- a/apps/sim/tools/brightdata/dataset_homedepot_products.ts +++ b/apps/sim/tools/brightdata/dataset_homedepot_products.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetHomedepotProductsTool: ToolConfig = { id: 'brightdata_dataset_homedepot_products', name: 'Bright Data Homedepot Products Dataset', - description: "Quickly read structured homedepot product data.\nRequires a valid homedepot product URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured homedepot product data.\nRequires a valid homedepot product URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_instagram_comments.ts b/apps/sim/tools/brightdata/dataset_instagram_comments.ts index 61892c6aa8..4944c65f20 100644 --- a/apps/sim/tools/brightdata/dataset_instagram_comments.ts +++ b/apps/sim/tools/brightdata/dataset_instagram_comments.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetInstagramCommentsTool: ToolConfig = { id: 'brightdata_dataset_instagram_comments', name: 'Bright Data Instagram Comments Dataset', - description: "Quickly read structured Instagram comments data.\nRequires a valid Instagram URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured Instagram comments data.\nRequires a valid Instagram URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_instagram_posts.ts b/apps/sim/tools/brightdata/dataset_instagram_posts.ts index 67af5930ac..23006fcf4f 100644 --- a/apps/sim/tools/brightdata/dataset_instagram_posts.ts +++ b/apps/sim/tools/brightdata/dataset_instagram_posts.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetInstagramPostsTool: ToolConfig = { id: 'brightdata_dataset_instagram_posts', name: 'Bright Data Instagram Posts Dataset', - description: "Quickly read structured Instagram post data.\nRequires a valid Instagram URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured Instagram post data.\nRequires a valid Instagram URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_instagram_profiles.ts b/apps/sim/tools/brightdata/dataset_instagram_profiles.ts index 5d5ea2d316..af246beea0 100644 --- a/apps/sim/tools/brightdata/dataset_instagram_profiles.ts +++ b/apps/sim/tools/brightdata/dataset_instagram_profiles.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetInstagramProfilesTool: ToolConfig = { id: 'brightdata_dataset_instagram_profiles', name: 'Bright Data Instagram Profiles Dataset', - description: "Quickly read structured Instagram profile data.\nRequires a valid Instagram URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured Instagram profile data.\nRequires a valid Instagram URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_instagram_reels.ts b/apps/sim/tools/brightdata/dataset_instagram_reels.ts index bd4e03ea28..9992453a45 100644 --- a/apps/sim/tools/brightdata/dataset_instagram_reels.ts +++ b/apps/sim/tools/brightdata/dataset_instagram_reels.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetInstagramReelsTool: ToolConfig = { id: 'brightdata_dataset_instagram_reels', name: 'Bright Data Instagram Reels Dataset', - description: "Quickly read structured Instagram reel data.\nRequires a valid Instagram URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured Instagram reel data.\nRequires a valid Instagram URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_linkedin_company_profile.ts b/apps/sim/tools/brightdata/dataset_linkedin_company_profile.ts index 7274454061..091ae634bb 100644 --- a/apps/sim/tools/brightdata/dataset_linkedin_company_profile.ts +++ b/apps/sim/tools/brightdata/dataset_linkedin_company_profile.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetLinkedinCompanyProfileTool: ToolConfig = { id: 'brightdata_dataset_linkedin_company_profile', name: 'Bright Data Linkedin Company Profile Dataset', - description: "Quickly read structured linkedin company profile data\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured linkedin company profile data\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_linkedin_job_listings.ts b/apps/sim/tools/brightdata/dataset_linkedin_job_listings.ts index f5964174e5..b5af2070c4 100644 --- a/apps/sim/tools/brightdata/dataset_linkedin_job_listings.ts +++ b/apps/sim/tools/brightdata/dataset_linkedin_job_listings.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetLinkedinJobListingsTool: ToolConfig = { id: 'brightdata_dataset_linkedin_job_listings', name: 'Bright Data Linkedin Job Listings Dataset', - description: "Quickly read structured linkedin job listings data\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured linkedin job listings data\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_linkedin_people_search.ts b/apps/sim/tools/brightdata/dataset_linkedin_people_search.ts index 8c514f671c..8cefdc32c5 100644 --- a/apps/sim/tools/brightdata/dataset_linkedin_people_search.ts +++ b/apps/sim/tools/brightdata/dataset_linkedin_people_search.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetLinkedinPeopleSearchTool: ToolConfig = { id: 'brightdata_dataset_linkedin_people_search', name: 'Bright Data Linkedin People Search Dataset', - description: "Quickly read structured linkedin people search data\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured linkedin people search data\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_linkedin_person_profile.ts b/apps/sim/tools/brightdata/dataset_linkedin_person_profile.ts index a8b3fa4823..937b50a2ef 100644 --- a/apps/sim/tools/brightdata/dataset_linkedin_person_profile.ts +++ b/apps/sim/tools/brightdata/dataset_linkedin_person_profile.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetLinkedinPersonProfileTool: ToolConfig = { id: 'brightdata_dataset_linkedin_person_profile', name: 'Bright Data Linkedin Person Profile Dataset', - description: "Quickly read structured linkedin people profile data.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured linkedin people profile data.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_linkedin_posts.ts b/apps/sim/tools/brightdata/dataset_linkedin_posts.ts index d15b199cde..027f9f2f3a 100644 --- a/apps/sim/tools/brightdata/dataset_linkedin_posts.ts +++ b/apps/sim/tools/brightdata/dataset_linkedin_posts.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetLinkedinPostsTool: ToolConfig = { id: 'brightdata_dataset_linkedin_posts', name: 'Bright Data Linkedin Posts Dataset', - description: "Quickly read structured linkedin posts data\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured linkedin posts data\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_npm_package.ts b/apps/sim/tools/brightdata/dataset_npm_package.ts index 9fbb58cdfb..f77a964078 100644 --- a/apps/sim/tools/brightdata/dataset_npm_package.ts +++ b/apps/sim/tools/brightdata/dataset_npm_package.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetNpmPackageTool: ToolConfig = { id: 'brightdata_dataset_npm_package', name: 'Bright Data Npm Package Dataset', - description: "Quickly read structured npm package data.\nRequires a valid npm package name (e.g., @brightdata/sdk).\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured npm package data.\nRequires a valid npm package name (e.g., @brightdata/sdk).\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_pypi_package.ts b/apps/sim/tools/brightdata/dataset_pypi_package.ts index b511e32961..476269ab60 100644 --- a/apps/sim/tools/brightdata/dataset_pypi_package.ts +++ b/apps/sim/tools/brightdata/dataset_pypi_package.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetPypiPackageTool: ToolConfig = { id: 'brightdata_dataset_pypi_package', name: 'Bright Data Pypi Package Dataset', - description: "Quickly read structured PyPI package data.\nRequires a valid PyPI package name (e.g., langchain-brightdata).\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured PyPI package data.\nRequires a valid PyPI package name (e.g., langchain-brightdata).\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_reddit_posts.ts b/apps/sim/tools/brightdata/dataset_reddit_posts.ts index 1fc3f26813..6758fd4463 100644 --- a/apps/sim/tools/brightdata/dataset_reddit_posts.ts +++ b/apps/sim/tools/brightdata/dataset_reddit_posts.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetRedditPostsTool: ToolConfig = { id: 'brightdata_dataset_reddit_posts', name: 'Bright Data Reddit Posts Dataset', - description: "Quickly read structured reddit posts data.\nRequires a valid reddit post URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured reddit posts data.\nRequires a valid reddit post URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_reuter_news.ts b/apps/sim/tools/brightdata/dataset_reuter_news.ts index 4a6f52a110..fdb1072640 100644 --- a/apps/sim/tools/brightdata/dataset_reuter_news.ts +++ b/apps/sim/tools/brightdata/dataset_reuter_news.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetReuterNewsTool: ToolConfig = { id: 'brightdata_dataset_reuter_news', name: 'Bright Data Reuter News Dataset', - description: "Quickly read structured reuter news data.\nRequires a valid reuter news report URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured reuter news data.\nRequires a valid reuter news report URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_tiktok_comments.ts b/apps/sim/tools/brightdata/dataset_tiktok_comments.ts index 99704f8fa6..69c216d37b 100644 --- a/apps/sim/tools/brightdata/dataset_tiktok_comments.ts +++ b/apps/sim/tools/brightdata/dataset_tiktok_comments.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetTiktokCommentsTool: ToolConfig = { id: 'brightdata_dataset_tiktok_comments', name: 'Bright Data Tiktok Comments Dataset', - description: "Quickly read structured Tiktok comments data.\nRequires a valid Tiktok video URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured Tiktok comments data.\nRequires a valid Tiktok video URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_tiktok_posts.ts b/apps/sim/tools/brightdata/dataset_tiktok_posts.ts index 73f57ae184..0a91c773ad 100644 --- a/apps/sim/tools/brightdata/dataset_tiktok_posts.ts +++ b/apps/sim/tools/brightdata/dataset_tiktok_posts.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetTiktokPostsTool: ToolConfig = { id: 'brightdata_dataset_tiktok_posts', name: 'Bright Data Tiktok Posts Dataset', - description: "Quickly read structured Tiktok post data.\nRequires a valid Tiktok post URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured Tiktok post data.\nRequires a valid Tiktok post URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_tiktok_profiles.ts b/apps/sim/tools/brightdata/dataset_tiktok_profiles.ts index e44a13c8b4..ec8ab5e004 100644 --- a/apps/sim/tools/brightdata/dataset_tiktok_profiles.ts +++ b/apps/sim/tools/brightdata/dataset_tiktok_profiles.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetTiktokProfilesTool: ToolConfig = { id: 'brightdata_dataset_tiktok_profiles', name: 'Bright Data Tiktok Profiles Dataset', - description: "Quickly read structured Tiktok profiles data.\nRequires a valid Tiktok profile URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured Tiktok profiles data.\nRequires a valid Tiktok profile URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_tiktok_shop.ts b/apps/sim/tools/brightdata/dataset_tiktok_shop.ts index 0b07724fa2..26ebf71066 100644 --- a/apps/sim/tools/brightdata/dataset_tiktok_shop.ts +++ b/apps/sim/tools/brightdata/dataset_tiktok_shop.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetTiktokShopTool: ToolConfig = { id: 'brightdata_dataset_tiktok_shop', name: 'Bright Data Tiktok Shop Dataset', - description: "Quickly read structured Tiktok shop data.\nRequires a valid Tiktok shop product URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured Tiktok shop data.\nRequires a valid Tiktok shop product URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_walmart_product.ts b/apps/sim/tools/brightdata/dataset_walmart_product.ts index d5de7b00cc..d5bc38aad7 100644 --- a/apps/sim/tools/brightdata/dataset_walmart_product.ts +++ b/apps/sim/tools/brightdata/dataset_walmart_product.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetWalmartProductTool: ToolConfig = { id: 'brightdata_dataset_walmart_product', name: 'Bright Data Walmart Product Dataset', - description: "Quickly read structured walmart product data.\nRequires a valid product URL with /ip/ in it.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured walmart product data.\nRequires a valid product URL with /ip/ in it.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_walmart_seller.ts b/apps/sim/tools/brightdata/dataset_walmart_seller.ts index 53ae740d16..b4bb24259d 100644 --- a/apps/sim/tools/brightdata/dataset_walmart_seller.ts +++ b/apps/sim/tools/brightdata/dataset_walmart_seller.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetWalmartSellerTool: ToolConfig = { id: 'brightdata_dataset_walmart_seller', name: 'Bright Data Walmart Seller Dataset', - description: "Quickly read structured walmart seller data.\nRequires a valid walmart seller URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured walmart seller data.\nRequires a valid walmart seller URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_x_posts.ts b/apps/sim/tools/brightdata/dataset_x_posts.ts index ac88fd9f56..14a218965e 100644 --- a/apps/sim/tools/brightdata/dataset_x_posts.ts +++ b/apps/sim/tools/brightdata/dataset_x_posts.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetXPostsTool: ToolConfig = { id: 'brightdata_dataset_x_posts', name: 'Bright Data X Posts Dataset', - description: "Quickly read structured X post data.\nRequires a valid X post URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured X post data.\nRequires a valid X post URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_yahoo_finance_business.ts b/apps/sim/tools/brightdata/dataset_yahoo_finance_business.ts index 0a8bfc143b..f3ec1c08a6 100644 --- a/apps/sim/tools/brightdata/dataset_yahoo_finance_business.ts +++ b/apps/sim/tools/brightdata/dataset_yahoo_finance_business.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetYahooFinanceBusinessTool: ToolConfig = { id: 'brightdata_dataset_yahoo_finance_business', name: 'Bright Data Yahoo Finance Business Dataset', - description: "Quickly read structured yahoo finance business data.\nRequires a valid yahoo finance business URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured yahoo finance business data.\nRequires a valid yahoo finance business URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_youtube_comments.ts b/apps/sim/tools/brightdata/dataset_youtube_comments.ts index 99ffa39aa7..77c06428c3 100644 --- a/apps/sim/tools/brightdata/dataset_youtube_comments.ts +++ b/apps/sim/tools/brightdata/dataset_youtube_comments.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetYoutubeCommentsTool: ToolConfig = { id: 'brightdata_dataset_youtube_comments', name: 'Bright Data Youtube Comments Dataset', - description: "Quickly read structured youtube comments data.\nRequires a valid youtube video URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured youtube comments data.\nRequires a valid youtube video URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_youtube_profiles.ts b/apps/sim/tools/brightdata/dataset_youtube_profiles.ts index df0c988e1e..192b246406 100644 --- a/apps/sim/tools/brightdata/dataset_youtube_profiles.ts +++ b/apps/sim/tools/brightdata/dataset_youtube_profiles.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetYoutubeProfilesTool: ToolConfig = { id: 'brightdata_dataset_youtube_profiles', name: 'Bright Data Youtube Profiles Dataset', - description: "Quickly read structured youtube profiles data.\nRequires a valid youtube profile URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured youtube profiles data.\nRequires a valid youtube profile URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_youtube_videos.ts b/apps/sim/tools/brightdata/dataset_youtube_videos.ts index 0e11bfbe9c..5308bdcfe0 100644 --- a/apps/sim/tools/brightdata/dataset_youtube_videos.ts +++ b/apps/sim/tools/brightdata/dataset_youtube_videos.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetYoutubeVideosTool: ToolConfig = { id: 'brightdata_dataset_youtube_videos', name: 'Bright Data Youtube Videos Dataset', - description: "Quickly read structured YouTube videos data.\nRequires a valid YouTube video URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured YouTube videos data.\nRequires a valid YouTube video URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_zara_products.ts b/apps/sim/tools/brightdata/dataset_zara_products.ts index 80267437a9..536e633c2e 100644 --- a/apps/sim/tools/brightdata/dataset_zara_products.ts +++ b/apps/sim/tools/brightdata/dataset_zara_products.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetZaraProductsTool: ToolConfig = { id: 'brightdata_dataset_zara_products', name: 'Bright Data Zara Products Dataset', - description: "Quickly read structured zara product data.\nRequires a valid zara product URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured zara product data.\nRequires a valid zara product URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_zillow_properties_listing.ts b/apps/sim/tools/brightdata/dataset_zillow_properties_listing.ts index 3e31b309bb..90aa09a0f5 100644 --- a/apps/sim/tools/brightdata/dataset_zillow_properties_listing.ts +++ b/apps/sim/tools/brightdata/dataset_zillow_properties_listing.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetZillowPropertiesListingTool: ToolConfig = { id: 'brightdata_dataset_zillow_properties_listing', name: 'Bright Data Zillow Properties Listing Dataset', - description: "Quickly read structured zillow properties listing data.\nRequires a valid zillow properties listing URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured zillow properties listing data.\nRequires a valid zillow properties listing URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/dataset_zoominfo_company_profile.ts b/apps/sim/tools/brightdata/dataset_zoominfo_company_profile.ts index e931b3d810..c1cae34157 100644 --- a/apps/sim/tools/brightdata/dataset_zoominfo_company_profile.ts +++ b/apps/sim/tools/brightdata/dataset_zoominfo_company_profile.ts @@ -7,7 +7,8 @@ import type { ToolConfig } from '@/tools/types' export const datasetZoominfoCompanyProfileTool: ToolConfig = { id: 'brightdata_dataset_zoominfo_company_profile', name: 'Bright Data Zoominfo Company Profile Dataset', - description: "Quickly read structured ZoomInfo company profile data.\nRequires a valid ZoomInfo company URL.\nThis can be a cache lookup, so it can be more reliable than scraping", + description: + 'Quickly read structured ZoomInfo company profile data.\nRequires a valid ZoomInfo company URL.\nThis can be a cache lookup, so it can be more reliable than scraping', version: '1.0.0', params: { diff --git a/apps/sim/tools/brightdata/index.ts b/apps/sim/tools/brightdata/index.ts index f18238e47b..ced9f57611 100644 --- a/apps/sim/tools/brightdata/index.ts +++ b/apps/sim/tools/brightdata/index.ts @@ -1,48 +1,48 @@ -export { scrapeMarkdownTool } from './scrape_markdown' -export { searchEngineTool } from './search_engine' export { datasetAmazonProductTool } from './dataset_amazon_product' export { datasetAmazonProductReviewsTool } from './dataset_amazon_product_reviews' export { datasetAmazonProductSearchTool } from './dataset_amazon_product_search' -export { datasetWalmartProductTool } from './dataset_walmart_product' -export { datasetWalmartSellerTool } from './dataset_walmart_seller' -export { datasetEbayProductTool } from './dataset_ebay_product' -export { datasetHomedepotProductsTool } from './dataset_homedepot_products' -export { datasetZaraProductsTool } from './dataset_zara_products' -export { datasetEtsyProductsTool } from './dataset_etsy_products' +export { datasetAppleAppStoreTool } from './dataset_apple_app_store' export { datasetBestbuyProductsTool } from './dataset_bestbuy_products' -export { datasetLinkedinPersonProfileTool } from './dataset_linkedin_person_profile' -export { datasetLinkedinCompanyProfileTool } from './dataset_linkedin_company_profile' -export { datasetLinkedinJobListingsTool } from './dataset_linkedin_job_listings' -export { datasetLinkedinPostsTool } from './dataset_linkedin_posts' -export { datasetLinkedinPeopleSearchTool } from './dataset_linkedin_people_search' +export { datasetBookingHotelListingsTool } from './dataset_booking_hotel_listings' export { datasetCrunchbaseCompanyTool } from './dataset_crunchbase_company' -export { datasetZoominfoCompanyProfileTool } from './dataset_zoominfo_company_profile' -export { datasetInstagramProfilesTool } from './dataset_instagram_profiles' -export { datasetInstagramPostsTool } from './dataset_instagram_posts' -export { datasetInstagramReelsTool } from './dataset_instagram_reels' -export { datasetInstagramCommentsTool } from './dataset_instagram_comments' -export { datasetFacebookPostsTool } from './dataset_facebook_posts' -export { datasetFacebookMarketplaceListingsTool } from './dataset_facebook_marketplace_listings' +export { datasetEbayProductTool } from './dataset_ebay_product' +export { datasetEtsyProductsTool } from './dataset_etsy_products' export { datasetFacebookCompanyReviewsTool } from './dataset_facebook_company_reviews' export { datasetFacebookEventsTool } from './dataset_facebook_events' -export { datasetTiktokProfilesTool } from './dataset_tiktok_profiles' -export { datasetTiktokPostsTool } from './dataset_tiktok_posts' -export { datasetTiktokShopTool } from './dataset_tiktok_shop' -export { datasetTiktokCommentsTool } from './dataset_tiktok_comments' +export { datasetFacebookMarketplaceListingsTool } from './dataset_facebook_marketplace_listings' +export { datasetFacebookPostsTool } from './dataset_facebook_posts' +export { datasetGithubRepositoryFileTool } from './dataset_github_repository_file' export { datasetGoogleMapsReviewsTool } from './dataset_google_maps_reviews' -export { datasetGoogleShoppingTool } from './dataset_google_shopping' export { datasetGooglePlayStoreTool } from './dataset_google_play_store' -export { datasetAppleAppStoreTool } from './dataset_apple_app_store' +export { datasetGoogleShoppingTool } from './dataset_google_shopping' +export { datasetHomedepotProductsTool } from './dataset_homedepot_products' +export { datasetInstagramCommentsTool } from './dataset_instagram_comments' +export { datasetInstagramPostsTool } from './dataset_instagram_posts' +export { datasetInstagramProfilesTool } from './dataset_instagram_profiles' +export { datasetInstagramReelsTool } from './dataset_instagram_reels' +export { datasetLinkedinCompanyProfileTool } from './dataset_linkedin_company_profile' +export { datasetLinkedinJobListingsTool } from './dataset_linkedin_job_listings' +export { datasetLinkedinPeopleSearchTool } from './dataset_linkedin_people_search' +export { datasetLinkedinPersonProfileTool } from './dataset_linkedin_person_profile' +export { datasetLinkedinPostsTool } from './dataset_linkedin_posts' +export { datasetNpmPackageTool } from './dataset_npm_package' +export { datasetPypiPackageTool } from './dataset_pypi_package' +export { datasetRedditPostsTool } from './dataset_reddit_posts' export { datasetReuterNewsTool } from './dataset_reuter_news' -export { datasetGithubRepositoryFileTool } from './dataset_github_repository_file' -export { datasetYahooFinanceBusinessTool } from './dataset_yahoo_finance_business' +export { datasetTiktokCommentsTool } from './dataset_tiktok_comments' +export { datasetTiktokPostsTool } from './dataset_tiktok_posts' +export { datasetTiktokProfilesTool } from './dataset_tiktok_profiles' +export { datasetTiktokShopTool } from './dataset_tiktok_shop' +export { datasetWalmartProductTool } from './dataset_walmart_product' +export { datasetWalmartSellerTool } from './dataset_walmart_seller' export { datasetXPostsTool } from './dataset_x_posts' -export { datasetZillowPropertiesListingTool } from './dataset_zillow_properties_listing' -export { datasetBookingHotelListingsTool } from './dataset_booking_hotel_listings' -export { datasetYoutubeProfilesTool } from './dataset_youtube_profiles' +export { datasetYahooFinanceBusinessTool } from './dataset_yahoo_finance_business' export { datasetYoutubeCommentsTool } from './dataset_youtube_comments' -export { datasetRedditPostsTool } from './dataset_reddit_posts' +export { datasetYoutubeProfilesTool } from './dataset_youtube_profiles' export { datasetYoutubeVideosTool } from './dataset_youtube_videos' -export { datasetNpmPackageTool } from './dataset_npm_package' -export { datasetPypiPackageTool } from './dataset_pypi_package' +export { datasetZaraProductsTool } from './dataset_zara_products' +export { datasetZillowPropertiesListingTool } from './dataset_zillow_properties_listing' +export { datasetZoominfoCompanyProfileTool } from './dataset_zoominfo_company_profile' +export { scrapeMarkdownTool } from './scrape_markdown' +export { searchEngineTool } from './search_engine' export * from './types' diff --git a/apps/sim/tools/brightdata/types.ts b/apps/sim/tools/brightdata/types.ts index 35d06bc18e..cf68b0fce0 100644 --- a/apps/sim/tools/brightdata/types.ts +++ b/apps/sim/tools/brightdata/types.ts @@ -6,12 +6,10 @@ export interface BrightDataAuthParams { browserZone?: string } - export interface ScrapeMarkdownParams extends BrightDataAuthParams { url: string } - export interface ScrapeMarkdownResponse extends ToolResponse { output: { markdown: string @@ -20,13 +18,11 @@ export interface ScrapeMarkdownResponse extends ToolResponse { } } - export interface SearchEngineParams extends BrightDataAuthParams { query: string maxResults?: number } - export interface SearchEngineResponse extends ToolResponse { output: { results: Array<{ @@ -37,14 +33,12 @@ export interface SearchEngineResponse extends ToolResponse { } } - export interface BrowserNavigateParams extends BrightDataAuthParams { url: string sessionKey?: string country?: string } - export interface BrowserNavigateResponse extends ToolResponse { output: { success: boolean @@ -54,12 +48,10 @@ export interface BrowserNavigateResponse extends ToolResponse { } } - export interface BrowserSnapshotParams extends BrightDataAuthParams { sessionKey?: string } - export interface BrowserSnapshotResponse extends ToolResponse { output: { html: string @@ -68,12 +60,10 @@ export interface BrowserSnapshotResponse extends ToolResponse { } } - export interface BrowserScreenshotParams extends BrightDataAuthParams { sessionKey?: string } - export interface BrowserScreenshotResponse extends ToolResponse { output: { image: string @@ -81,7 +71,6 @@ export interface BrowserScreenshotResponse extends ToolResponse { } } - export interface DatasetParams extends BrightDataAuthParams { datasetId: string url?: string @@ -89,7 +78,6 @@ export interface DatasetParams extends BrightDataAuthParams { [key: string]: unknown } - export interface DatasetResponse extends ToolResponse { output: { data: unknown @@ -97,7 +85,6 @@ export interface DatasetResponse extends ToolResponse { } } - export type BrightDataResponse = | ScrapeMarkdownResponse | SearchEngineResponse diff --git a/apps/sim/tools/registry.ts b/apps/sim/tools/registry.ts index 27f2d7c5cc..0a0f64b05b 100644 --- a/apps/sim/tools/registry.ts +++ b/apps/sim/tools/registry.ts @@ -52,53 +52,53 @@ import { asanaUpdateTaskTool, } from '@/tools/asana' import { - scrapeMarkdownTool, - searchEngineTool, - datasetAmazonProductTool, datasetAmazonProductReviewsTool, datasetAmazonProductSearchTool, - datasetWalmartProductTool, - datasetWalmartSellerTool, - datasetEbayProductTool, - datasetHomedepotProductsTool, - datasetZaraProductsTool, - datasetEtsyProductsTool, + datasetAmazonProductTool, + datasetAppleAppStoreTool, datasetBestbuyProductsTool, - datasetLinkedinPersonProfileTool, - datasetLinkedinCompanyProfileTool, - datasetLinkedinJobListingsTool, - datasetLinkedinPostsTool, - datasetLinkedinPeopleSearchTool, + datasetBookingHotelListingsTool, datasetCrunchbaseCompanyTool, - datasetZoominfoCompanyProfileTool, - datasetInstagramProfilesTool, - datasetInstagramPostsTool, - datasetInstagramReelsTool, - datasetInstagramCommentsTool, - datasetFacebookPostsTool, - datasetFacebookMarketplaceListingsTool, + datasetEbayProductTool, + datasetEtsyProductsTool, datasetFacebookCompanyReviewsTool, datasetFacebookEventsTool, - datasetTiktokProfilesTool, - datasetTiktokPostsTool, - datasetTiktokShopTool, - datasetTiktokCommentsTool, + datasetFacebookMarketplaceListingsTool, + datasetFacebookPostsTool, + datasetGithubRepositoryFileTool, datasetGoogleMapsReviewsTool, - datasetGoogleShoppingTool, datasetGooglePlayStoreTool, - datasetAppleAppStoreTool, + datasetGoogleShoppingTool, + datasetHomedepotProductsTool, + datasetInstagramCommentsTool, + datasetInstagramPostsTool, + datasetInstagramProfilesTool, + datasetInstagramReelsTool, + datasetLinkedinCompanyProfileTool, + datasetLinkedinJobListingsTool, + datasetLinkedinPeopleSearchTool, + datasetLinkedinPersonProfileTool, + datasetLinkedinPostsTool, + datasetNpmPackageTool, + datasetPypiPackageTool, + datasetRedditPostsTool, datasetReuterNewsTool, - datasetGithubRepositoryFileTool, - datasetYahooFinanceBusinessTool, + datasetTiktokCommentsTool, + datasetTiktokPostsTool, + datasetTiktokProfilesTool, + datasetTiktokShopTool, + datasetWalmartProductTool, + datasetWalmartSellerTool, datasetXPostsTool, - datasetZillowPropertiesListingTool, - datasetBookingHotelListingsTool, - datasetYoutubeProfilesTool, + datasetYahooFinanceBusinessTool, datasetYoutubeCommentsTool, - datasetRedditPostsTool, + datasetYoutubeProfilesTool, datasetYoutubeVideosTool, - datasetNpmPackageTool, - datasetPypiPackageTool, + datasetZaraProductsTool, + datasetZillowPropertiesListingTool, + datasetZoominfoCompanyProfileTool, + scrapeMarkdownTool, + searchEngineTool, } from '@/tools/brightdata' import { browserUseRunTaskTool } from '@/tools/browser_use' import { diff --git a/apps/sim/tsconfig.brightdata.json b/apps/sim/tsconfig.brightdata.json new file mode 100644 index 0000000000..5566445486 --- /dev/null +++ b/apps/sim/tsconfig.brightdata.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "include": [ + "blocks/blocks/brightdata.ts", + "tools/brightdata/**/*.ts", + "app/api/tools/brightdata/**/*.ts", + "tools/registry.ts", + "blocks/registry.ts" + ] +} From afcfc681806307d08cb3b58040017ff6b9e745f6 Mon Sep 17 00:00:00 2001 From: meirk-brd Date: Wed, 14 Jan 2026 10:07:00 +0200 Subject: [PATCH 4/4] delete custom tsconf --- apps/sim/tsconfig.brightdata.json | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 apps/sim/tsconfig.brightdata.json diff --git a/apps/sim/tsconfig.brightdata.json b/apps/sim/tsconfig.brightdata.json deleted file mode 100644 index 5566445486..0000000000 --- a/apps/sim/tsconfig.brightdata.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "./tsconfig.json", - "include": [ - "blocks/blocks/brightdata.ts", - "tools/brightdata/**/*.ts", - "app/api/tools/brightdata/**/*.ts", - "tools/registry.ts", - "blocks/registry.ts" - ] -}