From b98351fbe9157990de59bba44bf35fb7596372eb Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Sat, 17 Jan 2026 01:04:47 +0000 Subject: [PATCH 01/15] Add markdown alternate links for LLM training data discovery - Add to page headers pointing to .md version - Improve MDX-to-markdown compilation to produce clean markdown output - Preserve code blocks and frontmatter while stripping JSX components Co-Authored-By: Claude Opus 4.5 --- app/api/markdown/[[...slug]]/route.ts | 93 ++++++++++++++++++++++++++- app/layout.tsx | 5 ++ 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index 61006c812..8cd890f21 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -7,6 +7,90 @@ export const dynamic = "force-dynamic"; // Regex pattern for removing .md extension const MD_EXTENSION_REGEX = /\.md$/; +// Regex patterns for MDX to Markdown compilation (top-level for performance) +const FRONTMATTER_REGEX = /^---\n([\s\S]*?)\n---\n?/; +const IMPORT_FROM_REGEX = /^import\s+.*?from\s+['"].*?['"];?\s*$/gm; +const IMPORT_DIRECT_REGEX = /^import\s+['"].*?['"];?\s*$/gm; +const IMPORT_DESTRUCTURE_REGEX = + /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm; +const EXPORT_REGEX = + /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm; +const SELF_CLOSING_JSX_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*\/>/g; +const JSX_WITH_CHILDREN_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*>([\s\S]*?)<\/\1>/g; +const CODE_BLOCK_REGEX = /```[\s\S]*?```/g; +const JSX_EXPRESSION_REGEX = /\{[^}]+\}/g; +const EXCESSIVE_NEWLINES_REGEX = /\n{3,}/g; +const CODE_BLOCK_PLACEHOLDER_REGEX = /__CODE_BLOCK_(\d+)__/g; + +/** + * Compiles MDX content to clean markdown by: + * - Preserving frontmatter + * - Removing import statements + * - Converting JSX components to their text content + * - Preserving standard markdown + */ +function compileMdxToMarkdown(content: string): string { + let result = content; + + // Extract and preserve frontmatter if present + let frontmatter = ""; + const frontmatterMatch = result.match(FRONTMATTER_REGEX); + if (frontmatterMatch) { + frontmatter = frontmatterMatch[0]; + result = result.slice(frontmatterMatch[0].length); + } + + // Remove import statements (various formats) + result = result.replace(IMPORT_FROM_REGEX, ""); + result = result.replace(IMPORT_DIRECT_REGEX, ""); + result = result.replace(IMPORT_DESTRUCTURE_REGEX, ""); + + // Remove export statements (like export const metadata) + result = result.replace(EXPORT_REGEX, ""); + + // Process self-closing JSX components (e.g., or ) + // Handles components with dots like + result = result.replace(SELF_CLOSING_JSX_REGEX, ""); + + // Process JSX components with children - extract the text content + // Handles components with dots like content + // Keep processing until no more JSX components remain + let previousResult = ""; + while (previousResult !== result) { + previousResult = result; + // Match opening tag, capture tag name (with dots), and content until matching closing tag + result = result.replace(JSX_WITH_CHILDREN_REGEX, (_, _tag, innerContent) => + innerContent.trim() + ); + } + + // Remove any remaining JSX expressions like {variable} or {expression} + // But preserve code blocks by temporarily replacing them + const codeBlocks: string[] = []; + result = result.replace(CODE_BLOCK_REGEX, (match) => { + codeBlocks.push(match); + return `__CODE_BLOCK_${codeBlocks.length - 1}__`; + }); + + // Now remove JSX expressions outside code blocks + result = result.replace(JSX_EXPRESSION_REGEX, ""); + + // Restore code blocks + result = result.replace( + CODE_BLOCK_PLACEHOLDER_REGEX, + (_, index) => codeBlocks[Number.parseInt(index, 10)] + ); + + // Clean up excessive blank lines (more than 2 consecutive) + result = result.replace(EXCESSIVE_NEWLINES_REGEX, "\n\n"); + + // Trim leading/trailing whitespace + result = result.trim(); + + // Reconstruct with frontmatter + return `${frontmatter}${result}\n`; +} + export async function GET( request: NextRequest, _context: { params: Promise<{ slug?: string[] }> } @@ -31,13 +115,16 @@ export async function GET( return new NextResponse("Markdown file not found", { status: 404 }); } - const content = await readFile(filePath, "utf-8"); + const rawContent = await readFile(filePath, "utf-8"); + + // Compile MDX to clean markdown + const content = compileMdxToMarkdown(rawContent); - // Return the raw markdown with proper headers + // Return the compiled markdown with proper headers return new NextResponse(content, { status: 200, headers: { - "Content-Type": "text/plain; charset=utf-8", + "Content-Type": "text/markdown; charset=utf-8", "Content-Disposition": "inline", }, }); diff --git a/app/layout.tsx b/app/layout.tsx index 56fdf950f..adad9f6ad 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -104,6 +104,11 @@ export default async function RootLayout({ + {lang !== "en" && ( From a79d46a9fde0cba73c912580eeb7cf4ebaa3fd89 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Sat, 17 Jan 2026 01:09:28 +0000 Subject: [PATCH 02/15] Add fallback content for component-only pages in markdown API Pages that only contain React components (like the landing page) now return a helpful markdown response with the title, description, and a link to the full interactive page. Co-Authored-By: Claude Opus 4.5 --- app/api/markdown/[[...slug]]/route.ts | 36 +++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index 8cd890f21..71d0acb13 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -22,14 +22,34 @@ const JSX_EXPRESSION_REGEX = /\{[^}]+\}/g; const EXCESSIVE_NEWLINES_REGEX = /\n{3,}/g; const CODE_BLOCK_PLACEHOLDER_REGEX = /__CODE_BLOCK_(\d+)__/g; +// Regex for extracting frontmatter fields +const TITLE_REGEX = /title:\s*["']?([^"'\n]+)["']?/; +const DESCRIPTION_REGEX = /description:\s*["']?([^"'\n]+)["']?/; + +/** + * Extracts title and description from frontmatter + */ +function extractFrontmatterMeta(frontmatter: string): { + title: string; + description: string; +} { + const titleMatch = frontmatter.match(TITLE_REGEX); + const descriptionMatch = frontmatter.match(DESCRIPTION_REGEX); + return { + title: titleMatch?.[1] || "Arcade Documentation", + description: descriptionMatch?.[1] || "", + }; +} + /** * Compiles MDX content to clean markdown by: * - Preserving frontmatter * - Removing import statements * - Converting JSX components to their text content * - Preserving standard markdown + * - Providing fallback content for component-only pages */ -function compileMdxToMarkdown(content: string): string { +function compileMdxToMarkdown(content: string, pagePath: string): string { let result = content; // Extract and preserve frontmatter if present @@ -87,6 +107,18 @@ function compileMdxToMarkdown(content: string): string { // Trim leading/trailing whitespace result = result.trim(); + // If content is essentially empty (component-only page), provide fallback + if (!result || result.length < 10) { + const { title, description } = extractFrontmatterMeta(frontmatter); + const htmlUrl = `https://docs.arcade.dev${pagePath}`; + return `${frontmatter}# ${title} + +${description} + +This page contains interactive content. Visit the full page at: ${htmlUrl} +`; + } + // Reconstruct with frontmatter return `${frontmatter}${result}\n`; } @@ -118,7 +150,7 @@ export async function GET( const rawContent = await readFile(filePath, "utf-8"); // Compile MDX to clean markdown - const content = compileMdxToMarkdown(rawContent); + const content = compileMdxToMarkdown(rawContent, pathWithoutMd); // Return the compiled markdown with proper headers return new NextResponse(content, { From 37a7fb1fe2ecdae2cf277fa2cf514e6372999b28 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Sat, 17 Jan 2026 13:15:08 +0000 Subject: [PATCH 03/15] Fix indentation in compiled markdown output - Add dedent function to normalize indentation when extracting content from JSX components - Add normalizeIndentation function to clean up stray whitespace while preserving meaningful markdown indentation (nested lists, blockquotes) - Move list detection regex patterns to module top level for performance - Ensures code block markers (```) start at column 0 Co-Authored-By: Claude Opus 4.5 --- app/api/markdown/[[...slug]]/route.ts | 90 ++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index 71d0acb13..1b23977b9 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -22,10 +22,59 @@ const JSX_EXPRESSION_REGEX = /\{[^}]+\}/g; const EXCESSIVE_NEWLINES_REGEX = /\n{3,}/g; const CODE_BLOCK_PLACEHOLDER_REGEX = /__CODE_BLOCK_(\d+)__/g; +// Regex for detecting markdown list items and numbered lists +const UNORDERED_LIST_REGEX = /^[-*+]\s/; +const ORDERED_LIST_REGEX = /^\d+[.)]\s/; + // Regex for extracting frontmatter fields const TITLE_REGEX = /title:\s*["']?([^"'\n]+)["']?/; const DESCRIPTION_REGEX = /description:\s*["']?([^"'\n]+)["']?/; +// Regex for detecting leading whitespace on lines +const LEADING_WHITESPACE_REGEX = /^[ \t]+/; + +/** + * Removes consistent leading indentation from all lines of text. + * This normalizes content that was indented inside JSX components. + * Code block markers (```) are ignored when calculating minimum indent + * since they typically start at column 0 in MDX files. + */ +function dedent(text: string): string { + const lines = text.split("\n"); + + // Find minimum indentation, ignoring: + // - Empty lines + // - Code block markers (lines starting with ```) + let minIndent = Number.POSITIVE_INFINITY; + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed === "" || trimmed.startsWith("```")) { + continue; // Ignore empty lines and code block markers + } + const match = line.match(LEADING_WHITESPACE_REGEX); + const indent = match ? match[0].length : 0; + if (indent < minIndent) { + minIndent = indent; + } + } + + // If no indentation found, return as-is + if (minIndent === 0 || minIndent === Number.POSITIVE_INFINITY) { + return text; + } + + // Remove the minimum indentation from each line (except code block content) + return lines + .map((line) => { + // Don't modify empty lines or lines with less indentation than min + if (line.trim() === "" || line.length < minIndent) { + return line.trimStart(); + } + return line.slice(minIndent); + }) + .join("\n"); +} + /** * Extracts title and description from frontmatter */ @@ -41,6 +90,41 @@ function extractFrontmatterMeta(frontmatter: string): { }; } +/** + * Normalizes indentation in the final output. + * Removes stray leading whitespace outside code blocks while preserving + * meaningful markdown indentation (nested lists, blockquotes). + */ +function normalizeIndentation(text: string): string { + const finalLines: string[] = []; + let inCodeBlock = false; + + for (const line of text.split("\n")) { + if (line.trim().startsWith("```")) { + inCodeBlock = !inCodeBlock; + finalLines.push(line.trimStart()); // Code block markers should start at column 0 + } else if (inCodeBlock) { + finalLines.push(line); // Preserve indentation inside code blocks + } else { + const trimmed = line.trimStart(); + // Preserve indentation for nested list items and blockquotes + const isListItem = + UNORDERED_LIST_REGEX.test(trimmed) || ORDERED_LIST_REGEX.test(trimmed); + const isBlockquote = trimmed.startsWith(">"); + if ((isListItem || isBlockquote) && line.startsWith(" ")) { + // Keep markdown-meaningful indentation (but normalize to 2-space increments) + const leadingSpaces = line.length - line.trimStart().length; + const normalizedIndent = " ".repeat(Math.floor(leadingSpaces / 2)); + finalLines.push(normalizedIndent + trimmed); + } else { + finalLines.push(trimmed); // Remove leading whitespace for other lines + } + } + } + + return finalLines.join("\n"); +} + /** * Compiles MDX content to clean markdown by: * - Preserving frontmatter @@ -79,8 +163,9 @@ function compileMdxToMarkdown(content: string, pagePath: string): string { while (previousResult !== result) { previousResult = result; // Match opening tag, capture tag name (with dots), and content until matching closing tag + // Apply dedent to each extracted piece to normalize indentation result = result.replace(JSX_WITH_CHILDREN_REGEX, (_, _tag, innerContent) => - innerContent.trim() + dedent(innerContent.trim()) ); } @@ -101,6 +186,9 @@ function compileMdxToMarkdown(content: string, pagePath: string): string { (_, index) => codeBlocks[Number.parseInt(index, 10)] ); + // Normalize indentation (remove stray whitespace, preserve meaningful markdown indentation) + result = normalizeIndentation(result); + // Clean up excessive blank lines (more than 2 consecutive) result = result.replace(EXCESSIVE_NEWLINES_REGEX, "\n\n"); From 020d3635f5f1b23b6feab770d63f0f78eed13a02 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Sat, 17 Jan 2026 21:14:41 +0000 Subject: [PATCH 04/15] mindent fix --- app/api/markdown/[[...slug]]/route.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index 1b23977b9..b0b3e8dc9 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -66,10 +66,16 @@ function dedent(text: string): string { // Remove the minimum indentation from each line (except code block content) return lines .map((line) => { + const trimmed = line.trim(); // Don't modify empty lines or lines with less indentation than min - if (line.trim() === "" || line.length < minIndent) { + if (trimmed === "" || line.length < minIndent) { return line.trimStart(); } + // Preserve code block markers - just remove leading whitespace + // This matches the logic that ignores them when calculating minIndent + if (trimmed.startsWith("```")) { + return trimmed; + } return line.slice(minIndent); }) .join("\n"); From c6e4fb8fd666a65063362d82bd45b6c1d11fdff6 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Sat, 17 Jan 2026 21:43:20 +0000 Subject: [PATCH 05/15] Fix frontmatter regex to handle apostrophes in quoted strings The previous regex patterns `["']?([^"'\n]+)["']?` would truncate text at the first apostrophe (e.g., "Arcade's" became "Arcade"). This fix: - Uses separate patterns for double-quoted, single-quoted, and unquoted values - Requires closing quotes to be at end of line to prevent apostrophes from being misinterpreted as closing delimiters - Adds stripSurroundingQuotes helper for fallback cases Co-Authored-By: Claude Opus 4.5 --- app/api/markdown/[[...slug]]/route.ts | 43 +++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index b0b3e8dc9..10812365f 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -27,8 +27,12 @@ const UNORDERED_LIST_REGEX = /^[-*+]\s/; const ORDERED_LIST_REGEX = /^\d+[.)]\s/; // Regex for extracting frontmatter fields -const TITLE_REGEX = /title:\s*["']?([^"'\n]+)["']?/; -const DESCRIPTION_REGEX = /description:\s*["']?([^"'\n]+)["']?/; +// Handles: "double quoted", 'single quoted', or unquoted values +// Group 1 = double-quoted content, Group 2 = single-quoted content, Group 3 = unquoted/fallback +// Quoted patterns require closing quote at end of line to prevent apostrophes being misread as delimiters +const TITLE_REGEX = /title:\s*(?:"([^"]*)"\s*$|'([^']*)'\s*$|([^\n]+))/; +const DESCRIPTION_REGEX = + /description:\s*(?:"([^"]*)"\s*$|'([^']*)'\s*$|([^\n]+))/; // Regex for detecting leading whitespace on lines const LEADING_WHITESPACE_REGEX = /^[ \t]+/; @@ -82,7 +86,23 @@ function dedent(text: string): string { } /** - * Extracts title and description from frontmatter + * Strips surrounding quotes from a value if present. + * Used for unquoted fallback values that may contain quotes due to apostrophe handling. + */ +function stripSurroundingQuotes(value: string): string { + const trimmed = value.trim(); + if ( + (trimmed.startsWith('"') && trimmed.endsWith('"')) || + (trimmed.startsWith("'") && trimmed.endsWith("'")) + ) { + return trimmed.slice(1, -1); + } + return trimmed; +} + +/** + * Extracts title and description from frontmatter. + * Handles double-quoted, single-quoted, and unquoted YAML values. */ function extractFrontmatterMeta(frontmatter: string): { title: string; @@ -90,9 +110,22 @@ function extractFrontmatterMeta(frontmatter: string): { } { const titleMatch = frontmatter.match(TITLE_REGEX); const descriptionMatch = frontmatter.match(DESCRIPTION_REGEX); + + // Extract from whichever capture group matched: + // Group 1 = double-quoted, Group 2 = single-quoted, Group 3 = unquoted/fallback + // For group 3 (fallback), strip surrounding quotes if present + const title = + titleMatch?.[1] ?? + titleMatch?.[2] ?? + stripSurroundingQuotes(titleMatch?.[3] ?? ""); + const description = + descriptionMatch?.[1] ?? + descriptionMatch?.[2] ?? + stripSurroundingQuotes(descriptionMatch?.[3] ?? ""); + return { - title: titleMatch?.[1] || "Arcade Documentation", - description: descriptionMatch?.[1] || "", + title: title || "Arcade Documentation", + description, }; } From fd16de7dcb787e805ace673729aa9bc232b29956 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Sat, 17 Jan 2026 21:59:26 +0000 Subject: [PATCH 06/15] Skip alternate link for root pathname fallback When x-pathname header is not set, pathname defaults to "/" which would produce an invalid alternate link "https://docs.arcade.dev/.md". Only render the alternate link when we have a real page path. Co-Authored-By: Claude Opus 4.5 --- app/layout.tsx | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/app/layout.tsx b/app/layout.tsx index adad9f6ad..bc3a1f8b9 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -104,11 +104,13 @@ export default async function RootLayout({ - + {pathname !== "/" && ( + + )} {lang !== "en" && ( From d7b7c711018863a34b02b9b998b3711904439f98 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Tue, 20 Jan 2026 17:51:28 +0000 Subject: [PATCH 07/15] Generate static markdown files at build time - Add scripts/generate-markdown.ts to pre-render MDX to markdown - Update proxy.ts to serve static .md files from public/ - Delete API route in favor of static file serving - Add link rewriting to add /en/ prefix and .md extension - Add markdown-friendly component implementations - Fix localhost URL in gmail integration page Co-Authored-By: Claude Opus 4.5 --- .gitignore | 1 + app/api/markdown/[[...slug]]/route.ts | 295 --- .../integrations/productivity/gmail/page.mdx | 2 +- lib/mdx-to-markdown.tsx | 2314 +++++++++++++++++ package.json | 12 +- pnpm-lock.yaml | 181 +- proxy.ts | 25 +- scripts/generate-markdown.ts | 155 ++ 8 files changed, 2663 insertions(+), 322 deletions(-) delete mode 100644 app/api/markdown/[[...slug]]/route.ts create mode 100644 lib/mdx-to-markdown.tsx create mode 100644 scripts/generate-markdown.ts diff --git a/.gitignore b/.gitignore index c714c9723..5968906cb 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ node_modules .DS_Store .env.local public/sitemap*.xml +public/en/**/*.md .env _pagefind/ diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts deleted file mode 100644 index 10812365f..000000000 --- a/app/api/markdown/[[...slug]]/route.ts +++ /dev/null @@ -1,295 +0,0 @@ -import { access, readFile } from "node:fs/promises"; -import { join } from "node:path"; -import { type NextRequest, NextResponse } from "next/server"; - -export const dynamic = "force-dynamic"; - -// Regex pattern for removing .md extension -const MD_EXTENSION_REGEX = /\.md$/; - -// Regex patterns for MDX to Markdown compilation (top-level for performance) -const FRONTMATTER_REGEX = /^---\n([\s\S]*?)\n---\n?/; -const IMPORT_FROM_REGEX = /^import\s+.*?from\s+['"].*?['"];?\s*$/gm; -const IMPORT_DIRECT_REGEX = /^import\s+['"].*?['"];?\s*$/gm; -const IMPORT_DESTRUCTURE_REGEX = - /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm; -const EXPORT_REGEX = - /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm; -const SELF_CLOSING_JSX_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*\/>/g; -const JSX_WITH_CHILDREN_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*>([\s\S]*?)<\/\1>/g; -const CODE_BLOCK_REGEX = /```[\s\S]*?```/g; -const JSX_EXPRESSION_REGEX = /\{[^}]+\}/g; -const EXCESSIVE_NEWLINES_REGEX = /\n{3,}/g; -const CODE_BLOCK_PLACEHOLDER_REGEX = /__CODE_BLOCK_(\d+)__/g; - -// Regex for detecting markdown list items and numbered lists -const UNORDERED_LIST_REGEX = /^[-*+]\s/; -const ORDERED_LIST_REGEX = /^\d+[.)]\s/; - -// Regex for extracting frontmatter fields -// Handles: "double quoted", 'single quoted', or unquoted values -// Group 1 = double-quoted content, Group 2 = single-quoted content, Group 3 = unquoted/fallback -// Quoted patterns require closing quote at end of line to prevent apostrophes being misread as delimiters -const TITLE_REGEX = /title:\s*(?:"([^"]*)"\s*$|'([^']*)'\s*$|([^\n]+))/; -const DESCRIPTION_REGEX = - /description:\s*(?:"([^"]*)"\s*$|'([^']*)'\s*$|([^\n]+))/; - -// Regex for detecting leading whitespace on lines -const LEADING_WHITESPACE_REGEX = /^[ \t]+/; - -/** - * Removes consistent leading indentation from all lines of text. - * This normalizes content that was indented inside JSX components. - * Code block markers (```) are ignored when calculating minimum indent - * since they typically start at column 0 in MDX files. - */ -function dedent(text: string): string { - const lines = text.split("\n"); - - // Find minimum indentation, ignoring: - // - Empty lines - // - Code block markers (lines starting with ```) - let minIndent = Number.POSITIVE_INFINITY; - for (const line of lines) { - const trimmed = line.trim(); - if (trimmed === "" || trimmed.startsWith("```")) { - continue; // Ignore empty lines and code block markers - } - const match = line.match(LEADING_WHITESPACE_REGEX); - const indent = match ? match[0].length : 0; - if (indent < minIndent) { - minIndent = indent; - } - } - - // If no indentation found, return as-is - if (minIndent === 0 || minIndent === Number.POSITIVE_INFINITY) { - return text; - } - - // Remove the minimum indentation from each line (except code block content) - return lines - .map((line) => { - const trimmed = line.trim(); - // Don't modify empty lines or lines with less indentation than min - if (trimmed === "" || line.length < minIndent) { - return line.trimStart(); - } - // Preserve code block markers - just remove leading whitespace - // This matches the logic that ignores them when calculating minIndent - if (trimmed.startsWith("```")) { - return trimmed; - } - return line.slice(minIndent); - }) - .join("\n"); -} - -/** - * Strips surrounding quotes from a value if present. - * Used for unquoted fallback values that may contain quotes due to apostrophe handling. - */ -function stripSurroundingQuotes(value: string): string { - const trimmed = value.trim(); - if ( - (trimmed.startsWith('"') && trimmed.endsWith('"')) || - (trimmed.startsWith("'") && trimmed.endsWith("'")) - ) { - return trimmed.slice(1, -1); - } - return trimmed; -} - -/** - * Extracts title and description from frontmatter. - * Handles double-quoted, single-quoted, and unquoted YAML values. - */ -function extractFrontmatterMeta(frontmatter: string): { - title: string; - description: string; -} { - const titleMatch = frontmatter.match(TITLE_REGEX); - const descriptionMatch = frontmatter.match(DESCRIPTION_REGEX); - - // Extract from whichever capture group matched: - // Group 1 = double-quoted, Group 2 = single-quoted, Group 3 = unquoted/fallback - // For group 3 (fallback), strip surrounding quotes if present - const title = - titleMatch?.[1] ?? - titleMatch?.[2] ?? - stripSurroundingQuotes(titleMatch?.[3] ?? ""); - const description = - descriptionMatch?.[1] ?? - descriptionMatch?.[2] ?? - stripSurroundingQuotes(descriptionMatch?.[3] ?? ""); - - return { - title: title || "Arcade Documentation", - description, - }; -} - -/** - * Normalizes indentation in the final output. - * Removes stray leading whitespace outside code blocks while preserving - * meaningful markdown indentation (nested lists, blockquotes). - */ -function normalizeIndentation(text: string): string { - const finalLines: string[] = []; - let inCodeBlock = false; - - for (const line of text.split("\n")) { - if (line.trim().startsWith("```")) { - inCodeBlock = !inCodeBlock; - finalLines.push(line.trimStart()); // Code block markers should start at column 0 - } else if (inCodeBlock) { - finalLines.push(line); // Preserve indentation inside code blocks - } else { - const trimmed = line.trimStart(); - // Preserve indentation for nested list items and blockquotes - const isListItem = - UNORDERED_LIST_REGEX.test(trimmed) || ORDERED_LIST_REGEX.test(trimmed); - const isBlockquote = trimmed.startsWith(">"); - if ((isListItem || isBlockquote) && line.startsWith(" ")) { - // Keep markdown-meaningful indentation (but normalize to 2-space increments) - const leadingSpaces = line.length - line.trimStart().length; - const normalizedIndent = " ".repeat(Math.floor(leadingSpaces / 2)); - finalLines.push(normalizedIndent + trimmed); - } else { - finalLines.push(trimmed); // Remove leading whitespace for other lines - } - } - } - - return finalLines.join("\n"); -} - -/** - * Compiles MDX content to clean markdown by: - * - Preserving frontmatter - * - Removing import statements - * - Converting JSX components to their text content - * - Preserving standard markdown - * - Providing fallback content for component-only pages - */ -function compileMdxToMarkdown(content: string, pagePath: string): string { - let result = content; - - // Extract and preserve frontmatter if present - let frontmatter = ""; - const frontmatterMatch = result.match(FRONTMATTER_REGEX); - if (frontmatterMatch) { - frontmatter = frontmatterMatch[0]; - result = result.slice(frontmatterMatch[0].length); - } - - // Remove import statements (various formats) - result = result.replace(IMPORT_FROM_REGEX, ""); - result = result.replace(IMPORT_DIRECT_REGEX, ""); - result = result.replace(IMPORT_DESTRUCTURE_REGEX, ""); - - // Remove export statements (like export const metadata) - result = result.replace(EXPORT_REGEX, ""); - - // Process self-closing JSX components (e.g., or ) - // Handles components with dots like - result = result.replace(SELF_CLOSING_JSX_REGEX, ""); - - // Process JSX components with children - extract the text content - // Handles components with dots like content - // Keep processing until no more JSX components remain - let previousResult = ""; - while (previousResult !== result) { - previousResult = result; - // Match opening tag, capture tag name (with dots), and content until matching closing tag - // Apply dedent to each extracted piece to normalize indentation - result = result.replace(JSX_WITH_CHILDREN_REGEX, (_, _tag, innerContent) => - dedent(innerContent.trim()) - ); - } - - // Remove any remaining JSX expressions like {variable} or {expression} - // But preserve code blocks by temporarily replacing them - const codeBlocks: string[] = []; - result = result.replace(CODE_BLOCK_REGEX, (match) => { - codeBlocks.push(match); - return `__CODE_BLOCK_${codeBlocks.length - 1}__`; - }); - - // Now remove JSX expressions outside code blocks - result = result.replace(JSX_EXPRESSION_REGEX, ""); - - // Restore code blocks - result = result.replace( - CODE_BLOCK_PLACEHOLDER_REGEX, - (_, index) => codeBlocks[Number.parseInt(index, 10)] - ); - - // Normalize indentation (remove stray whitespace, preserve meaningful markdown indentation) - result = normalizeIndentation(result); - - // Clean up excessive blank lines (more than 2 consecutive) - result = result.replace(EXCESSIVE_NEWLINES_REGEX, "\n\n"); - - // Trim leading/trailing whitespace - result = result.trim(); - - // If content is essentially empty (component-only page), provide fallback - if (!result || result.length < 10) { - const { title, description } = extractFrontmatterMeta(frontmatter); - const htmlUrl = `https://docs.arcade.dev${pagePath}`; - return `${frontmatter}# ${title} - -${description} - -This page contains interactive content. Visit the full page at: ${htmlUrl} -`; - } - - // Reconstruct with frontmatter - return `${frontmatter}${result}\n`; -} - -export async function GET( - request: NextRequest, - _context: { params: Promise<{ slug?: string[] }> } -) { - try { - // Get the original pathname from the request - const url = new URL(request.url); - // Remove /api/markdown prefix to get the original path - const originalPath = url.pathname.replace("/api/markdown", ""); - - // Remove .md extension - const pathWithoutMd = originalPath.replace(MD_EXTENSION_REGEX, ""); - - // Map URL to file path - // e.g., /en/home/quickstart -> app/en/home/quickstart/page.mdx - const filePath = join(process.cwd(), "app", `${pathWithoutMd}/page.mdx`); - - // Check if file exists - try { - await access(filePath); - } catch { - return new NextResponse("Markdown file not found", { status: 404 }); - } - - const rawContent = await readFile(filePath, "utf-8"); - - // Compile MDX to clean markdown - const content = compileMdxToMarkdown(rawContent, pathWithoutMd); - - // Return the compiled markdown with proper headers - return new NextResponse(content, { - status: 200, - headers: { - "Content-Type": "text/markdown; charset=utf-8", - "Content-Disposition": "inline", - }, - }); - } catch (error) { - return new NextResponse(`Internal server error: ${error}`, { - status: 500, - }); - } -} diff --git a/app/en/resources/integrations/productivity/gmail/page.mdx b/app/en/resources/integrations/productivity/gmail/page.mdx index f20dc6303..7e68ebc22 100644 --- a/app/en/resources/integrations/productivity/gmail/page.mdx +++ b/app/en/resources/integrations/productivity/gmail/page.mdx @@ -272,7 +272,7 @@ Delete a draft email using the Gmail API. The `TrashEmail` tool is currently only available on a self-hosted instance of the Arcade Engine. To learn more about self-hosting, see the [self-hosting - documentation](http://localhost:3000/en/home/deployment/engine-configuration). + documentation](/guides/deployment-hosting/configure-engine).
diff --git a/lib/mdx-to-markdown.tsx b/lib/mdx-to-markdown.tsx new file mode 100644 index 000000000..0e8345df4 --- /dev/null +++ b/lib/mdx-to-markdown.tsx @@ -0,0 +1,2314 @@ +/** + * MDX to Markdown converter + * + * Compiles MDX content, renders it with markdown-friendly components, + * then converts the resulting HTML to clean Markdown using the unified ecosystem. + */ + +import { pathToFileURL } from "node:url"; +import { compile, run } from "@mdx-js/mdx"; +import type { ReactNode } from "react"; +import { createElement, Fragment } from "react"; +import { Fragment as JsxFragment, jsx, jsxs } from "react/jsx-runtime"; +import rehypeParse from "rehype-parse"; +import rehypeRemark from "rehype-remark"; +import remarkGfm from "remark-gfm"; +import remarkStringify from "remark-stringify"; +import { unified } from "unified"; + +// Regex patterns (at top level for performance) +const FILE_EXTENSION_REGEX = /\.[^.]+$/; +const UNDERSCORE_DASH_REGEX = /[_-]/g; +const FRONTMATTER_REGEX = /^---\n([\s\S]*?)\n---\n?/; +const TITLE_REGEX = /title:\s*(?:"([^"]*)"|'([^']*)'|([^\n]+))/; +const DESCRIPTION_REGEX = /description:\s*(?:"([^"]*)"|'([^']*)'|([^\n]+))/; + +// Types for mdast table nodes +type MdastTableCell = { type: "tableCell"; children: unknown[] }; +type MdastTableRow = { type: "tableRow"; children: MdastTableCell[] }; +type HtmlNode = { + type: string; + tagName?: string; + children?: HtmlNode[]; + properties?: Record; +}; +type StateAll = (node: HtmlNode) => unknown[]; + +/** Extract cells from a table row element */ +function extractCellsFromRow( + state: { all: StateAll }, + row: HtmlNode +): MdastTableCell[] { + const cells: MdastTableCell[] = []; + for (const cell of row.children || []) { + if ( + cell.type === "element" && + (cell.tagName === "th" || cell.tagName === "td") + ) { + cells.push({ type: "tableCell", children: state.all(cell) }); + } + } + return cells; +} + +/** Extract rows from a table section (thead, tbody, tfoot) or direct tr children */ +function extractRowsFromTableSection( + state: { all: StateAll }, + section: HtmlNode +): MdastTableRow[] { + const rows: MdastTableRow[] = []; + for (const child of section.children || []) { + if (child.type === "element" && child.tagName === "tr") { + const cells = extractCellsFromRow(state, child); + if (cells.length > 0) { + rows.push({ type: "tableRow", children: cells }); + } + } + } + return rows; +} + +/** Check if element is a table section (thead, tbody, tfoot) */ +function isTableSection(tagName: string | undefined): boolean { + return tagName === "thead" || tagName === "tbody" || tagName === "tfoot"; +} + +// Dynamic import to avoid Next.js RSC restrictions +let renderToStaticMarkup: typeof import("react-dom/server").renderToStaticMarkup; +async function getRenderer() { + if (!renderToStaticMarkup) { + const reactDomServer = await import("react-dom/server"); + renderToStaticMarkup = reactDomServer.renderToStaticMarkup; + } + return renderToStaticMarkup; +} + +/** + * Convert HTML to Markdown using unified ecosystem (rehype-remark) + * This is more reliable than turndown for complex HTML structures + */ +async function htmlToMarkdown(html: string): Promise { + const result = await unified() + .use(rehypeParse, { fragment: true }) + .use(rehypeRemark, { + handlers: { + // Custom handler for video elements + video: (_state, node) => { + const src = (node.properties?.src as string) || ""; + const filename = + src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Video"; + const title = filename.replace(UNDERSCORE_DASH_REGEX, " "); + return { + type: "paragraph", + children: [ + { + type: "link", + url: src, + children: [{ type: "text", value: `Video: ${title}` }], + }, + ], + }; + }, + // Custom handler for audio elements + audio: (_state, node) => { + const src = (node.properties?.src as string) || ""; + const filename = + src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Audio"; + const title = filename.replace(UNDERSCORE_DASH_REGEX, " "); + return { + type: "paragraph", + children: [ + { + type: "link", + url: src, + children: [{ type: "text", value: `Audio: ${title}` }], + }, + ], + }; + }, + // Custom handler for iframe elements + iframe: (_state, node) => { + const src = (node.properties?.src as string) || ""; + const title = + (node.properties?.title as string) || "Embedded content"; + return { + type: "paragraph", + children: [ + { + type: "link", + url: src, + children: [{ type: "text", value: title }], + }, + ], + }; + }, + // Custom handler for HTML tables - convert to markdown tables + // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Table parsing requires nested logic + table: (state, node) => { + const rows: MdastTableRow[] = []; + + for (const child of node.children || []) { + if (child.type !== "element") { + continue; + } + if (isTableSection(child.tagName)) { + rows.push( + ...extractRowsFromTableSection(state, child as HtmlNode) + ); + } else if (child.tagName === "tr") { + const cells = extractCellsFromRow(state, child as HtmlNode); + if (cells.length > 0) { + rows.push({ type: "tableRow", children: cells }); + } + } + } + + const colCount = rows[0]?.children?.length || 0; + return { + type: "table", + align: new Array(colCount).fill(null), + children: rows, + }; + }, + // These are handled by the table handler above, but we still need to define them + // to prevent "unknown node" errors when encountered outside tables + thead: (state, node) => { + const rows: unknown[] = []; + for (const child of node.children || []) { + if (child.type === "element" && child.tagName === "tr") { + rows.push(...state.all(child)); + } + } + return rows; + }, + tbody: (state, node) => { + const rows: unknown[] = []; + for (const child of node.children || []) { + if (child.type === "element" && child.tagName === "tr") { + rows.push(...state.all(child)); + } + } + return rows; + }, + tfoot: (state, node) => { + const rows: unknown[] = []; + for (const child of node.children || []) { + if (child.type === "element" && child.tagName === "tr") { + rows.push(...state.all(child)); + } + } + return rows; + }, + tr: (state, node) => { + const cells: { type: "tableCell"; children: unknown[] }[] = []; + for (const child of node.children || []) { + if ( + child.type === "element" && + (child.tagName === "th" || child.tagName === "td") + ) { + cells.push({ + type: "tableCell", + children: state.all(child), + }); + } + } + return { + type: "tableRow", + children: cells, + }; + }, + th: (state, node) => ({ + type: "tableCell", + children: state.all(node), + }), + td: (state, node) => ({ + type: "tableCell", + children: state.all(node), + }), + // Custom handler for callout divs - render as paragraph with bold label + // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Callout parsing logic + div: (state, node) => { + const className = + (node.properties?.className as string[])?.join(" ") || ""; + + // Check if this is a callout + if ( + className.includes("callout") || + className.includes("admonition") || + className.includes("warning") || + className.includes("info") || + className.includes("error") || + className.includes("tip") + ) { + let label = ""; + if (className.includes("warning")) { + label = "Warning"; + } else if (className.includes("error")) { + label = "Error"; + } else if (className.includes("tip")) { + label = "Tip"; + } else if (className.includes("info")) { + label = "Note"; + } + + // Process children and prepend bold label + const children = state.all(node); + if (label && children.length > 0) { + // Add bold label to the first paragraph's children + const firstChild = children[0]; + if (firstChild && firstChild.type === "paragraph") { + return [ + { + type: "paragraph", + children: [ + { + type: "strong", + children: [{ type: "text", value: `${label}:` }], + }, + { type: "text", value: " " }, + ...(firstChild.children || []), + ], + }, + ...children.slice(1), + ]; + } + } + return children; + } + + // Default: just return children (strip the div wrapper) + return state.all(node); + }, + }, + }) + .use(remarkGfm) // Enable GFM for tables, strikethrough, etc. + .use(remarkStringify, { + bullet: "-", + fences: true, + listItemIndent: "one", + }) + .process(html); + + return String(result); +} + +// ============================================ +// Markdown-Friendly Component Implementations +// ============================================ + +// Simple wrapper that just renders children +function PassThrough({ children }: { children?: ReactNode }) { + return createElement(Fragment, null, children); +} + +// Tabs - render all tab content with headers +function MarkdownTabs({ + children, + items, +}: { + children?: ReactNode; + items?: string[]; +}) { + // If we have items array, children are the tab panels + if (items && Array.isArray(items)) { + const childArray = Array.isArray(children) ? children : [children]; + return createElement( + "div", + null, + childArray.map((child, i) => + createElement( + "div", + { key: i }, + createElement("h4", null, items[i] || `Option ${i + 1}`), + child + ) + ) + ); + } + return createElement("div", null, children); +} + +// Tab content - just render the content +function MarkdownTab({ children }: { children?: ReactNode }) { + return createElement("div", null, children); +} + +// Assign Tab to Tabs for Tabs.Tab syntax +MarkdownTabs.Tab = MarkdownTab; + +// Steps - render as numbered sections +function MarkdownSteps({ children }: { children?: ReactNode }) { + return createElement("div", { className: "steps" }, children); +} + +// Callout - render as a styled div that turndown will convert +function MarkdownCallout({ + children, + type = "info", +}: { + children?: ReactNode; + type?: string; +}) { + return createElement("div", { className: `callout ${type}` }, children); +} + +// Cards - render as sections +function MarkdownCards({ children }: { children?: ReactNode }) { + return createElement("div", null, children); +} + +function MarkdownCard({ + children, + title, + href, +}: { + children?: ReactNode; + title?: string; + href?: string; +}) { + if (href) { + return createElement( + "div", + null, + title && createElement("h4", null, createElement("a", { href }, title)), + children + ); + } + return createElement( + "div", + null, + title && createElement("h4", null, title), + children + ); +} + +MarkdownCards.Card = MarkdownCard; + +// FileTree - render as a code block +function MarkdownFileTree({ children }: { children?: ReactNode }) { + return createElement("pre", null, createElement("code", null, children)); +} + +// Link components - render as standard links +function _MarkdownLink({ + children, + href, +}: { + children?: ReactNode; + href?: string; +}) { + return createElement("a", { href }, children); +} + +// ============================================ +// HTML Element Handlers +// ============================================ + +// Video - convert to a descriptive link +function MarkdownVideo({ + src, + title, + children, +}: { + src?: string; + title?: string; + children?: ReactNode; +}) { + if (!src) { + return createElement(Fragment, null, children); + } + const filename = + src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Video"; + const videoTitle = title || filename.replace(UNDERSCORE_DASH_REGEX, " "); + return createElement("p", null, `[Video: ${videoTitle}](${src})`); +} + +// Audio - convert to a descriptive link +function MarkdownAudio({ + src, + title, + children, +}: { + src?: string; + title?: string; + children?: ReactNode; +}) { + if (!src) { + return createElement(Fragment, null, children); + } + const filename = + src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Audio"; + const audioTitle = title || filename.replace(UNDERSCORE_DASH_REGEX, " "); + return createElement("p", null, `[Audio: ${audioTitle}](${src})`); +} + +// Image - keep as img for turndown to handle +function MarkdownImage({ + src, + alt, + title, +}: { + src?: string; + alt?: string; + title?: string; +}) { + return createElement("img", { src, alt: alt || title || "" }); +} + +// Iframe - convert to link +function MarkdownIframe({ src, title }: { src?: string; title?: string }) { + if (!src) { + return null; + } + const label = title || "Embedded content"; + return createElement("p", null, `[${label}](${src})`); +} + +// HR - render as markdown horizontal rule +function MarkdownHr() { + return createElement("hr", null); +} + +// BR - render as line break +function MarkdownBr() { + return createElement("br", null); +} + +// Container elements - just pass through children (strips the wrapper) +function MarkdownPassthrough({ children }: { children?: ReactNode }) { + return createElement(Fragment, null, children); +} + +// Figure/Figcaption - extract content +function MarkdownFigure({ children }: { children?: ReactNode }) { + return createElement("figure", null, children); +} + +function MarkdownFigcaption({ children }: { children?: ReactNode }) { + return createElement("figcaption", null, children); +} + +// Details/Summary - convert to blockquote-style +function MarkdownDetails({ children }: { children?: ReactNode }) { + return createElement("blockquote", null, children); +} + +function MarkdownSummary({ children }: { children?: ReactNode }) { + return createElement("strong", null, children); +} + +// Table elements - pass through for turndown +function MarkdownTable({ children }: { children?: ReactNode }) { + return createElement("table", null, children); +} + +function MarkdownThead({ children }: { children?: ReactNode }) { + return createElement("thead", null, children); +} + +function MarkdownTbody({ children }: { children?: ReactNode }) { + return createElement("tbody", null, children); +} + +function MarkdownTr({ children }: { children?: ReactNode }) { + return createElement("tr", null, children); +} + +function MarkdownTh({ children }: { children?: ReactNode }) { + return createElement("th", null, children); +} + +function MarkdownTd({ children }: { children?: ReactNode }) { + return createElement("td", null, children); +} + +// Definition lists +function MarkdownDl({ children }: { children?: ReactNode }) { + return createElement("dl", null, children); +} + +function MarkdownDt({ children }: { children?: ReactNode }) { + return createElement("dt", null, createElement("strong", null, children)); +} + +function MarkdownDd({ children }: { children?: ReactNode }) { + return createElement("dd", null, children); +} + +// Code block elements - preserve language and content +function MarkdownPre({ + children, + ...props +}: { + children?: ReactNode; + [key: string]: unknown; +}) { + // Extract data-language if present (MDX sometimes uses this) + const lang = props["data-language"] || ""; + return createElement("pre", { "data-language": lang }, children); +} + +function MarkdownCode({ + children, + className, + ...props +}: { + children?: ReactNode; + className?: string; + [key: string]: unknown; +}) { + // Preserve the language class for turndown + return createElement( + "code", + { className: className || "", ...props }, + children + ); +} + +// GuideOverview custom component +function MarkdownGuideOverview({ children }: { children?: ReactNode }) { + return createElement("div", null, children); +} + +function MarkdownGuideOverviewItem({ + children, + title, + href, +}: { + children?: ReactNode; + title?: string; + href?: string; +}) { + return createElement( + "div", + null, + title && + createElement( + "h4", + null, + href ? createElement("a", { href }, title) : title + ), + children + ); +} + +function MarkdownGuideOverviewOutcomes({ children }: { children?: ReactNode }) { + return createElement(Fragment, null, children); +} + +function MarkdownGuideOverviewPrerequisites({ + children, +}: { + children?: ReactNode; +}) { + return createElement( + "div", + null, + createElement("strong", null, "Prerequisites:"), + children + ); +} + +MarkdownGuideOverview.Item = MarkdownGuideOverviewItem; +MarkdownGuideOverview.Outcomes = MarkdownGuideOverviewOutcomes; +MarkdownGuideOverview.Prerequisites = MarkdownGuideOverviewPrerequisites; + +// CheatSheet components +function MarkdownCheatSheetGrid({ children }: { children?: ReactNode }) { + return createElement(Fragment, null, children); +} + +function MarkdownCheatSheetSection({ + children, + title, + icon, +}: { + children?: ReactNode; + title?: string; + icon?: string; +}) { + return createElement( + "div", + null, + title && createElement("h3", null, icon ? `${icon} ${title}` : title), + children + ); +} + +function MarkdownInfoBox({ + children, + type = "info", +}: { + children?: ReactNode; + type?: string; +}) { + // Use a div with callout class so the callout handler processes it + // The div handler in htmlToMarkdown will add the bold label + return createElement("div", { className: `callout ${type}` }, children); +} + +function MarkdownCommandItem({ children }: { children?: ReactNode }) { + return createElement("div", null, children); +} + +function MarkdownCommandList({ children }: { children?: ReactNode }) { + return createElement("div", null, children); +} + +function MarkdownCommandBlock({ children }: { children?: ReactNode }) { + return createElement("div", null, children); +} + +// GlossaryTerm - just render the text +function MarkdownGlossaryTerm({ + children, + term, +}: { + children?: ReactNode; + term?: string; +}) { + return createElement("strong", { title: term }, children); +} + +// DashboardLink - render as a link +function MarkdownDashboardLink({ + children, + href, +}: { + children?: ReactNode; + href?: string; +}) { + const dashboardUrl = href || "https://api.arcade.dev/dashboard"; + return createElement("a", { href: dashboardUrl }, children); +} + +// SignupLink - render as a link +function MarkdownSignupLink({ + children, +}: { + children?: ReactNode; + linkLocation?: string; +}) { + return createElement( + "a", + { href: "https://api.arcade.dev/dashboard/signup" }, + children + ); +} + +// Generic catch-all for unknown components +function _MarkdownGeneric({ children }: { children?: ReactNode }) { + return createElement("div", null, children); +} + +// ToolCard - render as a link with summary +function MarkdownToolCard({ + name, + summary, + link, + category: _category, +}: { + name?: string; + image?: string; + summary?: string; + link?: string; + category?: string; +}) { + return createElement( + "li", + null, + createElement("a", { href: link }, name), + summary ? ` - ${summary}` : "" + ); +} + +// MCPClientGrid - render as a list of MCP clients +function MarkdownMCPClientGrid() { + const mcpClients = [ + { + key: "cursor", + name: "Cursor", + description: "AI-powered code editor with built-in MCP support", + }, + { + key: "claude-desktop", + name: "Claude Desktop", + description: "Anthropic's desktop app for Claude with MCP integration", + }, + { + key: "visual-studio-code", + name: "Visual Studio Code", + description: "Microsoft's code editor with MCP extensions", + }, + ]; + + return createElement( + "ul", + null, + ...mcpClients.map((client, i) => + createElement( + "li", + { key: i }, + createElement( + "a", + { href: `/guides/tool-calling/mcp-clients/${client.key}` }, + client.name + ), + ` - ${client.description}` + ) + ) + ); +} + +// ContactCards - render contact information +function MarkdownContactCards() { + const contacts = [ + { + title: "Email Support", + description: + "Get help with technical issues, account questions, and general inquiries.", + href: "mailto:support@arcade.dev", + }, + { + title: "Contact Sales", + description: + "Discuss enterprise plans, pricing, and how Arcade can help your organization.", + href: "mailto:sales@arcade.dev", + }, + { + title: "Discord Community", + description: + "Join for real-time help, connect with developers, and stay updated.", + href: "https://discord.gg/GUZEMpEZ9p", + }, + { + title: "GitHub Issues & Discussions", + description: "Report bugs, request features, and contribute to Arcade.", + href: "https://github.com/ArcadeAI/arcade-mcp", + }, + { + title: "Security Research", + description: "Report security vulnerabilities responsibly.", + href: "/guides/security/security-research-program", + }, + { + title: "System Status", + description: "Check the current status of Arcade's services.", + href: "https://status.arcade.dev", + }, + ]; + + return createElement( + "ul", + null, + ...contacts.map((contact, i) => + createElement( + "li", + { key: i }, + createElement("a", { href: contact.href }, contact.title), + ` - ${contact.description}` + ) + ) + ); +} + +// SubpageList - render list of subpages (uses provided meta) +function MarkdownSubpageList({ + basePath, + meta, +}: { + basePath?: string; + meta?: Record; +}) { + if (!(meta && basePath)) { + return null; + } + + const subpages = Object.entries(meta).filter( + ([key]) => key !== "index" && key !== "*" + ); + + return createElement( + "ul", + null, + ...subpages.map(([key, title], i) => { + let linkText: string; + if (typeof title === "string") { + linkText = title; + } else if ( + typeof title === "object" && + title !== null && + "title" in title + ) { + linkText = (title as { title: string }).title; + } else { + linkText = String(title); + } + return createElement( + "li", + { key: i }, + createElement("a", { href: `${basePath}/${key}` }, linkText) + ); + }) + ); +} + +// ============================================ +// Landing Page Component - renders as markdown-friendly content +// ============================================ +function MarkdownLandingPage() { + return createElement( + "div", + null, + // Hero + createElement("h1", null, "MCP Runtime for AI agents that get things done"), + createElement( + "p", + null, + "Arcade handles OAuth, manages user tokens, and gives you 100+ pre-built integrations so your agents can take real action in production." + ), + createElement("hr", null), + + // Get Started links + createElement("h2", null, "Get Started"), + createElement( + "ul", + null, + createElement( + "li", + null, + createElement( + "a", + { href: "/get-started/quickstarts/call-tool-agent" }, + "Get Started with Arcade" + ) + ), + createElement( + "li", + null, + createElement("a", { href: "/resources/integrations" }, "Explore Tools") + ) + ), + + // Get Tools Section + createElement("h2", null, "Get Tools"), + createElement( + "ul", + null, + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations" }, + "Pre-built Integrations" + ), + " - Browse 100+ ready-to-use integrations for Gmail, Slack, GitHub, and more." + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/create-tools/tool-basics/build-mcp-server" }, + "Build Custom Tools" + ), + " - Create your own MCP servers and custom tools with our SDK." + ) + ), + + // Use Arcade Section + createElement("h2", null, "Use Arcade"), + createElement( + "ul", + null, + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/tool-calling/mcp-clients" }, + "Connect to Your IDE" + ), + " - Add tools to Cursor, VS Code, Claude Desktop, or any MCP client." + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/agent-frameworks" }, + "Power Your Agent" + ), + " - Integrate with LangChain, OpenAI Agents, CrewAI, Vercel AI, and more." + ) + ), + + // Popular Integrations + createElement("h2", null, "Popular Integrations"), + createElement( + "p", + null, + "Pre-built MCP servers ready to use with your agents. ", + createElement("a", { href: "/resources/integrations" }, "See all 100+") + ), + createElement( + "ul", + null, + // Row 1 + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/google-sheets" }, + "Google Sheets" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/jira" }, + "Jira" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/gmail" }, + "Gmail" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/confluence" }, + "Confluence" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/social-communication/slack" }, + "Slack" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/google-docs" }, + "Google Docs" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/google-slides" }, + "Google Slides" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/crm/hubspot" }, + "HubSpot" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/linear" }, + "Linear" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/google-drive" }, + "Google Drive" + ) + ), + // Row 2 + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/development/github" }, + "GitHub" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/social-communication/x" }, + "X" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { + href: "/resources/integrations/social-communication/microsoft-teams", + }, + "MS Teams" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/outlook-mail" }, + "Outlook" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/payments/stripe" }, + "Stripe" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/notion" }, + "Notion" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/asana" }, + "Asana" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/social-communication/reddit" }, + "Reddit" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/search/youtube" }, + "YouTube" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/resources/integrations/productivity/dropbox" }, + "Dropbox" + ) + ) + ), + + // Framework Compatibility + createElement("h2", null, "Works With Your Stack"), + createElement( + "p", + null, + "Arcade integrates with popular agent frameworks and LLM providers." + ), + createElement( + "ul", + null, + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/agent-frameworks/langchain/use-arcade-tools" }, + "LangChain" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/agent-frameworks/openai-agents/use-arcade-tools" }, + "OpenAI Agents" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/agent-frameworks/crewai/use-arcade-tools" }, + "CrewAI" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/agent-frameworks/vercelai" }, + "Vercel AI" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/agent-frameworks/google-adk/use-arcade-tools" }, + "Google ADK" + ) + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/guides/agent-frameworks/mastra/use-arcade-tools" }, + "Mastra" + ) + ) + ), + + // How Arcade Works + createElement("h2", null, "How Arcade Works"), + createElement( + "p", + null, + "Three core components that power your AI agents." + ), + createElement( + "ul", + null, + createElement( + "li", + null, + createElement( + "strong", + null, + createElement("a", { href: "/get-started/about-arcade" }, "Runtime") + ), + " - Your MCP server and agentic tool provider. Manages authentication, tool registration, and execution." + ), + createElement( + "li", + null, + createElement( + "strong", + null, + createElement( + "a", + { href: "/resources/integrations" }, + "Tool Catalog" + ) + ), + " - Catalog of pre-built tools and integrations. Browse 100+ ready-to-use MCP servers." + ), + createElement( + "li", + null, + createElement( + "strong", + null, + createElement( + "a", + { href: "/guides/tool-calling/custom-apps/auth-tool-calling" }, + "Agent Authorization" + ) + ), + " - Let agents act on behalf of users. Handle OAuth, API keys, and tokens for tools like Gmail and Google Drive." + ) + ), + + // Sample Applications + createElement("h2", null, "Sample Applications"), + createElement( + "p", + null, + "See Arcade in action with these example applications." + ), + createElement( + "ul", + null, + createElement( + "li", + null, + createElement("a", { href: "https://chat.arcade.dev/" }, "Arcade Chat"), + " - A chatbot that can help you with your daily tasks." + ), + createElement( + "li", + null, + createElement( + "a", + { href: "https://github.com/ArcadeAI/ArcadeSlackAgent" }, + "Archer" + ), + " - A bot for Slack that can act on your behalf." + ), + createElement( + "li", + null, + createElement( + "a", + { href: "https://github.com/dforwardfeed/slack-AIpodcast-summaries" }, + "YouTube Podcast Summarizer" + ), + " - A Slack bot that extracts and summarizes YouTube transcripts." + ) + ), + createElement( + "p", + null, + createElement("a", { href: "/resources/examples" }, "See all examples") + ), + + // Connect IDE callout + createElement("h2", null, "Connect Your IDE with Arcade's LLMs.txt"), + createElement( + "p", + null, + "Give Cursor, Claude Code, and other AI IDEs access to Arcade's documentation so they can write integration code for you. ", + createElement( + "a", + { href: "/get-started/setup/connect-arcade-docs" }, + "Learn more" + ) + ), + + // Quick Links + createElement("h2", null, "Quick Links"), + createElement( + "ul", + null, + createElement( + "li", + null, + createElement("a", { href: "/references/api" }, "API Reference") + ), + createElement( + "li", + null, + createElement( + "a", + { href: "/references/cli-cheat-sheet" }, + "CLI Cheat Sheet" + ) + ), + createElement( + "li", + null, + createElement("a", { href: "/resources/faq" }, "FAQ") + ), + createElement( + "li", + null, + createElement("a", { href: "/references/changelog" }, "Changelog") + ) + ) + ); +} + +// ============================================ +// MCP Servers Component - renders MCP server list as markdown +// ============================================ +function MarkdownMcpServers() { + // All available MCP servers (alphabetically sorted) + const allMcpServers = [ + { + label: "Airtable API", + href: "/resources/integrations/productivity/airtable-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Arcade Engine API", + href: "/resources/integrations/development/arcade-engine-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Asana", + href: "/resources/integrations/productivity/asana", + type: "arcade", + category: "productivity", + }, + { + label: "Asana API", + href: "/resources/integrations/productivity/asana-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Ashby API", + href: "/resources/integrations/productivity/ashby-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Box API", + href: "/resources/integrations/productivity/box-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Bright Data", + href: "/resources/integrations/development/brightdata", + type: "community", + category: "development", + }, + { + label: "Calendly API", + href: "/resources/integrations/productivity/calendly-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Clickhouse", + href: "/resources/integrations/databases/clickhouse", + type: "community", + category: "databases", + }, + { + label: "ClickUp", + href: "/resources/integrations/productivity/clickup", + type: "arcade", + category: "productivity", + }, + { + label: "ClickUp API", + href: "/resources/integrations/productivity/clickup-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Close.io", + href: "/resources/integrations/productivity/closeio", + type: "community", + category: "productivity", + }, + { + label: "Confluence", + href: "/resources/integrations/productivity/confluence", + type: "arcade", + category: "productivity", + }, + { + label: "Cursor Agents API", + href: "/resources/integrations/development/cursor-agents-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Customer.io API", + href: "/resources/integrations/customer-support/customerio-api", + type: "arcade_starter", + category: "customer-support", + }, + { + label: "Customer.io Pipelines API", + href: "/resources/integrations/customer-support/customerio-pipelines-api", + type: "arcade_starter", + category: "customer-support", + }, + { + label: "Customer.io Track API", + href: "/resources/integrations/customer-support/customerio-track-api", + type: "arcade_starter", + category: "customer-support", + }, + { + label: "Datadog API", + href: "/resources/integrations/development/datadog-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Discord", + href: "/resources/integrations/social-communication/discord", + type: "auth", + category: "social", + }, + { + label: "Dropbox", + href: "/resources/integrations/productivity/dropbox", + type: "arcade", + category: "productivity", + }, + { + label: "E2B", + href: "/resources/integrations/development/e2b", + type: "arcade", + category: "development", + }, + { + label: "Exa API", + href: "/resources/integrations/search/exa-api", + type: "arcade_starter", + category: "search", + }, + { + label: "Figma API", + href: "/resources/integrations/productivity/figma-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Firecrawl", + href: "/resources/integrations/development/firecrawl", + type: "arcade", + category: "development", + }, + { + label: "Freshservice API", + href: "/resources/integrations/customer-support/freshservice-api", + type: "arcade_starter", + category: "customer-support", + }, + { + label: "GitHub", + href: "/resources/integrations/development/github", + type: "arcade", + category: "development", + }, + { + label: "GitHub API", + href: "/resources/integrations/development/github-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Gmail", + href: "/resources/integrations/productivity/gmail", + type: "arcade", + category: "productivity", + }, + { + label: "Google Calendar", + href: "/resources/integrations/productivity/google-calendar", + type: "arcade", + category: "productivity", + }, + { + label: "Google Contacts", + href: "/resources/integrations/productivity/google-contacts", + type: "arcade", + category: "productivity", + }, + { + label: "Google Docs", + href: "/resources/integrations/productivity/google-docs", + type: "arcade", + category: "productivity", + }, + { + label: "Google Drive", + href: "/resources/integrations/productivity/google-drive", + type: "arcade", + category: "productivity", + }, + { + label: "Google Finance", + href: "/resources/integrations/search/google_finance", + type: "arcade", + category: "search", + }, + { + label: "Google Flights", + href: "/resources/integrations/search/google_flights", + type: "arcade", + category: "search", + }, + { + label: "Google Hotels", + href: "/resources/integrations/search/google_hotels", + type: "arcade", + category: "search", + }, + { + label: "Google Jobs", + href: "/resources/integrations/search/google_jobs", + type: "arcade", + category: "search", + }, + { + label: "Google Maps", + href: "/resources/integrations/search/google_maps", + type: "arcade", + category: "search", + }, + { + label: "Google News", + href: "/resources/integrations/search/google_news", + type: "arcade", + category: "search", + }, + { + label: "Google Search", + href: "/resources/integrations/search/google_search", + type: "arcade", + category: "search", + }, + { + label: "Google Sheets", + href: "/resources/integrations/productivity/google-sheets", + type: "arcade", + category: "productivity", + }, + { + label: "Google Shopping", + href: "/resources/integrations/search/google_shopping", + type: "arcade", + category: "search", + }, + { + label: "Google Slides", + href: "/resources/integrations/productivity/google-slides", + type: "arcade", + category: "productivity", + }, + { + label: "HubSpot", + href: "/resources/integrations/sales/hubspot", + type: "arcade", + category: "sales", + }, + { + label: "HubSpot Automation API", + href: "/resources/integrations/sales/hubspot-automation-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "HubSpot CMS API", + href: "/resources/integrations/sales/hubspot-cms-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "HubSpot Conversations API", + href: "/resources/integrations/sales/hubspot-conversations-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "HubSpot CRM API", + href: "/resources/integrations/sales/hubspot-crm-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "HubSpot Events API", + href: "/resources/integrations/sales/hubspot-events-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "HubSpot Marketing API", + href: "/resources/integrations/sales/hubspot-marketing-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "HubSpot Meetings API", + href: "/resources/integrations/sales/hubspot-meetings-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "HubSpot Users API", + href: "/resources/integrations/sales/hubspot-users-api", + type: "arcade_starter", + category: "sales", + }, + { + label: "Imgflip", + href: "/resources/integrations/entertainment/imgflip", + type: "arcade", + category: "entertainment", + }, + { + label: "Intercom API", + href: "/resources/integrations/customer-support/intercom-api", + type: "arcade_starter", + category: "customer-support", + }, + { + label: "Jira", + href: "/resources/integrations/productivity/jira", + type: "auth", + category: "productivity", + }, + { + label: "Linear", + href: "/resources/integrations/productivity/linear", + type: "arcade", + category: "productivity", + }, + { + label: "LinkedIn", + href: "/resources/integrations/social-communication/linkedin", + type: "arcade", + category: "social", + }, + { + label: "Luma API", + href: "/resources/integrations/productivity/luma-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Mailchimp API", + href: "/resources/integrations/productivity/mailchimp-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Microsoft SharePoint", + href: "/resources/integrations/productivity/sharepoint", + type: "arcade", + category: "productivity", + }, + { + label: "Microsoft Teams", + href: "/resources/integrations/social-communication/microsoft-teams", + type: "arcade", + category: "social", + }, + { + label: "Miro API", + href: "/resources/integrations/productivity/miro-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "MongoDB", + href: "/resources/integrations/databases/mongodb", + type: "community", + category: "databases", + }, + { + label: "Notion", + href: "/resources/integrations/productivity/notion", + type: "arcade", + category: "productivity", + }, + { + label: "Obsidian", + href: "/resources/integrations/productivity/obsidian", + type: "community", + category: "productivity", + }, + { + label: "Outlook Calendar", + href: "/resources/integrations/productivity/outlook-calendar", + type: "arcade", + category: "productivity", + }, + { + label: "Outlook Mail", + href: "/resources/integrations/productivity/outlook-mail", + type: "arcade", + category: "productivity", + }, + { + label: "PagerDuty API", + href: "/resources/integrations/development/pagerduty-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Postgres", + href: "/resources/integrations/databases/postgres", + type: "community", + category: "databases", + }, + { + label: "PostHog API", + href: "/resources/integrations/development/posthog-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Reddit", + href: "/resources/integrations/social-communication/reddit", + type: "arcade", + category: "social", + }, + { + label: "Salesforce", + href: "/resources/integrations/sales/salesforce", + type: "arcade", + category: "sales", + }, + { + label: "Slack", + href: "/resources/integrations/social-communication/slack", + type: "arcade", + category: "social", + }, + { + label: "Slack API", + href: "/resources/integrations/social-communication/slack-api", + type: "arcade_starter", + category: "social", + }, + { + label: "Spotify", + href: "/resources/integrations/entertainment/spotify", + type: "arcade", + category: "entertainment", + }, + { + label: "SquareUp API", + href: "/resources/integrations/productivity/squareup-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Stripe", + href: "/resources/integrations/payments/stripe", + type: "arcade", + category: "payments", + }, + { + label: "Stripe API", + href: "/resources/integrations/payments/stripe_api", + type: "arcade_starter", + category: "payments", + }, + { + label: "TickTick API", + href: "/resources/integrations/productivity/ticktick-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Trello API", + href: "/resources/integrations/productivity/trello-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "Twilio", + href: "/resources/integrations/social-communication/twilio", + type: "verified", + category: "social", + }, + { + label: "Twitch", + href: "/resources/integrations/entertainment/twitch", + type: "auth", + category: "entertainment", + }, + { + label: "Vercel API", + href: "/resources/integrations/development/vercel-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Walmart", + href: "/resources/integrations/search/walmart", + type: "arcade", + category: "search", + }, + { + label: "Weaviate API", + href: "/resources/integrations/development/weaviate-api", + type: "arcade_starter", + category: "development", + }, + { + label: "X", + href: "/resources/integrations/social-communication/x", + type: "arcade", + category: "social", + }, + { + label: "Xero API", + href: "/resources/integrations/productivity/xero-api", + type: "arcade_starter", + category: "productivity", + }, + { + label: "YouTube", + href: "/resources/integrations/search/youtube", + type: "arcade", + category: "search", + }, + { + label: "Zendesk", + href: "/resources/integrations/customer-support/zendesk", + type: "arcade", + category: "customer-support", + }, + { + label: "Zoho Books API", + href: "/resources/integrations/payments/zoho-books-api", + type: "arcade_starter", + category: "payments", + }, + { + label: "Zoho Creator API", + href: "/resources/integrations/productivity/zoho-creator-api", + type: "arcade_starter", + category: "development", + }, + { + label: "Zoom", + href: "/resources/integrations/social-communication/zoom", + type: "arcade", + category: "social", + }, + ]; + + const typeLabels: Record = { + arcade: "Arcade Optimized", + arcade_starter: "Arcade Starter", + verified: "Verified", + community: "Community", + auth: "Auth Provider", + }; + + const categoryLabels: Record = { + productivity: "Productivity", + development: "Development", + social: "Social", + search: "Search", + sales: "Sales", + payments: "Payments", + entertainment: "Entertainment", + databases: "Databases", + "customer-support": "Customer Support", + }; + + return createElement( + "div", + null, + createElement( + "p", + null, + "Registry of all MCP Servers available in the Arcade ecosystem. ", + createElement( + "a", + { href: "/guides/create-tools/tool-basics/build-mcp-server" }, + "Build your own MCP Server" + ), + "." + ), + createElement( + "ul", + null, + ...allMcpServers.map((server, i) => + createElement( + "li", + { key: i }, + createElement("a", { href: server.href }, server.label), + ` - ${typeLabels[server.type] || server.type}, ${categoryLabels[server.category] || server.category}` + ) + ) + ) + ); +} + +// All available markdown-friendly components +const markdownComponents: Record< + string, + React.ComponentType> +> = { + // Nextra components + Tabs: MarkdownTabs, + "Tabs.Tab": MarkdownTab, + Tab: MarkdownTab, + Steps: MarkdownSteps, + Callout: MarkdownCallout, + Cards: MarkdownCards, + "Cards.Card": MarkdownCard, + Card: MarkdownCard, + FileTree: MarkdownFileTree, + + // Custom components + GuideOverview: MarkdownGuideOverview, + "GuideOverview.Item": MarkdownGuideOverviewItem, + "GuideOverview.Outcomes": MarkdownGuideOverviewOutcomes, + "GuideOverview.Prerequisites": MarkdownGuideOverviewPrerequisites, + CheatSheetGrid: MarkdownCheatSheetGrid, + CheatSheetSection: MarkdownCheatSheetSection, + InfoBox: MarkdownInfoBox, + CommandItem: MarkdownCommandItem, + CommandList: MarkdownCommandList, + CommandBlock: MarkdownCommandBlock, + GlossaryTerm: MarkdownGlossaryTerm, + DashboardLink: MarkdownDashboardLink, + SignupLink: MarkdownSignupLink, + ToolCard: MarkdownToolCard, + MCPClientGrid: MarkdownMCPClientGrid, + ContactCards: MarkdownContactCards, + SubpageList: MarkdownSubpageList, + + // Page-level components + LandingPage: MarkdownLandingPage, + Toolkits: MarkdownMcpServers, + + // HTML-like components (uppercase - for custom components) + Video: MarkdownVideo, + Audio: MarkdownAudio, + Image: MarkdownImage, + + // ============================================ + // Lowercase HTML element overrides + // MDX passes these through as intrinsic elements + // ============================================ + + // Media elements - convert to links + video: MarkdownVideo, + audio: MarkdownAudio, + img: MarkdownImage, + iframe: MarkdownIframe, + embed: MarkdownPassthrough, + object: MarkdownPassthrough, + source: MarkdownPassthrough, + track: MarkdownPassthrough, + picture: MarkdownPassthrough, + + // Structural/semantic elements - strip wrapper, keep children + div: MarkdownPassthrough, + span: MarkdownPassthrough, + section: MarkdownPassthrough, + article: MarkdownPassthrough, + aside: MarkdownPassthrough, + header: MarkdownPassthrough, + footer: MarkdownPassthrough, + main: MarkdownPassthrough, + nav: MarkdownPassthrough, + address: MarkdownPassthrough, + hgroup: MarkdownPassthrough, + + // Figure elements + figure: MarkdownFigure, + figcaption: MarkdownFigcaption, + + // Interactive elements + details: MarkdownDetails, + summary: MarkdownSummary, + dialog: MarkdownPassthrough, + + // Self-closing elements + hr: MarkdownHr, + br: MarkdownBr, + wbr: MarkdownPassthrough, + + // Table elements - pass through for turndown + table: MarkdownTable, + thead: MarkdownThead, + tbody: MarkdownTbody, + tfoot: MarkdownTbody, + tr: MarkdownTr, + th: MarkdownTh, + td: MarkdownTd, + caption: MarkdownPassthrough, + colgroup: MarkdownPassthrough, + col: MarkdownPassthrough, + + // Code elements - preserve language info + pre: MarkdownPre, + code: MarkdownCode, + + // Definition lists + dl: MarkdownDl, + dt: MarkdownDt, + dd: MarkdownDd, + + // Form elements - strip (not useful in markdown) + form: MarkdownPassthrough, + input: MarkdownPassthrough, + button: MarkdownPassthrough, + select: MarkdownPassthrough, + option: MarkdownPassthrough, + optgroup: MarkdownPassthrough, + textarea: MarkdownPassthrough, + label: MarkdownPassthrough, + fieldset: MarkdownPassthrough, + legend: MarkdownPassthrough, + datalist: MarkdownPassthrough, + output: MarkdownPassthrough, + progress: MarkdownPassthrough, + meter: MarkdownPassthrough, + + // Script/style elements - remove entirely + script: () => null, + style: () => null, + noscript: MarkdownPassthrough, + template: MarkdownPassthrough, + slot: MarkdownPassthrough, + + // Canvas/SVG - remove (not representable in markdown) + canvas: () => null, + svg: () => null, + + // Misc inline elements - pass through + abbr: MarkdownPassthrough, + bdi: MarkdownPassthrough, + bdo: MarkdownPassthrough, + cite: MarkdownPassthrough, + data: MarkdownPassthrough, + dfn: MarkdownPassthrough, + kbd: MarkdownPassthrough, + mark: MarkdownPassthrough, + q: MarkdownPassthrough, + rb: MarkdownPassthrough, + rp: MarkdownPassthrough, + rt: MarkdownPassthrough, + rtc: MarkdownPassthrough, + ruby: MarkdownPassthrough, + s: MarkdownPassthrough, + samp: MarkdownPassthrough, + small: MarkdownPassthrough, + sub: MarkdownPassthrough, + sup: MarkdownPassthrough, + time: MarkdownPassthrough, + u: MarkdownPassthrough, + var: MarkdownPassthrough, + + // Map/area elements + map: MarkdownPassthrough, + area: MarkdownPassthrough, + + // Fallbacks + wrapper: PassThrough, +}; + +/** + * Strip import and export statements from MDX content + * This allows us to compile MDX without needing to resolve external modules + */ +function stripImportsAndExports(content: string): string { + let result = content; + + // Remove import statements (various formats) + // import X from 'module' + result = result.replace(/^import\s+.*?from\s+['"].*?['"];?\s*$/gm, ""); + // import 'module' + result = result.replace(/^import\s+['"].*?['"];?\s*$/gm, ""); + // import { X } from 'module' (multiline) + result = result.replace( + /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm, + "" + ); + + // Remove export statements + result = result.replace( + /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm, + "" + ); + + return result; +} + +/** + * Convert MDX content to clean Markdown + */ +export async function mdxToMarkdown( + mdxContent: string, + pagePath: string +): Promise { + // Extract frontmatter first + const frontmatterMatch = mdxContent.match(FRONTMATTER_REGEX); + let frontmatter = ""; + let contentWithoutFrontmatter = mdxContent; + + if (frontmatterMatch) { + frontmatter = frontmatterMatch[0]; + contentWithoutFrontmatter = mdxContent.slice(frontmatterMatch[0].length); + } + + try { + // Strip imports before compilation so MDX doesn't try to resolve them + const strippedContent = stripImportsAndExports(contentWithoutFrontmatter); + + // Compile MDX to JavaScript + // Include remarkGfm to properly parse GFM tables, strikethrough, etc. in the MDX source + const compiled = await compile(strippedContent, { + outputFormat: "function-body", + development: false, + remarkPlugins: [remarkGfm], + }); + + // Run the compiled code to get the component + // Use process.cwd() as baseUrl since we're not resolving any imports + const { default: MDXContent } = await run(String(compiled), { + Fragment: JsxFragment, + jsx, + jsxs, + baseUrl: pathToFileURL(process.cwd()).href, + }); + + // Render with markdown-friendly components + const element = createElement(MDXContent, { + components: markdownComponents, + }); + + // Convert React element to HTML string + const render = await getRenderer(); + const html = render(element); + + // Convert HTML to Markdown using unified ecosystem + let markdown = await htmlToMarkdown(html); + + // Clean up excessive whitespace + markdown = markdown.replace(/\n{3,}/g, "\n\n").trim(); + + // If result is empty, provide fallback + if (!markdown || markdown.length < 10) { + const title = extractTitle(frontmatter); + const description = extractDescription(frontmatter); + const htmlUrl = `https://docs.arcade.dev${pagePath}`; + return `${frontmatter}# ${title} + +${description} + +This page contains interactive content. Visit the full page at: ${htmlUrl} +`; + } + + return `${frontmatter}${markdown}\n`; + } catch (_error) { + return fallbackMdxToMarkdown(mdxContent, pagePath); + } +} + +/** + * Extract title from frontmatter + */ +function extractTitle(frontmatter: string): string { + const match = frontmatter.match(TITLE_REGEX); + return ( + match?.[1] || match?.[2] || match?.[3]?.trim() || "Arcade Documentation" + ); +} + +/** + * Extract description from frontmatter + */ +function extractDescription(frontmatter: string): string { + const match = frontmatter.match(DESCRIPTION_REGEX); + return match?.[1] || match?.[2] || match?.[3]?.trim() || ""; +} + +/** + * Fallback: Simple regex-based MDX to Markdown conversion + * Used when MDX compilation fails + */ +function fallbackMdxToMarkdown(content: string, pagePath: string): string { + let result = content; + + // Extract frontmatter + const frontmatterMatch = result.match(FRONTMATTER_REGEX); + let frontmatter = ""; + if (frontmatterMatch) { + frontmatter = frontmatterMatch[0]; + result = result.slice(frontmatterMatch[0].length); + } + + // Remove imports + result = result.replace(/^import\s+.*?from\s+['"].*?['"];?\s*$/gm, ""); + result = result.replace(/^import\s+['"].*?['"];?\s*$/gm, ""); + result = result.replace( + /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm, + "" + ); + + // Remove exports + result = result.replace( + /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm, + "" + ); + + // Remove self-closing JSX components (uppercase) + result = result.replace(/<([A-Z][a-zA-Z0-9.]*)[^>]*\/>/g, ""); + + // Remove self-closing HTML elements (lowercase) - but convert video/img to links + result = result.replace( + /]*src=["']([^"']+)["'][^>]*\/?>/gi, + "\n\n[Video]($1)\n\n" + ); + result = result.replace( + /]*src=["']([^"']+)["'][^>]*\/?>/gi, + "![]($1)" + ); + result = result.replace(/<[a-z][a-zA-Z0-9]*[^>]*\/>/g, ""); + + // Extract content from JSX with children (process iteratively for nesting) + let prev = ""; + while (prev !== result) { + prev = result; + // Uppercase components + result = result.replace( + /<([A-Z][a-zA-Z0-9.]*)[^>]*>([\s\S]*?)<\/\1>/g, + "$2" + ); + // Lowercase HTML elements (div, span, etc.) + result = result.replace( + /<(div|span|section|article|aside|header|footer|main|nav|video|figure|figcaption)[^>]*>([\s\S]*?)<\/\1>/gi, + "$2" + ); + } + + // Remove JSX expressions + result = result.replace(/\{[^}]+\}/g, ""); + + // Clean up + result = result.replace(/\n{3,}/g, "\n\n").trim(); + + if (!result || result.length < 10) { + const title = extractTitle(frontmatter); + const description = extractDescription(frontmatter); + const htmlUrl = `https://docs.arcade.dev${pagePath}`; + return `${frontmatter}# ${title} + +${description} + +This page contains interactive content. Visit the full page at: ${htmlUrl} +`; + } + + return `${frontmatter}${result}\n`; +} diff --git a/package.json b/package.json index ba7953195..f2316f47a 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "type": "module", "scripts": { "dev": "next dev --webpack", - "build": "next build --webpack", + "build": "pnpm run generate:markdown && next build --webpack", + "generate:markdown": "pnpm exec tsx scripts/generate-markdown.ts", "start": "next start", "lint": "pnpm dlx ultracite check", "format": "pnpm dlx ultracite fix", @@ -39,6 +40,7 @@ "homepage": "https://arcade.dev/", "dependencies": { "@arcadeai/design-system": "^3.26.0", + "@mdx-js/mdx": "^3.1.1", "@next/third-parties": "16.0.1", "@ory/client": "1.22.7", "@theguild/remark-mermaid": "0.3.0", @@ -54,8 +56,14 @@ "react-dom": "19.2.3", "react-hook-form": "7.65.0", "react-syntax-highlighter": "16.1.0", + "rehype-parse": "^9.0.1", + "rehype-remark": "^10.0.1", + "remark-gfm": "^4.0.1", + "remark-stringify": "^11.0.0", "swagger-ui-react": "^5.30.0", "tailwindcss-animate": "1.0.7", + "turndown": "^7.2.2", + "unified": "^11.0.5", "unist-util-visit": "5.0.0", "unist-util-visit-parents": "6.0.2", "zustand": "5.0.8" @@ -71,6 +79,7 @@ "@types/react": "19.2.7", "@types/react-dom": "19.2.3", "@types/react-syntax-highlighter": "15.5.13", + "@types/turndown": "^5.0.6", "@types/unist": "3.0.3", "commander": "14.0.2", "dotenv": "^17.2.3", @@ -87,6 +96,7 @@ "remark": "^15.0.1", "remark-rehype": "^11.1.2", "tailwindcss": "4.1.16", + "tsx": "^4.21.0", "typescript": "5.9.3", "ultracite": "6.1.0", "vitest": "4.0.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 436ac536b..7923f5808 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,7 +10,10 @@ importers: dependencies: '@arcadeai/design-system': specifier: ^3.26.0 - version: 3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2)) + version: 3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2)) + '@mdx-js/mdx': + specifier: ^3.1.1 + version: 3.1.1 '@next/third-parties': specifier: 16.0.1 version: 16.0.1(next@16.1.1(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3) @@ -56,12 +59,30 @@ importers: react-syntax-highlighter: specifier: 16.1.0 version: 16.1.0(react@19.2.3) + rehype-parse: + specifier: ^9.0.1 + version: 9.0.1 + rehype-remark: + specifier: ^10.0.1 + version: 10.0.1 + remark-gfm: + specifier: ^4.0.1 + version: 4.0.1 + remark-stringify: + specifier: ^11.0.0 + version: 11.0.0 swagger-ui-react: specifier: ^5.30.0 version: 5.31.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) tailwindcss-animate: specifier: 1.0.7 version: 1.0.7(tailwindcss@4.1.16) + turndown: + specifier: ^7.2.2 + version: 7.2.2 + unified: + specifier: ^11.0.5 + version: 11.0.5 unist-util-visit: specifier: 5.0.0 version: 5.0.0 @@ -102,6 +123,9 @@ importers: '@types/react-syntax-highlighter': specifier: 15.5.13 version: 15.5.13 + '@types/turndown': + specifier: ^5.0.6 + version: 5.0.6 '@types/unist': specifier: 3.0.3 version: 3.0.3 @@ -150,6 +174,9 @@ importers: tailwindcss: specifier: 4.1.16 version: 4.1.16 + tsx: + specifier: ^4.21.0 + version: 4.21.0 typescript: specifier: 5.9.3 version: 5.9.3 @@ -158,7 +185,7 @@ importers: version: 6.1.0(typescript@5.9.3) vitest: specifier: 4.0.5 - version: 4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) + version: 4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) zod: specifier: 4.1.12 version: 4.1.12 @@ -653,6 +680,9 @@ packages: '@mermaid-js/parser@0.6.3': resolution: {integrity: sha512-lnjOhe7zyHjc+If7yT4zoedx2vo4sHaTmtkl1+or8BRTnCtDmcTpAjpzDSfCZrshM5bCoz0GyidzadJAH1xobA==} + '@mixmark-io/domino@2.2.0': + resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==} + '@napi-rs/simple-git-android-arm-eabi@0.1.22': resolution: {integrity: sha512-JQZdnDNm8o43A5GOzwN/0Tz3CDBQtBUNqzVwEopm32uayjdjxev1Csp1JeaqF3v9djLDIvsSE39ecsN2LhCKKQ==} engines: {node: '>= 10'} @@ -2253,6 +2283,9 @@ packages: '@types/trusted-types@2.0.7': resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} + '@types/turndown@5.0.6': + resolution: {integrity: sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==} + '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} @@ -3015,6 +3048,9 @@ packages: resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} engines: {node: '>=16'} + get-tsconfig@4.13.0: + resolution: {integrity: sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==} + github-slugger@2.0.0: resolution: {integrity: sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==} @@ -3051,6 +3087,9 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} + hast-util-embedded@3.0.0: + resolution: {integrity: sha512-naH8sld4Pe2ep03qqULEtvYr7EjrLK2QHY8KJR6RJkTUjPGObe1vnx585uzem2hGra+s1q08DZZpfgDVYRbaXA==} + hast-util-from-dom@5.0.1: resolution: {integrity: sha512-N+LqofjR2zuzTjCPzyDUdSshy4Ma6li7p/c3pA78uTwzFgENbgbUrm2ugwsOdcjI1muO+o6Dgzp9p8WHtn/39Q==} @@ -3063,12 +3102,24 @@ packages: hast-util-from-parse5@8.0.3: resolution: {integrity: sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg==} + hast-util-has-property@3.0.0: + resolution: {integrity: sha512-MNilsvEKLFpV604hwfhVStK0usFY/QmM5zX16bo7EjnAEGofr5YyI37kzopBlZJkHD4t887i+q/C8/tr5Q94cA==} + + hast-util-is-body-ok-link@3.0.1: + resolution: {integrity: sha512-0qpnzOBLztXHbHQenVB8uNuxTnm/QBFUOmdOSsEn7GnBtyY07+ENTWVFBAnXd/zEgd9/SUG3lRY7hSIBWRgGpQ==} + hast-util-is-element@3.0.0: resolution: {integrity: sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==} + hast-util-minify-whitespace@1.0.1: + resolution: {integrity: sha512-L96fPOVpnclQE0xzdWb/D12VT5FabA7SnZOUMtL1DbXmYiHJMXZvFkIZfiMmTCNJHUeO2K9UYNXoVyfz+QHuOw==} + hast-util-parse-selector@4.0.0: resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==} + hast-util-phrasing@3.0.1: + resolution: {integrity: sha512-6h60VfI3uBQUxHqTyMymMZnEbNl1XmEGtOxxKYL7stY2o601COo62AWAYBQR9lZbYXYSBoxag8UpPRXK+9fqSQ==} + hast-util-raw@9.1.0: resolution: {integrity: sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==} @@ -3081,6 +3132,9 @@ packages: hast-util-to-jsx-runtime@2.3.6: resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==} + hast-util-to-mdast@10.1.2: + resolution: {integrity: sha512-FiCRI7NmOvM4y+f5w32jPRzcxDIz+PUqDwEqn1A+1q2cdp3B8Gx7aVrXORdOKjMNDQsD1ogOr896+0jJHW1EFQ==} + hast-util-to-parse5@8.0.1: resolution: {integrity: sha512-MlWT6Pjt4CG9lFCjiz4BH7l9wmrMkfkJYCxFwKQic8+RTZgWPuWxwAfjJElsXkex7DJjfSJsQIt931ilUgmwdA==} @@ -4165,6 +4219,9 @@ packages: rehype-katex@7.0.1: resolution: {integrity: sha512-OiM2wrZ/wuhKkigASodFoo8wimG3H12LWQaH8qSPVJn9apWKFSH3YOCtbKpBorTVw/eI7cuT21XBbvwEswbIOA==} + rehype-minify-whitespace@6.0.2: + resolution: {integrity: sha512-Zk0pyQ06A3Lyxhe9vGtOtzz3Z0+qZ5+7icZ/PL/2x1SHPbKao5oB/g/rlc6BCTajqBb33JcOe71Ye1oFsuYbnw==} + rehype-parse@9.0.1: resolution: {integrity: sha512-ksCzCD0Fgfh7trPDxr2rSylbwq9iYDkSn8TCDmEJ49ljEUBxDVCzCHv7QNzZOfODanX4+bWQ4WZqLCRWYLfhag==} @@ -4180,6 +4237,9 @@ packages: rehype-recma@1.0.0: resolution: {integrity: sha512-lqA4rGUf1JmacCNWWZx0Wv1dHqMwxzsDWYMTowuplHF3xH0N/MmrZ/G3BDZnzAkRmxDadujCjaKM2hqYdCBOGw==} + rehype-remark@10.0.1: + resolution: {integrity: sha512-EmDndlb5NVwXGfUa4c9GPK+lXeItTilLhE6ADSaQuHr4JUlKw9MidzGzx4HpqZrNCt6vnHmEifXQiiA+CEnjYQ==} + rehype-stringify@10.0.1: resolution: {integrity: sha512-k9ecfXHmIPuFVI61B9DeLPN0qFHfawM6RsuX48hoqlaKSF61RskNjSm1lI8PhBEM0MRdLxVVm4WmTqJQccH9mA==} @@ -4229,6 +4289,9 @@ packages: reselect@5.1.1: resolution: {integrity: sha512-K/BG6eIky/SBpzfHZv/dd+9JBFiS4SWV7FIujVyJRux6e45+73RaUHXLmIR1f7WOMaQ0U1km6qwklRQxpJJY0w==} + resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + restore-cursor@5.1.0: resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} engines: {node: '>=18'} @@ -4521,6 +4584,9 @@ packages: trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} + trim-trailing-lines@2.1.0: + resolution: {integrity: sha512-5UR5Biq4VlVOtzqkm2AZlgvSlDJtME46uV0br0gENbwN4l5+mMKT4b9gJKqWtuL2zAIqajGJGuvbCbcAJUZqBg==} + trough@2.2.0: resolution: {integrity: sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==} @@ -4568,6 +4634,14 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tsx@4.21.0: + resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} + engines: {node: '>=18.0.0'} + hasBin: true + + turndown@7.2.2: + resolution: {integrity: sha512-1F7db8BiExOKxjSMU2b7if62D/XOyQyZbPKq/nUwopfgnHlqXHqQ0lvfUTeUIr1lZJzOPFn43dODyMSIfvWRKQ==} + twoslash-protocol@0.3.4: resolution: {integrity: sha512-HHd7lzZNLUvjPzG/IE6js502gEzLC1x7HaO1up/f72d8G8ScWAs9Yfa97igelQRDl5h9tGcdFsRp+lNVre1EeQ==} @@ -4918,7 +4992,7 @@ snapshots: transitivePeerDependencies: - encoding - '@arcadeai/design-system@3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2))': + '@arcadeai/design-system@3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@arcadeai/arcadejs': 1.15.0 '@hookform/resolvers': 5.2.2(react-hook-form@7.65.0(react@19.2.3)) @@ -4943,7 +5017,7 @@ snapshots: '@radix-ui/react-tabs': 1.1.13(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-toggle': 1.1.10(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-tooltip': 1.2.8(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) - '@tailwindcss/vite': 4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2)) + '@tailwindcss/vite': 4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2)) class-variance-authority: 0.7.1 clsx: 2.1.1 cmdk: 1.1.1(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) @@ -5327,6 +5401,8 @@ snapshots: dependencies: langium: 3.3.1 + '@mixmark-io/domino@2.2.0': {} + '@napi-rs/simple-git-android-arm-eabi@0.1.22': optional: true @@ -6926,12 +7002,12 @@ snapshots: postcss: 8.5.6 tailwindcss: 4.1.16 - '@tailwindcss/vite@4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2))': + '@tailwindcss/vite@4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@tailwindcss/node': 4.1.14 '@tailwindcss/oxide': 4.1.14 tailwindcss: 4.1.14 - vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) + vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) '@tanstack/react-virtual@3.13.13(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: @@ -7158,6 +7234,8 @@ snapshots: '@types/trusted-types@2.0.7': optional: true + '@types/turndown@5.0.6': {} + '@types/unist@2.0.11': {} '@types/unist@3.0.3': {} @@ -7187,13 +7265,13 @@ snapshots: chai: 6.2.1 tinyrainbow: 3.0.3 - '@vitest/mocker@4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2))': + '@vitest/mocker@4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2))': dependencies: '@vitest/spy': 4.0.5 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) + vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) '@vitest/pretty-format@4.0.5': dependencies: @@ -7934,6 +8012,10 @@ snapshots: get-stream@8.0.1: {} + get-tsconfig@4.13.0: + dependencies: + resolve-pkg-maps: 1.0.0 + github-slugger@2.0.0: {} glob-parent@5.1.2: @@ -7967,6 +8049,11 @@ snapshots: dependencies: function-bind: 1.1.2 + hast-util-embedded@3.0.0: + dependencies: + '@types/hast': 3.0.4 + hast-util-is-element: 3.0.0 + hast-util-from-dom@5.0.1: dependencies: '@types/hast': 3.0.4 @@ -8000,14 +8087,38 @@ snapshots: vfile-location: 5.0.3 web-namespaces: 2.0.1 + hast-util-has-property@3.0.0: + dependencies: + '@types/hast': 3.0.4 + + hast-util-is-body-ok-link@3.0.1: + dependencies: + '@types/hast': 3.0.4 + hast-util-is-element@3.0.0: dependencies: '@types/hast': 3.0.4 + hast-util-minify-whitespace@1.0.1: + dependencies: + '@types/hast': 3.0.4 + hast-util-embedded: 3.0.0 + hast-util-is-element: 3.0.0 + hast-util-whitespace: 3.0.0 + unist-util-is: 6.0.1 + hast-util-parse-selector@4.0.0: dependencies: '@types/hast': 3.0.4 + hast-util-phrasing@3.0.1: + dependencies: + '@types/hast': 3.0.4 + hast-util-embedded: 3.0.0 + hast-util-has-property: 3.0.0 + hast-util-is-body-ok-link: 3.0.1 + hast-util-is-element: 3.0.0 + hast-util-raw@9.1.0: dependencies: '@types/hast': 3.0.4 @@ -8079,6 +8190,23 @@ snapshots: transitivePeerDependencies: - supports-color + hast-util-to-mdast@10.1.2: + dependencies: + '@types/hast': 3.0.4 + '@types/mdast': 4.0.4 + '@ungap/structured-clone': 1.3.0 + hast-util-phrasing: 3.0.1 + hast-util-to-html: 9.0.5 + hast-util-to-text: 4.0.2 + hast-util-whitespace: 3.0.0 + mdast-util-phrasing: 4.1.0 + mdast-util-to-hast: 13.2.1 + mdast-util-to-string: 4.0.0 + rehype-minify-whitespace: 6.0.2 + trim-trailing-lines: 2.1.0 + unist-util-position: 5.0.0 + unist-util-visit: 5.0.0 + hast-util-to-parse5@8.0.1: dependencies: '@types/hast': 3.0.4 @@ -9509,6 +9637,11 @@ snapshots: unist-util-visit-parents: 6.0.2 vfile: 6.0.3 + rehype-minify-whitespace@6.0.2: + dependencies: + '@types/hast': 3.0.4 + hast-util-minify-whitespace: 1.0.1 + rehype-parse@9.0.1: dependencies: '@types/hast': 3.0.4 @@ -9539,6 +9672,14 @@ snapshots: transitivePeerDependencies: - supports-color + rehype-remark@10.0.1: + dependencies: + '@types/hast': 3.0.4 + '@types/mdast': 4.0.4 + hast-util-to-mdast: 10.1.2 + unified: 11.0.5 + vfile: 6.0.3 + rehype-stringify@10.0.1: dependencies: '@types/hast': 3.0.4 @@ -9638,6 +9779,8 @@ snapshots: reselect@5.1.1: {} + resolve-pkg-maps@1.0.0: {} + restore-cursor@5.1.0: dependencies: onetime: 7.0.0 @@ -10029,6 +10172,8 @@ snapshots: trim-lines@3.0.1: {} + trim-trailing-lines@2.1.0: {} + trough@2.2.0: {} trpc-cli@0.12.1(@trpc/server@11.8.0(typescript@5.9.3))(zod@4.1.12): @@ -10053,6 +10198,17 @@ snapshots: tslib@2.8.1: {} + tsx@4.21.0: + dependencies: + esbuild: 0.27.1 + get-tsconfig: 4.13.0 + optionalDependencies: + fsevents: 2.3.3 + + turndown@7.2.2: + dependencies: + '@mixmark-io/domino': 2.2.0 + twoslash-protocol@0.3.4: {} twoslash@0.3.4(typescript@5.9.3): @@ -10238,7 +10394,7 @@ snapshots: d3-time: 3.1.0 d3-timer: 3.0.1 - vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2): + vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2): dependencies: esbuild: 0.27.1 fdir: 6.5.0(picomatch@4.0.3) @@ -10251,12 +10407,13 @@ snapshots: fsevents: 2.3.3 jiti: 2.6.1 lightningcss: 1.30.2 + tsx: 4.21.0 yaml: 2.8.2 - vitest@4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2): + vitest@4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2): dependencies: '@vitest/expect': 4.0.5 - '@vitest/mocker': 4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2)) + '@vitest/mocker': 4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2)) '@vitest/pretty-format': 4.0.5 '@vitest/runner': 4.0.5 '@vitest/snapshot': 4.0.5 @@ -10273,7 +10430,7 @@ snapshots: tinyexec: 0.3.2 tinyglobby: 0.2.15 tinyrainbow: 3.0.3 - vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) + vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) why-is-node-running: 2.3.0 optionalDependencies: '@types/debug': 4.1.12 diff --git a/proxy.ts b/proxy.ts index df8e7d83a..f9cd4b515 100644 --- a/proxy.ts +++ b/proxy.ts @@ -63,19 +63,18 @@ function pathnameIsMissingLocale(pathname: string): boolean { export function proxy(request: NextRequest) { const pathname = request.nextUrl.pathname; - // Handle .md requests without locale - redirect to add locale first - if (pathname.endsWith(".md") && pathnameIsMissingLocale(pathname)) { - const locale = getPreferredLocale(request); - const pathWithoutMd = pathname.replace(MD_EXTENSION_REGEX, ""); - const redirectPath = `/${locale}${pathWithoutMd}.md`; - return NextResponse.redirect(new URL(redirectPath, request.url)); - } - - // Rewrite .md requests (with locale) to the markdown API route - if (pathname.endsWith(".md") && !pathname.startsWith("/api/")) { - const url = request.nextUrl.clone(); - url.pathname = `/api/markdown${pathname}`; - return NextResponse.rewrite(url); + // .md requests are served as static files from public/ + // (generated at build time by scripts/generate-markdown.ts) + if (pathname.endsWith(".md")) { + // Add locale prefix if missing, then let Next.js serve from public/ + if (pathnameIsMissingLocale(pathname)) { + const locale = getPreferredLocale(request); + const pathWithoutMd = pathname.replace(MD_EXTENSION_REGEX, ""); + const redirectPath = `/${locale}${pathWithoutMd}.md`; + return NextResponse.redirect(new URL(redirectPath, request.url)); + } + // Let Next.js serve the static file from public/ + return NextResponse.next(); } // Redirect if there is no locale diff --git a/scripts/generate-markdown.ts b/scripts/generate-markdown.ts new file mode 100644 index 000000000..618f9953f --- /dev/null +++ b/scripts/generate-markdown.ts @@ -0,0 +1,155 @@ +/** + * Static Markdown Generation Script + * + * Generates pre-rendered markdown files from MDX pages for LLM consumption. + * Outputs to public/en/ so files are served directly by Next.js/CDN. + * + * Usage: pnpm dlx tsx scripts/generate-markdown.ts + */ + +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import fastGlob from "fast-glob"; +import { mdxToMarkdown } from "../lib/mdx-to-markdown"; + +const OUTPUT_DIR = join(process.cwd(), "public"); +const SEPARATOR_WIDTH = 50; +const RESOURCE_FILE_REGEX = + /\.(png|jpg|jpeg|gif|svg|webp|mp4|webm|pdf|zip|tar|gz)$/i; + +/** + * Rewrite internal links to point to .md files + * - /references/foo → /en/references/foo.md + * - /en/references/foo → /en/references/foo.md + * - External links, anchors, and resource links are unchanged + */ +function rewriteLinksToMarkdown(markdown: string): string { + // Match markdown links: [text](url) + return markdown.replace( + /\[([^\]]*)\]\(([^)]+)\)/g, + (match, text, url: string) => { + // Skip external links + if ( + url.startsWith("http://") || + url.startsWith("https://") || + url.startsWith("mailto:") + ) { + return match; + } + + // Skip anchor-only links + if (url.startsWith("#")) { + return match; + } + + // Skip resource links (images, videos, files) + if ( + RESOURCE_FILE_REGEX.test(url) || + url.startsWith("/images/") || + url.startsWith("/videos/") || + url.startsWith("/files/") + ) { + return match; + } + + // Skip if already has .md extension + if (url.endsWith(".md")) { + return match; + } + + // Handle anchor in URL + let anchor = ""; + let pathPart = url; + const anchorIndex = url.indexOf("#"); + if (anchorIndex !== -1) { + anchor = url.slice(anchorIndex); + pathPart = url.slice(0, anchorIndex); + } + + // Add /en prefix if not present (internal doc links) + if (pathPart.startsWith("/") && !pathPart.startsWith("/en/")) { + pathPart = `/en${pathPart}`; + } + + // Add .md extension + const newUrl = `${pathPart}.md${anchor}`; + return `[${text}](${newUrl})`; + } + ); +} + +async function generateMarkdownFiles() { + console.log("Generating static markdown files...\n"); + + // Find all MDX pages + const mdxFiles = await fastGlob("app/en/**/page.mdx", { + cwd: process.cwd(), + absolute: false, + }); + + console.log(`Found ${mdxFiles.length} MDX files\n`); + + let successCount = 0; + let errorCount = 0; + const errors: { file: string; error: string }[] = []; + + for (const mdxFile of mdxFiles) { + try { + // Read MDX content + const mdxPath = join(process.cwd(), mdxFile); + const mdxContent = await readFile(mdxPath, "utf-8"); + + // Compute paths + // app/en/references/auth-providers/page.mdx → /en/references/auth-providers + const relativePath = mdxFile + .replace("app/", "/") + .replace("/page.mdx", ""); + + // Convert to markdown + let markdown = await mdxToMarkdown(mdxContent, relativePath); + + // Rewrite links to point to .md files + markdown = rewriteLinksToMarkdown(markdown); + + // Output path: public/en/references/auth-providers.md + const outputPath = join(OUTPUT_DIR, `${relativePath}.md`); + + // Ensure directory exists + await mkdir(dirname(outputPath), { recursive: true }); + + // Write markdown file + await writeFile(outputPath, markdown, "utf-8"); + + successCount += 1; + process.stdout.write(`✓ ${relativePath}.md\n`); + } catch (error) { + errorCount += 1; + const errorMessage = + error instanceof Error ? error.message : String(error); + errors.push({ file: mdxFile, error: errorMessage }); + process.stdout.write(`✗ ${mdxFile}: ${errorMessage}\n`); + } + } + + console.log(`\n${"=".repeat(SEPARATOR_WIDTH)}`); + console.log(`Generated: ${successCount} files`); + if (errorCount > 0) { + console.log(`Errors: ${errorCount} files`); + console.log("\nFailed files:"); + for (const { file, error } of errors) { + console.log(` - ${file}: ${error}`); + } + } + console.log("=".repeat(SEPARATOR_WIDTH)); + + // Exit with error code if any files failed + if (errorCount > 0) { + process.exit(1); + } +} + +// Run the script +generateMarkdownFiles().catch((error) => { + console.error("Fatal error:", error); + process.exit(1); +}); From 0ea4dbb5d666cc97ff25f26552e027f5fd8f612e Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Tue, 20 Jan 2026 17:54:06 +0000 Subject: [PATCH 08/15] Revert "Generate static markdown files at build time" This reverts commit d7b7c711018863a34b02b9b998b3711904439f98. --- .gitignore | 1 - app/api/markdown/[[...slug]]/route.ts | 295 +++ .../integrations/productivity/gmail/page.mdx | 2 +- lib/mdx-to-markdown.tsx | 2314 ----------------- package.json | 12 +- pnpm-lock.yaml | 181 +- proxy.ts | 25 +- scripts/generate-markdown.ts | 155 -- 8 files changed, 322 insertions(+), 2663 deletions(-) create mode 100644 app/api/markdown/[[...slug]]/route.ts delete mode 100644 lib/mdx-to-markdown.tsx delete mode 100644 scripts/generate-markdown.ts diff --git a/.gitignore b/.gitignore index 5968906cb..c714c9723 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,6 @@ node_modules .DS_Store .env.local public/sitemap*.xml -public/en/**/*.md .env _pagefind/ diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts new file mode 100644 index 000000000..10812365f --- /dev/null +++ b/app/api/markdown/[[...slug]]/route.ts @@ -0,0 +1,295 @@ +import { access, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { type NextRequest, NextResponse } from "next/server"; + +export const dynamic = "force-dynamic"; + +// Regex pattern for removing .md extension +const MD_EXTENSION_REGEX = /\.md$/; + +// Regex patterns for MDX to Markdown compilation (top-level for performance) +const FRONTMATTER_REGEX = /^---\n([\s\S]*?)\n---\n?/; +const IMPORT_FROM_REGEX = /^import\s+.*?from\s+['"].*?['"];?\s*$/gm; +const IMPORT_DIRECT_REGEX = /^import\s+['"].*?['"];?\s*$/gm; +const IMPORT_DESTRUCTURE_REGEX = + /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm; +const EXPORT_REGEX = + /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm; +const SELF_CLOSING_JSX_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*\/>/g; +const JSX_WITH_CHILDREN_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*>([\s\S]*?)<\/\1>/g; +const CODE_BLOCK_REGEX = /```[\s\S]*?```/g; +const JSX_EXPRESSION_REGEX = /\{[^}]+\}/g; +const EXCESSIVE_NEWLINES_REGEX = /\n{3,}/g; +const CODE_BLOCK_PLACEHOLDER_REGEX = /__CODE_BLOCK_(\d+)__/g; + +// Regex for detecting markdown list items and numbered lists +const UNORDERED_LIST_REGEX = /^[-*+]\s/; +const ORDERED_LIST_REGEX = /^\d+[.)]\s/; + +// Regex for extracting frontmatter fields +// Handles: "double quoted", 'single quoted', or unquoted values +// Group 1 = double-quoted content, Group 2 = single-quoted content, Group 3 = unquoted/fallback +// Quoted patterns require closing quote at end of line to prevent apostrophes being misread as delimiters +const TITLE_REGEX = /title:\s*(?:"([^"]*)"\s*$|'([^']*)'\s*$|([^\n]+))/; +const DESCRIPTION_REGEX = + /description:\s*(?:"([^"]*)"\s*$|'([^']*)'\s*$|([^\n]+))/; + +// Regex for detecting leading whitespace on lines +const LEADING_WHITESPACE_REGEX = /^[ \t]+/; + +/** + * Removes consistent leading indentation from all lines of text. + * This normalizes content that was indented inside JSX components. + * Code block markers (```) are ignored when calculating minimum indent + * since they typically start at column 0 in MDX files. + */ +function dedent(text: string): string { + const lines = text.split("\n"); + + // Find minimum indentation, ignoring: + // - Empty lines + // - Code block markers (lines starting with ```) + let minIndent = Number.POSITIVE_INFINITY; + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed === "" || trimmed.startsWith("```")) { + continue; // Ignore empty lines and code block markers + } + const match = line.match(LEADING_WHITESPACE_REGEX); + const indent = match ? match[0].length : 0; + if (indent < minIndent) { + minIndent = indent; + } + } + + // If no indentation found, return as-is + if (minIndent === 0 || minIndent === Number.POSITIVE_INFINITY) { + return text; + } + + // Remove the minimum indentation from each line (except code block content) + return lines + .map((line) => { + const trimmed = line.trim(); + // Don't modify empty lines or lines with less indentation than min + if (trimmed === "" || line.length < minIndent) { + return line.trimStart(); + } + // Preserve code block markers - just remove leading whitespace + // This matches the logic that ignores them when calculating minIndent + if (trimmed.startsWith("```")) { + return trimmed; + } + return line.slice(minIndent); + }) + .join("\n"); +} + +/** + * Strips surrounding quotes from a value if present. + * Used for unquoted fallback values that may contain quotes due to apostrophe handling. + */ +function stripSurroundingQuotes(value: string): string { + const trimmed = value.trim(); + if ( + (trimmed.startsWith('"') && trimmed.endsWith('"')) || + (trimmed.startsWith("'") && trimmed.endsWith("'")) + ) { + return trimmed.slice(1, -1); + } + return trimmed; +} + +/** + * Extracts title and description from frontmatter. + * Handles double-quoted, single-quoted, and unquoted YAML values. + */ +function extractFrontmatterMeta(frontmatter: string): { + title: string; + description: string; +} { + const titleMatch = frontmatter.match(TITLE_REGEX); + const descriptionMatch = frontmatter.match(DESCRIPTION_REGEX); + + // Extract from whichever capture group matched: + // Group 1 = double-quoted, Group 2 = single-quoted, Group 3 = unquoted/fallback + // For group 3 (fallback), strip surrounding quotes if present + const title = + titleMatch?.[1] ?? + titleMatch?.[2] ?? + stripSurroundingQuotes(titleMatch?.[3] ?? ""); + const description = + descriptionMatch?.[1] ?? + descriptionMatch?.[2] ?? + stripSurroundingQuotes(descriptionMatch?.[3] ?? ""); + + return { + title: title || "Arcade Documentation", + description, + }; +} + +/** + * Normalizes indentation in the final output. + * Removes stray leading whitespace outside code blocks while preserving + * meaningful markdown indentation (nested lists, blockquotes). + */ +function normalizeIndentation(text: string): string { + const finalLines: string[] = []; + let inCodeBlock = false; + + for (const line of text.split("\n")) { + if (line.trim().startsWith("```")) { + inCodeBlock = !inCodeBlock; + finalLines.push(line.trimStart()); // Code block markers should start at column 0 + } else if (inCodeBlock) { + finalLines.push(line); // Preserve indentation inside code blocks + } else { + const trimmed = line.trimStart(); + // Preserve indentation for nested list items and blockquotes + const isListItem = + UNORDERED_LIST_REGEX.test(trimmed) || ORDERED_LIST_REGEX.test(trimmed); + const isBlockquote = trimmed.startsWith(">"); + if ((isListItem || isBlockquote) && line.startsWith(" ")) { + // Keep markdown-meaningful indentation (but normalize to 2-space increments) + const leadingSpaces = line.length - line.trimStart().length; + const normalizedIndent = " ".repeat(Math.floor(leadingSpaces / 2)); + finalLines.push(normalizedIndent + trimmed); + } else { + finalLines.push(trimmed); // Remove leading whitespace for other lines + } + } + } + + return finalLines.join("\n"); +} + +/** + * Compiles MDX content to clean markdown by: + * - Preserving frontmatter + * - Removing import statements + * - Converting JSX components to their text content + * - Preserving standard markdown + * - Providing fallback content for component-only pages + */ +function compileMdxToMarkdown(content: string, pagePath: string): string { + let result = content; + + // Extract and preserve frontmatter if present + let frontmatter = ""; + const frontmatterMatch = result.match(FRONTMATTER_REGEX); + if (frontmatterMatch) { + frontmatter = frontmatterMatch[0]; + result = result.slice(frontmatterMatch[0].length); + } + + // Remove import statements (various formats) + result = result.replace(IMPORT_FROM_REGEX, ""); + result = result.replace(IMPORT_DIRECT_REGEX, ""); + result = result.replace(IMPORT_DESTRUCTURE_REGEX, ""); + + // Remove export statements (like export const metadata) + result = result.replace(EXPORT_REGEX, ""); + + // Process self-closing JSX components (e.g., or ) + // Handles components with dots like + result = result.replace(SELF_CLOSING_JSX_REGEX, ""); + + // Process JSX components with children - extract the text content + // Handles components with dots like content + // Keep processing until no more JSX components remain + let previousResult = ""; + while (previousResult !== result) { + previousResult = result; + // Match opening tag, capture tag name (with dots), and content until matching closing tag + // Apply dedent to each extracted piece to normalize indentation + result = result.replace(JSX_WITH_CHILDREN_REGEX, (_, _tag, innerContent) => + dedent(innerContent.trim()) + ); + } + + // Remove any remaining JSX expressions like {variable} or {expression} + // But preserve code blocks by temporarily replacing them + const codeBlocks: string[] = []; + result = result.replace(CODE_BLOCK_REGEX, (match) => { + codeBlocks.push(match); + return `__CODE_BLOCK_${codeBlocks.length - 1}__`; + }); + + // Now remove JSX expressions outside code blocks + result = result.replace(JSX_EXPRESSION_REGEX, ""); + + // Restore code blocks + result = result.replace( + CODE_BLOCK_PLACEHOLDER_REGEX, + (_, index) => codeBlocks[Number.parseInt(index, 10)] + ); + + // Normalize indentation (remove stray whitespace, preserve meaningful markdown indentation) + result = normalizeIndentation(result); + + // Clean up excessive blank lines (more than 2 consecutive) + result = result.replace(EXCESSIVE_NEWLINES_REGEX, "\n\n"); + + // Trim leading/trailing whitespace + result = result.trim(); + + // If content is essentially empty (component-only page), provide fallback + if (!result || result.length < 10) { + const { title, description } = extractFrontmatterMeta(frontmatter); + const htmlUrl = `https://docs.arcade.dev${pagePath}`; + return `${frontmatter}# ${title} + +${description} + +This page contains interactive content. Visit the full page at: ${htmlUrl} +`; + } + + // Reconstruct with frontmatter + return `${frontmatter}${result}\n`; +} + +export async function GET( + request: NextRequest, + _context: { params: Promise<{ slug?: string[] }> } +) { + try { + // Get the original pathname from the request + const url = new URL(request.url); + // Remove /api/markdown prefix to get the original path + const originalPath = url.pathname.replace("/api/markdown", ""); + + // Remove .md extension + const pathWithoutMd = originalPath.replace(MD_EXTENSION_REGEX, ""); + + // Map URL to file path + // e.g., /en/home/quickstart -> app/en/home/quickstart/page.mdx + const filePath = join(process.cwd(), "app", `${pathWithoutMd}/page.mdx`); + + // Check if file exists + try { + await access(filePath); + } catch { + return new NextResponse("Markdown file not found", { status: 404 }); + } + + const rawContent = await readFile(filePath, "utf-8"); + + // Compile MDX to clean markdown + const content = compileMdxToMarkdown(rawContent, pathWithoutMd); + + // Return the compiled markdown with proper headers + return new NextResponse(content, { + status: 200, + headers: { + "Content-Type": "text/markdown; charset=utf-8", + "Content-Disposition": "inline", + }, + }); + } catch (error) { + return new NextResponse(`Internal server error: ${error}`, { + status: 500, + }); + } +} diff --git a/app/en/resources/integrations/productivity/gmail/page.mdx b/app/en/resources/integrations/productivity/gmail/page.mdx index 7e68ebc22..f20dc6303 100644 --- a/app/en/resources/integrations/productivity/gmail/page.mdx +++ b/app/en/resources/integrations/productivity/gmail/page.mdx @@ -272,7 +272,7 @@ Delete a draft email using the Gmail API. The `TrashEmail` tool is currently only available on a self-hosted instance of the Arcade Engine. To learn more about self-hosting, see the [self-hosting - documentation](/guides/deployment-hosting/configure-engine). + documentation](http://localhost:3000/en/home/deployment/engine-configuration).
diff --git a/lib/mdx-to-markdown.tsx b/lib/mdx-to-markdown.tsx deleted file mode 100644 index 0e8345df4..000000000 --- a/lib/mdx-to-markdown.tsx +++ /dev/null @@ -1,2314 +0,0 @@ -/** - * MDX to Markdown converter - * - * Compiles MDX content, renders it with markdown-friendly components, - * then converts the resulting HTML to clean Markdown using the unified ecosystem. - */ - -import { pathToFileURL } from "node:url"; -import { compile, run } from "@mdx-js/mdx"; -import type { ReactNode } from "react"; -import { createElement, Fragment } from "react"; -import { Fragment as JsxFragment, jsx, jsxs } from "react/jsx-runtime"; -import rehypeParse from "rehype-parse"; -import rehypeRemark from "rehype-remark"; -import remarkGfm from "remark-gfm"; -import remarkStringify from "remark-stringify"; -import { unified } from "unified"; - -// Regex patterns (at top level for performance) -const FILE_EXTENSION_REGEX = /\.[^.]+$/; -const UNDERSCORE_DASH_REGEX = /[_-]/g; -const FRONTMATTER_REGEX = /^---\n([\s\S]*?)\n---\n?/; -const TITLE_REGEX = /title:\s*(?:"([^"]*)"|'([^']*)'|([^\n]+))/; -const DESCRIPTION_REGEX = /description:\s*(?:"([^"]*)"|'([^']*)'|([^\n]+))/; - -// Types for mdast table nodes -type MdastTableCell = { type: "tableCell"; children: unknown[] }; -type MdastTableRow = { type: "tableRow"; children: MdastTableCell[] }; -type HtmlNode = { - type: string; - tagName?: string; - children?: HtmlNode[]; - properties?: Record; -}; -type StateAll = (node: HtmlNode) => unknown[]; - -/** Extract cells from a table row element */ -function extractCellsFromRow( - state: { all: StateAll }, - row: HtmlNode -): MdastTableCell[] { - const cells: MdastTableCell[] = []; - for (const cell of row.children || []) { - if ( - cell.type === "element" && - (cell.tagName === "th" || cell.tagName === "td") - ) { - cells.push({ type: "tableCell", children: state.all(cell) }); - } - } - return cells; -} - -/** Extract rows from a table section (thead, tbody, tfoot) or direct tr children */ -function extractRowsFromTableSection( - state: { all: StateAll }, - section: HtmlNode -): MdastTableRow[] { - const rows: MdastTableRow[] = []; - for (const child of section.children || []) { - if (child.type === "element" && child.tagName === "tr") { - const cells = extractCellsFromRow(state, child); - if (cells.length > 0) { - rows.push({ type: "tableRow", children: cells }); - } - } - } - return rows; -} - -/** Check if element is a table section (thead, tbody, tfoot) */ -function isTableSection(tagName: string | undefined): boolean { - return tagName === "thead" || tagName === "tbody" || tagName === "tfoot"; -} - -// Dynamic import to avoid Next.js RSC restrictions -let renderToStaticMarkup: typeof import("react-dom/server").renderToStaticMarkup; -async function getRenderer() { - if (!renderToStaticMarkup) { - const reactDomServer = await import("react-dom/server"); - renderToStaticMarkup = reactDomServer.renderToStaticMarkup; - } - return renderToStaticMarkup; -} - -/** - * Convert HTML to Markdown using unified ecosystem (rehype-remark) - * This is more reliable than turndown for complex HTML structures - */ -async function htmlToMarkdown(html: string): Promise { - const result = await unified() - .use(rehypeParse, { fragment: true }) - .use(rehypeRemark, { - handlers: { - // Custom handler for video elements - video: (_state, node) => { - const src = (node.properties?.src as string) || ""; - const filename = - src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Video"; - const title = filename.replace(UNDERSCORE_DASH_REGEX, " "); - return { - type: "paragraph", - children: [ - { - type: "link", - url: src, - children: [{ type: "text", value: `Video: ${title}` }], - }, - ], - }; - }, - // Custom handler for audio elements - audio: (_state, node) => { - const src = (node.properties?.src as string) || ""; - const filename = - src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Audio"; - const title = filename.replace(UNDERSCORE_DASH_REGEX, " "); - return { - type: "paragraph", - children: [ - { - type: "link", - url: src, - children: [{ type: "text", value: `Audio: ${title}` }], - }, - ], - }; - }, - // Custom handler for iframe elements - iframe: (_state, node) => { - const src = (node.properties?.src as string) || ""; - const title = - (node.properties?.title as string) || "Embedded content"; - return { - type: "paragraph", - children: [ - { - type: "link", - url: src, - children: [{ type: "text", value: title }], - }, - ], - }; - }, - // Custom handler for HTML tables - convert to markdown tables - // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Table parsing requires nested logic - table: (state, node) => { - const rows: MdastTableRow[] = []; - - for (const child of node.children || []) { - if (child.type !== "element") { - continue; - } - if (isTableSection(child.tagName)) { - rows.push( - ...extractRowsFromTableSection(state, child as HtmlNode) - ); - } else if (child.tagName === "tr") { - const cells = extractCellsFromRow(state, child as HtmlNode); - if (cells.length > 0) { - rows.push({ type: "tableRow", children: cells }); - } - } - } - - const colCount = rows[0]?.children?.length || 0; - return { - type: "table", - align: new Array(colCount).fill(null), - children: rows, - }; - }, - // These are handled by the table handler above, but we still need to define them - // to prevent "unknown node" errors when encountered outside tables - thead: (state, node) => { - const rows: unknown[] = []; - for (const child of node.children || []) { - if (child.type === "element" && child.tagName === "tr") { - rows.push(...state.all(child)); - } - } - return rows; - }, - tbody: (state, node) => { - const rows: unknown[] = []; - for (const child of node.children || []) { - if (child.type === "element" && child.tagName === "tr") { - rows.push(...state.all(child)); - } - } - return rows; - }, - tfoot: (state, node) => { - const rows: unknown[] = []; - for (const child of node.children || []) { - if (child.type === "element" && child.tagName === "tr") { - rows.push(...state.all(child)); - } - } - return rows; - }, - tr: (state, node) => { - const cells: { type: "tableCell"; children: unknown[] }[] = []; - for (const child of node.children || []) { - if ( - child.type === "element" && - (child.tagName === "th" || child.tagName === "td") - ) { - cells.push({ - type: "tableCell", - children: state.all(child), - }); - } - } - return { - type: "tableRow", - children: cells, - }; - }, - th: (state, node) => ({ - type: "tableCell", - children: state.all(node), - }), - td: (state, node) => ({ - type: "tableCell", - children: state.all(node), - }), - // Custom handler for callout divs - render as paragraph with bold label - // biome-ignore lint/complexity/noExcessiveCognitiveComplexity: Callout parsing logic - div: (state, node) => { - const className = - (node.properties?.className as string[])?.join(" ") || ""; - - // Check if this is a callout - if ( - className.includes("callout") || - className.includes("admonition") || - className.includes("warning") || - className.includes("info") || - className.includes("error") || - className.includes("tip") - ) { - let label = ""; - if (className.includes("warning")) { - label = "Warning"; - } else if (className.includes("error")) { - label = "Error"; - } else if (className.includes("tip")) { - label = "Tip"; - } else if (className.includes("info")) { - label = "Note"; - } - - // Process children and prepend bold label - const children = state.all(node); - if (label && children.length > 0) { - // Add bold label to the first paragraph's children - const firstChild = children[0]; - if (firstChild && firstChild.type === "paragraph") { - return [ - { - type: "paragraph", - children: [ - { - type: "strong", - children: [{ type: "text", value: `${label}:` }], - }, - { type: "text", value: " " }, - ...(firstChild.children || []), - ], - }, - ...children.slice(1), - ]; - } - } - return children; - } - - // Default: just return children (strip the div wrapper) - return state.all(node); - }, - }, - }) - .use(remarkGfm) // Enable GFM for tables, strikethrough, etc. - .use(remarkStringify, { - bullet: "-", - fences: true, - listItemIndent: "one", - }) - .process(html); - - return String(result); -} - -// ============================================ -// Markdown-Friendly Component Implementations -// ============================================ - -// Simple wrapper that just renders children -function PassThrough({ children }: { children?: ReactNode }) { - return createElement(Fragment, null, children); -} - -// Tabs - render all tab content with headers -function MarkdownTabs({ - children, - items, -}: { - children?: ReactNode; - items?: string[]; -}) { - // If we have items array, children are the tab panels - if (items && Array.isArray(items)) { - const childArray = Array.isArray(children) ? children : [children]; - return createElement( - "div", - null, - childArray.map((child, i) => - createElement( - "div", - { key: i }, - createElement("h4", null, items[i] || `Option ${i + 1}`), - child - ) - ) - ); - } - return createElement("div", null, children); -} - -// Tab content - just render the content -function MarkdownTab({ children }: { children?: ReactNode }) { - return createElement("div", null, children); -} - -// Assign Tab to Tabs for Tabs.Tab syntax -MarkdownTabs.Tab = MarkdownTab; - -// Steps - render as numbered sections -function MarkdownSteps({ children }: { children?: ReactNode }) { - return createElement("div", { className: "steps" }, children); -} - -// Callout - render as a styled div that turndown will convert -function MarkdownCallout({ - children, - type = "info", -}: { - children?: ReactNode; - type?: string; -}) { - return createElement("div", { className: `callout ${type}` }, children); -} - -// Cards - render as sections -function MarkdownCards({ children }: { children?: ReactNode }) { - return createElement("div", null, children); -} - -function MarkdownCard({ - children, - title, - href, -}: { - children?: ReactNode; - title?: string; - href?: string; -}) { - if (href) { - return createElement( - "div", - null, - title && createElement("h4", null, createElement("a", { href }, title)), - children - ); - } - return createElement( - "div", - null, - title && createElement("h4", null, title), - children - ); -} - -MarkdownCards.Card = MarkdownCard; - -// FileTree - render as a code block -function MarkdownFileTree({ children }: { children?: ReactNode }) { - return createElement("pre", null, createElement("code", null, children)); -} - -// Link components - render as standard links -function _MarkdownLink({ - children, - href, -}: { - children?: ReactNode; - href?: string; -}) { - return createElement("a", { href }, children); -} - -// ============================================ -// HTML Element Handlers -// ============================================ - -// Video - convert to a descriptive link -function MarkdownVideo({ - src, - title, - children, -}: { - src?: string; - title?: string; - children?: ReactNode; -}) { - if (!src) { - return createElement(Fragment, null, children); - } - const filename = - src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Video"; - const videoTitle = title || filename.replace(UNDERSCORE_DASH_REGEX, " "); - return createElement("p", null, `[Video: ${videoTitle}](${src})`); -} - -// Audio - convert to a descriptive link -function MarkdownAudio({ - src, - title, - children, -}: { - src?: string; - title?: string; - children?: ReactNode; -}) { - if (!src) { - return createElement(Fragment, null, children); - } - const filename = - src.split("/").pop()?.replace(FILE_EXTENSION_REGEX, "") || "Audio"; - const audioTitle = title || filename.replace(UNDERSCORE_DASH_REGEX, " "); - return createElement("p", null, `[Audio: ${audioTitle}](${src})`); -} - -// Image - keep as img for turndown to handle -function MarkdownImage({ - src, - alt, - title, -}: { - src?: string; - alt?: string; - title?: string; -}) { - return createElement("img", { src, alt: alt || title || "" }); -} - -// Iframe - convert to link -function MarkdownIframe({ src, title }: { src?: string; title?: string }) { - if (!src) { - return null; - } - const label = title || "Embedded content"; - return createElement("p", null, `[${label}](${src})`); -} - -// HR - render as markdown horizontal rule -function MarkdownHr() { - return createElement("hr", null); -} - -// BR - render as line break -function MarkdownBr() { - return createElement("br", null); -} - -// Container elements - just pass through children (strips the wrapper) -function MarkdownPassthrough({ children }: { children?: ReactNode }) { - return createElement(Fragment, null, children); -} - -// Figure/Figcaption - extract content -function MarkdownFigure({ children }: { children?: ReactNode }) { - return createElement("figure", null, children); -} - -function MarkdownFigcaption({ children }: { children?: ReactNode }) { - return createElement("figcaption", null, children); -} - -// Details/Summary - convert to blockquote-style -function MarkdownDetails({ children }: { children?: ReactNode }) { - return createElement("blockquote", null, children); -} - -function MarkdownSummary({ children }: { children?: ReactNode }) { - return createElement("strong", null, children); -} - -// Table elements - pass through for turndown -function MarkdownTable({ children }: { children?: ReactNode }) { - return createElement("table", null, children); -} - -function MarkdownThead({ children }: { children?: ReactNode }) { - return createElement("thead", null, children); -} - -function MarkdownTbody({ children }: { children?: ReactNode }) { - return createElement("tbody", null, children); -} - -function MarkdownTr({ children }: { children?: ReactNode }) { - return createElement("tr", null, children); -} - -function MarkdownTh({ children }: { children?: ReactNode }) { - return createElement("th", null, children); -} - -function MarkdownTd({ children }: { children?: ReactNode }) { - return createElement("td", null, children); -} - -// Definition lists -function MarkdownDl({ children }: { children?: ReactNode }) { - return createElement("dl", null, children); -} - -function MarkdownDt({ children }: { children?: ReactNode }) { - return createElement("dt", null, createElement("strong", null, children)); -} - -function MarkdownDd({ children }: { children?: ReactNode }) { - return createElement("dd", null, children); -} - -// Code block elements - preserve language and content -function MarkdownPre({ - children, - ...props -}: { - children?: ReactNode; - [key: string]: unknown; -}) { - // Extract data-language if present (MDX sometimes uses this) - const lang = props["data-language"] || ""; - return createElement("pre", { "data-language": lang }, children); -} - -function MarkdownCode({ - children, - className, - ...props -}: { - children?: ReactNode; - className?: string; - [key: string]: unknown; -}) { - // Preserve the language class for turndown - return createElement( - "code", - { className: className || "", ...props }, - children - ); -} - -// GuideOverview custom component -function MarkdownGuideOverview({ children }: { children?: ReactNode }) { - return createElement("div", null, children); -} - -function MarkdownGuideOverviewItem({ - children, - title, - href, -}: { - children?: ReactNode; - title?: string; - href?: string; -}) { - return createElement( - "div", - null, - title && - createElement( - "h4", - null, - href ? createElement("a", { href }, title) : title - ), - children - ); -} - -function MarkdownGuideOverviewOutcomes({ children }: { children?: ReactNode }) { - return createElement(Fragment, null, children); -} - -function MarkdownGuideOverviewPrerequisites({ - children, -}: { - children?: ReactNode; -}) { - return createElement( - "div", - null, - createElement("strong", null, "Prerequisites:"), - children - ); -} - -MarkdownGuideOverview.Item = MarkdownGuideOverviewItem; -MarkdownGuideOverview.Outcomes = MarkdownGuideOverviewOutcomes; -MarkdownGuideOverview.Prerequisites = MarkdownGuideOverviewPrerequisites; - -// CheatSheet components -function MarkdownCheatSheetGrid({ children }: { children?: ReactNode }) { - return createElement(Fragment, null, children); -} - -function MarkdownCheatSheetSection({ - children, - title, - icon, -}: { - children?: ReactNode; - title?: string; - icon?: string; -}) { - return createElement( - "div", - null, - title && createElement("h3", null, icon ? `${icon} ${title}` : title), - children - ); -} - -function MarkdownInfoBox({ - children, - type = "info", -}: { - children?: ReactNode; - type?: string; -}) { - // Use a div with callout class so the callout handler processes it - // The div handler in htmlToMarkdown will add the bold label - return createElement("div", { className: `callout ${type}` }, children); -} - -function MarkdownCommandItem({ children }: { children?: ReactNode }) { - return createElement("div", null, children); -} - -function MarkdownCommandList({ children }: { children?: ReactNode }) { - return createElement("div", null, children); -} - -function MarkdownCommandBlock({ children }: { children?: ReactNode }) { - return createElement("div", null, children); -} - -// GlossaryTerm - just render the text -function MarkdownGlossaryTerm({ - children, - term, -}: { - children?: ReactNode; - term?: string; -}) { - return createElement("strong", { title: term }, children); -} - -// DashboardLink - render as a link -function MarkdownDashboardLink({ - children, - href, -}: { - children?: ReactNode; - href?: string; -}) { - const dashboardUrl = href || "https://api.arcade.dev/dashboard"; - return createElement("a", { href: dashboardUrl }, children); -} - -// SignupLink - render as a link -function MarkdownSignupLink({ - children, -}: { - children?: ReactNode; - linkLocation?: string; -}) { - return createElement( - "a", - { href: "https://api.arcade.dev/dashboard/signup" }, - children - ); -} - -// Generic catch-all for unknown components -function _MarkdownGeneric({ children }: { children?: ReactNode }) { - return createElement("div", null, children); -} - -// ToolCard - render as a link with summary -function MarkdownToolCard({ - name, - summary, - link, - category: _category, -}: { - name?: string; - image?: string; - summary?: string; - link?: string; - category?: string; -}) { - return createElement( - "li", - null, - createElement("a", { href: link }, name), - summary ? ` - ${summary}` : "" - ); -} - -// MCPClientGrid - render as a list of MCP clients -function MarkdownMCPClientGrid() { - const mcpClients = [ - { - key: "cursor", - name: "Cursor", - description: "AI-powered code editor with built-in MCP support", - }, - { - key: "claude-desktop", - name: "Claude Desktop", - description: "Anthropic's desktop app for Claude with MCP integration", - }, - { - key: "visual-studio-code", - name: "Visual Studio Code", - description: "Microsoft's code editor with MCP extensions", - }, - ]; - - return createElement( - "ul", - null, - ...mcpClients.map((client, i) => - createElement( - "li", - { key: i }, - createElement( - "a", - { href: `/guides/tool-calling/mcp-clients/${client.key}` }, - client.name - ), - ` - ${client.description}` - ) - ) - ); -} - -// ContactCards - render contact information -function MarkdownContactCards() { - const contacts = [ - { - title: "Email Support", - description: - "Get help with technical issues, account questions, and general inquiries.", - href: "mailto:support@arcade.dev", - }, - { - title: "Contact Sales", - description: - "Discuss enterprise plans, pricing, and how Arcade can help your organization.", - href: "mailto:sales@arcade.dev", - }, - { - title: "Discord Community", - description: - "Join for real-time help, connect with developers, and stay updated.", - href: "https://discord.gg/GUZEMpEZ9p", - }, - { - title: "GitHub Issues & Discussions", - description: "Report bugs, request features, and contribute to Arcade.", - href: "https://github.com/ArcadeAI/arcade-mcp", - }, - { - title: "Security Research", - description: "Report security vulnerabilities responsibly.", - href: "/guides/security/security-research-program", - }, - { - title: "System Status", - description: "Check the current status of Arcade's services.", - href: "https://status.arcade.dev", - }, - ]; - - return createElement( - "ul", - null, - ...contacts.map((contact, i) => - createElement( - "li", - { key: i }, - createElement("a", { href: contact.href }, contact.title), - ` - ${contact.description}` - ) - ) - ); -} - -// SubpageList - render list of subpages (uses provided meta) -function MarkdownSubpageList({ - basePath, - meta, -}: { - basePath?: string; - meta?: Record; -}) { - if (!(meta && basePath)) { - return null; - } - - const subpages = Object.entries(meta).filter( - ([key]) => key !== "index" && key !== "*" - ); - - return createElement( - "ul", - null, - ...subpages.map(([key, title], i) => { - let linkText: string; - if (typeof title === "string") { - linkText = title; - } else if ( - typeof title === "object" && - title !== null && - "title" in title - ) { - linkText = (title as { title: string }).title; - } else { - linkText = String(title); - } - return createElement( - "li", - { key: i }, - createElement("a", { href: `${basePath}/${key}` }, linkText) - ); - }) - ); -} - -// ============================================ -// Landing Page Component - renders as markdown-friendly content -// ============================================ -function MarkdownLandingPage() { - return createElement( - "div", - null, - // Hero - createElement("h1", null, "MCP Runtime for AI agents that get things done"), - createElement( - "p", - null, - "Arcade handles OAuth, manages user tokens, and gives you 100+ pre-built integrations so your agents can take real action in production." - ), - createElement("hr", null), - - // Get Started links - createElement("h2", null, "Get Started"), - createElement( - "ul", - null, - createElement( - "li", - null, - createElement( - "a", - { href: "/get-started/quickstarts/call-tool-agent" }, - "Get Started with Arcade" - ) - ), - createElement( - "li", - null, - createElement("a", { href: "/resources/integrations" }, "Explore Tools") - ) - ), - - // Get Tools Section - createElement("h2", null, "Get Tools"), - createElement( - "ul", - null, - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations" }, - "Pre-built Integrations" - ), - " - Browse 100+ ready-to-use integrations for Gmail, Slack, GitHub, and more." - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/create-tools/tool-basics/build-mcp-server" }, - "Build Custom Tools" - ), - " - Create your own MCP servers and custom tools with our SDK." - ) - ), - - // Use Arcade Section - createElement("h2", null, "Use Arcade"), - createElement( - "ul", - null, - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/tool-calling/mcp-clients" }, - "Connect to Your IDE" - ), - " - Add tools to Cursor, VS Code, Claude Desktop, or any MCP client." - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/agent-frameworks" }, - "Power Your Agent" - ), - " - Integrate with LangChain, OpenAI Agents, CrewAI, Vercel AI, and more." - ) - ), - - // Popular Integrations - createElement("h2", null, "Popular Integrations"), - createElement( - "p", - null, - "Pre-built MCP servers ready to use with your agents. ", - createElement("a", { href: "/resources/integrations" }, "See all 100+") - ), - createElement( - "ul", - null, - // Row 1 - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/google-sheets" }, - "Google Sheets" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/jira" }, - "Jira" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/gmail" }, - "Gmail" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/confluence" }, - "Confluence" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/social-communication/slack" }, - "Slack" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/google-docs" }, - "Google Docs" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/google-slides" }, - "Google Slides" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/crm/hubspot" }, - "HubSpot" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/linear" }, - "Linear" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/google-drive" }, - "Google Drive" - ) - ), - // Row 2 - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/development/github" }, - "GitHub" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/social-communication/x" }, - "X" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { - href: "/resources/integrations/social-communication/microsoft-teams", - }, - "MS Teams" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/outlook-mail" }, - "Outlook" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/payments/stripe" }, - "Stripe" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/notion" }, - "Notion" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/asana" }, - "Asana" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/social-communication/reddit" }, - "Reddit" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/search/youtube" }, - "YouTube" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/resources/integrations/productivity/dropbox" }, - "Dropbox" - ) - ) - ), - - // Framework Compatibility - createElement("h2", null, "Works With Your Stack"), - createElement( - "p", - null, - "Arcade integrates with popular agent frameworks and LLM providers." - ), - createElement( - "ul", - null, - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/agent-frameworks/langchain/use-arcade-tools" }, - "LangChain" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/agent-frameworks/openai-agents/use-arcade-tools" }, - "OpenAI Agents" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/agent-frameworks/crewai/use-arcade-tools" }, - "CrewAI" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/agent-frameworks/vercelai" }, - "Vercel AI" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/agent-frameworks/google-adk/use-arcade-tools" }, - "Google ADK" - ) - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/guides/agent-frameworks/mastra/use-arcade-tools" }, - "Mastra" - ) - ) - ), - - // How Arcade Works - createElement("h2", null, "How Arcade Works"), - createElement( - "p", - null, - "Three core components that power your AI agents." - ), - createElement( - "ul", - null, - createElement( - "li", - null, - createElement( - "strong", - null, - createElement("a", { href: "/get-started/about-arcade" }, "Runtime") - ), - " - Your MCP server and agentic tool provider. Manages authentication, tool registration, and execution." - ), - createElement( - "li", - null, - createElement( - "strong", - null, - createElement( - "a", - { href: "/resources/integrations" }, - "Tool Catalog" - ) - ), - " - Catalog of pre-built tools and integrations. Browse 100+ ready-to-use MCP servers." - ), - createElement( - "li", - null, - createElement( - "strong", - null, - createElement( - "a", - { href: "/guides/tool-calling/custom-apps/auth-tool-calling" }, - "Agent Authorization" - ) - ), - " - Let agents act on behalf of users. Handle OAuth, API keys, and tokens for tools like Gmail and Google Drive." - ) - ), - - // Sample Applications - createElement("h2", null, "Sample Applications"), - createElement( - "p", - null, - "See Arcade in action with these example applications." - ), - createElement( - "ul", - null, - createElement( - "li", - null, - createElement("a", { href: "https://chat.arcade.dev/" }, "Arcade Chat"), - " - A chatbot that can help you with your daily tasks." - ), - createElement( - "li", - null, - createElement( - "a", - { href: "https://github.com/ArcadeAI/ArcadeSlackAgent" }, - "Archer" - ), - " - A bot for Slack that can act on your behalf." - ), - createElement( - "li", - null, - createElement( - "a", - { href: "https://github.com/dforwardfeed/slack-AIpodcast-summaries" }, - "YouTube Podcast Summarizer" - ), - " - A Slack bot that extracts and summarizes YouTube transcripts." - ) - ), - createElement( - "p", - null, - createElement("a", { href: "/resources/examples" }, "See all examples") - ), - - // Connect IDE callout - createElement("h2", null, "Connect Your IDE with Arcade's LLMs.txt"), - createElement( - "p", - null, - "Give Cursor, Claude Code, and other AI IDEs access to Arcade's documentation so they can write integration code for you. ", - createElement( - "a", - { href: "/get-started/setup/connect-arcade-docs" }, - "Learn more" - ) - ), - - // Quick Links - createElement("h2", null, "Quick Links"), - createElement( - "ul", - null, - createElement( - "li", - null, - createElement("a", { href: "/references/api" }, "API Reference") - ), - createElement( - "li", - null, - createElement( - "a", - { href: "/references/cli-cheat-sheet" }, - "CLI Cheat Sheet" - ) - ), - createElement( - "li", - null, - createElement("a", { href: "/resources/faq" }, "FAQ") - ), - createElement( - "li", - null, - createElement("a", { href: "/references/changelog" }, "Changelog") - ) - ) - ); -} - -// ============================================ -// MCP Servers Component - renders MCP server list as markdown -// ============================================ -function MarkdownMcpServers() { - // All available MCP servers (alphabetically sorted) - const allMcpServers = [ - { - label: "Airtable API", - href: "/resources/integrations/productivity/airtable-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Arcade Engine API", - href: "/resources/integrations/development/arcade-engine-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Asana", - href: "/resources/integrations/productivity/asana", - type: "arcade", - category: "productivity", - }, - { - label: "Asana API", - href: "/resources/integrations/productivity/asana-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Ashby API", - href: "/resources/integrations/productivity/ashby-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Box API", - href: "/resources/integrations/productivity/box-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Bright Data", - href: "/resources/integrations/development/brightdata", - type: "community", - category: "development", - }, - { - label: "Calendly API", - href: "/resources/integrations/productivity/calendly-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Clickhouse", - href: "/resources/integrations/databases/clickhouse", - type: "community", - category: "databases", - }, - { - label: "ClickUp", - href: "/resources/integrations/productivity/clickup", - type: "arcade", - category: "productivity", - }, - { - label: "ClickUp API", - href: "/resources/integrations/productivity/clickup-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Close.io", - href: "/resources/integrations/productivity/closeio", - type: "community", - category: "productivity", - }, - { - label: "Confluence", - href: "/resources/integrations/productivity/confluence", - type: "arcade", - category: "productivity", - }, - { - label: "Cursor Agents API", - href: "/resources/integrations/development/cursor-agents-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Customer.io API", - href: "/resources/integrations/customer-support/customerio-api", - type: "arcade_starter", - category: "customer-support", - }, - { - label: "Customer.io Pipelines API", - href: "/resources/integrations/customer-support/customerio-pipelines-api", - type: "arcade_starter", - category: "customer-support", - }, - { - label: "Customer.io Track API", - href: "/resources/integrations/customer-support/customerio-track-api", - type: "arcade_starter", - category: "customer-support", - }, - { - label: "Datadog API", - href: "/resources/integrations/development/datadog-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Discord", - href: "/resources/integrations/social-communication/discord", - type: "auth", - category: "social", - }, - { - label: "Dropbox", - href: "/resources/integrations/productivity/dropbox", - type: "arcade", - category: "productivity", - }, - { - label: "E2B", - href: "/resources/integrations/development/e2b", - type: "arcade", - category: "development", - }, - { - label: "Exa API", - href: "/resources/integrations/search/exa-api", - type: "arcade_starter", - category: "search", - }, - { - label: "Figma API", - href: "/resources/integrations/productivity/figma-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Firecrawl", - href: "/resources/integrations/development/firecrawl", - type: "arcade", - category: "development", - }, - { - label: "Freshservice API", - href: "/resources/integrations/customer-support/freshservice-api", - type: "arcade_starter", - category: "customer-support", - }, - { - label: "GitHub", - href: "/resources/integrations/development/github", - type: "arcade", - category: "development", - }, - { - label: "GitHub API", - href: "/resources/integrations/development/github-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Gmail", - href: "/resources/integrations/productivity/gmail", - type: "arcade", - category: "productivity", - }, - { - label: "Google Calendar", - href: "/resources/integrations/productivity/google-calendar", - type: "arcade", - category: "productivity", - }, - { - label: "Google Contacts", - href: "/resources/integrations/productivity/google-contacts", - type: "arcade", - category: "productivity", - }, - { - label: "Google Docs", - href: "/resources/integrations/productivity/google-docs", - type: "arcade", - category: "productivity", - }, - { - label: "Google Drive", - href: "/resources/integrations/productivity/google-drive", - type: "arcade", - category: "productivity", - }, - { - label: "Google Finance", - href: "/resources/integrations/search/google_finance", - type: "arcade", - category: "search", - }, - { - label: "Google Flights", - href: "/resources/integrations/search/google_flights", - type: "arcade", - category: "search", - }, - { - label: "Google Hotels", - href: "/resources/integrations/search/google_hotels", - type: "arcade", - category: "search", - }, - { - label: "Google Jobs", - href: "/resources/integrations/search/google_jobs", - type: "arcade", - category: "search", - }, - { - label: "Google Maps", - href: "/resources/integrations/search/google_maps", - type: "arcade", - category: "search", - }, - { - label: "Google News", - href: "/resources/integrations/search/google_news", - type: "arcade", - category: "search", - }, - { - label: "Google Search", - href: "/resources/integrations/search/google_search", - type: "arcade", - category: "search", - }, - { - label: "Google Sheets", - href: "/resources/integrations/productivity/google-sheets", - type: "arcade", - category: "productivity", - }, - { - label: "Google Shopping", - href: "/resources/integrations/search/google_shopping", - type: "arcade", - category: "search", - }, - { - label: "Google Slides", - href: "/resources/integrations/productivity/google-slides", - type: "arcade", - category: "productivity", - }, - { - label: "HubSpot", - href: "/resources/integrations/sales/hubspot", - type: "arcade", - category: "sales", - }, - { - label: "HubSpot Automation API", - href: "/resources/integrations/sales/hubspot-automation-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "HubSpot CMS API", - href: "/resources/integrations/sales/hubspot-cms-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "HubSpot Conversations API", - href: "/resources/integrations/sales/hubspot-conversations-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "HubSpot CRM API", - href: "/resources/integrations/sales/hubspot-crm-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "HubSpot Events API", - href: "/resources/integrations/sales/hubspot-events-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "HubSpot Marketing API", - href: "/resources/integrations/sales/hubspot-marketing-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "HubSpot Meetings API", - href: "/resources/integrations/sales/hubspot-meetings-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "HubSpot Users API", - href: "/resources/integrations/sales/hubspot-users-api", - type: "arcade_starter", - category: "sales", - }, - { - label: "Imgflip", - href: "/resources/integrations/entertainment/imgflip", - type: "arcade", - category: "entertainment", - }, - { - label: "Intercom API", - href: "/resources/integrations/customer-support/intercom-api", - type: "arcade_starter", - category: "customer-support", - }, - { - label: "Jira", - href: "/resources/integrations/productivity/jira", - type: "auth", - category: "productivity", - }, - { - label: "Linear", - href: "/resources/integrations/productivity/linear", - type: "arcade", - category: "productivity", - }, - { - label: "LinkedIn", - href: "/resources/integrations/social-communication/linkedin", - type: "arcade", - category: "social", - }, - { - label: "Luma API", - href: "/resources/integrations/productivity/luma-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Mailchimp API", - href: "/resources/integrations/productivity/mailchimp-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Microsoft SharePoint", - href: "/resources/integrations/productivity/sharepoint", - type: "arcade", - category: "productivity", - }, - { - label: "Microsoft Teams", - href: "/resources/integrations/social-communication/microsoft-teams", - type: "arcade", - category: "social", - }, - { - label: "Miro API", - href: "/resources/integrations/productivity/miro-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "MongoDB", - href: "/resources/integrations/databases/mongodb", - type: "community", - category: "databases", - }, - { - label: "Notion", - href: "/resources/integrations/productivity/notion", - type: "arcade", - category: "productivity", - }, - { - label: "Obsidian", - href: "/resources/integrations/productivity/obsidian", - type: "community", - category: "productivity", - }, - { - label: "Outlook Calendar", - href: "/resources/integrations/productivity/outlook-calendar", - type: "arcade", - category: "productivity", - }, - { - label: "Outlook Mail", - href: "/resources/integrations/productivity/outlook-mail", - type: "arcade", - category: "productivity", - }, - { - label: "PagerDuty API", - href: "/resources/integrations/development/pagerduty-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Postgres", - href: "/resources/integrations/databases/postgres", - type: "community", - category: "databases", - }, - { - label: "PostHog API", - href: "/resources/integrations/development/posthog-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Reddit", - href: "/resources/integrations/social-communication/reddit", - type: "arcade", - category: "social", - }, - { - label: "Salesforce", - href: "/resources/integrations/sales/salesforce", - type: "arcade", - category: "sales", - }, - { - label: "Slack", - href: "/resources/integrations/social-communication/slack", - type: "arcade", - category: "social", - }, - { - label: "Slack API", - href: "/resources/integrations/social-communication/slack-api", - type: "arcade_starter", - category: "social", - }, - { - label: "Spotify", - href: "/resources/integrations/entertainment/spotify", - type: "arcade", - category: "entertainment", - }, - { - label: "SquareUp API", - href: "/resources/integrations/productivity/squareup-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Stripe", - href: "/resources/integrations/payments/stripe", - type: "arcade", - category: "payments", - }, - { - label: "Stripe API", - href: "/resources/integrations/payments/stripe_api", - type: "arcade_starter", - category: "payments", - }, - { - label: "TickTick API", - href: "/resources/integrations/productivity/ticktick-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Trello API", - href: "/resources/integrations/productivity/trello-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "Twilio", - href: "/resources/integrations/social-communication/twilio", - type: "verified", - category: "social", - }, - { - label: "Twitch", - href: "/resources/integrations/entertainment/twitch", - type: "auth", - category: "entertainment", - }, - { - label: "Vercel API", - href: "/resources/integrations/development/vercel-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Walmart", - href: "/resources/integrations/search/walmart", - type: "arcade", - category: "search", - }, - { - label: "Weaviate API", - href: "/resources/integrations/development/weaviate-api", - type: "arcade_starter", - category: "development", - }, - { - label: "X", - href: "/resources/integrations/social-communication/x", - type: "arcade", - category: "social", - }, - { - label: "Xero API", - href: "/resources/integrations/productivity/xero-api", - type: "arcade_starter", - category: "productivity", - }, - { - label: "YouTube", - href: "/resources/integrations/search/youtube", - type: "arcade", - category: "search", - }, - { - label: "Zendesk", - href: "/resources/integrations/customer-support/zendesk", - type: "arcade", - category: "customer-support", - }, - { - label: "Zoho Books API", - href: "/resources/integrations/payments/zoho-books-api", - type: "arcade_starter", - category: "payments", - }, - { - label: "Zoho Creator API", - href: "/resources/integrations/productivity/zoho-creator-api", - type: "arcade_starter", - category: "development", - }, - { - label: "Zoom", - href: "/resources/integrations/social-communication/zoom", - type: "arcade", - category: "social", - }, - ]; - - const typeLabels: Record = { - arcade: "Arcade Optimized", - arcade_starter: "Arcade Starter", - verified: "Verified", - community: "Community", - auth: "Auth Provider", - }; - - const categoryLabels: Record = { - productivity: "Productivity", - development: "Development", - social: "Social", - search: "Search", - sales: "Sales", - payments: "Payments", - entertainment: "Entertainment", - databases: "Databases", - "customer-support": "Customer Support", - }; - - return createElement( - "div", - null, - createElement( - "p", - null, - "Registry of all MCP Servers available in the Arcade ecosystem. ", - createElement( - "a", - { href: "/guides/create-tools/tool-basics/build-mcp-server" }, - "Build your own MCP Server" - ), - "." - ), - createElement( - "ul", - null, - ...allMcpServers.map((server, i) => - createElement( - "li", - { key: i }, - createElement("a", { href: server.href }, server.label), - ` - ${typeLabels[server.type] || server.type}, ${categoryLabels[server.category] || server.category}` - ) - ) - ) - ); -} - -// All available markdown-friendly components -const markdownComponents: Record< - string, - React.ComponentType> -> = { - // Nextra components - Tabs: MarkdownTabs, - "Tabs.Tab": MarkdownTab, - Tab: MarkdownTab, - Steps: MarkdownSteps, - Callout: MarkdownCallout, - Cards: MarkdownCards, - "Cards.Card": MarkdownCard, - Card: MarkdownCard, - FileTree: MarkdownFileTree, - - // Custom components - GuideOverview: MarkdownGuideOverview, - "GuideOverview.Item": MarkdownGuideOverviewItem, - "GuideOverview.Outcomes": MarkdownGuideOverviewOutcomes, - "GuideOverview.Prerequisites": MarkdownGuideOverviewPrerequisites, - CheatSheetGrid: MarkdownCheatSheetGrid, - CheatSheetSection: MarkdownCheatSheetSection, - InfoBox: MarkdownInfoBox, - CommandItem: MarkdownCommandItem, - CommandList: MarkdownCommandList, - CommandBlock: MarkdownCommandBlock, - GlossaryTerm: MarkdownGlossaryTerm, - DashboardLink: MarkdownDashboardLink, - SignupLink: MarkdownSignupLink, - ToolCard: MarkdownToolCard, - MCPClientGrid: MarkdownMCPClientGrid, - ContactCards: MarkdownContactCards, - SubpageList: MarkdownSubpageList, - - // Page-level components - LandingPage: MarkdownLandingPage, - Toolkits: MarkdownMcpServers, - - // HTML-like components (uppercase - for custom components) - Video: MarkdownVideo, - Audio: MarkdownAudio, - Image: MarkdownImage, - - // ============================================ - // Lowercase HTML element overrides - // MDX passes these through as intrinsic elements - // ============================================ - - // Media elements - convert to links - video: MarkdownVideo, - audio: MarkdownAudio, - img: MarkdownImage, - iframe: MarkdownIframe, - embed: MarkdownPassthrough, - object: MarkdownPassthrough, - source: MarkdownPassthrough, - track: MarkdownPassthrough, - picture: MarkdownPassthrough, - - // Structural/semantic elements - strip wrapper, keep children - div: MarkdownPassthrough, - span: MarkdownPassthrough, - section: MarkdownPassthrough, - article: MarkdownPassthrough, - aside: MarkdownPassthrough, - header: MarkdownPassthrough, - footer: MarkdownPassthrough, - main: MarkdownPassthrough, - nav: MarkdownPassthrough, - address: MarkdownPassthrough, - hgroup: MarkdownPassthrough, - - // Figure elements - figure: MarkdownFigure, - figcaption: MarkdownFigcaption, - - // Interactive elements - details: MarkdownDetails, - summary: MarkdownSummary, - dialog: MarkdownPassthrough, - - // Self-closing elements - hr: MarkdownHr, - br: MarkdownBr, - wbr: MarkdownPassthrough, - - // Table elements - pass through for turndown - table: MarkdownTable, - thead: MarkdownThead, - tbody: MarkdownTbody, - tfoot: MarkdownTbody, - tr: MarkdownTr, - th: MarkdownTh, - td: MarkdownTd, - caption: MarkdownPassthrough, - colgroup: MarkdownPassthrough, - col: MarkdownPassthrough, - - // Code elements - preserve language info - pre: MarkdownPre, - code: MarkdownCode, - - // Definition lists - dl: MarkdownDl, - dt: MarkdownDt, - dd: MarkdownDd, - - // Form elements - strip (not useful in markdown) - form: MarkdownPassthrough, - input: MarkdownPassthrough, - button: MarkdownPassthrough, - select: MarkdownPassthrough, - option: MarkdownPassthrough, - optgroup: MarkdownPassthrough, - textarea: MarkdownPassthrough, - label: MarkdownPassthrough, - fieldset: MarkdownPassthrough, - legend: MarkdownPassthrough, - datalist: MarkdownPassthrough, - output: MarkdownPassthrough, - progress: MarkdownPassthrough, - meter: MarkdownPassthrough, - - // Script/style elements - remove entirely - script: () => null, - style: () => null, - noscript: MarkdownPassthrough, - template: MarkdownPassthrough, - slot: MarkdownPassthrough, - - // Canvas/SVG - remove (not representable in markdown) - canvas: () => null, - svg: () => null, - - // Misc inline elements - pass through - abbr: MarkdownPassthrough, - bdi: MarkdownPassthrough, - bdo: MarkdownPassthrough, - cite: MarkdownPassthrough, - data: MarkdownPassthrough, - dfn: MarkdownPassthrough, - kbd: MarkdownPassthrough, - mark: MarkdownPassthrough, - q: MarkdownPassthrough, - rb: MarkdownPassthrough, - rp: MarkdownPassthrough, - rt: MarkdownPassthrough, - rtc: MarkdownPassthrough, - ruby: MarkdownPassthrough, - s: MarkdownPassthrough, - samp: MarkdownPassthrough, - small: MarkdownPassthrough, - sub: MarkdownPassthrough, - sup: MarkdownPassthrough, - time: MarkdownPassthrough, - u: MarkdownPassthrough, - var: MarkdownPassthrough, - - // Map/area elements - map: MarkdownPassthrough, - area: MarkdownPassthrough, - - // Fallbacks - wrapper: PassThrough, -}; - -/** - * Strip import and export statements from MDX content - * This allows us to compile MDX without needing to resolve external modules - */ -function stripImportsAndExports(content: string): string { - let result = content; - - // Remove import statements (various formats) - // import X from 'module' - result = result.replace(/^import\s+.*?from\s+['"].*?['"];?\s*$/gm, ""); - // import 'module' - result = result.replace(/^import\s+['"].*?['"];?\s*$/gm, ""); - // import { X } from 'module' (multiline) - result = result.replace( - /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm, - "" - ); - - // Remove export statements - result = result.replace( - /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm, - "" - ); - - return result; -} - -/** - * Convert MDX content to clean Markdown - */ -export async function mdxToMarkdown( - mdxContent: string, - pagePath: string -): Promise { - // Extract frontmatter first - const frontmatterMatch = mdxContent.match(FRONTMATTER_REGEX); - let frontmatter = ""; - let contentWithoutFrontmatter = mdxContent; - - if (frontmatterMatch) { - frontmatter = frontmatterMatch[0]; - contentWithoutFrontmatter = mdxContent.slice(frontmatterMatch[0].length); - } - - try { - // Strip imports before compilation so MDX doesn't try to resolve them - const strippedContent = stripImportsAndExports(contentWithoutFrontmatter); - - // Compile MDX to JavaScript - // Include remarkGfm to properly parse GFM tables, strikethrough, etc. in the MDX source - const compiled = await compile(strippedContent, { - outputFormat: "function-body", - development: false, - remarkPlugins: [remarkGfm], - }); - - // Run the compiled code to get the component - // Use process.cwd() as baseUrl since we're not resolving any imports - const { default: MDXContent } = await run(String(compiled), { - Fragment: JsxFragment, - jsx, - jsxs, - baseUrl: pathToFileURL(process.cwd()).href, - }); - - // Render with markdown-friendly components - const element = createElement(MDXContent, { - components: markdownComponents, - }); - - // Convert React element to HTML string - const render = await getRenderer(); - const html = render(element); - - // Convert HTML to Markdown using unified ecosystem - let markdown = await htmlToMarkdown(html); - - // Clean up excessive whitespace - markdown = markdown.replace(/\n{3,}/g, "\n\n").trim(); - - // If result is empty, provide fallback - if (!markdown || markdown.length < 10) { - const title = extractTitle(frontmatter); - const description = extractDescription(frontmatter); - const htmlUrl = `https://docs.arcade.dev${pagePath}`; - return `${frontmatter}# ${title} - -${description} - -This page contains interactive content. Visit the full page at: ${htmlUrl} -`; - } - - return `${frontmatter}${markdown}\n`; - } catch (_error) { - return fallbackMdxToMarkdown(mdxContent, pagePath); - } -} - -/** - * Extract title from frontmatter - */ -function extractTitle(frontmatter: string): string { - const match = frontmatter.match(TITLE_REGEX); - return ( - match?.[1] || match?.[2] || match?.[3]?.trim() || "Arcade Documentation" - ); -} - -/** - * Extract description from frontmatter - */ -function extractDescription(frontmatter: string): string { - const match = frontmatter.match(DESCRIPTION_REGEX); - return match?.[1] || match?.[2] || match?.[3]?.trim() || ""; -} - -/** - * Fallback: Simple regex-based MDX to Markdown conversion - * Used when MDX compilation fails - */ -function fallbackMdxToMarkdown(content: string, pagePath: string): string { - let result = content; - - // Extract frontmatter - const frontmatterMatch = result.match(FRONTMATTER_REGEX); - let frontmatter = ""; - if (frontmatterMatch) { - frontmatter = frontmatterMatch[0]; - result = result.slice(frontmatterMatch[0].length); - } - - // Remove imports - result = result.replace(/^import\s+.*?from\s+['"].*?['"];?\s*$/gm, ""); - result = result.replace(/^import\s+['"].*?['"];?\s*$/gm, ""); - result = result.replace( - /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm, - "" - ); - - // Remove exports - result = result.replace( - /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm, - "" - ); - - // Remove self-closing JSX components (uppercase) - result = result.replace(/<([A-Z][a-zA-Z0-9.]*)[^>]*\/>/g, ""); - - // Remove self-closing HTML elements (lowercase) - but convert video/img to links - result = result.replace( - /]*src=["']([^"']+)["'][^>]*\/?>/gi, - "\n\n[Video]($1)\n\n" - ); - result = result.replace( - /]*src=["']([^"']+)["'][^>]*\/?>/gi, - "![]($1)" - ); - result = result.replace(/<[a-z][a-zA-Z0-9]*[^>]*\/>/g, ""); - - // Extract content from JSX with children (process iteratively for nesting) - let prev = ""; - while (prev !== result) { - prev = result; - // Uppercase components - result = result.replace( - /<([A-Z][a-zA-Z0-9.]*)[^>]*>([\s\S]*?)<\/\1>/g, - "$2" - ); - // Lowercase HTML elements (div, span, etc.) - result = result.replace( - /<(div|span|section|article|aside|header|footer|main|nav|video|figure|figcaption)[^>]*>([\s\S]*?)<\/\1>/gi, - "$2" - ); - } - - // Remove JSX expressions - result = result.replace(/\{[^}]+\}/g, ""); - - // Clean up - result = result.replace(/\n{3,}/g, "\n\n").trim(); - - if (!result || result.length < 10) { - const title = extractTitle(frontmatter); - const description = extractDescription(frontmatter); - const htmlUrl = `https://docs.arcade.dev${pagePath}`; - return `${frontmatter}# ${title} - -${description} - -This page contains interactive content. Visit the full page at: ${htmlUrl} -`; - } - - return `${frontmatter}${result}\n`; -} diff --git a/package.json b/package.json index f2316f47a..ba7953195 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,7 @@ "type": "module", "scripts": { "dev": "next dev --webpack", - "build": "pnpm run generate:markdown && next build --webpack", - "generate:markdown": "pnpm exec tsx scripts/generate-markdown.ts", + "build": "next build --webpack", "start": "next start", "lint": "pnpm dlx ultracite check", "format": "pnpm dlx ultracite fix", @@ -40,7 +39,6 @@ "homepage": "https://arcade.dev/", "dependencies": { "@arcadeai/design-system": "^3.26.0", - "@mdx-js/mdx": "^3.1.1", "@next/third-parties": "16.0.1", "@ory/client": "1.22.7", "@theguild/remark-mermaid": "0.3.0", @@ -56,14 +54,8 @@ "react-dom": "19.2.3", "react-hook-form": "7.65.0", "react-syntax-highlighter": "16.1.0", - "rehype-parse": "^9.0.1", - "rehype-remark": "^10.0.1", - "remark-gfm": "^4.0.1", - "remark-stringify": "^11.0.0", "swagger-ui-react": "^5.30.0", "tailwindcss-animate": "1.0.7", - "turndown": "^7.2.2", - "unified": "^11.0.5", "unist-util-visit": "5.0.0", "unist-util-visit-parents": "6.0.2", "zustand": "5.0.8" @@ -79,7 +71,6 @@ "@types/react": "19.2.7", "@types/react-dom": "19.2.3", "@types/react-syntax-highlighter": "15.5.13", - "@types/turndown": "^5.0.6", "@types/unist": "3.0.3", "commander": "14.0.2", "dotenv": "^17.2.3", @@ -96,7 +87,6 @@ "remark": "^15.0.1", "remark-rehype": "^11.1.2", "tailwindcss": "4.1.16", - "tsx": "^4.21.0", "typescript": "5.9.3", "ultracite": "6.1.0", "vitest": "4.0.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7923f5808..436ac536b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -10,10 +10,7 @@ importers: dependencies: '@arcadeai/design-system': specifier: ^3.26.0 - version: 3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2)) - '@mdx-js/mdx': - specifier: ^3.1.1 - version: 3.1.1 + version: 3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2)) '@next/third-parties': specifier: 16.0.1 version: 16.0.1(next@16.1.1(@opentelemetry/api@1.9.0)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react@19.2.3) @@ -59,30 +56,12 @@ importers: react-syntax-highlighter: specifier: 16.1.0 version: 16.1.0(react@19.2.3) - rehype-parse: - specifier: ^9.0.1 - version: 9.0.1 - rehype-remark: - specifier: ^10.0.1 - version: 10.0.1 - remark-gfm: - specifier: ^4.0.1 - version: 4.0.1 - remark-stringify: - specifier: ^11.0.0 - version: 11.0.0 swagger-ui-react: specifier: ^5.30.0 version: 5.31.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) tailwindcss-animate: specifier: 1.0.7 version: 1.0.7(tailwindcss@4.1.16) - turndown: - specifier: ^7.2.2 - version: 7.2.2 - unified: - specifier: ^11.0.5 - version: 11.0.5 unist-util-visit: specifier: 5.0.0 version: 5.0.0 @@ -123,9 +102,6 @@ importers: '@types/react-syntax-highlighter': specifier: 15.5.13 version: 15.5.13 - '@types/turndown': - specifier: ^5.0.6 - version: 5.0.6 '@types/unist': specifier: 3.0.3 version: 3.0.3 @@ -174,9 +150,6 @@ importers: tailwindcss: specifier: 4.1.16 version: 4.1.16 - tsx: - specifier: ^4.21.0 - version: 4.21.0 typescript: specifier: 5.9.3 version: 5.9.3 @@ -185,7 +158,7 @@ importers: version: 6.1.0(typescript@5.9.3) vitest: specifier: 4.0.5 - version: 4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) + version: 4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) zod: specifier: 4.1.12 version: 4.1.12 @@ -680,9 +653,6 @@ packages: '@mermaid-js/parser@0.6.3': resolution: {integrity: sha512-lnjOhe7zyHjc+If7yT4zoedx2vo4sHaTmtkl1+or8BRTnCtDmcTpAjpzDSfCZrshM5bCoz0GyidzadJAH1xobA==} - '@mixmark-io/domino@2.2.0': - resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==} - '@napi-rs/simple-git-android-arm-eabi@0.1.22': resolution: {integrity: sha512-JQZdnDNm8o43A5GOzwN/0Tz3CDBQtBUNqzVwEopm32uayjdjxev1Csp1JeaqF3v9djLDIvsSE39ecsN2LhCKKQ==} engines: {node: '>= 10'} @@ -2283,9 +2253,6 @@ packages: '@types/trusted-types@2.0.7': resolution: {integrity: sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==} - '@types/turndown@5.0.6': - resolution: {integrity: sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==} - '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} @@ -3048,9 +3015,6 @@ packages: resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} engines: {node: '>=16'} - get-tsconfig@4.13.0: - resolution: {integrity: sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==} - github-slugger@2.0.0: resolution: {integrity: sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==} @@ -3087,9 +3051,6 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} - hast-util-embedded@3.0.0: - resolution: {integrity: sha512-naH8sld4Pe2ep03qqULEtvYr7EjrLK2QHY8KJR6RJkTUjPGObe1vnx585uzem2hGra+s1q08DZZpfgDVYRbaXA==} - hast-util-from-dom@5.0.1: resolution: {integrity: sha512-N+LqofjR2zuzTjCPzyDUdSshy4Ma6li7p/c3pA78uTwzFgENbgbUrm2ugwsOdcjI1muO+o6Dgzp9p8WHtn/39Q==} @@ -3102,24 +3063,12 @@ packages: hast-util-from-parse5@8.0.3: resolution: {integrity: sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg==} - hast-util-has-property@3.0.0: - resolution: {integrity: sha512-MNilsvEKLFpV604hwfhVStK0usFY/QmM5zX16bo7EjnAEGofr5YyI37kzopBlZJkHD4t887i+q/C8/tr5Q94cA==} - - hast-util-is-body-ok-link@3.0.1: - resolution: {integrity: sha512-0qpnzOBLztXHbHQenVB8uNuxTnm/QBFUOmdOSsEn7GnBtyY07+ENTWVFBAnXd/zEgd9/SUG3lRY7hSIBWRgGpQ==} - hast-util-is-element@3.0.0: resolution: {integrity: sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==} - hast-util-minify-whitespace@1.0.1: - resolution: {integrity: sha512-L96fPOVpnclQE0xzdWb/D12VT5FabA7SnZOUMtL1DbXmYiHJMXZvFkIZfiMmTCNJHUeO2K9UYNXoVyfz+QHuOw==} - hast-util-parse-selector@4.0.0: resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==} - hast-util-phrasing@3.0.1: - resolution: {integrity: sha512-6h60VfI3uBQUxHqTyMymMZnEbNl1XmEGtOxxKYL7stY2o601COo62AWAYBQR9lZbYXYSBoxag8UpPRXK+9fqSQ==} - hast-util-raw@9.1.0: resolution: {integrity: sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==} @@ -3132,9 +3081,6 @@ packages: hast-util-to-jsx-runtime@2.3.6: resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==} - hast-util-to-mdast@10.1.2: - resolution: {integrity: sha512-FiCRI7NmOvM4y+f5w32jPRzcxDIz+PUqDwEqn1A+1q2cdp3B8Gx7aVrXORdOKjMNDQsD1ogOr896+0jJHW1EFQ==} - hast-util-to-parse5@8.0.1: resolution: {integrity: sha512-MlWT6Pjt4CG9lFCjiz4BH7l9wmrMkfkJYCxFwKQic8+RTZgWPuWxwAfjJElsXkex7DJjfSJsQIt931ilUgmwdA==} @@ -4219,9 +4165,6 @@ packages: rehype-katex@7.0.1: resolution: {integrity: sha512-OiM2wrZ/wuhKkigASodFoo8wimG3H12LWQaH8qSPVJn9apWKFSH3YOCtbKpBorTVw/eI7cuT21XBbvwEswbIOA==} - rehype-minify-whitespace@6.0.2: - resolution: {integrity: sha512-Zk0pyQ06A3Lyxhe9vGtOtzz3Z0+qZ5+7icZ/PL/2x1SHPbKao5oB/g/rlc6BCTajqBb33JcOe71Ye1oFsuYbnw==} - rehype-parse@9.0.1: resolution: {integrity: sha512-ksCzCD0Fgfh7trPDxr2rSylbwq9iYDkSn8TCDmEJ49ljEUBxDVCzCHv7QNzZOfODanX4+bWQ4WZqLCRWYLfhag==} @@ -4237,9 +4180,6 @@ packages: rehype-recma@1.0.0: resolution: {integrity: sha512-lqA4rGUf1JmacCNWWZx0Wv1dHqMwxzsDWYMTowuplHF3xH0N/MmrZ/G3BDZnzAkRmxDadujCjaKM2hqYdCBOGw==} - rehype-remark@10.0.1: - resolution: {integrity: sha512-EmDndlb5NVwXGfUa4c9GPK+lXeItTilLhE6ADSaQuHr4JUlKw9MidzGzx4HpqZrNCt6vnHmEifXQiiA+CEnjYQ==} - rehype-stringify@10.0.1: resolution: {integrity: sha512-k9ecfXHmIPuFVI61B9DeLPN0qFHfawM6RsuX48hoqlaKSF61RskNjSm1lI8PhBEM0MRdLxVVm4WmTqJQccH9mA==} @@ -4289,9 +4229,6 @@ packages: reselect@5.1.1: resolution: {integrity: sha512-K/BG6eIky/SBpzfHZv/dd+9JBFiS4SWV7FIujVyJRux6e45+73RaUHXLmIR1f7WOMaQ0U1km6qwklRQxpJJY0w==} - resolve-pkg-maps@1.0.0: - resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} - restore-cursor@5.1.0: resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} engines: {node: '>=18'} @@ -4584,9 +4521,6 @@ packages: trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} - trim-trailing-lines@2.1.0: - resolution: {integrity: sha512-5UR5Biq4VlVOtzqkm2AZlgvSlDJtME46uV0br0gENbwN4l5+mMKT4b9gJKqWtuL2zAIqajGJGuvbCbcAJUZqBg==} - trough@2.2.0: resolution: {integrity: sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==} @@ -4634,14 +4568,6 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} - tsx@4.21.0: - resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} - engines: {node: '>=18.0.0'} - hasBin: true - - turndown@7.2.2: - resolution: {integrity: sha512-1F7db8BiExOKxjSMU2b7if62D/XOyQyZbPKq/nUwopfgnHlqXHqQ0lvfUTeUIr1lZJzOPFn43dODyMSIfvWRKQ==} - twoslash-protocol@0.3.4: resolution: {integrity: sha512-HHd7lzZNLUvjPzG/IE6js502gEzLC1x7HaO1up/f72d8G8ScWAs9Yfa97igelQRDl5h9tGcdFsRp+lNVre1EeQ==} @@ -4992,7 +4918,7 @@ snapshots: transitivePeerDependencies: - encoding - '@arcadeai/design-system@3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2))': + '@arcadeai/design-system@3.26.0(@hookform/resolvers@5.2.2(react-hook-form@7.65.0(react@19.2.3)))(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(lucide-react@0.548.0(react@19.2.3))(react-dom@19.2.3(react@19.2.3))(react-hook-form@7.65.0(react@19.2.3))(react@19.2.3)(recharts@3.6.0(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react-is@16.13.1)(react@19.2.3)(redux@5.0.1))(tailwindcss@4.1.16)(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2))': dependencies: '@arcadeai/arcadejs': 1.15.0 '@hookform/resolvers': 5.2.2(react-hook-form@7.65.0(react@19.2.3)) @@ -5017,7 +4943,7 @@ snapshots: '@radix-ui/react-tabs': 1.1.13(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-toggle': 1.1.10(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) '@radix-ui/react-tooltip': 1.2.8(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) - '@tailwindcss/vite': 4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2)) + '@tailwindcss/vite': 4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2)) class-variance-authority: 0.7.1 clsx: 2.1.1 cmdk: 1.1.1(@types/react-dom@19.2.3(@types/react@19.2.7))(@types/react@19.2.7)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) @@ -5401,8 +5327,6 @@ snapshots: dependencies: langium: 3.3.1 - '@mixmark-io/domino@2.2.0': {} - '@napi-rs/simple-git-android-arm-eabi@0.1.22': optional: true @@ -7002,12 +6926,12 @@ snapshots: postcss: 8.5.6 tailwindcss: 4.1.16 - '@tailwindcss/vite@4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2))': + '@tailwindcss/vite@4.1.14(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2))': dependencies: '@tailwindcss/node': 4.1.14 '@tailwindcss/oxide': 4.1.14 tailwindcss: 4.1.14 - vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) + vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) '@tanstack/react-virtual@3.13.13(react-dom@19.2.3(react@19.2.3))(react@19.2.3)': dependencies: @@ -7234,8 +7158,6 @@ snapshots: '@types/trusted-types@2.0.7': optional: true - '@types/turndown@5.0.6': {} - '@types/unist@2.0.11': {} '@types/unist@3.0.3': {} @@ -7265,13 +7187,13 @@ snapshots: chai: 6.2.1 tinyrainbow: 3.0.3 - '@vitest/mocker@4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2))': + '@vitest/mocker@4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2))': dependencies: '@vitest/spy': 4.0.5 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) + vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) '@vitest/pretty-format@4.0.5': dependencies: @@ -8012,10 +7934,6 @@ snapshots: get-stream@8.0.1: {} - get-tsconfig@4.13.0: - dependencies: - resolve-pkg-maps: 1.0.0 - github-slugger@2.0.0: {} glob-parent@5.1.2: @@ -8049,11 +7967,6 @@ snapshots: dependencies: function-bind: 1.1.2 - hast-util-embedded@3.0.0: - dependencies: - '@types/hast': 3.0.4 - hast-util-is-element: 3.0.0 - hast-util-from-dom@5.0.1: dependencies: '@types/hast': 3.0.4 @@ -8087,38 +8000,14 @@ snapshots: vfile-location: 5.0.3 web-namespaces: 2.0.1 - hast-util-has-property@3.0.0: - dependencies: - '@types/hast': 3.0.4 - - hast-util-is-body-ok-link@3.0.1: - dependencies: - '@types/hast': 3.0.4 - hast-util-is-element@3.0.0: dependencies: '@types/hast': 3.0.4 - hast-util-minify-whitespace@1.0.1: - dependencies: - '@types/hast': 3.0.4 - hast-util-embedded: 3.0.0 - hast-util-is-element: 3.0.0 - hast-util-whitespace: 3.0.0 - unist-util-is: 6.0.1 - hast-util-parse-selector@4.0.0: dependencies: '@types/hast': 3.0.4 - hast-util-phrasing@3.0.1: - dependencies: - '@types/hast': 3.0.4 - hast-util-embedded: 3.0.0 - hast-util-has-property: 3.0.0 - hast-util-is-body-ok-link: 3.0.1 - hast-util-is-element: 3.0.0 - hast-util-raw@9.1.0: dependencies: '@types/hast': 3.0.4 @@ -8190,23 +8079,6 @@ snapshots: transitivePeerDependencies: - supports-color - hast-util-to-mdast@10.1.2: - dependencies: - '@types/hast': 3.0.4 - '@types/mdast': 4.0.4 - '@ungap/structured-clone': 1.3.0 - hast-util-phrasing: 3.0.1 - hast-util-to-html: 9.0.5 - hast-util-to-text: 4.0.2 - hast-util-whitespace: 3.0.0 - mdast-util-phrasing: 4.1.0 - mdast-util-to-hast: 13.2.1 - mdast-util-to-string: 4.0.0 - rehype-minify-whitespace: 6.0.2 - trim-trailing-lines: 2.1.0 - unist-util-position: 5.0.0 - unist-util-visit: 5.0.0 - hast-util-to-parse5@8.0.1: dependencies: '@types/hast': 3.0.4 @@ -9637,11 +9509,6 @@ snapshots: unist-util-visit-parents: 6.0.2 vfile: 6.0.3 - rehype-minify-whitespace@6.0.2: - dependencies: - '@types/hast': 3.0.4 - hast-util-minify-whitespace: 1.0.1 - rehype-parse@9.0.1: dependencies: '@types/hast': 3.0.4 @@ -9672,14 +9539,6 @@ snapshots: transitivePeerDependencies: - supports-color - rehype-remark@10.0.1: - dependencies: - '@types/hast': 3.0.4 - '@types/mdast': 4.0.4 - hast-util-to-mdast: 10.1.2 - unified: 11.0.5 - vfile: 6.0.3 - rehype-stringify@10.0.1: dependencies: '@types/hast': 3.0.4 @@ -9779,8 +9638,6 @@ snapshots: reselect@5.1.1: {} - resolve-pkg-maps@1.0.0: {} - restore-cursor@5.1.0: dependencies: onetime: 7.0.0 @@ -10172,8 +10029,6 @@ snapshots: trim-lines@3.0.1: {} - trim-trailing-lines@2.1.0: {} - trough@2.2.0: {} trpc-cli@0.12.1(@trpc/server@11.8.0(typescript@5.9.3))(zod@4.1.12): @@ -10198,17 +10053,6 @@ snapshots: tslib@2.8.1: {} - tsx@4.21.0: - dependencies: - esbuild: 0.27.1 - get-tsconfig: 4.13.0 - optionalDependencies: - fsevents: 2.3.3 - - turndown@7.2.2: - dependencies: - '@mixmark-io/domino': 2.2.0 - twoslash-protocol@0.3.4: {} twoslash@0.3.4(typescript@5.9.3): @@ -10394,7 +10238,7 @@ snapshots: d3-time: 3.1.0 d3-timer: 3.0.1 - vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2): + vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2): dependencies: esbuild: 0.27.1 fdir: 6.5.0(picomatch@4.0.3) @@ -10407,13 +10251,12 @@ snapshots: fsevents: 2.3.3 jiti: 2.6.1 lightningcss: 1.30.2 - tsx: 4.21.0 yaml: 2.8.2 - vitest@4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2): + vitest@4.0.5(@types/debug@4.1.12)(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2): dependencies: '@vitest/expect': 4.0.5 - '@vitest/mocker': 4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2)) + '@vitest/mocker': 4.0.5(vite@7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2)) '@vitest/pretty-format': 4.0.5 '@vitest/runner': 4.0.5 '@vitest/snapshot': 4.0.5 @@ -10430,7 +10273,7 @@ snapshots: tinyexec: 0.3.2 tinyglobby: 0.2.15 tinyrainbow: 3.0.3 - vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) + vite: 7.3.0(@types/node@24.9.2)(jiti@2.6.1)(lightningcss@1.30.2)(yaml@2.8.2) why-is-node-running: 2.3.0 optionalDependencies: '@types/debug': 4.1.12 diff --git a/proxy.ts b/proxy.ts index f9cd4b515..df8e7d83a 100644 --- a/proxy.ts +++ b/proxy.ts @@ -63,18 +63,19 @@ function pathnameIsMissingLocale(pathname: string): boolean { export function proxy(request: NextRequest) { const pathname = request.nextUrl.pathname; - // .md requests are served as static files from public/ - // (generated at build time by scripts/generate-markdown.ts) - if (pathname.endsWith(".md")) { - // Add locale prefix if missing, then let Next.js serve from public/ - if (pathnameIsMissingLocale(pathname)) { - const locale = getPreferredLocale(request); - const pathWithoutMd = pathname.replace(MD_EXTENSION_REGEX, ""); - const redirectPath = `/${locale}${pathWithoutMd}.md`; - return NextResponse.redirect(new URL(redirectPath, request.url)); - } - // Let Next.js serve the static file from public/ - return NextResponse.next(); + // Handle .md requests without locale - redirect to add locale first + if (pathname.endsWith(".md") && pathnameIsMissingLocale(pathname)) { + const locale = getPreferredLocale(request); + const pathWithoutMd = pathname.replace(MD_EXTENSION_REGEX, ""); + const redirectPath = `/${locale}${pathWithoutMd}.md`; + return NextResponse.redirect(new URL(redirectPath, request.url)); + } + + // Rewrite .md requests (with locale) to the markdown API route + if (pathname.endsWith(".md") && !pathname.startsWith("/api/")) { + const url = request.nextUrl.clone(); + url.pathname = `/api/markdown${pathname}`; + return NextResponse.rewrite(url); } // Redirect if there is no locale diff --git a/scripts/generate-markdown.ts b/scripts/generate-markdown.ts deleted file mode 100644 index 618f9953f..000000000 --- a/scripts/generate-markdown.ts +++ /dev/null @@ -1,155 +0,0 @@ -/** - * Static Markdown Generation Script - * - * Generates pre-rendered markdown files from MDX pages for LLM consumption. - * Outputs to public/en/ so files are served directly by Next.js/CDN. - * - * Usage: pnpm dlx tsx scripts/generate-markdown.ts - */ - -import { mkdir, readFile, writeFile } from "node:fs/promises"; -import { dirname, join } from "node:path"; -import fastGlob from "fast-glob"; -import { mdxToMarkdown } from "../lib/mdx-to-markdown"; - -const OUTPUT_DIR = join(process.cwd(), "public"); -const SEPARATOR_WIDTH = 50; -const RESOURCE_FILE_REGEX = - /\.(png|jpg|jpeg|gif|svg|webp|mp4|webm|pdf|zip|tar|gz)$/i; - -/** - * Rewrite internal links to point to .md files - * - /references/foo → /en/references/foo.md - * - /en/references/foo → /en/references/foo.md - * - External links, anchors, and resource links are unchanged - */ -function rewriteLinksToMarkdown(markdown: string): string { - // Match markdown links: [text](url) - return markdown.replace( - /\[([^\]]*)\]\(([^)]+)\)/g, - (match, text, url: string) => { - // Skip external links - if ( - url.startsWith("http://") || - url.startsWith("https://") || - url.startsWith("mailto:") - ) { - return match; - } - - // Skip anchor-only links - if (url.startsWith("#")) { - return match; - } - - // Skip resource links (images, videos, files) - if ( - RESOURCE_FILE_REGEX.test(url) || - url.startsWith("/images/") || - url.startsWith("/videos/") || - url.startsWith("/files/") - ) { - return match; - } - - // Skip if already has .md extension - if (url.endsWith(".md")) { - return match; - } - - // Handle anchor in URL - let anchor = ""; - let pathPart = url; - const anchorIndex = url.indexOf("#"); - if (anchorIndex !== -1) { - anchor = url.slice(anchorIndex); - pathPart = url.slice(0, anchorIndex); - } - - // Add /en prefix if not present (internal doc links) - if (pathPart.startsWith("/") && !pathPart.startsWith("/en/")) { - pathPart = `/en${pathPart}`; - } - - // Add .md extension - const newUrl = `${pathPart}.md${anchor}`; - return `[${text}](${newUrl})`; - } - ); -} - -async function generateMarkdownFiles() { - console.log("Generating static markdown files...\n"); - - // Find all MDX pages - const mdxFiles = await fastGlob("app/en/**/page.mdx", { - cwd: process.cwd(), - absolute: false, - }); - - console.log(`Found ${mdxFiles.length} MDX files\n`); - - let successCount = 0; - let errorCount = 0; - const errors: { file: string; error: string }[] = []; - - for (const mdxFile of mdxFiles) { - try { - // Read MDX content - const mdxPath = join(process.cwd(), mdxFile); - const mdxContent = await readFile(mdxPath, "utf-8"); - - // Compute paths - // app/en/references/auth-providers/page.mdx → /en/references/auth-providers - const relativePath = mdxFile - .replace("app/", "/") - .replace("/page.mdx", ""); - - // Convert to markdown - let markdown = await mdxToMarkdown(mdxContent, relativePath); - - // Rewrite links to point to .md files - markdown = rewriteLinksToMarkdown(markdown); - - // Output path: public/en/references/auth-providers.md - const outputPath = join(OUTPUT_DIR, `${relativePath}.md`); - - // Ensure directory exists - await mkdir(dirname(outputPath), { recursive: true }); - - // Write markdown file - await writeFile(outputPath, markdown, "utf-8"); - - successCount += 1; - process.stdout.write(`✓ ${relativePath}.md\n`); - } catch (error) { - errorCount += 1; - const errorMessage = - error instanceof Error ? error.message : String(error); - errors.push({ file: mdxFile, error: errorMessage }); - process.stdout.write(`✗ ${mdxFile}: ${errorMessage}\n`); - } - } - - console.log(`\n${"=".repeat(SEPARATOR_WIDTH)}`); - console.log(`Generated: ${successCount} files`); - if (errorCount > 0) { - console.log(`Errors: ${errorCount} files`); - console.log("\nFailed files:"); - for (const { file, error } of errors) { - console.log(` - ${file}: ${error}`); - } - } - console.log("=".repeat(SEPARATOR_WIDTH)); - - // Exit with error code if any files failed - if (errorCount > 0) { - process.exit(1); - } -} - -// Run the script -generateMarkdownFiles().catch((error) => { - console.error("Fatal error:", error); - process.exit(1); -}); From 6054f7612a187bb8f68f76715ca66958d38f0a59 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Tue, 20 Jan 2026 18:27:35 +0000 Subject: [PATCH 09/15] Resolve merge conflict in markdown route --- app/api/markdown/[[...slug]]/route.ts | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index 10812365f..8e84e8a4f 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -15,8 +15,17 @@ const IMPORT_DESTRUCTURE_REGEX = /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm; const EXPORT_REGEX = /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm; -const SELF_CLOSING_JSX_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*\/>/g; -const JSX_WITH_CHILDREN_REGEX = /<([A-Z][a-zA-Z0-9.]*)[^>]*>([\s\S]*?)<\/\1>/g; +// JSX attribute pattern that properly handles quoted strings containing ">" characters +// Matches: non-quote/non-angle chars, OR complete double-quoted strings, OR complete single-quoted strings +const JSX_ATTRS_PATTERN = "(?:[^>\"'\\n]|\"[^\"]*\"|'[^']*')*"; +const SELF_CLOSING_JSX_REGEX = new RegExp( + `<([A-Z][a-zA-Z0-9.]*)${JSX_ATTRS_PATTERN}\\/>`, + "g" +); +const JSX_WITH_CHILDREN_REGEX = new RegExp( + `<([A-Z][a-zA-Z0-9.]*)${JSX_ATTRS_PATTERN}>([\\s\\S]*?)<\\/\\1>`, + "g" +); const CODE_BLOCK_REGEX = /```[\s\S]*?```/g; const JSX_EXPRESSION_REGEX = /\{[^}]+\}/g; const EXCESSIVE_NEWLINES_REGEX = /\n{3,}/g; @@ -71,8 +80,11 @@ function dedent(text: string): string { return lines .map((line) => { const trimmed = line.trim(); + // Calculate leading whitespace length for this line + const leadingMatch = line.match(LEADING_WHITESPACE_REGEX); + const leadingLength = leadingMatch ? leadingMatch[0].length : 0; // Don't modify empty lines or lines with less indentation than min - if (trimmed === "" || line.length < minIndent) { + if (trimmed === "" || leadingLength < minIndent) { return line.trimStart(); } // Preserve code block markers - just remove leading whitespace @@ -219,10 +231,10 @@ function compileMdxToMarkdown(content: string, pagePath: string): string { // Now remove JSX expressions outside code blocks result = result.replace(JSX_EXPRESSION_REGEX, ""); - // Restore code blocks + // Restore code blocks (return original placeholder if index doesn't exist) result = result.replace( CODE_BLOCK_PLACEHOLDER_REGEX, - (_, index) => codeBlocks[Number.parseInt(index, 10)] + (match, index) => codeBlocks[Number.parseInt(index, 10)] ?? match ); // Normalize indentation (remove stray whitespace, preserve meaningful markdown indentation) From b0682e8bb3345cc4852b096759b4c6ad078792e5 Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Tue, 20 Jan 2026 19:49:47 +0000 Subject: [PATCH 10/15] Update markdown route --- app/api/markdown/[[...slug]]/route.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index 8e84e8a4f..753af3d59 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -15,9 +15,17 @@ const IMPORT_DESTRUCTURE_REGEX = /^import\s*\{[\s\S]*?\}\s*from\s*['"].*?['"];?\s*$/gm; const EXPORT_REGEX = /^export\s+(const|let|var|function|default)\s+[\s\S]*?(?=\n(?:import|export|#|\n|$))/gm; -// JSX attribute pattern that properly handles quoted strings containing ">" characters -// Matches: non-quote/non-angle chars, OR complete double-quoted strings, OR complete single-quoted strings -const JSX_ATTRS_PATTERN = "(?:[^>\"'\\n]|\"[^\"]*\"|'[^']*')*"; +// JSX attribute pattern that properly handles: +// - Quoted strings containing ">" characters +// - JSX expressions in curly braces containing ">" (arrow functions, comparisons) +// - Multiline attributes (newlines allowed between attributes) +// - Up to 3 levels of brace nesting for style={{outer: {inner: 1}}} patterns +// The brace pattern uses a recursive-like structure to handle nested braces +const BRACE_CONTENT_L0 = "[^{}]*"; // Innermost: no braces +const BRACE_CONTENT_L1 = `(?:${BRACE_CONTENT_L0}|\\{${BRACE_CONTENT_L0}\\})*`; // 1 level +const BRACE_CONTENT_L2 = `(?:${BRACE_CONTENT_L0}|\\{${BRACE_CONTENT_L1}\\})*`; // 2 levels +const BRACE_PATTERN = `\\{${BRACE_CONTENT_L2}\\}`; // Full brace expression (supports 3 levels) +const JSX_ATTRS_PATTERN = `(?:[^>"'{}]|"[^"]*"|'[^']*'|${BRACE_PATTERN})*`; const SELF_CLOSING_JSX_REGEX = new RegExp( `<([A-Z][a-zA-Z0-9.]*)${JSX_ATTRS_PATTERN}\\/>`, "g" From 4dc5764bf4335a1eb26f3d0f62d4e50ddb3e91a4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 23 Jan 2026 20:12:54 +0000 Subject: [PATCH 11/15] Fix JSX expression regex to handle nested braces Replace simple regex pattern that failed on nested braces with existing BRACE_PATTERN that properly handles up to 3 levels of nesting. This fixes issues with expressions like {obj || {default: true}} and {items.map(x => ({a: x}))} where stray closing braces would remain in the output. --- app/api/markdown/[[...slug]]/route.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index 753af3d59..9c0582365 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -35,7 +35,7 @@ const JSX_WITH_CHILDREN_REGEX = new RegExp( "g" ); const CODE_BLOCK_REGEX = /```[\s\S]*?```/g; -const JSX_EXPRESSION_REGEX = /\{[^}]+\}/g; +const JSX_EXPRESSION_REGEX = new RegExp(BRACE_PATTERN, "g"); const EXCESSIVE_NEWLINES_REGEX = /\n{3,}/g; const CODE_BLOCK_PLACEHOLDER_REGEX = /__CODE_BLOCK_(\d+)__/g; From 7a77d90127847200ab8383f0c3d158b709f331ad Mon Sep 17 00:00:00 2001 From: Rachel Lee Nabors Date: Fri, 23 Jan 2026 20:55:53 +0000 Subject: [PATCH 12/15] Improve markdown compilation for LLM consumption - Convert GuideOverview.Outcomes/Prerequisites/YouWillLearn to ## headers - Convert components to ![alt](src) markdown syntax - Add .md extension to internal links for LLM crawlers Co-Authored-By: Claude Opus 4.5 --- app/api/markdown/[[...slug]]/route.ts | 65 +++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index 9c0582365..f7351c7a4 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -39,6 +39,27 @@ const JSX_EXPRESSION_REGEX = new RegExp(BRACE_PATTERN, "g"); const EXCESSIVE_NEWLINES_REGEX = /\n{3,}/g; const CODE_BLOCK_PLACEHOLDER_REGEX = /__CODE_BLOCK_(\d+)__/g; +// GuideOverview component patterns - convert to markdown headers +const GUIDE_OVERVIEW_OUTCOMES_REGEX = + /\s*([\s\S]*?)\s*<\/GuideOverview\.Outcomes>/g; +const GUIDE_OVERVIEW_PREREQUISITES_REGEX = + /\s*([\s\S]*?)\s*<\/GuideOverview\.Prerequisites>/g; +const GUIDE_OVERVIEW_YOU_WILL_LEARN_REGEX = + /\s*([\s\S]*?)\s*<\/GuideOverview\.YouWillLearn>/g; + +// Image component pattern - extract alt and src for markdown image +// Handles both quoted strings and JSX expressions: alt="text" or alt={"text"}, src="/path" or src={"/path"} +const IMAGE_ALT_REGEX = /alt=(?:["']([^"']+)["']|\{["']([^"']+)["']\})/; +const IMAGE_SRC_REGEX = /src=(?:["']([^"']+)["']|\{["']([^"']+)["']\})/; +const IMAGE_COMPONENT_REGEX = /]*?\/>/g; + +// Internal markdown links - add .md extension +// Matches [text](/path) but not [text](http...) or [text](#anchor) +const INTERNAL_LINK_REGEX = /\[([^\]]+)\]\(\/([^)#][^)]*)\)/g; + +// Check if path has a file extension +const HAS_EXTENSION_REGEX = /\.[a-zA-Z0-9]+$/; + // Regex for detecting markdown list items and numbered lists const UNORDERED_LIST_REGEX = /^[-*+]\s/; const ORDERED_LIST_REGEX = /^\d+[.)]\s/; @@ -211,6 +232,40 @@ function compileMdxToMarkdown(content: string, pagePath: string): string { // Remove export statements (like export const metadata) result = result.replace(EXPORT_REGEX, ""); + // Convert GuideOverview components to markdown headers before generic JSX stripping + result = result.replace( + GUIDE_OVERVIEW_OUTCOMES_REGEX, + (_, inner) => `## Outcomes\n\n${dedent(inner.trim())}\n` + ); + result = result.replace( + GUIDE_OVERVIEW_PREREQUISITES_REGEX, + (_, inner) => `## Prerequisites\n\n${dedent(inner.trim())}\n` + ); + result = result.replace( + GUIDE_OVERVIEW_YOU_WILL_LEARN_REGEX, + (_, inner) => `## You Will Learn\n\n${dedent(inner.trim())}\n` + ); + + // Convert Image components to markdown image syntax + // Extract alt and src from component attributes (handles both quoted and JSX expression syntax) + result = result.replace(IMAGE_COMPONENT_REGEX, (match) => { + const altMatch = match.match(IMAGE_ALT_REGEX); + const srcMatch = match.match(IMAGE_SRC_REGEX); + + // Extract from whichever capture group matched (quoted or JSX expression) + const alt = altMatch?.[1] || altMatch?.[2]; + const src = srcMatch?.[1] || srcMatch?.[2]; + + if (alt && src) { + // Make src absolute if it starts with / + const fullSrc = src.startsWith("/") + ? `https://docs.arcade.dev${src}` + : src; + return `![${alt}](${fullSrc})`; + } + return ""; + }); + // Process self-closing JSX components (e.g., or ) // Handles components with dots like result = result.replace(SELF_CLOSING_JSX_REGEX, ""); @@ -245,6 +300,16 @@ function compileMdxToMarkdown(content: string, pagePath: string): string { (match, index) => codeBlocks[Number.parseInt(index, 10)] ?? match ); + // Convert internal links to .md links for LLM consumption + // [text](/path/to/page) -> [text](/path/to/page.md) + result = result.replace(INTERNAL_LINK_REGEX, (_, text, path) => { + // Don't add .md if path already has an extension + if (HAS_EXTENSION_REGEX.test(path)) { + return `[${text}](/${path})`; + } + return `[${text}](/${path}.md)`; + }); + // Normalize indentation (remove stray whitespace, preserve meaningful markdown indentation) result = normalizeIndentation(result); From 59b09f910909dc70b9bd086184257cc105523405 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 23 Jan 2026 21:16:15 +0000 Subject: [PATCH 13/15] Fix regex bugs in markdown API route - Fix IMAGE_COMPONENT_REGEX to use JSX_ATTRS_PATTERN for handling > in attributes - Fix INTERNAL_LINK_REGEX to correctly place .md before URL fragments - Fix IMAGE_ALT_REGEX and IMAGE_SRC_REGEX to handle apostrophes in quoted strings --- app/api/markdown/[[...slug]]/route.ts | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index f7351c7a4..dc40f56c6 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -49,13 +49,14 @@ const GUIDE_OVERVIEW_YOU_WILL_LEARN_REGEX = // Image component pattern - extract alt and src for markdown image // Handles both quoted strings and JSX expressions: alt="text" or alt={"text"}, src="/path" or src={"/path"} -const IMAGE_ALT_REGEX = /alt=(?:["']([^"']+)["']|\{["']([^"']+)["']\})/; -const IMAGE_SRC_REGEX = /src=(?:["']([^"']+)["']|\{["']([^"']+)["']\})/; -const IMAGE_COMPONENT_REGEX = /]*?\/>/g; +const IMAGE_ALT_REGEX = /alt=(?:"([^"]*)"|'([^']*)'|\{"([^"]*)"\}|\{'([^']*)'\})/; +const IMAGE_SRC_REGEX = /src=(?:"([^"]*)"|'([^']*)'|\{"([^"]*)"\}|\{'([^']*)'\})/; +const IMAGE_COMPONENT_REGEX = new RegExp(``,"g"); // Internal markdown links - add .md extension // Matches [text](/path) but not [text](http...) or [text](#anchor) -const INTERNAL_LINK_REGEX = /\[([^\]]+)\]\(\/([^)#][^)]*)\)/g; +// Captures path and fragment separately to insert .md before fragment +const INTERNAL_LINK_REGEX = /\[([^\]]+)\]\(\/([^)#][^)#]*)(#[^)]*)?\)/g; // Check if path has a file extension const HAS_EXTENSION_REGEX = /\.[a-zA-Z0-9]+$/; @@ -252,9 +253,9 @@ function compileMdxToMarkdown(content: string, pagePath: string): string { const altMatch = match.match(IMAGE_ALT_REGEX); const srcMatch = match.match(IMAGE_SRC_REGEX); - // Extract from whichever capture group matched (quoted or JSX expression) - const alt = altMatch?.[1] || altMatch?.[2]; - const src = srcMatch?.[1] || srcMatch?.[2]; + // Extract from whichever capture group matched (double quotes, single quotes, or JSX expression with either) + const alt = altMatch?.[1] || altMatch?.[2] || altMatch?.[3] || altMatch?.[4]; + const src = srcMatch?.[1] || srcMatch?.[2] || srcMatch?.[3] || srcMatch?.[4]; if (alt && src) { // Make src absolute if it starts with / @@ -302,12 +303,13 @@ function compileMdxToMarkdown(content: string, pagePath: string): string { // Convert internal links to .md links for LLM consumption // [text](/path/to/page) -> [text](/path/to/page.md) - result = result.replace(INTERNAL_LINK_REGEX, (_, text, path) => { + // [text](/path/to/page#section) -> [text](/path/to/page.md#section) + result = result.replace(INTERNAL_LINK_REGEX, (_, text, path, fragment) => { // Don't add .md if path already has an extension if (HAS_EXTENSION_REGEX.test(path)) { - return `[${text}](/${path})`; + return `[${text}](/${path}${fragment || ""})`; } - return `[${text}](/${path}.md)`; + return `[${text}](/${path}.md${fragment || ""})`; }); // Normalize indentation (remove stray whitespace, preserve meaningful markdown indentation) From b500ab5c49d1140a65e5809ff5da0ad2250ac6e6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 23 Jan 2026 21:32:18 +0000 Subject: [PATCH 14/15] Fix code block state tracking for embedded fence markers Track the indentation level of opening code fences and only treat appears inside code blocks (e.g., in Python strings containing markdown examples). --- app/api/markdown/[[...slug]]/route.ts | 37 +++++++++++++++++++-------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index dc40f56c6..b5b3fce77 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -179,26 +179,43 @@ function extractFrontmatterMeta(frontmatter: string): { function normalizeIndentation(text: string): string { const finalLines: string[] = []; let inCodeBlock = false; + let fenceIndent = 0; // Track indentation level of opening fence for (const line of text.split("\n")) { - if (line.trim().startsWith("```")) { - inCodeBlock = !inCodeBlock; - finalLines.push(line.trimStart()); // Code block markers should start at column 0 + const trimmed = line.trim(); + const leadingSpaces = line.length - trimmed.length; + + if (trimmed.startsWith("```")) { + if (!inCodeBlock) { + // Opening fence - track its indentation + inCodeBlock = true; + fenceIndent = leadingSpaces; + finalLines.push(line.trimStart()); // Code block markers should start at column 0 + } else if (leadingSpaces <= fenceIndent) { + // Closing fence - only if not indented more than opening fence + inCodeBlock = false; + fenceIndent = 0; + finalLines.push(line.trimStart()); + } else { + // ``` inside code block (indented more than opening fence) + finalLines.push(line); // Preserve as code block content + } } else if (inCodeBlock) { finalLines.push(line); // Preserve indentation inside code blocks } else { - const trimmed = line.trimStart(); + const trimmedLine = line.trimStart(); // Preserve indentation for nested list items and blockquotes const isListItem = - UNORDERED_LIST_REGEX.test(trimmed) || ORDERED_LIST_REGEX.test(trimmed); - const isBlockquote = trimmed.startsWith(">"); + UNORDERED_LIST_REGEX.test(trimmedLine) || + ORDERED_LIST_REGEX.test(trimmedLine); + const isBlockquote = trimmedLine.startsWith(">"); if ((isListItem || isBlockquote) && line.startsWith(" ")) { // Keep markdown-meaningful indentation (but normalize to 2-space increments) - const leadingSpaces = line.length - line.trimStart().length; - const normalizedIndent = " ".repeat(Math.floor(leadingSpaces / 2)); - finalLines.push(normalizedIndent + trimmed); + const leadingSpacesCount = line.length - line.trimStart().length; + const normalizedIndent = " ".repeat(Math.floor(leadingSpacesCount / 2)); + finalLines.push(normalizedIndent + trimmedLine); } else { - finalLines.push(trimmed); // Remove leading whitespace for other lines + finalLines.push(trimmedLine); // Remove leading whitespace for other lines } } } From c43e8429e788f4d159c9cd4195152a5f6fb1ea4e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 23 Jan 2026 21:47:11 +0000 Subject: [PATCH 15/15] Fix indentation bug by calling dedent before trim The bug was caused by calling trim() before dedent(), which removed leading whitespace from the first line and caused dedent to calculate minIndent as 0. This left subsequent lines with incorrect indentation. Fixed by swapping the order to dedent(content).trim() at all four occurrences (lines 256, 260, 264, and 300). --- app/api/markdown/[[...slug]]/route.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/api/markdown/[[...slug]]/route.ts b/app/api/markdown/[[...slug]]/route.ts index b5b3fce77..52e6ad0c7 100644 --- a/app/api/markdown/[[...slug]]/route.ts +++ b/app/api/markdown/[[...slug]]/route.ts @@ -253,15 +253,15 @@ function compileMdxToMarkdown(content: string, pagePath: string): string { // Convert GuideOverview components to markdown headers before generic JSX stripping result = result.replace( GUIDE_OVERVIEW_OUTCOMES_REGEX, - (_, inner) => `## Outcomes\n\n${dedent(inner.trim())}\n` + (_, inner) => `## Outcomes\n\n${dedent(inner).trim()}\n` ); result = result.replace( GUIDE_OVERVIEW_PREREQUISITES_REGEX, - (_, inner) => `## Prerequisites\n\n${dedent(inner.trim())}\n` + (_, inner) => `## Prerequisites\n\n${dedent(inner).trim()}\n` ); result = result.replace( GUIDE_OVERVIEW_YOU_WILL_LEARN_REGEX, - (_, inner) => `## You Will Learn\n\n${dedent(inner.trim())}\n` + (_, inner) => `## You Will Learn\n\n${dedent(inner).trim()}\n` ); // Convert Image components to markdown image syntax @@ -297,7 +297,7 @@ function compileMdxToMarkdown(content: string, pagePath: string): string { // Match opening tag, capture tag name (with dots), and content until matching closing tag // Apply dedent to each extracted piece to normalize indentation result = result.replace(JSX_WITH_CHILDREN_REGEX, (_, _tag, innerContent) => - dedent(innerContent.trim()) + dedent(innerContent).trim() ); }