From 29ffeb426f07b08770504849535b7259fb8ee6cb Mon Sep 17 00:00:00 2001 From: jottakka Date: Thu, 15 Jan 2026 12:44:43 -0300 Subject: [PATCH 1/2] adding components and doc generator first files --- .vscode/settings.json | 2 + .../__tests__/AvailableToolsTable.test.tsx | 73 + .../DocumentationChunkRenderer.test.tsx | 160 ++ .../__tests__/DynamicCodeBlock.test.tsx | 87 + .../__tests__/ParametersTable.test.tsx | 52 + .../__tests__/ScopesDisplay.test.tsx | 31 + .../__tests__/ToolSection.test.tsx | 32 + .../__tests__/ToolkitHeader.test.tsx | 235 +++ .../components/AvailableToolsTable.tsx | 107 + .../components/DocumentationChunkRenderer.tsx | 150 ++ .../components/DynamicCodeBlock.tsx | 392 ++++ .../components/ParametersTable.tsx | 125 ++ .../toolkit-docs/components/ScopesDisplay.tsx | 87 + .../toolkit-docs/components/ToolSection.tsx | 189 ++ .../toolkit-docs/components/ToolkitHeader.tsx | 199 ++ .../toolkit-docs/components/ToolkitPage.tsx | 83 + .../toolkit-docs/components/index.ts | 12 + app/_components/toolkit-docs/index.ts | 48 + app/_components/toolkit-docs/types/index.ts | 427 ++++ .../development/github/preview/page.mdx | 13 + next-env.d.ts | 2 +- package.json | 1 + planningdoc.md | 663 +++++++ pnpm-lock.yaml | 32 + toolkit-docs-generator/.gitignore | 27 + toolkit-docs-generator/README.md | 160 ++ toolkit-docs-generator/biome.jsonc | 27 + .../mock-data/engine-api-response.json | 335 ++++ .../mock-data/metadata.json | 62 + toolkit-docs-generator/package.json | 52 + toolkit-docs-generator/pnpm-lock.yaml | 1721 +++++++++++++++++ toolkit-docs-generator/src/cli/index.ts | 428 ++++ toolkit-docs-generator/src/generator/index.ts | 4 + .../src/generator/json-generator.ts | 158 ++ toolkit-docs-generator/src/index.ts | 19 + toolkit-docs-generator/src/llm/client.ts | 116 ++ toolkit-docs-generator/src/llm/index.ts | 2 + .../src/llm/tool-example-generator.ts | 192 ++ .../src/merger/data-merger.ts | 337 ++++ toolkit-docs-generator/src/merger/index.ts | 4 + .../src/sources/custom-sections-file.ts | 103 + .../src/sources/in-memory.ts | 259 +++ toolkit-docs-generator/src/sources/index.ts | 13 + .../src/sources/interfaces.ts | 33 + .../src/sources/internal.ts | 52 + .../src/sources/mock-engine-api.ts | 214 ++ .../src/sources/mock-metadata.ts | 114 ++ .../src/sources/toolkit-data-source.ts | 208 ++ toolkit-docs-generator/src/types/index.ts | 400 ++++ toolkit-docs-generator/src/utils/fp.ts | 184 ++ toolkit-docs-generator/src/utils/index.ts | 4 + .../tests/fixtures/engine-api-response.json | 335 ++++ .../tests/fixtures/github-toolkit.json | 459 +++++ .../tests/fixtures/metadata.json | 62 + .../tests/fixtures/slack-toolkit.json | 256 +++ .../tests/llm/tool-example-generator.test.ts | 116 ++ .../tests/merger/data-merger.test.ts | 667 +++++++ .../tests/sources/in-memory.test.ts | 370 ++++ .../tests/sources/toolkit-data-source.test.ts | 127 ++ toolkit-docs-generator/tsconfig.json | 26 + toolkit-docs-generator/vitest.config.ts | 39 + 61 files changed, 10886 insertions(+), 1 deletion(-) create mode 100644 app/_components/toolkit-docs/__tests__/AvailableToolsTable.test.tsx create mode 100644 app/_components/toolkit-docs/__tests__/DocumentationChunkRenderer.test.tsx create mode 100644 app/_components/toolkit-docs/__tests__/DynamicCodeBlock.test.tsx create mode 100644 app/_components/toolkit-docs/__tests__/ParametersTable.test.tsx create mode 100644 app/_components/toolkit-docs/__tests__/ScopesDisplay.test.tsx create mode 100644 app/_components/toolkit-docs/__tests__/ToolSection.test.tsx create mode 100644 app/_components/toolkit-docs/__tests__/ToolkitHeader.test.tsx create mode 100644 app/_components/toolkit-docs/components/AvailableToolsTable.tsx create mode 100644 app/_components/toolkit-docs/components/DocumentationChunkRenderer.tsx create mode 100644 app/_components/toolkit-docs/components/DynamicCodeBlock.tsx create mode 100644 app/_components/toolkit-docs/components/ParametersTable.tsx create mode 100644 app/_components/toolkit-docs/components/ScopesDisplay.tsx create mode 100644 app/_components/toolkit-docs/components/ToolSection.tsx create mode 100644 app/_components/toolkit-docs/components/ToolkitHeader.tsx create mode 100644 app/_components/toolkit-docs/components/ToolkitPage.tsx create mode 100644 app/_components/toolkit-docs/components/index.ts create mode 100644 app/_components/toolkit-docs/index.ts create mode 100644 app/_components/toolkit-docs/types/index.ts create mode 100644 app/en/resources/integrations/development/github/preview/page.mdx create mode 100644 planningdoc.md create mode 100644 toolkit-docs-generator/.gitignore create mode 100644 toolkit-docs-generator/README.md create mode 100644 toolkit-docs-generator/biome.jsonc create mode 100644 toolkit-docs-generator/mock-data/engine-api-response.json create mode 100644 toolkit-docs-generator/mock-data/metadata.json create mode 100644 toolkit-docs-generator/package.json create mode 100644 toolkit-docs-generator/pnpm-lock.yaml create mode 100644 toolkit-docs-generator/src/cli/index.ts create mode 100644 toolkit-docs-generator/src/generator/index.ts create mode 100644 toolkit-docs-generator/src/generator/json-generator.ts create mode 100644 toolkit-docs-generator/src/index.ts create mode 100644 toolkit-docs-generator/src/llm/client.ts create mode 100644 toolkit-docs-generator/src/llm/index.ts create mode 100644 toolkit-docs-generator/src/llm/tool-example-generator.ts create mode 100644 toolkit-docs-generator/src/merger/data-merger.ts create mode 100644 toolkit-docs-generator/src/merger/index.ts create mode 100644 toolkit-docs-generator/src/sources/custom-sections-file.ts create mode 100644 toolkit-docs-generator/src/sources/in-memory.ts create mode 100644 toolkit-docs-generator/src/sources/index.ts create mode 100644 toolkit-docs-generator/src/sources/interfaces.ts create mode 100644 toolkit-docs-generator/src/sources/internal.ts create mode 100644 toolkit-docs-generator/src/sources/mock-engine-api.ts create mode 100644 toolkit-docs-generator/src/sources/mock-metadata.ts create mode 100644 toolkit-docs-generator/src/sources/toolkit-data-source.ts create mode 100644 toolkit-docs-generator/src/types/index.ts create mode 100644 toolkit-docs-generator/src/utils/fp.ts create mode 100644 toolkit-docs-generator/src/utils/index.ts create mode 100644 toolkit-docs-generator/tests/fixtures/engine-api-response.json create mode 100644 toolkit-docs-generator/tests/fixtures/github-toolkit.json create mode 100644 toolkit-docs-generator/tests/fixtures/metadata.json create mode 100644 toolkit-docs-generator/tests/fixtures/slack-toolkit.json create mode 100644 toolkit-docs-generator/tests/llm/tool-example-generator.test.ts create mode 100644 toolkit-docs-generator/tests/merger/data-merger.test.ts create mode 100644 toolkit-docs-generator/tests/sources/in-memory.test.ts create mode 100644 toolkit-docs-generator/tests/sources/toolkit-data-source.test.ts create mode 100644 toolkit-docs-generator/tsconfig.json create mode 100644 toolkit-docs-generator/vitest.config.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index d19b8e1a8..e59685ab4 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,6 +7,8 @@ ], "editor.wordWrap": "bounded", "editor.wordWrapColumn": 120, + "editor.cursorWidth": 3, + "editor.cursorBlinking": "solid", "editor.formatOnSave": true, "[javascript][typescript][json][jsonc][tsx][jsx][css]": { "editor.formatOnSave": true diff --git a/app/_components/toolkit-docs/__tests__/AvailableToolsTable.test.tsx b/app/_components/toolkit-docs/__tests__/AvailableToolsTable.test.tsx new file mode 100644 index 000000000..149654e60 --- /dev/null +++ b/app/_components/toolkit-docs/__tests__/AvailableToolsTable.test.tsx @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; + +import { + buildToolsTableData, + toToolAnchorId, +} from "../components/AvailableToolsTable"; + +describe("AvailableToolsTable helpers", () => { + it("builds table data with fallback descriptions", () => { + const data = buildToolsTableData([ + { + name: "CreateIssue", + qualifiedName: "Github.CreateIssue", + description: "Create an issue", + secretsInfo: [ + { name: "GITHUB_API_KEY", type: "api_key" }, + { name: "SECONDARY_TOKEN", type: "token" }, + { name: "ANOTHER_API_KEY", type: "api_key" }, + ], + }, + { + name: "ListPullRequests", + qualifiedName: "Github.ListPullRequests", + description: null, + secrets: ["WEBHOOK_SECRET"], + }, + { + name: "GetRepo", + qualifiedName: "Github.GetRepo", + description: null, + secrets: [], + }, + ]); + + expect(data).toEqual([ + ["Github.CreateIssue", "Create an issue", "API key, Token"], + ["Github.ListPullRequests", "No description provided.", "WEBHOOK_SECRET"], + ["Github.GetRepo", "No description provided.", "None"], + ]); + }); + + it("includes secret type docs base URL when provided", () => { + const data = buildToolsTableData( + [ + { + name: "CreateIssue", + qualifiedName: "Github.CreateIssue", + description: "Create an issue", + secretsInfo: [{ name: "GITHUB_API_KEY", type: "api_key" }], + }, + ], + { + secretsDisplay: "types", + secretTypeDocsBaseUrl: "/references/secrets", + } + ); + + expect(data).toEqual([ + [ + "Github.CreateIssue", + "Create an issue", + "API key (/references/secrets/api_key)", + ], + ]); + }); + + it("matches the anchor id logic used by TableOfContents", () => { + expect(toToolAnchorId("Github.CreateIssue")).toBe("githubcreateissue"); + expect(toToolAnchorId("Slack Api.Send Message")).toBe( + "slack-api.send-message".replace(".", "") + ); + }); +}); diff --git a/app/_components/toolkit-docs/__tests__/DocumentationChunkRenderer.test.tsx b/app/_components/toolkit-docs/__tests__/DocumentationChunkRenderer.test.tsx new file mode 100644 index 000000000..29662d8fe --- /dev/null +++ b/app/_components/toolkit-docs/__tests__/DocumentationChunkRenderer.test.tsx @@ -0,0 +1,160 @@ +import { describe, expect, it } from "vitest"; + +import type { DocumentationChunk } from "../types"; +import { hasChunksAt } from "../components/DocumentationChunkRenderer"; + +/** + * Unit tests for DocumentationChunkRenderer utility functions + * + * Note: Component rendering tests require additional dependencies + * (@testing-library/react). These tests focus on the pure utility functions. + */ + +describe("hasChunksAt", () => { + const createChunk = ( + overrides: Partial = {} + ): DocumentationChunk => ({ + type: "callout", + location: "description", + position: "before", + content: "Test content", + ...overrides, + }); + + it("returns true when chunks exist at location and position", () => { + const chunks: DocumentationChunk[] = [ + createChunk({ location: "header", position: "after" }), + ]; + + expect(hasChunksAt(chunks, "header", "after")).toBe(true); + }); + + it("returns false when no chunks at location", () => { + const chunks: DocumentationChunk[] = [ + createChunk({ location: "header", position: "after" }), + ]; + + expect(hasChunksAt(chunks, "parameters", "after")).toBe(false); + }); + + it("returns false when no chunks at position", () => { + const chunks: DocumentationChunk[] = [ + createChunk({ location: "header", position: "after" }), + ]; + + expect(hasChunksAt(chunks, "header", "before")).toBe(false); + }); + + it("returns false for empty array", () => { + expect(hasChunksAt([], "header", "after")).toBe(false); + }); + + it("returns true when multiple chunks match", () => { + const chunks: DocumentationChunk[] = [ + createChunk({ location: "description", position: "before" }), + createChunk({ location: "description", position: "before" }), + createChunk({ location: "description", position: "after" }), + ]; + + expect(hasChunksAt(chunks, "description", "before")).toBe(true); + }); + + it("works with all location types", () => { + const locations: DocumentationChunk["location"][] = [ + "header", + "description", + "parameters", + "auth", + "secrets", + "output", + "footer", + ]; + + for (const location of locations) { + const chunks: DocumentationChunk[] = [ + createChunk({ location, position: "after" }), + ]; + expect(hasChunksAt(chunks, location, "after")).toBe(true); + } + }); + + it("works with all position types", () => { + const positions: DocumentationChunk["position"][] = [ + "before", + "after", + "replace", + ]; + + for (const position of positions) { + const chunks: DocumentationChunk[] = [ + createChunk({ location: "header", position }), + ]; + expect(hasChunksAt(chunks, "header", position)).toBe(true); + } + }); +}); + +describe("DocumentationChunk type validation", () => { + it("validates chunk type property", () => { + const validTypes: DocumentationChunk["type"][] = [ + "callout", + "markdown", + "code", + "warning", + "info", + "tip", + ]; + + for (const type of validTypes) { + const chunk: DocumentationChunk = { + type, + location: "header", + position: "after", + content: "test", + }; + expect(chunk.type).toBe(type); + } + }); + + it("validates optional title property", () => { + const chunkWithTitle: DocumentationChunk = { + type: "callout", + location: "header", + position: "after", + content: "test", + title: "My Title", + }; + + const chunkWithoutTitle: DocumentationChunk = { + type: "callout", + location: "header", + position: "after", + content: "test", + }; + + expect(chunkWithTitle.title).toBe("My Title"); + expect(chunkWithoutTitle.title).toBeUndefined(); + }); + + it("validates optional variant property", () => { + const validVariants: DocumentationChunk["variant"][] = [ + "default", + "destructive", + "warning", + "info", + "success", + undefined, + ]; + + for (const variant of validVariants) { + const chunk: DocumentationChunk = { + type: "callout", + location: "header", + position: "after", + content: "test", + variant, + }; + expect(chunk.variant).toBe(variant); + } + }); +}); diff --git a/app/_components/toolkit-docs/__tests__/DynamicCodeBlock.test.tsx b/app/_components/toolkit-docs/__tests__/DynamicCodeBlock.test.tsx new file mode 100644 index 000000000..39edb3760 --- /dev/null +++ b/app/_components/toolkit-docs/__tests__/DynamicCodeBlock.test.tsx @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; + +import type { ToolCodeExample } from "../types"; +import { + buildToolInput, + generateJavaScriptExample, + generatePythonExample, +} from "../components/DynamicCodeBlock"; + +describe("DynamicCodeBlock helpers", () => { + it("builds tool input with placeholders for required values", () => { + const codeExample: ToolCodeExample = { + toolName: "Github.CreateIssue", + parameters: { + title: { + value: null, + type: "string", + required: true, + }, + body: { + value: null, + type: "string", + required: false, + }, + }, + requiresAuth: true, + authProvider: "github", + }; + + const toolInput = buildToolInput(codeExample.parameters); + + expect(toolInput).toEqual({ + title: "your_value", + }); + }); + + it("generates JavaScript example with auth flow", () => { + const codeExample: ToolCodeExample = { + toolName: "Github.CreateIssue", + parameters: { + owner: { + value: "arcadeai", + type: "string", + required: true, + }, + labels: { + value: ["bug", "high-priority"], + type: "array", + required: false, + }, + }, + requiresAuth: true, + authProvider: "github", + tabLabel: "Call the Tool with User Authorization", + }; + + const output = generateJavaScriptExample(codeExample); + + expect(output).toContain("client.tools.authorize"); + expect(output).toContain("user_id: USER_ID"); + expect(output).toContain('const TOOL_NAME = "Github.CreateIssue";'); + expect(output).toContain('owner: "arcadeai"'); + expect(output).toContain('labels: ["bug", "high-priority"]'); + }); + + it("generates Python example without auth flow when not required", () => { + const codeExample: ToolCodeExample = { + toolName: "Slack.ListChannels", + parameters: { + limit: { + value: 50, + type: "integer", + required: false, + }, + }, + requiresAuth: false, + }; + + const output = generatePythonExample(codeExample); + + expect(output).not.toContain("authorize"); + expect(output).not.toContain("USER_ID"); + expect(output).toContain('TOOL_NAME = "Slack.ListChannels"'); + expect(output).toContain("tool_input = {"); + expect(output).toContain('"limit": 50'); + }); +}); diff --git a/app/_components/toolkit-docs/__tests__/ParametersTable.test.tsx b/app/_components/toolkit-docs/__tests__/ParametersTable.test.tsx new file mode 100644 index 000000000..a969fad7d --- /dev/null +++ b/app/_components/toolkit-docs/__tests__/ParametersTable.test.tsx @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; + +import type { ToolParameter } from "../types"; +import { + formatEnumValues, + formatParameterType, +} from "../components/ParametersTable"; + +describe("ParametersTable helpers", () => { + describe("formatParameterType", () => { + it("formats array types with inner type", () => { + const param: ToolParameter = { + name: "labels", + type: "array", + innerType: "string", + required: false, + description: "Labels to add", + enum: null, + }; + + expect(formatParameterType(param)).toBe("array"); + }); + + it("returns base type when no inner type", () => { + const param: ToolParameter = { + name: "count", + type: "integer", + required: false, + description: "Count", + enum: null, + }; + + expect(formatParameterType(param)).toBe("integer"); + }); + }); + + describe("formatEnumValues", () => { + it("filters empty enum values", () => { + const values = ["open", "", "closed", " ", "all"]; + + expect(formatEnumValues(values)).toEqual(["open", "closed", "all"]); + }); + + it("returns empty array when enum is null", () => { + expect(formatEnumValues(null)).toEqual([]); + }); + + it("returns empty array when enum is undefined", () => { + expect(formatEnumValues(undefined)).toEqual([]); + }); + }); +}); diff --git a/app/_components/toolkit-docs/__tests__/ScopesDisplay.test.tsx b/app/_components/toolkit-docs/__tests__/ScopesDisplay.test.tsx new file mode 100644 index 000000000..ace4f6512 --- /dev/null +++ b/app/_components/toolkit-docs/__tests__/ScopesDisplay.test.tsx @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; + +import { normalizeScopes } from "../components/ScopesDisplay"; + +describe("ScopesDisplay helpers", () => { + it("trims scope values and removes duplicates", () => { + const scopes = [ + "repo", + " user:email ", + "repo", + "public_repo", + "user:email", + ]; + + expect(normalizeScopes(scopes)).toEqual([ + "repo", + "user:email", + "public_repo", + ]); + }); + + it("removes empty and whitespace-only scopes", () => { + const scopes = ["", " ", "repo", "\n", "user:email"]; + + expect(normalizeScopes(scopes)).toEqual(["repo", "user:email"]); + }); + + it("returns empty array when given empty array", () => { + expect(normalizeScopes([])).toEqual([]); + }); +}); diff --git a/app/_components/toolkit-docs/__tests__/ToolSection.test.tsx b/app/_components/toolkit-docs/__tests__/ToolSection.test.tsx new file mode 100644 index 000000000..cab856296 --- /dev/null +++ b/app/_components/toolkit-docs/__tests__/ToolSection.test.tsx @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; + +import type { DocumentationChunk } from "../types"; +import { shouldRenderDefaultSection } from "../components/ToolSection"; + +describe("ToolSection helpers", () => { + it("returns true when no replace chunks exist", () => { + const chunks: DocumentationChunk[] = [ + { + type: "callout", + location: "description", + position: "before", + content: "Info", + }, + ]; + + expect(shouldRenderDefaultSection(chunks, "description")).toBe(true); + }); + + it("returns false when replace chunk exists for location", () => { + const chunks: DocumentationChunk[] = [ + { + type: "markdown", + location: "parameters", + position: "replace", + content: "Custom parameters", + }, + ]; + + expect(shouldRenderDefaultSection(chunks, "parameters")).toBe(false); + }); +}); diff --git a/app/_components/toolkit-docs/__tests__/ToolkitHeader.test.tsx b/app/_components/toolkit-docs/__tests__/ToolkitHeader.test.tsx new file mode 100644 index 000000000..130c0aba7 --- /dev/null +++ b/app/_components/toolkit-docs/__tests__/ToolkitHeader.test.tsx @@ -0,0 +1,235 @@ +import { describe, expect, it } from "vitest"; + +import type { + ToolkitHeaderProps, + ToolkitMetadata, + ToolkitAuth, + ToolkitCategory, + ToolkitType, +} from "../types"; + +/** + * Unit tests for ToolkitHeader types and props validation + * + * Note: Component rendering tests require additional dependencies + * (@testing-library/react). These tests focus on type validation + * and prop structure. + */ + +describe("ToolkitHeaderProps type validation", () => { + const defaultMetadata: ToolkitMetadata = { + category: "development", + iconUrl: "https://example.com/icon.svg", + isBYOC: false, + isPro: false, + type: "arcade", + docsLink: "https://docs.arcade.dev/github", + }; + + const defaultAuth: ToolkitAuth = { + type: "oauth2", + providerId: "github", + allScopes: ["repo", "user:email"], + }; + + it("validates required props structure", () => { + const props: ToolkitHeaderProps = { + id: "Github", + label: "GitHub", + description: "GitHub repository tools", + metadata: defaultMetadata, + auth: defaultAuth, + }; + + expect(props.id).toBe("Github"); + expect(props.label).toBe("GitHub"); + expect(props.description).toBe("GitHub repository tools"); + expect(props.metadata).toEqual(defaultMetadata); + expect(props.auth).toEqual(defaultAuth); + }); + + it("validates optional props", () => { + const props: ToolkitHeaderProps = { + id: "Github", + label: "GitHub", + description: "GitHub repository tools", + metadata: defaultMetadata, + auth: defaultAuth, + version: "1.0.0", + author: "Custom Author", + summary: "A summary of the toolkit", + }; + + expect(props.version).toBe("1.0.0"); + expect(props.author).toBe("Custom Author"); + expect(props.summary).toBe("A summary of the toolkit"); + }); + + it("allows null description", () => { + const props: ToolkitHeaderProps = { + id: "Github", + label: "GitHub", + description: null, + metadata: defaultMetadata, + auth: defaultAuth, + }; + + expect(props.description).toBeNull(); + }); + + it("allows null auth", () => { + const props: ToolkitHeaderProps = { + id: "Github", + label: "GitHub", + description: "Description", + metadata: defaultMetadata, + auth: null, + }; + + expect(props.auth).toBeNull(); + }); +}); + +describe("ToolkitMetadata type validation", () => { + it("validates all category values", () => { + const categories: ToolkitCategory[] = [ + "productivity", + "social", + "development", + "entertainment", + "search", + "payments", + "sales", + "databases", + "customer-support", + ]; + + for (const category of categories) { + const metadata: ToolkitMetadata = { + category, + iconUrl: "https://example.com/icon.svg", + isBYOC: false, + isPro: false, + type: "arcade", + docsLink: "https://docs.arcade.dev", + }; + expect(metadata.category).toBe(category); + } + }); + + it("validates all type values", () => { + const types: ToolkitType[] = [ + "arcade", + "arcade_starter", + "verified", + "community", + "auth", + ]; + + for (const type of types) { + const metadata: ToolkitMetadata = { + category: "development", + iconUrl: "https://example.com/icon.svg", + isBYOC: false, + isPro: false, + type, + docsLink: "https://docs.arcade.dev", + }; + expect(metadata.type).toBe(type); + } + }); + + it("validates boolean flags", () => { + const metadata: ToolkitMetadata = { + category: "development", + iconUrl: "https://example.com/icon.svg", + isBYOC: true, + isPro: true, + type: "arcade", + docsLink: "https://docs.arcade.dev", + isComingSoon: true, + isHidden: false, + }; + + expect(metadata.isBYOC).toBe(true); + expect(metadata.isPro).toBe(true); + expect(metadata.isComingSoon).toBe(true); + expect(metadata.isHidden).toBe(false); + }); +}); + +describe("ToolkitAuth type validation", () => { + it("validates oauth2 auth type", () => { + const auth: ToolkitAuth = { + type: "oauth2", + providerId: "github", + allScopes: ["repo", "user:email"], + }; + + expect(auth.type).toBe("oauth2"); + expect(auth.providerId).toBe("github"); + expect(auth.allScopes).toEqual(["repo", "user:email"]); + }); + + it("validates api_key auth type", () => { + const auth: ToolkitAuth = { + type: "api_key", + providerId: null, + allScopes: [], + }; + + expect(auth.type).toBe("api_key"); + expect(auth.providerId).toBeNull(); + expect(auth.allScopes).toEqual([]); + }); + + it("validates mixed auth type", () => { + const auth: ToolkitAuth = { + type: "mixed", + providerId: "github", + allScopes: ["repo"], + }; + + expect(auth.type).toBe("mixed"); + expect(auth.providerId).toBe("github"); + expect(auth.allScopes).toEqual(["repo"]); + }); + + it("validates none auth type", () => { + const auth: ToolkitAuth = { + type: "none", + providerId: null, + allScopes: [], + }; + + expect(auth.type).toBe("none"); + }); + + it("validates empty scopes array", () => { + const auth: ToolkitAuth = { + type: "oauth2", + providerId: "custom", + allScopes: [], + }; + + expect(auth.allScopes).toEqual([]); + expect(auth.allScopes.length).toBe(0); + }); + + it("validates multiple scopes", () => { + const auth: ToolkitAuth = { + type: "oauth2", + providerId: "google", + allScopes: [ + "https://www.googleapis.com/auth/gmail.readonly", + "https://www.googleapis.com/auth/gmail.send", + "https://www.googleapis.com/auth/gmail.compose", + ], + }; + + expect(auth.allScopes.length).toBe(3); + expect(auth.allScopes).toContain( + "https://www.googleapis.com/auth/gmail.readonly" + ); + }); +}); diff --git a/app/_components/toolkit-docs/components/AvailableToolsTable.tsx b/app/_components/toolkit-docs/components/AvailableToolsTable.tsx new file mode 100644 index 000000000..e5677d75e --- /dev/null +++ b/app/_components/toolkit-docs/components/AvailableToolsTable.tsx @@ -0,0 +1,107 @@ +"use client"; + +import TableOfContents from "../../table-of-contents"; +import type { AvailableToolsTableProps, SecretType } from "../types"; + +export function toToolAnchorId(value: string): string { + return value.toLowerCase().replace(/\s+/g, "-").replace(".", ""); +} + +export function buildToolsTableData( + tools: AvailableToolsTableProps["tools"], + options?: Pick< + AvailableToolsTableProps, + "secretsDisplay" | "secretTypeLabels" | "secretTypeDocsBaseUrl" + > +): string[][] { + const secretTypeLabels: Record = { + api_key: "API key", + token: "Token", + client_secret: "Client secret", + webhook_secret: "Webhook secret", + private_key: "Private key", + password: "Password", + unknown: "Unknown", + ...options?.secretTypeLabels, + }; + + const normalizeBaseUrl = (baseUrl: string | undefined): string | undefined => { + if (!baseUrl) { + return undefined; + } + return baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl; + }; + + const baseUrl = normalizeBaseUrl(options?.secretTypeDocsBaseUrl); + + const formatSecretTypeLabel = (type: SecretType): string => { + const label = secretTypeLabels[type] ?? "Unknown"; + return baseUrl ? `${label} (${baseUrl}/${type})` : label; + }; + + const formatSecretSummary = ( + tool: AvailableToolsTableProps["tools"][number] + ): string => { + const displayMode = options?.secretsDisplay ?? "summary"; + const secretsInfo = tool.secretsInfo ?? []; + const secrets = tool.secrets ?? []; + + if (displayMode === "names") { + return secrets.length === 0 ? "None" : secrets.join(", "); + } + + if ( + displayMode === "types" || + (displayMode === "summary" && secretsInfo.length > 0) + ) { + const uniqueTypes = Array.from( + new Set(secretsInfo.map((secret) => secret.type)) + ); + return uniqueTypes.length === 0 + ? "None" + : uniqueTypes.map((type) => formatSecretTypeLabel(type)).join(", "); + } + + return secrets.length === 0 ? "None" : secrets.join(", "); + }; + + return tools.map((tool) => [ + tool.qualifiedName, + tool.description ?? "No description provided.", + formatSecretSummary(tool), + ]); +} + +/** + * AvailableToolsTable + * + * Renders a table of tools with clickable rows. + */ +export function AvailableToolsTable({ + tools, + secretsColumnLabel = "Secrets", + secretsDisplay = "summary", + secretTypeLabels, + secretTypeDocsBaseUrl, +}: AvailableToolsTableProps) { + if (!tools || tools.length === 0) { + return ( +

+ No tools available. +

+ ); + } + + return ( + + ); +} + +export default AvailableToolsTable; diff --git a/app/_components/toolkit-docs/components/DocumentationChunkRenderer.tsx b/app/_components/toolkit-docs/components/DocumentationChunkRenderer.tsx new file mode 100644 index 000000000..9e69c824b --- /dev/null +++ b/app/_components/toolkit-docs/components/DocumentationChunkRenderer.tsx @@ -0,0 +1,150 @@ +"use client"; + +import { Callout } from "nextra/components"; +import type React from "react"; +import ReactMarkdown from "react-markdown"; + +import type { + DocumentationChunk, + DocumentationChunkLocation, + DocumentationChunkPosition, + DocumentationChunkRendererProps, +} from "../types"; + +/** + * Maps chunk types to Nextra Callout types + */ +const CALLOUT_TYPE_MAP: Record = { + callout: "default", + info: "info", + tip: "info", + warning: "warning", + error: "error", +}; + +/** + * Maps chunk variants to Nextra Callout types + */ +const VARIANT_TYPE_MAP: Record = { + default: "default", + info: "info", + success: "info", + warning: "warning", + destructive: "error", +}; + +/** + * Renders markdown content from a string. + */ +function MarkdownContent({ content }: { content: string }) { + return {content}; +} + +/** + * Renders a single documentation chunk + */ +function ChunkContent({ chunk }: { chunk: DocumentationChunk }) { + const { type, content, title, variant } = chunk; + + // Handle code blocks + if (type === "code") { + return ( + 
+        {content}
+
+ ); + } + + // Handle plain markdown + if (type === "markdown") { + return ( +
+ +
+ ); + } + + // Handle callout types (callout, info, tip, warning) + // Determine the callout type from chunk type or variant + let calloutType: "default" | "info" | "warning" | "error" = "default"; + + if (variant && VARIANT_TYPE_MAP[variant]) { + calloutType = VARIANT_TYPE_MAP[variant]; + } else if (CALLOUT_TYPE_MAP[type]) { + calloutType = CALLOUT_TYPE_MAP[type]; + } + + return ( + + + + ); +} + +/** + * DocumentationChunkRenderer + * + * A generic component for rendering custom documentation content at specified + * injection points. Filters chunks by location and position, then renders + * the appropriate component based on chunk type. + * + * @example + * ```tsx + * + * ``` + * + * Supported chunk types: + * - `callout`: Default callout box + * - `info`: Blue info callout + * - `tip`: Blue tip callout + * - `warning`: Yellow warning callout + * - `markdown`: Raw markdown content + * - `code`: Code block + */ +export function DocumentationChunkRenderer({ + chunks, + location, + position, + className, +}: DocumentationChunkRendererProps): React.ReactElement | null { + // Filter chunks that match the specified location and position + const matchingChunks = chunks.filter( + (chunk) => chunk.location === location && chunk.position === position + ); + + // Return null if no matching chunks + if (matchingChunks.length === 0) { + return null; + } + + return ( +
+ {matchingChunks.map((chunk, index) => ( + + ))} +
+ ); +} + +/** + * Helper function to check if chunks exist for a given location and position + * Useful for conditional rendering + */ +export function hasChunksAt( + chunks: DocumentationChunk[], + location: DocumentationChunkLocation, + position: DocumentationChunkPosition +): boolean { + return chunks.some( + (chunk) => chunk.location === location && chunk.position === position + ); +} + +export default DocumentationChunkRenderer; diff --git a/app/_components/toolkit-docs/components/DynamicCodeBlock.tsx b/app/_components/toolkit-docs/components/DynamicCodeBlock.tsx new file mode 100644 index 000000000..82184c5b1 --- /dev/null +++ b/app/_components/toolkit-docs/components/DynamicCodeBlock.tsx @@ -0,0 +1,392 @@ +"use client"; + +import { Button } from "@arcadeai/design-system"; +import { ChevronDown } from "lucide-react"; +import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; +import { + atomDark, + oneLight, +} from "react-syntax-highlighter/dist/cjs/styles/prism"; +import { useEffect, useMemo, useState } from "react"; + +import { CopyButton } from "../../tabbed-code-block/copy-button"; +import { LanguageTabs } from "../../tabbed-code-block/language-tabs"; +import type { + DynamicCodeBlockProps, + ExampleParameterValue, + ToolCodeExample, +} from "../types"; + +type LanguageKey = "python" | "javascript"; + +const LANGUAGE_LABELS: Record = { + python: "Python", + javascript: "JavaScript", +}; + +const DEFAULT_LANGUAGES: LanguageKey[] = ["python", "javascript"]; + +const PLACEHOLDER_VALUES: Record = { + string: "your_value", + integer: 123, + boolean: true, + array: [], + object: {}, +}; + +const INDENT_SIZE: Record = { + python: 4, + javascript: 2, +}; + +function indent(level: number, language: LanguageKey): string { + return " ".repeat(level * INDENT_SIZE[language]); +} + +function isPlainObject(value: unknown): value is Record { + return ( + typeof value === "object" && + value !== null && + !Array.isArray(value) && + !(value instanceof Date) + ); +} + +function formatKey(key: string, language: LanguageKey): string { + if (language === "python") { + return JSON.stringify(key); + } + + const isValidIdentifier = /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key); + return isValidIdentifier ? key : JSON.stringify(key); +} + +function serializePrimitive(value: unknown, language: LanguageKey): string { + if (value === null) { + return language === "python" ? "None" : "null"; + } + + switch (typeof value) { + case "string": + return JSON.stringify(value); + case "number": + return Number.isFinite(value) ? String(value) : "null"; + case "boolean": + return language === "python" ? (value ? "True" : "False") : String(value); + default: + return language === "python" ? "None" : "null"; + } +} + +function shouldInline(values: string[]): boolean { + return values.every((value) => !value.includes("\n")) && values.join(", ").length < 80; +} + +export function serializeValue( + value: unknown, + language: LanguageKey, + level: number +): string { + if (Array.isArray(value)) { + if (value.length === 0) { + return "[]"; + } + + const items = value.map((item) => serializeValue(item, language, level + 1)); + if (shouldInline(items)) { + return `[${items.join(", ")}]`; + } + + const lines = items.map((item) => `${indent(level + 1, language)}${item}`); + return `[\n${lines.join(",\n")}\n${indent(level, language)}]`; + } + + if (isPlainObject(value)) { + const entries = Object.entries(value); + if (entries.length === 0) { + return "{}"; + } + + const lines = entries.map(([key, entryValue]) => { + const formattedKey = formatKey(key, language); + const formattedValue = serializeValue(entryValue, language, level + 1); + return `${indent(level + 1, language)}${formattedKey}: ${formattedValue}`; + }); + + return `{\n${lines.join(",\n")}\n${indent(level, language)}}`; + } + + return serializePrimitive(value, language); +} + +export function resolveExampleValue(param: ExampleParameterValue): unknown { + if (param.value === null || param.value === undefined) { + return param.required ? PLACEHOLDER_VALUES[param.type] : undefined; + } + + return param.value; +} + +export function buildToolInput( + parameters: ToolCodeExample["parameters"] +): Record { + const entries = Object.entries(parameters) + .map(([key, param]) => [key, resolveExampleValue(param)] as const) + .filter(([, value]) => value !== undefined); + + return Object.fromEntries(entries); +} + +function buildExecuteArgs(language: LanguageKey, includeUserId: boolean): string { + if (language === "python") { + const args = [ + "tool_name=TOOL_NAME,", + "input=tool_input,", + includeUserId ? "user_id=USER_ID," : null, + ].filter(Boolean); + + return [ + "response = client.tools.execute(", + ...args.map((arg) => `${indent(1, language)}${arg}`), + ")", + ].join("\n"); + } + + const args = [ + "tool_name: TOOL_NAME,", + "input: toolInput,", + includeUserId ? "user_id: USER_ID," : null, + ].filter(Boolean); + + return [ + "const response = await client.tools.execute({", + ...args.map((arg) => `${indent(1, language)}${arg}`), + "});", + ].join("\n"); +} + +export function generateJavaScriptExample(codeExample: ToolCodeExample): string { + const toolInput = buildToolInput(codeExample.parameters); + const toolInputLiteral = serializeValue(toolInput, "javascript", 0); + const lines: string[] = [ + 'import { Arcade } from "@arcadeai/arcadejs";', + "", + "const client = new Arcade(); // Automatically finds the `ARCADE_API_KEY` env variable", + "", + ]; + + if (codeExample.requiresAuth) { + lines.push('const USER_ID = "{arcade_user_id}";'); + } + + lines.push(`const TOOL_NAME = "${codeExample.toolName}";`, ""); + + if (codeExample.requiresAuth) { + lines.push( + "const authResponse = await client.tools.authorize({", + `${indent(1, "javascript")}tool_name: TOOL_NAME,`, + `${indent(1, "javascript")}user_id: USER_ID,`, + "});", + "", + 'if (authResponse.status !== "completed") {', + `${indent(1, "javascript")}console.log(\`Click this link to authorize: \${authResponse.url}\`);`, + "}", + "", + "await client.auth.waitForCompletion(authResponse);", + "" + ); + } + + lines.push(`const toolInput = ${toolInputLiteral};`, ""); + + lines.push(buildExecuteArgs("javascript", codeExample.requiresAuth), ""); + + lines.push("console.log(response);"); + + return lines.join("\n"); +} + +export function generatePythonExample(codeExample: ToolCodeExample): string { + const toolInput = buildToolInput(codeExample.parameters); + const toolInputLiteral = serializeValue(toolInput, "python", 0); + const lines: string[] = ["from arcadepy import Arcade", "", "client = Arcade()", ""]; + + if (codeExample.requiresAuth) { + lines.push('USER_ID = "{arcade_user_id}"'); + } + + lines.push(`TOOL_NAME = "${codeExample.toolName}"`, ""); + + if (codeExample.requiresAuth) { + lines.push( + "auth_response = client.tools.authorize(", + `${indent(1, "python")}tool_name=TOOL_NAME,`, + `${indent(1, "python")}user_id=USER_ID,`, + ")", + "", + 'if auth_response.status != "completed":', + `${indent(1, "python")}print(f"Click this link to authorize: {auth_response.url}")`, + "", + "client.auth.wait_for_completion(auth_response)", + "" + ); + } + + lines.push(`tool_input = ${toolInputLiteral}`, ""); + + lines.push(buildExecuteArgs("python", codeExample.requiresAuth), ""); + + lines.push("print(response)"); + + return lines.join("\n"); +} + +/** + * DynamicCodeBlock + * + * Generates and renders JavaScript and Python code examples dynamically + * from a ToolCodeExample configuration. + */ +export function DynamicCodeBlock({ + codeExample, + languages = DEFAULT_LANGUAGES, + tabLabel, +}: DynamicCodeBlockProps) { + const [isDarkMode, setIsDarkMode] = useState(false); + const [isExpanded, setIsExpanded] = useState(false); + const languageLabels = useMemo( + () => languages.map((lang) => LANGUAGE_LABELS[lang]), + [languages] + ); + + const [currentLanguage, setCurrentLanguage] = useState( + languageLabels[0] ?? "Python" + ); + + useEffect(() => { + const updateTheme = () => { + const html = document.documentElement; + const hasClassDark = html.classList.contains("dark"); + const hasClassLight = html.classList.contains("light"); + const hasDataThemeDark = html.getAttribute("data-theme") === "dark"; + const hasDataThemeLight = html.getAttribute("data-theme") === "light"; + const systemPrefersDark = + window.matchMedia?.("(prefers-color-scheme: dark)").matches ?? false; + + let isDark: boolean; + if (hasClassDark || hasDataThemeDark) { + isDark = true; + } else if (hasClassLight || hasDataThemeLight) { + isDark = false; + } else { + isDark = systemPrefersDark; + } + + setIsDarkMode(isDark); + }; + + updateTheme(); + + const observer = new MutationObserver(updateTheme); + observer.observe(document.documentElement, { + attributes: true, + attributeFilter: ["class", "data-theme"], + }); + + const mediaQuery = window.matchMedia?.("(prefers-color-scheme: dark)"); + mediaQuery?.addEventListener("change", updateTheme); + + return () => { + observer.disconnect(); + mediaQuery?.removeEventListener("change", updateTheme); + }; + }, []); + + const codeByLanguage = useMemo( + () => ({ + Python: generatePythonExample(codeExample), + JavaScript: generateJavaScriptExample(codeExample), + }), + [codeExample] + ); + + const selectedCode = codeByLanguage[currentLanguage as "Python" | "JavaScript"]; + const displayLabel = tabLabel ?? codeExample.tabLabel; + + const syntaxTheme = isDarkMode ? atomDark : oneLight; + const lowerCaseLanguage = currentLanguage.toLowerCase(); + + if (!isExpanded) { + return ( +
+ {displayLabel && ( +

{displayLabel}

+ )} + +
+ ); + } + + return ( +
+ {displayLabel && ( +

{displayLabel}

+ )} +
+
+ {languageLabels.length > 1 && ( + + )} +
+ +
+
+
+
+ + {lowerCaseLanguage} + +
+ +
+
+ + {selectedCode} + +
+
+
+ ); +} + +export default DynamicCodeBlock; diff --git a/app/_components/toolkit-docs/components/ParametersTable.tsx b/app/_components/toolkit-docs/components/ParametersTable.tsx new file mode 100644 index 000000000..d776fa019 --- /dev/null +++ b/app/_components/toolkit-docs/components/ParametersTable.tsx @@ -0,0 +1,125 @@ +"use client"; + +import type { ParametersTableProps, ToolParameter } from "../types"; + +/** + * Formats a parameter type for display. + */ +export function formatParameterType(param: ToolParameter): string { + if (param.type === "array" && param.innerType) { + return `array<${param.innerType}>`; + } + + return param.type; +} + +/** + * Formats enum values for display. + */ +export function formatEnumValues( + enumValues: string[] | null | undefined +): string[] { + if (!enumValues || enumValues.length === 0) { + return []; + } + + return enumValues.filter((value) => value.trim().length > 0); +} + +/** + * ParametersTable + * + * Renders a table of tool parameters with type, required, and description. + */ +export function ParametersTable({ + parameters, + enumBaseUrl, +}: ParametersTableProps) { + if (!parameters || parameters.length === 0) { + return ( +

+ No parameters required. +

+ ); + } + + return ( + + + + + + + + + + + {parameters.map((param, index) => { + const enumValues = formatEnumValues(param.enum); + const rowClass = + index % 2 === 0 + ? "bg-neutral-dark" + : "border-neutral-dark-medium border-b bg-transparent"; + + return ( + + + + + + + ); + })} + +
+ Parameter + + Type + + Required + + Description +
+ {param.name} + + {formatParameterType(param)} + + {param.required ? "Yes" : "No"} + + {param.description ?? "No description provided."} + {enumValues.length > 0 && ( +
+ Options:{" "} + {enumValues.map((value, valueIndex) => { + const content = {value}; + const separator = + valueIndex < enumValues.length - 1 ? ", " : ""; + + if (enumBaseUrl) { + return ( + + + {content} + + {separator} + + ); + } + + return ( + + {content} + {separator} + + ); + })} +
+ )} +
+ ); +} + +export default ParametersTable; diff --git a/app/_components/toolkit-docs/components/ScopesDisplay.tsx b/app/_components/toolkit-docs/components/ScopesDisplay.tsx new file mode 100644 index 000000000..79db24130 --- /dev/null +++ b/app/_components/toolkit-docs/components/ScopesDisplay.tsx @@ -0,0 +1,87 @@ +"use client"; + +import { Callout } from "nextra/components"; + +import type { ScopesDisplayProps } from "../types"; + +/** + * Normalizes scope values by trimming and removing duplicates. + */ +export function normalizeScopes(scopes: string[]): string[] { + const uniqueScopes = new Set(); + + for (const scope of scopes) { + const trimmed = scope.trim(); + if (trimmed.length > 0) { + uniqueScopes.add(trimmed); + } + } + + return Array.from(uniqueScopes); +} + +function ScopesInline({ scopes }: { scopes: string[] }) { + if (scopes.length === 0) { + return ( + + No scopes required. + + ); + } + + return ( +
+ {scopes.map((scope) => ( + + {scope} + + ))} +
+ ); +} + +function ScopesList({ scopes }: { scopes: string[] }) { + if (scopes.length === 0) { + return ( +

No scopes required.

+ ); + } + + return ( +
    + {scopes.map((scope) => ( +
  • + {scope} +
    • + ))} +
+ ); +} + +/** + * ScopesDisplay + * + * Renders OAuth scopes inline or inside a callout. + */ +export function ScopesDisplay({ + scopes, + variant = "inline", + title, +}: ScopesDisplayProps) { + const normalizedScopes = normalizeScopes(scopes); + + if (variant === "callout") { + return ( + + + + ); + } + + return ; +} + +export default ScopesDisplay; diff --git a/app/_components/toolkit-docs/components/ToolSection.tsx b/app/_components/toolkit-docs/components/ToolSection.tsx new file mode 100644 index 000000000..bf19a2b42 --- /dev/null +++ b/app/_components/toolkit-docs/components/ToolSection.tsx @@ -0,0 +1,189 @@ +"use client"; + +import { DocumentationChunkRenderer, hasChunksAt } from "./DocumentationChunkRenderer"; +import { DynamicCodeBlock } from "./DynamicCodeBlock"; +import { ParametersTable } from "./ParametersTable"; +import { ScopesDisplay } from "./ScopesDisplay"; +import { toToolAnchorId } from "./AvailableToolsTable"; +import type { ToolSectionProps } from "../types"; + +export function shouldRenderDefaultSection( + chunks: ToolSectionProps["tool"]["documentationChunks"], + location: "description" | "parameters" | "auth" | "secrets" | "output" +): boolean { + return !hasChunksAt(chunks, location, "replace"); +} + +/** + * ToolSection + * + * Renders a single tool section with parameters, scopes, secrets, output, and example. + */ +export function ToolSection({ tool }: ToolSectionProps) { + const anchorId = toToolAnchorId(tool.qualifiedName); + const scopes = tool.auth?.scopes ?? []; + const secretsInfo = tool.secretsInfo ?? []; + + const showDescription = shouldRenderDefaultSection( + tool.documentationChunks, + "description" + ); + const showParameters = shouldRenderDefaultSection( + tool.documentationChunks, + "parameters" + ); + const showAuth = shouldRenderDefaultSection(tool.documentationChunks, "auth"); + const showSecrets = shouldRenderDefaultSection( + tool.documentationChunks, + "secrets" + ); + const showOutput = shouldRenderDefaultSection(tool.documentationChunks, "output"); + + return ( +
+

{tool.qualifiedName}

+ + + {showDescription && ( +

+ {tool.description ?? "No description provided."} +

+ )} + + + +

Parameters

+ + {showParameters && } + + + +

Scopes

+ + {showAuth && } + + + +

Secrets

+ + {showSecrets && ( +
+ {tool.secrets.length === 0 ? ( +

No secrets required.

+ ) : ( +
    + {secretsInfo.length > 0 + ? secretsInfo.map((secret) => ( +
  • + {secret.name} + + ({secret.type}) + +
    • + )) + : tool.secrets.map((secret) => ( +
  • + {secret} +
    • + ))} +
+ )} +
+ )} + + + +

Output

+ + {showOutput && ( +
+ {tool.output ? ( + <> +

+ Type: {tool.output.type} +

+ {tool.output.description && ( +

{tool.output.description}

+ )} + + ) : ( +

No output schema provided.

+ )} +
+ )} + + + +

Example

+ {tool.codeExample ? ( + + ) : ( +

+ No example available for this tool. +

+ )} +
+ ); +} + +export default ToolSection; diff --git a/app/_components/toolkit-docs/components/ToolkitHeader.tsx b/app/_components/toolkit-docs/components/ToolkitHeader.tsx new file mode 100644 index 000000000..ead70c1a3 --- /dev/null +++ b/app/_components/toolkit-docs/components/ToolkitHeader.tsx @@ -0,0 +1,199 @@ +"use client"; + +import { + Badge, + ByocBadge, + getToolkitIconByName, + ProBadge, +} from "@arcadeai/design-system"; +import type React from "react"; + +import type { ToolkitHeaderProps, ToolkitType } from "../types"; + +/** + * Configuration for toolkit type badges + */ +const TYPE_CONFIG: Record< + Exclude, + { label: string; color: string } +> = { + arcade: { label: "Arcade", color: "text-brand-accent" }, + arcade_starter: { label: "Starter", color: "text-gray-500" }, + verified: { label: "Verified", color: "text-green-600" }, + community: { label: "Community", color: "text-purple-600" }, +}; + +/** + * Renders toolkit type, BYOC, and Pro badges + */ +function ToolkitBadges({ + type, + isByoc, + isPro, +}: { + type: ToolkitType; + isByoc: boolean; + isPro: boolean; +}) { + const typeInfo = type !== "auth" ? TYPE_CONFIG[type] : null; + + const showBadges = isPro || isByoc || typeInfo; + + if (!showBadges) { + return null; + } + + return ( +
+ {typeInfo && ( + + {typeInfo.label} + + )} + {isByoc && } + {isPro && } +
+ ); +} + +/** + * ToolkitHeader + * + * Renders the header section for a toolkit documentation page, including: + * - Toolkit icon (from Design System) + * - Toolkit name badges (type, BYOC, Pro) + * - Description + * - Author + * - Auth provider link + * + * @example + * ```tsx + * + * ``` + */ +export function ToolkitHeader({ + id, + label, + description, + metadata, + auth, + version, + author = "Arcade", +}: ToolkitHeaderProps): React.ReactElement { + // Get the icon component from Design System + const IconComponent = getToolkitIconByName(label); + + // Determine auth display + const authProviderName = auth?.providerId + ? auth.providerId.charAt(0).toUpperCase() + auth.providerId.slice(1) + : null; + const authDocsUrl = auth?.providerId + ? `/references/auth-providers/${auth.providerId.toLowerCase()}` + : null; + const authProviderLink = + authProviderName && authDocsUrl ? ( + + {authProviderName} auth provider + + ) : null; + + return ( +
+
+ {/* Icon */} + {IconComponent && ( +
+
+ +
+
+ )} + + {/* Content */} +
+ {/* Badges */} + + + {/* Description */} + {description && ( +

+ Description: + {description} +

+ )} + + {/* Author */} +

+ Author: + {author} +

+ + {/* Version (optional) */} + {version && ( +

+ Version: + {version} +

+ )} + + {/* Code link */} + {id && ( +

+ Code: + + GitHub + +

+ )} + + {/* Auth info */} +

+ Auth: + {auth?.type === "oauth2" ? ( + <> + User authorization + {authProviderLink && <> via the {authProviderLink}} + + ) : auth?.type === "api_key" ? ( + "API key authentication" + ) : auth?.type === "mixed" ? ( + <> + User authorization + {authProviderLink && <> via the {authProviderLink}} and API + key authentication + + ) : ( + "No authentication required" + )} +

+
+
+
+ ); +} + +export default ToolkitHeader; diff --git a/app/_components/toolkit-docs/components/ToolkitPage.tsx b/app/_components/toolkit-docs/components/ToolkitPage.tsx new file mode 100644 index 000000000..3b210e93a --- /dev/null +++ b/app/_components/toolkit-docs/components/ToolkitPage.tsx @@ -0,0 +1,83 @@ +import ReactMarkdown from "react-markdown"; + +import { AvailableToolsTable } from "./AvailableToolsTable"; +import { DocumentationChunkRenderer } from "./DocumentationChunkRenderer"; +import { ToolkitHeader } from "./ToolkitHeader"; +import { ToolSection } from "./ToolSection"; +import type { ToolkitPageProps } from "../types"; + +/** + * ToolkitPage + * + * Composes the full toolkit documentation page from JSON data. + */ +export function ToolkitPage({ data }: ToolkitPageProps) { + const tools = data.tools ?? []; + + return ( +
+ + + + + + {data.summary && ( +
+ {data.summary} +
+ )} + +

Available tools

+ ({ + name: tool.name, + qualifiedName: tool.qualifiedName, + description: tool.description, + secrets: tool.secrets, + secretsInfo: tool.secretsInfo, + }))} + /> + + {tools.map((tool) => ( + + ))} + + + + +
+ ); +} + +export default ToolkitPage; diff --git a/app/_components/toolkit-docs/components/index.ts b/app/_components/toolkit-docs/components/index.ts new file mode 100644 index 000000000..6ca047eae --- /dev/null +++ b/app/_components/toolkit-docs/components/index.ts @@ -0,0 +1,12 @@ +/** + * Toolkit Documentation Components + */ + +export { DocumentationChunkRenderer, hasChunksAt } from "./DocumentationChunkRenderer"; +export { DynamicCodeBlock } from "./DynamicCodeBlock"; +export { AvailableToolsTable } from "./AvailableToolsTable"; +export { ParametersTable } from "./ParametersTable"; +export { ScopesDisplay } from "./ScopesDisplay"; +export { ToolSection } from "./ToolSection"; +export { ToolkitHeader } from "./ToolkitHeader"; +export { ToolkitPage } from "./ToolkitPage"; diff --git a/app/_components/toolkit-docs/index.ts b/app/_components/toolkit-docs/index.ts new file mode 100644 index 000000000..a3316e4bd --- /dev/null +++ b/app/_components/toolkit-docs/index.ts @@ -0,0 +1,48 @@ +/** + * Toolkit Documentation Components + * + * This module provides React components for rendering toolkit documentation + * from JSON data sources. It enables dynamic documentation generation + * while preserving the ability to inject custom content. + * + * @example + * ```tsx + * import { ToolkitHeader, DocumentationChunkRenderer } from '@/app/_components/toolkit-docs'; + * import toolkitData from '@/data/toolkits/github.json'; + * + * export default function GitHubPage() { + * return ( + * <> + * + * + * + * ); + * } + * ``` + */ + +// Types +export * from "./types"; + +// Components +export { + DocumentationChunkRenderer, + hasChunksAt, +} from "./components/DocumentationChunkRenderer"; +export { AvailableToolsTable } from "./components/AvailableToolsTable"; +export { DynamicCodeBlock } from "./components/DynamicCodeBlock"; +export { ParametersTable } from "./components/ParametersTable"; +export { ScopesDisplay } from "./components/ScopesDisplay"; +export { ToolSection } from "./components/ToolSection"; +export { ToolkitHeader } from "./components/ToolkitHeader"; +export { ToolkitPage } from "./components/ToolkitPage"; diff --git a/app/_components/toolkit-docs/types/index.ts b/app/_components/toolkit-docs/types/index.ts new file mode 100644 index 000000000..8e9a924f0 --- /dev/null +++ b/app/_components/toolkit-docs/types/index.ts @@ -0,0 +1,427 @@ +/** + * Type definitions for toolkit documentation MDX components + * + * These types are designed for React component props and are compatible + * with the JSON data structure from toolkit-docs-generator. + */ + +// ============================================================================ +// Documentation Chunk Types +// ============================================================================ + +/** + * Type of documentation chunk content + */ +export type DocumentationChunkType = + | "callout" + | "markdown" + | "code" + | "warning" + | "info" + | "tip"; + +/** + * Location where the chunk should be injected + */ +export type DocumentationChunkLocation = + | "header" + | "description" + | "parameters" + | "auth" + | "secrets" + | "output" + | "footer"; + +/** + * Position relative to the location + */ +export type DocumentationChunkPosition = "before" | "after" | "replace"; + +/** + * Callout variant for styling + */ +export type DocumentationChunkVariant = + | "default" + | "destructive" + | "warning" + | "info" + | "success"; + +/** + * A documentation chunk represents custom content to inject into docs + */ +export interface DocumentationChunk { + /** Type of content */ + type: DocumentationChunkType; + /** Where to inject the content */ + location: DocumentationChunkLocation; + /** Position relative to location */ + position: DocumentationChunkPosition; + /** The actual content (markdown string) */ + content: string; + /** Optional title for callouts */ + title?: string; + /** Optional variant for styling */ + variant?: DocumentationChunkVariant; +} + +// ============================================================================ +// Tool Parameter Types +// ============================================================================ + +/** + * Tool parameter definition + */ +export interface ToolParameter { + /** Parameter name */ + name: string; + /** Parameter type (string, integer, boolean, array, object) */ + type: string; + /** For array types, the inner element type */ + innerType?: string; + /** Whether the parameter is required */ + required: boolean; + /** Parameter description */ + description: string | null; + /** Enum values if this is an enum parameter */ + enum: string[] | null; + /** Whether the parameter can be inferred by an LLM */ + inferrable?: boolean; + /** Default value if not provided */ + default?: unknown; +} + +// ============================================================================ +// Tool Auth Types +// ============================================================================ + +/** + * Tool-level authentication requirements + */ +export interface ToolAuth { + /** Auth provider ID (e.g., "github", "google") */ + providerId: string | null; + /** Provider type (e.g., "oauth2", "api_key") */ + providerType: string; + /** Required OAuth scopes for this specific tool */ + scopes: string[]; +} + +// ============================================================================ +// Tool Output Types +// ============================================================================ + +/** + * Tool output schema + */ +export interface ToolOutput { + /** Output type (object, array, string, etc.) */ + type: string; + /** Output description */ + description: string | null; +} + +// ============================================================================ +// Tool Secrets Types +// ============================================================================ + +export type SecretType = + | "api_key" + | "token" + | "client_secret" + | "webhook_secret" + | "private_key" + | "password" + | "unknown"; + +export interface ToolSecret { + /** Secret name */ + name: string; + /** Secret type classification */ + type: SecretType; +} + +// ============================================================================ +// Code Example Types +// ============================================================================ + +/** + * Parameter value with type information for code generation + */ +export interface ExampleParameterValue { + /** The example value to use in generated code */ + value: unknown; + /** Parameter type for proper serialization */ + type: "string" | "integer" | "boolean" | "array" | "object"; + /** Whether this parameter is required */ + required: boolean; +} + +/** + * Tool code example configuration + * Used to generate Python/JavaScript example code + */ +export interface ToolCodeExample { + /** Full tool name (e.g., "Github.SetStarred") */ + toolName: string; + /** Parameter values with type info */ + parameters: Record; + /** Whether this tool requires user authorization */ + requiresAuth: boolean; + /** Auth provider ID if auth is required */ + authProvider?: string; + /** Optional tab label for the code example */ + tabLabel?: string; +} + +// ============================================================================ +// Tool Definition Types +// ============================================================================ + +/** + * Complete tool definition with all documentation data + */ +export interface ToolDefinition { + /** Tool name (e.g., "CreateIssue") */ + name: string; + /** Qualified name (e.g., "Github.CreateIssue") */ + qualifiedName: string; + /** Fully qualified name with version (e.g., "Github.CreateIssue@1.0.0") */ + fullyQualifiedName: string; + /** Tool description */ + description: string | null; + /** Tool parameters */ + parameters: ToolParameter[]; + /** Tool authentication requirements */ + auth: ToolAuth | null; + /** Required secrets */ + secrets: string[]; + /** Classified secrets (LLM-generated) */ + secretsInfo?: ToolSecret[]; + /** Tool output schema */ + output: ToolOutput | null; + /** Custom documentation chunks for this tool */ + documentationChunks: DocumentationChunk[]; + /** Generated code example configuration */ + codeExample?: ToolCodeExample; +} + +// ============================================================================ +// Toolkit Metadata Types +// ============================================================================ + +/** + * Toolkit category for navigation grouping + */ +export type ToolkitCategory = + | "productivity" + | "social" + | "development" + | "entertainment" + | "search" + | "payments" + | "sales" + | "databases" + | "customer-support"; + +/** + * Toolkit type classification + */ +export type ToolkitType = + | "arcade" + | "arcade_starter" + | "verified" + | "community" + | "auth"; + +/** + * Toolkit metadata from Design System + */ +export interface ToolkitMetadata { + /** Category for navigation grouping */ + category: ToolkitCategory; + /** Icon URL */ + iconUrl: string; + /** Whether this toolkit requires BYOC (Bring Your Own Credentials) */ + isBYOC: boolean; + /** Whether this is a Pro feature */ + isPro: boolean; + /** Toolkit type classification */ + type: ToolkitType; + /** Link to documentation */ + docsLink: string; + /** Whether this toolkit is coming soon */ + isComingSoon?: boolean; + /** Whether this toolkit is hidden */ + isHidden?: boolean; +} + +// ============================================================================ +// Toolkit Auth Types +// ============================================================================ + +/** + * Toolkit-level authentication type + */ +export type ToolkitAuthType = "oauth2" | "api_key" | "mixed" | "none"; + +/** + * Toolkit-level authentication summary + */ +export interface ToolkitAuth { + /** Auth type */ + type: ToolkitAuthType; + /** Auth provider ID */ + providerId: string | null; + /** Union of all scopes required by tools in this toolkit */ + allScopes: string[]; +} + +// ============================================================================ +// Complete Toolkit Data Type +// ============================================================================ + +/** + * Complete toolkit data structure for rendering documentation + * This is the main type consumed by the ToolkitPage component + */ +export interface ToolkitData { + /** Unique toolkit ID (e.g., "Github") */ + id: string; + /** Human-readable label (e.g., "GitHub") */ + label: string; + /** Toolkit version (e.g., "1.0.0") */ + version: string; + /** Toolkit description */ + description: string | null; + /** LLM-generated summary */ + summary?: string; + /** Metadata from Design System */ + metadata: ToolkitMetadata; + /** Authentication requirements */ + auth: ToolkitAuth | null; + /** All tools in this toolkit */ + tools: ToolDefinition[]; + /** Toolkit-level documentation chunks */ + documentationChunks: DocumentationChunk[]; + /** Custom imports for MDX */ + customImports: string[]; + /** Sub-pages that exist for this toolkit */ + subPages: string[]; + /** Generation timestamp */ + generatedAt?: string; +} + +// ============================================================================ +// Component Props Types +// ============================================================================ + +/** + * Props for DocumentationChunkRenderer component + */ +export interface DocumentationChunkRendererProps { + /** Array of documentation chunks to filter and render */ + chunks: DocumentationChunk[]; + /** Filter by location */ + location: DocumentationChunkLocation; + /** Filter by position */ + position: DocumentationChunkPosition; + /** Optional className for the wrapper */ + className?: string; +} + +/** + * Props for ToolkitHeader component + */ +export interface ToolkitHeaderProps { + /** Toolkit ID for icon lookup */ + id: string; + /** Display label */ + label: string; + /** Toolkit description */ + description: string | null; + /** Summary text (optional) */ + summary?: string; + /** Toolkit metadata */ + metadata: ToolkitMetadata; + /** Authentication info */ + auth: ToolkitAuth | null; + /** Toolkit version */ + version?: string; + /** Author name (defaults to "Arcade") */ + author?: string; +} + +/** + * Props for ParametersTable component + */ +export interface ParametersTableProps { + /** Array of parameters to render */ + parameters: ToolParameter[]; + /** Base URL for enum references (optional) */ + enumBaseUrl?: string; +} + +/** + * Props for ScopesDisplay component + */ +export interface ScopesDisplayProps { + /** Array of OAuth scopes */ + scopes: string[]; + /** Display variant */ + variant?: "inline" | "callout"; + /** Optional title for the callout */ + title?: string; +} + +/** + * Props for DynamicCodeBlock component + */ +export interface DynamicCodeBlockProps { + /** Code example configuration */ + codeExample: ToolCodeExample; + /** Languages to generate (defaults to both) */ + languages?: ("python" | "javascript")[]; + /** Tab label override */ + tabLabel?: string; +} + +/** + * Props for ToolSection component + */ +export interface ToolSectionProps { + /** Tool definition */ + tool: ToolDefinition; + /** Toolkit ID (for generating anchors) */ + toolkitId: string; +} + +/** + * Props for AvailableToolsTable component + */ +export interface AvailableToolsTableProps { + /** Tools to display in the table */ + tools: Array<{ + name: string; + qualifiedName: string; + description: string | null; + secrets?: string[]; + secretsInfo?: ToolSecret[]; + }>; + /** Optional label for the secrets column */ + secretsColumnLabel?: string; + /** How to summarize secrets in the table */ + secretsDisplay?: "summary" | "names" | "types"; + /** Override labels for secret types */ + secretTypeLabels?: Partial>; + /** Base URL for linking secret type docs */ + secretTypeDocsBaseUrl?: string; +} + +/** + * Props for ToolkitPage component + */ +export interface ToolkitPageProps { + /** Complete toolkit data */ + data: ToolkitData; +} diff --git a/app/en/resources/integrations/development/github/preview/page.mdx b/app/en/resources/integrations/development/github/preview/page.mdx new file mode 100644 index 000000000..18065a9a4 --- /dev/null +++ b/app/en/resources/integrations/development/github/preview/page.mdx @@ -0,0 +1,13 @@ +--- +title: "Toolkit docs preview" +description: "Preview the JSON-driven toolkit documentation components." +--- + +import githubData from "@/toolkit-docs-generator/tests/fixtures/github-toolkit.json"; +import { ToolkitPage } from "@/app/_components/toolkit-docs"; + +# Toolkit docs preview + +Use this page to preview the JSON-driven toolkit components with real fixture data. + + diff --git a/next-env.d.ts b/next-env.d.ts index 9edff1c7c..c4b7818fb 100644 --- a/next-env.d.ts +++ b/next-env.d.ts @@ -1,6 +1,6 @@ /// /// -import "./.next/types/routes.d.ts"; +import "./.next/dev/types/routes.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/package.json b/package.json index d7fd42008..57206346d 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "react": "19.2.3", "react-dom": "19.2.3", "react-hook-form": "7.65.0", + "react-markdown": "^10.1.0", "react-syntax-highlighter": "16.1.0", "swagger-ui-react": "^5.30.0", "tailwindcss-animate": "1.0.7", diff --git a/planningdoc.md b/planningdoc.md new file mode 100644 index 000000000..15470fb88 --- /dev/null +++ b/planningdoc.md @@ -0,0 +1,663 @@ +# User Stories + +### 1. The Toolkit Developer + +- As a **Toolkit Developer**, I want to **have my toolkit's documentation updated automatically when I publish a new version**, so that **I don't have to manually run documentation scripts**. + +### 2. The Application Developer (Consumer) + +- As an **Application Developer**, I want to **access the latest information on required scopes, parameters, and tool definitions in the documentation**, so that **I can implement integrations correctly without relying on outdated or incomplete references**. + +### 3. The Documentation Maintainer + +- As a **Documentation Maintainer**, I want to **write custom warnings and context for tools that persist across automated updates**, so that **valuable human knowledge isn't overwritten by the generation script**. + +### 4. The Product Manager + +- As a **Product Manager**, I want to **ensure all toolkits in the documentation have consistent branding (icons, categories)**, so that **the user experience is unified across the marketing site and documentation**. + +# Engineering Planning + +At the moment toolkit docs are built either manually or using the document generator tool built by Renato. With the increase of toolkits and many required updates, this process is not scalating well. We need to create a new process to build toolkit documentations that are more maintainable and easier to update, also taking the opportunity to improve metadata documentation, as scopes (what was the primary goal for this proj now is a side effect of the sugested process). + +Renato's tool lives in the arcade docs repo. It is a CLI tool that requires direct access to the python toolkit source code. So any changes in the toolkits after the documentation is built will require a new manual run in order to have an updated documentation. We will improve this process by getting the tool data from the engine API and combining it with design system metadata to programmatically deal with the documentation generation. + +## Tool documentation json as new source of data + +Right now tools are mdx files that follow a standard format, for the most part, also there are some LLM or human generated sections for details that require a little more care and need to be brought to the users attention for a better understanding of requirements or limitations of the tool. Instead of still using this approach, MDX components will be used and populated using a new json file that will contain all the required info for the tool, like description, parameters, scopes, secrets, etc, and written MD specific sections, like those generated by some form of reasoning. + +NOTE: Latter we can replace this json for an engine api call, when tools metadate feature to accept dynamic data as we can move all the info we need to there. + +### Data Merge Architecture + +``` +┌─────────────────────────────────────────────────────────────────────────────────────────┐ +│ DATA MERGE PIPELINE │ +└─────────────────────────────────────────────────────────────────────────────────────────┘ + + SOURCE A: ENGINE API SOURCE B: DESIGN SYSTEM SOURCE C: EXISTING DOCS + (Dynamic Tool Data) (Toolkit Metadata) (Manual Content) + ┌───────────────────┐ ┌───────────────────┐ ┌───────────────────┐ + │ /v1/tools/all │ │ /metadata/toolkits│ │ .mdx files │ + │ │ │ │ │ │ + │ • Tools │ │ • Category │ │ • Custom Callouts │ + │ • Parameters │ │ • Icon URL │ │ • Warnings │ + │ • Scopes ✅ │ │ • BYOC/Pro flags │ │ • Extra Context │ + │ • Auth Info │ │ • Labels │ │ • Sub-pages │ + └─────────┬─────────┘ └─────────┬─────────┘ └─────────┬─────────┘ + │ │ │ + └──────────────┬──────────────┴──────────────────────────────┘ + │ + ▼ + ┌───────────────────────┐ + │ MERGER SCRIPT │ + │ (In new repo) │ + └──────────┬────────────┘ + │ + ▼ + ┌───────────────────────┐ + │ MERGED JSON DATA │ + │ (One file per kit) │ + └──────────┬────────────┘ +``` + +With that goal in mind, we need to define a new json format to store the toolkit documentation data, that must, for at least the moment, be gathered from multiple sources, those being: + +- **Engine API**: We must define a new endpoint that will fetch all tools definitions for a toolkit, something like the tools list endpoint but we can make it more focused to this project goal. Probably it will be still a private endpoint that will require a valid API key. +- **Design System**: We will use the design system dependency to get the toolkit metadata, like category, icon, type, flags, etc. Docs already has it as a node pack dependency. +- **Current MDX doc files**: We will use the current MDX doc files to get the custom sections that are written by hand and need to be brought to the users attention for a better understanding of requirements or limitations of the tool. It will be work for a one time script to extract what we need. + +### JSON Spec (Annotated with Sources) + +The merged JSON format is designed to be the single source of truth for rendering a toolkit's documentation page. It is divided into five logical sections: + +1. **Identification**: Basic identity fields like `id` (slug), `label` (display name), `version`, and `description`. These are used for page titles, URLs, and SEO. +2. **Metadata**: Presentation and business logic fields sourced from the Design System. This includes the `category` (for navigation grouping), `icon_url` (for the page header), and flags like `is_byoc` (requires own credentials) and `is_pro` (paid feature). +3. **Authentication**: A summary of the authentication requirements for the toolkit. It aggregates `default_scopes` from all tools to show a high-level view of permissions. +4. **Tools**: The core content. A list of all tools in the toolkit, sourced directly from the Engine API. Each tool object groups **all** relevant data for that tool: + - `name`, `description`, `parameters` (inputs) + - `auth` requirements (specific scopes for _this_ tool) + - `secrets` + - `output` schema + - `documentation`: Tool-specific custom content (e.g., callouts, warnings) extracted from existing docs. +5. **Documentation Content**: Toolkit-level custom content that doesn't belong to a specific tool, such as general setup instructions or extended auth details. + +```jsonc +{ + // --- IDENTIFICATION --- + "id": "Github", // [Source: Design System] Unique ID + "label": "GitHub", // [Source: Design System] Display name + "version": "1.0.0", // [Source: Engine API] Toolkit version + "description": "...", // [Source: Engine API] Toolkit description + + // --- METADATA --- + "metadata": { + "category": "development", // [Source: Design System] + "icon_url": "...", // [Source: Design System] + "is_byoc": false, // [Source: Design System] + "is_pro": false, // [Source: Design System] + "toolkit_type": "arcade", // [Source: Design System] + "docs_link": "..." // [Source: Design System] + }, + + // --- AUTHENTICATION --- + "auth": { + "type": "oauth2", // [Source: Engine API] Derived from tool requirements + "provider_id": "github", // [Source: Engine API] + "all_scopes": [ + // [Source: Engine API] Union of ALL scopes required by tools + "repo", + "user:email" + ] + }, + + // --- TOOLS --- + "tools": [ + { + "name": "CreateIssue", // [Source: Engine API] + "qualified_name": "Github...", // [Source: Engine API] + "description": "Creates...", // [Source: Engine API] + "parameters": [ + // [Source: Engine API] + { + "name": "title", + "type": "string", + "required": true, + "description": "..." + } + ], + "auth": { + "scopes": ["repo"] // [Source: Engine API] Specific scopes for THIS tool + }, + "secrets": [], // [Source: Engine API] + + // [Source: Existing MDX Docs] Tool-specific manual content + "documentation": { + "before_description": { + "type": "callout", + "content": "Warning: This tool has specific rate limits..." + }, + "after_parameters": { + "type": "markdown", + "content": "Note on parameter usage..." + } + } + } + ], + + // --- TOOLKIT DOCUMENTATION --- + "docs": { + // [Source: Existing MDX Docs] Toolkit-level manual content + "custom_imports": [ + "import MyComponent from \"@/app/_components/my-component\";" + ], + "sections": { + "auth_extended": { + "type": "markdown", + "content": "Extended explanation about auth setup..." + }, + "reference_extended": { + "type": "markdown", + "content": "Additional reference material..." + } + } + } +} +``` + +## Engine new endpoint to fetch tool data + +For collecting data from the tools in Engine, it is desired to be able to get a list of tools having the provider identifier and the latest version, being the last useful for when there are some tool definition updates getting the data only for this new version and not for all tools with other versions. +Right now engine has a list endpoint that returns the complete tool definition for toolkits, but it does not allow the filtering by version and also has a chuck size limiting (I was informed that is like a few thousends, so not exactly a issue). Those are small details that would probably require some extra work either on the engine or in the consumer side, but I believe that a secret endpoint that we can use as base the list tools one and tailor for the documentation seems a more interesting approach, as one would freely update it depending of the needs for the whole documentation generation workflow and also will avoid being affected by changes or limitations that are required by the list tools endpoint. + +Note: After a meeting with Eric and Sergion related to the tool metadata project, I believe that the use of a new dedicate endpoint for the documentation generation got more reasons to be the choice, as we might add in the future some dicts as metadata and do some processing to return it to the docs. + +Said that, for this endpoint the OpenAPI scheme looks like this: + +### OpenAPI Schema + +```yaml +/v1/tools/all: + get: + security: + - Bearer: [] + summary: List All Tools + description: Returns all tools without pagination, with optional filters. Returns simplified tool schema (no formatted schemas). + operationId: tools-list-all + tags: + - Tools + parameters: + - name: provider + in: query + description: Filter by auth provider ID (e.g., google, slack, github) + required: false + schema: + type: string + - name: version + in: query + description: Filter by toolkit version + required: false + schema: + type: string + - name: toolkit + in: query + description: Filter by toolkit name + required: false + schema: + type: string + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/ToolListAllResponse" + "401": + description: Unauthorized + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "500": + description: Internal Server Error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" +``` + +### Response Schema + +```yaml +components: + schemas: + ToolListAllResponse: + type: object + required: + - items + - total_count + properties: + items: + type: array + items: + $ref: "#/components/schemas/ToolSimplified" + total_count: + type: integer + format: int64 + description: Total number of tools returned + + ToolSimplified: + type: object + required: + - fully_qualified_name + - qualified_name + - name + - toolkit + - input + properties: + fully_qualified_name: + type: string + description: Full tool identifier including version (e.g., Google.Calendar.CreateEvent@1.0.0) + example: "Google.Calendar.CreateEvent@1.0.0" + qualified_name: + type: string + description: Tool identifier without version (e.g., Google.Calendar.CreateEvent) + example: "Google.Calendar.CreateEvent" + name: + type: string + description: Tool name only + example: "CreateEvent" + description: + type: string + nullable: true + description: Tool description + toolkit: + $ref: "#/components/schemas/ToolkitInfo" + input: + $ref: "#/components/schemas/ToolInput" + output: + $ref: "#/components/schemas/ToolOutput" + nullable: true + requirements: + $ref: "#/components/schemas/ToolRequirementsSimplified" + nullable: true + + ToolkitInfo: + type: object + required: + - name + - version + properties: + name: + type: string + example: "Google" + description: + type: string + nullable: true + version: + type: string + example: "1.0.0" + + ToolInput: + type: object + required: + - parameters + properties: + parameters: + type: array + items: + $ref: "#/components/schemas/ToolParameter" + + ToolParameter: + type: object + required: + - name + - required + - value_schema + properties: + name: + type: string + required: + type: boolean + description: + type: string + nullable: true + value_schema: + $ref: "#/components/schemas/ValueSchema" + inferrable: + type: boolean + default: true + + ValueSchema: + type: object + required: + - val_type + properties: + val_type: + type: string + description: Data type (string, integer, boolean, array, object, etc.) + inner_val_type: + type: string + nullable: true + description: For array types, the type of array elements + enum: + type: array + items: + type: string + nullable: true + + ToolOutput: + type: object + properties: + available_modes: + type: array + items: + type: string + description: + type: string + nullable: true + value_schema: + $ref: "#/components/schemas/ValueSchema" + nullable: true + + ToolRequirementsSimplified: + type: object + description: Simplified requirements - only keys and provider info, no status + properties: + authorization: + $ref: "#/components/schemas/AuthRequirementSimplified" + nullable: true + secrets: + type: array + items: + type: string + description: List of required secret keys + example: ["API_KEY", "WEBHOOK_SECRET"] + + AuthRequirementSimplified: + type: object + description: OAuth2 authorization requirement info (no status) + properties: + id: + type: string + description: Authorization requirement ID + provider_id: + type: string + nullable: true + description: Provider identifier (e.g., google, slack) + example: "google" + provider_type: + type: string + description: Type of provider (oauth2, api_key, etc.) + example: "oauth2" + scopes: + type: array + items: + type: string + nullable: true + description: Required OAuth2 scopes + example: + [ + "https://www.googleapis.com/auth/calendar", + "https://www.googleapis.com/auth/calendar.events", + ] +``` + +### Planning: Engine Code Implementation + +To implement this new endpoint in the Engine codebase, those next steps will be followed: + +_NOTE: I need feedback from engine engineers herem so they can say if something is wrong or missing._ + +1. **Define Schema Types**: + + - Create `internal/api/schemas/tools_all.go` to define `ToolListAllResponse`, `ToolSimplified`, and related structs. + - These structs should mirror the OpenAPI spec, ensuring all necessary fields (especially `scopes`) are included. + +2. **Implement Handler**: + + - Add `ListAllToolsHandler` in `internal/api/tool_handlers.go`. + - This handler needs to: + - Accept `provider`, `version`, and `toolkit` query parameters. + - Query the tool catalog/registry efficiently (avoiding pagination logic used in the standard list endpoint). + - Map the internal tool representation to `ToolSimplified` structs. + - Extract auth requirements and scopes correctly. + +3. **Register Route**: + + - In `internal/api/server.go`, register the new route `GET /tools/all`. + - Ensure it's protected with appropriate authentication (Bearer token) or API key validation, as this is intended for internal/build-time use. + +4. **Documentation**: + + - None docs, at least public available, as it will be a internal use only endpoint. + +5. **Testing**: + + - Add unit tests for the new handler to verify filtering logic and response format. + +## Design system extraction + +The Design System project contains metadata related to marketing and presentation, such as icons, categories, labels, and feature flags (e.g., `isBYOC`). This data is already available as a dependency in the `docs` project, which simplifies access. + +We will access this data primarily through the **`lib/metadata`** module exports. Specifically, we will use the `TOOLKIT_CATALOGUE` or the `TOOLKITS` array to look up metadata by toolkit ID. + +### Data Source Details + +- **Location**: `lib/metadata/` directory. +- **Key Files**: + - `lib/metadata/toolkits.ts`: Contains the main `TOOLKITS` array definition. + - `lib/metadata/index.ts`: Exports `TOOLKIT_CATALOGUE` (a record/map for easy lookup by ID). +- **Data Structure (`Toolkit` type)**: + - `id`: The unique identifier (e.g., "Github", "Gmail"). We will use this to match against Engine API data. + - `label`: Human-readable name (e.g., "GitHub"). + - `category`: Functional category (e.g., "development", "productivity"). + - `publicIconUrl`: URL to the SVG icon. + - `isBYOC`: Boolean flag indicating if users need their own credentials. + - `isPro`: Boolean flag indicating if the toolkit is a paid feature. + - `type`: Classification like "arcade", "verified", "community". + - `docsLink`: Full URL to existing docs (useful for cross-referencing). + +### Extraction Strategy + +The merger script will import the `TOOLKIT_CATALOGUE` from the design system dependency. For each toolkit found in the Engine API response, it will look up the corresponding metadata using the toolkit ID (normalizing case if necessary) and merge the fields into the final JSON structure under the `metadata` key. + +## One time script for extract custom sections from already existing docs + +We need to preserve valuable, manually written context from the current documentation while migrating to the automated system. This will be achieved by a one-time extraction script. + +### Requirements + +- **Input**: All MDX files in `app/en/resources/integrations/**/*.mdx`. +- **Output**: A JSON file (`custom_sections.json`) mapping toolkit IDs to their custom content. +- **Logic**: The script must parse MDX files and identify "Custom" vs "Generated" sections based on known patterns (e.g., `` is generated, but a specific `` might be custom). + +### Extraction Targets (Custom Sections) + +The script should specifically look for and extract: + +1. **Custom Imports**: Any imports that are not part of the standard set (e.g., `import ScopePicker...`). +2. **Toolkit-Level Callouts**: Warnings or info boxes appearing after the header but before the tools list (e.g., Jira's "Handling multiple Atlassian Clouds"). +3. **Extended Auth/Reference**: Text that appears after the standard `## Auth` or `## Reference` headers but isn't just the standard link or enum list. +4. **Per-Tool Callouts**: Info boxes inside a specific tool's section (e.g., Gmail's "This tool requires the following scope"). +5. **Sub-pages**: Detect if a toolkit has sub-directories like `environment-variables/` and note them for preservation. + +### Output Structure + +The extracted data will follow the structure defined in the "Documentation Content" section of the main JSON spec, enabling easy merging later. + +## Using and updating the documentation generator to build tool json files + +At the moment the make docs commands requires access to some python tools source code, to create/update the documentation in the doc repo using MDX files. This process should be updated to use the generated tool data json, instead of MDX files. + +The generated docs beside generating structured output based using a defined code logic it also uses LLMs to generate some sections that serves as summary and of the toolkit and its capabilities. This feature will probably be kept for helping the addition of new toolkits but a further analyses on how that would be dealt with in the updated process should be tested, as having it updating sections automatically without human supervision can cause some issue. + +One important change that will be added is to kept it easy to change the source of the json with tools data. In this project the json files will be used to generated the docs, but in the future when the addition of metadata to tools definitions we might have all that is required from an engine endpoint. + +### Current Flow + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ USER RUNS SCRIPT │ +│ python -m make_toolkit_docs │ +└─────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────┐ +│ 1. DISCOVER TOOLKITS │ +│ File: discovery.py │ +│ • Scans home directory for "toolkits" folders │ +│ • Looks for Python files with @tool decorator │ +└─────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────┐ +│ 2. USER SELECTS TOOLKIT │ +│ File: __main__.py │ +│ • Shows fuzzy-searchable list │ +│ • User picks one toolkit to document │ +└─────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────┐ +│ 3. INSTALL TOOLKIT (Dependency!) │ +│ File: __main__.py │ +│ • Runs: uv pip install -e {toolkit_dir} │ +│ • Makes toolkit importable in Python │ +└─────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────┐ +│ 4. LOAD TOOLS (Dependency!) │ +│ File: utils.py → get_list_of_tools() │ +│ • Uses arcade_core to discover tools in runtime │ +│ • Returns list of ToolDefinition objects │ +└─────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────┐ +│ 5. BUILD MDX │ +│ File: docs_builder.py → build_toolkit_mdx() │ +│ • Creates header, TOC, tool specs, footer │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +### Proposed Flow (Decoupled) + +``` +┌─────────────────────────────────────────────────────────────────────┐ +│ USER RUNS SCRIPT │ +│ python -m make_toolkit_docs │ +└─────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────┐ +│ 1. FETCH DATA (New Source) │ +│ • Reads Merged JSON (API + Metadata + Docs) │ +│ • NO local Python installation required │ +│ • NO runtime tool loading │ +└─────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────┐ +│ 2. PARSE DATA │ +│ • Deserializes JSON into Tool Definition Objects │ +│ • Extracts scopes, metadata, and custom sections │ +└─────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────┐ +│ 3. BUILD MDX │ +│ File: docs_builder.py → build_toolkit_mdx() │ +│ • Uses templates to render content │ +│ • Injects custom sections at defined points │ +│ • Adds new "Scopes" section per tool │ +└─────────────────────────────────────────────────────────────────────┘ +``` + +## Tool usage code examples generation at run time + +The make docs command right now adds a code snippet example for each tool, a js and a python. Most of that code is exact the same for all tools, so instead of creating 2 for each, templates will be used, where only will be required adding the tool call. This approach can simplify updates for all code examples, as they will be rendered from a single source. +Other point that is important is to garanteen that the generate code is syntaxally correct, so a costumer will be able to copy and paste it and run it without compile time errrors. Even if the code is syntactaclly correct, it maybe still be not functional, as some inputs require soem previous knowledge of format or data from the provider, so we wont garanteen that the code will produce a proper result only that inputs follow the right sort of input type as requested in the tool definition. + +## CI/CD Pipeline to Build and Deploy Documentation + +Documentation updates are currently manual or done locally via the `make docs` command. To automate this, we will add a CI/CD pipeline step to the existing toolkit publishing workflow. + +This workflow should trigger when new toolkit versions are released to staging. Since a single release can involve updates to **multiple toolkits simultaneously**, the pipeline must be capable of iterating through a list of changed toolkits and updating the documentation for all of them in a single batch. + +### Key Requirements + +1. **Trigger**: New release (staging) event, carrying a payload of one or more updated toolkit names/versions. +2. **Versioning Check**: For each toolkit in the payload, verify if the version has actually changed to avoid redundant updates. +3. **Data Gathering**: Loop through all changed toolkits: + - Fetch latest tool definitions from Engine API. + - Fetch metadata from Design System. +4. **Generation**: + - Update the Merged JSON file for each toolkit. +5. **PR Creation**: Create a **single Pull Request** containing updates for all affected toolkits to minimize noise. +6. **Approval**: Manual approval required to merge and publish. + +### Pipeline Workflow + +``` +┌────────────────────────────────────────────────────────────┐ +│ RELEASE EVENT (STAGING) │ +│ Payload: [Toolkit A, Toolkit B, ...] │ +└─────────────────────────────┬──────────────────────────────┘ + │ + ▼ +┌─────────────────────────────┴──────────────────────────────┐ +│ ITERATE THROUGH CHANGED TOOLKITS │ +│ │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ 1. CHECK VERSION │ │ +│ │ • Has Toolkit X version changed? │ │ +│ │ • IF NO: Skip to next │ │ +│ │ • IF YES: Add to update list │ │ +│ └──────────────────────────┬───────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ 2. GATHER & MERGE DATA │ │ +│ │ • Fetch Tool Definitions (Engine API) │ │ +│ │ • Fetch Metadata (Design System) │ │ +│ │ • Write/Update {toolkit_id}.json │ │ +│ └──────────────────────────┬───────────────────────────┘ │ +│ │ │ +│ ▼ │ +│ ┌──────────────────────────────────────────────────────┐ │ +│ │ 3. GENERATE DOCUMENTATION │ │ +│ │ • Run make_toolkit_docs --from-json │ │ +│ └──────────────────────────────────────────────────────┘ │ +│ │ +└─────────────────────────────┬──────────────────────────────┘ + │ + ▼ +┌────────────────────────────────────────────────────────────┐ +│ 4. CREATE PULL REQUEST │ +│ • Commit changes for ALL updated toolkits │ +│ • Open single PR in docs repo │ +└─────────────────────────────┬──────────────────────────────┘ + │ + ▼ +┌────────────────────────────────────────────────────────────┐ +│ 5. MANUAL APPROVAL │ +│ • Human reviews changes │ +│ • Merge PR -> Deploys to Production │ +└────────────────────────────────────────────────────────────┘ +``` + +## Future changes + +After a call with Sergio and Eric we discussed about the current project Eric is working on, where he will enable the addition of metadata to the tools, like tags and categories. Right now in the scope of his work are only already defined fields to be added, a dynamic payload was suggested, as it would allow enriching the tool definition with the data that is stored in design system thus all data that one would need for the docs would have a single source, that would be the Engine API. This would turn all the work related to the processing and merging of tool date from engine + design system unnecessary. +As currently there is not planning on when this feature will be added to the engine, what we can do is making the code modular enough so the data source can be easily swapped. +Besides the centralized source of tools other info would be available that can be added already in the docs when the first iteration of the feature is ready. In case of Eric merging it before this project is finished it probably is worth it to add the support for the metadata he is currently adding even if it would still require both sources for tools definitions. + +# Share Planning + +For the sharing plan the documentation itself is a manner of sharing the required knowledge to understand our tools with the community. Besides that we can add in the weekly newsletter so we can inform consumers that now our docs are more reliable. + +# Measurement Planning + +For measurement plan, we can use metrics from the access of tools documentation page and also try to collect user's feedback on how they feel about the documentation after some time of the release. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b5c31436f..b91de1eb9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -56,6 +56,9 @@ importers: react-hook-form: specifier: 7.65.0 version: 7.65.0(react@19.2.3) + react-markdown: + specifier: ^10.1.0 + version: 10.1.0(@types/react@19.2.7)(react@19.2.3) react-syntax-highlighter: specifier: 16.1.0 version: 16.1.0(react@19.2.3) @@ -2922,6 +2925,9 @@ packages: highlightjs-vue@1.0.0: resolution: {integrity: sha512-PDEfEF102G23vHmPhLyPboFCD+BkMGu+GuJe2d9/eH4FsCwvgBpnc9n0pGE+ffKdph38s6foEZiEjdgHdzp+IA==} + html-url-attributes@3.0.1: + resolution: {integrity: sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==} + html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} @@ -3864,6 +3870,12 @@ packages: react-is@16.13.1: resolution: {integrity: sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==} + react-markdown@10.1.0: + resolution: {integrity: sha512-qKxVopLT/TyA6BX3Ue5NwabOsAzm0Q7kAPwq6L+wWDwisYs7R8vZ0nRXqq6rkueboxpkjvLGU9fWifiX/ZZFxQ==} + peerDependencies: + '@types/react': '>=18' + react: '>=18' + react-medium-image-zoom@5.4.0: resolution: {integrity: sha512-BsE+EnFVQzFIlyuuQrZ9iTwyKpKkqdFZV1ImEQN573QPqGrIUuNni7aF+sZwDcxlsuOMayCr6oO/PZR/yJnbRg==} peerDependencies: @@ -7753,6 +7765,8 @@ snapshots: highlightjs-vue@1.0.0: {} + html-url-attributes@3.0.1: {} + html-void-elements@3.0.0: {} human-signals@5.0.0: {} @@ -8976,6 +8990,24 @@ snapshots: react-is@16.13.1: {} + react-markdown@10.1.0(@types/react@19.2.7)(react@19.2.3): + dependencies: + '@types/hast': 3.0.4 + '@types/mdast': 4.0.4 + '@types/react': 19.2.7 + devlop: 1.1.0 + hast-util-to-jsx-runtime: 2.3.6 + html-url-attributes: 3.0.1 + mdast-util-to-hast: 13.2.1 + react: 19.2.3 + remark-parse: 11.0.0 + remark-rehype: 11.1.2 + unified: 11.0.5 + unist-util-visit: 5.0.0 + vfile: 6.0.3 + transitivePeerDependencies: + - supports-color + react-medium-image-zoom@5.4.0(react-dom@19.2.3(react@19.2.3))(react@19.2.3): dependencies: react: 19.2.3 diff --git a/toolkit-docs-generator/.gitignore b/toolkit-docs-generator/.gitignore new file mode 100644 index 000000000..38c0afc46 --- /dev/null +++ b/toolkit-docs-generator/.gitignore @@ -0,0 +1,27 @@ +# Dependencies +node_modules/ + +# Build output +dist/ + +# Output directory +output/ + +# IDE +.idea/ +.vscode/ + +# OS files +.DS_Store +Thumbs.db + +# Test coverage +coverage/ + +# Logs +*.log +npm-debug.log* + +# Environment +.env +.env.local diff --git a/toolkit-docs-generator/README.md b/toolkit-docs-generator/README.md new file mode 100644 index 000000000..8f51bbf8a --- /dev/null +++ b/toolkit-docs-generator/README.md @@ -0,0 +1,160 @@ +# Toolkit Docs Generator + +A Node.js CLI tool for generating Arcade toolkit documentation JSON from multiple data sources. + +## Overview + +This tool combines data from: +- **Engine API**: Tool definitions, parameters, auth requirements, scopes +- **Design System**: Toolkit metadata (category, icons, flags) +- **Custom Sections**: Extracted documentation content from existing MDX files + +## Quick start + +```bash +# Install dependencies +pnpm install + +# Build the CLI +pnpm build + +# Generate docs for a toolkit (LLM required for examples) +pnpm start generate \ + --providers "Github:1.0.0" \ + --mock-data-dir ./mock-data \ + --llm-provider openai \ + --llm-model gpt-4.1-mini \ + --llm-api-key "$OPENAI_API_KEY" \ + -o ./output + +# Or generate for all toolkits +pnpm start generate-all \ + --mock-data-dir ./mock-data \ + --llm-provider anthropic \ + --llm-model claude-3-5-sonnet-latest \ + --llm-api-key "$ANTHROPIC_API_KEY" \ + -o ./output +``` + +## Configuration + +### Environment variables + +```bash +# Required for LLM-based examples +LLM_PROVIDER=openai +LLM_MODEL=gpt-4.1-mini +OPENAI_API_KEY=your-openai-key + +# Or use Anthropic +LLM_PROVIDER=anthropic +LLM_MODEL=claude-3-5-sonnet-latest +ANTHROPIC_API_KEY=your-anthropic-key +``` + +### CLI options + +```bash +Usage: toolkit-docs-generator [command] [options] + +Commands: + generate Generate documentation for selected toolkits + generate-all Generate documentation for all toolkits + validate Validate a generated JSON file + list-toolkits List available toolkits from mock data + +Options: + -o, --output Output directory (default: ./output) + --mock-data-dir Path to mock data directory + --llm-provider LLM provider (openai|anthropic) + --llm-model LLM model to use + --llm-api-key LLM API key + --llm-base-url LLM base URL + --llm-temperature LLM temperature + --llm-max-tokens LLM max tokens + --llm-system-prompt LLM system prompt override + --custom-sections Path to custom sections JSON + --verbose Enable verbose logging +``` + +## Development + +```bash +# Run in watch mode +pnpm dev + +# Run tests +pnpm test + +# Type check +pnpm typecheck + +# Lint +pnpm lint +``` + +## Architecture + +See [PLANNING.md](./PLANNING.md) for detailed architecture and implementation tickets. + +``` +toolkit-docs-generator/ +├── src/ +│ ├── cli/ # CLI commands and interface +│ ├── sources/ # Data source implementations +│ ├── merger/ # Data merging logic +│ ├── llm/ # LLM service for examples/summaries +│ ├── generator/ # JSON output generation +│ ├── types/ # TypeScript type definitions +│ └── utils/ # Shared utilities +├── tests/ # Test files +├── package.json +├── tsconfig.json +└── README.md +``` + +## Data source abstraction + +The tool uses a single toolkit data source interface, making it easy to swap implementations: + +```typescript +// Use mock data (current) +const toolkitDataSource = createMockToolkitDataSource({ dataDir: "./mock-data" }); +``` + +## Output format + +Generated JSON follows this structure: + +```json +{ + "id": "Github", + "label": "GitHub", + "version": "1.0.0", + "description": "...", + "metadata": { + "category": "development", + "iconUrl": "...", + "isBYOC": false, + "isPro": false + }, + "auth": { + "type": "oauth2", + "providerId": "github", + "allScopes": ["repo", "user:email"] + }, + "tools": [ + { + "name": "CreateIssue", + "description": "...", + "parameters": [...], + "auth": { "scopes": ["repo"] }, + "codeExample": { "toolName": "Github.CreateIssue", "parameters": {} } + } + ] +} +``` + +## License + +MIT diff --git a/toolkit-docs-generator/biome.jsonc b/toolkit-docs-generator/biome.jsonc new file mode 100644 index 000000000..fbf703846 --- /dev/null +++ b/toolkit-docs-generator/biome.jsonc @@ -0,0 +1,27 @@ +{ + "$schema": "./node_modules/@biomejs/biome/configuration_schema.json", + "extends": ["ultracite/biome/core"], + "linter": { + "rules": { + "performance": { + "noBarrelFile": "off", + "noAccumulatingSpread": "off", + "useTopLevelRegex": "off" + }, + "suspicious": { + "useGuardForIn": "off", + "useAwait": "off", + "noExplicitAny": "warn" + }, + "style": { + "noParameterProperties": "off", + "useConsistentMemberAccessibility": "off", + "useBlockStatements": "off", + "useNodejsImportProtocol": "off" + }, + "complexity": { + "noExcessiveCognitiveComplexity": "warn" + } + } + } +} diff --git a/toolkit-docs-generator/mock-data/engine-api-response.json b/toolkit-docs-generator/mock-data/engine-api-response.json new file mode 100644 index 000000000..8161a95dc --- /dev/null +++ b/toolkit-docs-generator/mock-data/engine-api-response.json @@ -0,0 +1,335 @@ +{ + "items": [ + { + "fully_qualified_name": "Github.CreateIssue@1.0.0", + "qualified_name": "Github.CreateIssue", + "name": "CreateIssue", + "description": "Create a new issue in a GitHub repository. Optionally add the issue to a project by specifying the project number.", + "toolkit": { + "name": "Github", + "version": "1.0.0", + "description": "GitHub repository and collaboration tools" + }, + "input": { + "parameters": [ + { + "name": "owner", + "required": true, + "description": "The owner (user or organization) of the repository", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "repo", + "required": true, + "description": "The name of the repository", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "title", + "required": true, + "description": "The title of the issue", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "body", + "required": false, + "description": "The body content of the issue (supports Markdown)", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "labels", + "required": false, + "description": "Labels to add to the issue", + "value_schema": { + "val_type": "array", + "inner_val_type": "string", + "enum": null + }, + "inferrable": true + } + ] + }, + "output": { + "available_modes": ["value"], + "description": "The created issue object with id, number, url, and other details", + "value_schema": { + "val_type": "object", + "inner_val_type": null, + "enum": null + } + }, + "requirements": { + "authorization": { + "id": "github_oauth", + "provider_id": "github", + "provider_type": "oauth2", + "scopes": ["repo"] + }, + "secrets": null + } + }, + { + "fully_qualified_name": "Github.SetStarred@1.0.0", + "qualified_name": "Github.SetStarred", + "name": "SetStarred", + "description": "Star or unstar a GitHub repository for the authenticated user.", + "toolkit": { + "name": "Github", + "version": "1.0.0", + "description": "GitHub repository and collaboration tools" + }, + "input": { + "parameters": [ + { + "name": "owner", + "required": true, + "description": "The owner of the repository to star/unstar", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "repo", + "required": true, + "description": "The name of the repository to star/unstar", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "starred", + "required": true, + "description": "True to star the repository, false to unstar it", + "value_schema": { + "val_type": "boolean", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + } + ] + }, + "output": { + "available_modes": ["value"], + "description": "Confirmation of the star/unstar action", + "value_schema": { + "val_type": "object", + "inner_val_type": null, + "enum": null + } + }, + "requirements": { + "authorization": { + "id": "github_oauth", + "provider_id": "github", + "provider_type": "oauth2", + "scopes": ["public_repo"] + }, + "secrets": null + } + }, + { + "fully_qualified_name": "Github.ListPullRequests@1.0.0", + "qualified_name": "Github.ListPullRequests", + "name": "ListPullRequests", + "description": "List pull requests in a GitHub repository with optional filtering.", + "toolkit": { + "name": "Github", + "version": "1.0.0", + "description": "GitHub repository and collaboration tools" + }, + "input": { + "parameters": [ + { + "name": "owner", + "required": true, + "description": "The owner of the repository", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "repo", + "required": true, + "description": "The name of the repository", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "state", + "required": false, + "description": "Filter by pull request state", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": ["open", "closed", "all"] + }, + "inferrable": true + }, + { + "name": "per_page", + "required": false, + "description": "Number of results per page (max 100)", + "value_schema": { + "val_type": "integer", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + } + ] + }, + "output": { + "available_modes": ["value"], + "description": "List of pull request objects", + "value_schema": { + "val_type": "array", + "inner_val_type": null, + "enum": null + } + }, + "requirements": { + "authorization": { + "id": "github_oauth", + "provider_id": "github", + "provider_type": "oauth2", + "scopes": ["repo"] + }, + "secrets": null + } + }, + { + "fully_qualified_name": "Slack.SendMessage@1.2.0", + "qualified_name": "Slack.SendMessage", + "name": "SendMessage", + "description": "Send a message to a Slack channel or direct message.", + "toolkit": { + "name": "Slack", + "version": "1.2.0", + "description": "Slack communication tools" + }, + "input": { + "parameters": [ + { + "name": "channel_id", + "required": true, + "description": "The ID of the channel or conversation", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "text", + "required": true, + "description": "The message text", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + } + ] + }, + "output": { + "available_modes": ["value"], + "description": "Message send confirmation", + "value_schema": { + "val_type": "object", + "inner_val_type": null, + "enum": null + } + }, + "requirements": { + "authorization": { + "id": "slack_oauth", + "provider_id": "slack", + "provider_type": "oauth2", + "scopes": ["chat:write"] + }, + "secrets": null + } + }, + { + "fully_qualified_name": "Slack.ListChannels@1.2.0", + "qualified_name": "Slack.ListChannels", + "name": "ListChannels", + "description": "List channels in the workspace.", + "toolkit": { + "name": "Slack", + "version": "1.2.0", + "description": "Slack communication tools" + }, + "input": { + "parameters": [ + { + "name": "limit", + "required": false, + "description": "Maximum number of channels to return", + "value_schema": { + "val_type": "integer", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + } + ] + }, + "output": { + "available_modes": ["value"], + "description": "List of channel objects", + "value_schema": { + "val_type": "array", + "inner_val_type": null, + "enum": null + } + }, + "requirements": { + "authorization": { + "id": "slack_oauth", + "provider_id": "slack", + "provider_type": "oauth2", + "scopes": ["channels:read"] + }, + "secrets": null + } + } + ], + "total_count": 5 +} diff --git a/toolkit-docs-generator/mock-data/metadata.json b/toolkit-docs-generator/mock-data/metadata.json new file mode 100644 index 000000000..ad2301af4 --- /dev/null +++ b/toolkit-docs-generator/mock-data/metadata.json @@ -0,0 +1,62 @@ +{ + "Github": { + "id": "Github", + "label": "GitHub", + "category": "development", + "iconUrl": "https://design-system.arcade.dev/icons/github.svg", + "isBYOC": false, + "isPro": false, + "type": "arcade", + "docsLink": "https://docs.arcade.dev/en/mcp-servers/development/github", + "isComingSoon": false, + "isHidden": false + }, + "Slack": { + "id": "Slack", + "label": "Slack", + "category": "social", + "iconUrl": "https://design-system.arcade.dev/icons/slack.svg", + "isBYOC": false, + "isPro": false, + "type": "arcade", + "docsLink": "https://docs.arcade.dev/en/mcp-servers/social/slack", + "isComingSoon": false, + "isHidden": false + }, + "Gmail": { + "id": "Gmail", + "label": "Gmail", + "category": "productivity", + "iconUrl": "https://design-system.arcade.dev/icons/gmail.svg", + "isBYOC": false, + "isPro": false, + "type": "arcade", + "docsLink": "https://docs.arcade.dev/en/mcp-servers/productivity/gmail", + "isComingSoon": false, + "isHidden": false + }, + "Jira": { + "id": "Jira", + "label": "Jira", + "category": "development", + "iconUrl": "https://design-system.arcade.dev/icons/jira.svg", + "isBYOC": true, + "isPro": false, + "type": "arcade", + "docsLink": "https://docs.arcade.dev/en/mcp-servers/development/jira", + "isComingSoon": false, + "isHidden": false + }, + "Stripe": { + "id": "Stripe", + "label": "Stripe", + "category": "payments", + "iconUrl": "https://design-system.arcade.dev/icons/stripe.svg", + "isBYOC": false, + "isPro": true, + "type": "arcade", + "docsLink": "https://docs.arcade.dev/en/mcp-servers/payments/stripe", + "isComingSoon": false, + "isHidden": false + } +} diff --git a/toolkit-docs-generator/package.json b/toolkit-docs-generator/package.json new file mode 100644 index 000000000..f97dcfe83 --- /dev/null +++ b/toolkit-docs-generator/package.json @@ -0,0 +1,52 @@ +{ + "name": "toolkit-docs-generator", + "version": "1.0.0", + "description": "CLI tool for generating Arcade toolkit documentation JSON from multiple data sources", + "type": "module", + "main": "dist/index.js", + "bin": { + "toolkit-docs-generator": "dist/cli/index.js" + }, + "scripts": { + "build": "tsup src/cli/index.ts src/index.ts --format esm --dts --clean", + "dev": "tsup src/cli/index.ts src/index.ts --format esm --watch", + "start": "node dist/cli/index.js", + "lint": "pnpm dlx ultracite check", + "lint:fix": "pnpm dlx ultracite fix", + "typecheck": "tsc --noEmit", + "test": "vitest --run", + "test:watch": "vitest --watch", + "test:coverage": "vitest --coverage", + "check": "pnpm lint && pnpm typecheck && pnpm test" + }, + "keywords": [ + "arcade", + "documentation", + "toolkit", + "generator", + "cli" + ], + "author": "Arcade", + "license": "MIT", + "engines": { + "node": "22.x", + "pnpm": ">=9.15.4" + }, + "dependencies": { + "@anthropic-ai/sdk": "0.71.2", + "chalk": "^5.4.0", + "commander": "^14.0.2", + "openai": "^6.7.0", + "ora": "^9.0.0", + "zod": "^4.1.12" + }, + "devDependencies": { + "@biomejs/biome": "2.3.11", + "@types/node": "^24.9.2", + "tsup": "^8.3.0", + "typescript": "^5.9.3", + "ultracite": "^7.0.11", + "vitest": "^4.0.5" + }, + "packageManager": "pnpm@10.11.0" +} diff --git a/toolkit-docs-generator/pnpm-lock.yaml b/toolkit-docs-generator/pnpm-lock.yaml new file mode 100644 index 000000000..2635a519f --- /dev/null +++ b/toolkit-docs-generator/pnpm-lock.yaml @@ -0,0 +1,1721 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + '@anthropic-ai/sdk': + specifier: 0.71.2 + version: 0.71.2(zod@4.3.5) + chalk: + specifier: ^5.4.0 + version: 5.6.2 + commander: + specifier: ^14.0.2 + version: 14.0.2 + openai: + specifier: ^6.7.0 + version: 6.16.0(zod@4.3.5) + ora: + specifier: ^9.0.0 + version: 9.0.0 + zod: + specifier: ^4.1.12 + version: 4.3.5 + devDependencies: + '@biomejs/biome': + specifier: 2.3.11 + version: 2.3.11 + '@types/node': + specifier: ^24.9.2 + version: 24.10.8 + tsup: + specifier: ^8.3.0 + version: 8.5.1(postcss@8.5.6)(typescript@5.9.3) + typescript: + specifier: ^5.9.3 + version: 5.9.3 + ultracite: + specifier: ^7.0.11 + version: 7.0.11(typescript@5.9.3) + vitest: + specifier: ^4.0.5 + version: 4.0.17(@types/node@24.10.8) + +packages: + + '@anthropic-ai/sdk@0.71.2': + resolution: {integrity: sha512-TGNDEUuEstk/DKu0/TflXAEt+p+p/WhTlFzEnoosvbaDU2LTjm42igSdlL0VijrKpWejtOKxX0b8A7uc+XiSAQ==} + hasBin: true + peerDependencies: + zod: ^3.25.0 || ^4.0.0 + peerDependenciesMeta: + zod: + optional: true + + '@babel/runtime@7.28.6': + resolution: {integrity: sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==} + engines: {node: '>=6.9.0'} + + '@biomejs/biome@2.3.11': + resolution: {integrity: sha512-/zt+6qazBWguPG6+eWmiELqO+9jRsMZ/DBU3lfuU2ngtIQYzymocHhKiZRyrbra4aCOoyTg/BmY+6WH5mv9xmQ==} + engines: {node: '>=14.21.3'} + hasBin: true + + '@biomejs/cli-darwin-arm64@2.3.11': + resolution: {integrity: sha512-/uXXkBcPKVQY7rc9Ys2CrlirBJYbpESEDme7RKiBD6MmqR2w3j0+ZZXRIL2xiaNPsIMMNhP1YnA+jRRxoOAFrA==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [darwin] + + '@biomejs/cli-darwin-x64@2.3.11': + resolution: {integrity: sha512-fh7nnvbweDPm2xEmFjfmq7zSUiox88plgdHF9OIW4i99WnXrAC3o2P3ag9judoUMv8FCSUnlwJCM1B64nO5Fbg==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [darwin] + + '@biomejs/cli-linux-arm64-musl@2.3.11': + resolution: {integrity: sha512-XPSQ+XIPZMLaZ6zveQdwNjbX+QdROEd1zPgMwD47zvHV+tCGB88VH+aynyGxAHdzL+Tm/+DtKST5SECs4iwCLg==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [linux] + + '@biomejs/cli-linux-arm64@2.3.11': + resolution: {integrity: sha512-l4xkGa9E7Uc0/05qU2lMYfN1H+fzzkHgaJoy98wO+b/7Gl78srbCRRgwYSW+BTLixTBrM6Ede5NSBwt7rd/i6g==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [linux] + + '@biomejs/cli-linux-x64-musl@2.3.11': + resolution: {integrity: sha512-vU7a8wLs5C9yJ4CB8a44r12aXYb8yYgBn+WeyzbMjaCMklzCv1oXr8x+VEyWodgJt9bDmhiaW/I0RHbn7rsNmw==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [linux] + + '@biomejs/cli-linux-x64@2.3.11': + resolution: {integrity: sha512-/1s9V/H3cSe0r0Mv/Z8JryF5x9ywRxywomqZVLHAoa/uN0eY7F8gEngWKNS5vbbN/BsfpCG5yeBT5ENh50Frxg==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [linux] + + '@biomejs/cli-win32-arm64@2.3.11': + resolution: {integrity: sha512-PZQ6ElCOnkYapSsysiTy0+fYX+agXPlWugh6+eQ6uPKI3vKAqNp6TnMhoM3oY2NltSB89hz59o8xIfOdyhi9Iw==} + engines: {node: '>=14.21.3'} + cpu: [arm64] + os: [win32] + + '@biomejs/cli-win32-x64@2.3.11': + resolution: {integrity: sha512-43VrG813EW+b5+YbDbz31uUsheX+qFKCpXeY9kfdAx+ww3naKxeVkTD9zLIWxUPfJquANMHrmW3wbe/037G0Qg==} + engines: {node: '>=14.21.3'} + cpu: [x64] + os: [win32] + + '@clack/core@0.5.0': + resolution: {integrity: sha512-p3y0FIOwaYRUPRcMO7+dlmLh8PSRcrjuTndsiA0WAFbWES0mLZlrjVoBRZ9DzkPFJZG6KGkJmoEAY0ZcVWTkow==} + + '@clack/prompts@0.11.0': + resolution: {integrity: sha512-pMN5FcrEw9hUkZA4f+zLlzivQSeQf5dRGJjSUbvVYDLvpKCdQx5OaknvKzgbtXOizhP+SJJJjqEbOe55uKKfAw==} + + '@esbuild/aix-ppc64@0.27.2': + resolution: {integrity: sha512-GZMB+a0mOMZs4MpDbj8RJp4cw+w1WV5NYD6xzgvzUJ5Ek2jerwfO2eADyI6ExDSUED+1X8aMbegahsJi+8mgpw==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.27.2': + resolution: {integrity: sha512-pvz8ZZ7ot/RBphf8fv60ljmaoydPU12VuXHImtAs0XhLLw+EXBi2BLe3OYSBslR4rryHvweW5gmkKFwTiFy6KA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.27.2': + resolution: {integrity: sha512-DVNI8jlPa7Ujbr1yjU2PfUSRtAUZPG9I1RwW4F4xFB1Imiu2on0ADiI/c3td+KmDtVKNbi+nffGDQMfcIMkwIA==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.27.2': + resolution: {integrity: sha512-z8Ank4Byh4TJJOh4wpz8g2vDy75zFL0TlZlkUkEwYXuPSgX8yzep596n6mT7905kA9uHZsf/o2OJZubl2l3M7A==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.27.2': + resolution: {integrity: sha512-davCD2Zc80nzDVRwXTcQP/28fiJbcOwvdolL0sOiOsbwBa72kegmVU0Wrh1MYrbuCL98Omp5dVhQFWRKR2ZAlg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.27.2': + resolution: {integrity: sha512-ZxtijOmlQCBWGwbVmwOF/UCzuGIbUkqB1faQRf5akQmxRJ1ujusWsb3CVfk/9iZKr2L5SMU5wPBi1UWbvL+VQA==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.27.2': + resolution: {integrity: sha512-lS/9CN+rgqQ9czogxlMcBMGd+l8Q3Nj1MFQwBZJyoEKI50XGxwuzznYdwcav6lpOGv5BqaZXqvBSiB/kJ5op+g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.27.2': + resolution: {integrity: sha512-tAfqtNYb4YgPnJlEFu4c212HYjQWSO/w/h/lQaBK7RbwGIkBOuNKQI9tqWzx7Wtp7bTPaGC6MJvWI608P3wXYA==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.27.2': + resolution: {integrity: sha512-hYxN8pr66NsCCiRFkHUAsxylNOcAQaxSSkHMMjcpx0si13t1LHFphxJZUiGwojB1a/Hd5OiPIqDdXONia6bhTw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.27.2': + resolution: {integrity: sha512-vWfq4GaIMP9AIe4yj1ZUW18RDhx6EPQKjwe7n8BbIecFtCQG4CfHGaHuh7fdfq+y3LIA2vGS/o9ZBGVxIDi9hw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.27.2': + resolution: {integrity: sha512-MJt5BRRSScPDwG2hLelYhAAKh9imjHK5+NE/tvnRLbIqUWa+0E9N4WNMjmp/kXXPHZGqPLxggwVhz7QP8CTR8w==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.27.2': + resolution: {integrity: sha512-lugyF1atnAT463aO6KPshVCJK5NgRnU4yb3FUumyVz+cGvZbontBgzeGFO1nF+dPueHD367a2ZXe1NtUkAjOtg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.27.2': + resolution: {integrity: sha512-nlP2I6ArEBewvJ2gjrrkESEZkB5mIoaTswuqNFRv/WYd+ATtUpe9Y09RnJvgvdag7he0OWgEZWhviS1OTOKixw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.27.2': + resolution: {integrity: sha512-C92gnpey7tUQONqg1n6dKVbx3vphKtTHJaNG2Ok9lGwbZil6DrfyecMsp9CrmXGQJmZ7iiVXvvZH6Ml5hL6XdQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.27.2': + resolution: {integrity: sha512-B5BOmojNtUyN8AXlK0QJyvjEZkWwy/FKvakkTDCziX95AowLZKR6aCDhG7LeF7uMCXEJqwa8Bejz5LTPYm8AvA==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.27.2': + resolution: {integrity: sha512-p4bm9+wsPwup5Z8f4EpfN63qNagQ47Ua2znaqGH6bqLlmJ4bx97Y9JdqxgGZ6Y8xVTixUnEkoKSHcpRlDnNr5w==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.27.2': + resolution: {integrity: sha512-uwp2Tip5aPmH+NRUwTcfLb+W32WXjpFejTIOWZFw/v7/KnpCDKG66u4DLcurQpiYTiYwQ9B7KOeMJvLCu/OvbA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.27.2': + resolution: {integrity: sha512-Kj6DiBlwXrPsCRDeRvGAUb/LNrBASrfqAIok+xB0LxK8CHqxZ037viF13ugfsIpePH93mX7xfJp97cyDuTZ3cw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.27.2': + resolution: {integrity: sha512-HwGDZ0VLVBY3Y+Nw0JexZy9o/nUAWq9MlV7cahpaXKW6TOzfVno3y3/M8Ga8u8Yr7GldLOov27xiCnqRZf0tCA==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.27.2': + resolution: {integrity: sha512-DNIHH2BPQ5551A7oSHD0CKbwIA/Ox7+78/AWkbS5QoRzaqlev2uFayfSxq68EkonB+IKjiuxBFoV8ESJy8bOHA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.27.2': + resolution: {integrity: sha512-/it7w9Nb7+0KFIzjalNJVR5bOzA9Vay+yIPLVHfIQYG/j+j9VTH84aNB8ExGKPU4AzfaEvN9/V4HV+F+vo8OEg==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.27.2': + resolution: {integrity: sha512-LRBbCmiU51IXfeXk59csuX/aSaToeG7w48nMwA6049Y4J4+VbWALAuXcs+qcD04rHDuSCSRKdmY63sruDS5qag==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.27.2': + resolution: {integrity: sha512-kMtx1yqJHTmqaqHPAzKCAkDaKsffmXkPHThSfRwZGyuqyIeBvf08KSsYXl+abf5HDAPMJIPnbBfXvP2ZC2TfHg==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.27.2': + resolution: {integrity: sha512-Yaf78O/B3Kkh+nKABUF++bvJv5Ijoy9AN1ww904rOXZFLWVc5OLOfL56W+C8F9xn5JQZa3UX6m+IktJnIb1Jjg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.27.2': + resolution: {integrity: sha512-Iuws0kxo4yusk7sw70Xa2E2imZU5HoixzxfGCdxwBdhiDgt9vX9VUCBhqcwY7/uh//78A1hMkkROMJq9l27oLQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.27.2': + resolution: {integrity: sha512-sRdU18mcKf7F+YgheI/zGf5alZatMUTKj/jNS6l744f9u3WFu4v7twcUI9vu4mknF4Y9aDlblIie0IM+5xxaqQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@isaacs/balanced-match@4.0.1': + resolution: {integrity: sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ==} + engines: {node: 20 || >=22} + + '@isaacs/brace-expansion@5.0.0': + resolution: {integrity: sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA==} + engines: {node: 20 || >=22} + + '@jridgewell/gen-mapping@0.3.13': + resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/sourcemap-codec@1.5.5': + resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + + '@rollup/rollup-android-arm-eabi@4.55.1': + resolution: {integrity: sha512-9R0DM/ykwfGIlNu6+2U09ga0WXeZ9MRC2Ter8jnz8415VbuIykVuc6bhdrbORFZANDmTDvq26mJrEVTl8TdnDg==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.55.1': + resolution: {integrity: sha512-eFZCb1YUqhTysgW3sj/55du5cG57S7UTNtdMjCW7LwVcj3dTTcowCsC8p7uBdzKsZYa8J7IDE8lhMI+HX1vQvg==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.55.1': + resolution: {integrity: sha512-p3grE2PHcQm2e8PSGZdzIhCKbMCw/xi9XvMPErPhwO17vxtvCN5FEA2mSLgmKlCjHGMQTP6phuQTYWUnKewwGg==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.55.1': + resolution: {integrity: sha512-rDUjG25C9qoTm+e02Esi+aqTKSBYwVTaoS1wxcN47/Luqef57Vgp96xNANwt5npq9GDxsH7kXxNkJVEsWEOEaQ==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.55.1': + resolution: {integrity: sha512-+JiU7Jbp5cdxekIgdte0jfcu5oqw4GCKr6i3PJTlXTCU5H5Fvtkpbs4XJHRmWNXF+hKmn4v7ogI5OQPaupJgOg==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.55.1': + resolution: {integrity: sha512-V5xC1tOVWtLLmr3YUk2f6EJK4qksksOYiz/TCsFHu/R+woubcLWdC9nZQmwjOAbmExBIVKsm1/wKmEy4z4u4Bw==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.55.1': + resolution: {integrity: sha512-Rn3n+FUk2J5VWx+ywrG/HGPTD9jXNbicRtTM11e/uorplArnXZYsVifnPPqNNP5BsO3roI4n8332ukpY/zN7rQ==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm-musleabihf@4.55.1': + resolution: {integrity: sha512-grPNWydeKtc1aEdrJDWk4opD7nFtQbMmV7769hiAaYyUKCT1faPRm2av8CX1YJsZ4TLAZcg9gTR1KvEzoLjXkg==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm64-gnu@4.55.1': + resolution: {integrity: sha512-a59mwd1k6x8tXKcUxSyISiquLwB5pX+fJW9TkWU46lCqD/GRDe9uDN31jrMmVP3feI3mhAdvcCClhV8V5MhJFQ==} + cpu: [arm64] + os: [linux] + + '@rollup/rollup-linux-arm64-musl@4.55.1': + resolution: {integrity: sha512-puS1MEgWX5GsHSoiAsF0TYrpomdvkaXm0CofIMG5uVkP6IBV+ZO9xhC5YEN49nsgYo1DuuMquF9+7EDBVYu4uA==} + cpu: [arm64] + os: [linux] + + '@rollup/rollup-linux-loong64-gnu@4.55.1': + resolution: {integrity: sha512-r3Wv40in+lTsULSb6nnoudVbARdOwb2u5fpeoOAZjFLznp6tDU8kd+GTHmJoqZ9lt6/Sys33KdIHUaQihFcu7g==} + cpu: [loong64] + os: [linux] + + '@rollup/rollup-linux-loong64-musl@4.55.1': + resolution: {integrity: sha512-MR8c0+UxAlB22Fq4R+aQSPBayvYa3+9DrwG/i1TKQXFYEaoW3B5b/rkSRIypcZDdWjWnpcvxbNaAJDcSbJU3Lw==} + cpu: [loong64] + os: [linux] + + '@rollup/rollup-linux-ppc64-gnu@4.55.1': + resolution: {integrity: sha512-3KhoECe1BRlSYpMTeVrD4sh2Pw2xgt4jzNSZIIPLFEsnQn9gAnZagW9+VqDqAHgm1Xc77LzJOo2LdigS5qZ+gw==} + cpu: [ppc64] + os: [linux] + + '@rollup/rollup-linux-ppc64-musl@4.55.1': + resolution: {integrity: sha512-ziR1OuZx0vdYZZ30vueNZTg73alF59DicYrPViG0NEgDVN8/Jl87zkAPu4u6VjZST2llgEUjaiNl9JM6HH1Vdw==} + cpu: [ppc64] + os: [linux] + + '@rollup/rollup-linux-riscv64-gnu@4.55.1': + resolution: {integrity: sha512-uW0Y12ih2XJRERZ4jAfKamTyIHVMPQnTZcQjme2HMVDAHY4amf5u414OqNYC+x+LzRdRcnIG1YodLrrtA8xsxw==} + cpu: [riscv64] + os: [linux] + + '@rollup/rollup-linux-riscv64-musl@4.55.1': + resolution: {integrity: sha512-u9yZ0jUkOED1BFrqu3BwMQoixvGHGZ+JhJNkNKY/hyoEgOwlqKb62qu+7UjbPSHYjiVy8kKJHvXKv5coH4wDeg==} + cpu: [riscv64] + os: [linux] + + '@rollup/rollup-linux-s390x-gnu@4.55.1': + resolution: {integrity: sha512-/0PenBCmqM4ZUd0190j7J0UsQ/1nsi735iPRakO8iPciE7BQ495Y6msPzaOmvx0/pn+eJVVlZrNrSh4WSYLxNg==} + cpu: [s390x] + os: [linux] + + '@rollup/rollup-linux-x64-gnu@4.55.1': + resolution: {integrity: sha512-a8G4wiQxQG2BAvo+gU6XrReRRqj+pLS2NGXKm8io19goR+K8lw269eTrPkSdDTALwMmJp4th2Uh0D8J9bEV1vg==} + cpu: [x64] + os: [linux] + + '@rollup/rollup-linux-x64-musl@4.55.1': + resolution: {integrity: sha512-bD+zjpFrMpP/hqkfEcnjXWHMw5BIghGisOKPj+2NaNDuVT+8Ds4mPf3XcPHuat1tz89WRL+1wbcxKY3WSbiT7w==} + cpu: [x64] + os: [linux] + + '@rollup/rollup-openbsd-x64@4.55.1': + resolution: {integrity: sha512-eLXw0dOiqE4QmvikfQ6yjgkg/xDM+MdU9YJuP4ySTibXU0oAvnEWXt7UDJmD4UkYialMfOGFPJnIHSe/kdzPxg==} + cpu: [x64] + os: [openbsd] + + '@rollup/rollup-openharmony-arm64@4.55.1': + resolution: {integrity: sha512-xzm44KgEP11te3S2HCSyYf5zIzWmx3n8HDCc7EE59+lTcswEWNpvMLfd9uJvVX8LCg9QWG67Xt75AuHn4vgsXw==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.55.1': + resolution: {integrity: sha512-yR6Bl3tMC/gBok5cz/Qi0xYnVbIxGx5Fcf/ca0eB6/6JwOY+SRUcJfI0OpeTpPls7f194as62thCt/2BjxYN8g==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.55.1': + resolution: {integrity: sha512-3fZBidchE0eY0oFZBnekYCfg+5wAB0mbpCBuofh5mZuzIU/4jIVkbESmd2dOsFNS78b53CYv3OAtwqkZZmU5nA==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.55.1': + resolution: {integrity: sha512-xGGY5pXj69IxKb4yv/POoocPy/qmEGhimy/FoTpTSVju3FYXUQQMFCaZZXJVidsmGxRioZAwpThl/4zX41gRKg==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.55.1': + resolution: {integrity: sha512-SPEpaL6DX4rmcXtnhdrQYgzQ5W2uW3SCJch88lB2zImhJRhIIK44fkUrgIV/Q8yUNfw5oyZ5vkeQsZLhCb06lw==} + cpu: [x64] + os: [win32] + + '@standard-schema/spec@1.1.0': + resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + + '@trpc/server@11.8.1': + resolution: {integrity: sha512-P4rzZRpEL7zDFgjxK65IdyH0e41FMFfTkQkuq0BA5tKcr7E6v9/v38DEklCpoDN6sPiB1Sigy/PUEzHENhswDA==} + peerDependencies: + typescript: '>=5.7.2' + + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + + '@types/estree@1.0.8': + resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + + '@types/node@24.10.8': + resolution: {integrity: sha512-r0bBaXu5Swb05doFYO2kTWHMovJnNVbCsII0fhesM8bNRlLhXIuckley4a2DaD+vOdmm5G+zGkQZAPZsF80+YQ==} + + '@vitest/expect@4.0.17': + resolution: {integrity: sha512-mEoqP3RqhKlbmUmntNDDCJeTDavDR+fVYkSOw8qRwJFaW/0/5zA9zFeTrHqNtcmwh6j26yMmwx2PqUDPzt5ZAQ==} + + '@vitest/mocker@4.0.17': + resolution: {integrity: sha512-+ZtQhLA3lDh1tI2wxe3yMsGzbp7uuJSWBM1iTIKCbppWTSBN09PUC+L+fyNlQApQoR+Ps8twt2pbSSXg2fQVEQ==} + peerDependencies: + msw: ^2.4.9 + vite: ^6.0.0 || ^7.0.0-0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@4.0.17': + resolution: {integrity: sha512-Ah3VAYmjcEdHg6+MwFE17qyLqBHZ+ni2ScKCiW2XrlSBV4H3Z7vYfPfz7CWQ33gyu76oc0Ai36+kgLU3rfF4nw==} + + '@vitest/runner@4.0.17': + resolution: {integrity: sha512-JmuQyf8aMWoo/LmNFppdpkfRVHJcsgzkbCA+/Bk7VfNH7RE6Ut2qxegeyx2j3ojtJtKIbIGy3h+KxGfYfk28YQ==} + + '@vitest/snapshot@4.0.17': + resolution: {integrity: sha512-npPelD7oyL+YQM2gbIYvlavlMVWUfNNGZPcu0aEUQXt7FXTuqhmgiYupPnAanhKvyP6Srs2pIbWo30K0RbDtRQ==} + + '@vitest/spy@4.0.17': + resolution: {integrity: sha512-I1bQo8QaP6tZlTomQNWKJE6ym4SHf3oLS7ceNjozxxgzavRAgZDc06T7kD8gb9bXKEgcLNt00Z+kZO6KaJ62Ew==} + + '@vitest/utils@4.0.17': + resolution: {integrity: sha512-RG6iy+IzQpa9SB8HAFHJ9Y+pTzI+h8553MrciN9eC6TFBErqrQaTas4vG+MVj8S4uKk8uTT2p0vgZPnTdxd96w==} + + acorn@8.15.0: + resolution: {integrity: sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==} + engines: {node: '>=0.4.0'} + hasBin: true + + ansi-regex@6.2.2: + resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==} + engines: {node: '>=12'} + + any-promise@1.3.0: + resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} + + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + + bundle-require@5.1.0: + resolution: {integrity: sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + peerDependencies: + esbuild: '>=0.18' + + cac@6.7.14: + resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} + engines: {node: '>=8'} + + chai@6.2.2: + resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} + engines: {node: '>=18'} + + chalk@5.6.2: + resolution: {integrity: sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + + chokidar@4.0.3: + resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} + engines: {node: '>= 14.16.0'} + + citty@0.1.6: + resolution: {integrity: sha512-tskPPKEs8D2KPafUypv2gxwJP8h/OaJmC82QQGGDQcHvXX43xF2VDACcJVmZ0EuSxkpO9Kc4MlrA3q0+FG58AQ==} + + cli-cursor@5.0.0: + resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==} + engines: {node: '>=18'} + + cli-spinners@3.4.0: + resolution: {integrity: sha512-bXfOC4QcT1tKXGorxL3wbJm6XJPDqEnij2gQ2m7ESQuE+/z9YFIWnl/5RpTiKWbMq3EVKR4fRLJGn6DVfu0mpw==} + engines: {node: '>=18.20'} + + commander@14.0.2: + resolution: {integrity: sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==} + engines: {node: '>=20'} + + commander@4.1.1: + resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} + engines: {node: '>= 6'} + + confbox@0.1.8: + resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} + + confbox@0.2.2: + resolution: {integrity: sha512-1NB+BKqhtNipMsov4xI/NnhCKp9XG9NamYp5PVm9klAT0fsrNPjaFICsCFhNhwZJKNh7zB/3q8qXz0E9oaMNtQ==} + + consola@3.4.2: + resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} + engines: {node: ^14.18.0 || >=16.10.0} + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + deepmerge@4.3.1: + resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} + engines: {node: '>=0.10.0'} + + es-module-lexer@1.7.0: + resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + + esbuild@0.27.2: + resolution: {integrity: sha512-HyNQImnsOC7X9PMNaCIeAm4ISCQXs5a5YasTXVliKv4uuBo1dKrG0A+uQS8M5eXjVMnLg3WgXaKvprHlFJQffw==} + engines: {node: '>=18'} + hasBin: true + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + + expect-type@1.3.0: + resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} + engines: {node: '>=12.0.0'} + + exsolve@1.0.8: + resolution: {integrity: sha512-LmDxfWXwcTArk8fUEnOfSZpHOJ6zOMUJKOtFLFqJLoKJetuQG874Uc7/Kki7zFLzYybmZhp1M7+98pfMqeX8yA==} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + fix-dts-default-cjs-exports@1.0.1: + resolution: {integrity: sha512-pVIECanWFC61Hzl2+oOCtoJ3F17kglZC/6N94eRWycFgBH35hHx0Li604ZIzhseh97mf2p0cv7vVrOZGoqhlEg==} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + get-east-asian-width@1.4.0: + resolution: {integrity: sha512-QZjmEOC+IT1uk6Rx0sX22V6uHWVwbdbxf1faPqJ1QhLdGgsRGCZoyaQBm/piRdJy/D2um6hM1UP7ZEeQ4EkP+Q==} + engines: {node: '>=18'} + + glob@13.0.0: + resolution: {integrity: sha512-tvZgpqk6fz4BaNZ66ZsRaZnbHvP/jG3uKJvAZOwEVUL4RTA5nJeeLYfyN9/VA8NX/V3IBG+hkeuGpKjvELkVhA==} + engines: {node: 20 || >=22} + + is-interactive@2.0.0: + resolution: {integrity: sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==} + engines: {node: '>=12'} + + is-unicode-supported@2.1.0: + resolution: {integrity: sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==} + engines: {node: '>=18'} + + joycon@3.1.1: + resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} + engines: {node: '>=10'} + + json-schema-to-ts@3.1.1: + resolution: {integrity: sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==} + engines: {node: '>=16'} + + jsonc-parser@3.3.1: + resolution: {integrity: sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==} + + lilconfig@3.1.3: + resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} + engines: {node: '>=14'} + + lines-and-columns@1.2.4: + resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + + load-tsconfig@0.2.5: + resolution: {integrity: sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + log-symbols@7.0.1: + resolution: {integrity: sha512-ja1E3yCr9i/0hmBVaM0bfwDjnGy8I/s6PP4DFp+yP+a+mrHO4Rm7DtmnqROTUkHIkqffC84YY7AeqX6oFk0WFg==} + engines: {node: '>=18'} + + lru-cache@11.2.4: + resolution: {integrity: sha512-B5Y16Jr9LB9dHVkh6ZevG+vAbOsNOYCX+sXvFWFu7B3Iz5mijW3zdbMyhsh8ANd2mSWBYdJgnqi+mL7/LrOPYg==} + engines: {node: 20 || >=22} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + mimic-function@5.0.1: + resolution: {integrity: sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==} + engines: {node: '>=18'} + + minimatch@10.1.1: + resolution: {integrity: sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==} + engines: {node: 20 || >=22} + + minipass@7.1.2: + resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} + engines: {node: '>=16 || 14 >=14.17'} + + mlly@1.8.0: + resolution: {integrity: sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + mz@2.7.0: + resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} + + nanoid@3.3.11: + resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + nypm@0.6.2: + resolution: {integrity: sha512-7eM+hpOtrKrBDCh7Ypu2lJ9Z7PNZBdi/8AT3AX8xoCj43BBVHD0hPSTEvMtkMpfs8FCqBGhxB+uToIQimA111g==} + engines: {node: ^14.16.0 || >=16.10.0} + hasBin: true + + object-assign@4.1.1: + resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} + engines: {node: '>=0.10.0'} + + obug@2.1.1: + resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} + + onetime@7.0.0: + resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==} + engines: {node: '>=18'} + + openai@6.16.0: + resolution: {integrity: sha512-fZ1uBqjFUjXzbGc35fFtYKEOxd20kd9fDpFeqWtsOZWiubY8CZ1NAlXHW3iathaFvqmNtCWMIsosCuyeI7Joxg==} + hasBin: true + peerDependencies: + ws: ^8.18.0 + zod: ^3.25 || ^4.0 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + + ora@9.0.0: + resolution: {integrity: sha512-m0pg2zscbYgWbqRR6ABga5c3sZdEon7bSgjnlXC64kxtxLOyjRcbbUkLj7HFyy/FTD+P2xdBWu8snGhYI0jc4A==} + engines: {node: '>=20'} + + path-scurry@2.0.1: + resolution: {integrity: sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==} + engines: {node: 20 || >=22} + + pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.3: + resolution: {integrity: sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==} + engines: {node: '>=12'} + + pirates@4.0.7: + resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==} + engines: {node: '>= 6'} + + pkg-types@1.3.1: + resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} + + pkg-types@2.3.0: + resolution: {integrity: sha512-SIqCzDRg0s9npO5XQ3tNZioRY1uK06lA41ynBC1YmFTmnY6FjUjVt6s4LoADmwoig1qqD0oK8h1p/8mlMx8Oig==} + + postcss-load-config@6.0.1: + resolution: {integrity: sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==} + engines: {node: '>= 18'} + peerDependencies: + jiti: '>=1.21.0' + postcss: '>=8.0.9' + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + jiti: + optional: true + postcss: + optional: true + tsx: + optional: true + yaml: + optional: true + + postcss@8.5.6: + resolution: {integrity: sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==} + engines: {node: ^10 || ^12 || >=14} + + readdirp@4.1.2: + resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} + engines: {node: '>= 14.18.0'} + + resolve-from@5.0.0: + resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} + engines: {node: '>=8'} + + restore-cursor@5.1.0: + resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} + engines: {node: '>=18'} + + rollup@4.55.1: + resolution: {integrity: sha512-wDv/Ht1BNHB4upNbK74s9usvl7hObDnvVzknxqY/E/O3X6rW1U1rV1aENEfJ54eFZDTNo7zv1f5N4edCluH7+A==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + + sisteransi@1.0.5: + resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + source-map@0.7.6: + resolution: {integrity: sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==} + engines: {node: '>= 12'} + + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + std-env@3.10.0: + resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + + stdin-discarder@0.2.2: + resolution: {integrity: sha512-UhDfHmA92YAlNnCfhmq0VeNL5bDbiZGg7sZ2IvPsXubGkiNa9EC+tUTsjBRsYUAz87btI6/1wf4XoVvQ3uRnmQ==} + engines: {node: '>=18'} + + string-width@8.1.0: + resolution: {integrity: sha512-Kxl3KJGb/gxkaUMOjRsQ8IrXiGW75O4E3RPjFIINOVH8AMl2SQ/yWdTzWwF3FevIX9LcMAjJW+GRwAlAbTSXdg==} + engines: {node: '>=20'} + + strip-ansi@7.1.2: + resolution: {integrity: sha512-gmBGslpoQJtgnMAvOVqGZpEz9dyoKTCzy2nfz/n8aIFhN/jCE/rCmcxabB6jOOHV+0WNnylOxaxBQPSvcWklhA==} + engines: {node: '>=12'} + + sucrase@3.35.1: + resolution: {integrity: sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw==} + engines: {node: '>=16 || 14 >=14.17'} + hasBin: true + + thenify-all@1.6.0: + resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} + engines: {node: '>=0.8'} + + thenify@3.3.1: + resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} + + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + + tinyexec@0.3.2: + resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + + tinyexec@1.0.2: + resolution: {integrity: sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg==} + engines: {node: '>=18'} + + tinyglobby@0.2.15: + resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} + engines: {node: '>=12.0.0'} + + tinyrainbow@3.0.3: + resolution: {integrity: sha512-PSkbLUoxOFRzJYjjxHJt9xro7D+iilgMX/C9lawzVuYiIdcihh9DXmVibBe8lmcFrRi/VzlPjBxbN7rH24q8/Q==} + engines: {node: '>=14.0.0'} + + tree-kill@1.2.2: + resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} + hasBin: true + + trpc-cli@0.12.2: + resolution: {integrity: sha512-kGNCiyOimGlfcZFImbWzFF2Nn3TMnenwUdyuckiN5SEaceJbIac7+Iau3WsVHjQpoNgugFruZMDOKf8GNQNtJw==} + engines: {node: '>=18'} + hasBin: true + peerDependencies: + '@orpc/server': ^1.0.0 + '@trpc/server': ^10.45.2 || ^11.0.1 + '@valibot/to-json-schema': ^1.1.0 + effect: ^3.14.2 || ^4.0.0 + valibot: ^1.1.0 + zod: ^3.24.0 || ^4.0.0 + peerDependenciesMeta: + '@orpc/server': + optional: true + '@trpc/server': + optional: true + '@valibot/to-json-schema': + optional: true + effect: + optional: true + valibot: + optional: true + zod: + optional: true + + ts-algebra@2.0.0: + resolution: {integrity: sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==} + + ts-interface-checker@0.1.13: + resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} + + tsup@8.5.1: + resolution: {integrity: sha512-xtgkqwdhpKWr3tKPmCkvYmS9xnQK3m3XgxZHwSUjvfTjp7YfXe5tT3GgWi0F2N+ZSMsOeWeZFh7ZZFg5iPhing==} + engines: {node: '>=18'} + hasBin: true + peerDependencies: + '@microsoft/api-extractor': ^7.36.0 + '@swc/core': ^1 + postcss: ^8.4.12 + typescript: '>=4.5.0' + peerDependenciesMeta: + '@microsoft/api-extractor': + optional: true + '@swc/core': + optional: true + postcss: + optional: true + typescript: + optional: true + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + ufo@1.6.2: + resolution: {integrity: sha512-heMioaxBcG9+Znsda5Q8sQbWnLJSl98AFDXTO80wELWEzX3hordXsTdxrIfMQoO9IY1MEnoGoPjpoKpMj+Yx0Q==} + + ultracite@7.0.11: + resolution: {integrity: sha512-YeuJTf/Tu12l7K0qvl1G525NfrPUay6UFVt6TrTL8QfXcFZBI+Tzr0Dg14Hxcb964pRQF5PjBaUk3bJCNQHu+g==} + hasBin: true + peerDependencies: + oxlint: ^1.0.0 + peerDependenciesMeta: + oxlint: + optional: true + + undici-types@7.16.0: + resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} + + vite@7.3.1: + resolution: {integrity: sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + peerDependencies: + '@types/node': ^20.19.0 || >=22.12.0 + jiti: '>=1.21.0' + less: ^4.0.0 + lightningcss: ^1.21.0 + sass: ^1.70.0 + sass-embedded: ^1.70.0 + stylus: '>=0.54.8' + sugarss: ^5.0.0 + terser: ^5.16.0 + tsx: ^4.8.1 + yaml: ^2.4.2 + peerDependenciesMeta: + '@types/node': + optional: true + jiti: + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + tsx: + optional: true + yaml: + optional: true + + vitest@4.0.17: + resolution: {integrity: sha512-FQMeF0DJdWY0iOnbv466n/0BudNdKj1l5jYgl5JVTwjSsZSlqyXFt/9+1sEyhR6CLowbZpV7O1sCHrzBhucKKg==} + engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@opentelemetry/api': ^1.9.0 + '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 + '@vitest/browser-playwright': 4.0.17 + '@vitest/browser-preview': 4.0.17 + '@vitest/browser-webdriverio': 4.0.17 + '@vitest/ui': 4.0.17 + happy-dom: '*' + jsdom: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@opentelemetry/api': + optional: true + '@types/node': + optional: true + '@vitest/browser-playwright': + optional: true + '@vitest/browser-preview': + optional: true + '@vitest/browser-webdriverio': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + + yoctocolors@2.1.2: + resolution: {integrity: sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug==} + engines: {node: '>=18'} + + zod@4.3.5: + resolution: {integrity: sha512-k7Nwx6vuWx1IJ9Bjuf4Zt1PEllcwe7cls3VNzm4CQ1/hgtFUK2bRNG3rvnpPUhFjmqJKAKtjV576KnUkHocg/g==} + +snapshots: + + '@anthropic-ai/sdk@0.71.2(zod@4.3.5)': + dependencies: + json-schema-to-ts: 3.1.1 + optionalDependencies: + zod: 4.3.5 + + '@babel/runtime@7.28.6': {} + + '@biomejs/biome@2.3.11': + optionalDependencies: + '@biomejs/cli-darwin-arm64': 2.3.11 + '@biomejs/cli-darwin-x64': 2.3.11 + '@biomejs/cli-linux-arm64': 2.3.11 + '@biomejs/cli-linux-arm64-musl': 2.3.11 + '@biomejs/cli-linux-x64': 2.3.11 + '@biomejs/cli-linux-x64-musl': 2.3.11 + '@biomejs/cli-win32-arm64': 2.3.11 + '@biomejs/cli-win32-x64': 2.3.11 + + '@biomejs/cli-darwin-arm64@2.3.11': + optional: true + + '@biomejs/cli-darwin-x64@2.3.11': + optional: true + + '@biomejs/cli-linux-arm64-musl@2.3.11': + optional: true + + '@biomejs/cli-linux-arm64@2.3.11': + optional: true + + '@biomejs/cli-linux-x64-musl@2.3.11': + optional: true + + '@biomejs/cli-linux-x64@2.3.11': + optional: true + + '@biomejs/cli-win32-arm64@2.3.11': + optional: true + + '@biomejs/cli-win32-x64@2.3.11': + optional: true + + '@clack/core@0.5.0': + dependencies: + picocolors: 1.1.1 + sisteransi: 1.0.5 + + '@clack/prompts@0.11.0': + dependencies: + '@clack/core': 0.5.0 + picocolors: 1.1.1 + sisteransi: 1.0.5 + + '@esbuild/aix-ppc64@0.27.2': + optional: true + + '@esbuild/android-arm64@0.27.2': + optional: true + + '@esbuild/android-arm@0.27.2': + optional: true + + '@esbuild/android-x64@0.27.2': + optional: true + + '@esbuild/darwin-arm64@0.27.2': + optional: true + + '@esbuild/darwin-x64@0.27.2': + optional: true + + '@esbuild/freebsd-arm64@0.27.2': + optional: true + + '@esbuild/freebsd-x64@0.27.2': + optional: true + + '@esbuild/linux-arm64@0.27.2': + optional: true + + '@esbuild/linux-arm@0.27.2': + optional: true + + '@esbuild/linux-ia32@0.27.2': + optional: true + + '@esbuild/linux-loong64@0.27.2': + optional: true + + '@esbuild/linux-mips64el@0.27.2': + optional: true + + '@esbuild/linux-ppc64@0.27.2': + optional: true + + '@esbuild/linux-riscv64@0.27.2': + optional: true + + '@esbuild/linux-s390x@0.27.2': + optional: true + + '@esbuild/linux-x64@0.27.2': + optional: true + + '@esbuild/netbsd-arm64@0.27.2': + optional: true + + '@esbuild/netbsd-x64@0.27.2': + optional: true + + '@esbuild/openbsd-arm64@0.27.2': + optional: true + + '@esbuild/openbsd-x64@0.27.2': + optional: true + + '@esbuild/openharmony-arm64@0.27.2': + optional: true + + '@esbuild/sunos-x64@0.27.2': + optional: true + + '@esbuild/win32-arm64@0.27.2': + optional: true + + '@esbuild/win32-ia32@0.27.2': + optional: true + + '@esbuild/win32-x64@0.27.2': + optional: true + + '@isaacs/balanced-match@4.0.1': {} + + '@isaacs/brace-expansion@5.0.0': + dependencies: + '@isaacs/balanced-match': 4.0.1 + + '@jridgewell/gen-mapping@0.3.13': + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/sourcemap-codec@1.5.5': {} + + '@jridgewell/trace-mapping@0.3.31': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + + '@rollup/rollup-android-arm-eabi@4.55.1': + optional: true + + '@rollup/rollup-android-arm64@4.55.1': + optional: true + + '@rollup/rollup-darwin-arm64@4.55.1': + optional: true + + '@rollup/rollup-darwin-x64@4.55.1': + optional: true + + '@rollup/rollup-freebsd-arm64@4.55.1': + optional: true + + '@rollup/rollup-freebsd-x64@4.55.1': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.55.1': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.55.1': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.55.1': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.55.1': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.55.1': + optional: true + + '@rollup/rollup-linux-loong64-musl@4.55.1': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.55.1': + optional: true + + '@rollup/rollup-linux-ppc64-musl@4.55.1': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.55.1': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.55.1': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.55.1': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.55.1': + optional: true + + '@rollup/rollup-linux-x64-musl@4.55.1': + optional: true + + '@rollup/rollup-openbsd-x64@4.55.1': + optional: true + + '@rollup/rollup-openharmony-arm64@4.55.1': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.55.1': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.55.1': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.55.1': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.55.1': + optional: true + + '@standard-schema/spec@1.1.0': {} + + '@trpc/server@11.8.1(typescript@5.9.3)': + dependencies: + typescript: 5.9.3 + + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + + '@types/deep-eql@4.0.2': {} + + '@types/estree@1.0.8': {} + + '@types/node@24.10.8': + dependencies: + undici-types: 7.16.0 + + '@vitest/expect@4.0.17': + dependencies: + '@standard-schema/spec': 1.1.0 + '@types/chai': 5.2.3 + '@vitest/spy': 4.0.17 + '@vitest/utils': 4.0.17 + chai: 6.2.2 + tinyrainbow: 3.0.3 + + '@vitest/mocker@4.0.17(vite@7.3.1(@types/node@24.10.8))': + dependencies: + '@vitest/spy': 4.0.17 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.3.1(@types/node@24.10.8) + + '@vitest/pretty-format@4.0.17': + dependencies: + tinyrainbow: 3.0.3 + + '@vitest/runner@4.0.17': + dependencies: + '@vitest/utils': 4.0.17 + pathe: 2.0.3 + + '@vitest/snapshot@4.0.17': + dependencies: + '@vitest/pretty-format': 4.0.17 + magic-string: 0.30.21 + pathe: 2.0.3 + + '@vitest/spy@4.0.17': {} + + '@vitest/utils@4.0.17': + dependencies: + '@vitest/pretty-format': 4.0.17 + tinyrainbow: 3.0.3 + + acorn@8.15.0: {} + + ansi-regex@6.2.2: {} + + any-promise@1.3.0: {} + + assertion-error@2.0.1: {} + + bundle-require@5.1.0(esbuild@0.27.2): + dependencies: + esbuild: 0.27.2 + load-tsconfig: 0.2.5 + + cac@6.7.14: {} + + chai@6.2.2: {} + + chalk@5.6.2: {} + + chokidar@4.0.3: + dependencies: + readdirp: 4.1.2 + + citty@0.1.6: + dependencies: + consola: 3.4.2 + + cli-cursor@5.0.0: + dependencies: + restore-cursor: 5.1.0 + + cli-spinners@3.4.0: {} + + commander@14.0.2: {} + + commander@4.1.1: {} + + confbox@0.1.8: {} + + confbox@0.2.2: {} + + consola@3.4.2: {} + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + deepmerge@4.3.1: {} + + es-module-lexer@1.7.0: {} + + esbuild@0.27.2: + optionalDependencies: + '@esbuild/aix-ppc64': 0.27.2 + '@esbuild/android-arm': 0.27.2 + '@esbuild/android-arm64': 0.27.2 + '@esbuild/android-x64': 0.27.2 + '@esbuild/darwin-arm64': 0.27.2 + '@esbuild/darwin-x64': 0.27.2 + '@esbuild/freebsd-arm64': 0.27.2 + '@esbuild/freebsd-x64': 0.27.2 + '@esbuild/linux-arm': 0.27.2 + '@esbuild/linux-arm64': 0.27.2 + '@esbuild/linux-ia32': 0.27.2 + '@esbuild/linux-loong64': 0.27.2 + '@esbuild/linux-mips64el': 0.27.2 + '@esbuild/linux-ppc64': 0.27.2 + '@esbuild/linux-riscv64': 0.27.2 + '@esbuild/linux-s390x': 0.27.2 + '@esbuild/linux-x64': 0.27.2 + '@esbuild/netbsd-arm64': 0.27.2 + '@esbuild/netbsd-x64': 0.27.2 + '@esbuild/openbsd-arm64': 0.27.2 + '@esbuild/openbsd-x64': 0.27.2 + '@esbuild/openharmony-arm64': 0.27.2 + '@esbuild/sunos-x64': 0.27.2 + '@esbuild/win32-arm64': 0.27.2 + '@esbuild/win32-ia32': 0.27.2 + '@esbuild/win32-x64': 0.27.2 + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.8 + + expect-type@1.3.0: {} + + exsolve@1.0.8: {} + + fdir@6.5.0(picomatch@4.0.3): + optionalDependencies: + picomatch: 4.0.3 + + fix-dts-default-cjs-exports@1.0.1: + dependencies: + magic-string: 0.30.21 + mlly: 1.8.0 + rollup: 4.55.1 + + fsevents@2.3.3: + optional: true + + get-east-asian-width@1.4.0: {} + + glob@13.0.0: + dependencies: + minimatch: 10.1.1 + minipass: 7.1.2 + path-scurry: 2.0.1 + + is-interactive@2.0.0: {} + + is-unicode-supported@2.1.0: {} + + joycon@3.1.1: {} + + json-schema-to-ts@3.1.1: + dependencies: + '@babel/runtime': 7.28.6 + ts-algebra: 2.0.0 + + jsonc-parser@3.3.1: {} + + lilconfig@3.1.3: {} + + lines-and-columns@1.2.4: {} + + load-tsconfig@0.2.5: {} + + log-symbols@7.0.1: + dependencies: + is-unicode-supported: 2.1.0 + yoctocolors: 2.1.2 + + lru-cache@11.2.4: {} + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + + mimic-function@5.0.1: {} + + minimatch@10.1.1: + dependencies: + '@isaacs/brace-expansion': 5.0.0 + + minipass@7.1.2: {} + + mlly@1.8.0: + dependencies: + acorn: 8.15.0 + pathe: 2.0.3 + pkg-types: 1.3.1 + ufo: 1.6.2 + + ms@2.1.3: {} + + mz@2.7.0: + dependencies: + any-promise: 1.3.0 + object-assign: 4.1.1 + thenify-all: 1.6.0 + + nanoid@3.3.11: {} + + nypm@0.6.2: + dependencies: + citty: 0.1.6 + consola: 3.4.2 + pathe: 2.0.3 + pkg-types: 2.3.0 + tinyexec: 1.0.2 + + object-assign@4.1.1: {} + + obug@2.1.1: {} + + onetime@7.0.0: + dependencies: + mimic-function: 5.0.1 + + openai@6.16.0(zod@4.3.5): + optionalDependencies: + zod: 4.3.5 + + ora@9.0.0: + dependencies: + chalk: 5.6.2 + cli-cursor: 5.0.0 + cli-spinners: 3.4.0 + is-interactive: 2.0.0 + is-unicode-supported: 2.1.0 + log-symbols: 7.0.1 + stdin-discarder: 0.2.2 + string-width: 8.1.0 + strip-ansi: 7.1.2 + + path-scurry@2.0.1: + dependencies: + lru-cache: 11.2.4 + minipass: 7.1.2 + + pathe@2.0.3: {} + + picocolors@1.1.1: {} + + picomatch@4.0.3: {} + + pirates@4.0.7: {} + + pkg-types@1.3.1: + dependencies: + confbox: 0.1.8 + mlly: 1.8.0 + pathe: 2.0.3 + + pkg-types@2.3.0: + dependencies: + confbox: 0.2.2 + exsolve: 1.0.8 + pathe: 2.0.3 + + postcss-load-config@6.0.1(postcss@8.5.6): + dependencies: + lilconfig: 3.1.3 + optionalDependencies: + postcss: 8.5.6 + + postcss@8.5.6: + dependencies: + nanoid: 3.3.11 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + readdirp@4.1.2: {} + + resolve-from@5.0.0: {} + + restore-cursor@5.1.0: + dependencies: + onetime: 7.0.0 + signal-exit: 4.1.0 + + rollup@4.55.1: + dependencies: + '@types/estree': 1.0.8 + optionalDependencies: + '@rollup/rollup-android-arm-eabi': 4.55.1 + '@rollup/rollup-android-arm64': 4.55.1 + '@rollup/rollup-darwin-arm64': 4.55.1 + '@rollup/rollup-darwin-x64': 4.55.1 + '@rollup/rollup-freebsd-arm64': 4.55.1 + '@rollup/rollup-freebsd-x64': 4.55.1 + '@rollup/rollup-linux-arm-gnueabihf': 4.55.1 + '@rollup/rollup-linux-arm-musleabihf': 4.55.1 + '@rollup/rollup-linux-arm64-gnu': 4.55.1 + '@rollup/rollup-linux-arm64-musl': 4.55.1 + '@rollup/rollup-linux-loong64-gnu': 4.55.1 + '@rollup/rollup-linux-loong64-musl': 4.55.1 + '@rollup/rollup-linux-ppc64-gnu': 4.55.1 + '@rollup/rollup-linux-ppc64-musl': 4.55.1 + '@rollup/rollup-linux-riscv64-gnu': 4.55.1 + '@rollup/rollup-linux-riscv64-musl': 4.55.1 + '@rollup/rollup-linux-s390x-gnu': 4.55.1 + '@rollup/rollup-linux-x64-gnu': 4.55.1 + '@rollup/rollup-linux-x64-musl': 4.55.1 + '@rollup/rollup-openbsd-x64': 4.55.1 + '@rollup/rollup-openharmony-arm64': 4.55.1 + '@rollup/rollup-win32-arm64-msvc': 4.55.1 + '@rollup/rollup-win32-ia32-msvc': 4.55.1 + '@rollup/rollup-win32-x64-gnu': 4.55.1 + '@rollup/rollup-win32-x64-msvc': 4.55.1 + fsevents: 2.3.3 + + siginfo@2.0.0: {} + + signal-exit@4.1.0: {} + + sisteransi@1.0.5: {} + + source-map-js@1.2.1: {} + + source-map@0.7.6: {} + + stackback@0.0.2: {} + + std-env@3.10.0: {} + + stdin-discarder@0.2.2: {} + + string-width@8.1.0: + dependencies: + get-east-asian-width: 1.4.0 + strip-ansi: 7.1.2 + + strip-ansi@7.1.2: + dependencies: + ansi-regex: 6.2.2 + + sucrase@3.35.1: + dependencies: + '@jridgewell/gen-mapping': 0.3.13 + commander: 4.1.1 + lines-and-columns: 1.2.4 + mz: 2.7.0 + pirates: 4.0.7 + tinyglobby: 0.2.15 + ts-interface-checker: 0.1.13 + + thenify-all@1.6.0: + dependencies: + thenify: 3.3.1 + + thenify@3.3.1: + dependencies: + any-promise: 1.3.0 + + tinybench@2.9.0: {} + + tinyexec@0.3.2: {} + + tinyexec@1.0.2: {} + + tinyglobby@0.2.15: + dependencies: + fdir: 6.5.0(picomatch@4.0.3) + picomatch: 4.0.3 + + tinyrainbow@3.0.3: {} + + tree-kill@1.2.2: {} + + trpc-cli@0.12.2(@trpc/server@11.8.1(typescript@5.9.3))(zod@4.3.5): + dependencies: + commander: 14.0.2 + optionalDependencies: + '@trpc/server': 11.8.1(typescript@5.9.3) + zod: 4.3.5 + + ts-algebra@2.0.0: {} + + ts-interface-checker@0.1.13: {} + + tsup@8.5.1(postcss@8.5.6)(typescript@5.9.3): + dependencies: + bundle-require: 5.1.0(esbuild@0.27.2) + cac: 6.7.14 + chokidar: 4.0.3 + consola: 3.4.2 + debug: 4.4.3 + esbuild: 0.27.2 + fix-dts-default-cjs-exports: 1.0.1 + joycon: 3.1.1 + picocolors: 1.1.1 + postcss-load-config: 6.0.1(postcss@8.5.6) + resolve-from: 5.0.0 + rollup: 4.55.1 + source-map: 0.7.6 + sucrase: 3.35.1 + tinyexec: 0.3.2 + tinyglobby: 0.2.15 + tree-kill: 1.2.2 + optionalDependencies: + postcss: 8.5.6 + typescript: 5.9.3 + transitivePeerDependencies: + - jiti + - supports-color + - tsx + - yaml + + typescript@5.9.3: {} + + ufo@1.6.2: {} + + ultracite@7.0.11(typescript@5.9.3): + dependencies: + '@clack/prompts': 0.11.0 + '@trpc/server': 11.8.1(typescript@5.9.3) + deepmerge: 4.3.1 + glob: 13.0.0 + jsonc-parser: 3.3.1 + nypm: 0.6.2 + trpc-cli: 0.12.2(@trpc/server@11.8.1(typescript@5.9.3))(zod@4.3.5) + zod: 4.3.5 + transitivePeerDependencies: + - '@orpc/server' + - '@valibot/to-json-schema' + - effect + - typescript + - valibot + + undici-types@7.16.0: {} + + vite@7.3.1(@types/node@24.10.8): + dependencies: + esbuild: 0.27.2 + fdir: 6.5.0(picomatch@4.0.3) + picomatch: 4.0.3 + postcss: 8.5.6 + rollup: 4.55.1 + tinyglobby: 0.2.15 + optionalDependencies: + '@types/node': 24.10.8 + fsevents: 2.3.3 + + vitest@4.0.17(@types/node@24.10.8): + dependencies: + '@vitest/expect': 4.0.17 + '@vitest/mocker': 4.0.17(vite@7.3.1(@types/node@24.10.8)) + '@vitest/pretty-format': 4.0.17 + '@vitest/runner': 4.0.17 + '@vitest/snapshot': 4.0.17 + '@vitest/spy': 4.0.17 + '@vitest/utils': 4.0.17 + es-module-lexer: 1.7.0 + expect-type: 1.3.0 + magic-string: 0.30.21 + obug: 2.1.1 + pathe: 2.0.3 + picomatch: 4.0.3 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 1.0.2 + tinyglobby: 0.2.15 + tinyrainbow: 3.0.3 + vite: 7.3.1(@types/node@24.10.8) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 24.10.8 + transitivePeerDependencies: + - jiti + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - terser + - tsx + - yaml + + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 + + yoctocolors@2.1.2: {} + + zod@4.3.5: {} diff --git a/toolkit-docs-generator/src/cli/index.ts b/toolkit-docs-generator/src/cli/index.ts new file mode 100644 index 000000000..0de097bc2 --- /dev/null +++ b/toolkit-docs-generator/src/cli/index.ts @@ -0,0 +1,428 @@ +#!/usr/bin/env node + +/** + * Toolkit Docs Generator CLI + * + * Entry point for the CLI application. + * Run with: npx toolkit-docs-generator + * + * Input format for providers: + * --providers "Github:1.0.0,Slack:2.1.0" + * --providers "Github" (uses latest version) + */ + +import chalk from "chalk"; +import { Command } from "commander"; +import ora from "ora"; +import { join, resolve } from "path"; +import { createJsonGenerator } from "../generator/json-generator.js"; +import { + createLlmClient, + type LlmProvider, + LlmToolExampleGenerator, +} from "../llm/index.js"; +import type { MergeResult } from "../merger/data-merger.js"; +import { createDataMerger } from "../merger/data-merger.js"; +import { createCustomSectionsFileSource } from "../sources/custom-sections-file.js"; +import { createEmptyCustomSectionsSource } from "../sources/in-memory.js"; +import { createMockToolkitDataSource } from "../sources/toolkit-data-source.js"; +import { type ProviderVersion, ProviderVersionSchema } from "../types/index.js"; + +const program = new Command(); + +/** + * Parse providers string into array of ProviderVersion objects + * @param input - Comma-separated string like "Github:1.0.0,Slack:2.1.0" + */ +const parseProviders = (input: string): ProviderVersion[] => { + const parts = input + .split(",") + .map((p) => p.trim()) + .filter(Boolean); + return parts.map((part) => { + const [provider, version] = part.split(":").map((s) => s.trim()); + if (!provider) { + throw new Error( + `Invalid provider format: "${part}". Expected "Provider" or "Provider:version"` + ); + } + const parsed = ProviderVersionSchema.safeParse({ + provider, + version: version || undefined, + }); + if (!parsed.success) { + throw new Error(`Invalid provider: ${parsed.error.message}`); + } + return parsed.data; + }); +}; + +/** + * Get the default fixture paths (for mock mode) + */ +const getDefaultMockDataDir = (): string => join(process.cwd(), "mock-data"); + +const resolveLlmProvider = (value?: string): LlmProvider => { + if (value === "openai" || value === "anthropic") { + return value; + } + throw new Error( + 'LLM provider is required. Use --llm-provider "openai" or "anthropic".' + ); +}; + +const resolveLlmConfig = (options: { + llmProvider?: string; + llmApiKey?: string; + llmModel?: string; + llmBaseUrl?: string; + llmTemperature?: number; + llmMaxTokens?: number; + llmSystemPrompt?: string; +}) => { + const provider = resolveLlmProvider( + options.llmProvider ?? process.env.LLM_PROVIDER + ); + const model = options.llmModel ?? process.env.LLM_MODEL; + + if (!model) { + throw new Error("LLM model is required. Use --llm-model or LLM_MODEL."); + } + + const apiKey = + options.llmApiKey ?? + (provider === "openai" + ? process.env.OPENAI_API_KEY + : process.env.ANTHROPIC_API_KEY); + + if (!apiKey) { + throw new Error( + provider === "openai" + ? "OpenAI API key is required. Use --llm-api-key or OPENAI_API_KEY." + : "Anthropic API key is required. Use --llm-api-key or ANTHROPIC_API_KEY." + ); + } + + const client = createLlmClient({ + provider, + config: { + apiKey, + ...(options.llmBaseUrl ? { baseUrl: options.llmBaseUrl } : {}), + }, + }); + + return new LlmToolExampleGenerator({ + client, + model, + ...(options.llmTemperature !== undefined + ? { temperature: options.llmTemperature } + : {}), + ...(options.llmMaxTokens !== undefined + ? { maxTokens: options.llmMaxTokens } + : {}), + ...(options.llmSystemPrompt + ? { systemPrompt: options.llmSystemPrompt } + : {}), + }); +}; + +const processProviders = async ( + providers: ProviderVersion[], + merger: ReturnType, + spinner: ReturnType, + verbose: boolean +): Promise => { + const results: MergeResult[] = []; + + for (const pv of providers) { + spinner.start(`Processing ${pv.provider}...`); + + try { + const result = await merger.mergeToolkit(pv.provider, pv.version); + results.push(result); + + if (result.warnings.length > 0 && verbose) { + spinner.warn(`${pv.provider}: ${result.warnings.join(", ")}`); + } else { + spinner.succeed(`${pv.provider}: ${result.toolkit.tools.length} tools`); + } + } catch (error) { + spinner.fail(`${pv.provider}: ${error}`); + } + } + + return results; +}; + +program + .name("toolkit-docs-generator") + .description("Generate documentation JSON for Arcade toolkits") + .version("1.0.0"); + +program + .command("generate") + .description("Generate documentation for specified providers") + .requiredOption( + "-p, --providers ", + 'Comma-separated list of providers with optional versions (e.g., "Github:1.0.0,Slack:2.1.0" or "Github,Slack")' + ) + .option("-o, --output ", "Output directory", "./output") + .option("--mock-data-dir ", "Path to mock data directory") + .option("--llm-provider ", "LLM provider (openai|anthropic)") + .option("--llm-model ", "LLM model to use") + .option("--llm-api-key ", "LLM API key") + .option("--llm-base-url ", "LLM base URL") + .option("--llm-temperature ", "LLM temperature", (value) => + Number.parseFloat(value) + ) + .option("--llm-max-tokens ", "LLM max tokens", (value) => + Number.parseInt(value, 10) + ) + .option("--llm-system-prompt ", "LLM system prompt override") + .option("--custom-sections ", "Path to custom sections JSON") + .option("--verbose", "Enable verbose logging", false) + .action( + async (options: { + providers: string; + output: string; + mockDataDir?: string; + llmProvider?: string; + llmModel?: string; + llmApiKey?: string; + llmBaseUrl?: string; + llmTemperature?: number; + llmMaxTokens?: number; + llmSystemPrompt?: string; + customSections?: string; + verbose: boolean; + }) => { + const spinner = ora("Parsing input...").start(); + + try { + // Parse providers + const providers = parseProviders(options.providers); + spinner.succeed(`Parsed ${providers.length} provider(s)`); + + if (options.verbose) { + console.log(chalk.cyan("\nProviders to process:")); + for (const pv of providers) { + const version = pv.version ?? "latest"; + console.log(chalk.dim(` - ${pv.provider}:${version}`)); + } + } + + // Initialize sources + spinner.start("Initializing data sources..."); + + const mockDataDir = options.mockDataDir ?? getDefaultMockDataDir(); + const toolkitDataSource = createMockToolkitDataSource({ + dataDir: mockDataDir, + }); + const toolExampleGenerator = resolveLlmConfig(options); + + // Custom sections source + const customSectionsSource = options.customSections + ? createCustomSectionsFileSource(options.customSections) + : createEmptyCustomSectionsSource(); + + spinner.succeed("Data sources initialized"); + + // Create merger using unified source + const merger = createDataMerger({ + toolkitDataSource, + customSectionsSource, + toolExampleGenerator, + }); + + // Create generator + const generator = createJsonGenerator({ + outputDir: resolve(options.output), + prettyPrint: true, + generateIndex: true, + }); + + // Process each provider + const allResults = await processProviders( + providers, + merger, + spinner, + options.verbose + ); + + // Generate output files + if (allResults.length > 0) { + spinner.start("Writing output files..."); + + const toolkits = allResults.map((r) => r.toolkit); + const genResult = await generator.generateAll(toolkits); + + if (genResult.errors.length > 0) { + spinner.warn(`Written with errors: ${genResult.errors.join(", ")}`); + } else { + spinner.succeed(`Written ${genResult.filesWritten.length} file(s)`); + } + + // Print summary + console.log(chalk.green("\n✓ Generation complete\n")); + console.log(chalk.dim("Files:")); + for (const file of genResult.filesWritten) { + console.log(chalk.dim(` ${file}`)); + } + } + } catch (error) { + spinner.fail( + `Error: ${error instanceof Error ? error.message : String(error)}` + ); + process.exit(1); + } + } + ); + +program + .command("generate-all") + .description("Generate documentation for all toolkits in mock data") + .option("-o, --output ", "Output directory", "./output") + .option("--mock-data-dir ", "Path to mock data directory") + .option("--llm-provider ", "LLM provider (openai|anthropic)") + .option("--llm-model ", "LLM model to use") + .option("--llm-api-key ", "LLM API key") + .option("--llm-base-url ", "LLM base URL") + .option("--llm-temperature ", "LLM temperature", (value) => + Number.parseFloat(value) + ) + .option("--llm-max-tokens ", "LLM max tokens", (value) => + Number.parseInt(value, 10) + ) + .option("--llm-system-prompt ", "LLM system prompt override") + .option("--custom-sections ", "Path to custom sections JSON") + .option("--verbose", "Enable verbose logging", false) + .action( + async (options: { + output: string; + mockDataDir?: string; + llmProvider?: string; + llmModel?: string; + llmApiKey?: string; + llmBaseUrl?: string; + llmTemperature?: number; + llmMaxTokens?: number; + llmSystemPrompt?: string; + customSections?: string; + verbose: boolean; + }) => { + const spinner = ora("Initializing...").start(); + + try { + const mockDataDir = options.mockDataDir ?? getDefaultMockDataDir(); + const toolkitDataSource = createMockToolkitDataSource({ + dataDir: mockDataDir, + }); + const toolExampleGenerator = resolveLlmConfig(options); + + const customSectionsSource = options.customSections + ? createCustomSectionsFileSource(options.customSections) + : createEmptyCustomSectionsSource(); + + spinner.succeed("Data sources initialized"); + + // Create merger using unified source + const merger = createDataMerger({ + toolkitDataSource, + customSectionsSource, + toolExampleGenerator, + }); + + spinner.start("Merging all toolkits..."); + const results = await merger.mergeAllToolkits(); + spinner.succeed(`Merged ${results.length} toolkit(s)`); + + // Generate output + const generator = createJsonGenerator({ + outputDir: resolve(options.output), + prettyPrint: true, + generateIndex: true, + }); + + spinner.start("Writing output files..."); + const toolkits = results.map((r) => r.toolkit); + const genResult = await generator.generateAll(toolkits); + + if (genResult.errors.length > 0) { + spinner.warn(`Written with errors: ${genResult.errors.join(", ")}`); + } else { + spinner.succeed(`Written ${genResult.filesWritten.length} file(s)`); + } + + console.log(chalk.green("\n✓ Generation complete\n")); + } catch (error) { + spinner.fail( + `Error: ${error instanceof Error ? error.message : String(error)}` + ); + process.exit(1); + } + } + ); + +program + .command("validate ") + .description("Validate a generated JSON file against the schema") + .action(async (file: string) => { + const { readFile } = await import("fs/promises"); + const { MergedToolkitSchema } = await import("../types/index.js"); + + try { + const content = await readFile(file, "utf-8"); + const json = JSON.parse(content); + const result = MergedToolkitSchema.safeParse(json); + + if (result.success) { + console.log(chalk.green(`✓ ${file} is valid`)); + } else { + console.log(chalk.red(`✗ ${file} is invalid:`)); + console.log(chalk.dim(result.error.message)); + process.exit(1); + } + } catch (error) { + console.log(chalk.red(`✗ Failed to validate: ${error}`)); + process.exit(1); + } + }); + +program + .command("list-toolkits") + .description("List toolkits available in mock data") + .option("--mock-data-dir ", "Path to mock data directory") + .option("--json", "Output as JSON", false) + .action(async (options: { mockDataDir?: string; json: boolean }) => { + try { + const toolkits = new Map(); + const mockDataDir = options.mockDataDir ?? getDefaultMockDataDir(); + const toolkitDataSource = createMockToolkitDataSource({ + dataDir: mockDataDir, + }); + const toolkitData = await toolkitDataSource.fetchAllToolkitsData(); + + for (const [id, data] of toolkitData) { + toolkits.set(id, data.tools.length); + } + + if (options.json) { + const data = Array.from(toolkits.entries()).map(([id, count]) => ({ + id, + toolCount: count, + })); + console.log(JSON.stringify(data, null, 2)); + } else { + console.log(chalk.bold(`\nToolkits in fixture (${toolkits.size}):\n`)); + for (const [id, count] of toolkits) { + console.log(` ${chalk.green(id.padEnd(20))} ${count} tool(s)`); + } + } + } catch (error) { + console.log(chalk.red(`Error: ${error}`)); + process.exit(1); + } + }); + +// Parse command line arguments +program.parse(); diff --git a/toolkit-docs-generator/src/generator/index.ts b/toolkit-docs-generator/src/generator/index.ts new file mode 100644 index 000000000..37b71f395 --- /dev/null +++ b/toolkit-docs-generator/src/generator/index.ts @@ -0,0 +1,4 @@ +/** + * Generator module exports + */ +export * from "./json-generator.js"; diff --git a/toolkit-docs-generator/src/generator/json-generator.ts b/toolkit-docs-generator/src/generator/json-generator.ts new file mode 100644 index 000000000..62f050b09 --- /dev/null +++ b/toolkit-docs-generator/src/generator/json-generator.ts @@ -0,0 +1,158 @@ +/** + * JSON Generator + * + * Outputs merged toolkit data as JSON files. + */ +import { mkdir, writeFile } from "fs/promises"; +import { dirname, join } from "path"; +import type { + MergedToolkit, + ToolkitIndex, + ToolkitIndexEntry, +} from "../types/index.js"; +import { MergedToolkitSchema } from "../types/index.js"; + +// ============================================================================ +// Generator Configuration +// ============================================================================ + +export interface JsonGeneratorConfig { + /** Output directory path */ + outputDir: string; + /** Whether to pretty-print JSON (default: true) */ + prettyPrint?: boolean; + /** Whether to generate an index file (default: true) */ + generateIndex?: boolean; + /** Whether to validate output with Zod (default: true) */ + validateOutput?: boolean; +} + +export interface GeneratorResult { + /** List of files that were written */ + filesWritten: string[]; + /** List of errors that occurred */ + errors: string[]; +} + +// ============================================================================ +// JSON Generator +// ============================================================================ + +/** + * Generator that outputs merged toolkit data as JSON files + */ +export class JsonGenerator { + private readonly outputDir: string; + private readonly prettyPrint: boolean; + private readonly generateIndex: boolean; + private readonly validateOutput: boolean; + + constructor(config: JsonGeneratorConfig) { + this.outputDir = config.outputDir; + this.prettyPrint = config.prettyPrint ?? true; + this.generateIndex = config.generateIndex ?? true; + this.validateOutput = config.validateOutput ?? true; + } + + /** + * Generate JSON file for a single toolkit + */ + async generateToolkitFile(toolkit: MergedToolkit): Promise { + // Validate if enabled + if (this.validateOutput) { + const result = MergedToolkitSchema.safeParse(toolkit); + if (!result.success) { + throw new Error( + `Validation failed for ${toolkit.id}: ${result.error.message}` + ); + } + } + + const fileName = `${toolkit.id.toLowerCase()}.json`; + const filePath = join(this.outputDir, fileName); + + // Ensure directory exists + await mkdir(dirname(filePath), { recursive: true }); + + // Write file + const content = this.prettyPrint + ? JSON.stringify(toolkit, null, 2) + : JSON.stringify(toolkit); + + await writeFile(filePath, content, "utf-8"); + return filePath; + } + + /** + * Generate JSON files for multiple toolkits + */ + async generateAll( + toolkits: readonly MergedToolkit[] + ): Promise { + const filesWritten: string[] = []; + const errors: string[] = []; + + // Generate per-toolkit files + for (const toolkit of toolkits) { + try { + const filePath = await this.generateToolkitFile(toolkit); + filesWritten.push(filePath); + } catch (error) { + errors.push(`Failed to write ${toolkit.id}: ${error}`); + } + } + + // Generate index file + if (this.generateIndex && toolkits.length > 0) { + try { + const indexPath = await this.generateIndexFile(toolkits); + filesWritten.push(indexPath); + } catch (error) { + errors.push(`Failed to write index: ${error}`); + } + } + + return { filesWritten, errors }; + } + + /** + * Generate index file with toolkit summaries + */ + private async generateIndexFile( + toolkits: readonly MergedToolkit[] + ): Promise { + const entries: ToolkitIndexEntry[] = toolkits.map((t) => ({ + id: t.id, + label: t.label, + version: t.version, + category: t.metadata.category, + toolCount: t.tools.length, + authType: t.auth?.type ?? "none", + })); + + const index: ToolkitIndex = { + generatedAt: new Date().toISOString(), + version: "1.0.0", + toolkits: entries, + }; + + const filePath = join(this.outputDir, "index.json"); + + await mkdir(dirname(filePath), { recursive: true }); + + const content = this.prettyPrint + ? JSON.stringify(index, null, 2) + : JSON.stringify(index); + + await writeFile(filePath, content, "utf-8"); + return filePath; + } +} + +// ============================================================================ +// Factory +// ============================================================================ + +export const createJsonGenerator = ( + config: JsonGeneratorConfig +): JsonGenerator => new JsonGenerator(config); diff --git a/toolkit-docs-generator/src/index.ts b/toolkit-docs-generator/src/index.ts new file mode 100644 index 000000000..97f39ff57 --- /dev/null +++ b/toolkit-docs-generator/src/index.ts @@ -0,0 +1,19 @@ +/** + * Toolkit Docs Generator + * + * A library for generating Arcade toolkit documentation JSON + * from multiple data sources. + */ + +// Generator +export * from "./generator/index.js"; +// LLM +export * from "./llm/index.js"; +// Merger +export * from "./merger/index.js"; +// Sources +export * from "./sources/index.js"; +// Types +export * from "./types/index.js"; +// Utils +export * from "./utils/index.js"; diff --git a/toolkit-docs-generator/src/llm/client.ts b/toolkit-docs-generator/src/llm/client.ts new file mode 100644 index 000000000..2d2b7fa63 --- /dev/null +++ b/toolkit-docs-generator/src/llm/client.ts @@ -0,0 +1,116 @@ +import Anthropic from "@anthropic-ai/sdk"; +import OpenAI from "openai"; + +export type LlmProvider = "openai" | "anthropic"; + +export interface LlmRequest { + readonly model: string; + readonly prompt: string; + readonly system?: string; + readonly temperature?: number; + readonly maxTokens?: number; +} + +export interface LlmClient { + readonly provider: LlmProvider; + readonly generateText: (request: LlmRequest) => Promise; +} + +export interface OpenAiClientConfig { + readonly apiKey: string; + readonly baseUrl?: string; +} + +export class OpenAiClient implements LlmClient { + readonly provider = "openai" as const; + private readonly client: OpenAI; + + constructor(config: OpenAiClientConfig) { + this.client = new OpenAI({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + }); + } + + async generateText(request: LlmRequest): Promise { + const payload: OpenAI.Responses.ResponseCreateParamsNonStreaming = { + model: request.model, + input: request.prompt, + }; + + if (request.system !== undefined) { + payload.instructions = request.system; + } + if (request.temperature !== undefined) { + payload.temperature = request.temperature; + } + if (request.maxTokens !== undefined) { + payload.max_output_tokens = request.maxTokens; + } + + const response = await this.client.responses.create(payload); + + if (!response.output_text) { + throw new Error("OpenAI response missing output_text"); + } + + return response.output_text.trim(); + } +} + +export interface AnthropicClientConfig { + readonly apiKey: string; + readonly baseUrl?: string; +} + +export class AnthropicClient implements LlmClient { + readonly provider = "anthropic" as const; + private readonly client: Anthropic; + + constructor(config: AnthropicClientConfig) { + this.client = new Anthropic({ + apiKey: config.apiKey, + baseURL: config.baseUrl, + }); + } + + async generateText(request: LlmRequest): Promise { + const payload: Anthropic.MessageCreateParamsNonStreaming = { + model: request.model, + max_tokens: request.maxTokens ?? 512, + messages: [{ role: "user", content: request.prompt }], + }; + + if (request.temperature !== undefined) { + payload.temperature = request.temperature; + } + if (request.system !== undefined) { + payload.system = request.system; + } + + const response = await this.client.messages.create(payload); + + const content = response.content + .map((item) => ("text" in item ? item.text : "")) + .join("") + .trim(); + + if (!content) { + throw new Error("Anthropic response missing text content"); + } + + return content; + } +} + +export type LlmClientConfig = + | { provider: "openai"; config: OpenAiClientConfig } + | { provider: "anthropic"; config: AnthropicClientConfig }; + +export const createLlmClient = (config: LlmClientConfig): LlmClient => { + if (config.provider === "openai") { + return new OpenAiClient(config.config); + } + + return new AnthropicClient(config.config); +}; diff --git a/toolkit-docs-generator/src/llm/index.ts b/toolkit-docs-generator/src/llm/index.ts new file mode 100644 index 000000000..460ef494d --- /dev/null +++ b/toolkit-docs-generator/src/llm/index.ts @@ -0,0 +1,2 @@ +export * from "./client.js"; +export * from "./tool-example-generator.js"; diff --git a/toolkit-docs-generator/src/llm/tool-example-generator.ts b/toolkit-docs-generator/src/llm/tool-example-generator.ts new file mode 100644 index 000000000..28a8af9cc --- /dev/null +++ b/toolkit-docs-generator/src/llm/tool-example-generator.ts @@ -0,0 +1,192 @@ +import type { + ToolExampleGenerator, + ToolExampleResult, +} from "../merger/data-merger.js"; +import { + type ExampleParameterValue, + type SecretType, + SecretTypeSchema, + type ToolDefinition, + type ToolSecret, +} from "../types/index.js"; +import type { LlmClient } from "./client.js"; + +export interface LlmToolExampleGeneratorConfig { + readonly client: LlmClient; + readonly model: string; + readonly temperature?: number; + readonly maxTokens?: number; + readonly systemPrompt?: string; +} + +const defaultSystemPrompt = + "Return only valid JSON. No markdown, no extra text."; + +const buildPrompt = (tool: ToolDefinition): string => { + const parameterLines = tool.parameters.map((param) => { + const description = param.description ?? "No description"; + return `- ${param.name} (${param.type}, required: ${param.required}): ${description}`; + }); + const secrets = tool.secrets.length > 0 ? tool.secrets.join(", ") : "None"; + + return [ + "Generate example values for the tool parameters below.", + "Return a JSON object with this shape:", + '{ "parameters": { "": "" }, "secrets": { "": "" } }', + "Secret types must be one of: api_key, token, client_secret, webhook_secret, private_key, password, unknown.", + "", + `Tool: ${tool.qualifiedName}`, + `Description: ${tool.description ?? "No description"}`, + "Parameters:", + ...parameterLines, + `Secrets: ${secrets}`, + ].join("\n"); +}; + +const extractJson = (text: string): string => { + const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/); + if (fenced?.[1]) { + return fenced[1].trim(); + } + return text.trim(); +}; + +const parseJsonObject = (text: string): Record => { + const jsonText = extractJson(text); + const parsed = JSON.parse(jsonText) as unknown; + + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("LLM response is not a JSON object"); + } + + return parsed as Record; +}; + +const mapParamType = ( + type: string +): "string" | "integer" | "boolean" | "array" | "object" => { + switch (type) { + case "string": + return "string"; + case "integer": + case "number": + return "integer"; + case "boolean": + return "boolean"; + case "array": + return "array"; + case "object": + return "object"; + default: + return "string"; + } +}; + +const normalizeExampleValue = (paramType: string, value: unknown): unknown => { + if (value === null || value === undefined) { + return null; + } + + const mappedType = mapParamType(paramType); + + if (mappedType === "integer") { + if (typeof value === "string") { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : value; + } + return value; + } + + if (mappedType === "boolean") { + if (typeof value === "string") { + const lowered = value.toLowerCase(); + if (lowered === "true") { + return true; + } + if (lowered === "false") { + return false; + } + } + return value; + } + + return value; +}; + +export class LlmToolExampleGenerator implements ToolExampleGenerator { + private readonly client: LlmClient; + private readonly model: string; + private readonly temperature: number | undefined; + private readonly maxTokens: number | undefined; + private readonly systemPrompt: string; + + constructor(config: LlmToolExampleGeneratorConfig) { + this.client = config.client; + this.model = config.model; + this.temperature = config.temperature; + this.maxTokens = config.maxTokens; + this.systemPrompt = config.systemPrompt ?? defaultSystemPrompt; + } + + async generate(tool: ToolDefinition): Promise { + const prompt = buildPrompt(tool); + const request = { + model: this.model, + prompt, + system: this.systemPrompt, + } as const; + + const response = await this.client.generateText({ + ...request, + ...(this.temperature !== undefined + ? { temperature: this.temperature } + : {}), + ...(this.maxTokens !== undefined ? { maxTokens: this.maxTokens } : {}), + }); + + const payload = parseJsonObject(response); + const rawParameters = + (payload.parameters as Record | undefined) ?? {}; + const rawSecrets = + (payload.secrets as Record | undefined) ?? {}; + const parameters: Record = {}; + + for (const param of tool.parameters) { + const rawValue = rawParameters[param.name]; + parameters[param.name] = { + value: normalizeExampleValue(param.type, rawValue), + type: mapParamType(param.type), + required: param.required, + }; + } + + const normalizedSecrets = new Map(); + for (const [key, value] of Object.entries(rawSecrets)) { + if (typeof value === "string") { + const parsed = SecretTypeSchema.safeParse(value); + normalizedSecrets.set( + key.toLowerCase(), + parsed.success ? parsed.data : "unknown" + ); + } + } + + const secretsInfo: ToolSecret[] = tool.secrets.map((secret) => ({ + name: secret, + type: normalizedSecrets.get(secret.toLowerCase()) ?? "unknown", + })); + + return { + codeExample: { + toolName: tool.qualifiedName, + parameters, + requiresAuth: tool.auth !== null, + authProvider: tool.auth?.providerId ?? undefined, + tabLabel: tool.auth + ? "Call the Tool with User Authorization" + : "Call the Tool", + }, + secretsInfo, + }; + } +} diff --git a/toolkit-docs-generator/src/merger/data-merger.ts b/toolkit-docs-generator/src/merger/data-merger.ts new file mode 100644 index 000000000..2bf806505 --- /dev/null +++ b/toolkit-docs-generator/src/merger/data-merger.ts @@ -0,0 +1,337 @@ +/** + * Data Merger + * + * Combines data from Engine API, Design System, and Custom Sections + * into the final MergedToolkit format. + */ + +import type { ICustomSectionsSource } from "../sources/interfaces.js"; +import type { IToolkitDataSource } from "../sources/toolkit-data-source.js"; +import type { + CustomSections, + DocumentationChunk, + MergedTool, + MergedToolkit, + MergedToolkitAuth, + MergedToolkitMetadata, + ToolCodeExample, + ToolDefinition, + ToolkitAuthType, + ToolkitMetadata, +} from "../types/index.js"; + +// ============================================================================ +// Merger Configuration +// ============================================================================ + +export interface DataMergerConfig { + toolkitDataSource: IToolkitDataSource; + customSectionsSource: ICustomSectionsSource; + toolExampleGenerator: ToolExampleGenerator; +} + +export interface MergeResult { + toolkit: MergedToolkit; + warnings: string[]; +} + +export interface ToolExampleResult { + codeExample: ToolCodeExample; + secretsInfo: MergedTool["secretsInfo"]; +} + +export interface ToolExampleGenerator { + generate: (tool: ToolDefinition) => Promise; +} + +// ============================================================================ +// Pure Functions for Data Transformation +// ============================================================================ + +/** + * Group tools by their toolkit name (first part of qualified name) + */ +export const groupToolsByToolkit = ( + tools: readonly ToolDefinition[] +): ReadonlyMap => { + const groups = new Map(); + + for (const tool of tools) { + const toolkitName = tool.qualifiedName.split(".")[0]; + if (!toolkitName) continue; + + const existing = groups.get(toolkitName) ?? []; + groups.set(toolkitName, [...existing, tool]); + } + + return groups; +}; +/** + * Compute the union of all scopes from tools + */ +export const computeAllScopes = ( + tools: readonly ToolDefinition[] +): string[] => { + const scopeSet = new Set(); + + for (const tool of tools) { + if (tool.auth?.scopes) { + for (const scope of tool.auth.scopes) { + scopeSet.add(scope); + } + } + } + + return Array.from(scopeSet).sort(); +}; + +/** + * Determine the auth type from tools + */ +export const determineAuthType = ( + tools: readonly ToolDefinition[] +): ToolkitAuthType => { + const hasOAuth = tools.some((tool) => tool.auth?.providerType === "oauth2"); + const hasApiKey = tools.some( + (tool) => tool.auth && tool.auth.providerType !== "oauth2" + ); + + if (hasOAuth && hasApiKey) { + return "mixed"; + } + + if (hasOAuth) { + return "oauth2"; + } + + if (hasApiKey) { + return "api_key"; + } + + return "none"; +}; + +/** + * Get the provider ID from tools + */ +export const getProviderId = ( + tools: readonly ToolDefinition[] +): string | null => { + const toolWithAuth = tools.find((t) => t.auth?.providerId); + return toolWithAuth?.auth?.providerId ?? null; +}; + +/** + * Extract version from fully qualified name + */ +export const extractVersion = (fullyQualifiedName: string): string => { + const parts = fullyQualifiedName.split("@"); + return parts[1] ?? "0.0.0"; +}; +/** + * Create default metadata for toolkits not found in Design System + */ +const getDefaultMetadata = (toolkitId: string): MergedToolkitMetadata => ({ + category: "development", + iconUrl: `https://design-system.arcade.dev/icons/${toolkitId.toLowerCase()}.svg`, + isBYOC: false, + isPro: false, + type: "arcade", + docsLink: `https://docs.arcade.dev/en/mcp-servers/development/${toolkitId.toLowerCase()}`, + isComingSoon: false, + isHidden: false, +}); + +/** + * Transform ToolkitMetadata to MergedToolkitMetadata (without id/label) + */ +const transformMetadata = ( + metadata: ToolkitMetadata +): MergedToolkitMetadata => ({ + category: metadata.category, + iconUrl: metadata.iconUrl, + isBYOC: metadata.isBYOC, + isPro: metadata.isPro, + type: metadata.type, + docsLink: metadata.docsLink, + isComingSoon: metadata.isComingSoon, + isHidden: metadata.isHidden, +}); + +/** + * Transform a tool definition into a merged tool + */ +const transformTool = async ( + tool: ToolDefinition, + toolChunks: { [key: string]: DocumentationChunk[] }, + toolExampleGenerator: ToolExampleGenerator +): Promise => { + const exampleResult = await toolExampleGenerator.generate(tool); + + return { + name: tool.name, + qualifiedName: tool.qualifiedName, + fullyQualifiedName: tool.fullyQualifiedName, + description: tool.description, + parameters: tool.parameters, + auth: tool.auth, + secrets: tool.secrets, + secretsInfo: exampleResult.secretsInfo, + output: tool.output, + documentationChunks: toolChunks[tool.name] ?? [], + codeExample: exampleResult.codeExample, + }; +}; + +// ============================================================================ +// Main Merger Function +// ============================================================================ + +/** + * Merge data from all sources for a single toolkit + */ +export const mergeToolkit = async ( + toolkitId: string, + tools: readonly ToolDefinition[], + metadata: ToolkitMetadata | null, + customSections: CustomSections | null, + toolExampleGenerator: ToolExampleGenerator +): Promise => { + const warnings: string[] = []; + + if (tools.length === 0) { + warnings.push(`No tools found for toolkit: ${toolkitId}`); + } + + if (!metadata) { + warnings.push( + `No metadata found for toolkit: ${toolkitId} - using defaults` + ); + } + + // Get version from first tool + const firstTool = tools[0]; + const version = firstTool + ? extractVersion(firstTool.fullyQualifiedName) + : "0.0.0"; + + // Get toolkit description from first tool's toolkit info + const description = firstTool?.description ?? null; + + // Build auth info + const authType = determineAuthType(tools); + const providerId = getProviderId(tools); + const allScopes = computeAllScopes(tools); + + const auth: MergedToolkitAuth | null = + authType !== "none" + ? { + type: authType, + providerId, + allScopes, + } + : null; + + // Transform tools + const toolChunks = (customSections?.toolChunks ?? {}) as { + [key: string]: DocumentationChunk[]; + }; + const mergedTools = await Promise.all( + tools.map((tool) => transformTool(tool, toolChunks, toolExampleGenerator)) + ); + + // Build final toolkit + const toolkit: MergedToolkit = { + id: toolkitId, + label: metadata?.label ?? toolkitId, + version, + description, + metadata: metadata + ? transformMetadata(metadata) + : getDefaultMetadata(toolkitId), + auth, + tools: mergedTools, + documentationChunks: customSections?.documentationChunks ?? [], + customImports: customSections?.customImports ?? [], + subPages: customSections?.subPages ?? [], + generatedAt: new Date().toISOString(), + }; + + return { toolkit, warnings }; +}; + +// ============================================================================ +// Data Merger Class +// ============================================================================ + +/** + * Data merger that combines all sources + */ +export class DataMerger { + private readonly toolkitDataSource: IToolkitDataSource; + private readonly customSectionsSource: ICustomSectionsSource; + private readonly toolExampleGenerator: ToolExampleGenerator; + + constructor(config: DataMergerConfig) { + this.toolkitDataSource = config.toolkitDataSource; + this.customSectionsSource = config.customSectionsSource; + this.toolExampleGenerator = config.toolExampleGenerator; + } + + /** + * Merge data for a single toolkit + */ + async mergeToolkit( + toolkitId: string, + version?: string + ): Promise { + const toolkitData = await this.toolkitDataSource.fetchToolkitData( + toolkitId, + version + ); + + // Fetch custom sections + const customSections = + await this.customSectionsSource.getCustomSections(toolkitId); + + return mergeToolkit( + toolkitId, + toolkitData.tools, + toolkitData.metadata, + customSections, + this.toolExampleGenerator + ); + } + + /** + * Merge data for all toolkits + */ + async mergeAllToolkits(): Promise { + const results: MergeResult[] = []; + + const allToolkitsData = await this.toolkitDataSource.fetchAllToolkitsData(); + + for (const [toolkitId, toolkitData] of allToolkitsData) { + const customSections = + await this.customSectionsSource.getCustomSections(toolkitId); + + const result = await mergeToolkit( + toolkitId, + toolkitData.tools, + toolkitData.metadata, + customSections, + this.toolExampleGenerator + ); + results.push(result); + } + + return results; + } +} + +// ============================================================================ +// Factory +// ============================================================================ + +export const createDataMerger = (config: DataMergerConfig): DataMerger => + new DataMerger(config); diff --git a/toolkit-docs-generator/src/merger/index.ts b/toolkit-docs-generator/src/merger/index.ts new file mode 100644 index 000000000..86f15c8fe --- /dev/null +++ b/toolkit-docs-generator/src/merger/index.ts @@ -0,0 +1,4 @@ +/** + * Merger module exports + */ +export * from "./data-merger.js"; diff --git a/toolkit-docs-generator/src/sources/custom-sections-file.ts b/toolkit-docs-generator/src/sources/custom-sections-file.ts new file mode 100644 index 000000000..db96db711 --- /dev/null +++ b/toolkit-docs-generator/src/sources/custom-sections-file.ts @@ -0,0 +1,103 @@ +/** + * Custom Sections File Source + * + * Loads custom documentation sections from a JSON file. + * This file is produced by the one-time MDX extraction script. + */ +import { access, readFile } from "fs/promises"; +import { z } from "zod"; +import type { CustomSections } from "../types/index.js"; +import { DocumentationChunkSchema } from "../types/index.js"; +import { normalizeId } from "../utils/fp.js"; +import type { ICustomSectionsSource } from "./interfaces.js"; + +// ============================================================================ +// File Schema +// ============================================================================ + +const CustomSectionsFileSchema = z.record( + z.string(), + z.object({ + documentationChunks: z.array(DocumentationChunkSchema).default([]), + customImports: z.array(z.string()).default([]), + subPages: z.array(z.string()).default([]), + toolChunks: z + .record(z.string(), z.array(DocumentationChunkSchema)) + .default({}), + }) +); + +type CustomSectionsFile = z.infer; + +// ============================================================================ +// Custom Sections File Source +// ============================================================================ + +export interface CustomSectionsFileConfig { + filePath: string; +} + +/** + * Source that loads custom documentation sections from a JSON file + */ +export class CustomSectionsFileSource implements ICustomSectionsSource { + private readonly filePath: string; + private cachedData: CustomSectionsFile | null = null; + + constructor(config: CustomSectionsFileConfig) { + this.filePath = config.filePath; + } + + private async loadFile(): Promise { + if (this.cachedData !== null) { + return this.cachedData; + } + + try { + await access(this.filePath); + const content = await readFile(this.filePath, "utf-8"); + const json = JSON.parse(content); + this.cachedData = CustomSectionsFileSchema.parse(json); + return this.cachedData; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + // File doesn't exist - return empty data + this.cachedData = {}; + return this.cachedData; + } + throw error; + } + } + + async getCustomSections(toolkitId: string): Promise { + const data = await this.loadFile(); + + // Try exact match + if (data[toolkitId]) { + return data[toolkitId] as CustomSections; + } + + // Try normalized match + const normalizedId = normalizeId(toolkitId); + const entry = Object.entries(data).find( + ([key]) => normalizeId(key) === normalizedId + ); + + return entry ? (entry[1] as CustomSections) : null; + } + + async getAllCustomSections(): Promise< + Readonly> + > { + const data = await this.loadFile(); + return data as Record; + } +} + +// ============================================================================ +// Factory +// ============================================================================ + +export const createCustomSectionsFileSource = ( + filePath: string +): ICustomSectionsSource => new CustomSectionsFileSource({ filePath }); diff --git a/toolkit-docs-generator/src/sources/in-memory.ts b/toolkit-docs-generator/src/sources/in-memory.ts new file mode 100644 index 000000000..837e9aaf9 --- /dev/null +++ b/toolkit-docs-generator/src/sources/in-memory.ts @@ -0,0 +1,259 @@ +/** + * In-memory implementations of data sources for testing + * + * These implementations use real data structures and logic, + * avoiding mocks while enabling isolated testing. + */ +import type { + CustomSections, + ToolDefinition, + ToolkitMetadata, +} from "../types/index.js"; +import { normalizeId } from "../utils/fp.js"; +import type { ICustomSectionsSource } from "./interfaces.js"; +import type { + FetchOptions, + IMetadataSource, + IToolDataSource, +} from "./internal.js"; + +// ============================================================================ +// In-Memory Tool Data Source +// ============================================================================ + +/** + * In-memory implementation of IToolDataSource for testing + * + * Use this instead of mocking the interface. Simply provide + * realistic test data in the constructor. + * + * @example + * ```typescript + * const source = new InMemoryToolDataSource([ + * { name: 'CreateIssue', qualifiedName: 'Github.CreateIssue', ... }, + * { name: 'SetStarred', qualifiedName: 'Github.SetStarred', ... }, + * ]); + * const tools = await source.fetchToolsByToolkit('Github'); + * ``` + */ +export class InMemoryToolDataSource implements IToolDataSource { + private readonly tools: readonly ToolDefinition[]; + + constructor(tools: readonly ToolDefinition[]) { + this.tools = tools; + } + + async fetchToolsByToolkit( + toolkitId: string + ): Promise { + const normalizedId = normalizeId(toolkitId); + return this.tools.filter((tool) => { + const toolToolkitId = tool.qualifiedName.split(".")[0]; + return toolToolkitId && normalizeId(toolToolkitId) === normalizedId; + }); + } + + async fetchAllTools( + options?: FetchOptions + ): Promise { + let result = [...this.tools]; + + if (options?.toolkitId) { + const normalizedId = normalizeId(options.toolkitId); + result = result.filter((tool) => { + const toolToolkitId = tool.qualifiedName.split(".")[0]; + return toolToolkitId && normalizeId(toolToolkitId) === normalizedId; + }); + } + + if (options?.version) { + result = result.filter((tool) => { + const version = tool.fullyQualifiedName.split("@")[1]; + return version === options.version; + }); + } + + if (options?.providerId) { + result = result.filter( + (tool) => tool.auth?.providerId === options.providerId + ); + } + + return result; + } + + async isAvailable(): Promise { + return true; + } +} + +// ============================================================================ +// In-Memory Metadata Source +// ============================================================================ + +/** + * In-memory implementation of IMetadataSource for testing + * + * @example + * ```typescript + * const source = new InMemoryMetadataSource([ + * { id: 'Github', label: 'GitHub', category: 'development', ... }, + * { id: 'Slack', label: 'Slack', category: 'social', ... }, + * ]); + * const metadata = await source.getToolkitMetadata('Github'); + * ``` + */ +export class InMemoryMetadataSource implements IMetadataSource { + private readonly metadata: ReadonlyMap; + + constructor(toolkits: readonly ToolkitMetadata[]) { + const map = new Map(); + for (const toolkit of toolkits) { + // Store by exact ID + map.set(toolkit.id, toolkit); + // Also store by normalized ID for case-insensitive lookup + map.set(normalizeId(toolkit.id), toolkit); + } + this.metadata = map; + } + + async getToolkitMetadata(toolkitId: string): Promise { + // Try exact match first + const exact = this.metadata.get(toolkitId); + if (exact) return exact; + + // Try normalized match + const normalized = this.metadata.get(normalizeId(toolkitId)); + return normalized ?? null; + } + + async getAllToolkitsMetadata(): Promise { + // Return unique toolkits (filter out normalized duplicates) + const seen = new Set(); + const result: ToolkitMetadata[] = []; + + for (const [key, value] of this.metadata) { + // Only include if this is the original ID (not normalized) + if (key === value.id && !seen.has(value.id)) { + seen.add(value.id); + result.push(value); + } + } + + return result; + } + + async listToolkitIds(): Promise { + const metadata = await this.getAllToolkitsMetadata(); + return metadata.map((m) => m.id); + } +} + +// ============================================================================ +// In-Memory Custom Sections Source +// ============================================================================ + +/** + * In-memory implementation of ICustomSectionsSource for testing + * + * @example + * ```typescript + * const source = new InMemoryCustomSectionsSource({ + * Github: { + * documentationChunks: [{ type: 'warning', ... }], + * customImports: [], + * subPages: [], + * toolChunks: {}, + * }, + * }); + * const sections = await source.getCustomSections('Github'); + * ``` + */ +export class InMemoryCustomSectionsSource implements ICustomSectionsSource { + private readonly sections: ReadonlyMap; + + constructor(sections: Readonly>) { + const map = new Map(); + for (const [key, value] of Object.entries(sections)) { + map.set(key, value); + map.set(normalizeId(key), value); + } + this.sections = map; + } + + async getCustomSections(toolkitId: string): Promise { + const exact = this.sections.get(toolkitId); + if (exact) return exact; + + const normalized = this.sections.get(normalizeId(toolkitId)); + return normalized ?? null; + } + + async getAllCustomSections(): Promise< + Readonly> + > { + const result: Record = {}; + const seen = new Set(); + + for (const [key, value] of this.sections) { + // Only include original keys + if (!seen.has(normalizeId(key))) { + seen.add(normalizeId(key)); + result[key] = value; + } + } + + return result; + } +} + +// ============================================================================ +// Empty Custom Sections Source +// ============================================================================ + +/** + * Empty implementation that always returns null/empty + * Useful when custom sections are not needed + */ +export class EmptyCustomSectionsSource implements ICustomSectionsSource { + async getCustomSections(_toolkitId: string): Promise { + return null; + } + + async getAllCustomSections(): Promise< + Readonly> + > { + return {}; + } +} + +// ============================================================================ +// Factory Functions +// ============================================================================ + +/** + * Create an in-memory tool data source from test fixtures + */ +export const createInMemoryToolDataSource = ( + tools: readonly ToolDefinition[] +): IToolDataSource => new InMemoryToolDataSource(tools); + +/** + * Create an in-memory metadata source from test fixtures + */ +export const createInMemoryMetadataSource = ( + toolkits: readonly ToolkitMetadata[] +): IMetadataSource => new InMemoryMetadataSource(toolkits); + +/** + * Create an in-memory custom sections source from test fixtures + */ +export const createInMemoryCustomSectionsSource = ( + sections: Readonly> +): ICustomSectionsSource => new InMemoryCustomSectionsSource(sections); + +/** + * Create an empty custom sections source + */ +export const createEmptyCustomSectionsSource = (): ICustomSectionsSource => + new EmptyCustomSectionsSource(); diff --git a/toolkit-docs-generator/src/sources/index.ts b/toolkit-docs-generator/src/sources/index.ts new file mode 100644 index 000000000..25577c682 --- /dev/null +++ b/toolkit-docs-generator/src/sources/index.ts @@ -0,0 +1,13 @@ +/** + * Data sources exports + */ + +export * from "./custom-sections-file.js"; +export * from "./in-memory.js"; +export * from "./interfaces.js"; +export * from "./mock-engine-api.js"; +export * from "./mock-metadata.js"; +export * from "./toolkit-data-source.js"; + +// Note: Design System source will be added when the API is ready +// It will require @arcadeai/design-system package to be installed diff --git a/toolkit-docs-generator/src/sources/interfaces.ts b/toolkit-docs-generator/src/sources/interfaces.ts new file mode 100644 index 000000000..4f9255e06 --- /dev/null +++ b/toolkit-docs-generator/src/sources/interfaces.ts @@ -0,0 +1,33 @@ +/** + * Public source interfaces for the toolkit docs generator + * + */ +import type { CustomSections } from "../types/index.js"; + +// ============================================================================ +// Custom Sections Source Interface +// ============================================================================ + +/** + * Interface for fetching custom documentation sections + * + * Implementations: + * - CustomSectionsFileSource: Loads from extracted JSON file + * - EmptyCustomSectionsSource: Returns empty sections (for new toolkits) + */ +export interface ICustomSectionsSource { + /** + * Get custom sections for a specific toolkit + * @param toolkitId - The toolkit identifier + */ + readonly getCustomSections: ( + toolkitId: string + ) => Promise; + + /** + * Get all custom sections + */ + readonly getAllCustomSections: () => Promise< + Readonly> + >; +} diff --git a/toolkit-docs-generator/src/sources/internal.ts b/toolkit-docs-generator/src/sources/internal.ts new file mode 100644 index 000000000..dba59d7f9 --- /dev/null +++ b/toolkit-docs-generator/src/sources/internal.ts @@ -0,0 +1,52 @@ +/** + * Internal source interfaces + * + * These interfaces are used only inside the toolkit data source implementations. + * Do not export them from the public sources index. + */ +import type { ToolDefinition, ToolkitMetadata } from "../types/index.js"; + +// ============================================================================ +// Fetch Options +// ============================================================================ + +export interface FetchOptions { + /** Filter by toolkit ID */ + readonly toolkitId?: string; + /** Filter by toolkit version */ + readonly version?: string; + /** Filter by auth provider ID */ + readonly providerId?: string; +} + +// ============================================================================ +// Tool Data Source Interface (internal) +// ============================================================================ + +export interface IToolDataSource { + /** Fetch tools for a specific toolkit */ + readonly fetchToolsByToolkit: ( + toolkitId: string + ) => Promise; + /** Fetch all tools from the source */ + readonly fetchAllTools: ( + options?: FetchOptions + ) => Promise; + /** Check if the data source is available */ + readonly isAvailable: () => Promise; +} + +// ============================================================================ +// Metadata Source Interface (internal) +// ============================================================================ + +export interface IMetadataSource { + /** Get metadata for a specific toolkit */ + readonly getToolkitMetadata: ( + toolkitId: string + ) => Promise; + /** Get all toolkit metadata */ + readonly getAllToolkitsMetadata: () => Promise; + /** List all available toolkit IDs */ + readonly listToolkitIds: () => Promise; +} diff --git a/toolkit-docs-generator/src/sources/mock-engine-api.ts b/toolkit-docs-generator/src/sources/mock-engine-api.ts new file mode 100644 index 000000000..1bbe66b65 --- /dev/null +++ b/toolkit-docs-generator/src/sources/mock-engine-api.ts @@ -0,0 +1,214 @@ +/** + * Mock Engine API Source + * + * This source loads tool definitions from JSON fixtures, simulating + * what the real Engine API will return. Replace with EngineApiSource + * when the API endpoint is ready. + */ +import { readFile } from "fs/promises"; +import { z } from "zod"; +import type { ToolDefinition, ToolParameter } from "../types/index.js"; +import { normalizeId } from "../utils/fp.js"; +import type { FetchOptions, IToolDataSource } from "./internal.js"; + +// ============================================================================ +// Engine API Response Schemas (matches planningdoc.md spec) +// ============================================================================ + +const ApiValueSchemaSchema = z.object({ + val_type: z.string(), + inner_val_type: z.string().nullable(), + enum: z.array(z.string()).nullable(), +}); + +const ApiParameterSchema = z.object({ + name: z.string(), + required: z.boolean(), + description: z.string().nullable(), + value_schema: ApiValueSchemaSchema, + inferrable: z.boolean().default(true), +}); + +const ApiToolkitInfoSchema = z.object({ + name: z.string(), + version: z.string(), + description: z.string().nullable(), +}); + +const ApiInputSchema = z.object({ + parameters: z.array(ApiParameterSchema), +}); + +const ApiOutputSchema = z + .object({ + available_modes: z.array(z.string()).optional(), + description: z.string().nullable(), + value_schema: ApiValueSchemaSchema.nullable(), + }) + .nullable(); + +const ApiAuthRequirementSchema = z + .object({ + id: z.string(), + provider_id: z.string().nullable(), + provider_type: z.string(), + scopes: z.array(z.string()).nullable(), + }) + .nullable(); + +const ApiRequirementsSchema = z + .object({ + authorization: ApiAuthRequirementSchema, + secrets: z.array(z.string()).nullable(), + }) + .nullable(); + +const ApiToolSchema = z.object({ + fully_qualified_name: z.string(), + qualified_name: z.string(), + name: z.string(), + description: z.string().nullable(), + toolkit: ApiToolkitInfoSchema, + input: ApiInputSchema, + output: ApiOutputSchema, + requirements: ApiRequirementsSchema, +}); + +const ApiResponseSchema = z.object({ + items: z.array(ApiToolSchema), + total_count: z.number(), +}); + +type ApiTool = z.infer; + +// ============================================================================ +// Transform API response to internal types +// ============================================================================ + +const transformParameter = ( + apiParam: z.infer +): ToolParameter => ({ + name: apiParam.name, + type: apiParam.value_schema.val_type, + innerType: apiParam.value_schema.inner_val_type ?? undefined, + required: apiParam.required, + description: apiParam.description, + enum: apiParam.value_schema.enum, + inferrable: apiParam.inferrable, +}); + +const transformTool = (apiTool: ApiTool): ToolDefinition => ({ + name: apiTool.name, + qualifiedName: apiTool.qualified_name, + fullyQualifiedName: apiTool.fully_qualified_name, + description: apiTool.description, + parameters: apiTool.input.parameters.map(transformParameter), + auth: apiTool.requirements?.authorization + ? { + providerId: apiTool.requirements.authorization.provider_id, + providerType: apiTool.requirements.authorization.provider_type, + scopes: apiTool.requirements.authorization.scopes ?? [], + } + : null, + secrets: apiTool.requirements?.secrets ?? [], + output: apiTool.output + ? { + type: apiTool.output.value_schema?.val_type ?? "unknown", + description: apiTool.output.description, + } + : null, +}); + +// ============================================================================ +// Mock Engine API Source +// ============================================================================ + +export interface MockEngineApiConfig { + /** Path to the JSON fixture file */ + fixtureFilePath: string; +} + +/** + * Mock implementation of IToolDataSource that loads from JSON fixtures + * + * Use this until the real Engine API endpoint is available. + * The fixture format matches the expected API response schema. + */ +export class MockEngineApiSource implements IToolDataSource { + private readonly fixtureFilePath: string; + private cachedData: ToolDefinition[] | null = null; + + constructor(config: MockEngineApiConfig) { + this.fixtureFilePath = config.fixtureFilePath; + } + + private async loadFixture(): Promise { + if (this.cachedData !== null) { + return this.cachedData; + } + + const content = await readFile(this.fixtureFilePath, "utf-8"); + const json = JSON.parse(content); + const parsed = ApiResponseSchema.parse(json); + this.cachedData = parsed.items.map(transformTool); + return this.cachedData; + } + + async fetchToolsByToolkit( + toolkitId: string + ): Promise { + const tools = await this.loadFixture(); + const normalizedId = normalizeId(toolkitId); + + return tools.filter((tool) => { + const toolToolkitId = tool.qualifiedName.split(".")[0]; + return toolToolkitId && normalizeId(toolToolkitId) === normalizedId; + }); + } + + async fetchAllTools( + options?: FetchOptions + ): Promise { + let tools = await this.loadFixture(); + + if (options?.toolkitId) { + const normalizedId = normalizeId(options.toolkitId); + tools = tools.filter((tool) => { + const toolToolkitId = tool.qualifiedName.split(".")[0]; + return toolToolkitId && normalizeId(toolToolkitId) === normalizedId; + }); + } + + if (options?.version) { + tools = tools.filter((tool) => { + const version = tool.fullyQualifiedName.split("@")[1]; + return version === options.version; + }); + } + + if (options?.providerId) { + tools = tools.filter( + (tool) => tool.auth?.providerId === options.providerId + ); + } + + return tools; + } + + async isAvailable(): Promise { + try { + await this.loadFixture(); + return true; + } catch { + return false; + } + } +} + +// ============================================================================ +// Factory +// ============================================================================ + +export const createMockEngineApiSource = ( + fixtureFilePath: string +): IToolDataSource => new MockEngineApiSource({ fixtureFilePath }); diff --git a/toolkit-docs-generator/src/sources/mock-metadata.ts b/toolkit-docs-generator/src/sources/mock-metadata.ts new file mode 100644 index 000000000..68bc44a3f --- /dev/null +++ b/toolkit-docs-generator/src/sources/mock-metadata.ts @@ -0,0 +1,114 @@ +/** + * Mock Metadata Source + * + * This source loads toolkit metadata from a JSON fixture file, + * simulating what the Design System package provides. + * Use this until the Design System package is properly installed. + */ +import { access, readFile } from "fs/promises"; +import { z } from "zod"; +import type { ToolkitMetadata } from "../types/index.js"; +import { ToolkitMetadataSchema } from "../types/index.js"; +import { normalizeId } from "../utils/fp.js"; +import type { IMetadataSource } from "./internal.js"; + +// ============================================================================ +// File Schema +// ============================================================================ + +const MetadataFileSchema = z.record(z.string(), ToolkitMetadataSchema); + +type MetadataFile = z.infer; + +// ============================================================================ +// Mock Metadata Source +// ============================================================================ + +export interface MockMetadataConfig { + /** Path to the JSON fixture file */ + fixtureFilePath: string; +} + +/** + * Mock implementation of IMetadataSource that loads from JSON fixtures + */ +export class MockMetadataSource implements IMetadataSource { + private readonly fixtureFilePath: string; + private cachedData: MetadataFile | null = null; + private normalizedIndex: Map | null = null; + + constructor(config: MockMetadataConfig) { + this.fixtureFilePath = config.fixtureFilePath; + } + + private async loadFixture(): Promise { + if (this.cachedData !== null) { + return this.cachedData; + } + + try { + await access(this.fixtureFilePath); + const content = await readFile(this.fixtureFilePath, "utf-8"); + const json = JSON.parse(content); + this.cachedData = MetadataFileSchema.parse(json); + this.normalizedIndex = this.buildNormalizedIndex(this.cachedData); + return this.cachedData; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + this.cachedData = {}; + this.normalizedIndex = new Map(); + return this.cachedData; + } + throw error; + } + } + + private buildNormalizedIndex( + data: MetadataFile + ): Map { + const index = new Map(); + for (const [_, metadata] of Object.entries(data)) { + const normalized = normalizeId(metadata.id); + index.set(normalized, metadata); + } + return index; + } + + async getToolkitMetadata(toolkitId: string): Promise { + const data = await this.loadFixture(); + + // Try exact match first + const exact = data[toolkitId]; + if (exact) { + return exact; + } + + // Try normalized match + if (this.normalizedIndex) { + const normalized = this.normalizedIndex.get(normalizeId(toolkitId)); + if (normalized) { + return normalized; + } + } + + return null; + } + + async getAllToolkitsMetadata(): Promise { + const data = await this.loadFixture(); + return Object.values(data) as ToolkitMetadata[]; + } + + async listToolkitIds(): Promise { + const data = await this.loadFixture(); + return Object.keys(data); + } +} + +// ============================================================================ +// Factory +// ============================================================================ + +export const createMockMetadataSource = ( + fixtureFilePath: string +): IMetadataSource => new MockMetadataSource({ fixtureFilePath }); diff --git a/toolkit-docs-generator/src/sources/toolkit-data-source.ts b/toolkit-docs-generator/src/sources/toolkit-data-source.ts new file mode 100644 index 000000000..b547942b8 --- /dev/null +++ b/toolkit-docs-generator/src/sources/toolkit-data-source.ts @@ -0,0 +1,208 @@ +/** + * Unified Toolkit Data Source + * + * This abstraction combines tool definitions and metadata into a single interface. + * This allows us to swap implementations easily when the data comes from a single source + * in the future (e.g., when Engine API includes metadata). + */ + +import { join } from "path"; +import type { ToolDefinition, ToolkitMetadata } from "../types/index.js"; +import type { IMetadataSource, IToolDataSource } from "./internal.js"; +import { createMockEngineApiSource } from "./mock-engine-api.js"; +import { createMockMetadataSource } from "./mock-metadata.js"; + +// ============================================================================ +// Unified Toolkit Data Interface +// ============================================================================ + +/** + * Combined toolkit data containing both tools and metadata + */ +export interface ToolkitData { + /** Tool definitions from Engine API */ + readonly tools: readonly ToolDefinition[]; + /** Metadata from Design System */ + readonly metadata: ToolkitMetadata | null; +} + +// ============================================================================ +// Unified Toolkit Data Source Interface +// ============================================================================ + +/** + * Interface for fetching combined toolkit data (tools + metadata) + * + * This abstraction allows us to: + * 1. Currently combine separate Engine API and Design System sources + * 2. Future: Use a single unified source when Engine API includes metadata + * + * Implementations: + * - CombinedToolkitDataSource: Combines IToolDataSource + IMetadataSource + * - UnifiedToolkitDataSource: Single source (future implementation) + */ +export interface IToolkitDataSource { + /** + * Fetch combined data for a specific toolkit + * @param toolkitId - The toolkit identifier (e.g., "Github", "Slack") + * @param version - Optional version filter + */ + readonly fetchToolkitData: ( + toolkitId: string, + version?: string + ) => Promise; + + /** + * Fetch combined data for all toolkits + */ + readonly fetchAllToolkitsData: () => Promise< + ReadonlyMap + >; + + /** + * Check if the data source is available + */ + readonly isAvailable: () => Promise; +} + +// ============================================================================ +// Combined Implementation (Current: Separate Sources) +// ============================================================================ + +/** + * Configuration for combined toolkit data source + */ +export interface CombinedToolkitDataSourceConfig { + /** Source for tool definitions */ + readonly toolSource: IToolDataSource; + /** Source for toolkit metadata */ + readonly metadataSource: IMetadataSource; +} + +/** + * Combined implementation that merges separate tool and metadata sources + * + * This is the current implementation that combines: + * - Engine API (via IToolDataSource) + * - Design System (via IMetadataSource) + * + * In the future, this can be replaced with UnifiedToolkitDataSource + * when Engine API includes metadata. + */ +export class CombinedToolkitDataSource implements IToolkitDataSource { + private readonly toolSource: IToolDataSource; + private readonly metadataSource: IMetadataSource; + + constructor(config: CombinedToolkitDataSourceConfig) { + this.toolSource = config.toolSource; + this.metadataSource = config.metadataSource; + } + + async fetchToolkitData( + toolkitId: string, + version?: string + ): Promise { + // Fetch tools and metadata in parallel + const [tools, metadata] = await Promise.all([ + this.toolSource.fetchToolsByToolkit(toolkitId), + this.metadataSource.getToolkitMetadata(toolkitId), + ]); + + // Filter tools by version if specified + const filteredTools = version + ? tools.filter((tool) => { + const toolVersion = tool.fullyQualifiedName.split("@")[1]; + return toolVersion === version; + }) + : tools; + + return { + tools: filteredTools, + metadata, + }; + } + + async fetchAllToolkitsData(): Promise> { + // Fetch all tools and metadata in parallel + const [allTools, allMetadata] = await Promise.all([ + this.toolSource.fetchAllTools(), + this.metadataSource.getAllToolkitsMetadata(), + ]); + + // Group tools by toolkit ID + const toolkitGroups = new Map(); + for (const tool of allTools) { + const toolkitId = tool.qualifiedName.split(".")[0]; + if (toolkitId) { + const existing = toolkitGroups.get(toolkitId) ?? []; + toolkitGroups.set(toolkitId, [...existing, tool]); + } + } + + // Create metadata lookup map + const metadataMap = new Map(); + for (const metadata of allMetadata) { + metadataMap.set(metadata.id, metadata); + } + + // Combine into ToolkitData map + const result = new Map(); + for (const [toolkitId, tools] of toolkitGroups) { + result.set(toolkitId, { + tools, + metadata: metadataMap.get(toolkitId) ?? null, + }); + } + + return result; + } + + async isAvailable(): Promise { + const [toolAvailable, metadataAvailable] = await Promise.all([ + this.toolSource.isAvailable(), + Promise.resolve(true), // Metadata source is always available (returns null if not found) + ]); + return toolAvailable && metadataAvailable; + } +} + +// ============================================================================ +// Factory +// ============================================================================ + +/** + * Create a combined toolkit data source from separate sources + */ +export const createCombinedToolkitDataSource = ( + config: CombinedToolkitDataSourceConfig +): IToolkitDataSource => new CombinedToolkitDataSource(config); + +// ============================================================================ +// Mock Toolkit Data Source (Current: JSON fixtures) +// ============================================================================ + +/** + * Configuration for mock toolkit data source + */ +export interface MockToolkitDataSourceConfig { + /** Directory containing mock data fixtures */ + readonly dataDir: string; +} + +/** + * Create a mock toolkit data source using JSON fixtures. + * + * This hides the fact that tools and metadata come from separate sources, + * while keeping a single abstraction for the rest of the system. + */ +export const createMockToolkitDataSource = ( + config: MockToolkitDataSourceConfig +): IToolkitDataSource => { + const toolFixturePath = join(config.dataDir, "engine-api-response.json"); + const metadataFixturePath = join(config.dataDir, "metadata.json"); + + return createCombinedToolkitDataSource({ + toolSource: createMockEngineApiSource(toolFixturePath), + metadataSource: createMockMetadataSource(metadataFixturePath), + }); +}; diff --git a/toolkit-docs-generator/src/types/index.ts b/toolkit-docs-generator/src/types/index.ts new file mode 100644 index 000000000..1740ae39d --- /dev/null +++ b/toolkit-docs-generator/src/types/index.ts @@ -0,0 +1,400 @@ +/** + * Core type definitions for the toolkit docs generator + */ +import { z } from "zod"; + +// ============================================================================ +// CLI Input Types +// ============================================================================ + +/** + * Input format for specifying which providers/toolkits to process + * Format: "Provider:version" e.g., "Github:1.0.0" + */ +export const ProviderVersionSchema = z.object({ + provider: z.string().min(1), + version: z.string().optional(), // If not provided, use latest +}); + +export type ProviderVersion = z.infer; + +export const GenerateInputSchema = z.object({ + providers: z.array(ProviderVersionSchema).min(1), + outputDir: z.string().default("./output"), + skipExamples: z.boolean().default(false), + skipSummary: z.boolean().default(false), + customSectionsFile: z.string().optional(), +}); + +export type GenerateInput = z.infer; + +// ============================================================================ +// Tool Parameter Schema +// ============================================================================ + +export const ToolParameterSchema = z.object({ + name: z.string(), + type: z.string(), + innerType: z.string().optional(), + required: z.boolean(), + description: z.string().nullable(), + enum: z.array(z.string()).nullable(), + inferrable: z.boolean().default(true), +}); + +export type ToolParameter = z.infer; + +// ============================================================================ +// Tool Auth Schema +// ============================================================================ + +export const ToolAuthSchema = z.object({ + providerId: z.string().nullable(), + providerType: z.string(), + scopes: z.array(z.string()), +}); + +export type ToolAuth = z.infer; + +// ============================================================================ +// Tool Output Schema +// ============================================================================ + +export const ToolOutputSchema = z.object({ + type: z.string(), + description: z.string().nullable(), +}); + +export type ToolOutput = z.infer; + +// ============================================================================ +// Tool Secrets Schema +// ============================================================================ + +export const SecretTypeSchema = z.enum([ + "api_key", + "token", + "client_secret", + "webhook_secret", + "private_key", + "password", + "unknown", +]); + +export type SecretType = z.infer; + +export const ToolSecretSchema = z.object({ + name: z.string(), + type: SecretTypeSchema, +}); + +export type ToolSecret = z.infer; + +// ============================================================================ +// Tool Definition Schema (from Engine API) +// ============================================================================ + +export const ToolDefinitionSchema = z.object({ + name: z.string(), + qualifiedName: z.string(), + fullyQualifiedName: z.string(), + description: z.string().nullable(), + parameters: z.array(ToolParameterSchema), + auth: ToolAuthSchema.nullable(), + secrets: z.array(z.string()), + output: ToolOutputSchema.nullable(), +}); + +export type ToolDefinition = z.infer; + +// ============================================================================ +// Toolkit Metadata Schema (from Design System) +// ============================================================================ + +export const ToolkitCategorySchema = z.enum([ + "productivity", + "social", + "development", + "entertainment", + "search", + "payments", + "sales", + "databases", + "customer-support", +]); + +export type ToolkitCategory = z.infer; + +export const ToolkitTypeSchema = z.enum([ + "arcade", + "arcade_starter", + "verified", + "community", + "auth", +]); + +export type ToolkitType = z.infer; + +export const ToolkitMetadataSchema = z.object({ + id: z.string(), + label: z.string(), + category: ToolkitCategorySchema, + iconUrl: z.string(), + isBYOC: z.boolean(), + isPro: z.boolean(), + type: ToolkitTypeSchema, + docsLink: z.string(), + isComingSoon: z.boolean(), + isHidden: z.boolean(), +}); + +export type ToolkitMetadata = z.infer; + +// ============================================================================ +// Documentation Chunk Schema (for custom content injection) +// ============================================================================ + +/** + * Type of documentation chunk content + * - callout: Warning, info, or tip box + * - markdown: Raw markdown content + * - code: Code block with language + * - warning: Highlighted warning message + * - info: Informational note + * - tip: Helpful tip + */ +export const DocumentationChunkTypeSchema = z.enum([ + "callout", + "markdown", + "code", + "warning", + "info", + "tip", +]); + +export type DocumentationChunkType = z.infer< + typeof DocumentationChunkTypeSchema +>; + +/** + * Location where the chunk should be injected + * - header: After the toolkit header, before tools list + * - description: Around the tool description + * - parameters: Around the parameters section + * - auth: Around the auth/scopes section + * - secrets: Around the secrets section + * - output: Around the output section + * - footer: After all tools, before the footer + */ +export const DocumentationChunkLocationSchema = z.enum([ + "header", + "description", + "parameters", + "auth", + "secrets", + "output", + "footer", +]); + +export type DocumentationChunkLocation = z.infer< + typeof DocumentationChunkLocationSchema +>; + +/** + * Position relative to the location + */ +export const DocumentationChunkPositionSchema = z.enum([ + "before", + "after", + "replace", +]); + +export type DocumentationChunkPosition = z.infer< + typeof DocumentationChunkPositionSchema +>; + +/** + * A documentation chunk represents custom content to inject into docs + */ +export const DocumentationChunkSchema = z.object({ + /** Type of content */ + type: DocumentationChunkTypeSchema, + /** Where to inject the content */ + location: DocumentationChunkLocationSchema, + /** Position relative to location (before, after, replace) */ + position: DocumentationChunkPositionSchema, + /** The actual content (markdown string) */ + content: z.string(), + /** Optional title for callouts */ + title: z.string().optional(), + /** Optional variant for styling (e.g., "destructive" for warnings) */ + variant: z + .enum(["default", "destructive", "warning", "info", "success"]) + .optional(), +}); + +export type DocumentationChunk = z.infer; + +// ============================================================================ +// Tool Code Example Schema (for generating example code) +// ============================================================================ + +/** + * Parameter value with type information for code generation + */ +export const ExampleParameterValueSchema = z.object({ + /** The example value to use in code */ + value: z.unknown(), + /** Parameter type */ + type: z.enum(["string", "integer", "boolean", "array", "object"]), + /** Whether this parameter is required */ + required: z.boolean(), +}); + +export type ExampleParameterValue = z.infer; + +/** + * Tool code example configuration + * Used to generate Python/JavaScript example code + */ +export const ToolCodeExampleSchema = z.object({ + /** Full tool name (e.g., "Github.SetStarred") */ + toolName: z.string(), + /** Parameter values with type info */ + parameters: z.record(z.string(), ExampleParameterValueSchema), + /** Whether this tool requires user authorization */ + requiresAuth: z.boolean(), + /** Auth provider ID if auth is required */ + authProvider: z.string().optional(), + /** Optional tab label for the code example */ + tabLabel: z.string().optional(), +}); + +export type ToolCodeExample = z.infer; + +// ============================================================================ +// Custom Sections Schema (extracted from MDX) +// ============================================================================ + +export const CustomSectionsSchema = z.object({ + /** Toolkit-level documentation chunks */ + documentationChunks: z.array(DocumentationChunkSchema).default([]), + /** Custom imports needed for the MDX file */ + customImports: z.array(z.string()).default([]), + /** Sub-pages that exist for this toolkit */ + subPages: z.array(z.string()).default([]), + /** Per-tool documentation chunks (keyed by tool name) */ + toolChunks: z + .record(z.string(), z.array(DocumentationChunkSchema)) + .default({}), +}); + +export type CustomSections = z.infer; + +// ============================================================================ +// Merged Tool Schema (output format) +// ============================================================================ + +export const MergedToolSchema = z.object({ + name: z.string(), + qualifiedName: z.string(), + fullyQualifiedName: z.string(), + description: z.string().nullable(), + parameters: z.array(ToolParameterSchema), + auth: ToolAuthSchema.nullable(), + secrets: z.array(z.string()), + secretsInfo: z.array(ToolSecretSchema).default([]), + output: ToolOutputSchema.nullable(), + /** Custom documentation chunks for this tool */ + documentationChunks: z.array(DocumentationChunkSchema).default([]), + /** Generated code example configuration */ + codeExample: ToolCodeExampleSchema.optional(), +}); + +export type MergedTool = z.infer; + +// ============================================================================ +// Merged Toolkit Schema (output format) +// ============================================================================ + +export const ToolkitAuthTypeSchema = z.enum([ + "oauth2", + "api_key", + "mixed", + "none", +]); + +export type ToolkitAuthType = z.infer; + +export const MergedToolkitMetadataSchema = z.object({ + category: ToolkitCategorySchema, + iconUrl: z.string(), + isBYOC: z.boolean(), + isPro: z.boolean(), + type: ToolkitTypeSchema, + docsLink: z.string(), + isComingSoon: z.boolean(), + isHidden: z.boolean(), +}); + +export type MergedToolkitMetadata = z.infer; + +export const MergedToolkitAuthSchema = z.object({ + type: ToolkitAuthTypeSchema, + providerId: z.string().nullable(), + allScopes: z.array(z.string()), +}); + +export type MergedToolkitAuth = z.infer; + +export const MergedToolkitSchema = z.object({ + /** Unique toolkit ID (e.g., "Github") */ + id: z.string(), + /** Human-readable label (e.g., "GitHub") */ + label: z.string(), + /** Toolkit version (e.g., "1.0.0") */ + version: z.string(), + /** Toolkit description */ + description: z.string().nullable(), + /** LLM-generated summary (optional) */ + summary: z.string().optional(), + /** Metadata from Design System */ + metadata: MergedToolkitMetadataSchema, + /** Authentication requirements */ + auth: MergedToolkitAuthSchema.nullable(), + /** All tools in this toolkit */ + tools: z.array(MergedToolSchema), + /** Toolkit-level documentation chunks */ + documentationChunks: z.array(DocumentationChunkSchema).default([]), + /** Custom imports for MDX */ + customImports: z.array(z.string()).default([]), + /** Sub-pages that exist for this toolkit */ + subPages: z.array(z.string()).default([]), + /** Generation metadata */ + generatedAt: z.string().optional(), +}); + +export type MergedToolkit = z.infer; + +// ============================================================================ +// Index Output Schema +// ============================================================================ + +export const ToolkitIndexEntrySchema = z.object({ + id: z.string(), + label: z.string(), + version: z.string(), + category: ToolkitCategorySchema, + toolCount: z.number(), + authType: ToolkitAuthTypeSchema, +}); + +export type ToolkitIndexEntry = z.infer; + +export const ToolkitIndexSchema = z.object({ + generatedAt: z.string(), + version: z.string(), + toolkits: z.array(ToolkitIndexEntrySchema), +}); + +export type ToolkitIndex = z.infer; diff --git a/toolkit-docs-generator/src/utils/fp.ts b/toolkit-docs-generator/src/utils/fp.ts new file mode 100644 index 000000000..19061323c --- /dev/null +++ b/toolkit-docs-generator/src/utils/fp.ts @@ -0,0 +1,184 @@ +/** + * Functional programming utilities + */ + +/** + * Pipe a value through a series of functions + * @example + * pipe(5, double, addOne) // 11 + */ +export const pipe = (value: T, ...fns: Array<(arg: T) => T>): T => + fns.reduce((acc, fn) => fn(acc), value); + +/** + * Compose functions from right to left + * @example + * const doubleAndAddOne = compose(addOne, double); + * doubleAndAddOne(5) // 11 + */ +export const compose = + (...fns: Array<(arg: T) => T>) => + (value: T): T => + fns.reduceRight((acc, fn) => fn(acc), value); + +/** + * Create a function that always returns the same value + */ +export const constant = + (value: T) => + (): T => + value; + +/** + * Identity function - returns its argument unchanged + */ +export const identity = (value: T): T => value; + +/** + * Safely get a property from an object + */ +export const prop = + (key: K) => + (obj: T): T[K] => + obj[key]; + +/** + * Check if a value is not null or undefined + */ +export const isNotNullish = (value: T | null | undefined): value is T => + value !== null && value !== undefined; + +/** + * Filter out null and undefined values from an array + */ +export const filterNullish = (arr: readonly (T | null | undefined)[]): T[] => + arr.filter(isNotNullish); + +/** + * Group array items by a key function + */ +export const groupBy = ( + arr: readonly T[], + keyFn: (item: T) => K +): Record => + arr.reduce( + (acc, item) => { + const key = keyFn(item); + const existing = acc[key] ?? []; + return { ...acc, [key]: [...existing, item] }; + }, + {} as Record + ); + +/** + * Create a unique array by a key function + */ +export const uniqueBy = ( + arr: readonly T[], + keyFn: (item: T) => K +): T[] => { + const seen = new Set(); + return arr.filter((item) => { + const key = keyFn(item); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +}; + +/** + * Flatten an array of arrays + */ +export const flatten = (arr: readonly (readonly T[])[]): T[] => + arr.flatMap(identity); + +/** + * Map over an object's values + */ +export const mapValues = ( + obj: Readonly>, + fn: (value: T, key: string) => U +): Record => + Object.fromEntries( + Object.entries(obj).map(([key, value]) => [key, fn(value, key)]) + ); + +/** + * Pick specified keys from an object + */ +export const pick = ( + obj: T, + keys: readonly K[] +): Pick => + keys.reduce((acc, key) => ({ ...acc, [key]: obj[key] }), {} as Pick); + +/** + * Omit specified keys from an object + */ +export const omit = ( + obj: T, + keys: readonly K[] +): Omit => { + const keysSet = new Set(keys); + return Object.fromEntries( + Object.entries(obj).filter(([key]) => !keysSet.has(key as keyof T)) + ) as Omit; +}; + +/** + * Deep merge two objects + */ +export const deepMerge = ( + target: T, + source: Partial +): T => { + const result = { ...target }; + + for (const key in source) { + const sourceValue = source[key]; + const targetValue = result[key]; + + if ( + sourceValue !== undefined && + typeof sourceValue === "object" && + sourceValue !== null && + !Array.isArray(sourceValue) && + typeof targetValue === "object" && + targetValue !== null && + !Array.isArray(targetValue) + ) { + (result as Record)[key] = deepMerge( + targetValue as object, + sourceValue as object + ); + } else if (sourceValue !== undefined) { + (result as Record)[key] = sourceValue; + } + } + + return result; +}; + +/** + * Normalize a string for comparison (lowercase, remove hyphens/underscores) + */ +export const normalizeId = (id: string): string => + id.toLowerCase().replace(/[-_]/g, ""); + +/** + * Convert PascalCase to snake_case + */ +export const pascalToSnakeCase = (str: string): string => + str + .replace(/([A-Z])/g, "_$1") + .toLowerCase() + .replace(/^_/, ""); + +/** + * Convert snake_case to PascalCase + */ +export const snakeToPascalCase = (str: string): string => + str + .split("_") + .map((part) => part.charAt(0).toUpperCase() + part.slice(1).toLowerCase()) + .join(""); diff --git a/toolkit-docs-generator/src/utils/index.ts b/toolkit-docs-generator/src/utils/index.ts new file mode 100644 index 000000000..a8422a3bc --- /dev/null +++ b/toolkit-docs-generator/src/utils/index.ts @@ -0,0 +1,4 @@ +/** + * Utility exports + */ +export * from "./fp.js"; diff --git a/toolkit-docs-generator/tests/fixtures/engine-api-response.json b/toolkit-docs-generator/tests/fixtures/engine-api-response.json new file mode 100644 index 000000000..8161a95dc --- /dev/null +++ b/toolkit-docs-generator/tests/fixtures/engine-api-response.json @@ -0,0 +1,335 @@ +{ + "items": [ + { + "fully_qualified_name": "Github.CreateIssue@1.0.0", + "qualified_name": "Github.CreateIssue", + "name": "CreateIssue", + "description": "Create a new issue in a GitHub repository. Optionally add the issue to a project by specifying the project number.", + "toolkit": { + "name": "Github", + "version": "1.0.0", + "description": "GitHub repository and collaboration tools" + }, + "input": { + "parameters": [ + { + "name": "owner", + "required": true, + "description": "The owner (user or organization) of the repository", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "repo", + "required": true, + "description": "The name of the repository", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "title", + "required": true, + "description": "The title of the issue", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "body", + "required": false, + "description": "The body content of the issue (supports Markdown)", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "labels", + "required": false, + "description": "Labels to add to the issue", + "value_schema": { + "val_type": "array", + "inner_val_type": "string", + "enum": null + }, + "inferrable": true + } + ] + }, + "output": { + "available_modes": ["value"], + "description": "The created issue object with id, number, url, and other details", + "value_schema": { + "val_type": "object", + "inner_val_type": null, + "enum": null + } + }, + "requirements": { + "authorization": { + "id": "github_oauth", + "provider_id": "github", + "provider_type": "oauth2", + "scopes": ["repo"] + }, + "secrets": null + } + }, + { + "fully_qualified_name": "Github.SetStarred@1.0.0", + "qualified_name": "Github.SetStarred", + "name": "SetStarred", + "description": "Star or unstar a GitHub repository for the authenticated user.", + "toolkit": { + "name": "Github", + "version": "1.0.0", + "description": "GitHub repository and collaboration tools" + }, + "input": { + "parameters": [ + { + "name": "owner", + "required": true, + "description": "The owner of the repository to star/unstar", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "repo", + "required": true, + "description": "The name of the repository to star/unstar", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "starred", + "required": true, + "description": "True to star the repository, false to unstar it", + "value_schema": { + "val_type": "boolean", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + } + ] + }, + "output": { + "available_modes": ["value"], + "description": "Confirmation of the star/unstar action", + "value_schema": { + "val_type": "object", + "inner_val_type": null, + "enum": null + } + }, + "requirements": { + "authorization": { + "id": "github_oauth", + "provider_id": "github", + "provider_type": "oauth2", + "scopes": ["public_repo"] + }, + "secrets": null + } + }, + { + "fully_qualified_name": "Github.ListPullRequests@1.0.0", + "qualified_name": "Github.ListPullRequests", + "name": "ListPullRequests", + "description": "List pull requests in a GitHub repository with optional filtering.", + "toolkit": { + "name": "Github", + "version": "1.0.0", + "description": "GitHub repository and collaboration tools" + }, + "input": { + "parameters": [ + { + "name": "owner", + "required": true, + "description": "The owner of the repository", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "repo", + "required": true, + "description": "The name of the repository", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "state", + "required": false, + "description": "Filter by pull request state", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": ["open", "closed", "all"] + }, + "inferrable": true + }, + { + "name": "per_page", + "required": false, + "description": "Number of results per page (max 100)", + "value_schema": { + "val_type": "integer", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + } + ] + }, + "output": { + "available_modes": ["value"], + "description": "List of pull request objects", + "value_schema": { + "val_type": "array", + "inner_val_type": null, + "enum": null + } + }, + "requirements": { + "authorization": { + "id": "github_oauth", + "provider_id": "github", + "provider_type": "oauth2", + "scopes": ["repo"] + }, + "secrets": null + } + }, + { + "fully_qualified_name": "Slack.SendMessage@1.2.0", + "qualified_name": "Slack.SendMessage", + "name": "SendMessage", + "description": "Send a message to a Slack channel or direct message.", + "toolkit": { + "name": "Slack", + "version": "1.2.0", + "description": "Slack communication tools" + }, + "input": { + "parameters": [ + { + "name": "channel_id", + "required": true, + "description": "The ID of the channel or conversation", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + }, + { + "name": "text", + "required": true, + "description": "The message text", + "value_schema": { + "val_type": "string", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + } + ] + }, + "output": { + "available_modes": ["value"], + "description": "Message send confirmation", + "value_schema": { + "val_type": "object", + "inner_val_type": null, + "enum": null + } + }, + "requirements": { + "authorization": { + "id": "slack_oauth", + "provider_id": "slack", + "provider_type": "oauth2", + "scopes": ["chat:write"] + }, + "secrets": null + } + }, + { + "fully_qualified_name": "Slack.ListChannels@1.2.0", + "qualified_name": "Slack.ListChannels", + "name": "ListChannels", + "description": "List channels in the workspace.", + "toolkit": { + "name": "Slack", + "version": "1.2.0", + "description": "Slack communication tools" + }, + "input": { + "parameters": [ + { + "name": "limit", + "required": false, + "description": "Maximum number of channels to return", + "value_schema": { + "val_type": "integer", + "inner_val_type": null, + "enum": null + }, + "inferrable": true + } + ] + }, + "output": { + "available_modes": ["value"], + "description": "List of channel objects", + "value_schema": { + "val_type": "array", + "inner_val_type": null, + "enum": null + } + }, + "requirements": { + "authorization": { + "id": "slack_oauth", + "provider_id": "slack", + "provider_type": "oauth2", + "scopes": ["channels:read"] + }, + "secrets": null + } + } + ], + "total_count": 5 +} diff --git a/toolkit-docs-generator/tests/fixtures/github-toolkit.json b/toolkit-docs-generator/tests/fixtures/github-toolkit.json new file mode 100644 index 000000000..bf89775f7 --- /dev/null +++ b/toolkit-docs-generator/tests/fixtures/github-toolkit.json @@ -0,0 +1,459 @@ +{ + "id": "Github", + "label": "GitHub", + "version": "1.0.0", + "description": "GitHub repository and collaboration tools for managing repositories, issues, pull requests, and more.", + "summary": "The Arcade GitHub toolkit provides a comprehensive set of tools for interacting with GitHub. These tools make it easy to build agents and AI apps that can:\n\n- Create and manage repositories, issues, and pull requests\n- Star and unstar repositories\n- Search for code, issues, and repositories\n- Manage repository collaborators and permissions\n- Access user profile information", + "metadata": { + "category": "development", + "iconUrl": "https://design-system.arcade.dev/icons/github.svg", + "isBYOC": false, + "isPro": false, + "type": "arcade", + "docsLink": "https://docs.arcade.dev/en/mcp-servers/development/github", + "isComingSoon": false, + "isHidden": false + }, + "auth": { + "type": "oauth2", + "providerId": "github", + "allScopes": ["repo", "user:email", "read:user", "public_repo"] + }, + "tools": [ + { + "name": "CreateIssue", + "qualifiedName": "Github.CreateIssue", + "fullyQualifiedName": "Github.CreateIssue@1.0.0", + "description": "Create a new issue in a GitHub repository. Optionally add the issue to a project by specifying the project number.", + "parameters": [ + { + "name": "owner", + "type": "string", + "required": true, + "description": "The owner (user or organization) of the repository", + "enum": null, + "inferrable": true + }, + { + "name": "repo", + "type": "string", + "required": true, + "description": "The name of the repository", + "enum": null, + "inferrable": true + }, + { + "name": "title", + "type": "string", + "required": true, + "description": "The title of the issue", + "enum": null, + "inferrable": true + }, + { + "name": "body", + "type": "string", + "required": false, + "description": "The body content of the issue (supports Markdown)", + "enum": null, + "inferrable": true + }, + { + "name": "labels", + "type": "array", + "innerType": "string", + "required": false, + "description": "Labels to add to the issue", + "enum": null, + "inferrable": true + }, + { + "name": "assignees", + "type": "array", + "innerType": "string", + "required": false, + "description": "GitHub usernames to assign to the issue", + "enum": null, + "inferrable": true + }, + { + "name": "project_number", + "type": "integer", + "required": false, + "description": "Project number to add the issue to (if any)", + "enum": null, + "inferrable": false + } + ], + "auth": { + "providerId": "github", + "providerType": "oauth2", + "scopes": ["repo"] + }, + "secrets": [], + "output": { + "type": "object", + "description": "The created issue object with id, number, url, and other details" + }, + "documentationChunks": [ + { + "type": "info", + "location": "parameters", + "position": "after", + "content": "When adding an issue to a project, make sure the authenticated user has write access to the project.", + "title": "Project Access" + } + ], + "codeExample": { + "toolName": "Github.CreateIssue", + "parameters": { + "owner": { + "value": "arcadeai", + "type": "string", + "required": true + }, + "repo": { + "value": "arcade-ai", + "type": "string", + "required": true + }, + "title": { + "value": "Bug: Login button not working", + "type": "string", + "required": true + }, + "body": { + "value": "## Description\nThe login button on the home page is unresponsive.\n\n## Steps to reproduce\n1. Go to home page\n2. Click login button\n3. Nothing happens", + "type": "string", + "required": false + }, + "labels": { + "value": ["bug", "high-priority"], + "type": "array", + "required": false + } + }, + "requiresAuth": true, + "authProvider": "github", + "tabLabel": "Call the Tool with User Authorization" + } + }, + { + "name": "SetStarred", + "qualifiedName": "Github.SetStarred", + "fullyQualifiedName": "Github.SetStarred@1.0.0", + "description": "Star or unstar a GitHub repository for the authenticated user.", + "parameters": [ + { + "name": "owner", + "type": "string", + "required": true, + "description": "The owner of the repository to star/unstar", + "enum": null, + "inferrable": true + }, + { + "name": "repo", + "type": "string", + "required": true, + "description": "The name of the repository to star/unstar", + "enum": null, + "inferrable": true + }, + { + "name": "starred", + "type": "boolean", + "required": true, + "description": "True to star the repository, false to unstar it", + "enum": null, + "inferrable": true + } + ], + "auth": { + "providerId": "github", + "providerType": "oauth2", + "scopes": ["public_repo"] + }, + "secrets": [], + "output": { + "type": "object", + "description": "Confirmation of the star/unstar action" + }, + "documentationChunks": [], + "codeExample": { + "toolName": "Github.SetStarred", + "parameters": { + "owner": { + "value": "arcadeai", + "type": "string", + "required": true + }, + "repo": { + "value": "arcade-ai", + "type": "string", + "required": true + }, + "starred": { + "value": true, + "type": "boolean", + "required": true + } + }, + "requiresAuth": true, + "authProvider": "github", + "tabLabel": "Call the Tool with User Authorization" + } + }, + { + "name": "GetRepository", + "qualifiedName": "Github.GetRepository", + "fullyQualifiedName": "Github.GetRepository@1.0.0", + "description": "Get detailed information about a GitHub repository including stars, forks, open issues, and more.", + "parameters": [ + { + "name": "owner", + "type": "string", + "required": true, + "description": "The owner of the repository", + "enum": null, + "inferrable": true + }, + { + "name": "repo", + "type": "string", + "required": true, + "description": "The name of the repository", + "enum": null, + "inferrable": true + } + ], + "auth": { + "providerId": "github", + "providerType": "oauth2", + "scopes": ["repo"] + }, + "secrets": [], + "output": { + "type": "object", + "description": "Repository details including name, description, stars, forks, language, and more" + }, + "documentationChunks": [], + "codeExample": { + "toolName": "Github.GetRepository", + "parameters": { + "owner": { + "value": "arcadeai", + "type": "string", + "required": true + }, + "repo": { + "value": "arcade-ai", + "type": "string", + "required": true + } + }, + "requiresAuth": true, + "authProvider": "github", + "tabLabel": "Call the Tool with User Authorization" + } + }, + { + "name": "ListPullRequests", + "qualifiedName": "Github.ListPullRequests", + "fullyQualifiedName": "Github.ListPullRequests@1.0.0", + "description": "List pull requests in a GitHub repository with optional filtering by state, head, base, and sort order.", + "parameters": [ + { + "name": "owner", + "type": "string", + "required": true, + "description": "The owner of the repository", + "enum": null, + "inferrable": true + }, + { + "name": "repo", + "type": "string", + "required": true, + "description": "The name of the repository", + "enum": null, + "inferrable": true + }, + { + "name": "state", + "type": "string", + "required": false, + "description": "Filter by pull request state", + "enum": ["open", "closed", "all"], + "inferrable": true + }, + { + "name": "sort", + "type": "string", + "required": false, + "description": "What to sort results by", + "enum": ["created", "updated", "popularity", "long-running"], + "inferrable": true + }, + { + "name": "direction", + "type": "string", + "required": false, + "description": "Sort direction", + "enum": ["asc", "desc"], + "inferrable": true + }, + { + "name": "per_page", + "type": "integer", + "required": false, + "description": "Number of results per page (max 100)", + "enum": null, + "inferrable": true + } + ], + "auth": { + "providerId": "github", + "providerType": "oauth2", + "scopes": ["repo"] + }, + "secrets": [], + "output": { + "type": "array", + "description": "List of pull request objects" + }, + "documentationChunks": [ + { + "type": "tip", + "location": "description", + "position": "after", + "content": "Use `state: 'all'` to retrieve both open and closed pull requests in a single request.", + "title": "Pro tip" + } + ], + "codeExample": { + "toolName": "Github.ListPullRequests", + "parameters": { + "owner": { + "value": "arcadeai", + "type": "string", + "required": true + }, + "repo": { + "value": "arcade-ai", + "type": "string", + "required": true + }, + "state": { + "value": "open", + "type": "string", + "required": false + }, + "sort": { + "value": "updated", + "type": "string", + "required": false + }, + "per_page": { + "value": 10, + "type": "integer", + "required": false + } + }, + "requiresAuth": true, + "authProvider": "github", + "tabLabel": "Call the Tool with User Authorization" + } + }, + { + "name": "SearchCode", + "qualifiedName": "Github.SearchCode", + "fullyQualifiedName": "Github.SearchCode@1.0.0", + "description": "Search for code across all public repositories or within specific repositories using GitHub's code search.", + "parameters": [ + { + "name": "query", + "type": "string", + "required": true, + "description": "The search query. Supports GitHub search syntax including qualifiers like `repo:`, `language:`, `path:`", + "enum": null, + "inferrable": true + }, + { + "name": "per_page", + "type": "integer", + "required": false, + "description": "Number of results per page (max 100, default 30)", + "enum": null, + "inferrable": true + }, + { + "name": "page", + "type": "integer", + "required": false, + "description": "Page number for pagination", + "enum": null, + "inferrable": true + } + ], + "auth": { + "providerId": "github", + "providerType": "oauth2", + "scopes": ["repo"] + }, + "secrets": [], + "output": { + "type": "object", + "description": "Search results containing matched code snippets and file locations" + }, + "documentationChunks": [ + { + "type": "warning", + "location": "description", + "position": "before", + "content": "GitHub's code search API has rate limits. Authenticated requests allow 30 requests per minute. Consider implementing pagination and caching for large searches.", + "title": "Rate Limits", + "variant": "warning" + }, + { + "type": "info", + "location": "parameters", + "position": "after", + "content": "Use qualifiers to narrow your search:\n- `repo:owner/name` - Search in a specific repository\n- `language:python` - Filter by programming language\n- `path:src/` - Search in specific directories\n- `extension:ts` - Filter by file extension", + "title": "Search Qualifiers" + } + ], + "codeExample": { + "toolName": "Github.SearchCode", + "parameters": { + "query": { + "value": "ToolDefinition language:typescript repo:arcadeai/arcade-ai", + "type": "string", + "required": true + }, + "per_page": { + "value": 10, + "type": "integer", + "required": false + } + }, + "requiresAuth": true, + "authProvider": "github", + "tabLabel": "Call the Tool with User Authorization" + } + } + ], + "documentationChunks": [ + { + "type": "callout", + "location": "header", + "position": "after", + "content": "The GitHub toolkit requires OAuth2 authentication. Users will be prompted to authorize access to their GitHub account when first using these tools.", + "title": "Authentication Required", + "variant": "info" + } + ], + "customImports": [], + "subPages": [], + "generatedAt": "2026-01-14T12:00:00Z" +} diff --git a/toolkit-docs-generator/tests/fixtures/metadata.json b/toolkit-docs-generator/tests/fixtures/metadata.json new file mode 100644 index 000000000..ad2301af4 --- /dev/null +++ b/toolkit-docs-generator/tests/fixtures/metadata.json @@ -0,0 +1,62 @@ +{ + "Github": { + "id": "Github", + "label": "GitHub", + "category": "development", + "iconUrl": "https://design-system.arcade.dev/icons/github.svg", + "isBYOC": false, + "isPro": false, + "type": "arcade", + "docsLink": "https://docs.arcade.dev/en/mcp-servers/development/github", + "isComingSoon": false, + "isHidden": false + }, + "Slack": { + "id": "Slack", + "label": "Slack", + "category": "social", + "iconUrl": "https://design-system.arcade.dev/icons/slack.svg", + "isBYOC": false, + "isPro": false, + "type": "arcade", + "docsLink": "https://docs.arcade.dev/en/mcp-servers/social/slack", + "isComingSoon": false, + "isHidden": false + }, + "Gmail": { + "id": "Gmail", + "label": "Gmail", + "category": "productivity", + "iconUrl": "https://design-system.arcade.dev/icons/gmail.svg", + "isBYOC": false, + "isPro": false, + "type": "arcade", + "docsLink": "https://docs.arcade.dev/en/mcp-servers/productivity/gmail", + "isComingSoon": false, + "isHidden": false + }, + "Jira": { + "id": "Jira", + "label": "Jira", + "category": "development", + "iconUrl": "https://design-system.arcade.dev/icons/jira.svg", + "isBYOC": true, + "isPro": false, + "type": "arcade", + "docsLink": "https://docs.arcade.dev/en/mcp-servers/development/jira", + "isComingSoon": false, + "isHidden": false + }, + "Stripe": { + "id": "Stripe", + "label": "Stripe", + "category": "payments", + "iconUrl": "https://design-system.arcade.dev/icons/stripe.svg", + "isBYOC": false, + "isPro": true, + "type": "arcade", + "docsLink": "https://docs.arcade.dev/en/mcp-servers/payments/stripe", + "isComingSoon": false, + "isHidden": false + } +} diff --git a/toolkit-docs-generator/tests/fixtures/slack-toolkit.json b/toolkit-docs-generator/tests/fixtures/slack-toolkit.json new file mode 100644 index 000000000..ce3f5f4d3 --- /dev/null +++ b/toolkit-docs-generator/tests/fixtures/slack-toolkit.json @@ -0,0 +1,256 @@ +{ + "id": "Slack", + "label": "Slack", + "version": "1.2.0", + "description": "Slack communication and collaboration tools for sending messages, managing channels, and interacting with workspaces.", + "summary": "The Arcade Slack toolkit provides tools for interacting with Slack workspaces. Build agents and AI apps that can:\n\n- Send and manage messages in channels and DMs\n- Create, archive, and manage channels\n- List and search workspace members\n- React to messages with emoji\n- Upload and share files", + "metadata": { + "category": "social", + "iconUrl": "https://design-system.arcade.dev/icons/slack.svg", + "isBYOC": false, + "isPro": false, + "type": "arcade", + "docsLink": "https://docs.arcade.dev/en/mcp-servers/social/slack", + "isComingSoon": false, + "isHidden": false + }, + "auth": { + "type": "oauth2", + "providerId": "slack", + "allScopes": [ + "channels:read", + "channels:write", + "chat:write", + "users:read", + "files:write", + "reactions:write" + ] + }, + "tools": [ + { + "name": "SendMessage", + "qualifiedName": "Slack.SendMessage", + "fullyQualifiedName": "Slack.SendMessage@1.2.0", + "description": "Send a message to a Slack channel or direct message. Supports rich text formatting with blocks.", + "parameters": [ + { + "name": "channel_id", + "type": "string", + "required": true, + "description": "The ID of the channel or conversation to send the message to", + "enum": null, + "inferrable": true + }, + { + "name": "text", + "type": "string", + "required": true, + "description": "The message text. Supports Slack's mrkdwn formatting", + "enum": null, + "inferrable": true + }, + { + "name": "thread_ts", + "type": "string", + "required": false, + "description": "Timestamp of the parent message to reply in thread", + "enum": null, + "inferrable": false + }, + { + "name": "unfurl_links", + "type": "boolean", + "required": false, + "description": "Whether to enable URL unfurling", + "enum": null, + "inferrable": true + } + ], + "auth": { + "providerId": "slack", + "providerType": "oauth2", + "scopes": ["chat:write"] + }, + "secrets": [], + "output": { + "type": "object", + "description": "Message send confirmation with timestamp and channel" + }, + "documentationChunks": [ + { + "type": "info", + "location": "parameters", + "position": "after", + "content": "To get the `channel_id`, use the `ListChannels` tool or find it in the channel details in Slack (right-click channel > View channel details > scroll to bottom).", + "title": "Finding Channel IDs" + } + ], + "codeExample": { + "toolName": "Slack.SendMessage", + "parameters": { + "channel_id": { + "value": "C0123456789", + "type": "string", + "required": true + }, + "text": { + "value": "Hello from Arcade! :wave:", + "type": "string", + "required": true + } + }, + "requiresAuth": true, + "authProvider": "slack", + "tabLabel": "Call the Tool with User Authorization" + } + }, + { + "name": "ListChannels", + "qualifiedName": "Slack.ListChannels", + "fullyQualifiedName": "Slack.ListChannels@1.2.0", + "description": "List public and private channels in the workspace that the authenticated user has access to.", + "parameters": [ + { + "name": "limit", + "type": "integer", + "required": false, + "description": "Maximum number of channels to return (default 100, max 1000)", + "enum": null, + "inferrable": true + }, + { + "name": "types", + "type": "string", + "required": false, + "description": "Comma-separated list of channel types to include", + "enum": ["public_channel", "private_channel", "mpim", "im"], + "inferrable": true + }, + { + "name": "exclude_archived", + "type": "boolean", + "required": false, + "description": "Whether to exclude archived channels", + "enum": null, + "inferrable": true + } + ], + "auth": { + "providerId": "slack", + "providerType": "oauth2", + "scopes": ["channels:read"] + }, + "secrets": [], + "output": { + "type": "array", + "description": "List of channel objects with id, name, and metadata" + }, + "documentationChunks": [], + "codeExample": { + "toolName": "Slack.ListChannels", + "parameters": { + "limit": { + "value": 50, + "type": "integer", + "required": false + }, + "exclude_archived": { + "value": true, + "type": "boolean", + "required": false + } + }, + "requiresAuth": true, + "authProvider": "slack", + "tabLabel": "Call the Tool with User Authorization" + } + }, + { + "name": "AddReaction", + "qualifiedName": "Slack.AddReaction", + "fullyQualifiedName": "Slack.AddReaction@1.2.0", + "description": "Add an emoji reaction to a message in Slack.", + "parameters": [ + { + "name": "channel_id", + "type": "string", + "required": true, + "description": "The ID of the channel containing the message", + "enum": null, + "inferrable": true + }, + { + "name": "timestamp", + "type": "string", + "required": true, + "description": "The timestamp of the message to react to", + "enum": null, + "inferrable": false + }, + { + "name": "emoji", + "type": "string", + "required": true, + "description": "The emoji name without colons (e.g., 'thumbsup' not ':thumbsup:')", + "enum": null, + "inferrable": true + } + ], + "auth": { + "providerId": "slack", + "providerType": "oauth2", + "scopes": ["reactions:write"] + }, + "secrets": [], + "output": { + "type": "object", + "description": "Confirmation of the reaction being added" + }, + "documentationChunks": [ + { + "type": "tip", + "location": "parameters", + "position": "after", + "content": "Common emoji names: `thumbsup`, `thumbsdown`, `heart`, `eyes`, `white_check_mark`, `x`", + "title": "Common Emoji" + } + ], + "codeExample": { + "toolName": "Slack.AddReaction", + "parameters": { + "channel_id": { + "value": "C0123456789", + "type": "string", + "required": true + }, + "timestamp": { + "value": "1234567890.123456", + "type": "string", + "required": true + }, + "emoji": { + "value": "white_check_mark", + "type": "string", + "required": true + } + }, + "requiresAuth": true, + "authProvider": "slack", + "tabLabel": "Call the Tool with User Authorization" + } + } + ], + "documentationChunks": [ + { + "type": "warning", + "location": "header", + "position": "after", + "content": "Slack rate limits vary by method. Most methods allow 1+ request per second, but some (like `chat.postMessage`) are limited to 1 request per second per channel. Implement appropriate rate limiting in your application.", + "title": "Rate Limits", + "variant": "warning" + } + ], + "customImports": [], + "subPages": ["environment-variables"], + "generatedAt": "2026-01-14T12:00:00Z" +} diff --git a/toolkit-docs-generator/tests/llm/tool-example-generator.test.ts b/toolkit-docs-generator/tests/llm/tool-example-generator.test.ts new file mode 100644 index 000000000..6cafa9aad --- /dev/null +++ b/toolkit-docs-generator/tests/llm/tool-example-generator.test.ts @@ -0,0 +1,116 @@ +/** + * Tests for the LLM tool example generator + */ +import { describe, expect, it } from "vitest"; +import type { LlmClient } from "../../src/llm/client.js"; +import { LlmToolExampleGenerator } from "../../src/llm/tool-example-generator.js"; +import type { ToolDefinition } from "../../src/types/index.js"; + +const createTool = ( + overrides: Partial = {} +): ToolDefinition => ({ + name: "CreateIssue", + qualifiedName: "Github.CreateIssue", + fullyQualifiedName: "Github.CreateIssue@1.0.0", + description: "Create an issue", + parameters: [ + { + name: "owner", + type: "string", + required: true, + description: "Repository owner", + enum: null, + inferrable: true, + }, + { + name: "repo", + type: "string", + required: true, + description: "Repository name", + enum: null, + inferrable: true, + }, + ], + auth: { + providerId: "github", + providerType: "oauth2", + scopes: ["repo"], + }, + secrets: ["API_KEY"], + output: null, + ...overrides, +}); + +describe("LlmToolExampleGenerator", () => { + it("should map LLM JSON output to parameter values", async () => { + const fakeClient: LlmClient = { + provider: "openai", + generateText: async () => + JSON.stringify({ + parameters: { owner: "arcadeai", repo: "docs" }, + secrets: { API_KEY: "api_key" }, + }), + }; + const generator = new LlmToolExampleGenerator({ + client: fakeClient, + model: "gpt-4.1-mini", + }); + + const result = await generator.generate(createTool()); + + expect(result.codeExample.parameters.owner?.value).toBe("arcadeai"); + expect(result.codeExample.parameters.repo?.value).toBe("docs"); + expect(result.secretsInfo).toEqual([{ name: "API_KEY", type: "api_key" }]); + }); + + it("should coerce numeric strings for integer parameters", async () => { + const fakeClient: LlmClient = { + provider: "openai", + generateText: async () => + JSON.stringify({ + parameters: { owner: "arcadeai", repo: "docs", per_page: "30" }, + secrets: { API_KEY: "api_key" }, + }), + }; + const generator = new LlmToolExampleGenerator({ + client: fakeClient, + model: "gpt-4.1-mini", + }); + + const tool = createTool({ + parameters: [ + ...createTool().parameters, + { + name: "per_page", + type: "integer", + required: false, + description: null, + enum: null, + inferrable: true, + }, + ], + }); + + const result = await generator.generate(tool); + + expect(result.codeExample.parameters.per_page?.value).toBe(30); + }); + + it("should accept JSON wrapped in code fences", async () => { + const fakeClient: LlmClient = { + provider: "openai", + generateText: async () => + '```json\n{"parameters":{"owner":"arcadeai","repo":"toolkit"},"secrets":{"API_KEY":"api_key"}}\n```', + }; + const generator = new LlmToolExampleGenerator({ + client: fakeClient, + model: "gpt-4.1-mini", + }); + + const result = await generator.generate(createTool()); + + expect(result.codeExample.parameters.owner?.value).toBe("arcadeai"); + expect(result.codeExample.parameters.repo?.value).toBe("toolkit"); + expect(result.secretsInfo).toEqual([{ name: "API_KEY", type: "api_key" }]); + }); +}); diff --git a/toolkit-docs-generator/tests/merger/data-merger.test.ts b/toolkit-docs-generator/tests/merger/data-merger.test.ts new file mode 100644 index 000000000..c13b468a6 --- /dev/null +++ b/toolkit-docs-generator/tests/merger/data-merger.test.ts @@ -0,0 +1,667 @@ +/** + * Tests for the Data Merger + * + * These tests use in-memory implementations (NOT mocks) to verify + * the merge logic works correctly. + */ +import { describe, expect, it } from "vitest"; +import { + computeAllScopes, + DataMerger, + determineAuthType, + extractVersion, + getProviderId, + groupToolsByToolkit, + mergeToolkit, + type ToolExampleGenerator, +} from "../../src/merger/data-merger.js"; +import { + EmptyCustomSectionsSource, + InMemoryMetadataSource, + InMemoryToolDataSource, +} from "../../src/sources/in-memory.js"; +import { createCombinedToolkitDataSource } from "../../src/sources/toolkit-data-source.js"; +import type { + CustomSections, + ToolDefinition, + ToolkitMetadata, +} from "../../src/types/index.js"; + +// ============================================================================ +// Test Fixtures - Realistic data matching production schema +// ============================================================================ + +const createTool = ( + overrides: Partial = {} +): ToolDefinition => ({ + name: "TestTool", + qualifiedName: "TestKit.TestTool", + fullyQualifiedName: "TestKit.TestTool@1.0.0", + description: "A test tool", + parameters: [ + { + name: "param1", + type: "string", + required: true, + description: "First parameter", + enum: null, + inferrable: true, + }, + ], + auth: { + providerId: "test", + providerType: "oauth2", + scopes: ["read", "write"], + }, + secrets: [], + output: { + type: "object", + description: "Result", + }, + ...overrides, +}); + +const createMetadata = ( + overrides: Partial = {} +): ToolkitMetadata => ({ + id: "TestKit", + label: "Test Kit", + category: "development", + iconUrl: "https://example.com/icon.svg", + isBYOC: false, + isPro: false, + type: "arcade", + docsLink: "https://docs.example.com/test", + isComingSoon: false, + isHidden: false, + ...overrides, +}); + +const createCustomSections = ( + overrides: Partial = {} +): CustomSections => ({ + documentationChunks: [], + customImports: [], + subPages: [], + toolChunks: {}, + ...overrides, +}); + +const createStubGenerator = (): ToolExampleGenerator => ({ + generate: async (tool) => ({ + codeExample: { + toolName: tool.qualifiedName, + parameters: {}, + requiresAuth: tool.auth !== null, + authProvider: tool.auth?.providerId ?? undefined, + tabLabel: tool.auth + ? "Call the Tool with User Authorization" + : "Call the Tool", + }, + secretsInfo: tool.secrets.map((secret) => ({ + name: secret, + type: "unknown", + })), + }), +}); + +// ============================================================================ +// Pure Function Tests +// ============================================================================ + +describe("groupToolsByToolkit", () => { + it("should group tools by their toolkit name", () => { + const tools = [ + createTool({ qualifiedName: "Github.CreateIssue" }), + createTool({ qualifiedName: "Github.ListPRs" }), + createTool({ qualifiedName: "Slack.SendMessage" }), + ]; + + const result = groupToolsByToolkit(tools); + + expect(result.size).toBe(2); + expect(result.get("Github")).toHaveLength(2); + expect(result.get("Slack")).toHaveLength(1); + }); + + it("should handle empty array", () => { + const result = groupToolsByToolkit([]); + expect(result.size).toBe(0); + }); + + it("should skip tools with invalid qualified names", () => { + const tools = [ + createTool({ qualifiedName: "Github.CreateIssue" }), + createTool({ qualifiedName: "" }), // Invalid + ]; + + const result = groupToolsByToolkit(tools); + + expect(result.size).toBe(1); + expect(result.get("Github")).toHaveLength(1); + }); +}); + +describe("computeAllScopes", () => { + it("should compute union of all scopes", () => { + const tools = [ + createTool({ + auth: { + providerId: "test", + providerType: "oauth2", + scopes: ["read", "write"], + }, + }), + createTool({ + auth: { + providerId: "test", + providerType: "oauth2", + scopes: ["write", "delete"], + }, + }), + ]; + + const result = computeAllScopes(tools); + + expect(result).toHaveLength(3); + expect(result).toContain("read"); + expect(result).toContain("write"); + expect(result).toContain("delete"); + }); + + it("should return sorted array", () => { + const tools = [ + createTool({ + auth: { + providerId: "test", + providerType: "oauth2", + scopes: ["z_scope", "a_scope"], + }, + }), + ]; + + const result = computeAllScopes(tools); + + expect(result).toEqual(["a_scope", "z_scope"]); + }); + + it("should return empty array for tools without auth", () => { + const tools = [createTool({ auth: null })]; + + const result = computeAllScopes(tools); + + expect(result).toEqual([]); + }); +}); + +describe("determineAuthType", () => { + it("should return oauth2 for oauth2 provider type", () => { + const tools = [ + createTool({ + auth: { providerId: "github", providerType: "oauth2", scopes: [] }, + }), + ]; + + const result = determineAuthType(tools); + + expect(result).toBe("oauth2"); + }); + + it("should return api_key for non-oauth2 provider type", () => { + const tools = [ + createTool({ + auth: { providerId: "api", providerType: "api_key", scopes: [] }, + }), + ]; + + const result = determineAuthType(tools); + + expect(result).toBe("api_key"); + }); + + it("should return mixed when oauth2 and api_key both exist", () => { + const tools = [ + createTool({ + auth: { providerId: "github", providerType: "oauth2", scopes: [] }, + }), + createTool({ + auth: { providerId: "stripe", providerType: "api_key", scopes: [] }, + }), + ]; + + const result = determineAuthType(tools); + + expect(result).toBe("mixed"); + }); + + it("should ignore secrets when no auth exists", () => { + const tools = [createTool({ auth: null, secrets: ["API_KEY"] })]; + + const result = determineAuthType(tools); + + expect(result).toBe("none"); + }); + + it("should return none when no auth and no secrets", () => { + const tools = [createTool({ auth: null, secrets: [] })]; + + const result = determineAuthType(tools); + + expect(result).toBe("none"); + }); +}); + +describe("getProviderId", () => { + it("should return provider ID from first tool with auth", () => { + const tools = [ + createTool({ auth: null }), + createTool({ + auth: { providerId: "github", providerType: "oauth2", scopes: [] }, + }), + ]; + + const result = getProviderId(tools); + + expect(result).toBe("github"); + }); + + it("should return null when no tools have auth", () => { + const tools = [createTool({ auth: null })]; + + const result = getProviderId(tools); + + expect(result).toBeNull(); + }); +}); + +describe("extractVersion", () => { + it("should extract version from fully qualified name", () => { + expect(extractVersion("Github.CreateIssue@1.0.0")).toBe("1.0.0"); + expect(extractVersion("Slack.SendMessage@2.1.3")).toBe("2.1.3"); + }); + + it("should return 0.0.0 for names without version", () => { + expect(extractVersion("Github.CreateIssue")).toBe("0.0.0"); + }); +}); + +describe("ToolExampleGenerator", () => { + it("should generate an example config for a tool", async () => { + const tool = createTool({ + name: "CreateIssue", + qualifiedName: "Github.CreateIssue", + }); + const generator = createStubGenerator(); + + const result = await generator.generate(tool); + + expect(result.codeExample.toolName).toBe("Github.CreateIssue"); + expect(result.codeExample.requiresAuth).toBe(true); + expect(result.codeExample.authProvider).toBe("test"); + }); + + it("should set requiresAuth to false when no auth", async () => { + const tool = createTool({ auth: null }); + const generator = createStubGenerator(); + + const result = await generator.generate(tool); + + expect(result.codeExample.requiresAuth).toBe(false); + expect(result.codeExample.authProvider).toBeUndefined(); + }); +}); + +// ============================================================================ +// mergeToolkit Function Tests +// ============================================================================ + +describe("mergeToolkit", () => { + it("should merge tools, metadata, and custom sections", async () => { + const tools = [ + createTool({ + name: "Tool1", + qualifiedName: "TestKit.Tool1", + fullyQualifiedName: "TestKit.Tool1@1.2.0", + auth: { + providerId: "test", + providerType: "oauth2", + scopes: ["scope1"], + }, + }), + createTool({ + name: "Tool2", + qualifiedName: "TestKit.Tool2", + fullyQualifiedName: "TestKit.Tool2@1.2.0", + auth: { + providerId: "test", + providerType: "oauth2", + scopes: ["scope2"], + }, + }), + ]; + const metadata = createMetadata({ id: "TestKit", label: "Test Kit" }); + const customSections = createCustomSections({ + documentationChunks: [ + { + type: "warning", + location: "header", + position: "after", + content: "Warning!", + }, + ], + }); + + const result = await mergeToolkit( + "TestKit", + tools, + metadata, + customSections, + createStubGenerator() + ); + + expect(result.toolkit.id).toBe("TestKit"); + expect(result.toolkit.label).toBe("Test Kit"); + expect(result.toolkit.version).toBe("1.2.0"); + expect(result.toolkit.tools).toHaveLength(2); + expect(result.toolkit.auth?.allScopes).toContain("scope1"); + expect(result.toolkit.auth?.allScopes).toContain("scope2"); + expect(result.toolkit.documentationChunks).toHaveLength(1); + expect(result.warnings).toHaveLength(0); + }); + + it("should use default metadata when not provided", async () => { + const tools = [createTool({ qualifiedName: "Unknown.Tool" })]; + + const result = await mergeToolkit( + "Unknown", + tools, + null, + null, + createStubGenerator() + ); + + expect(result.toolkit.label).toBe("Unknown"); + expect(result.toolkit.metadata.category).toBe("development"); + expect(result.warnings).toContain( + "No metadata found for toolkit: Unknown - using defaults" + ); + }); + + it("should warn when no tools found", async () => { + const result = await mergeToolkit( + "Empty", + [], + createMetadata(), + null, + createStubGenerator() + ); + + expect(result.toolkit.tools).toHaveLength(0); + expect(result.warnings).toContain("No tools found for toolkit: Empty"); + }); + + it("should set auth to null when auth type is none", async () => { + const tools = [createTool({ auth: null, secrets: [] })]; + + const result = await mergeToolkit( + "NoAuth", + tools, + null, + null, + createStubGenerator() + ); + + expect(result.toolkit.auth).toBeNull(); + }); + + it("should include per-tool documentation chunks", async () => { + const tools = [createTool({ name: "SpecialTool" })]; + const customSections = createCustomSections({ + toolChunks: { + SpecialTool: [ + { + type: "info", + location: "parameters", + position: "after", + content: "Note about params", + }, + ], + }, + }); + + const result = await mergeToolkit( + "TestKit", + tools, + null, + customSections, + createStubGenerator() + ); + + expect(result.toolkit.tools[0]?.documentationChunks).toHaveLength(1); + }); + + it("should include secret classifications for tools", async () => { + const generator: ToolExampleGenerator = { + generate: async () => ({ + codeExample: { + toolName: "TestKit.CreateIssue", + parameters: {}, + requiresAuth: true, + authProvider: "test", + tabLabel: "Call the Tool with User Authorization", + }, + secretsInfo: [ + { name: "API_KEY", type: "api_key" }, + { name: "WEBHOOK_SECRET", type: "webhook_secret" }, + { name: "ACCESS_TOKEN", type: "token" }, + ], + }), + }; + const tools = [ + createTool({ + secrets: ["API_KEY", "WEBHOOK_SECRET", "ACCESS_TOKEN"], + }), + ]; + + const result = await mergeToolkit("TestKit", tools, null, null, generator); + + expect(result.toolkit.tools[0]?.secretsInfo).toEqual([ + { name: "API_KEY", type: "api_key" }, + { name: "WEBHOOK_SECRET", type: "webhook_secret" }, + { name: "ACCESS_TOKEN", type: "token" }, + ]); + }); +}); + +// ============================================================================ +// DataMerger Class Tests +// ============================================================================ + +describe("DataMerger", () => { + const githubTool1 = createTool({ + name: "CreateIssue", + qualifiedName: "Github.CreateIssue", + fullyQualifiedName: "Github.CreateIssue@1.0.0", + auth: { providerId: "github", providerType: "oauth2", scopes: ["repo"] }, + }); + + const githubTool2 = createTool({ + name: "SetStarred", + qualifiedName: "Github.SetStarred", + fullyQualifiedName: "Github.SetStarred@1.0.0", + auth: { + providerId: "github", + providerType: "oauth2", + scopes: ["public_repo"], + }, + }); + + const slackTool = createTool({ + name: "SendMessage", + qualifiedName: "Slack.SendMessage", + fullyQualifiedName: "Slack.SendMessage@1.2.0", + auth: { + providerId: "slack", + providerType: "oauth2", + scopes: ["chat:write"], + }, + }); + + const githubMetadata = createMetadata({ + id: "Github", + label: "GitHub", + category: "development", + }); + const slackMetadata = createMetadata({ + id: "Slack", + label: "Slack", + category: "social", + }); + + describe("mergeToolkit", () => { + it("should merge data for a single toolkit", async () => { + const toolkitDataSource = createCombinedToolkitDataSource({ + toolSource: new InMemoryToolDataSource([ + githubTool1, + githubTool2, + slackTool, + ]), + metadataSource: new InMemoryMetadataSource([ + githubMetadata, + slackMetadata, + ]), + }); + const merger = new DataMerger({ + toolkitDataSource, + customSectionsSource: new EmptyCustomSectionsSource(), + toolExampleGenerator: createStubGenerator(), + }); + + const result = await merger.mergeToolkit("Github"); + + expect(result.toolkit.id).toBe("Github"); + expect(result.toolkit.label).toBe("GitHub"); + expect(result.toolkit.tools).toHaveLength(2); + expect(result.toolkit.auth?.allScopes).toContain("repo"); + expect(result.toolkit.auth?.allScopes).toContain("public_repo"); + }); + + it("should merge data using unified toolkit data source", async () => { + const toolkitDataSource = createCombinedToolkitDataSource({ + toolSource: new InMemoryToolDataSource([ + githubTool1, + githubTool2, + slackTool, + ]), + metadataSource: new InMemoryMetadataSource([ + githubMetadata, + slackMetadata, + ]), + }); + const merger = new DataMerger({ + toolkitDataSource, + customSectionsSource: new EmptyCustomSectionsSource(), + toolExampleGenerator: createStubGenerator(), + }); + + const result = await merger.mergeToolkit("Github"); + + expect(result.toolkit.id).toBe("Github"); + expect(result.toolkit.label).toBe("GitHub"); + expect(result.toolkit.tools).toHaveLength(2); + expect(result.toolkit.auth?.allScopes).toContain("repo"); + expect(result.toolkit.auth?.allScopes).toContain("public_repo"); + }); + + it("should filter by version when specified", async () => { + const tools = [ + createTool({ + qualifiedName: "Test.Tool", + fullyQualifiedName: "Test.Tool@1.0.0", + }), + createTool({ + qualifiedName: "Test.Tool", + fullyQualifiedName: "Test.Tool@2.0.0", + }), + ]; + + const toolkitDataSource = createCombinedToolkitDataSource({ + toolSource: new InMemoryToolDataSource(tools), + metadataSource: new InMemoryMetadataSource([]), + }); + const merger = new DataMerger({ + toolkitDataSource, + customSectionsSource: new EmptyCustomSectionsSource(), + toolExampleGenerator: createStubGenerator(), + }); + + const result = await merger.mergeToolkit("Test", "1.0.0"); + + expect(result.toolkit.tools).toHaveLength(1); + expect(result.toolkit.version).toBe("1.0.0"); + }); + + it("should handle case-insensitive toolkit IDs", async () => { + const toolkitDataSource = createCombinedToolkitDataSource({ + toolSource: new InMemoryToolDataSource([githubTool1]), + metadataSource: new InMemoryMetadataSource([githubMetadata]), + }); + const merger = new DataMerger({ + toolkitDataSource, + customSectionsSource: new EmptyCustomSectionsSource(), + toolExampleGenerator: createStubGenerator(), + }); + + const result = await merger.mergeToolkit("github"); // lowercase + + expect(result.toolkit.id).toBe("github"); + expect(result.toolkit.label).toBe("GitHub"); + }); + }); + + describe("mergeAllToolkits", () => { + it("should merge all toolkits found in tools", async () => { + const toolkitDataSource = createCombinedToolkitDataSource({ + toolSource: new InMemoryToolDataSource([ + githubTool1, + githubTool2, + slackTool, + ]), + metadataSource: new InMemoryMetadataSource([ + githubMetadata, + slackMetadata, + ]), + }); + const merger = new DataMerger({ + toolkitDataSource, + customSectionsSource: new EmptyCustomSectionsSource(), + toolExampleGenerator: createStubGenerator(), + }); + + const results = await merger.mergeAllToolkits(); + + expect(results).toHaveLength(2); + + const githubResult = results.find((r) => r.toolkit.id === "Github"); + const slackResult = results.find((r) => r.toolkit.id === "Slack"); + + expect(githubResult?.toolkit.tools).toHaveLength(2); + expect(slackResult?.toolkit.tools).toHaveLength(1); + }); + + it("should return empty array when no tools", async () => { + const toolkitDataSource = createCombinedToolkitDataSource({ + toolSource: new InMemoryToolDataSource([]), + metadataSource: new InMemoryMetadataSource([]), + }); + const merger = new DataMerger({ + toolkitDataSource, + customSectionsSource: new EmptyCustomSectionsSource(), + }); + + const results = await merger.mergeAllToolkits(); + + expect(results).toHaveLength(0); + }); + }); +}); diff --git a/toolkit-docs-generator/tests/sources/in-memory.test.ts b/toolkit-docs-generator/tests/sources/in-memory.test.ts new file mode 100644 index 000000000..917b4a93b --- /dev/null +++ b/toolkit-docs-generator/tests/sources/in-memory.test.ts @@ -0,0 +1,370 @@ +/** + * Tests for in-memory data source implementations + * + * These tests demonstrate the testing philosophy: + * - Use real implementations instead of mocks + * - Test with realistic fixture data + * - Focus on behavior, not implementation details + */ +import { describe, expect, it } from "vitest"; +import { + EmptyCustomSectionsSource, + InMemoryCustomSectionsSource, + InMemoryMetadataSource, + InMemoryToolDataSource, +} from "../../src/sources/in-memory.js"; +import type { + CustomSections, + ToolDefinition, + ToolkitMetadata, +} from "../../src/types/index.js"; + +// ============================================================================ +// Test Fixtures - Realistic data matching production schema +// ============================================================================ + +const createTestTool = ( + overrides: Partial = {} +): ToolDefinition => ({ + name: "CreateIssue", + qualifiedName: "Github.CreateIssue", + fullyQualifiedName: "Github.CreateIssue@1.0.0", + description: "Create a new issue in a GitHub repository", + parameters: [ + { + name: "owner", + type: "string", + required: true, + description: "Repository owner", + enum: null, + inferrable: true, + }, + { + name: "repo", + type: "string", + required: true, + description: "Repository name", + enum: null, + inferrable: true, + }, + ], + auth: { + providerId: "github", + providerType: "oauth2", + scopes: ["repo"], + }, + secrets: [], + output: { + type: "object", + description: "Created issue details", + }, + ...overrides, +}); + +const createTestMetadata = ( + overrides: Partial = {} +): ToolkitMetadata => ({ + id: "Github", + label: "GitHub", + category: "development", + iconUrl: "https://design-system.arcade.dev/icons/github.svg", + isBYOC: false, + isPro: false, + type: "arcade", + docsLink: "https://docs.arcade.dev/en/mcp-servers/development/github", + isComingSoon: false, + isHidden: false, + ...overrides, +}); + +const createTestCustomSections = ( + overrides: Partial = {} +): CustomSections => ({ + documentationChunks: [ + { + type: "warning", + location: "header", + position: "after", + content: "This toolkit requires OAuth2 authentication.", + }, + ], + customImports: [], + subPages: [], + toolChunks: {}, + ...overrides, +}); + +// ============================================================================ +// InMemoryToolDataSource Tests +// ============================================================================ + +describe("InMemoryToolDataSource", () => { + const githubTool1 = createTestTool({ + name: "CreateIssue", + qualifiedName: "Github.CreateIssue", + fullyQualifiedName: "Github.CreateIssue@1.0.0", + }); + + const githubTool2 = createTestTool({ + name: "SetStarred", + qualifiedName: "Github.SetStarred", + fullyQualifiedName: "Github.SetStarred@1.0.0", + }); + + const slackTool = createTestTool({ + name: "SendMessage", + qualifiedName: "Slack.SendMessage", + fullyQualifiedName: "Slack.SendMessage@1.2.0", + auth: { + providerId: "slack", + providerType: "oauth2", + scopes: ["chat:write"], + }, + }); + + const testTools = [githubTool1, githubTool2, slackTool]; + + describe("fetchToolsByToolkit", () => { + it("should return tools for the specified toolkit", async () => { + const source = new InMemoryToolDataSource(testTools); + + const result = await source.fetchToolsByToolkit("Github"); + + expect(result).toHaveLength(2); + expect(result.map((t) => t.name)).toContain("CreateIssue"); + expect(result.map((t) => t.name)).toContain("SetStarred"); + }); + + it("should handle case-insensitive toolkit IDs", async () => { + const source = new InMemoryToolDataSource(testTools); + + const result = await source.fetchToolsByToolkit("github"); + + expect(result).toHaveLength(2); + }); + + it("should return empty array for unknown toolkit", async () => { + const source = new InMemoryToolDataSource(testTools); + + const result = await source.fetchToolsByToolkit("Unknown"); + + expect(result).toHaveLength(0); + }); + }); + + describe("fetchAllTools", () => { + it("should return all tools when no options provided", async () => { + const source = new InMemoryToolDataSource(testTools); + + const result = await source.fetchAllTools(); + + expect(result).toHaveLength(3); + }); + + it("should filter by toolkitId option", async () => { + const source = new InMemoryToolDataSource(testTools); + + const result = await source.fetchAllTools({ toolkitId: "Slack" }); + + expect(result).toHaveLength(1); + expect(result[0]?.name).toBe("SendMessage"); + }); + + it("should filter by version option", async () => { + const source = new InMemoryToolDataSource(testTools); + + const result = await source.fetchAllTools({ version: "1.2.0" }); + + expect(result).toHaveLength(1); + expect(result[0]?.name).toBe("SendMessage"); + }); + + it("should filter by providerId option", async () => { + const source = new InMemoryToolDataSource(testTools); + + const result = await source.fetchAllTools({ providerId: "slack" }); + + expect(result).toHaveLength(1); + expect(result[0]?.name).toBe("SendMessage"); + }); + + it("should combine multiple filters", async () => { + const source = new InMemoryToolDataSource(testTools); + + const result = await source.fetchAllTools({ + toolkitId: "Github", + version: "1.0.0", + }); + + expect(result).toHaveLength(2); + }); + }); + + describe("isAvailable", () => { + it("should always return true", async () => { + const source = new InMemoryToolDataSource([]); + + const result = await source.isAvailable(); + + expect(result).toBe(true); + }); + }); +}); + +// ============================================================================ +// InMemoryMetadataSource Tests +// ============================================================================ + +describe("InMemoryMetadataSource", () => { + const githubMetadata = createTestMetadata({ + id: "Github", + label: "GitHub", + category: "development", + }); + + const slackMetadata = createTestMetadata({ + id: "Slack", + label: "Slack", + category: "social", + }); + + const testMetadata = [githubMetadata, slackMetadata]; + + describe("getToolkitMetadata", () => { + it("should return metadata for exact ID match", async () => { + const source = new InMemoryMetadataSource(testMetadata); + + const result = await source.getToolkitMetadata("Github"); + + expect(result).not.toBeNull(); + expect(result?.id).toBe("Github"); + expect(result?.label).toBe("GitHub"); + }); + + it("should handle case-insensitive lookup", async () => { + const source = new InMemoryMetadataSource(testMetadata); + + const result = await source.getToolkitMetadata("github"); + + expect(result).not.toBeNull(); + expect(result?.id).toBe("Github"); + }); + + it("should return null for unknown toolkit", async () => { + const source = new InMemoryMetadataSource(testMetadata); + + const result = await source.getToolkitMetadata("Unknown"); + + expect(result).toBeNull(); + }); + }); + + describe("getAllToolkitsMetadata", () => { + it("should return all unique toolkits", async () => { + const source = new InMemoryMetadataSource(testMetadata); + + const result = await source.getAllToolkitsMetadata(); + + expect(result).toHaveLength(2); + expect(result.map((m) => m.id)).toContain("Github"); + expect(result.map((m) => m.id)).toContain("Slack"); + }); + }); + + describe("listToolkitIds", () => { + it("should return all toolkit IDs", async () => { + const source = new InMemoryMetadataSource(testMetadata); + + const result = await source.listToolkitIds(); + + expect(result).toHaveLength(2); + expect(result).toContain("Github"); + expect(result).toContain("Slack"); + }); + }); +}); + +// ============================================================================ +// InMemoryCustomSectionsSource Tests +// ============================================================================ + +describe("InMemoryCustomSectionsSource", () => { + const githubSections = createTestCustomSections({ + documentationChunks: [ + { + type: "warning", + location: "header", + position: "after", + content: "GitHub OAuth required", + }, + ], + }); + + const slackSections = createTestCustomSections({ + subPages: ["environment-variables"], + }); + + const testSections = { + Github: githubSections, + Slack: slackSections, + }; + + describe("getCustomSections", () => { + it("should return sections for exact ID match", async () => { + const source = new InMemoryCustomSectionsSource(testSections); + + const result = await source.getCustomSections("Github"); + + expect(result).not.toBeNull(); + expect(result?.documentationChunks).toHaveLength(1); + }); + + it("should handle case-insensitive lookup", async () => { + const source = new InMemoryCustomSectionsSource(testSections); + + const result = await source.getCustomSections("github"); + + expect(result).not.toBeNull(); + }); + + it("should return null for unknown toolkit", async () => { + const source = new InMemoryCustomSectionsSource(testSections); + + const result = await source.getCustomSections("Unknown"); + + expect(result).toBeNull(); + }); + }); + + describe("getAllCustomSections", () => { + it("should return all sections", async () => { + const source = new InMemoryCustomSectionsSource(testSections); + + const result = await source.getAllCustomSections(); + + expect(Object.keys(result)).toHaveLength(2); + }); + }); +}); + +// ============================================================================ +// EmptyCustomSectionsSource Tests +// ============================================================================ + +describe("EmptyCustomSectionsSource", () => { + it("should always return null for getCustomSections", async () => { + const source = new EmptyCustomSectionsSource(); + + const result = await source.getCustomSections("Github"); + + expect(result).toBeNull(); + }); + + it("should return empty object for getAllCustomSections", async () => { + const source = new EmptyCustomSectionsSource(); + + const result = await source.getAllCustomSections(); + + expect(result).toEqual({}); + }); +}); diff --git a/toolkit-docs-generator/tests/sources/toolkit-data-source.test.ts b/toolkit-docs-generator/tests/sources/toolkit-data-source.test.ts new file mode 100644 index 000000000..e49347854 --- /dev/null +++ b/toolkit-docs-generator/tests/sources/toolkit-data-source.test.ts @@ -0,0 +1,127 @@ +/** + * Tests for the Combined Toolkit Data Source + * + * These tests use real in-memory sources to verify the combined + * abstraction returns tools + metadata together. + */ +import { describe, expect, it } from "vitest"; +import { + InMemoryMetadataSource, + InMemoryToolDataSource, +} from "../../src/sources/in-memory.js"; +import { createCombinedToolkitDataSource } from "../../src/sources/toolkit-data-source.js"; +import type { ToolDefinition, ToolkitMetadata } from "../../src/types/index.js"; + +const createTool = ( + overrides: Partial = {} +): ToolDefinition => ({ + name: "TestTool", + qualifiedName: "Github.TestTool", + fullyQualifiedName: "Github.TestTool@1.0.0", + description: "A test tool", + parameters: [], + auth: null, + secrets: [], + output: null, + ...overrides, +}); + +const createMetadata = ( + overrides: Partial = {} +): ToolkitMetadata => ({ + id: "Github", + label: "GitHub", + category: "development", + iconUrl: "https://example.com/github.svg", + isBYOC: false, + isPro: false, + type: "arcade", + docsLink: "https://docs.example.com/github", + isComingSoon: false, + isHidden: false, + ...overrides, +}); + +describe("CombinedToolkitDataSource", () => { + it("should return tools and metadata for a toolkit", async () => { + const toolSource = new InMemoryToolDataSource([ + createTool({ qualifiedName: "Github.CreateIssue" }), + createTool({ qualifiedName: "Github.ListPullRequests" }), + ]); + const metadataSource = new InMemoryMetadataSource([createMetadata()]); + const dataSource = createCombinedToolkitDataSource({ + toolSource, + metadataSource, + }); + + const result = await dataSource.fetchToolkitData("Github"); + + expect(result.tools).toHaveLength(2); + expect(result.metadata?.label).toBe("GitHub"); + }); + + it("should filter tools by version when provided", async () => { + const toolSource = new InMemoryToolDataSource([ + createTool({ + qualifiedName: "Github.CreateIssue", + fullyQualifiedName: "Github.CreateIssue@1.0.0", + }), + createTool({ + qualifiedName: "Github.CreateIssue", + fullyQualifiedName: "Github.CreateIssue@2.0.0", + }), + ]); + const metadataSource = new InMemoryMetadataSource([createMetadata()]); + const dataSource = createCombinedToolkitDataSource({ + toolSource, + metadataSource, + }); + + const result = await dataSource.fetchToolkitData("Github", "1.0.0"); + + expect(result.tools).toHaveLength(1); + expect(result.tools[0]?.fullyQualifiedName).toContain("@1.0.0"); + }); + + it("should return null metadata when not found", async () => { + const toolSource = new InMemoryToolDataSource([ + createTool({ qualifiedName: "Github.CreateIssue" }), + ]); + const metadataSource = new InMemoryMetadataSource([]); + const dataSource = createCombinedToolkitDataSource({ + toolSource, + metadataSource, + }); + + const result = await dataSource.fetchToolkitData("Github"); + + expect(result.metadata).toBeNull(); + }); + + it("should return all toolkits with metadata when available", async () => { + const toolSource = new InMemoryToolDataSource([ + createTool({ + qualifiedName: "Github.CreateIssue", + fullyQualifiedName: "Github.CreateIssue@1.0.0", + }), + createTool({ + qualifiedName: "Slack.SendMessage", + fullyQualifiedName: "Slack.SendMessage@1.2.0", + }), + ]); + const metadataSource = new InMemoryMetadataSource([ + createMetadata(), + createMetadata({ id: "Slack", label: "Slack", category: "social" }), + ]); + const dataSource = createCombinedToolkitDataSource({ + toolSource, + metadataSource, + }); + + const result = await dataSource.fetchAllToolkitsData(); + + expect(result.size).toBe(2); + expect(result.get("Github")?.metadata?.label).toBe("GitHub"); + expect(result.get("Slack")?.metadata?.label).toBe("Slack"); + }); +}); diff --git a/toolkit-docs-generator/tsconfig.json b/toolkit-docs-generator/tsconfig.json new file mode 100644 index 000000000..8acdb3fea --- /dev/null +++ b/toolkit-docs-generator/tsconfig.json @@ -0,0 +1,26 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "outDir": "./dist", + "rootDir": "./src", + "resolveJsonModule": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "exactOptionalPropertyTypes": true, + "noUncheckedIndexedAccess": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "**/*.test.ts"] +} diff --git a/toolkit-docs-generator/vitest.config.ts b/toolkit-docs-generator/vitest.config.ts new file mode 100644 index 000000000..f7e5aeeda --- /dev/null +++ b/toolkit-docs-generator/vitest.config.ts @@ -0,0 +1,39 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + // Enable globals like describe, it, expect without imports + globals: true, + + // Test environment + environment: "node", + + // Include test files + include: ["tests/**/*.test.ts"], + + // Coverage configuration + coverage: { + provider: "v8", + reporter: ["text", "json", "html"], + exclude: [ + "node_modules/", + "dist/", + "tests/", + "**/*.d.ts", + "vitest.config.ts", + ], + // Require 80% coverage + thresholds: { + lines: 80, + functions: 80, + branches: 80, + statements: 80, + }, + }, + + // TypeScript configuration + typecheck: { + enabled: true, + }, + }, +}); From 47220efa2defd16d5d80c0ddb142eadb30cd4ae8 Mon Sep 17 00:00:00 2001 From: jottakka Date: Fri, 23 Jan 2026 18:01:27 -0300 Subject: [PATCH 2/2] [TOO-314] Tool documentation auto generation tool and replacing hardcode mdx for component based one --- app/_components/dashboard-link.tsx | 33 +- app/_components/posthog.tsx | 27 +- app/_components/scope-picker.test.tsx | 28 + app/_components/scope-picker.tsx | 361 +++++++++--- .../__tests__/AvailableToolsTable.test.tsx | 92 +-- .../__tests__/ToolkitPage.test.tsx | 11 + .../components/AvailableToolsTable.tsx | 555 ++++++++++++++++-- .../components/DynamicCodeBlock.tsx | 268 +++++---- .../components/ParametersTable.tsx | 151 ++--- .../toolkit-docs/components/ScopesDisplay.tsx | 37 +- .../toolkit-docs/components/ToolSection.tsx | 431 ++++++++++---- .../components/ToolkitDocsPreview.tsx | 25 + .../toolkit-docs/components/ToolkitHeader.tsx | 228 ++++--- .../toolkit-docs/components/ToolkitPage.tsx | 336 ++++++++++- app/_components/toolkit-docs/constants.ts | 183 ++++++ app/_components/toolkit-docs/index.ts | 3 + app/_components/toolkit-docs/types/index.ts | 37 ++ app/_lib/__tests__/toolkit-data.test.ts | 72 +++ app/_lib/toolkit-data.ts | 55 ++ app/api/toolkit-data/[toolkitId]/route.ts | 19 + app/en/resources/integrations/_meta.tsx | 4 + .../development/github/preview/page.mdx | 4 +- .../preview/ToolkitPreviewIndex.tsx | 101 ++++ .../[toolkitId]/ToolkitPreviewContent.tsx | 47 ++ .../preview/[toolkitId]/_meta.tsx | 14 + .../integrations/preview/[toolkitId]/page.mdx | 8 + .../resources/integrations/preview/_meta.tsx | 16 + .../resources/integrations/preview/page.mdx | 12 + app/globals.css | 9 + toolkit-docs-generator/src/cli/index.ts | 515 ++++++++++++++-- toolkit-docs-generator/src/generator/index.ts | 1 + .../src/generator/json-generator.ts | 21 +- .../src/generator/output-verifier.ts | 179 ++++++ toolkit-docs-generator/src/llm/index.ts | 1 + .../src/llm/toolkit-summary-generator.ts | 148 +++++ .../src/merger/data-merger.ts | 288 ++++++++- .../src/sources/engine-api.ts | 171 ++++++ toolkit-docs-generator/src/sources/index.ts | 1 + .../src/sources/mock-engine-api.ts | 120 +--- .../src/sources/tool-metadata-schema.ts | 145 +++++ .../src/sources/toolkit-data-source.ts | 23 + toolkit-docs-generator/src/types/index.ts | 1 + .../src/utils/concurrency.ts | 75 +++ toolkit-docs-generator/src/utils/index.ts | 1 + .../tests/generator/output-verifier.test.ts | 233 ++++++++ .../llm/toolkit-summary-generator.test.ts | 89 +++ .../tests/merger/data-merger.test.ts | 191 ++++++ .../tests/sources/engine-api.test.ts | 253 ++++++++ 48 files changed, 4857 insertions(+), 766 deletions(-) create mode 100644 app/_components/scope-picker.test.tsx create mode 100644 app/_components/toolkit-docs/__tests__/ToolkitPage.test.tsx create mode 100644 app/_components/toolkit-docs/components/ToolkitDocsPreview.tsx create mode 100644 app/_components/toolkit-docs/constants.ts create mode 100644 app/_lib/__tests__/toolkit-data.test.ts create mode 100644 app/_lib/toolkit-data.ts create mode 100644 app/api/toolkit-data/[toolkitId]/route.ts create mode 100644 app/en/resources/integrations/preview/ToolkitPreviewIndex.tsx create mode 100644 app/en/resources/integrations/preview/[toolkitId]/ToolkitPreviewContent.tsx create mode 100644 app/en/resources/integrations/preview/[toolkitId]/_meta.tsx create mode 100644 app/en/resources/integrations/preview/[toolkitId]/page.mdx create mode 100644 app/en/resources/integrations/preview/_meta.tsx create mode 100644 app/en/resources/integrations/preview/page.mdx create mode 100644 toolkit-docs-generator/src/generator/output-verifier.ts create mode 100644 toolkit-docs-generator/src/llm/toolkit-summary-generator.ts create mode 100644 toolkit-docs-generator/src/sources/engine-api.ts create mode 100644 toolkit-docs-generator/src/sources/tool-metadata-schema.ts create mode 100644 toolkit-docs-generator/src/utils/concurrency.ts create mode 100644 toolkit-docs-generator/tests/generator/output-verifier.test.ts create mode 100644 toolkit-docs-generator/tests/llm/toolkit-summary-generator.test.ts create mode 100644 toolkit-docs-generator/tests/sources/engine-api.test.ts diff --git a/app/_components/dashboard-link.tsx b/app/_components/dashboard-link.tsx index 3c01a32e7..1907a0e68 100644 --- a/app/_components/dashboard-link.tsx +++ b/app/_components/dashboard-link.tsx @@ -1,12 +1,37 @@ import Link from "next/link"; const DASHBOARD_BASE_URL = - process.env.NEXT_PUBLIC_DASHBOARD_URL || "https://api.arcade.dev"; + process.env.NEXT_PUBLIC_DASHBOARD_URL || "https://app.arcade.dev"; + +const normalizePath = (value: string): string => + value.replace(/\/+$/, "").replace(/^\/+/, ""); + +const resolveDashboardPrefix = (): string => { + const baseUrl = DASHBOARD_BASE_URL.toLowerCase(); + const explicitPrefix = process.env.NEXT_PUBLIC_DASHBOARD_PATH_PREFIX; + + if (explicitPrefix !== undefined) { + return normalizePath(explicitPrefix); + } + + if (baseUrl.includes("/dashboard") || baseUrl.includes("app.arcade.dev")) { + return ""; + } + + return "dashboard"; +}; + +const DASHBOARD_PATH_PREFIX = resolveDashboardPrefix(); +const BASE_URL = DASHBOARD_BASE_URL.replace(/\/+$/, ""); export const getDashboardUrl = (path = "") => - path - ? `${DASHBOARD_BASE_URL}/dashboard/${path}` - : `${DASHBOARD_BASE_URL}/dashboard`; + [ + BASE_URL, + DASHBOARD_PATH_PREFIX, + path ? normalizePath(path) : "", + ] + .filter(Boolean) + .join("/"); type DashboardLinkProps = { path?: string; diff --git a/app/_components/posthog.tsx b/app/_components/posthog.tsx index 9a27c6e08..032b96212 100644 --- a/app/_components/posthog.tsx +++ b/app/_components/posthog.tsx @@ -6,11 +6,21 @@ import posthog from "posthog-js"; import { PostHogProvider } from "posthog-js/react"; import { useEffect } from "react"; +function createNoopPosthogClient() { + return { + capture: () => {}, + identify: () => {}, + reset: () => {}, + // biome-ignore lint/suspicious/noExplicitAny: Minimal no-op client for disabled analytics + } as any; +} + export const PostHog = ({ children }: { children: React.ReactNode }) => { const pathname = usePathname(); + const isEnabled = Boolean(process.env.NEXT_PUBLIC_POSTHOG_KEY); useEffect(() => { - if (process.env.NEXT_PUBLIC_POSTHOG_KEY) { + if (isEnabled) { posthog.init(process.env.NEXT_PUBLIC_POSTHOG_KEY, { api_host: process.env.NEXT_PUBLIC_POSTHOG_HOST || "https://us.i.posthog.com", @@ -34,18 +44,21 @@ export const PostHog = ({ children }: { children: React.ReactNode }) => { }, }); } else { - // biome-ignore lint/suspicious/noConsole: This is ok for PostHog - console.warn("Analytics is disabled because no key is set"); + // No key: keep analytics fully disabled and avoid noisy console errors. } - }, []); + }, [isEnabled]); // Track page views when pathname changes // biome-ignore lint/correctness/useExhaustiveDependencies: pathname is required for route change tracking useEffect(() => { - if (process.env.NEXT_PUBLIC_POSTHOG_KEY) { + if (isEnabled) { posthog?.capture("$pageview"); } - }, [pathname]); + }, [pathname, isEnabled]); - return {children}; + return ( + + {children} + + ); }; diff --git a/app/_components/scope-picker.test.tsx b/app/_components/scope-picker.test.tsx new file mode 100644 index 000000000..aeb81438c --- /dev/null +++ b/app/_components/scope-picker.test.tsx @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; + +import { getRequiredScopes, getSelectedToolNames } from "./scope-picker"; + +describe("scope picker helpers", () => { + const tools = [ + { name: "SendEmail", scopes: ["scope.send", " scope.send "] }, + { name: "ListEmails", scopes: ["scope.read", ""] }, + ]; + + it("filters selected tools to known names only", () => { + const selected = new Set(["SendEmail", "UnknownTool"]); + + expect(getSelectedToolNames(tools, selected)).toEqual(["SendEmail"]); + }); + + it("builds required scopes for selected tools", () => { + const selected = new Set(["SendEmail", "ListEmails"]); + + expect(getRequiredScopes(tools, selected)).toEqual(["scope.read", "scope.send"]); + }); + + it("returns empty scopes when no valid tools selected", () => { + const selected = new Set(["UnknownTool"]); + + expect(getRequiredScopes(tools, selected)).toEqual([]); + }); +}); diff --git a/app/_components/scope-picker.tsx b/app/_components/scope-picker.tsx index bad282f7d..ac2eda0a5 100644 --- a/app/_components/scope-picker.tsx +++ b/app/_components/scope-picker.tsx @@ -1,19 +1,119 @@ "use client"; +import { Check, Copy, KeyRound, ShieldCheck, Wrench } from "lucide-react"; import { usePostHog } from "posthog-js/react"; -import { useState } from "react"; +import { useCallback, useMemo, useState } from "react"; type Tool = { name: string; scopes: string[]; + secrets?: string[]; }; type ScopePickerProps = { tools: Tool[]; + selectedTools?: string[]; + onSelectedToolsChange?: (selectedTools: string[]) => void; }; -export default function ScopePicker({ tools }: ScopePickerProps) { - const [selectedTools, setSelectedTools] = useState>(new Set()); +function CopyButton({ + text, + label, + variant = "default", +}: { + text: string; + label: string; + variant?: "default" | "small"; +}) { + const [copied, setCopied] = useState(false); + + const handleCopy = useCallback(async () => { + try { + await navigator.clipboard.writeText(text); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + } catch { + console.error("Failed to copy"); + } + }, [text]); + + if (variant === "small") { + return ( + + ); + } + + return ( + + ); +} + +export function getSelectedToolNames( + tools: Tool[], + selectedTools: Set +): string[] { + const validToolNames = new Set(tools.map((tool) => tool.name)); + return Array.from(selectedTools).filter((name) => validToolNames.has(name)); +} + +export function getRequiredScopes( + tools: Tool[], + selectedTools: Set +): string[] { + const selectedToolNames = new Set(getSelectedToolNames(tools, selectedTools)); + const normalizedScopes = tools + .filter((tool) => selectedToolNames.has(tool.name)) + .flatMap((tool) => tool.scopes) + .map((scope) => scope.trim()) + .filter((scope) => scope.length > 0); + + return Array.from(new Set(normalizedScopes)).sort(); +} + +export function getRequiredSecrets( + tools: Tool[], + selectedTools: Set +): string[] { + const selectedToolNames = new Set(getSelectedToolNames(tools, selectedTools)); + const secrets = tools + .filter((tool) => selectedToolNames.has(tool.name)) + .flatMap((tool) => tool.secrets ?? []) + .map((secret) => secret.trim()) + .filter((secret) => secret.length > 0); + + return Array.from(new Set(secrets)).sort(); +} + +export default function ScopePicker({ + tools, + selectedTools, + onSelectedToolsChange, +}: ScopePickerProps) { + const [internalSelectedTools, setInternalSelectedTools] = useState>( + new Set() + ); + const [showAdvanced, setShowAdvanced] = useState(false); + const isControlled = Array.isArray(selectedTools); + const selectedToolsSet = useMemo( + () => (isControlled ? new Set(selectedTools) : internalSelectedTools), + [isControlled, selectedTools, internalSelectedTools] + ); const posthog = usePostHog(); const trackScopeCalculatorUsed = ( @@ -29,56 +129,85 @@ export default function ScopePicker({ tools }: ScopePickerProps) { }); }; + const updateSelectedTools = ( + nextSelected: Set, + action: string, + toolName?: string + ) => { + const nextList = Array.from(nextSelected).sort(); + trackScopeCalculatorUsed(action, toolName, nextSelected.size); + if (isControlled) { + onSelectedToolsChange?.(nextList); + } else { + setInternalSelectedTools(nextSelected); + } + }; + const toggleTool = (toolName: string) => { - const newSelected = new Set(selectedTools); + const newSelected = new Set(selectedToolsSet); const isSelecting = !newSelected.has(toolName); if (newSelected.has(toolName)) { newSelected.delete(toolName); } else { newSelected.add(toolName); } - setSelectedTools(newSelected); - trackScopeCalculatorUsed( + updateSelectedTools( + newSelected, isSelecting ? "tool_selected" : "tool_deselected", - toolName, - newSelected.size + toolName ); }; const selectAll = () => { - setSelectedTools(new Set(tools.map((t) => t.name))); - trackScopeCalculatorUsed("select_all", undefined, tools.length); + updateSelectedTools(new Set(tools.map((t) => t.name)), "select_all"); }; const clearAll = () => { - setSelectedTools(new Set()); - trackScopeCalculatorUsed("clear_all", undefined, 0); + updateSelectedTools(new Set(), "clear_all"); }; - // Get unique scopes from selected tools - const requiredScopes = Array.from( - new Set( - tools.filter((t) => selectedTools.has(t.name)).flatMap((t) => t.scopes) - ) - ).sort(); + const selectedToolNames = getSelectedToolNames(tools, selectedToolsSet); + const requiredScopes = getRequiredScopes(tools, selectedToolsSet); + const requiredSecrets = getRequiredSecrets(tools, selectedToolsSet); + + const scopesAsText = requiredScopes.join("\n"); + const secretsAsText = requiredSecrets.join("\n"); + const toolNamesAsText = selectedToolNames.join(", "); + const selectedToolsAsJson = JSON.stringify( + tools + .filter((t) => selectedToolsSet.has(t.name)) + .map((t) => ({ name: t.name, scopes: t.scopes, secrets: t.secrets ?? [] })), + null, + 2 + ); + + const hasRequirements = requiredScopes.length > 0 || requiredSecrets.length > 0; + const hasScopes = requiredScopes.length > 0; return ( -
-
-
-

- Scope calculator +
+ {/* Header */} +
+
+

+ + Selected tools + {selectedToolNames.length > 0 && ( + + {selectedToolNames.length} of {tools.length} + + )}

-
+
-

- Select the tools you plan to use to see the required OAuth scopes. -

+ {/* Tools Grid */}
-
- {tools.map((tool) => ( - - ))} +
+ {tools.map((tool) => { + const toolHasScopes = (tool.scopes?.length ?? 0) > 0; + const toolHasSecrets = (tool.secrets?.length ?? 0) > 0; + return ( + + ); + })}
-
-

- Required scopes{" "} - {selectedTools.size > 0 && ( - - ({requiredScopes.length}) - + {/* Copy Actions */} + {selectedToolNames.length > 0 && ( +
+ + + {showAdvanced && requiredScopes.length > 0 && ( + + )} + {requiredSecrets.length > 0 && ( + )} +
+ )} + + {/* Requirements Summary */} +
+

+ Requirements

- {requiredScopes.length > 0 ? ( -
    + {selectedToolNames.length === 0 ? ( +

    + Select tools to see requirements +

    + ) : ( + <> + {/* Scopes indicator - only shown when advanced is enabled and there are scopes */} + {showAdvanced && requiredScopes.length > 0 && ( +
    + + + {requiredScopes.length} OAuth scope{requiredScopes.length > 1 ? "s" : ""} required + +
    + )} + + {/* Secrets - always show the actual secret names */} +
    + 0 ? "text-amber-400" : "text-muted-foreground/50"}`} /> + {requiredSecrets.length > 0 ? ( +
    + Secrets: + {requiredSecrets.map((secret) => ( + + {secret} + + ))} +
    + ) : ( + No secrets required + )} +
    + + {/* Show OAuth Scopes Toggle - inside the box, only when there are scopes */} + {requiredScopes.length > 0 && ( +
    + +
    + )} + + )} +
+ + {/* OAuth Scopes Details - only shown when advanced is enabled and there are scopes */} + {showAdvanced && selectedToolNames.length > 0 && requiredScopes.length > 0 && ( +
+
+

+ + Required OAuth scopes + + {requiredScopes.length} + +

+ +
+
{requiredScopes.map((scope) => ( -
  • {scope} -
    • + ))} - - ) : ( -

    - Select tools above to see required scopes -

    - )} -
    +
    +

    + )}
    ); } + +export { CopyButton }; diff --git a/app/_components/toolkit-docs/__tests__/AvailableToolsTable.test.tsx b/app/_components/toolkit-docs/__tests__/AvailableToolsTable.test.tsx index 149654e60..a955cd035 100644 --- a/app/_components/toolkit-docs/__tests__/AvailableToolsTable.test.tsx +++ b/app/_components/toolkit-docs/__tests__/AvailableToolsTable.test.tsx @@ -1,13 +1,15 @@ import { describe, expect, it } from "vitest"; import { - buildToolsTableData, + buildScopeDisplayItems, + buildSecretDisplayItems, + filterTools, toToolAnchorId, } from "../components/AvailableToolsTable"; describe("AvailableToolsTable helpers", () => { - it("builds table data with fallback descriptions", () => { - const data = buildToolsTableData([ + it("builds secret display items from secrets info", () => { + const items = buildSecretDisplayItems( { name: "CreateIssue", qualifiedName: "Github.CreateIssue", @@ -15,55 +17,65 @@ describe("AvailableToolsTable helpers", () => { secretsInfo: [ { name: "GITHUB_API_KEY", type: "api_key" }, { name: "SECONDARY_TOKEN", type: "token" }, - { name: "ANOTHER_API_KEY", type: "api_key" }, ], }, - { - name: "ListPullRequests", - qualifiedName: "Github.ListPullRequests", - description: null, - secrets: ["WEBHOOK_SECRET"], - }, - { - name: "GetRepo", - qualifiedName: "Github.GetRepo", - description: null, - secrets: [], - }, - ]); + { secretsDisplay: "summary" } + ); - expect(data).toEqual([ - ["Github.CreateIssue", "Create an issue", "API key, Token"], - ["Github.ListPullRequests", "No description provided.", "WEBHOOK_SECRET"], - ["Github.GetRepo", "No description provided.", "None"], + expect(items).toEqual([ + { label: "API key", href: undefined }, + { label: "Token", href: undefined }, ]); }); - it("includes secret type docs base URL when provided", () => { - const data = buildToolsTableData( - [ - { - name: "CreateIssue", - qualifiedName: "Github.CreateIssue", - description: "Create an issue", - secretsInfo: [{ name: "GITHUB_API_KEY", type: "api_key" }], - }, - ], + it("adds hrefs for secret type docs when base URL is provided", () => { + const items = buildSecretDisplayItems( { - secretsDisplay: "types", - secretTypeDocsBaseUrl: "/references/secrets", - } + name: "CreateIssue", + qualifiedName: "Github.CreateIssue", + description: "Create an issue", + secretsInfo: [{ name: "GITHUB_API_KEY", type: "api_key" }], + }, + { secretsDisplay: "types", secretTypeDocsBaseUrl: "/references/secrets" } ); - expect(data).toEqual([ - [ - "Github.CreateIssue", - "Create an issue", - "API key (/references/secrets/api_key)", - ], + expect(items).toEqual([ + { label: "API key", href: "/references/secrets/api_key" }, ]); }); + it("filters tools by query and filter mode", () => { + const tools = [ + { + name: "CreateIssue", + qualifiedName: "Github.CreateIssue", + description: "Create an issue", + scopes: ["repo"], + secrets: ["API_KEY"], + }, + { + name: "ListRepos", + qualifiedName: "Github.ListRepos", + description: "List repositories", + scopes: [], + secrets: [], + }, + ]; + + expect(filterTools(tools, "list", "all", [])).toEqual([tools[1]]); + expect(filterTools(tools, "", "has_scopes", [])).toEqual([tools[0]]); + expect(filterTools(tools, "", "no_secrets", [])).toEqual([tools[1]]); + expect(filterTools(tools, "", "all", ["repo"])).toEqual([tools[0]]); + expect(filterTools(tools, "", "all", ["missing"])).toEqual([]); + }); + + it("builds scope display items from scopes", () => { + expect( + buildScopeDisplayItems([" scope.one ", "", "scope.two", "scope.one"]) + ).toEqual(["scope.one", "scope.two"]); + }); + + it("matches the anchor id logic used by TableOfContents", () => { expect(toToolAnchorId("Github.CreateIssue")).toBe("githubcreateissue"); expect(toToolAnchorId("Slack Api.Send Message")).toBe( diff --git a/app/_components/toolkit-docs/__tests__/ToolkitPage.test.tsx b/app/_components/toolkit-docs/__tests__/ToolkitPage.test.tsx new file mode 100644 index 000000000..054e88a15 --- /dev/null +++ b/app/_components/toolkit-docs/__tests__/ToolkitPage.test.tsx @@ -0,0 +1,11 @@ +import { describe, expect, it } from "vitest"; + +import { buildPipPackageName } from "../components/ToolkitPage"; + +describe("ToolkitPage helpers", () => { + it("builds a pip package name from toolkit id", () => { + expect(buildPipPackageName("Github")).toBe("arcade_github"); + expect(buildPipPackageName("Google Sheets")).toBe("arcade_google_sheets"); + expect(buildPipPackageName("Slack-Api")).toBe("arcade_slack_api"); + }); +}); diff --git a/app/_components/toolkit-docs/components/AvailableToolsTable.tsx b/app/_components/toolkit-docs/components/AvailableToolsTable.tsx index e5677d75e..a5a5530af 100644 --- a/app/_components/toolkit-docs/components/AvailableToolsTable.tsx +++ b/app/_components/toolkit-docs/components/AvailableToolsTable.tsx @@ -1,75 +1,297 @@ "use client"; -import TableOfContents from "../../table-of-contents"; +import { Check, KeyRound } from "lucide-react"; +import { useEffect, useMemo, useRef, useState } from "react"; + +import { SCROLLING_CELL } from "../constants"; import type { AvailableToolsTableProps, SecretType } from "../types"; +import { normalizeScopes } from "./ScopesDisplay"; + +/** + * A cell content wrapper that auto-scrolls on hover when content overflows. + * Scroll duration is proportional to content length. + * Only scrolls when this specific cell is hovered, with a delay. + */ +function ScrollingCell({ + children, + className = "", + pixelsPerSecond = SCROLLING_CELL.pixelsPerSecond, + delay = SCROLLING_CELL.delayMs, +}: { + children: React.ReactNode; + className?: string; + pixelsPerSecond?: number; + delay?: number; +}) { + const containerRef = useRef(null); + const contentRef = useRef(null); + const [isOverflowing, setIsOverflowing] = useState(false); + const [isHovered, setIsHovered] = useState(false); + const [scrollOffset, setScrollOffset] = useState(0); + const [duration, setDuration] = useState(0); + const hoverTimeoutRef = useRef(null); + + useEffect(() => { + const checkOverflow = () => { + if (containerRef.current && contentRef.current) { + const containerWidth = containerRef.current.offsetWidth; + const contentWidth = contentRef.current.scrollWidth; + const overflow = contentWidth - containerWidth; + setIsOverflowing(overflow > 0); + setScrollOffset(overflow + SCROLLING_CELL.extraPadding); + // Calculate duration based on scroll distance + setDuration(Math.max(SCROLLING_CELL.minDurationMs, (overflow + SCROLLING_CELL.extraPadding) / pixelsPerSecond * 1000)); + } + }; + checkOverflow(); + window.addEventListener("resize", checkOverflow); + return () => window.removeEventListener("resize", checkOverflow); + }, [children, pixelsPerSecond]); + + const handleMouseEnter = () => { + if (isOverflowing) { + hoverTimeoutRef.current = setTimeout(() => { + setIsHovered(true); + }, delay); + } + }; + + const handleMouseLeave = () => { + if (hoverTimeoutRef.current) { + clearTimeout(hoverTimeoutRef.current); + hoverTimeoutRef.current = null; + } + setIsHovered(false); + }; + + useEffect(() => { + return () => { + if (hoverTimeoutRef.current) { + clearTimeout(hoverTimeoutRef.current); + } + }; + }, []); + + return ( +
    +
    + {children} +
    + {isOverflowing && ( +
    + )} +
    + ); +} export function toToolAnchorId(value: string): string { return value.toLowerCase().replace(/\s+/g, "-").replace(".", ""); } -export function buildToolsTableData( - tools: AvailableToolsTableProps["tools"], +export type AvailableToolsFilter = + | "all" + | "has_scopes" + | "no_scopes" + | "has_secrets" + | "no_secrets"; + +export type AvailableToolsSort = + | "name_asc" + | "name_desc" + | "scopes_first" + | "secrets_first" + | "selected_first"; + +type SecretDisplayItem = { + label: string; + href?: string; +}; + +const DEFAULT_SECRET_LABELS: Record = { + api_key: "API key", + token: "Token", + client_secret: "Client secret", + webhook_secret: "Webhook secret", + private_key: "Private key", + password: "Password", + unknown: "Unknown", +}; + +function normalizeBaseUrl(baseUrl?: string): string | undefined { + if (!baseUrl) { + return undefined; + } + return baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl; +} + +export function buildSecretDisplayItems( + tool: AvailableToolsTableProps["tools"][number], options?: Pick< AvailableToolsTableProps, "secretsDisplay" | "secretTypeLabels" | "secretTypeDocsBaseUrl" > -): string[][] { - const secretTypeLabels: Record = { - api_key: "API key", - token: "Token", - client_secret: "Client secret", - webhook_secret: "Webhook secret", - private_key: "Private key", - password: "Password", - unknown: "Unknown", +): SecretDisplayItem[] { + const displayMode = options?.secretsDisplay ?? "summary"; + const secretsInfo = tool.secretsInfo ?? []; + const secrets = tool.secrets ?? []; + const secretTypeLabels = { + ...DEFAULT_SECRET_LABELS, ...options?.secretTypeLabels, }; + const baseUrl = normalizeBaseUrl(options?.secretTypeDocsBaseUrl); - const normalizeBaseUrl = (baseUrl: string | undefined): string | undefined => { - if (!baseUrl) { - return undefined; - } - return baseUrl.endsWith("/") ? baseUrl.slice(0, -1) : baseUrl; - }; + if (displayMode === "names") { + return secrets.map((secret) => ({ label: secret })); + } - const baseUrl = normalizeBaseUrl(options?.secretTypeDocsBaseUrl); + if ( + displayMode === "types" || + (displayMode === "summary" && secretsInfo.length > 0) + ) { + const uniqueTypes = Array.from( + new Set(secretsInfo.map((secret) => secret.type)) + ); - const formatSecretTypeLabel = (type: SecretType): string => { - const label = secretTypeLabels[type] ?? "Unknown"; - return baseUrl ? `${label} (${baseUrl}/${type})` : label; + return uniqueTypes.map((type) => ({ + label: secretTypeLabels[type] ?? "Unknown", + href: baseUrl ? `${baseUrl}/${type}` : undefined, + })); + } + + return secrets.map((secret) => ({ label: secret })); +} + +export function buildScopeDisplayItems(scopes: string[]): string[] { + return normalizeScopes(scopes); +} + +export function sortTools( + tools: AvailableToolsTableProps["tools"], + sort: AvailableToolsSort, + selectedTools?: Set +): AvailableToolsTableProps["tools"] { + const sorted = [...tools]; + + const countSecrets = (tool: (typeof tools)[number]): number => { + const names = [ + ...(tool.secrets ?? []), + ...((tool.secretsInfo ?? []).map((secret) => secret.name)), + ]; + return new Set(names).size; }; - const formatSecretSummary = ( - tool: AvailableToolsTableProps["tools"][number] - ): string => { - const displayMode = options?.secretsDisplay ?? "summary"; - const secretsInfo = tool.secretsInfo ?? []; - const secrets = tool.secrets ?? []; + switch (sort) { + case "name_asc": + return sorted.sort((a, b) => a.name.localeCompare(b.name)); + case "name_desc": + return sorted.sort((a, b) => b.name.localeCompare(a.name)); + case "scopes_first": + return sorted.sort((a, b) => { + const aScopes = (a.scopes ?? []).length; + const bScopes = (b.scopes ?? []).length; + if (aScopes !== bScopes) return bScopes - aScopes; + return a.name.localeCompare(b.name); + }); + case "secrets_first": + return sorted.sort((a, b) => { + const aSecrets = countSecrets(a); + const bSecrets = countSecrets(b); + if (aSecrets !== bSecrets) return bSecrets - aSecrets; + return a.name.localeCompare(b.name); + }); + case "selected_first": + return sorted.sort((a, b) => { + const aSelected = selectedTools?.has(a.name) ? 1 : 0; + const bSelected = selectedTools?.has(b.name) ? 1 : 0; + if (aSelected !== bSelected) return bSelected - aSelected; + return a.name.localeCompare(b.name); + }); + default: + return sorted; + } +} + +export function filterTools( + tools: AvailableToolsTableProps["tools"], + query: string, + filter: AvailableToolsFilter, + selectedScopes: string[] +): AvailableToolsTableProps["tools"] { + const normalizedQuery = query.trim().toLowerCase(); - if (displayMode === "names") { - return secrets.length === 0 ? "None" : secrets.join(", "); + return tools.filter((tool) => { + const haystack = [ + tool.name, + tool.qualifiedName, + tool.description ?? "", + ] + .join(" ") + .toLowerCase(); + const matchesQuery = + normalizedQuery.length === 0 || haystack.includes(normalizedQuery); + + if (!matchesQuery) { + return false; } - if ( - displayMode === "types" || - (displayMode === "summary" && secretsInfo.length > 0) - ) { - const uniqueTypes = Array.from( - new Set(secretsInfo.map((secret) => secret.type)) - ); - return uniqueTypes.length === 0 - ? "None" - : uniqueTypes.map((type) => formatSecretTypeLabel(type)).join(", "); + const toolScopes = buildScopeDisplayItems(tool.scopes ?? []); + const hasScopes = toolScopes.length > 0; + const hasSecrets = + (tool.secretsInfo?.length ?? 0) > 0 || (tool.secrets?.length ?? 0) > 0; + + const matchesScopes = + selectedScopes.length === 0 || + selectedScopes.some((scope) => toolScopes.includes(scope)); + + if (!matchesScopes) { + return false; } - return secrets.length === 0 ? "None" : secrets.join(", "); - }; + switch (filter) { + case "has_scopes": + return hasScopes; + case "no_scopes": + return !hasScopes; + case "has_secrets": + return hasSecrets; + case "no_secrets": + return !hasSecrets; + case "all": + default: + return true; + } + }); +} - return tools.map((tool) => [ - tool.qualifiedName, - tool.description ?? "No description provided.", - formatSecretSummary(tool), - ]); +function truncateItems(items: string[], maxItems: number) { + if (items.length <= maxItems) { + return { visibleItems: items, remainingCount: 0 }; + } + return { + visibleItems: items.slice(0, maxItems), + remainingCount: items.length - maxItems, + }; } /** @@ -83,6 +305,17 @@ export function AvailableToolsTable({ secretsDisplay = "summary", secretTypeLabels, secretTypeDocsBaseUrl, + enableSearch = true, + enableFilters = true, + enableScopeFilter = true, + searchPlaceholder = "Search tools...", + filterLabel = "Filter", + scopeFilterLabel = "Filter by scope", + scopeFilterDescription = "Select scopes to narrow the tool list.", + defaultFilter = "all", + selectedTools, + onToggleSelection, + showSelection = false, }: AvailableToolsTableProps) { if (!tools || tools.length === 0) { return ( @@ -92,15 +325,227 @@ export function AvailableToolsTable({ ); } + const [query, setQuery] = useState(""); + const [filter, setFilter] = useState(defaultFilter); + const [sort, setSort] = useState("name_asc"); + const [selectedScopes, setSelectedScopes] = useState([]); + + const availableScopes = useMemo(() => { + const allScopes = tools.flatMap((tool) => + buildScopeDisplayItems(tool.scopes ?? []) + ); + return Array.from(new Set(allScopes)).sort(); + }, [tools]); + + const filteredTools = useMemo(() => { + const filtered = filterTools(tools, query, filter, selectedScopes); + return sortTools(filtered, sort, selectedTools); + }, [tools, query, filter, selectedScopes, sort, selectedTools]); + + const toggleScope = (scope: string) => { + setSelectedScopes((current) => { + if (current.includes(scope)) { + return current.filter((item) => item !== scope); + } + return [...current, scope]; + }); + }; + + const clearScopes = () => { + setSelectedScopes([]); + }; + return ( - +
    + {(enableSearch || enableFilters) && ( +
    + {enableSearch && ( +
    + + + + + setQuery(event.target.value)} + placeholder={searchPlaceholder} + type="search" + value={query} + /> +
    + )} + {enableFilters && ( + + )} + + + {filteredTools.length} of {tools.length} + +
    + )} + {filteredTools.length === 0 ? ( +
    +

    No tools match your search.

    +
    + ) : ( +
    +
    + + + + {showSelection && ( + + )} + + + + + + + {filteredTools.map((tool, index) => { + const scopes = buildScopeDisplayItems(tool.scopes ?? []); + const { visibleItems, remainingCount } = truncateItems(scopes, 2); + const secretItems = buildSecretDisplayItems(tool, { + secretsDisplay, + secretTypeLabels, + secretTypeDocsBaseUrl, + }); + const isSelected = selectedTools?.has(tool.name) ?? false; + const rowBg = isSelected + ? "bg-brand-accent/10" + : index % 2 === 0 + ? "bg-neutral-dark/20" + : "bg-neutral-dark/5"; + + return ( + { + window.location.hash = `#${toToolAnchorId(tool.qualifiedName)}`; + }} + > + {showSelection && ( + + )} + + + + + ); + })} + +
    + + + Tool name + + Description + + + + {secretsColumnLabel} + +
    + + + + + {tool.qualifiedName} + + + + + + {tool.description ?? "No description provided."} + + + + {secretItems.length === 0 ? ( + — + ) : ( +
    + {secretItems.map((item) => + item.href ? ( + + + {item.label} + + + ) : ( + + {item.label} + + ) + )} +
    + )} +
    +
    +
    + )} +
    ); } diff --git a/app/_components/toolkit-docs/components/DynamicCodeBlock.tsx b/app/_components/toolkit-docs/components/DynamicCodeBlock.tsx index 82184c5b1..7afa40af1 100644 --- a/app/_components/toolkit-docs/components/DynamicCodeBlock.tsx +++ b/app/_components/toolkit-docs/components/DynamicCodeBlock.tsx @@ -1,22 +1,39 @@ "use client"; -import { Button } from "@arcadeai/design-system"; -import { ChevronDown } from "lucide-react"; +import { X } from "lucide-react"; import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; -import { - atomDark, - oneLight, -} from "react-syntax-highlighter/dist/cjs/styles/prism"; +import { atomDark } from "react-syntax-highlighter/dist/cjs/styles/prism"; import { useEffect, useMemo, useState } from "react"; import { CopyButton } from "../../tabbed-code-block/copy-button"; -import { LanguageTabs } from "../../tabbed-code-block/language-tabs"; import type { DynamicCodeBlockProps, ExampleParameterValue, ToolCodeExample, } from "../types"; +/** + * Python logo SVG icon + */ +function PythonIcon({ className }: { className?: string }) { + return ( + + + + ); +} + +/** + * TypeScript logo SVG icon + */ +function TypeScriptIcon({ className }: { className?: string }) { + return ( + + + + ); +} + type LanguageKey = "python" | "javascript"; const LANGUAGE_LABELS: Record = { @@ -241,6 +258,85 @@ export function generatePythonExample(codeExample: ToolCodeExample): string { return lines.join("\n"); } +/** + * Code popup modal for displaying examples + */ +function CodePopup({ + code, + language, + onClose, +}: { + code: string; + language: "python" | "typescript"; + onClose: () => void; +}) { + const displayName = language === "python" ? "Python" : "TypeScript"; + const Icon = language === "python" ? PythonIcon : TypeScriptIcon; + const iconColor = language === "python" ? "text-[#3776AB]" : "text-[#3178C6]"; + + // Close on escape key + useEffect(() => { + const handleEscape = (e: KeyboardEvent) => { + if (e.key === "Escape") onClose(); + }; + document.addEventListener("keydown", handleEscape); + return () => document.removeEventListener("keydown", handleEscape); + }, [onClose]); + + return ( +
    +
    e.stopPropagation()} + > + {/* Header */} +
    +
    +
    + +
    + + {displayName} Example + +
    +
    + + +
    +
    + + {/* Code */} +
    + + {code} + +
    +
    +
    + ); +} + /** * DynamicCodeBlock * @@ -252,139 +348,61 @@ export function DynamicCodeBlock({ languages = DEFAULT_LANGUAGES, tabLabel, }: DynamicCodeBlockProps) { - const [isDarkMode, setIsDarkMode] = useState(false); - const [isExpanded, setIsExpanded] = useState(false); - const languageLabels = useMemo( - () => languages.map((lang) => LANGUAGE_LABELS[lang]), - [languages] - ); - - const [currentLanguage, setCurrentLanguage] = useState( - languageLabels[0] ?? "Python" - ); - - useEffect(() => { - const updateTheme = () => { - const html = document.documentElement; - const hasClassDark = html.classList.contains("dark"); - const hasClassLight = html.classList.contains("light"); - const hasDataThemeDark = html.getAttribute("data-theme") === "dark"; - const hasDataThemeLight = html.getAttribute("data-theme") === "light"; - const systemPrefersDark = - window.matchMedia?.("(prefers-color-scheme: dark)").matches ?? false; - - let isDark: boolean; - if (hasClassDark || hasDataThemeDark) { - isDark = true; - } else if (hasClassLight || hasDataThemeLight) { - isDark = false; - } else { - isDark = systemPrefersDark; - } - - setIsDarkMode(isDark); - }; - - updateTheme(); - - const observer = new MutationObserver(updateTheme); - observer.observe(document.documentElement, { - attributes: true, - attributeFilter: ["class", "data-theme"], - }); - - const mediaQuery = window.matchMedia?.("(prefers-color-scheme: dark)"); - mediaQuery?.addEventListener("change", updateTheme); - - return () => { - observer.disconnect(); - mediaQuery?.removeEventListener("change", updateTheme); - }; - }, []); + const [activePopup, setActivePopup] = useState<"python" | "typescript" | null>(null); const codeByLanguage = useMemo( () => ({ - Python: generatePythonExample(codeExample), - JavaScript: generateJavaScriptExample(codeExample), + python: generatePythonExample(codeExample), + typescript: generateJavaScriptExample(codeExample), }), [codeExample] ); - const selectedCode = codeByLanguage[currentLanguage as "Python" | "JavaScript"]; const displayLabel = tabLabel ?? codeExample.tabLabel; - const syntaxTheme = isDarkMode ? atomDark : oneLight; - const lowerCaseLanguage = currentLanguage.toLowerCase(); - - if (!isExpanded) { - return ( -
    - {displayLabel && ( -

    {displayLabel}

    - )} - -
    - ); - } + const showPython = languages.includes("python"); + const showTypeScript = languages.includes("javascript"); return ( -
    +
    {displayLabel && ( -

    {displayLabel}

    +

    {displayLabel}

    )} -
    -
    - {languageLabels.length > 1 && ( - - )} -
    - -
    -
    -
    -
    - - {lowerCaseLanguage} - -
    - -
    -
    - + {showPython && ( +
    +
    + +
    + Python example + + )} + {showTypeScript && ( + + )}
    + + {/* Popup Modal */} + {activePopup && ( + setActivePopup(null)} + /> + )}
    ); } diff --git a/app/_components/toolkit-docs/components/ParametersTable.tsx b/app/_components/toolkit-docs/components/ParametersTable.tsx index d776fa019..160d26d5e 100644 --- a/app/_components/toolkit-docs/components/ParametersTable.tsx +++ b/app/_components/toolkit-docs/components/ParametersTable.tsx @@ -1,5 +1,7 @@ "use client"; +import { CheckCircle2 } from "lucide-react"; + import type { ParametersTableProps, ToolParameter } from "../types"; /** @@ -37,88 +39,97 @@ export function ParametersTable({ }: ParametersTableProps) { if (!parameters || parameters.length === 0) { return ( -

    - No parameters required. -

    +
    +

    No parameters required.

    +
    ); } return ( - - - - - - - - - - - {parameters.map((param, index) => { - const enumValues = formatEnumValues(param.enum); - const rowClass = - index % 2 === 0 - ? "bg-neutral-dark" - : "border-neutral-dark-medium border-b bg-transparent"; +
    +
    - Parameter - - Type - - Required - - Description -
    + + + + + + + + + + {parameters.map((param, index) => { + const enumValues = formatEnumValues(param.enum); - return ( - - - - - + + + + - - ); - })} - -
    + Parameter + + Type + + Req. + + Description +
    - {param.name} - - {formatParameterType(param)} - - {param.required ? "Yes" : "No"} - - {param.description ?? "No description provided."} - {enumValues.length > 0 && ( -
    - Options:{" "} - {enumValues.map((value, valueIndex) => { - const content = {value}; - const separator = - valueIndex < enumValues.length - 1 ? ", " : ""; + return ( +
    +
    + {param.required && ( + + )} + {param.name} +
    +     		+ + {formatParameterType(param)} + + + {param.required ? ( + + ) : ( + — + )} + + {param.description ?? "No description provided."} + {enumValues.length > 0 && ( +
    + {enumValues.map((value) => { + const chip = ( + + {value} + + ); - if (enumBaseUrl) { - return ( - + if (enumBaseUrl) { + return ( - {content} + {chip} - {separator} - - ); - } + ); + } - return ( - - {content} - {separator} - - ); - })} -
    - )} -
    + return {chip}; + })} +
    + )} + + + ); + })} + + +
    ); } diff --git a/app/_components/toolkit-docs/components/ScopesDisplay.tsx b/app/_components/toolkit-docs/components/ScopesDisplay.tsx index 79db24130..b65bd6017 100644 --- a/app/_components/toolkit-docs/components/ScopesDisplay.tsx +++ b/app/_components/toolkit-docs/components/ScopesDisplay.tsx @@ -1,6 +1,6 @@ "use client"; -import { Callout } from "nextra/components"; +import { ShieldCheck } from "lucide-react"; import type { ScopesDisplayProps } from "../types"; @@ -23,8 +23,8 @@ export function normalizeScopes(scopes: string[]): string[] { function ScopesInline({ scopes }: { scopes: string[] }) { if (scopes.length === 0) { return ( - - No scopes required. + + None required ); } @@ -33,8 +33,10 @@ function ScopesInline({ scopes }: { scopes: string[] }) {
    {scopes.map((scope) => ( {scope} @@ -46,18 +48,22 @@ function ScopesInline({ scopes }: { scopes: string[] }) { function ScopesList({ scopes }: { scopes: string[] }) { if (scopes.length === 0) { return ( -

    No scopes required.

    +

    None required

    ); } return ( -
      +
      {scopes.map((scope) => ( -
    • - {scope} -
      • + + {scope} + ))} -
    +
    ); } @@ -74,10 +80,17 @@ export function ScopesDisplay({ const normalizedScopes = normalizeScopes(scopes); if (variant === "callout") { + const heading = title?.trim(); return ( - +
    + {heading && ( +
    + + {heading} +
    + )} - +
    ); } diff --git a/app/_components/toolkit-docs/components/ToolSection.tsx b/app/_components/toolkit-docs/components/ToolSection.tsx index bf19a2b42..40080afbe 100644 --- a/app/_components/toolkit-docs/components/ToolSection.tsx +++ b/app/_components/toolkit-docs/components/ToolSection.tsx @@ -1,12 +1,101 @@ "use client"; +import { Button } from "@arcadeai/design-system"; +import { + Check, + Copy, + ExternalLink, + KeyRound, + Lock, + ShieldCheck, + Wrench, +} from "lucide-react"; +import { useCallback, useState } from "react"; + import { DocumentationChunkRenderer, hasChunksAt } from "./DocumentationChunkRenderer"; import { DynamicCodeBlock } from "./DynamicCodeBlock"; import { ParametersTable } from "./ParametersTable"; import { ScopesDisplay } from "./ScopesDisplay"; import { toToolAnchorId } from "./AvailableToolsTable"; +import { getDashboardUrl } from "../../dashboard-link"; +import { useOrySession } from "../../../_lib/ory-session-context"; import type { ToolSectionProps } from "../types"; +function CopyToolButton({ tool }: { tool: ToolSectionProps["tool"] }) { + const [copied, setCopied] = useState(false); + + const handleCopy = useCallback(async () => { + const toolDefinition = { + name: tool.qualifiedName, + description: tool.description, + parameters: tool.parameters.map((p) => ({ + name: p.name, + type: p.type, + required: p.required, + description: p.description, + ...(p.enum ? { enum: p.enum } : {}), + })), + scopes: tool.auth?.scopes ?? [], + secrets: tool.secrets, + output: tool.output, + }; + + try { + await navigator.clipboard.writeText(JSON.stringify(toolDefinition, null, 2)); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + } catch { + console.error("Failed to copy tool definition"); + } + }, [tool]); + + return ( + + ); +} + +function CopyScopesButton({ scopes }: { scopes: string[] }) { + const [copied, setCopied] = useState(false); + + const handleCopy = useCallback(async () => { + try { + await navigator.clipboard.writeText(scopes.join("\n")); + setCopied(true); + setTimeout(() => setCopied(false), 2000); + } catch { + console.error("Failed to copy scopes"); + } + }, [scopes]); + + return ( + + ); +} + export function shouldRenderDefaultSection( chunks: ToolSectionProps["tool"]["documentationChunks"], location: "description" | "parameters" | "auth" | "secrets" | "output" @@ -19,10 +108,19 @@ export function shouldRenderDefaultSection( * * Renders a single tool section with parameters, scopes, secrets, output, and example. */ -export function ToolSection({ tool }: ToolSectionProps) { +export function ToolSection({ + tool, + isSelected = false, + showSelection = false, + onToggleSelection, +}: ToolSectionProps) { + const [showAdvanced, setShowAdvanced] = useState(false); const anchorId = toToolAnchorId(tool.qualifiedName); const scopes = tool.auth?.scopes ?? []; const secretsInfo = tool.secretsInfo ?? []; + const hasScopes = scopes.length > 0; + const hasSecrets = + (tool.secrets?.length ?? 0) > 0 || (tool.secretsInfo?.length ?? 0) > 0; const showDescription = shouldRenderDefaultSection( tool.documentationChunks, @@ -39,17 +137,64 @@ export function ToolSection({ tool }: ToolSectionProps) { ); const showOutput = shouldRenderDefaultSection(tool.documentationChunks, "output"); + const { email, loading } = useOrySession(); + const isLoggedIn = !loading && !!email; + const loginUrl = getDashboardUrl(); + const executeUrl = getDashboardUrl( + `playground/execute?toolId=${encodeURIComponent(tool.qualifiedName)}` + ); + + const getActionHref = (href: string) => (isLoggedIn ? href : loginUrl); + const getActionTitle = (label: string) => + isLoggedIn ? label : "Sign in to use this in the Arcade dashboard"; + + const actionHint = isLoggedIn + ? "Run this tool directly in the Arcade dashboard." + : "Sign in to run tools. You'll be redirected to the dashboard login."; + return ( -
    -

    {tool.qualifiedName}

    +
    + {/* Tool Header */} +
    +
    +
    + +
    +

    + {tool.qualifiedName} +

    +
    +
    + + {showSelection && ( + + )} +
    +
    + {/* Description */} {showDescription && ( -

    +

    {tool.description ?? "No description provided."}

    )} @@ -64,122 +209,200 @@ export function ToolSection({ tool }: ToolSectionProps) { position="after" /> -

    Parameters

    - - {showParameters && } - - + {/* Parameters */} +
    +

    + Parameters +

    + + {showParameters && } + + +
    -

    Scopes

    - - {showAuth && } - - + {/* Requirements Summary */} +
    +

    + Requirements +

    + + {/* Scopes indicator - only shown when advanced is enabled and there are scopes */} + {showAdvanced && hasScopes && ( +
    + + + Requires {scopes.length} OAuth scope{scopes.length > 1 ? "s" : ""} + +
    + )} -

    Secrets

    - - {showSecrets && ( -
    - {tool.secrets.length === 0 ? ( -

    No secrets required.

    - ) : ( -
      + {/* Secrets - always show the actual secret names */} +
      + + {hasSecrets ? ( +
      + Secrets: {secretsInfo.length > 0 ? secretsInfo.map((secret) => ( -
    • - {secret.name} - - ({secret.type}) - -
      • + + {secret.name} + ({secret.type}) + )) : tool.secrets.map((secret) => ( -
    • - {secret} -
      • + + {secret} + ))} -
    +
    + ) : ( + No secrets required )}
    + + {/* Show OAuth Scopes Toggle - inside the box, only when there are scopes */} + {hasScopes && ( +
    + +
    + )} +
    + + {/* OAuth Scopes Details - only shown when advanced mode is enabled and there are scopes */} + {showAdvanced && hasScopes && ( +
    +
    +

    + + Required OAuth scopes +

    + +
    + + {showAuth && } + + +
    )} - - -

    Output

    - - {showOutput && ( -
    - {tool.output ? ( - <> -

    - Type: {tool.output.type} -

    - {tool.output.description && ( -

    {tool.output.description}

    + {/* Output */} +
    +

    + Output +

    + + {showOutput && ( +
    + {tool.output ? ( +
    + Type: + + {tool.output.type} + + {tool.output.description && ( + — {tool.output.description} + )} +
    + ) : ( +

    No output schema provided.

    + )} +
    + )} + + +
    + + {/* Try it in Arcade */} + -

    Example

    + {/* Code Example */} {tool.codeExample ? ( - +
    + +
    ) : ( -

    - No example available for this tool. +

    + No code example available for this tool.

    )} diff --git a/app/_components/toolkit-docs/components/ToolkitDocsPreview.tsx b/app/_components/toolkit-docs/components/ToolkitDocsPreview.tsx new file mode 100644 index 000000000..29cbe4e43 --- /dev/null +++ b/app/_components/toolkit-docs/components/ToolkitDocsPreview.tsx @@ -0,0 +1,25 @@ +import type { ToolkitData } from "../types"; +import { readToolkitData } from "@/app/_lib/toolkit-data"; +import { ToolkitPage } from "./ToolkitPage"; + +type ToolkitDocsPreviewProps = { + toolkitId: string; + fallbackData?: ToolkitData; +}; + +export async function ToolkitDocsPreview({ + toolkitId, + fallbackData, +}: ToolkitDocsPreviewProps) { + const toolkitData = (await readToolkitData(toolkitId)) ?? fallbackData; + + if (!toolkitData) { + return ( +

    + Toolkit data not found for {toolkitId}. +

    + ); + } + + return ; +} diff --git a/app/_components/toolkit-docs/components/ToolkitHeader.tsx b/app/_components/toolkit-docs/components/ToolkitHeader.tsx index ead70c1a3..37bf1fb15 100644 --- a/app/_components/toolkit-docs/components/ToolkitHeader.tsx +++ b/app/_components/toolkit-docs/components/ToolkitHeader.tsx @@ -6,23 +6,22 @@ import { getToolkitIconByName, ProBadge, } from "@arcadeai/design-system"; +import { KeyRound, Wrench } from "lucide-react"; import type React from "react"; +import { TYPE_CONFIG } from "../../../en/resources/integrations/components/type-config"; +import { + AUTH_TYPE_LABELS, + DEFAULT_AUTHOR, + getAuthProviderDocsUrl, + getGitHubRepoUrl, + getPackageName, + getPyPIUrl, + LICENSE_BADGE, + PYPI_BADGES, +} from "../constants"; import type { ToolkitHeaderProps, ToolkitType } from "../types"; -/** - * Configuration for toolkit type badges - */ -const TYPE_CONFIG: Record< - Exclude, - { label: string; color: string } -> = { - arcade: { label: "Arcade", color: "text-brand-accent" }, - arcade_starter: { label: "Starter", color: "text-gray-500" }, - verified: { label: "Verified", color: "text-green-600" }, - community: { label: "Community", color: "text-purple-600" }, -}; - /** * Renders toolkit type, BYOC, and Pro badges */ @@ -36,6 +35,7 @@ function ToolkitBadges({ isPro: boolean; }) { const typeInfo = type !== "auth" ? TYPE_CONFIG[type] : null; + const TypeIcon = typeInfo?.icon; const showBadges = isPro || isByoc || typeInfo; @@ -45,12 +45,13 @@ function ToolkitBadges({ return (
    - {typeInfo && ( + {typeInfo && TypeIcon && ( - {typeInfo.label} + + {typeInfo.label} )} {isByoc && } @@ -87,17 +88,20 @@ export function ToolkitHeader({ metadata, auth, version, - author = "Arcade", + author = DEFAULT_AUTHOR, + toolStats, }: ToolkitHeaderProps): React.ReactElement { // Get the icon component from Design System const IconComponent = getToolkitIconByName(label); + const iconUrl = metadata.iconUrl; + const packageName = getPackageName(id); // Determine auth display const authProviderName = auth?.providerId ? auth.providerId.charAt(0).toUpperCase() + auth.providerId.slice(1) : null; const authDocsUrl = auth?.providerId - ? `/references/auth-providers/${auth.providerId.toLowerCase()}` + ? getAuthProviderDocsUrl(auth.providerId) : null; const authProviderLink = authProviderName && authDocsUrl ? ( @@ -111,21 +115,33 @@ export function ToolkitHeader({ return (
    -
    - {/* Icon */} - {IconComponent && ( -
    -
    - +
    + {/* Icon - centered vertically */} + {IconComponent ? ( +
    +
    +
    - )} + ) : iconUrl ? ( +
    +
    + {`${label} +
    +
    + ) : null} {/* Content */} -
    +
    {/* Badges */} - Description: +

    {description}

    )} - {/* Author */} -

    - Author: - {author} -

    - - {/* Version (optional) */} - {version && ( -

    - Version: - {version} -

    + {/* Info Grid */} +
    + {/* Author */} +
    + Author: + {author} +
    + + {/* Version */} + {version && ( +
    + Version: + + {version} + +
    + )} + + {/* Code link */} + {id && ( +
    + Code: + + GitHub + + + + +
    + )} + + {/* Auth info */} +
    + Auth: + + {auth?.type === "oauth2" ? ( + <> + {AUTH_TYPE_LABELS.oauth2} + {authProviderLink && <> via the {authProviderLink}} + + ) : auth?.type === "api_key" ? ( + AUTH_TYPE_LABELS.api_key + ) : auth?.type === "mixed" ? ( + <> + {AUTH_TYPE_LABELS.mixed} + {authProviderLink && <> via the {authProviderLink}} {AUTH_TYPE_LABELS.mixedSuffix} + + ) : ( + AUTH_TYPE_LABELS.none + )} + +
    +
    + + {/* Tool Stats */} + {toolStats && ( +
    + {/* Total tools */} +
    +
    + +
    +
    + {toolStats.total} + tools +
    +
    + + {/* Tools with secrets */} + {toolStats.withSecrets > 0 && ( +
    +
    + +
    +
    + {toolStats.withSecrets} + require secrets +
    +
    + )} +
    )} +
    +
    - {/* Code link */} - {id && ( -

    - Code: + {/* PyPI Badges */} + {id && ( +

    +
    + {PYPI_BADGES.map((badge) => ( - GitHub + {badge.alt} -

    - )} - - {/* Auth info */} -

    - Auth: - {auth?.type === "oauth2" ? ( - <> - User authorization - {authProviderLink && <> via the {authProviderLink}} - - ) : auth?.type === "api_key" ? ( - "API key authentication" - ) : auth?.type === "mixed" ? ( - <> - User authorization - {authProviderLink && <> via the {authProviderLink}} and API - key authentication - - ) : ( - "No authentication required" - )} -

    + ))} + + {LICENSE_BADGE.alt} + +
    -
    + )}
    ); } diff --git a/app/_components/toolkit-docs/components/ToolkitPage.tsx b/app/_components/toolkit-docs/components/ToolkitPage.tsx index 3b210e93a..4aeb53370 100644 --- a/app/_components/toolkit-docs/components/ToolkitPage.tsx +++ b/app/_components/toolkit-docs/components/ToolkitPage.tsx @@ -1,10 +1,193 @@ +"use client"; + +import { KeyRound } from "lucide-react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import ReactMarkdown from "react-markdown"; -import { AvailableToolsTable } from "./AvailableToolsTable"; -import { DocumentationChunkRenderer } from "./DocumentationChunkRenderer"; +import ScopePicker from "../../scope-picker"; +import ToolFooter from "../../tool-footer"; +import { AvailableToolsTable, toToolAnchorId } from "./AvailableToolsTable"; +import { DocumentationChunkRenderer, hasChunksAt } from "./DocumentationChunkRenderer"; import { ToolkitHeader } from "./ToolkitHeader"; import { ToolSection } from "./ToolSection"; -import type { ToolkitPageProps } from "../types"; +import type { ToolkitPageProps, ToolDefinition, ToolkitCategory } from "../types"; + +export function buildPipPackageName(toolkitId: string): string { + const normalized = toolkitId.toLowerCase().replace(/[^a-z0-9]+/g, "_"); + return `arcade_${normalized}`; +} + +function toTitleCaseCategory(category: ToolkitCategory): string { + return category + .split("-") + .map((part) => part.charAt(0).toUpperCase() + part.slice(1)) + .join(" "); +} + +/** + * Breadcrumb bar like older integration pages. + * + * Note: preview pages are dynamic, so we render this in-page. + */ +function BreadcrumbBar({ + label, + category, +}: { + label: string; + category: ToolkitCategory; +}) { + return ( + + ); +} + +/** + * Right sidebar that lists the tools on the page. + * Kept fixed and flush to the window edge, without narrowing main content. + * Auto-scrolls and highlights the currently visible section. + */ +function ToolsOnThisPage({ tools }: { tools: ToolDefinition[] }) { + const [activeId, setActiveId] = useState(null); + const sidebarRef = useRef(null); + const itemRefs = useRef>(new Map()); + + // Build list of all section IDs to observe + const sectionIds = useMemo(() => { + const ids = ["available-tools"]; + for (const tool of tools) { + ids.push(toToolAnchorId(tool.qualifiedName)); + } + ids.push("get-building"); + return ids; + }, [tools]); + + // Intersection Observer to track visible sections + useEffect(() => { + const observer = new IntersectionObserver( + (entries) => { + // Find the first visible section (from top) + const visibleEntries = entries + .filter((entry) => entry.isIntersecting) + .sort((a, b) => a.boundingClientRect.top - b.boundingClientRect.top); + + if (visibleEntries.length > 0) { + setActiveId(visibleEntries[0].target.id); + } + }, + { + rootMargin: "-80px 0px -60% 0px", + threshold: 0, + } + ); + + // Observe all sections + for (const id of sectionIds) { + const element = document.getElementById(id); + if (element) { + observer.observe(element); + } + } + + return () => observer.disconnect(); + }, [sectionIds]); + + // Auto-scroll sidebar to keep active item visible + useEffect(() => { + if (activeId && sidebarRef.current) { + const activeItem = itemRefs.current.get(activeId); + if (activeItem) { + const sidebar = sidebarRef.current; + const itemTop = activeItem.offsetTop; + const itemHeight = activeItem.offsetHeight; + const sidebarScrollTop = sidebar.scrollTop; + const sidebarHeight = sidebar.clientHeight; + + // Check if item is outside visible area + if (itemTop < sidebarScrollTop + 60) { + sidebar.scrollTo({ top: Math.max(0, itemTop - 60), behavior: "smooth" }); + } else if (itemTop + itemHeight > sidebarScrollTop + sidebarHeight - 60) { + sidebar.scrollTo({ + top: itemTop + itemHeight - sidebarHeight + 60, + behavior: "smooth", + }); + } + } + } + }, [activeId]); + + const setItemRef = useCallback((id: string, el: HTMLAnchorElement | null) => { + if (el) { + itemRefs.current.set(id, el); + } else { + itemRefs.current.delete(id); + } + }, []); + + const getLinkClasses = (id: string) => { + const isActive = activeId === id; + return isActive + ? "text-brand-accent font-medium border-l-2 border-brand-accent -ml-[2px] pl-[14px]" + : "text-muted-foreground hover:text-brand-accent"; + }; + + return ( + + ); +} /** * ToolkitPage @@ -12,21 +195,86 @@ import type { ToolkitPageProps } from "../types"; * Composes the full toolkit documentation page from JSON data. */ export function ToolkitPage({ data }: ToolkitPageProps) { - const tools = data.tools ?? []; + const rawTools = data.tools ?? []; + // Temporary UI fallback for toolkit-level secrets that are documented but not yet + // emitted by the Engine tool metadata endpoint. + // + // Example: GitHub toolkit docs require `GITHUB_SERVER_URL` for all tools. + const toolkitSecretOverrides = + data.id.toLowerCase() === "github" ? (["GITHUB_SERVER_URL"] as const) : []; + const tools = rawTools.map((tool) => { + if (toolkitSecretOverrides.length === 0) { + return tool; + } + return { + ...tool, + secrets: Array.from( + new Set([...(tool.secrets ?? []), ...toolkitSecretOverrides]) + ), + }; + }); + const [selectedTools, setSelectedTools] = useState>(new Set()); + const selectionTools = tools.map((tool) => { + const secrets = + (tool.secrets ?? []).length > 0 + ? (tool.secrets ?? []) + : (tool.secretsInfo ?? []).map((secret) => secret.name); + + return { + name: tool.name, + scopes: tool.auth?.scopes ?? [], + secrets, + }; + }); + const shouldShowSelection = tools.length > 0; + + // Compute tool stats + const toolStats = { + total: tools.length, + withScopes: tools.filter((tool) => (tool.auth?.scopes ?? []).length > 0).length, + withSecrets: tools.filter( + (tool) => + (tool.secretsInfo?.length ?? 0) > 0 || (tool.secrets?.length ?? 0) > 0 + ).length, + }; + const showToolFooter = !hasChunksAt(data.documentationChunks, "footer", "replace"); + const pipPackageName = + data.pipPackageName ?? buildPipPackageName(data.id); + + const handleScopeSelectionChange = (toolNames: string[]) => { + setSelectedTools(new Set(toolNames)); + }; + + const toggleToolSelection = (toolName: string) => { + setSelectedTools((prevSelected) => { + const nextSelected = new Set(prevSelected); + if (nextSelected.has(toolName)) { + nextSelected.delete(toolName); + } else { + nextSelected.add(toolName); + } + return nextSelected; + }); + }; return ( -
    +
    + +

    + {data.label} +

    {data.summary && ( -
    +
    {data.summary}
    )} -

    Available tools

    +
    +

    + + + + + + Available tools + + {tools.length} + +

    +
    ({ name: tool.name, qualifiedName: tool.qualifiedName, description: tool.description, secrets: tool.secrets, secretsInfo: tool.secretsInfo, + scopes: tool.auth?.scopes ?? [], }))} /> + {shouldShowSelection && ( +
    + +
    + )} + {tools.map((tool) => ( - + ))} - - - +
    + + {showToolFooter && } + + +
    + +
    ); } diff --git a/app/_components/toolkit-docs/constants.ts b/app/_components/toolkit-docs/constants.ts new file mode 100644 index 000000000..eb516eac0 --- /dev/null +++ b/app/_components/toolkit-docs/constants.ts @@ -0,0 +1,183 @@ +/** + * Toolkit documentation constants + * + * Centralized configuration for URLs, badges, labels, and other constants + * used throughout the toolkit documentation components. + */ + +// ============================================================================= +// External URLs +// ============================================================================= + +/** + * GitHub organization URL + */ +export const GITHUB_ORG_URL = "https://github.com/arcadeai"; + +/** + * GitHub repository URL pattern for toolkit code + * Use: `${GITHUB_ORG_URL}/arcade_${toolkitId.toLowerCase()}` + */ +export const GITHUB_REPO_PREFIX = "arcade_"; + +/** + * License file URL + */ +export const LICENSE_URL = `${GITHUB_ORG_URL}/arcade-ai/blob/main/LICENSE`; + +/** + * PyPI base URL + */ +export const PYPI_BASE_URL = "https://pypi.org/project"; + +/** + * Shields.io base URL for badges + */ +export const SHIELDS_IO_BASE_URL = "https://img.shields.io"; + +/** + * Arcade Dashboard URLs + */ +export const ARCADE_DASHBOARD = { + baseUrl: "https://app.arcade.dev", + playgroundExecute: "https://app.arcade.dev/playground/execute", + login: "https://app.arcade.dev/login", +} as const; + +// ============================================================================= +// Internal Routes +// ============================================================================= + +/** + * Auth provider documentation base path + */ +export const AUTH_PROVIDER_DOCS_PATH = "/references/auth-providers"; + +// ============================================================================= +// Package Naming +// ============================================================================= + +/** + * Package name prefix for PyPI packages + */ +export const PACKAGE_PREFIX = "arcade_"; + +/** + * Generate PyPI package name from toolkit ID + */ +export function getPackageName(toolkitId: string): string { + return `${PACKAGE_PREFIX}${toolkitId.toLowerCase()}`; +} + +/** + * Generate GitHub repository URL from toolkit ID + */ +export function getGitHubRepoUrl(toolkitId: string): string { + return `${GITHUB_ORG_URL}/${GITHUB_REPO_PREFIX}${toolkitId.toLowerCase()}`; +} + +/** + * Generate PyPI project URL from package name + */ +export function getPyPIUrl(packageName: string): string { + return `${PYPI_BASE_URL}/${packageName}/`; +} + +/** + * Generate auth provider docs URL from provider ID + */ +export function getAuthProviderDocsUrl(providerId: string): string { + return `${AUTH_PROVIDER_DOCS_PATH}/${providerId.toLowerCase()}`; +} + +// ============================================================================= +// Badges Configuration +// ============================================================================= + +export type BadgeConfig = { + alt: string; + src: string | ((packageName: string) => string); + href: string | ((packageName: string) => string); +}; + +/** + * PyPI badge configurations + */ +export const PYPI_BADGES: BadgeConfig[] = [ + { + alt: "PyPI Version", + src: (pkg) => `${SHIELDS_IO_BASE_URL}/pypi/v/${pkg}`, + href: (pkg) => getPyPIUrl(pkg), + }, + { + alt: "Python Versions", + src: (pkg) => `${SHIELDS_IO_BASE_URL}/pypi/pyversions/${pkg}`, + href: (pkg) => getPyPIUrl(pkg), + }, + { + alt: "Wheel Status", + src: (pkg) => `${SHIELDS_IO_BASE_URL}/pypi/wheel/${pkg}`, + href: (pkg) => getPyPIUrl(pkg), + }, + { + alt: "Downloads", + src: (pkg) => `${SHIELDS_IO_BASE_URL}/pypi/dm/${pkg}`, + href: (pkg) => getPyPIUrl(pkg), + }, +]; + +/** + * License badge configuration + */ +export const LICENSE_BADGE: BadgeConfig = { + alt: "License", + src: `${SHIELDS_IO_BASE_URL}/badge/License-MIT-yellow.svg`, + href: LICENSE_URL, +}; + +// ============================================================================= +// Default Values +// ============================================================================= + +/** + * Default toolkit author + */ +export const DEFAULT_AUTHOR = "Arcade"; + +// ============================================================================= +// Auth Type Labels +// ============================================================================= + +export const AUTH_TYPE_LABELS = { + oauth2: "User authorization", + api_key: "API key authentication", + mixed: "User authorization", + mixedSuffix: "and API key authentication", + none: "No authentication required", +} as const; + +// ============================================================================= +// UI Constants +// ============================================================================= + +/** + * Scrolling cell animation settings + */ +export const SCROLLING_CELL = { + pixelsPerSecond: 50, + delayMs: 300, + minDurationMs: 1000, + returnDurationMs: 300, + extraPadding: 16, +} as const; + +/** + * Icon sizes used throughout components + */ +export const ICON_SIZES = { + toolkitHeader: "h-20 w-20", + toolkitHeaderSmall: "h-16 w-16", + badge: "h-3.5 w-3.5", + stat: "h-4 w-4", + inline: "h-3 w-3", +} as const; diff --git a/app/_components/toolkit-docs/index.ts b/app/_components/toolkit-docs/index.ts index a3316e4bd..c942b9052 100644 --- a/app/_components/toolkit-docs/index.ts +++ b/app/_components/toolkit-docs/index.ts @@ -34,6 +34,9 @@ // Types export * from "./types"; +// Constants +export * from "./constants"; + // Components export { DocumentationChunkRenderer, diff --git a/app/_components/toolkit-docs/types/index.ts b/app/_components/toolkit-docs/types/index.ts index 8e9a924f0..65f122667 100644 --- a/app/_components/toolkit-docs/types/index.ts +++ b/app/_components/toolkit-docs/types/index.ts @@ -308,6 +308,8 @@ export interface ToolkitData { customImports: string[]; /** Sub-pages that exist for this toolkit */ subPages: string[]; + /** Optional pip package name override */ + pipPackageName?: string; /** Generation timestamp */ generatedAt?: string; } @@ -350,6 +352,12 @@ export interface ToolkitHeaderProps { version?: string; /** Author name (defaults to "Arcade") */ author?: string; + /** Tool statistics */ + toolStats?: { + total: number; + withScopes: number; + withSecrets: number; + }; } /** @@ -394,6 +402,12 @@ export interface ToolSectionProps { tool: ToolDefinition; /** Toolkit ID (for generating anchors) */ toolkitId: string; + /** Whether the tool is selected in the selected tools panel */ + isSelected?: boolean; + /** Show selection checkbox */ + showSelection?: boolean; + /** Toggle selection handler */ + onToggleSelection?: (toolName: string) => void; } /** @@ -407,6 +421,7 @@ export interface AvailableToolsTableProps { description: string | null; secrets?: string[]; secretsInfo?: ToolSecret[]; + scopes?: string[]; }>; /** Optional label for the secrets column */ secretsColumnLabel?: string; @@ -416,6 +431,28 @@ export interface AvailableToolsTableProps { secretTypeLabels?: Partial>; /** Base URL for linking secret type docs */ secretTypeDocsBaseUrl?: string; + /** Enable search input */ + enableSearch?: boolean; + /** Enable filters */ + enableFilters?: boolean; + /** Enable scope filter chips */ + enableScopeFilter?: boolean; + /** Search input placeholder */ + searchPlaceholder?: string; + /** Filter label */ + filterLabel?: string; + /** Scope filter label */ + scopeFilterLabel?: string; + /** Scope filter helper text */ + scopeFilterDescription?: string; + /** Default filter selection */ + defaultFilter?: "all" | "has_scopes" | "no_scopes" | "has_secrets" | "no_secrets"; + /** Currently selected tool names */ + selectedTools?: Set; + /** Handler for toggling tool selection */ + onToggleSelection?: (toolName: string) => void; + /** Whether to show selection checkboxes */ + showSelection?: boolean; } /** diff --git a/app/_lib/__tests__/toolkit-data.test.ts b/app/_lib/__tests__/toolkit-data.test.ts new file mode 100644 index 000000000..5ac34cd65 --- /dev/null +++ b/app/_lib/__tests__/toolkit-data.test.ts @@ -0,0 +1,72 @@ +import { mkdtemp, readFile, rm, writeFile } from "fs/promises"; +import { tmpdir } from "os"; +import { join } from "path"; +import { describe, expect, it } from "vitest"; + +import { readToolkitData, readToolkitIndex } from "../toolkit-data"; + +const loadFixture = async (fileName: string): Promise => { + const fixturesDir = new URL("../../../toolkit-docs-generator/tests/fixtures/", import.meta.url); + const filePath = new URL(fileName, fixturesDir); + return await readFile(filePath, "utf-8"); +}; + +const withTempDir = async (fn: (dir: string) => Promise) => { + const dir = await mkdtemp(join(tmpdir(), "toolkit-data-")); + try { + await fn(dir); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}; + +describe("toolkit data loader", () => { + it("reads toolkit data from the provided directory", async () => { + await withTempDir(async (dir) => { + const fixture = await loadFixture("github-toolkit.json"); + await writeFile(join(dir, "github.json"), fixture, "utf-8"); + + const data = await readToolkitData("Github", { dataDir: dir }); + + expect(data?.id).toBe("Github"); + expect(data?.tools.length).toBeGreaterThan(0); + }); + }); + + it("returns null when toolkit data is missing", async () => { + await withTempDir(async (dir) => { + const data = await readToolkitData("Missing", { dataDir: dir }); + + expect(data).toBeNull(); + }); + }); + + it("reads index data from the provided directory", async () => { + await withTempDir(async (dir) => { + const indexFixture = JSON.stringify( + { + generatedAt: "2026-01-15T00:00:00.000Z", + version: "1.0.0", + toolkits: [ + { + id: "Github", + label: "GitHub", + version: "1.0.0", + category: "development", + toolCount: 3, + authType: "oauth2", + }, + ], + }, + null, + 2 + ); + await writeFile(join(dir, "index.json"), indexFixture, "utf-8"); + + const index = await readToolkitIndex({ dataDir: dir }); + + expect(index?.toolkits).toHaveLength(1); + expect(index?.toolkits[0]?.id).toBe("Github"); + }); + }); +}); diff --git a/app/_lib/toolkit-data.ts b/app/_lib/toolkit-data.ts new file mode 100644 index 000000000..17ce6e38e --- /dev/null +++ b/app/_lib/toolkit-data.ts @@ -0,0 +1,55 @@ +import { readFile } from "fs/promises"; +import { join } from "path"; +import type { ToolkitData } from "@/app/_components/toolkit-docs/types"; + +export type ToolkitIndexEntry = { + id: string; + label: string; + version: string; + category: string; + toolCount: number; + authType: string; +}; + +export type ToolkitIndex = { + generatedAt: string; + version: string; + toolkits: ToolkitIndexEntry[]; +}; + +type ToolkitDataOptions = { + dataDir?: string; +}; + +const DEFAULT_DATA_DIR = join(process.cwd(), "data", "toolkits"); + +const resolveDataDir = (options?: ToolkitDataOptions): string => + options?.dataDir ?? process.env.TOOLKIT_DATA_DIR ?? DEFAULT_DATA_DIR; + +export const readToolkitData = async ( + toolkitId: string, + options?: ToolkitDataOptions +): Promise => { + const fileName = `${toolkitId.toLowerCase()}.json`; + const filePath = join(resolveDataDir(options), fileName); + + try { + const content = await readFile(filePath, "utf-8"); + return JSON.parse(content) as ToolkitData; + } catch { + return null; + } +}; + +export const readToolkitIndex = async ( + options?: ToolkitDataOptions +): Promise => { + const filePath = join(resolveDataDir(options), "index.json"); + + try { + const content = await readFile(filePath, "utf-8"); + return JSON.parse(content) as ToolkitIndex; + } catch { + return null; + } +}; diff --git a/app/api/toolkit-data/[toolkitId]/route.ts b/app/api/toolkit-data/[toolkitId]/route.ts new file mode 100644 index 000000000..750f450f3 --- /dev/null +++ b/app/api/toolkit-data/[toolkitId]/route.ts @@ -0,0 +1,19 @@ +import { NextResponse } from "next/server"; +import { readToolkitData } from "@/app/_lib/toolkit-data"; + +export async function GET( + _request: Request, + { params }: { params: Promise<{ toolkitId: string }> } +) { + const { toolkitId } = await params; + const data = await readToolkitData(toolkitId); + + if (!data) { + return NextResponse.json( + { error: `Toolkit not found: ${toolkitId}` }, + { status: 404 } + ); + } + + return NextResponse.json(data); +} diff --git a/app/en/resources/integrations/_meta.tsx b/app/en/resources/integrations/_meta.tsx index 9c0691602..285e5d526 100644 --- a/app/en/resources/integrations/_meta.tsx +++ b/app/en/resources/integrations/_meta.tsx @@ -49,6 +49,10 @@ const meta: MetaRecord = { "contribute-a-server": { title: "Contribute a Server", }, + preview: { + title: "Preview (Dev)", + display: "hidden", + }, }; export default meta; diff --git a/app/en/resources/integrations/development/github/preview/page.mdx b/app/en/resources/integrations/development/github/preview/page.mdx index 18065a9a4..ef424a374 100644 --- a/app/en/resources/integrations/development/github/preview/page.mdx +++ b/app/en/resources/integrations/development/github/preview/page.mdx @@ -4,10 +4,10 @@ description: "Preview the JSON-driven toolkit documentation components." --- import githubData from "@/toolkit-docs-generator/tests/fixtures/github-toolkit.json"; -import { ToolkitPage } from "@/app/_components/toolkit-docs"; +import { ToolkitDocsPreview } from "@/app/_components/toolkit-docs/components/ToolkitDocsPreview"; # Toolkit docs preview Use this page to preview the JSON-driven toolkit components with real fixture data. - + diff --git a/app/en/resources/integrations/preview/ToolkitPreviewIndex.tsx b/app/en/resources/integrations/preview/ToolkitPreviewIndex.tsx new file mode 100644 index 000000000..887568861 --- /dev/null +++ b/app/en/resources/integrations/preview/ToolkitPreviewIndex.tsx @@ -0,0 +1,101 @@ +import Link from "next/link"; +import { readToolkitIndex } from "@/app/_lib/toolkit-data"; + +const AUTH_TYPE_STYLES: Record = { + oauth2: "bg-blue-500/10 text-blue-400 border-blue-500/30", + api_key: "bg-amber-500/10 text-amber-400 border-amber-500/30", + mixed: "bg-purple-500/10 text-purple-400 border-purple-500/30", + none: "bg-green-500/10 text-green-400 border-green-500/30", +}; + +const AUTH_TYPE_LABELS: Record = { + oauth2: "OAuth", + api_key: "API Key", + mixed: "Mixed", + none: "None", +}; + +export async function ToolkitPreviewIndex() { + const index = await readToolkitIndex(); + + if (!index) { + return ( +
    +

    + No toolkit index found. Run the generator first. +

    +
    + ); + } + + const groupedByCategory = index.toolkits.reduce( + (acc, toolkit) => { + const category = toolkit.category || "other"; + if (!acc[category]) { + acc[category] = []; + } + acc[category].push(toolkit); + return acc; + }, + {} as Record + ); + + const categories = Object.keys(groupedByCategory).sort(); + + return ( +
    +
    + + {index.toolkits.length} toolkits generated + + + {new Date(index.generatedAt).toLocaleString()} + +
    + + {categories.map((category) => ( +
    +

    + + {category.replace(/-/g, " ")} + + {groupedByCategory[category].length} + +

    +
    + {groupedByCategory[category] + .sort((a, b) => a.label.localeCompare(b.label)) + .map((toolkit) => ( + +
    + + {toolkit.label} + + + v{toolkit.version} + +
    +
    + + {toolkit.toolCount} tools + + + {AUTH_TYPE_LABELS[toolkit.authType] || toolkit.authType} + +
    + + ))} +
    +
    + ))} +
    + ); +} diff --git a/app/en/resources/integrations/preview/[toolkitId]/ToolkitPreviewContent.tsx b/app/en/resources/integrations/preview/[toolkitId]/ToolkitPreviewContent.tsx new file mode 100644 index 000000000..234c7ecc7 --- /dev/null +++ b/app/en/resources/integrations/preview/[toolkitId]/ToolkitPreviewContent.tsx @@ -0,0 +1,47 @@ +"use client"; + +import { useParams } from "next/navigation"; +import { useEffect, useState } from "react"; +import { ToolkitPage } from "@/app/_components/toolkit-docs/components/ToolkitPage"; +import type { ToolkitData } from "@/app/_components/toolkit-docs/types"; + +export function ToolkitPreviewContent() { + const params = useParams(); + const toolkitId = params.toolkitId as string; + const [data, setData] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + useEffect(() => { + const loadData = async () => { + try { + const response = await fetch(`/api/toolkit-data/${toolkitId}`); + if (!response.ok) { + throw new Error(`Toolkit not found: ${toolkitId}`); + } + const toolkitData = await response.json(); + setData(toolkitData); + } catch (err) { + setError(err instanceof Error ? err.message : "Failed to load toolkit"); + } finally { + setLoading(false); + } + }; + + loadData(); + }, [toolkitId]); + + if (loading) { + return

    Loading toolkit data...

    ; + } + + if (error || !data) { + return ( +

    + {error || `Toolkit "${toolkitId}" not found.`} +

    + ); + } + + return ; +} diff --git a/app/en/resources/integrations/preview/[toolkitId]/_meta.tsx b/app/en/resources/integrations/preview/[toolkitId]/_meta.tsx new file mode 100644 index 000000000..a472ede3d --- /dev/null +++ b/app/en/resources/integrations/preview/[toolkitId]/_meta.tsx @@ -0,0 +1,14 @@ +import type { MetaRecord } from "nextra"; + +const meta: MetaRecord = { + "*": { + theme: { + breadcrumb: false, + toc: false, + copyPage: true, + }, + }, +}; + +export default meta; + diff --git a/app/en/resources/integrations/preview/[toolkitId]/page.mdx b/app/en/resources/integrations/preview/[toolkitId]/page.mdx new file mode 100644 index 000000000..d1e18e863 --- /dev/null +++ b/app/en/resources/integrations/preview/[toolkitId]/page.mdx @@ -0,0 +1,8 @@ +--- +title: "Toolkit preview" +description: "Preview generated toolkit documentation." +--- + +import { ToolkitPreviewContent } from "./ToolkitPreviewContent"; + + diff --git a/app/en/resources/integrations/preview/_meta.tsx b/app/en/resources/integrations/preview/_meta.tsx new file mode 100644 index 000000000..daa058e41 --- /dev/null +++ b/app/en/resources/integrations/preview/_meta.tsx @@ -0,0 +1,16 @@ +import type { MetaRecord } from "nextra"; + +const meta: MetaRecord = { + "*": { + theme: { + // Preview pages are dynamic/client-rendered. + // We render breadcrumb + "On this page" ourselves to match existing layouts. + breadcrumb: false, + toc: false, + copyPage: true, + }, + }, +}; + +export default meta; + diff --git a/app/en/resources/integrations/preview/page.mdx b/app/en/resources/integrations/preview/page.mdx new file mode 100644 index 000000000..2a6631d05 --- /dev/null +++ b/app/en/resources/integrations/preview/page.mdx @@ -0,0 +1,12 @@ +--- +title: "Toolkit previews" +description: "Preview generated toolkit documentation." +--- + +import { ToolkitPreviewIndex } from "./ToolkitPreviewIndex"; + +# Toolkit documentation previews + +Preview pages for all generated toolkit JSON data. + + diff --git a/app/globals.css b/app/globals.css index fa510d573..29871489a 100644 --- a/app/globals.css +++ b/app/globals.css @@ -97,3 +97,12 @@ nav > div:has(.nextra-search) { .guide-overview ul ul { margin-top: 0; } + +@keyframes tool-name-marquee { + 0% { + transform: translateX(0); + } + 100% { + transform: translateX(-100%); + } +} diff --git a/toolkit-docs-generator/src/cli/index.ts b/toolkit-docs-generator/src/cli/index.ts index 0de097bc2..328f7a213 100644 --- a/toolkit-docs-generator/src/cli/index.ts +++ b/toolkit-docs-generator/src/cli/index.ts @@ -12,21 +12,33 @@ */ import chalk from "chalk"; +import { readFile, readdir, rm } from "fs/promises"; import { Command } from "commander"; import ora from "ora"; import { join, resolve } from "path"; -import { createJsonGenerator } from "../generator/json-generator.js"; +import { createJsonGenerator, verifyOutputDir } from "../generator/index.js"; import { createLlmClient, + type LlmClient, type LlmProvider, LlmToolExampleGenerator, + LlmToolkitSummaryGenerator, } from "../llm/index.js"; import type { MergeResult } from "../merger/data-merger.js"; import { createDataMerger } from "../merger/data-merger.js"; import { createCustomSectionsFileSource } from "../sources/custom-sections-file.js"; import { createEmptyCustomSectionsSource } from "../sources/in-memory.js"; -import { createMockToolkitDataSource } from "../sources/toolkit-data-source.js"; -import { type ProviderVersion, ProviderVersionSchema } from "../types/index.js"; +import { createMockMetadataSource } from "../sources/mock-metadata.js"; +import { + createEngineToolkitDataSource, + createMockToolkitDataSource, +} from "../sources/toolkit-data-source.js"; +import { + MergedToolkitSchema, + type MergedToolkit, + type ProviderVersion, + ProviderVersionSchema, +} from "../types/index.js"; const program = new Command(); @@ -62,6 +74,34 @@ const parseProviders = (input: string): ProviderVersion[] => { */ const getDefaultMockDataDir = (): string => join(process.cwd(), "mock-data"); +/** + * Get the default output directory for docs JSON. + */ +const getDefaultOutputDir = (): string => { + const cwd = process.cwd(); + if (cwd.endsWith("toolkit-docs-generator")) { + return resolve(cwd, "..", "data", "toolkits"); + } + return resolve(cwd, "data", "toolkits"); +}; + +const clearOutputDir = async ( + outputDir: string, + verbose: boolean +): Promise => { + const resolvedDir = resolve(outputDir); + const repoRoot = resolve(process.cwd()); + if (resolvedDir === "/" || resolvedDir === repoRoot) { + throw new Error( + `Refusing to overwrite output directory: ${resolvedDir}` + ); + } + await rm(resolvedDir, { recursive: true, force: true }); + if (verbose) { + console.log(chalk.dim(`Cleared output directory: ${resolvedDir}`)); + } +}; + const resolveLlmProvider = (value?: string): LlmProvider => { if (value === "openai" || value === "anthropic") { return value; @@ -79,7 +119,13 @@ const resolveLlmConfig = (options: { llmTemperature?: number; llmMaxTokens?: number; llmSystemPrompt?: string; -}) => { +}): { + client: LlmClient; + model: string; + temperature?: number; + maxTokens?: number; + systemPrompt?: string; +} => { const provider = resolveLlmProvider( options.llmProvider ?? process.env.LLM_PROVIDER ); @@ -111,7 +157,7 @@ const resolveLlmConfig = (options: { }, }); - return new LlmToolExampleGenerator({ + return { client, model, ...(options.llmTemperature !== undefined @@ -123,7 +169,101 @@ const resolveLlmConfig = (options: { ...(options.llmSystemPrompt ? { systemPrompt: options.llmSystemPrompt } : {}), - }); + }; +}; + +const resolveEngineConfig = (options: { + engineApiUrl?: string; + engineApiKey?: string; + enginePageSize?: number; +}) => { + const baseUrl = options.engineApiUrl ?? process.env.ENGINE_API_URL; + const apiKey = options.engineApiKey ?? process.env.ENGINE_API_KEY; + + if (!baseUrl && !apiKey) { + return null; + } + + if (!baseUrl || !apiKey) { + throw new Error( + "Engine API requires both --engine-api-url and --engine-api-key (or ENGINE_API_URL/ENGINE_API_KEY)." + ); + } + + return { + baseUrl, + apiKey, + ...(options.enginePageSize ? { pageSize: options.enginePageSize } : {}), + }; +}; + +const normalizeToolkitKey = (toolkitId: string): string => + toolkitId.toLowerCase(); + +const loadPreviousToolkit = async ( + filePath: string +): Promise => { + try { + const content = await readFile(filePath, "utf-8"); + const parsed = JSON.parse(content) as unknown; + const result = MergedToolkitSchema.safeParse(parsed); + if (!result.success) { + return null; + } + return result.data; + } catch { + return null; + } +}; + +const loadPreviousToolkitsForProviders = async ( + dir: string, + providers: ProviderVersion[], + verbose: boolean +): Promise> => { + const previousToolkits = new Map(); + + for (const provider of providers) { + const filePath = join(dir, `${provider.provider.toLowerCase()}.json`); + const toolkit = await loadPreviousToolkit(filePath); + if (toolkit) { + previousToolkits.set(normalizeToolkitKey(toolkit.id), toolkit); + } else if (verbose) { + console.log( + chalk.dim(`No previous output found for ${provider.provider}.`) + ); + } + } + + return previousToolkits; +}; + +const loadPreviousToolkitsFromDir = async ( + dir: string, + verbose: boolean +): Promise> => { + const previousToolkits = new Map(); + + try { + const entries = await readdir(dir); + const jsonFiles = entries.filter( + (entry) => entry.endsWith(".json") && entry !== "index.json" + ); + + for (const fileName of jsonFiles) { + const filePath = join(dir, fileName); + const toolkit = await loadPreviousToolkit(filePath); + if (toolkit) { + previousToolkits.set(normalizeToolkitKey(toolkit.id), toolkit); + } + } + } catch (error) { + if (verbose) { + console.log(chalk.dim(`Failed to read previous output: ${error}`)); + } + } + + return previousToolkits; }; const processProviders = async ( @@ -161,13 +301,26 @@ program program .command("generate") - .description("Generate documentation for specified providers") - .requiredOption( + .description("Generate documentation for specific providers or all toolkits") + .option( "-p, --providers ", 'Comma-separated list of providers with optional versions (e.g., "Github:1.0.0,Slack:2.1.0" or "Github,Slack")' ) - .option("-o, --output ", "Output directory", "./output") + .option("--all", "Generate documentation for all toolkits", false) + .option("-o, --output ", "Output directory", getDefaultOutputDir()) .option("--mock-data-dir ", "Path to mock data directory") + .option("--metadata-file ", "Path to metadata JSON file") + .option("--engine-api-url ", "Engine API base URL") + .option("--engine-api-key ", "Engine API key") + .option("--engine-page-size ", "Engine API page size", (value) => + Number.parseInt(value, 10) + ) + .option("--previous-output ", "Path to previous output directory") + .option("--force-regenerate", "Regenerate all examples and summary", false) + .option( + "--overwrite-output", + "Delete output directory before writing new JSON" + ) .option("--llm-provider ", "LLM provider (openai|anthropic)") .option("--llm-model ", "LLM model to use") .option("--llm-api-key ", "LLM API key") @@ -179,13 +332,38 @@ program Number.parseInt(value, 10) ) .option("--llm-system-prompt ", "LLM system prompt override") + .option( + "--llm-concurrency ", + "Max concurrent LLM calls per toolkit (default: 5)", + (value) => Number.parseInt(value, 10) + ) + .option( + "--toolkit-concurrency ", + "Max concurrent toolkit processing (default: 3)", + (value) => Number.parseInt(value, 10) + ) + .option( + "--no-index-from-output", + "Do not rebuild index from output directory" + ) + .option("--skip-examples", "Skip LLM example generation", false) + .option("--skip-summary", "Skip LLM summary generation", false) + .option("--no-verify-output", "Skip output verification") .option("--custom-sections ", "Path to custom sections JSON") .option("--verbose", "Enable verbose logging", false) .action( async (options: { - providers: string; + providers?: string; + all: boolean; output: string; mockDataDir?: string; + metadataFile?: string; + engineApiUrl?: string; + engineApiKey?: string; + enginePageSize?: number; + previousOutput?: string; + forceRegenerate: boolean; + overwriteOutput?: boolean; llmProvider?: string; llmModel?: string; llmApiKey?: string; @@ -193,32 +371,100 @@ program llmTemperature?: number; llmMaxTokens?: number; llmSystemPrompt?: string; + llmConcurrency?: number; + toolkitConcurrency?: number; + indexFromOutput: boolean; + skipExamples: boolean; + skipSummary: boolean; + verifyOutput: boolean; customSections?: string; verbose: boolean; }) => { const spinner = ora("Parsing input...").start(); try { - // Parse providers - const providers = parseProviders(options.providers); - spinner.succeed(`Parsed ${providers.length} provider(s)`); - - if (options.verbose) { - console.log(chalk.cyan("\nProviders to process:")); - for (const pv of providers) { - const version = pv.version ?? "latest"; - console.log(chalk.dim(` - ${pv.provider}:${version}`)); + const runAll = options.all; + if (runAll && options.providers) { + throw new Error('Use either "--all" or "--providers", not both.'); + } + + const providers = runAll + ? [] + : options.providers + ? parseProviders(options.providers) + : null; + + if (!runAll && !providers) { + throw new Error('Missing required option "--providers" or "--all".'); + } + + if (providers) { + spinner.succeed(`Parsed ${providers.length} provider(s)`); + + if (options.verbose) { + console.log(chalk.cyan("\nProviders to process:")); + for (const pv of providers) { + const version = pv.version ?? "latest"; + console.log(chalk.dim(` - ${pv.provider}:${version}`)); + } } + } else { + spinner.succeed("Parsed all toolkits"); } // Initialize sources spinner.start("Initializing data sources..."); const mockDataDir = options.mockDataDir ?? getDefaultMockDataDir(); - const toolkitDataSource = createMockToolkitDataSource({ - dataDir: mockDataDir, - }); - const toolExampleGenerator = resolveLlmConfig(options); + const metadataFile = + options.metadataFile ?? join(mockDataDir, "metadata.json"); + const metadataSource = createMockMetadataSource(metadataFile); + const engineConfig = resolveEngineConfig(options); + const toolkitDataSource = engineConfig + ? createEngineToolkitDataSource({ + engine: engineConfig, + metadataSource, + }) + : createMockToolkitDataSource({ + dataDir: mockDataDir, + }); + const needsExamples = !options.skipExamples; + const needsSummary = !options.skipSummary; + const needsLlm = needsExamples || needsSummary; + + const llmConfig = needsLlm ? resolveLlmConfig(options) : null; + + let toolExampleGenerator: LlmToolExampleGenerator | undefined; + if (needsExamples) { + if (!llmConfig) { + throw new Error("LLM configuration is required for examples."); + } + toolExampleGenerator = new LlmToolExampleGenerator(llmConfig); + } + + let toolkitSummaryGenerator: LlmToolkitSummaryGenerator | undefined; + if (needsSummary) { + if (!llmConfig) { + throw new Error("LLM configuration is required for summaries."); + } + toolkitSummaryGenerator = new LlmToolkitSummaryGenerator(llmConfig); + } + const previousOutputDir = options.forceRegenerate + ? undefined + : options.previousOutput ?? + (options.overwriteOutput ? undefined : options.output); + const previousToolkits = previousOutputDir + ? runAll + ? await loadPreviousToolkitsFromDir( + previousOutputDir, + options.verbose + ) + : await loadPreviousToolkitsForProviders( + previousOutputDir, + providers ?? [], + options.verbose + ) + : undefined; // Custom sections source const customSectionsSource = options.customSections @@ -227,11 +473,36 @@ program spinner.succeed("Data sources initialized"); + if (options.overwriteOutput) { + spinner.start("Clearing output directory..."); + await clearOutputDir(options.output, options.verbose); + spinner.succeed("Output directory cleared"); + } + + // Track progress for --all mode + const completedToolkits: string[] = []; + const onToolkitProgress = ( + toolkitId: string, + status: "start" | "done" + ) => { + if (status === "start") { + spinner.text = `Processing ${toolkitId}...`; + } else { + completedToolkits.push(toolkitId); + spinner.text = `Processed ${completedToolkits.length} toolkit(s) (latest: ${toolkitId})`; + } + }; + // Create merger using unified source const merger = createDataMerger({ toolkitDataSource, customSectionsSource, - toolExampleGenerator, + ...(toolExampleGenerator ? { toolExampleGenerator } : {}), + ...(toolkitSummaryGenerator ? { toolkitSummaryGenerator } : {}), + ...(previousToolkits ? { previousToolkits } : {}), + ...(options.llmConcurrency ? { llmConcurrency: options.llmConcurrency } : {}), + ...(options.toolkitConcurrency ? { toolkitConcurrency: options.toolkitConcurrency } : {}), + ...(runAll ? { onToolkitProgress } : {}), }); // Create generator @@ -239,21 +510,30 @@ program outputDir: resolve(options.output), prettyPrint: true, generateIndex: true, + indexSource: options.indexFromOutput ? "output" : "current", }); - // Process each provider - const allResults = await processProviders( - providers, - merger, - spinner, - options.verbose - ); + // Process toolkits + if (runAll) { + spinner.start("Processing toolkits..."); + } + const allResults = runAll + ? await merger.mergeAllToolkits() + : await processProviders( + providers ?? [], + merger, + spinner, + options.verbose + ); + if (runAll) { + spinner.succeed(`Processed ${allResults.length} toolkit(s)`); + } // Generate output files if (allResults.length > 0) { spinner.start("Writing output files..."); - const toolkits = allResults.map((r) => r.toolkit); + const toolkits = allResults.map((r) => r.toolkit); const genResult = await generator.generateAll(toolkits); if (genResult.errors.length > 0) { @@ -269,6 +549,19 @@ program console.log(chalk.dim(` ${file}`)); } } + + if (options.verifyOutput) { + spinner.start("Verifying output..."); + const verification = await verifyOutputDir(resolve(options.output)); + if (!verification.valid) { + spinner.fail("Output verification failed."); + for (const error of verification.errors) { + console.log(chalk.red(` - ${error}`)); + } + process.exit(1); + } + spinner.succeed("Output verified"); + } } catch (error) { spinner.fail( `Error: ${error instanceof Error ? error.message : String(error)}` @@ -281,8 +574,20 @@ program program .command("generate-all") .description("Generate documentation for all toolkits in mock data") - .option("-o, --output ", "Output directory", "./output") + .option("-o, --output ", "Output directory", getDefaultOutputDir()) .option("--mock-data-dir ", "Path to mock data directory") + .option("--metadata-file ", "Path to metadata JSON file") + .option("--engine-api-url ", "Engine API base URL") + .option("--engine-api-key ", "Engine API key") + .option("--engine-page-size ", "Engine API page size", (value) => + Number.parseInt(value, 10) + ) + .option("--previous-output ", "Path to previous output directory") + .option("--force-regenerate", "Regenerate all examples and summary", false) + .option( + "--overwrite-output", + "Delete output directory before writing new JSON" + ) .option("--llm-provider ", "LLM provider (openai|anthropic)") .option("--llm-model ", "LLM model to use") .option("--llm-api-key ", "LLM API key") @@ -294,12 +599,36 @@ program Number.parseInt(value, 10) ) .option("--llm-system-prompt ", "LLM system prompt override") + .option( + "--llm-concurrency ", + "Max concurrent LLM calls per toolkit (default: 5)", + (value) => Number.parseInt(value, 10) + ) + .option( + "--toolkit-concurrency ", + "Max concurrent toolkit processing (default: 3)", + (value) => Number.parseInt(value, 10) + ) + .option( + "--no-index-from-output", + "Do not rebuild index from output directory" + ) + .option("--skip-examples", "Skip LLM example generation", false) + .option("--skip-summary", "Skip LLM summary generation", false) + .option("--no-verify-output", "Skip output verification") .option("--custom-sections ", "Path to custom sections JSON") .option("--verbose", "Enable verbose logging", false) .action( async (options: { output: string; mockDataDir?: string; + metadataFile?: string; + engineApiUrl?: string; + engineApiKey?: string; + enginePageSize?: number; + previousOutput?: string; + forceRegenerate: boolean; + overwriteOutput?: boolean; llmProvider?: string; llmModel?: string; llmApiKey?: string; @@ -307,6 +636,12 @@ program llmTemperature?: number; llmMaxTokens?: number; llmSystemPrompt?: string; + llmConcurrency?: number; + toolkitConcurrency?: number; + indexFromOutput: boolean; + skipExamples: boolean; + skipSummary: boolean; + verifyOutput: boolean; customSections?: string; verbose: boolean; }) => { @@ -314,10 +649,49 @@ program try { const mockDataDir = options.mockDataDir ?? getDefaultMockDataDir(); - const toolkitDataSource = createMockToolkitDataSource({ - dataDir: mockDataDir, - }); - const toolExampleGenerator = resolveLlmConfig(options); + const metadataFile = + options.metadataFile ?? join(mockDataDir, "metadata.json"); + const metadataSource = createMockMetadataSource(metadataFile); + const engineConfig = resolveEngineConfig(options); + const toolkitDataSource = engineConfig + ? createEngineToolkitDataSource({ + engine: engineConfig, + metadataSource, + }) + : createMockToolkitDataSource({ + dataDir: mockDataDir, + }); + const needsExamples = !options.skipExamples; + const needsSummary = !options.skipSummary; + const needsLlm = needsExamples || needsSummary; + + const llmConfig = needsLlm ? resolveLlmConfig(options) : null; + + let toolExampleGenerator: LlmToolExampleGenerator | undefined; + if (needsExamples) { + if (!llmConfig) { + throw new Error("LLM configuration is required for examples."); + } + toolExampleGenerator = new LlmToolExampleGenerator(llmConfig); + } + + let toolkitSummaryGenerator: LlmToolkitSummaryGenerator | undefined; + if (needsSummary) { + if (!llmConfig) { + throw new Error("LLM configuration is required for summaries."); + } + toolkitSummaryGenerator = new LlmToolkitSummaryGenerator(llmConfig); + } + const previousOutputDir = options.forceRegenerate + ? undefined + : options.previousOutput ?? + (options.overwriteOutput ? undefined : options.output); + const previousToolkits = previousOutputDir + ? await loadPreviousToolkitsFromDir( + previousOutputDir, + options.verbose + ) + : undefined; const customSectionsSource = options.customSections ? createCustomSectionsFileSource(options.customSections) @@ -325,22 +699,48 @@ program spinner.succeed("Data sources initialized"); + if (options.overwriteOutput) { + spinner.start("Clearing output directory..."); + await clearOutputDir(options.output, options.verbose); + spinner.succeed("Output directory cleared"); + } + + // Track progress + const completedToolkits: string[] = []; + const onToolkitProgress = ( + toolkitId: string, + status: "start" | "done" + ) => { + if (status === "start") { + spinner.text = `Processing ${toolkitId}...`; + } else { + completedToolkits.push(toolkitId); + spinner.text = `Processed ${completedToolkits.length} toolkit(s) (latest: ${toolkitId})`; + } + }; + // Create merger using unified source const merger = createDataMerger({ toolkitDataSource, customSectionsSource, - toolExampleGenerator, + ...(toolExampleGenerator ? { toolExampleGenerator } : {}), + ...(toolkitSummaryGenerator ? { toolkitSummaryGenerator } : {}), + ...(previousToolkits ? { previousToolkits } : {}), + ...(options.llmConcurrency ? { llmConcurrency: options.llmConcurrency } : {}), + ...(options.toolkitConcurrency ? { toolkitConcurrency: options.toolkitConcurrency } : {}), + onToolkitProgress, }); - spinner.start("Merging all toolkits..."); + spinner.start("Processing toolkits..."); const results = await merger.mergeAllToolkits(); - spinner.succeed(`Merged ${results.length} toolkit(s)`); + spinner.succeed(`Processed ${results.length} toolkit(s)`); // Generate output const generator = createJsonGenerator({ outputDir: resolve(options.output), prettyPrint: true, generateIndex: true, + indexSource: options.indexFromOutput ? "output" : "current", }); spinner.start("Writing output files..."); @@ -353,6 +753,19 @@ program spinner.succeed(`Written ${genResult.filesWritten.length} file(s)`); } + if (options.verifyOutput) { + spinner.start("Verifying output..."); + const verification = await verifyOutputDir(resolve(options.output)); + if (!verification.valid) { + spinner.fail("Output verification failed."); + for (const error of verification.errors) { + console.log(chalk.red(` - ${error}`)); + } + process.exit(1); + } + spinner.succeed("Output verified"); + } + console.log(chalk.green("\n✓ Generation complete\n")); } catch (error) { spinner.fail( @@ -424,5 +837,27 @@ program } }); +program + .command("verify-output") + .description("Verify output directory structure and schema") + .option("-o, --output ", "Output directory", getDefaultOutputDir()) + .action(async (options: { output: string }) => { + try { + const verification = await verifyOutputDir(resolve(options.output)); + if (!verification.valid) { + console.log(chalk.red("Output verification failed:")); + for (const error of verification.errors) { + console.log(chalk.red(` - ${error}`)); + } + process.exit(1); + } + + console.log(chalk.green("✓ Output verified")); + } catch (error) { + console.log(chalk.red(`Error: ${error}`)); + process.exit(1); + } + }); + // Parse command line arguments program.parse(); diff --git a/toolkit-docs-generator/src/generator/index.ts b/toolkit-docs-generator/src/generator/index.ts index 37b71f395..b3c123b1e 100644 --- a/toolkit-docs-generator/src/generator/index.ts +++ b/toolkit-docs-generator/src/generator/index.ts @@ -2,3 +2,4 @@ * Generator module exports */ export * from "./json-generator.js"; +export * from "./output-verifier.js"; \ No newline at end of file diff --git a/toolkit-docs-generator/src/generator/json-generator.ts b/toolkit-docs-generator/src/generator/json-generator.ts index 62f050b09..2e13a54d1 100644 --- a/toolkit-docs-generator/src/generator/json-generator.ts +++ b/toolkit-docs-generator/src/generator/json-generator.ts @@ -11,6 +11,7 @@ import type { ToolkitIndexEntry, } from "../types/index.js"; import { MergedToolkitSchema } from "../types/index.js"; +import { readToolkitsFromDir } from "./output-verifier.js"; // ============================================================================ // Generator Configuration @@ -23,6 +24,8 @@ export interface JsonGeneratorConfig { prettyPrint?: boolean; /** Whether to generate an index file (default: true) */ generateIndex?: boolean; + /** Where to source toolkits for the index file */ + indexSource?: "current" | "output"; /** Whether to validate output with Zod (default: true) */ validateOutput?: boolean; } @@ -45,12 +48,14 @@ export class JsonGenerator { private readonly outputDir: string; private readonly prettyPrint: boolean; private readonly generateIndex: boolean; + private readonly indexSource: "current" | "output"; private readonly validateOutput: boolean; constructor(config: JsonGeneratorConfig) { this.outputDir = config.outputDir; this.prettyPrint = config.prettyPrint ?? true; this.generateIndex = config.generateIndex ?? true; + this.indexSource = config.indexSource ?? "current"; this.validateOutput = config.validateOutput ?? true; } @@ -105,7 +110,11 @@ export class JsonGenerator { // Generate index file if (this.generateIndex && toolkits.length > 0) { try { - const indexPath = await this.generateIndexFile(toolkits); + const indexToolkits = + this.indexSource === "output" + ? await this.getToolkitsFromOutputDir(errors) + : toolkits; + const indexPath = await this.generateIndexFile(indexToolkits); filesWritten.push(indexPath); } catch (error) { errors.push(`Failed to write index: ${error}`); @@ -147,6 +156,16 @@ export class JsonGenerator { await writeFile(filePath, content, "utf-8"); return filePath; } + + private async getToolkitsFromOutputDir( + errors: string[] + ): Promise { + const readResult = await readToolkitsFromDir(this.outputDir); + if (readResult.errors.length > 0) { + errors.push(...readResult.errors); + } + return readResult.toolkits; + } } // ============================================================================ diff --git a/toolkit-docs-generator/src/generator/output-verifier.ts b/toolkit-docs-generator/src/generator/output-verifier.ts new file mode 100644 index 000000000..10d9b82c1 --- /dev/null +++ b/toolkit-docs-generator/src/generator/output-verifier.ts @@ -0,0 +1,179 @@ +import { readFile, readdir } from "fs/promises"; +import { basename, join } from "path"; +import type { MergedToolkit, ToolkitIndex } from "../types/index.js"; +import { + MergedToolkitSchema, + ToolkitIndexSchema, +} from "../types/index.js"; + +export interface OutputVerificationResult { + valid: boolean; + errors: string[]; + warnings: string[]; +} + +export interface ToolkitReadResult { + toolkits: MergedToolkit[]; + errors: string[]; +} + +const isToolkitFile = (fileName: string): boolean => + fileName.endsWith(".json") && fileName !== "index.json"; + +const normalizeToolkitKey = (toolkitId: string): string => + toolkitId.toLowerCase(); + +const readToolkitFile = async ( + filePath: string +): Promise => { + try { + const content = await readFile(filePath, "utf-8"); + const parsed = JSON.parse(content) as unknown; + const result = MergedToolkitSchema.safeParse(parsed); + if (!result.success) { + return null; + } + return result.data; + } catch { + return null; + } +}; + +export const readToolkitsFromDir = async ( + dir: string +): Promise => { + const toolkits: MergedToolkit[] = []; + const errors: string[] = []; + + let entries: string[] = []; + try { + entries = await readdir(dir); + } catch (error) { + return { + toolkits: [], + errors: [`Failed to read output directory: ${error}`], + }; + } + + const jsonFiles = entries.filter(isToolkitFile); + for (const fileName of jsonFiles) { + const filePath = join(dir, fileName); + const toolkit = await readToolkitFile(filePath); + if (!toolkit) { + errors.push(`Invalid toolkit JSON: ${fileName}`); + continue; + } + toolkits.push(toolkit); + } + + return { toolkits, errors }; +}; + +const readIndexFile = async (dir: string): Promise => { + const indexPath = join(dir, "index.json"); + try { + const content = await readFile(indexPath, "utf-8"); + const parsed = JSON.parse(content) as unknown; + const result = ToolkitIndexSchema.safeParse(parsed); + if (!result.success) { + return null; + } + return result.data; + } catch { + return null; + } +}; + +export const verifyOutputDir = async ( + dir: string +): Promise => { + const errors: string[] = []; + const warnings: string[] = []; + + const { toolkits, errors: toolkitErrors } = await readToolkitsFromDir(dir); + errors.push(...toolkitErrors); + + if (toolkits.length === 0) { + errors.push("No toolkit JSON files found in output directory."); + } + + const toolkitById = new Map(); + for (const toolkit of toolkits) { + const key = normalizeToolkitKey(toolkit.id); + toolkitById.set(key, toolkit); + } + + const index = await readIndexFile(dir); + if (!index) { + errors.push("Missing or invalid index.json in output directory."); + } else { + const indexIds = new Set( + index.toolkits.map((entry) => normalizeToolkitKey(entry.id)) + ); + + for (const [toolkitId, toolkit] of toolkitById) { + if (!indexIds.has(toolkitId)) { + errors.push(`index.json missing toolkit entry: ${toolkit.id}`); + continue; + } + + const entry = index.toolkits.find( + (item) => normalizeToolkitKey(item.id) === toolkitId + ); + if (!entry) { + continue; + } + + if (entry.version !== toolkit.version) { + errors.push( + `index.json version mismatch for ${toolkit.id}: ${entry.version} vs ${toolkit.version}` + ); + } + + if (entry.category !== toolkit.metadata.category) { + errors.push( + `index.json category mismatch for ${toolkit.id}: ${entry.category} vs ${toolkit.metadata.category}` + ); + } + + if (entry.toolCount !== toolkit.tools.length) { + errors.push( + `index.json toolCount mismatch for ${toolkit.id}: ${entry.toolCount} vs ${toolkit.tools.length}` + ); + } + + const authType = toolkit.auth?.type ?? "none"; + if (entry.authType !== authType) { + errors.push( + `index.json authType mismatch for ${toolkit.id}: ${entry.authType} vs ${authType}` + ); + } + } + + for (const entry of index.toolkits) { + const toolkitId = normalizeToolkitKey(entry.id); + if (!toolkitById.has(toolkitId)) { + errors.push(`index.json entry has no matching file: ${entry.id}`); + } + } + } + + const fileNames = await readdir(dir).catch(() => []); + const jsonFiles = fileNames.filter(isToolkitFile); + for (const fileName of jsonFiles) { + const baseName = basename(fileName, ".json"); + if (baseName !== baseName.toLowerCase()) { + errors.push(`File name must be lowercase: ${fileName}`); + } + const idFromFile = baseName.toLowerCase(); + if (!toolkitById.has(idFromFile)) { + errors.push(`File name does not match toolkit id: ${fileName}`); + } + } + + return { + valid: errors.length === 0, + errors, + warnings, + }; +}; diff --git a/toolkit-docs-generator/src/llm/index.ts b/toolkit-docs-generator/src/llm/index.ts index 460ef494d..93538c018 100644 --- a/toolkit-docs-generator/src/llm/index.ts +++ b/toolkit-docs-generator/src/llm/index.ts @@ -1,2 +1,3 @@ export * from "./client.js"; export * from "./tool-example-generator.js"; +export * from "./toolkit-summary-generator.js"; \ No newline at end of file diff --git a/toolkit-docs-generator/src/llm/toolkit-summary-generator.ts b/toolkit-docs-generator/src/llm/toolkit-summary-generator.ts new file mode 100644 index 000000000..82c963f90 --- /dev/null +++ b/toolkit-docs-generator/src/llm/toolkit-summary-generator.ts @@ -0,0 +1,148 @@ +import type { ToolkitSummaryGenerator } from "../merger/data-merger.js"; +import type { + MergedTool, + MergedToolkit, + SecretType, +} from "../types/index.js"; +import type { LlmClient } from "./client.js"; + +export interface LlmToolkitSummaryGeneratorConfig { + readonly client: LlmClient; + readonly model: string; + readonly temperature?: number; + readonly maxTokens?: number; + readonly systemPrompt?: string; +} + +const defaultSystemPrompt = + "Return only valid JSON. No markdown, no extra text."; + +const formatToolLines = (tools: MergedTool[]): string => { + if (tools.length === 0) { + return "None"; + } + + return tools + .map( + (tool) => + `- ${tool.qualifiedName}: ${tool.description ?? "No description"}` + ) + .join("\n"); +}; + +const formatAuth = (toolkit: MergedToolkit): string => { + if (!toolkit.auth) { + return "none"; + } + + const scopes = + toolkit.auth.allScopes.length > 0 + ? toolkit.auth.allScopes.join(", ") + : "None"; + const provider = toolkit.auth.providerId ?? "unknown"; + + return `${toolkit.auth.type}; provider: ${provider}; scopes: ${scopes}`; +}; + +const collectSecrets = (tools: MergedTool[]) => { + const secretNames = new Set(); + const secretTypes = new Set(); + + for (const tool of tools) { + for (const name of tool.secrets) { + secretNames.add(name); + } + for (const secret of tool.secretsInfo) { + secretTypes.add(secret.type); + } + } + + return { + names: Array.from(secretNames), + types: Array.from(secretTypes), + }; +}; + +const buildPrompt = (toolkit: MergedToolkit): string => { + const secrets = collectSecrets(toolkit.tools); + + return [ + "Write a concise summary for Arcade toolkit docs.", + 'Return JSON: {"summary": ""}', + "", + "Requirements:", + "- 60 to 140 words.", + "- Start with 1 to 2 sentences that explain the provider and what the toolkit enables.", + "- Add a **Capabilities** section with 3 to 5 bullet points.", + "- Do not list tools one by one. Summarize shared capabilities.", + "- If auth type is oauth2 or mixed, add an **OAuth** section with provider and scopes.", + "- If auth type is api_key or mixed, mention API key usage in **OAuth**.", + "- If any secrets exist, add a **Secrets** section describing secret types and examples.", + "- Use Markdown. Keep it concise and developer-focused.", + "", + `Toolkit: ${toolkit.label} (${toolkit.id})`, + `Description: ${toolkit.description ?? "No description"}`, + `Auth: ${formatAuth(toolkit)}`, + `Secret types: ${secrets.types.length > 0 ? secrets.types.join(", ") : "None"}`, + `Secret names: ${secrets.names.length > 0 ? secrets.names.join(", ") : "None"}`, + `Tools (${toolkit.tools.length}):`, + formatToolLines(toolkit.tools), + ].join("\n"); +}; + +const extractJson = (text: string): string => { + const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/); + if (fenced?.[1]) { + return fenced[1].trim(); + } + return text.trim(); +}; + +const parseJsonObject = (text: string): Record => { + const jsonText = extractJson(text); + const parsed = JSON.parse(jsonText) as unknown; + + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error("LLM response is not a JSON object"); + } + + return parsed as Record; +}; + +export class LlmToolkitSummaryGenerator implements ToolkitSummaryGenerator { + private readonly client: LlmClient; + private readonly model: string; + private readonly temperature: number | undefined; + private readonly maxTokens: number | undefined; + private readonly systemPrompt: string; + + constructor(config: LlmToolkitSummaryGeneratorConfig) { + this.client = config.client; + this.model = config.model; + this.temperature = config.temperature; + this.maxTokens = config.maxTokens; + this.systemPrompt = config.systemPrompt ?? defaultSystemPrompt; + } + + async generate(toolkit: MergedToolkit): Promise { + const prompt = buildPrompt(toolkit); + const response = await this.client.generateText({ + model: this.model, + prompt, + system: this.systemPrompt, + ...(this.temperature !== undefined + ? { temperature: this.temperature } + : {}), + ...(this.maxTokens !== undefined ? { maxTokens: this.maxTokens } : {}), + }); + + const payload = parseJsonObject(response); + const summary = payload.summary; + + if (typeof summary !== "string" || summary.trim().length === 0) { + throw new Error("LLM response missing summary"); + } + + return summary.trim(); + } +} diff --git a/toolkit-docs-generator/src/merger/data-merger.ts b/toolkit-docs-generator/src/merger/data-merger.ts index 2bf806505..de006476b 100644 --- a/toolkit-docs-generator/src/merger/data-merger.ts +++ b/toolkit-docs-generator/src/merger/data-merger.ts @@ -19,6 +19,7 @@ import type { ToolkitAuthType, ToolkitMetadata, } from "../types/index.js"; +import { mapWithConcurrency } from "../utils/concurrency.js"; // ============================================================================ // Merger Configuration @@ -27,7 +28,15 @@ import type { export interface DataMergerConfig { toolkitDataSource: IToolkitDataSource; customSectionsSource: ICustomSectionsSource; - toolExampleGenerator: ToolExampleGenerator; + toolExampleGenerator?: ToolExampleGenerator; + toolkitSummaryGenerator?: ToolkitSummaryGenerator; + previousToolkits?: ReadonlyMap; + /** Maximum concurrent LLM calls for tool examples (default: 5) */ + llmConcurrency?: number; + /** Maximum concurrent toolkit processing (default: 3) */ + toolkitConcurrency?: number; + /** Progress callback for toolkit processing */ + onToolkitProgress?: (toolkitId: string, status: "start" | "done") => void; } export interface MergeResult { @@ -44,6 +53,16 @@ export interface ToolExampleGenerator { generate: (tool: ToolDefinition) => Promise; } +export interface ToolkitSummaryGenerator { + generate: (toolkit: MergedToolkit) => Promise; +} + +interface MergeToolkitOptions { + previousToolkit?: MergedToolkit; + /** Maximum concurrent LLM calls for tool examples (default: 5) */ + llmConcurrency?: number; +} + // ============================================================================ // Pure Functions for Data Transformation // ============================================================================ @@ -85,6 +104,95 @@ export const computeAllScopes = ( return Array.from(scopeSet).sort(); }; +const normalizeList = (values: readonly string[]): string[] => + Array.from(values).sort(); + +const stableStringify = (value: unknown): string => { + if (Array.isArray(value)) { + return `[${value.map((item) => stableStringify(item)).join(",")}]`; + } + + if (value && typeof value === "object") { + const entries = Object.entries(value as Record).sort( + ([keyA], [keyB]) => keyA.localeCompare(keyB) + ); + return `{${entries + .map(([key, entryValue]) => { + return `${JSON.stringify(key)}:${stableStringify(entryValue)}`; + }) + .join(",")}}`; + } + + return JSON.stringify(value); +}; + +const buildToolSignatureInput = ( + tool: ToolDefinition | MergedTool +): Record => ({ + name: tool.name, + qualifiedName: tool.qualifiedName, + description: tool.description ?? null, + parameters: tool.parameters + .map((param) => ({ + name: param.name, + type: param.type, + innerType: param.innerType ?? null, + required: param.required, + description: param.description ?? null, + enum: param.enum ? normalizeList(param.enum) : null, + inferrable: param.inferrable ?? true, + })) + .sort((left, right) => left.name.localeCompare(right.name)), + auth: tool.auth + ? { + providerId: tool.auth.providerId ?? null, + providerType: tool.auth.providerType, + scopes: normalizeList(tool.auth.scopes), + } + : null, + secrets: normalizeList(tool.secrets), + output: tool.output + ? { + type: tool.output.type, + description: tool.output.description ?? null, + } + : null, +}); + +const buildToolSignature = (tool: ToolDefinition | MergedTool): string => + stableStringify(buildToolSignatureInput(tool)); + +const buildToolkitSummarySignature = (toolkit: MergedToolkit): string => + stableStringify({ + id: toolkit.id, + label: toolkit.label, + description: toolkit.description ?? null, + auth: toolkit.auth + ? { + type: toolkit.auth.type, + providerId: toolkit.auth.providerId ?? null, + allScopes: normalizeList(toolkit.auth.allScopes), + } + : null, + tools: toolkit.tools + .map((tool) => ({ + qualifiedName: tool.qualifiedName, + signature: buildToolSignature(tool), + })) + .sort((left, right) => left.qualifiedName.localeCompare(right.qualifiedName)), + }); + +const shouldReuseExample = ( + tool: ToolDefinition, + previousTool: MergedTool +): boolean => { + if (!previousTool.codeExample) { + return false; + } + + return buildToolSignature(tool) === buildToolSignature(previousTool); +}; + /** * Determine the auth type from tools */ @@ -164,8 +272,40 @@ const transformMetadata = ( const transformTool = async ( tool: ToolDefinition, toolChunks: { [key: string]: DocumentationChunk[] }, - toolExampleGenerator: ToolExampleGenerator + toolExampleGenerator: ToolExampleGenerator | undefined, + previousTool?: MergedTool ): Promise => { + if (previousTool && shouldReuseExample(tool, previousTool)) { + return { + name: tool.name, + qualifiedName: tool.qualifiedName, + fullyQualifiedName: tool.fullyQualifiedName, + description: tool.description, + parameters: tool.parameters, + auth: tool.auth, + secrets: tool.secrets, + secretsInfo: previousTool.secretsInfo ?? [], + output: tool.output, + documentationChunks: toolChunks[tool.name] ?? [], + codeExample: previousTool.codeExample, + }; + } + + if (!toolExampleGenerator) { + return { + name: tool.name, + qualifiedName: tool.qualifiedName, + fullyQualifiedName: tool.fullyQualifiedName, + description: tool.description, + parameters: tool.parameters, + auth: tool.auth, + secrets: tool.secrets, + secretsInfo: [], + output: tool.output, + documentationChunks: toolChunks[tool.name] ?? [], + }; + } + const exampleResult = await toolExampleGenerator.generate(tool); return { @@ -195,7 +335,8 @@ export const mergeToolkit = async ( tools: readonly ToolDefinition[], metadata: ToolkitMetadata | null, customSections: CustomSections | null, - toolExampleGenerator: ToolExampleGenerator + toolExampleGenerator: ToolExampleGenerator | undefined, + options: MergeToolkitOptions = {} ): Promise => { const warnings: string[] = []; @@ -215,8 +356,9 @@ export const mergeToolkit = async ( ? extractVersion(firstTool.fullyQualifiedName) : "0.0.0"; - // Get toolkit description from first tool's toolkit info - const description = firstTool?.description ?? null; + // Prefer toolkit-level description when available + const description = + firstTool?.toolkitDescription ?? firstTool?.description ?? null; // Build auth info const authType = determineAuthType(tools); @@ -236,8 +378,24 @@ export const mergeToolkit = async ( const toolChunks = (customSections?.toolChunks ?? {}) as { [key: string]: DocumentationChunk[]; }; - const mergedTools = await Promise.all( - tools.map((tool) => transformTool(tool, toolChunks, toolExampleGenerator)) + const previousToolByQualifiedName = new Map(); + if (options.previousToolkit) { + for (const tool of options.previousToolkit.tools) { + previousToolByQualifiedName.set(tool.qualifiedName, tool); + } + } + + const llmConcurrency = options.llmConcurrency ?? 5; + const mergedTools = await mapWithConcurrency( + tools, + (tool) => + transformTool( + tool, + toolChunks, + toolExampleGenerator, + previousToolByQualifiedName.get(tool.qualifiedName) + ), + llmConcurrency ); // Build final toolkit @@ -270,12 +428,69 @@ export const mergeToolkit = async ( export class DataMerger { private readonly toolkitDataSource: IToolkitDataSource; private readonly customSectionsSource: ICustomSectionsSource; - private readonly toolExampleGenerator: ToolExampleGenerator; + private readonly toolExampleGenerator: ToolExampleGenerator | undefined; + private readonly toolkitSummaryGenerator: ToolkitSummaryGenerator | undefined; + private readonly previousToolkits: + | ReadonlyMap + | undefined; + private readonly llmConcurrency: number; + private readonly toolkitConcurrency: number; + private readonly onToolkitProgress: + | ((toolkitId: string, status: "start" | "done") => void) + | undefined; constructor(config: DataMergerConfig) { this.toolkitDataSource = config.toolkitDataSource; this.customSectionsSource = config.customSectionsSource; this.toolExampleGenerator = config.toolExampleGenerator; + this.toolkitSummaryGenerator = config.toolkitSummaryGenerator; + this.previousToolkits = config.previousToolkits; + this.llmConcurrency = config.llmConcurrency ?? 5; + this.toolkitConcurrency = config.toolkitConcurrency ?? 3; + this.onToolkitProgress = config.onToolkitProgress; + } + + private getPreviousToolkit(toolkitId: string): MergedToolkit | undefined { + if (!this.previousToolkits) { + return undefined; + } + + return ( + this.previousToolkits.get(toolkitId.toLowerCase()) ?? + this.previousToolkits.get(toolkitId) + ); + } + + private async maybeGenerateSummary( + result: MergeResult, + previousToolkit?: MergedToolkit + ): Promise { + if (previousToolkit?.summary) { + const currentSignature = buildToolkitSummarySignature(result.toolkit); + const previousSignature = buildToolkitSummarySignature(previousToolkit); + if (currentSignature === previousSignature) { + result.toolkit.summary = previousToolkit.summary; + return; + } + } + + if (!this.toolkitSummaryGenerator) { + return; + } + + if (result.toolkit.summary) { + return; + } + + try { + const summary = await this.toolkitSummaryGenerator.generate(result.toolkit); + result.toolkit.summary = summary; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + result.warnings.push( + `Summary generation failed for ${result.toolkit.id}: ${message}` + ); + } } /** @@ -294,36 +509,59 @@ export class DataMerger { const customSections = await this.customSectionsSource.getCustomSections(toolkitId); - return mergeToolkit( + const previousToolkit = this.getPreviousToolkit(toolkitId); + const result = await mergeToolkit( toolkitId, toolkitData.tools, toolkitData.metadata, customSections, - this.toolExampleGenerator + this.toolExampleGenerator, + { + ...(previousToolkit ? { previousToolkit } : {}), + llmConcurrency: this.llmConcurrency, + } ); + await this.maybeGenerateSummary(result, previousToolkit); + + return result; } /** * Merge data for all toolkits */ async mergeAllToolkits(): Promise { - const results: MergeResult[] = []; - const allToolkitsData = await this.toolkitDataSource.fetchAllToolkitsData(); - for (const [toolkitId, toolkitData] of allToolkitsData) { - const customSections = - await this.customSectionsSource.getCustomSections(toolkitId); - - const result = await mergeToolkit( - toolkitId, - toolkitData.tools, - toolkitData.metadata, - customSections, - this.toolExampleGenerator - ); - results.push(result); - } + const toolkitEntries = Array.from(allToolkitsData.entries()); + + const results = await mapWithConcurrency( + toolkitEntries, + async ([toolkitId, toolkitData]) => { + this.onToolkitProgress?.(toolkitId, "start"); + + const customSections = + await this.customSectionsSource.getCustomSections(toolkitId); + + const previousToolkit = this.getPreviousToolkit(toolkitId); + const result = await mergeToolkit( + toolkitId, + toolkitData.tools, + toolkitData.metadata, + customSections, + this.toolExampleGenerator, + { + ...(previousToolkit ? { previousToolkit } : {}), + llmConcurrency: this.llmConcurrency, + } + ); + await this.maybeGenerateSummary(result, previousToolkit); + + this.onToolkitProgress?.(toolkitId, "done"); + + return result; + }, + this.toolkitConcurrency + ); return results; } diff --git a/toolkit-docs-generator/src/sources/engine-api.ts b/toolkit-docs-generator/src/sources/engine-api.ts new file mode 100644 index 000000000..0ff61c000 --- /dev/null +++ b/toolkit-docs-generator/src/sources/engine-api.ts @@ -0,0 +1,171 @@ +import type { ToolDefinition } from "../types/index.js"; +import type { FetchOptions, IToolDataSource } from "./internal.js"; +import { + parseToolMetadataError, + parseToolMetadataResponse, +} from "./tool-metadata-schema.js"; + +export interface EngineApiSourceConfig { + /** Base URL for Engine (e.g., https://api.arcade.dev) */ + readonly baseUrl: string; + /** Engine API key (Bearer token) */ + readonly apiKey: string; + /** Optional fetch implementation for testing */ + readonly fetchFn?: typeof fetch; + /** Page size for pagination */ + readonly pageSize?: number; + /** Include all versions in results (required for version filtering) */ + readonly includeAllVersions?: boolean; +} + +type ToolMetadataPage = { + items: ToolDefinition[]; + totalCount: number; +}; + +const DEFAULT_PAGE_SIZE = 100; +const MAX_PAGE_SIZE = 100; + +const normalizePageSize = (value?: number): number => { + if (!value || Number.isNaN(value) || value <= 0) { + return DEFAULT_PAGE_SIZE; + } + return Math.min(value, MAX_PAGE_SIZE); +}; + +const normalizeBaseUrl = (baseUrl: string): string => baseUrl.replace(/\/+$/, ""); + +const buildEndpointUrl = (baseUrl: string): string => { + const normalized = normalizeBaseUrl(baseUrl); + if (normalized.endsWith("/v1")) { + return `${normalized}/tool_metadata`; + } + return `${normalized}/v1/tool_metadata`; +}; + +export class EngineApiSource implements IToolDataSource { + private readonly endpoint: string; + private readonly apiKey: string; + private readonly fetchFn: typeof fetch; + private readonly pageSize: number; + private readonly includeAllVersions: boolean; + + constructor(config: EngineApiSourceConfig) { + this.endpoint = buildEndpointUrl(config.baseUrl); + this.apiKey = config.apiKey; + this.fetchFn = config.fetchFn ?? fetch; + this.pageSize = normalizePageSize(config.pageSize); + this.includeAllVersions = config.includeAllVersions ?? false; + } + + private async fetchPage( + options: FetchOptions | undefined, + offset: number + ): Promise { + const url = new URL(this.endpoint); + url.searchParams.set("limit", String(this.pageSize)); + url.searchParams.set("offset", String(offset)); + const includeAllVersions = options?.version ? true : this.includeAllVersions; + if (includeAllVersions) { + url.searchParams.set("include_all_versions", "true"); + } + + if (options?.toolkitId) { + url.searchParams.set("toolkit", options.toolkitId); + } + if (options?.version) { + url.searchParams.set("version", options.version); + } + if (options?.providerId) { + url.searchParams.set("provider_id", options.providerId); + } + + const response = await this.fetchFn(url.toString(), { + headers: { + Authorization: `Bearer ${this.apiKey}`, + Accept: "application/json", + }, + }); + + if (!response.ok) { + const errorDetail = await this.getErrorDetail(response); + throw new Error( + `Engine API error ${response.status}: ${errorDetail ?? response.statusText}` + ); + } + + const payload = (await response.json()) as unknown; + const parsed = parseToolMetadataResponse(payload); + return { + items: parsed.items, + totalCount: parsed.totalCount, + }; + } + + async fetchToolsByToolkit( + toolkitId: string + ): Promise { + return this.fetchAllTools({ toolkitId }); + } + + async fetchAllTools( + options?: FetchOptions + ): Promise { + const tools: ToolDefinition[] = []; + let offset = 0; + let totalCount = 0; + + while (true) { + const page = await this.fetchPage(options, offset); + if (offset === 0) { + totalCount = page.totalCount; + } + tools.push(...page.items); + + if (page.items.length === 0) { + break; + } + + offset += page.items.length; + if (offset >= totalCount) { + break; + } + } + + return tools; + } + + async isAvailable(): Promise { + try { + await this.fetchPage(undefined, 0); + return true; + } catch { + return false; + } + } + + private async getErrorDetail( + response: Response + ): Promise { + const contentType = response.headers.get("content-type") ?? ""; + if (!contentType.includes("application/json")) { + return null; + } + + try { + const payload = (await response.json()) as unknown; + const parsed = parseToolMetadataError(payload); + if (parsed) { + return `${parsed.name}: ${parsed.message}`; + } + } catch { + return null; + } + + return null; + } +} + +export const createEngineApiSource = ( + config: EngineApiSourceConfig +): IToolDataSource => new EngineApiSource(config); diff --git a/toolkit-docs-generator/src/sources/index.ts b/toolkit-docs-generator/src/sources/index.ts index 25577c682..6c6621b44 100644 --- a/toolkit-docs-generator/src/sources/index.ts +++ b/toolkit-docs-generator/src/sources/index.ts @@ -3,6 +3,7 @@ */ export * from "./custom-sections-file.js"; +export * from "./engine-api.js"; export * from "./in-memory.js"; export * from "./interfaces.js"; export * from "./mock-engine-api.js"; diff --git a/toolkit-docs-generator/src/sources/mock-engine-api.ts b/toolkit-docs-generator/src/sources/mock-engine-api.ts index 1bbe66b65..cb681eebb 100644 --- a/toolkit-docs-generator/src/sources/mock-engine-api.ts +++ b/toolkit-docs-generator/src/sources/mock-engine-api.ts @@ -6,122 +6,10 @@ * when the API endpoint is ready. */ import { readFile } from "fs/promises"; -import { z } from "zod"; -import type { ToolDefinition, ToolParameter } from "../types/index.js"; +import type { ToolDefinition } from "../types/index.js"; import { normalizeId } from "../utils/fp.js"; import type { FetchOptions, IToolDataSource } from "./internal.js"; - -// ============================================================================ -// Engine API Response Schemas (matches planningdoc.md spec) -// ============================================================================ - -const ApiValueSchemaSchema = z.object({ - val_type: z.string(), - inner_val_type: z.string().nullable(), - enum: z.array(z.string()).nullable(), -}); - -const ApiParameterSchema = z.object({ - name: z.string(), - required: z.boolean(), - description: z.string().nullable(), - value_schema: ApiValueSchemaSchema, - inferrable: z.boolean().default(true), -}); - -const ApiToolkitInfoSchema = z.object({ - name: z.string(), - version: z.string(), - description: z.string().nullable(), -}); - -const ApiInputSchema = z.object({ - parameters: z.array(ApiParameterSchema), -}); - -const ApiOutputSchema = z - .object({ - available_modes: z.array(z.string()).optional(), - description: z.string().nullable(), - value_schema: ApiValueSchemaSchema.nullable(), - }) - .nullable(); - -const ApiAuthRequirementSchema = z - .object({ - id: z.string(), - provider_id: z.string().nullable(), - provider_type: z.string(), - scopes: z.array(z.string()).nullable(), - }) - .nullable(); - -const ApiRequirementsSchema = z - .object({ - authorization: ApiAuthRequirementSchema, - secrets: z.array(z.string()).nullable(), - }) - .nullable(); - -const ApiToolSchema = z.object({ - fully_qualified_name: z.string(), - qualified_name: z.string(), - name: z.string(), - description: z.string().nullable(), - toolkit: ApiToolkitInfoSchema, - input: ApiInputSchema, - output: ApiOutputSchema, - requirements: ApiRequirementsSchema, -}); - -const ApiResponseSchema = z.object({ - items: z.array(ApiToolSchema), - total_count: z.number(), -}); - -type ApiTool = z.infer; - -// ============================================================================ -// Transform API response to internal types -// ============================================================================ - -const transformParameter = ( - apiParam: z.infer -): ToolParameter => ({ - name: apiParam.name, - type: apiParam.value_schema.val_type, - innerType: apiParam.value_schema.inner_val_type ?? undefined, - required: apiParam.required, - description: apiParam.description, - enum: apiParam.value_schema.enum, - inferrable: apiParam.inferrable, -}); - -const transformTool = (apiTool: ApiTool): ToolDefinition => ({ - name: apiTool.name, - qualifiedName: apiTool.qualified_name, - fullyQualifiedName: apiTool.fully_qualified_name, - description: apiTool.description, - parameters: apiTool.input.parameters.map(transformParameter), - auth: apiTool.requirements?.authorization - ? { - providerId: apiTool.requirements.authorization.provider_id, - providerType: apiTool.requirements.authorization.provider_type, - scopes: apiTool.requirements.authorization.scopes ?? [], - } - : null, - secrets: apiTool.requirements?.secrets ?? [], - output: apiTool.output - ? { - type: apiTool.output.value_schema?.val_type ?? "unknown", - description: apiTool.output.description, - } - : null, -}); - -// ============================================================================ -// Mock Engine API Source -// ============================================================================ +import { parseToolMetadataResponse } from "./tool-metadata-schema.js"; export interface MockEngineApiConfig { /** Path to the JSON fixture file */ @@ -149,8 +37,8 @@ export class MockEngineApiSource implements IToolDataSource { const content = await readFile(this.fixtureFilePath, "utf-8"); const json = JSON.parse(content); - const parsed = ApiResponseSchema.parse(json); - this.cachedData = parsed.items.map(transformTool); + const parsed = parseToolMetadataResponse(json); + this.cachedData = parsed.items; return this.cachedData; } diff --git a/toolkit-docs-generator/src/sources/tool-metadata-schema.ts b/toolkit-docs-generator/src/sources/tool-metadata-schema.ts new file mode 100644 index 000000000..cc770b15a --- /dev/null +++ b/toolkit-docs-generator/src/sources/tool-metadata-schema.ts @@ -0,0 +1,145 @@ +import { z } from "zod"; +import type { ToolDefinition, ToolParameter } from "../types/index.js"; + +const ToolMetadataValueSchema = z.object({ + val_type: z.string(), + inner_val_type: z.string().nullable().optional(), + enum: z.array(z.string()).nullable().optional(), +}); + +const ToolMetadataParameterSchema = z.object({ + name: z.string(), + required: z.boolean(), + description: z.string().nullable(), + value_schema: ToolMetadataValueSchema, + inferrable: z.boolean().default(true), +}); + +const ToolMetadataToolkitSchema = z.object({ + name: z.string(), + version: z.string(), + description: z.string().nullable(), +}); + +const ToolMetadataInputSchema = z.object({ + parameters: z.array(ToolMetadataParameterSchema), +}); + +const ToolMetadataOutputSchema = z + .object({ + description: z.string().nullable(), + value_schema: ToolMetadataValueSchema.nullable(), + }) + .nullable() + .optional(); + +const ToolMetadataAuthorizationSchema = z + .object({ + id: z.string().nullable().optional(), + provider_id: z.string().nullable().optional(), + provider_type: z.string().nullable().optional(), + scopes: z.array(z.string()), + }) + .nullable(); + +const ToolMetadataSecretSchema = z.union([ + z.object({ key: z.string() }), + z.string(), +]); + +const ToolMetadataRequirementsSchema = z + .object({ + authorization: ToolMetadataAuthorizationSchema.optional(), + secrets: z.array(ToolMetadataSecretSchema).nullable().optional(), + }) + .nullable() + .optional(); + +const ToolMetadataItemSchema = z.object({ + fully_qualified_name: z.string(), + qualified_name: z.string(), + name: z.string(), + description: z.string().nullable(), + toolkit: ToolMetadataToolkitSchema, + input: ToolMetadataInputSchema, + output: ToolMetadataOutputSchema, + requirements: ToolMetadataRequirementsSchema, +}); + +export const ToolMetadataResponseSchema = z.object({ + items: z.array(ToolMetadataItemSchema), + total_count: z.number(), +}); + +const ToolMetadataErrorSchema = z.object({ + name: z.string(), + message: z.string(), +}); + +type ToolMetadataItem = z.infer; + +const transformParameter = ( + apiParam: z.infer +): ToolParameter => ({ + name: apiParam.name, + type: apiParam.value_schema.val_type, + innerType: apiParam.value_schema.inner_val_type ?? undefined, + required: apiParam.required, + description: apiParam.description, + enum: apiParam.value_schema.enum ?? null, + inferrable: apiParam.inferrable, +}); + +const normalizeSecrets = ( + secrets: z.infer[] | null | undefined +): string[] => { + if (!secrets || secrets.length === 0) { + return []; + } + + return secrets + .map((secret) => (typeof secret === "string" ? secret : secret.key)) + .filter((secret) => secret.length > 0); +}; + +export const transformToolMetadataItem = ( + apiTool: ToolMetadataItem +): ToolDefinition => ({ + name: apiTool.name, + qualifiedName: apiTool.qualified_name, + fullyQualifiedName: apiTool.fully_qualified_name, + description: apiTool.description, + toolkitDescription: apiTool.toolkit.description, + parameters: apiTool.input.parameters.map(transformParameter), + auth: apiTool.requirements?.authorization + ? { + providerId: apiTool.requirements.authorization.provider_id ?? null, + providerType: apiTool.requirements.authorization.provider_type ?? "unknown", + scopes: apiTool.requirements.authorization.scopes ?? [], + } + : null, + secrets: normalizeSecrets(apiTool.requirements?.secrets), + output: apiTool.output + ? { + type: apiTool.output.value_schema?.val_type ?? "unknown", + description: apiTool.output.description ?? null, + } + : null, +}); + +export const parseToolMetadataResponse = ( + payload: unknown +): { items: ToolDefinition[]; totalCount: number } => { + const parsed = ToolMetadataResponseSchema.parse(payload); + return { + items: parsed.items.map(transformToolMetadataItem), + totalCount: parsed.total_count, + }; +}; + +export const parseToolMetadataError = ( + payload: unknown +): { name: string; message: string } | null => { + const parsed = ToolMetadataErrorSchema.safeParse(payload); + return parsed.success ? parsed.data : null; +}; diff --git a/toolkit-docs-generator/src/sources/toolkit-data-source.ts b/toolkit-docs-generator/src/sources/toolkit-data-source.ts index b547942b8..7146de11a 100644 --- a/toolkit-docs-generator/src/sources/toolkit-data-source.ts +++ b/toolkit-docs-generator/src/sources/toolkit-data-source.ts @@ -9,6 +9,10 @@ import { join } from "path"; import type { ToolDefinition, ToolkitMetadata } from "../types/index.js"; import type { IMetadataSource, IToolDataSource } from "./internal.js"; +import { + createEngineApiSource, + type EngineApiSourceConfig, +} from "./engine-api.js"; import { createMockEngineApiSource } from "./mock-engine-api.js"; import { createMockMetadataSource } from "./mock-metadata.js"; @@ -177,6 +181,25 @@ export const createCombinedToolkitDataSource = ( config: CombinedToolkitDataSourceConfig ): IToolkitDataSource => new CombinedToolkitDataSource(config); +// ============================================================================ +// Engine Toolkit Data Source +// ============================================================================ + +export interface EngineToolkitDataSourceConfig { + /** Engine API configuration */ + readonly engine: EngineApiSourceConfig; + /** Source for toolkit metadata */ + readonly metadataSource: IMetadataSource; +} + +export const createEngineToolkitDataSource = ( + config: EngineToolkitDataSourceConfig +): IToolkitDataSource => + createCombinedToolkitDataSource({ + toolSource: createEngineApiSource(config.engine), + metadataSource: config.metadataSource, + }); + // ============================================================================ // Mock Toolkit Data Source (Current: JSON fixtures) // ============================================================================ diff --git a/toolkit-docs-generator/src/types/index.ts b/toolkit-docs-generator/src/types/index.ts index 1740ae39d..3d2077aa2 100644 --- a/toolkit-docs-generator/src/types/index.ts +++ b/toolkit-docs-generator/src/types/index.ts @@ -99,6 +99,7 @@ export const ToolDefinitionSchema = z.object({ qualifiedName: z.string(), fullyQualifiedName: z.string(), description: z.string().nullable(), + toolkitDescription: z.string().nullable().optional(), parameters: z.array(ToolParameterSchema), auth: ToolAuthSchema.nullable(), secrets: z.array(z.string()), diff --git a/toolkit-docs-generator/src/utils/concurrency.ts b/toolkit-docs-generator/src/utils/concurrency.ts new file mode 100644 index 000000000..afd31cc99 --- /dev/null +++ b/toolkit-docs-generator/src/utils/concurrency.ts @@ -0,0 +1,75 @@ +/** + * Concurrency utilities for parallel processing with rate limiting + */ + +/** + * A simple concurrency limiter that allows running async tasks with a maximum + * number of concurrent executions. + */ +export class ConcurrencyLimiter { + private readonly maxConcurrency: number; + private running = 0; + private queue: Array<() => void> = []; + + constructor(maxConcurrency: number) { + this.maxConcurrency = Math.max(1, maxConcurrency); + } + + /** + * Run a task with concurrency limiting + */ + async run(task: () => Promise): Promise { + await this.acquire(); + try { + return await task(); + } finally { + this.release(); + } + } + + private async acquire(): Promise { + if (this.running < this.maxConcurrency) { + this.running++; + return; + } + + return new Promise((resolve) => { + this.queue.push(resolve); + }); + } + + private release(): void { + this.running--; + const next = this.queue.shift(); + if (next) { + this.running++; + next(); + } + } +} + +/** + * Run tasks in parallel with a concurrency limit + * + * @param items - Items to process + * @param fn - Async function to apply to each item + * @param concurrency - Maximum concurrent executions (default: 5) + * @returns Array of results in the same order as input + */ +export const mapWithConcurrency = async ( + items: readonly T[], + fn: (item: T, index: number) => Promise, + concurrency = 5 +): Promise => { + const limiter = new ConcurrencyLimiter(concurrency); + return Promise.all( + items.map((item, index) => limiter.run(() => fn(item, index))) + ); +}; + +/** + * Create a concurrency limiter + */ +export const createConcurrencyLimiter = ( + maxConcurrency: number +): ConcurrencyLimiter => new ConcurrencyLimiter(maxConcurrency); diff --git a/toolkit-docs-generator/src/utils/index.ts b/toolkit-docs-generator/src/utils/index.ts index a8422a3bc..ca937e025 100644 --- a/toolkit-docs-generator/src/utils/index.ts +++ b/toolkit-docs-generator/src/utils/index.ts @@ -2,3 +2,4 @@ * Utility exports */ export * from "./fp.js"; +export * from "./concurrency.js"; diff --git a/toolkit-docs-generator/tests/generator/output-verifier.test.ts b/toolkit-docs-generator/tests/generator/output-verifier.test.ts new file mode 100644 index 000000000..f87eb3e05 --- /dev/null +++ b/toolkit-docs-generator/tests/generator/output-verifier.test.ts @@ -0,0 +1,233 @@ +import { mkdtemp, readFile, rename, rm, writeFile } from "fs/promises"; +import { tmpdir } from "os"; +import { join } from "path"; +import { describe, expect, it } from "vitest"; + +import { + createJsonGenerator, + verifyOutputDir, +} from "../../src/generator/index.js"; +import type { MergedToolkit } from "../../src/types/index.js"; + +const loadFixture = async (fileName: string): Promise => { + const fixturesDir = new URL("../fixtures/", import.meta.url); + const filePath = new URL(fileName, fixturesDir); + const content = await readFile(filePath, "utf-8"); + return JSON.parse(content) as MergedToolkit; +}; + +const withTempDir = async (fn: (dir: string) => Promise) => { + const dir = await mkdtemp(join(tmpdir(), "toolkit-docs-")); + try { + await fn(dir); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}; + +describe("verifyOutputDir", () => { + it("flags an empty output directory", async () => { + await withTempDir(async (dir) => { + const result = await verifyOutputDir(dir); + + expect(result.valid).toBe(false); + expect( + result.errors.some((error) => + error.includes("No toolkit JSON files found") + ) + ).toBe(true); + expect( + result.errors.some((error) => + error.includes("Missing or invalid index.json") + ) + ).toBe(true); + }); + }); + + it("accepts a valid output directory", async () => { + await withTempDir(async (dir) => { + const toolkits = await Promise.all([ + loadFixture("github-toolkit.json"), + loadFixture("slack-toolkit.json"), + ]); + const generator = createJsonGenerator({ + outputDir: dir, + prettyPrint: false, + generateIndex: true, + }); + + await generator.generateAll(toolkits); + + const result = await verifyOutputDir(dir); + + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + }); + + it("flags missing index.json", async () => { + await withTempDir(async (dir) => { + const toolkits = await Promise.all([ + loadFixture("github-toolkit.json"), + loadFixture("slack-toolkit.json"), + ]); + const generator = createJsonGenerator({ + outputDir: dir, + prettyPrint: false, + generateIndex: true, + }); + + await generator.generateAll(toolkits); + await rm(join(dir, "index.json")); + + const result = await verifyOutputDir(dir); + + expect(result.valid).toBe(false); + expect(result.errors.some((error) => error.includes("index.json"))).toBe( + true + ); + }); + }); + + it("flags index entries that do not match toolkit metadata", async () => { + await withTempDir(async (dir) => { + const toolkits = await Promise.all([ + loadFixture("github-toolkit.json"), + loadFixture("slack-toolkit.json"), + ]); + const generator = createJsonGenerator({ + outputDir: dir, + prettyPrint: false, + generateIndex: true, + }); + + await generator.generateAll(toolkits); + + const indexPath = join(dir, "index.json"); + const index = JSON.parse(await readFile(indexPath, "utf-8")) as { + toolkits: Array>; + }; + const githubEntry = index.toolkits.find( + (entry) => entry.id === "Github" + ); + if (githubEntry) { + githubEntry.version = "9.9.9"; + githubEntry.category = "social"; + githubEntry.toolCount = 999; + githubEntry.authType = "none"; + } + await writeFile(indexPath, JSON.stringify(index, null, 2), "utf-8"); + + const result = await verifyOutputDir(dir); + + expect(result.valid).toBe(false); + expect( + result.errors.some((error) => + error.includes("version mismatch for Github") + ) + ).toBe(true); + expect( + result.errors.some((error) => + error.includes("category mismatch for Github") + ) + ).toBe(true); + expect( + result.errors.some((error) => + error.includes("toolCount mismatch for Github") + ) + ).toBe(true); + expect( + result.errors.some((error) => + error.includes("authType mismatch for Github") + ) + ).toBe(true); + }); + }); + + it("flags index entries without matching files", async () => { + await withTempDir(async (dir) => { + const toolkits = await Promise.all([ + loadFixture("github-toolkit.json"), + loadFixture("slack-toolkit.json"), + ]); + const generator = createJsonGenerator({ + outputDir: dir, + prettyPrint: false, + generateIndex: true, + }); + + await generator.generateAll(toolkits); + + const indexPath = join(dir, "index.json"); + const index = JSON.parse(await readFile(indexPath, "utf-8")) as { + toolkits: Array>; + }; + index.toolkits.push({ + id: "Ghost", + label: "Ghost", + version: "1.0.0", + category: "development", + toolCount: 0, + authType: "none", + }); + await writeFile(indexPath, JSON.stringify(index, null, 2), "utf-8"); + + const result = await verifyOutputDir(dir); + + expect(result.valid).toBe(false); + expect( + result.errors.some((error) => + error.includes("index.json entry has no matching file: Ghost") + ) + ).toBe(true); + }); + }); + + it("flags toolkit files that are not lowercase", async () => { + await withTempDir(async (dir) => { + const toolkits = await Promise.all([loadFixture("github-toolkit.json")]); + const generator = createJsonGenerator({ + outputDir: dir, + prettyPrint: false, + generateIndex: true, + }); + + await generator.generateAll(toolkits); + + await rename(join(dir, "github.json"), join(dir, "Github.json")); + + const result = await verifyOutputDir(dir); + + expect(result.valid).toBe(false); + expect( + result.errors.some((error) => + error.includes("File name must be lowercase: Github.json") + ) + ).toBe(true); + }); + }); + + it("flags toolkit files that do not match toolkit id", async () => { + await withTempDir(async (dir) => { + const toolkits = await Promise.all([loadFixture("github-toolkit.json")]); + const generator = createJsonGenerator({ + outputDir: dir, + prettyPrint: false, + generateIndex: true, + }); + + await generator.generateAll(toolkits); + + await rename(join(dir, "github.json"), join(dir, "mismatch.json")); + + const result = await verifyOutputDir(dir); + + expect(result.valid).toBe(false); + expect( + result.errors.some((error) => + error.includes("File name does not match toolkit id: mismatch.json") + ) + ).toBe(true); + }); + }); +}); diff --git a/toolkit-docs-generator/tests/llm/toolkit-summary-generator.test.ts b/toolkit-docs-generator/tests/llm/toolkit-summary-generator.test.ts new file mode 100644 index 000000000..9ff852ecc --- /dev/null +++ b/toolkit-docs-generator/tests/llm/toolkit-summary-generator.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; + +import { LlmToolkitSummaryGenerator } from "../../src/llm/toolkit-summary-generator.js"; +import type { LlmClient } from "../../src/llm/client.js"; +import type { MergedToolkit } from "../../src/types/index.js"; + +const createToolkit = (overrides: Partial = {}): MergedToolkit => ({ + id: "Github", + label: "GitHub", + version: "1.0.0", + description: "Tools for GitHub automation.", + metadata: { + category: "development", + iconUrl: "https://design-system.arcade.dev/icons/github.svg", + isBYOC: false, + isPro: false, + type: "arcade", + docsLink: "https://docs.arcade.dev/en/mcp-servers/development/github", + isComingSoon: false, + isHidden: false, + }, + auth: { + type: "oauth2", + providerId: "github", + allScopes: ["repo"], + }, + tools: [ + { + name: "CreateIssue", + qualifiedName: "Github.CreateIssue", + fullyQualifiedName: "Github.CreateIssue@1.0.0", + description: "Create issues in a repository.", + parameters: [], + auth: { + providerId: "github", + providerType: "oauth2", + scopes: ["repo"], + }, + secrets: ["GITHUB_API_KEY"], + secretsInfo: [{ name: "GITHUB_API_KEY", type: "api_key" }], + output: null, + documentationChunks: [], + }, + ], + documentationChunks: [], + customImports: [], + subPages: [], + generatedAt: "2026-01-15T00:00:00.000Z", + ...overrides, +}); + +describe("LlmToolkitSummaryGenerator", () => { + it("parses summary from a JSON response", async () => { + const client: LlmClient = { + generateText: async () => + '```json\n{"summary":"Concise summary."}\n```', + }; + const generator = new LlmToolkitSummaryGenerator({ + client, + model: "test-model", + }); + + const summary = await generator.generate(createToolkit()); + + expect(summary).toBe("Concise summary."); + }); + + it("includes tool descriptions and auth info in the prompt", async () => { + let capturedPrompt = ""; + const client: LlmClient = { + generateText: async ({ prompt }) => { + capturedPrompt = prompt; + return '{"summary":"OK"}'; + }, + }; + const generator = new LlmToolkitSummaryGenerator({ + client, + model: "test-model", + }); + + await generator.generate(createToolkit()); + + expect(capturedPrompt).toContain("Github.CreateIssue"); + expect(capturedPrompt).toContain("Create issues in a repository."); + expect(capturedPrompt).toContain("Auth:"); + expect(capturedPrompt).toContain("Secret types:"); + expect(capturedPrompt).toContain("Secret names:"); + }); +}); diff --git a/toolkit-docs-generator/tests/merger/data-merger.test.ts b/toolkit-docs-generator/tests/merger/data-merger.test.ts index c13b468a6..0535e963a 100644 --- a/toolkit-docs-generator/tests/merger/data-merger.test.ts +++ b/toolkit-docs-generator/tests/merger/data-merger.test.ts @@ -14,6 +14,7 @@ import { groupToolsByToolkit, mergeToolkit, type ToolExampleGenerator, + type ToolkitSummaryGenerator, } from "../../src/merger/data-merger.js"; import { EmptyCustomSectionsSource, @@ -38,6 +39,7 @@ const createTool = ( qualifiedName: "TestKit.TestTool", fullyQualifiedName: "TestKit.TestTool@1.0.0", description: "A test tool", + toolkitDescription: "Toolkit description", parameters: [ { name: "param1", @@ -105,6 +107,56 @@ const createStubGenerator = (): ToolExampleGenerator => ({ }), }); +const createStubSummaryGenerator = ( + summary: string +): ToolkitSummaryGenerator => ({ + generate: async (toolkit) => `${summary} (${toolkit.id})`, +}); + +const createCountingGenerator = () => { + let calls = 0; + const generator: ToolExampleGenerator = { + generate: async (tool) => { + calls += 1; + return { + codeExample: { + toolName: tool.qualifiedName, + parameters: {}, + requiresAuth: tool.auth !== null, + authProvider: tool.auth?.providerId ?? undefined, + tabLabel: tool.auth + ? "Call the Tool with User Authorization" + : "Call the Tool", + }, + secretsInfo: tool.secrets.map((secret) => ({ + name: secret, + type: "unknown", + })), + }; + }, + }; + + return { + generator, + getCalls: () => calls, + }; +}; + +const createCountingSummaryGenerator = () => { + let calls = 0; + const generator: ToolkitSummaryGenerator = { + generate: async () => { + calls += 1; + return "Generated summary"; + }, + }; + + return { + generator, + getCalls: () => calls, + }; +}; + // ============================================================================ // Pure Function Tests // ============================================================================ @@ -471,6 +523,37 @@ describe("mergeToolkit", () => { { name: "ACCESS_TOKEN", type: "token" }, ]); }); + + it("should omit code examples when generator is missing", async () => { + const tools = [createTool()]; + + const result = await mergeToolkit("TestKit", tools, null, null, undefined); + + expect(result.toolkit.tools[0]?.codeExample).toBeUndefined(); + expect(result.toolkit.tools[0]?.secretsInfo).toEqual([]); + }); + + it("should reuse previous code examples without generator", async () => { + const tools = [createTool({ qualifiedName: "TestKit.Tool1" })]; + const previousResult = await mergeToolkit( + "TestKit", + tools, + null, + null, + createStubGenerator() + ); + + const result = await mergeToolkit("TestKit", tools, null, null, undefined, { + previousToolkit: previousResult.toolkit, + }); + + expect(result.toolkit.tools[0]?.codeExample).toEqual( + previousResult.toolkit.tools[0]?.codeExample + ); + expect(result.toolkit.tools[0]?.secretsInfo).toEqual( + previousResult.toolkit.tools[0]?.secretsInfo + ); + }); }); // ============================================================================ @@ -541,11 +624,118 @@ describe("DataMerger", () => { expect(result.toolkit.id).toBe("Github"); expect(result.toolkit.label).toBe("GitHub"); + expect(result.toolkit.description).toBe("Toolkit description"); expect(result.toolkit.tools).toHaveLength(2); expect(result.toolkit.auth?.allScopes).toContain("repo"); expect(result.toolkit.auth?.allScopes).toContain("public_repo"); }); + it("adds a summary when a summary generator is provided", async () => { + const toolkitDataSource = createCombinedToolkitDataSource({ + toolSource: new InMemoryToolDataSource([githubTool1]), + metadataSource: new InMemoryMetadataSource([githubMetadata]), + }); + const merger = new DataMerger({ + toolkitDataSource, + customSectionsSource: new EmptyCustomSectionsSource(), + toolExampleGenerator: createStubGenerator(), + toolkitSummaryGenerator: createStubSummaryGenerator("Toolkit summary"), + }); + + const result = await merger.mergeToolkit("Github"); + + expect(result.toolkit.summary).toBe("Toolkit summary (Github)"); + }); + + it("reuses the previous summary when toolkit input is unchanged", async () => { + const toolkitDataSource = createCombinedToolkitDataSource({ + toolSource: new InMemoryToolDataSource([githubTool1]), + metadataSource: new InMemoryMetadataSource([githubMetadata]), + }); + const previousResult = await mergeToolkit( + "Github", + [githubTool1], + githubMetadata, + null, + createStubGenerator() + ); + previousResult.toolkit.summary = "Cached summary"; + + const countingSummary = createCountingSummaryGenerator(); + const merger = new DataMerger({ + toolkitDataSource, + customSectionsSource: new EmptyCustomSectionsSource(), + toolExampleGenerator: createStubGenerator(), + toolkitSummaryGenerator: countingSummary.generator, + previousToolkits: new Map([["github", previousResult.toolkit]]), + }); + + const result = await merger.mergeToolkit("Github"); + + expect(countingSummary.getCalls()).toBe(0); + expect(result.toolkit.summary).toBe("Cached summary"); + }); + + it("reuses previous examples when the tool is unchanged", async () => { + const toolkitDataSource = createCombinedToolkitDataSource({ + toolSource: new InMemoryToolDataSource([githubTool1]), + metadataSource: new InMemoryMetadataSource([githubMetadata]), + }); + const previousResult = await mergeToolkit( + "Github", + [githubTool1], + githubMetadata, + null, + createStubGenerator() + ); + const counting = createCountingGenerator(); + const merger = new DataMerger({ + toolkitDataSource, + customSectionsSource: new EmptyCustomSectionsSource(), + toolExampleGenerator: counting.generator, + previousToolkits: new Map([["github", previousResult.toolkit]]), + }); + + const result = await merger.mergeToolkit("Github"); + + expect(counting.getCalls()).toBe(0); + expect(result.toolkit.tools[0]?.codeExample).toEqual( + previousResult.toolkit.tools[0]?.codeExample + ); + }); + + it("calls the generator when the tool definition changes", async () => { + const updatedTool = createTool({ + name: "CreateIssue", + qualifiedName: "Github.CreateIssue", + fullyQualifiedName: "Github.CreateIssue@1.0.0", + description: "Updated description", + auth: { providerId: "github", providerType: "oauth2", scopes: ["repo"] }, + }); + const toolkitDataSource = createCombinedToolkitDataSource({ + toolSource: new InMemoryToolDataSource([updatedTool]), + metadataSource: new InMemoryMetadataSource([githubMetadata]), + }); + const previousResult = await mergeToolkit( + "Github", + [githubTool1], + githubMetadata, + null, + createStubGenerator() + ); + const counting = createCountingGenerator(); + const merger = new DataMerger({ + toolkitDataSource, + customSectionsSource: new EmptyCustomSectionsSource(), + toolExampleGenerator: counting.generator, + previousToolkits: new Map([["github", previousResult.toolkit]]), + }); + + await merger.mergeToolkit("Github"); + + expect(counting.getCalls()).toBe(1); + }); + it("should merge data using unified toolkit data source", async () => { const toolkitDataSource = createCombinedToolkitDataSource({ toolSource: new InMemoryToolDataSource([ @@ -657,6 +847,7 @@ describe("DataMerger", () => { const merger = new DataMerger({ toolkitDataSource, customSectionsSource: new EmptyCustomSectionsSource(), + toolExampleGenerator: createStubGenerator(), }); const results = await merger.mergeAllToolkits(); diff --git a/toolkit-docs-generator/tests/sources/engine-api.test.ts b/toolkit-docs-generator/tests/sources/engine-api.test.ts new file mode 100644 index 000000000..f06a27022 --- /dev/null +++ b/toolkit-docs-generator/tests/sources/engine-api.test.ts @@ -0,0 +1,253 @@ +import { describe, expect, it } from "vitest"; + +import { EngineApiSource } from "../../src/sources/engine-api.js"; + +type ToolMetadataItem = { + fully_qualified_name: string; + qualified_name: string; + name: string; + description: string | null; + toolkit: { + name: string; + version: string; + description: string | null; + }; + input: { + parameters: Array<{ + name: string; + required: boolean; + description: string | null; + value_schema: { + val_type: string; + inner_val_type: string | null; + enum: string[] | null; + }; + inferrable: boolean; + }>; + }; + output?: { + description: string | null; + value_schema: { + val_type: string; + inner_val_type: string | null; + enum: string[] | null; + } | null; + } | null; + requirements?: { + authorization: { + id?: string | null; + provider_id: string | null; + provider_type: string | null; + scopes: string[]; + } | null; + secrets: Array<{ key: string }> | null; + } | null; +}; + +const createItems = (): ToolMetadataItem[] => [ + { + fully_qualified_name: "Github.CreateIssue@1.0.0", + qualified_name: "Github.CreateIssue", + name: "CreateIssue", + description: "Create issue", + toolkit: { + name: "Github", + version: "1.0.0", + description: "GitHub toolkit", + }, + input: { parameters: [] }, + output: null, + requirements: { + authorization: { + provider_id: "github", + provider_type: "oauth2", + scopes: ["repo"], + }, + secrets: [{ key: "GITHUB_API_KEY" }], + }, + }, + { + fully_qualified_name: "Slack.SendMessage@1.2.0", + qualified_name: "Slack.SendMessage", + name: "SendMessage", + description: "Send message", + toolkit: { + name: "Slack", + version: "1.2.0", + description: "Slack toolkit", + }, + input: { parameters: [] }, + output: { + description: "Confirmation", + value_schema: null, + }, + requirements: { + authorization: { + provider_id: null, + provider_type: "oauth2", + scopes: ["chat:write"], + }, + secrets: null, + }, + }, +]; + +const createFetchStub = (items: ToolMetadataItem[], status = 200) => { + return async (input: RequestInfo | URL) => { + if (status !== 200) { + return new Response("error", { status }); + } + + const url = new URL(input.toString()); + const limit = Number(url.searchParams.get("limit") ?? items.length); + const offset = Number(url.searchParams.get("offset") ?? 0); + const toolkit = url.searchParams.get("toolkit"); + const providerId = url.searchParams.get("provider_id"); + const version = url.searchParams.get("version"); + + let filtered = items; + if (toolkit) { + filtered = filtered.filter((item) => item.toolkit.name === toolkit); + } + if (providerId) { + filtered = filtered.filter( + (item) => item.requirements?.authorization?.provider_id === providerId + ); + } + if (version) { + filtered = filtered.filter((item) => + item.fully_qualified_name.endsWith(`@${version}`) + ); + } + + const pageItems = filtered.slice(offset, offset + limit); + + return new Response( + JSON.stringify({ + items: pageItems, + total_count: filtered.length, + }), + { + status: 200, + headers: { "Content-Type": "application/json" }, + } + ); + }; +}; + +const createErrorFetchStub = (status: number, payload: unknown) => { + return async () => + new Response(JSON.stringify(payload), { + status, + headers: { "Content-Type": "application/json" }, + }); +}; + +const createInspectFetchStub = ( + inspect: (params: URLSearchParams) => void +) => { + return async (input: RequestInfo | URL) => { + const url = new URL(input.toString()); + inspect(url.searchParams); + return new Response( + JSON.stringify({ + items: [], + total_count: 0, + }), + { + status: 200, + headers: { "Content-Type": "application/json" }, + } + ); + }; +}; + +describe("EngineApiSource", () => { + it("fetches and transforms tool metadata with pagination", async () => { + const items = createItems(); + const source = new EngineApiSource({ + baseUrl: "https://api.arcade.dev", + apiKey: "test", + pageSize: 1, + fetchFn: createFetchStub(items), + }); + + const tools = await source.fetchAllTools(); + + expect(tools).toHaveLength(2); + expect(tools[0]?.toolkitDescription).toBe("GitHub toolkit"); + expect(tools[0]?.secrets).toEqual(["GITHUB_API_KEY"]); + expect(tools[1]?.output?.type).toBe("unknown"); + expect(tools[1]?.auth?.providerId).toBeNull(); + }); + + it("filters tools by toolkit and provider", async () => { + const items = createItems(); + const source = new EngineApiSource({ + baseUrl: "https://api.arcade.dev", + apiKey: "test", + fetchFn: createFetchStub(items), + }); + + const tools = await source.fetchAllTools({ + toolkitId: "Github", + providerId: "github", + }); + + expect(tools).toHaveLength(1); + expect(tools[0]?.qualifiedName).toBe("Github.CreateIssue"); + }); + + it("returns false when the endpoint is not available", async () => { + const source = new EngineApiSource({ + baseUrl: "https://api.arcade.dev", + apiKey: "test", + fetchFn: createFetchStub(createItems(), 500), + }); + + const available = await source.isAvailable(); + + expect(available).toBe(false); + }); + + it("includes error details from API responses", async () => { + const source = new EngineApiSource({ + baseUrl: "https://api.arcade.dev", + apiKey: "test", + fetchFn: createErrorFetchStub(400, { + name: "BadRequest", + message: "Invalid query", + }), + }); + + await expect(source.fetchAllTools()).rejects.toThrow( + "BadRequest: Invalid query" + ); + }); + + it("clamps page size to the maximum limit", async () => { + const source = new EngineApiSource({ + baseUrl: "https://api.arcade.dev", + apiKey: "test", + pageSize: 500, + fetchFn: createInspectFetchStub((params) => { + expect(params.get("limit")).toBe("100"); + }), + }); + + await source.fetchAllTools(); + }); + + it("forces include_all_versions when filtering by version", async () => { + const source = new EngineApiSource({ + baseUrl: "https://api.arcade.dev", + apiKey: "test", + fetchFn: createInspectFetchStub((params) => { + expect(params.get("include_all_versions")).toBe("true"); + expect(params.get("version")).toBe("0.1.3"); + }), + }); + + await source.fetchAllTools({ version: "0.1.3" }); + }); +});