diff --git a/src/index.ts b/src/index.ts index 1b297a8..de52de6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,12 @@ * - Added search-places tool to perform Google Places searches * - Added search-news tool to perform Google News searches * - Added get-google-reviews tool to fetch Google reviews for a place + * - Added scrape tool to extract content from a URL + * - Added doc-to-text tool to convert documents to plain text + * - Added extract-document tool to extract structured data from documents + * - Added crawl tool to crawl a website and extract content + * - Added add-to-knowledge-base tool to add content to a knowledge base + * - Added search-knowledge-base tool to search a knowledge base */ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; @@ -542,6 +548,391 @@ server.tool( } ); +// Tool to scrape content from a URL +server.tool( + "scrape", + { + url: z.string().url(), + format: z.enum(["markdown", "html", "screenshot"]).optional(), + cleaned: z.boolean().optional(), + renderJs: z.boolean().optional(), + }, + async ({ url, format, cleaned, renderJs }) => { + // Get API key from environment variable + const apiKey = process.env.DUMPLING_API_KEY; + if (!apiKey) { + throw new Error("DUMPLING_API_KEY environment variable not set"); + } + + try { + const response = await fetch(`${NWS_API_BASE}/api/v1/scrape`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + url, + format, + cleaned, + renderJs, + }), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `Failed to scrape content: ${response.status} ${errorText}` + ); + } + + const data = await response.json(); + + return { + content: [ + { + type: "text", + text: JSON.stringify( + { + title: data.title, + content: data.content, + metadata: data.metadata, + url: data.url, + format: data.format, + cleaned: data.cleaned, + }, + null, + 2 + ), + }, + ], + }; + } catch (error) { + console.error("Error scraping content:", error); + throw error; + } + } +); + +// Tool to convert documents to plain text +server.tool( + "doc-to-text", + { + url: z.string().url().optional(), + file: z.string().optional(), // Base64 encoded file content + pages: z.string().optional(), + }, + async ({ url, file, pages }) => { + // Ensure either url or file is provided + if (!url && !file) { + throw new Error("Either url or file is required"); + } + + // Get API key from environment variable + const apiKey = process.env.DUMPLING_API_KEY; + if (!apiKey) { + throw new Error("DUMPLING_API_KEY environment variable not set"); + } + + try { + const response = await fetch(`${NWS_API_BASE}/api/v1/doc-to-text`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + url, + file, + pages, + }), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `Failed to convert document to text: ${response.status} ${errorText}` + ); + } + + const data = await response.json(); + + return { + content: [ + { + type: "text", + text: data.text, + }, + ], + }; + } catch (error) { + console.error("Error converting document to text:", error); + throw error; + } + } +); + +// Tool to extract structured data from documents +server.tool( + "extract-document", + { + prompt: z.string(), + files: z.array( + z.object({ + url: z.string().url().optional(), + content: z.string().optional(), // Base64 encoded file content + filename: z.string().optional(), + }) + ), + jsonMode: z.boolean().optional(), + }, + async ({ prompt, files, jsonMode }) => { + // Get API key from environment variable + const apiKey = process.env.DUMPLING_API_KEY; + if (!apiKey) { + throw new Error("DUMPLING_API_KEY environment variable not set"); + } + + try { + const response = await fetch(`${NWS_API_BASE}/api/v1/extract-document`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + prompt, + files, + jsonMode, + }), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `Failed to extract document data: ${response.status} ${errorText}` + ); + } + + const data = await response.json(); + + return { + content: [ + { + type: "text", + text: JSON.stringify( + { + output: data.output, + creditUsage: data.creditUsage, + }, + null, + 2 + ), + }, + ], + }; + } catch (error) { + console.error("Error extracting document data:", error); + throw error; + } + } +); + +// Tool to crawl a website and extract content +server.tool( + "crawl", + { + url: z.string().url(), + limit: z.number().optional(), + depth: z.number().optional(), + format: z.enum(["markdown", "html", "screenshot"]).optional(), + cleaned: z.boolean().optional(), + renderJs: z.boolean().optional(), + }, + async ({ url, limit, depth, format, cleaned, renderJs }) => { + // Get API key from environment variable + const apiKey = process.env.DUMPLING_API_KEY; + if (!apiKey) { + throw new Error("DUMPLING_API_KEY environment variable not set"); + } + + try { + const response = await fetch(`${NWS_API_BASE}/api/v1/crawl`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + url, + limit, + depth, + format, + cleaned, + renderJs, + }), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `Failed to crawl website: ${response.status} ${errorText}` + ); + } + + const data = await response.json(); + + return { + content: [ + { + type: "text", + text: JSON.stringify( + { + pages: data.pages, + creditUsage: data.creditUsage, + }, + null, + 2 + ), + }, + ], + }; + } catch (error) { + console.error("Error crawling website:", error); + throw error; + } + } +); + +// Tool to add content to a knowledge base +server.tool( + "add-to-knowledge-base", + { + knowledgeBaseId: z.string(), + content: z.string(), + title: z.string().optional(), + url: z.string().url().optional(), + metadata: z.record(z.any()).optional(), + }, + async ({ knowledgeBaseId, content, title, url, metadata }) => { + // Get API key from environment variable + const apiKey = process.env.DUMPLING_API_KEY; + if (!apiKey) { + throw new Error("DUMPLING_API_KEY environment variable not set"); + } + + try { + const response = await fetch( + `${NWS_API_BASE}/api/v1/add-to-knowledge-base`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + knowledgeBaseId, + content, + title, + url, + metadata, + }), + } + ); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `Failed to add content to knowledge base: ${response.status} ${errorText}` + ); + } + + const data = await response.json(); + + return { + content: [ + { + type: "text", + text: JSON.stringify( + { + resource: data.resource, + creditUsage: data.creditUsage, + }, + null, + 2 + ), + }, + ], + }; + } catch (error) { + console.error("Error adding content to knowledge base:", error); + throw error; + } + } +); + +// Tool to search a knowledge base +server.tool( + "search-knowledge-base", + { + knowledgeBaseId: z.string(), + query: z.string(), + resultCount: z.number().optional(), + }, + async ({ knowledgeBaseId, query, resultCount }) => { + // Get API key from environment variable + const apiKey = process.env.DUMPLING_API_KEY; + if (!apiKey) { + throw new Error("DUMPLING_API_KEY environment variable not set"); + } + + try { + const response = await fetch( + `${NWS_API_BASE}/api/v1/search-knowledge-base`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + knowledgeBaseId, + query, + resultCount, + }), + } + ); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error( + `Failed to search knowledge base: ${response.status} ${errorText}` + ); + } + + const data = await response.json(); + + return { + content: [ + { + type: "text", + text: JSON.stringify( + { + results: data.results, + }, + null, + 2 + ), + }, + ], + }; + } catch (error) { + console.error("Error searching knowledge base:", error); + throw error; + } + } +); + async function main() { const transport = new StdioServerTransport(); await server.connect(transport);