Connecting Live Data — APIs, Scraping & Databases

// Reusable API client with timeout, retries, and error handling

export async function fetchAPI(url, options = {}) {
  const {
    headers = {},
    timeout = 10000,       // 10s default timeout
    retries = 2,            // Retry twice on failure
    retryDelay = 1000,      // 1s between retries
  } = options;

  for (let attempt = 0; attempt <= retries; attempt++) {
    try {
      const controller = new AbortController();
      const timer = setTimeout(() => controller.abort(), timeout);

      const res = await fetch(url, {
        headers: { "Accept": "application/json", ...headers },
        signal: controller.signal
      });

      clearTimeout(timer);

      if (res.status === 429) {
        // Rate limited — wait and retry
        const wait = parseInt(res.headers.get("Retry-After") || "5") * 1000;
        await new Promise(r => setTimeout(r, wait));
        continue;
      }

      if (!res.ok) {
        throw new Error(`HTTP ${res.status}: ${res.statusText}`);
      }

      return await res.json();
    } catch (err) {
      if (attempt === retries) throw err;
      await new Promise(r => setTimeout(r, retryDelay * (attempt + 1)));
    }
  }
}

// Pattern 1: API Key in query string
const url = `https://api.example.com/data?apikey=${process.env.API_KEY}`;

// Pattern 2: API Key in header
await fetchAPI(url, {
  headers: { "X-API-Key": process.env.API_KEY }
});

// Pattern 3: Bearer token (OAuth)
await fetchAPI(url, {
  headers: { "Authorization": `Bearer ${process.env.TOKEN}` }
});

// Pattern 4: Basic auth
const creds = Buffer.from(`${user}:${pass}`).toString("base64");
await fetchAPI(url, {
  headers: { "Authorization": `Basic ${creds}` }
});

import { z } from "zod";
import { fetchAPI } from "../lib/api-client.js";

export function registerStockTools(server) {
  const RAPID_KEY = process.env.RAPIDAPI_KEY;

  server.tool(
    "get_stock_quote",
    "Get real-time stock quote with price, change, volume, and key metrics",
    {
      symbol: z.string()
        .toUpperCase()
        .describe("Stock ticker (e.g., AAPL, MSFT, TSLA)"),
      include_extended: z.boolean()
        .optional().default(false)
        .describe("Include pre/post market data")
    },
    async ({ symbol, include_extended }) => {
      const data = await fetchAPI(
        `https://yahoo-finance15.p.rapidapi.com/api/v1/markets/quote?ticker=${symbol}`,
        { headers: { "x-rapidapi-key": RAPID_KEY } }
      );

      const q = data.body;
      const result = {
        symbol: q.symbol,
        name: q.shortName,
        price: q.regularMarketPrice,
        change: q.regularMarketChange,
        changePercent: `${q.regularMarketChangePercent.toFixed(2)}%`,
        volume: q.regularMarketVolume,
        marketCap: q.marketCap,
        peRatio: q.trailingPE,
        fiftyTwoWeekHigh: q.fiftyTwoWeekHigh,
        fiftyTwoWeekLow: q.fiftyTwoWeekLow
      };

      if (include_extended && q.preMarketPrice) {
        result.preMarket = q.preMarketPrice;
        result.afterHours = q.postMarketPrice;
      }

      return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
    }
  );
}

export function registerCryptoTools(server) {
  server.tool(
    "get_crypto_price",
    "Get cryptocurrency price, market cap, and 24h stats",
    {
      coin: z.string().toLowerCase()
        .describe("Coin ID (e.g., bitcoin, ethereum, solana)"),
      currency: z.enum(["usd", "eur", "gbp"])
        .optional().default("usd")
        .describe("Fiat currency for prices")
    },
    async ({ coin, currency }) => {
      const data = await fetchAPI(
        `https://api.coingecko.com/api/v3/coins/${coin}`
        + `?localization=false&tickers=false&community_data=false`
      );

      return { content: [{ type: "text", text: JSON.stringify({
        name: data.name,
        symbol: data.symbol.toUpperCase(),
        price: data.market_data.current_price[currency],
        change_24h: `${data.market_data.price_change_percentage_24h.toFixed(2)}%`,
        market_cap: data.market_data.market_cap[currency],
        volume_24h: data.market_data.total_volume[currency],
        ath: data.market_data.ath[currency],
        ath_change: `${data.market_data.ath_change_percentage[currency].toFixed(1)}%`
      }, null, 2) }] };
    }
  );
}

npm install cheerio

import * as cheerio from "cheerio";

export function registerScraperTools(server) {
  server.tool(
    "scrape_headlines",
    "Scrape top headlines from a financial news site",
    {
      source: z.enum(["reuters", "bloomberg", "cnbc"])
        .describe("News source to scrape"),
      limit: z.number().min(1).max(20).optional().default(10)
        .describe("Number of headlines")
    },
    async ({ source, limit }) => {
      const urls = {
        reuters: "https://www.reuters.com/markets/",
        bloomberg: "https://www.bloomberg.com/markets",
        cnbc: "https://www.cnbc.com/world/?region=world"
      };

      const res = await fetch(urls[source], {
        headers: {
          "User-Agent": "Mozilla/5.0 (compatible; MCPBot/1.0)"
        }
      });
      const html = await res.text();
      const $ = cheerio.load(html);

      const headlines = [];
      $("h3, [data-testid='Heading'] a, .Card-title").each((i, el) => {
        if (headlines.length >= limit) return false;
        const text = $(el).text().trim();
        const link = $(el).closest("a").attr("href") || $(el).find("a").attr("href");
        if (text.length > 10) {
          headlines.push({ title: text, url: link || "" });
        }
      });

      return { content: [{ type: "text",
        text: JSON.stringify({ source, count: headlines.length, headlines }, null, 2)
      }] };
    }
  );
}

import Database from "better-sqlite3";

export function registerSQLiteTools(server) {
  const dbPath = process.env.SQLITE_DB || "./data.db";
  const db = new Database(dbPath, { readonly: true });

  server.tool(
    "query_database",
    "Execute a read-only SQL query against the SQLite database",
    {
      sql: z.string().describe("SQL SELECT query (read-only)"),
      limit: z.number().optional().default(50)
        .describe("Max rows to return (default 50)")
    },
    async ({ sql, limit }) => {
      // Safety: only allow SELECT statements
      const normalized = sql.trim().toUpperCase();
      if (!normalized.startsWith("SELECT") && !normalized.startsWith("WITH")) {
        return { content: [{ type: "text",
          text: "Error: Only SELECT and WITH (CTE) queries are allowed."
        }], isError: true };
      }

      try {
        const safeSql = sql.replace(/;\s*$/, "") + ` LIMIT ${limit}`;
        const rows = db.prepare(safeSql).all();
        return { content: [{ type: "text",
          text: JSON.stringify({ rowCount: rows.length, rows }, null, 2)
        }] };
      } catch (err) {
        return { content: [{ type: "text",
          text: `SQL Error: ${err.message}`
        }], isError: true };
      }
    }
  );

  // Resource: Database schema
  server.resource("schema", "db://schema",
    { description: "Database table schemas", mimeType: "application/json" },
    async () => {
      const tables = db.prepare(
        "SELECT name, sql FROM sqlite_master WHERE type='table'"
      ).all();
      return { contents: [{ uri: "db://schema",
        mimeType: "application/json",
        text: JSON.stringify(tables, null, 2)
      }] };
    }
  );
}

import duckdb from "duckdb";

export function registerDuckDBTools(server) {
  const db = new duckdb.Database(":memory:");

  server.tool(
    "analyze_csv",
    "Run SQL analytics on a CSV/Parquet file using DuckDB",
    {
      file: z.string().describe("Path to CSV or Parquet file"),
      query: z.string().describe("SQL query (use 'data' as table name)")
    },
    async ({ file, query }) => {
      return new Promise((resolve, reject) => {
        const conn = db.connect();
        // Auto-detect file format
        const viewSQL = file.endsWith(".parquet")
          ? `CREATE OR REPLACE VIEW data AS SELECT * FROM '${file}'`
          : `CREATE OR REPLACE VIEW data AS SELECT * FROM read_csv_auto('${file}')`;

        conn.run(viewSQL, () => {
          conn.all(query, (err, rows) => {
            if (err) resolve({ content: [{ type:"text", text:`Error: ${err.message}` }], isError:true });
            else resolve({ content: [{ type:"text", text: JSON.stringify(rows, null, 2) }] });
          });
        });
      });
    }
  );
}

import { readFile, readdir, stat } from "node:fs/promises";
import { join, extname } from "node:path";

export function registerFileTools(server) {
  const ROOT = process.env.DATA_DIR || "./data";

  server.tool(
    "read_file",
    "Read a JSON, CSV, or text file from the data directory",
    {
      path: z.string().describe("Relative path within data directory")
    },
    async ({ path }) => {
      // Security: prevent directory traversal
      if (path.includes("..")) {
        return { content: [{ type:"text", text:"Error: Path traversal not allowed" }], isError:true };
      }

      const fullPath = join(ROOT, path);
      const content = await readFile(fullPath, "utf-8");

      // Parse JSON automatically
      if (extname(path) === ".json") {
        const parsed = JSON.parse(content);
        return { content: [{ type:"text",
          text: JSON.stringify(parsed, null, 2)
        }] };
      }

      return { content: [{ type:"text", text: content }] };
    }
  );

  // Resource: List available files
  server.resource("files", "files://list",
    { description: "List available data files" },
    async () => {
      const entries = await readdir(ROOT, { recursive: true });
      return { contents: [{ uri:"files://list", mimeType:"application/json",
        text: JSON.stringify(entries)
      }] };
    }
  );
}

export class TTLCache {
  constructor(defaultTTL = 60000) { // 60s default
    this.cache = new Map();
    this.defaultTTL = defaultTTL;
  }

  get(key) {
    const entry = this.cache.get(key);
    if (!entry) return null;
    if (Date.now() > entry.expires) {
      this.cache.delete(key);
      return null;
    }
    return entry.value;
  }

  set(key, value, ttl = this.defaultTTL) {
    this.cache.set(key, {
      value,
      expires: Date.now() + ttl
    });
  }

  wrap(key, fn, ttl) {
    return async (...args) => {
      const cached = this.get(key);
      if (cached) return cached;
      const result = await fn(...args);
      this.set(key, result, ttl);
      return result;
    };
  }
}

const cache = new TTLCache(300000); // 5 min default

server.tool("get_stock_quote", "...", { symbol: z.string() },
  async ({ symbol }) => {
    const cacheKey = `quote:${symbol}`;
    let data = cache.get(cacheKey);

    if (!data) {
      data = await fetchAPI(`.../${symbol}`);
      cache.set(cacheKey, data, 60000); // Cache 60s for real-time data
    }

    return { content: [{ type:"text", text: JSON.stringify(data) }] };
  }
);

// Wrapper that catches all errors and returns MCP-formatted error responses
export function safeTool(handler) {
  return async (params) => {
    try {
      return await handler(params);
    } catch (error) {
      const message = error.name === "AbortError"
        ? "Request timed out. The API may be slow or unreachable."
        : error.message.includes("429")
        ? "Rate limited. Please wait a moment and try again."
        : error.message.includes("ENOTFOUND")
        ? "Cannot reach the API. Check your network connection."
        : `Error: ${error.message}`;

      return {
        content: [{ type: "text", text: message }],
        isError: true
      };
    }
  };
}

// Usage
server.tool("get_quote", "...", { symbol: z.string() },
  safeTool(async ({ symbol }) => {
    // Your logic here — any thrown error is caught and formatted
  })
);