Express

npm install spidra-express spidra

SPIDRA_API_KEY=spd_live_your_api_key
SPIDRA_WEBHOOK_SECRET=whsec_your_signing_secret

# Optional overrides
SPIDRA_BASE_URL=https://api.spidra.io/api
SPIDRA_MAX_RETRIES=3
SPIDRA_BACKOFF_FACTOR=1.0

import express from "express";
import { spidraMiddleware, spidraErrorHandler } from "spidra-express";

const app = express();
app.use(express.json());

// Attach req.spidra
app.use(spidraMiddleware());

app.post("/api/extract-product", async (req, res, next) => {
  try {
    const { url } = req.body;

    const result = await req.spidra.scrape({
      urls: [{ url }],
      prompt: "Extract product title, price, and availability",
      output: "json",
    });

    res.json({ success: true, product: result.data?.[0]?.content });
  } catch (err) {
    next(err);
  }
});

// Format Spidra errors into matching HTTP status codes
app.use(spidraErrorHandler());

app.listen(3000, () => console.log("Server listening on port 3000"));

import express from "express";
import { createSpidraRouter } from "spidra-express";

const app = express();

// Exposes /api/spidra/scrape, /crawl, /search, /batch, /map, /parse, /account, /usage
app.use("/api/spidra", createSpidraRouter());

app.listen(3000);

import express from "express";
import { spidraWebhookMiddleware } from "spidra-express";

const app = express();

// Use express.raw() to capture the exact bytes for signature verification
app.post(
  "/api/webhooks/spidra",
  express.raw({ type: "application/json" }),
  spidraWebhookMiddleware({
    onEvent: async (event, req, res) => {
      console.log(`Verified webhook event: ${event.event}`);

      if (event.event === "crawl.completed") {
        console.log(`Crawl job ${event.jobId} completed!`);
      }
    },
  })
);

app.post("/scrape-quotes", async (req, res, next) => {
  try {
    const result = await req.spidra.scrape({
      urls: [
        {
          url: "https://quotes.toscrape.com/login",
          actions: [
            { type: "type", selector: "#username", value: "admin" },
            { type: "type", selector: "#password", value: "admin" },
            { type: "click", selector: "input[type='submit']" },
            { type: "waitFor", selector: "a[href='/logout']" },
          ],
        },
      ],
      prompt: "Extract top 5 quotes with author",
      output: "json",
      useProxy: true,
      proxyCountry: "us",
    });

    res.json(result);
  } catch (err) {
    next(err);
  }
});

app.post("/crawl-site", async (req, res, next) => {
  try {
    const { url } = req.body;

    const crawlJob = await req.spidra.crawl({
      baseUrl: url,
      crawlInstruction: "Follow documentation guides, skip release notes",
      transformInstruction: "Extract page title and key takeaways",
      maxPages: 25,
      includePaths: ["/guides/*"],
    });

    res.json({ pages: crawlJob.result });
  } catch (err) {
    next(err);
  }
});

app.get("/search", async (req, res, next) => {
  try {
    const query = req.query.q as string;

    const results = await req.spidra.search({
      query,
      sources: ["web", "news"],
      limit: 10,
    });

    res.json(results);
  } catch (err) {
    next(err);
  }
});

app.post("/batch", async (req, res, next) => {
  try {
    const { urls } = req.body;

    const batch = await req.spidra.batchScrape({
      urls,
      prompt: "Extract page title and meta description",
      output: "json",
    });

    res.json(batch);
  } catch (err) {
    next(err);
  }
});

app.get("/map", async (req, res, next) => {
  try {
    const url = req.query.url as string;
    const map = await req.spidra.map({ url, limit: 100 });
    res.json(map);
  } catch (err) {
    next(err);
  }
});

import { readFile } from "fs/promises";

app.post("/parse-doc", async (req, res, next) => {
  try {
    const fileBytes = await readFile("./invoice.pdf");
    const parsed = await req.spidra.parse({
      file: fileBytes,
      fileName: "invoice.pdf",
    }, {
      prompt: "Extract invoice number, total amount, and line items",
    });

    res.json({ markdown: parsed.markdown, data: parsed.json });
  } catch (err) {
    next(err);
  }
});

// Run Google Maps leads template
app.post("/leads", async (req, res, next) => {
  try {
    const run = await req.spidra.runTemplate("google-maps-leads", {
      query: "dentists",
      areas: ["Miami, FL"],
    });

    res.json(run.rows);
  } catch (err) {
    next(err);
  }
});

// Setup automated change monitor
app.post("/monitors", async (req, res, next) => {
  try {
    const monitor = await req.spidra.createMonitor({
      name: "Price Watch",
      sources: [{ kind: "urls", urls: ["https://example.com/pricing"] }],
      schedule: { every: "1d", at: "08:00" },
      alertOn: "page",
      goal: "Notify if pricing or tier names change",
      notify: { webhook: { url: "https://my-app.com/api/webhooks/spidra" } },
    });

    res.json(monitor);
  } catch (err) {
    next(err);
  }
});

import {
  SpidraAuthenticationError,
  SpidraRateLimitError,
  SpidraPaymentRequiredError,
  SpidraError,
} from "spidra-express";

app.use((err: any, req: any, res: any, next: any) => {
  if (err instanceof SpidraAuthenticationError) {
    return res.status(401).json({ error: "Invalid Spidra API Key" });
  }
  if (err instanceof SpidraPaymentRequiredError) {
    return res.status(402).json({ error: "Spidra credit balance exhausted" });
  }
  if (err instanceof SpidraRateLimitError) {
    return res.status(429).json({ error: "Rate limited", retryAfter: err.retryAfter });
  }
  if (err instanceof SpidraError) {
    return res.status(err.status || 500).json({ error: err.message, code: err.code });
  }
  next(err);
});