Next.js

npm install spidra-nextjs spidra

SPIDRA_API_KEY=spd_live_your_api_key
SPIDRA_WEBHOOK_SECRET=whsec_your_signing_secret

# Optional overrides
SPIDRA_BASE_URL=https://api.spidra.io/api
SPIDRA_MAX_RETRIES=3
SPIDRA_BACKOFF_FACTOR=1.0

// app/products/page.tsx
import { getSpidraClient } from "spidra-nextjs";

export default async function ProductsPage() {
  const spidra = getSpidraClient();

  const result = await spidra.scrape({
    urls: [{ url: "https://books.toscrape.com" }],
    prompt: "Extract the top 5 books with title, price, and rating",
    output: "json",
  });

  const books = result.data?.[0]?.content?.books ?? [];

  return (
    <main className="p-8 max-w-4xl mx-auto">
      <h1 className="text-2xl font-bold mb-4">Book Listings</h1>
      <div className="grid gap-4">
        {books.map((book: any, idx: number) => (
          <div key={idx} className="border p-4 rounded shadow-sm">
            <h2 className="font-semibold text-lg">{book.title}</h2>
            <p className="text-emerald-600 font-medium">{book.price}</p>
          </div>
        ))}
      </div>
    </main>
  );
}

// app/actions/scrape.ts
"use server";

import { createSpidraAction } from "spidra-nextjs";

export const scrapeUrlAction = createSpidraAction(async (client, targetUrl: string) => {
  return await client.scrape({
    urls: [{ url: targetUrl }],
    prompt: "Extract page headline and a 3-bullet summary",
    output: "json",
  });
});

// app/components/ScraperForm.tsx
"use client";

import { useState } from "react";
import { scrapeUrlAction } from "@/app/actions/scrape";

export function ScraperForm() {
  const [url, setUrl] = useState("");
  const [loading, setLoading] = useState(false);
  const [data, setData] = useState<any>(null);

  async function handleScrape(e: React.FormEvent) {
    e.preventDefault();
    setLoading(true);
    const res = await scrapeUrlAction(url);
    setLoading(false);
    if (res.success) {
      setData(res.data);
    } else {
      alert(`Error: ${res.error}`);
    }
  }

  return (
    <form onSubmit={handleScrape} className="space-y-4">
      <input
        type="url"
        value={url}
        onChange={(e) => setUrl(e.target.value)}
        placeholder="https://example.com"
        className="w-full border p-2 rounded"
        required
      />
      <button type="submit" disabled={loading} className="bg-blue-600 text-white px-4 py-2 rounded">
        {loading ? "Extracting..." : "Extract Data"}
      </button>
      {data && <pre className="bg-gray-100 p-4 rounded mt-4">{JSON.stringify(data, null, 2)}</pre>}
    </form>
  );
}

// app/api/spidra/[...action]/route.ts
import { createSpidraRouteHandler } from "spidra-nextjs";

export const { GET, POST } = createSpidraRouteHandler();

// app/api/webhooks/spidra/route.ts
import { createSpidraWebhookHandler } from "spidra-nextjs";

export const POST = createSpidraWebhookHandler({
  onEvent: async (event) => {
    if (event.event === "crawl.completed") {
      console.log(`Crawl job ${event.jobId} completed!`);
      // Update database or trigger revalidation
    } else if (event.event === "crawl.page") {
      console.log("New page extracted:", event.page?.url);
    }
  },
});

import { getSpidraClient } from "spidra-nextjs";

const spidra = getSpidraClient();

const result = await spidra.scrape({
  urls: [
    {
      url: "https://quotes.toscrape.com/login",
      actions: [
        { type: "type", selector: "#username", value: "admin" },
        { type: "type", selector: "#password", value: "admin" },
        { type: "click", selector: "input[type='submit']" },
        { type: "waitFor", selector: "a[href='/logout']" },
      ],
    },
  ],
  prompt: "Extract the top 5 quotes with author",
  output: "json",
  useProxy: true,
  proxyCountry: "us",
});

const crawlJob = await spidra.crawl({
  baseUrl: "https://docs.example.com",
  crawlInstruction: "Follow documentation guides, skip release notes and changelogs",
  transformInstruction: "Extract the page title and key code snippets",
  maxPages: 30,
  includePaths: ["/guides/*", "/tutorials/*"],
});

const results = await spidra.search({
  query: "Next.js App Router performance optimizations",
  sources: ["web", "news"],
  limit: 10,
  timeRange: "week",
});

const batch = await spidra.startBatchScrape({
  urls: [
    "https://example.com/page-1",
    "https://example.com/page-2",
    "https://example.com/page-3",
  ],
  prompt: "Extract title and meta description",
});

const watcher = spidra.watchBatch(batch.batchId);
watcher.on("item", (item) => {
  console.log(`Finished ${item.url}:`, item.status);
});

const map = await spidra.map({
  url: "https://example.com",
  limit: 100,
});

import { readFile } from "fs/promises";

const pdfBytes = await readFile("./contract.pdf");

const parsed = await spidra.parse({
  file: pdfBytes,
  fileName: "contract.pdf",
}, {
  prompt: "Extract effective date, parties, and total agreement amount",
});

// Run Google Maps leads template
const run = await spidra.runTemplate("google-maps-leads", {
  query: "plumbers",
  areas: ["Austin, TX"],
});

// Setup automated change monitor
const monitor = await spidra.createMonitor({
  name: "Pricing Watcher",
  sources: [{ kind: "urls", urls: ["https://example.com/pricing"] }],
  schedule: { every: "1d", at: "09:00" },
  alertOn: "page",
  goal: "Notify if pricing amounts change",
  notify: {
    webhook: { url: "https://my-app.com/api/webhooks/spidra" },
  },
});

import {
  SpidraAuthenticationError,
  SpidraRateLimitError,
  SpidraPaymentRequiredError,
  SpidraError,
} from "spidra-nextjs";

try {
  await spidra.scrape({ urls: [{ url: "https://example.com" }] });
} catch (err) {
  if (err instanceof SpidraAuthenticationError) {
    console.error("Invalid API Key");
  } else if (err instanceof SpidraPaymentRequiredError) {
    console.error("Credits exhausted");
  } else if (err instanceof SpidraRateLimitError) {
    console.warn(`Rate limited. Retry after ${err.retryAfter}s`);
  } else if (err instanceof SpidraError) {
    console.error(`Spidra error (${err.status}): ${err.message}`);
  }
}