Node

npm install spidra

import { SpidraClient } from 'spidra';

const spidra = new SpidraClient({ apiKey: process.env.SPIDRA_API_KEY });

const result = await spidra.search({
	query: 'best espresso machine 2026',
	sources: ['web', 'news'],
});

for (const hit of result.data.web ?? []) {
	console.log(hit.title, hit.url);
}

const result = await spidra.search({
	query: 'transformer attention mechanisms',
	sources: ['research'],
});

for (const paper of result.data.research ?? []) {
	console.log(paper.title, paper.authors, paper.doi);
}

const result = await spidra.search({
	query: 'memory leak is:issue repo:vercel/next.js',
	sources: ['developer'],
});

const page1 = await spidra.search({ query: 'espresso machine reviews', limit: 10 });
const page2 = await spidra.search({
	query: 'espresso machine reviews',
	pageTokens: page1.data.nextPageTokens,
});

const result = await spidra.search({
	query: 'product launch',
	sources: ['news'],
	timeRange: 'week',
});

const result = await spidra.search({
	query: 'espresso machine reviews',
	includeDomains: ['reddit.com'],
});

const result = await spidra.search({
	query: 'posthog before_send config',
	scrapeOptions: { formats: ['markdown'], maxResults: 5 },
});

result.data.web?.[0].markdown; // the scraped page's content, right there on the result

const { jobId } = await spidra.startSearch({ query: 'electric cars' });
const status = await spidra.getSearch(jobId);

const result = await spidra.scrape({
	urls: [{ url: 'https://spidra.io/pricing' }],
	prompt: 'Extract all pricing plans with name, price, and included features',
	output: 'json',
});

console.log(result.content);
// { plans: [{ name: "Starter", price: "$9/mo", features: [...] }, ...] }

// Submit, returns immediately with a jobId
const { jobId } = await spidra.startScrape({
	urls: [{ url: 'https://spidra.io' }],
	prompt: 'Extract the main headline',
});

// Check status at any time
const status = await spidra.getScrape(jobId);

if (status.status === 'completed') {
	console.log(status.result.content);
} else if (status.status === 'failed') {
	console.error(status.error);
}

const result = await spidra.scrape({
	urls: [{ url: 'https://github.careers' }],
	prompt: 'Extract the job listing details',
	output: 'json',
	schema: {
		type: 'object',
		required: ['title', 'company', 'remote'],
		properties: {
			title: { type: 'string' },
			company: { type: 'string' },
			remote: { type: ['boolean', 'null'] },
			salary_min: { type: ['number', 'null'] },
			salary_max: { type: ['number', 'null'] },
			skills: { type: 'array', items: { type: 'string' } },
		},
	},
});

import { z } from 'zod';

const JobListing = z.object({
	title: z.string(),
	company: z.string(),
	remote: z.boolean().nullable(),
	skills: z.array(z.string()),
});

const result = await spidra.scrape({
	urls: [{ url: 'https://github.careers' }],
	prompt: 'Extract the job listing details',
	output: 'json',
	schema: JobListing,
});

result.content.title; // typed as string, no casting needed

const result = await spidra.scrape({
	urls: [{ url: 'https://www.amazon.de/gp/bestsellers' }],
	prompt: 'List the top 10 products with name and price',
	useProxy: true,
	proxyCountry: 'de',
});

const result = await spidra.scrape({
	urls: [{ url: 'https://app.spidra.io/dashboard' }],
	prompt: 'Extract the monthly revenue and active user count',
	cookies: 'session=abc123; auth_token=xyz789',
});

const result = await spidra.scrape({
	urls: [
		{
			url: 'https://spidra.io/products',
			actions: [
				{ type: 'click', selector: '#accept-cookies' },
				{ type: 'wait', duration: 1000 },
				{ type: 'scroll', to: '80%' },
			],
		},
	],
	prompt: 'Extract all product names and prices',
});

{ type: 'click', selector: "button[data-testid='submit']" }   // CSS selector
{ type: 'click', value: 'Accept all cookies button' }          // plain English
{ type: 'type',  selector: "input[name='q']", value: 'wireless headphones' }
{ type: 'wait',  duration: 2000 }
{ type: 'scroll', to: '100%' }

const result = await spidra.scrape({
	urls: [
		{
			url: 'https://books.toscrape.com',
			actions: [
				{
					type: 'forEach',
					observe: 'Find all book cards in the product grid',
					mode: 'inline',
					captureSelector: 'article.product_pod',
					maxItems: 20,
					itemPrompt:
						'Extract title, price, and star rating as JSON: {title, price, star_rating}',
				},
			],
		},
	],
	prompt: 'Return a clean JSON array of all books',
	output: 'json',
});

{
  type:            'forEach',
  observe:         'Find all book title links in the product grid',
  mode:            'navigate',
  captureSelector: 'article.product_page',
  maxItems:        10,
  waitAfterClick:  800,
  itemPrompt:      'Extract title, price, star rating, and availability as JSON',
}

{
  type:            'forEach',
  observe:         'Find all room type cards',
  mode:            'click',
  captureSelector: "[role='dialog']",
  maxItems:        8,
  waitAfterClick:  1200,
  itemPrompt:      'Extract room name, bed type, price per night, and amenities as JSON',
}

{
  type:     'forEach',
  observe:  'Find all book title links',
  mode:     'navigate',
  maxItems: 40,
  pagination: {
    nextSelector: 'li.next > a',
    maxPages:     3,   // 3 additional pages beyond the first
  },
}

{
  type:            'forEach',
  observe:         'Find all book title links',
  mode:            'navigate',
  captureSelector: 'article.product_page',
  maxItems:        5,
  waitAfterClick:  1000,
  actions: [
    { type: 'scroll', to: '50%' },
  ],
  itemPrompt: 'Extract title, price, and full description as JSON',
}

const controller = new AbortController();

const result = await spidra.scrape(params, {
	pollInterval: 3000,              // ms between status checks (default: 3000)
	timeout: 600_000,                // max wait in ms (default: none, waits until done)
	signal: controller.signal,       // stop waiting without cancelling the job
});

const batch = await spidra.batchScrape({
	urls: [
		'https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html',
		'https://books.toscrape.com/catalogue/tipping-the-velvet_999/index.html',
		'https://books.toscrape.com/catalogue/soumission_998/index.html',
	],
	prompt: 'Extract product name, price, and availability',
	output: 'json',
	useProxy: true,
});

console.log(`${batch.completedCount}/${batch.totalUrls} succeeded`);

for (const item of batch.items) {
	if (item.status === 'completed') console.log(item.url, item.result);
	if (item.status === 'failed') console.error(item.url, item.error);
}

const { batchId } = await spidra.startBatchScrape({
	urls: ['https://spidra.io/blog', 'https://spidra.io/pricing'],
	prompt: 'Extract the page title',
});

// Later...
const result = await spidra.getBatchScrape(batchId);
console.log(result.status, result.completedCount, '/', result.totalUrls);

const { batchId } = await spidra.startBatchScrape({ urls, prompt: 'Extract product data' });

const watcher = spidra.watchBatch(batchId);

watcher.on('item', (item) => {
	console.log(item.url, item.status, item.result); // fires once per finished URL
});

const final = await watcher.wait(); // resolves with the terminal batch state

const result = await spidra.getBatchScrape(batchId);

if (result.failedCount > 0) {
	const { retriedCount } = await spidra.retryBatchScrape(batchId);
	console.log(`Retrying ${retriedCount} items`);
}

const { cancelledItems, creditsRefunded } = await spidra.cancelBatchScrape(batchId);
console.log(
	`Cancelled ${cancelledItems} items, refunded ${creditsRefunded} credits`,
);

const { jobs, pagination } = await spidra.listBatchScrapes({ page: 1, limit: 20 });

for (const job of jobs) {
	console.log(job.uuid, job.status, `${job.completedCount}/${job.totalUrls}`);
}

const params = {
	template: 'https://www.indeed.com/jobs?q={title}&l={location}&fromage=7&sort=date',
	variables: {
		title: ['merchandiser', 'category manager'],
		location: ['Columbus, OH', 'Dallas, TX', 'Atlanta, GA'],
	},
	itemsPath: 'jobs', // where the list of items sits in each page's result
	dedupeBy: 'id', // the field that identifies an item
	output: 'json' as const,
	target: 500, // optional: stop once this many unique items are collected
};

const estimate = spidra.estimateBulk(params);
// { pages: 6, batches: 1, credits: 6, aiTokensExtra: false }

const { items, stats } = await spidra.bulkScrape(params);
// stats: { pages: 6, failedPages: 0, rawItems: 230, uniqueItems: 209, duplicatesRemoved: 21, creditsUsed: 6 }

const { items } = await spidra.bulkScrape({
	template: 'https://example-shop.com/search?q={keyword}',
	variables: { keyword: ['desk lamp', 'standing desk', 'monitor arm'] },
	schema: {
		type: 'object',
		properties: {
			products: {
				type: 'array',
				items: {
					type: 'object',
					properties: { sku: { type: 'string' }, name: { type: 'string' }, price: { type: 'number' } },
				},
			},
		},
	},
	itemsPath: 'products',
	dedupeBy: 'sku',
});

const job = await spidra.crawl({
	baseUrl: 'https://posthog.com/blog',
	crawlInstruction: 'Follow blog post links only, skip tag and category pages',
	transformInstruction: 'Extract the title, author, publish date, and a one-sentence summary',
	maxPages: 30,
	useProxy: true,
});

for (const page of job.result) {
	console.log(page.url, page.data);
}

const job = await spidra.crawl({
	baseUrl: 'https://docs.spidra.io',
	crawlInstruction: 'Crawl all documentation pages',
	maxPages: 50,
});

for (const page of job.result) {
	// page.data contains the raw markdown of each page
	console.log(page.url, page.data);
}

const job = await spidra.crawl({
	baseUrl: 'https://github.careers',
	crawlInstruction: 'Crawl all job listing pages',
	schema: {
		type: 'object',
		properties: {
			title:    { type: 'string' },
			location: { type: 'string' },
			salary:   { type: 'string' },
			remote:   { type: 'boolean' },
		},
	},
	maxPages: 20,
});

const job = await spidra.crawl({
	baseUrl: 'https://spidra.io',
	crawlInstruction: 'Crawl all documentation pages',
	transformInstruction: 'Extract the page title and main content',
	includePaths: ['/docs/*'],
	excludePaths: ['/docs/changelog/*', '/docs/legacy/*'],
	maxPages: 30,
});

const { jobId } = await spidra.startCrawl({
	baseUrl: 'https://docs.spidra.io',
	crawlInstruction: 'Find all documentation pages',
	transformInstruction: 'Extract the page title and main content summary',
	maxPages: 50,
});

// Poll manually
const status = await spidra.getCrawl(jobId);
// status.status: "waiting" | "active" | "running" | "completed" | "failed" | "cancelled"

const { jobId } = await spidra.startCrawl({
	baseUrl: 'https://posthog.com/blog',
	crawlInstruction: 'Follow blog post links only',
	transformInstruction: 'Extract title, author, and publish date',
	maxPages: 50,
});

const watcher = spidra.watchCrawl(jobId);

watcher.on('page', (page) => {
	console.log(page.url, page.data); // fires once per crawled page
});
watcher.on('error', (err) => console.error(err));

const final = await watcher.wait();

await spidra.cancelCrawl(jobId);

// Retrieve whatever completed before cancellation
const { pages } = await spidra.crawlPages(jobId);
for (const page of pages) {
	if (page.status === 'success') console.log(page.url, page.data);
}

const { pages } = await spidra.crawlPages(jobId);

for (const page of pages) {
	console.log(page.url, page.status);
	// page.html: signed URL for the raw HTML snapshot
	// page.markdown: signed URL for the Markdown version
}

const { jobId: newJobId } = await spidra.crawlExtract(
	sourceJobId,
	'Extract only the product SKUs and prices as a CSV',
);

const result = await spidra.getCrawl(newJobId);

// List past crawl jobs
const { jobs, total, page, totalPages } = await spidra.crawlHistory({
	page: 1,
	limit: 10,
});

// Total crawl job count for your account
const { total: totalCrawls } = await spidra.crawlStats();

const details = await spidra.crawlJobDetails(jobId);
console.log(details.status, details.pages_crawled, details.credits_used);

const retry = await spidra.retryCrawlPage(jobId, pageId);
console.log(retry.data);

const blob = await spidra.downloadCrawlResults(jobId, ['markdown', 'data']);
// write `blob` to disk, or stream it straight to a response

const { report } = await spidra.crawlReport(jobId);
console.log(report?.understoodAs, report?.navigationPages);
for (const s of report?.skipped ?? []) console.log(s.reason, s.count);

// The links for one reason, 200 at a time
const res = await spidra.crawlReport(jobId, { reason: 'not_relevant', limit: 200, offset: 0 });

import { readFile } from 'node:fs/promises';

const doc = await spidra.parse(
  { file: await readFile('report.pdf'), fileName: 'report.pdf' },
  { pages: '1-10', includePages: true, prompt: 'Invoice number, total and due date' }
);
console.log(doc.markdown, doc.json);

const paper = await spidra.parse({ url: 'https://arxiv.org/pdf/1706.03762' }, { maxPages: 3 });

const { links, total, scanned } = await spidra.map({
  url: 'https://docs.python.org/3/',
  search: 'asyncio', // optional: best matches first
  limit: 20,
});

for (const link of links) console.log(link.url, link.title);

const { logs, total } = await spidra.scrapeLogs({
	status: 'failed', // "success" | "failed"
	searchTerm: 'amazon.com',
	channel: 'api', // "api" | "playground"
	dateStart: '2024-01-01',
	dateEnd: '2024-12-31',
	page: 1,
	limit: 20,
});

for (const log of logs) {
	console.log(log.urls[0]?.url, log.status, log.credits_used);
}

const log = await spidra.getScrapeLog(logUuid);
console.log(log.result_data); // full AI output for that job

// Range options: "7d" | "30d" | "weekly"
const rows = await spidra.usage('30d');

for (const row of rows) {
	console.log(row.date, row.requests, row.credits);
}

const spidra = new SpidraClient({
	apiKey: process.env.SPIDRA_API_KEY,
	maxRetries: 3,      // retry attempts for transient failures (default: 3, 0 disables)
	backoffFactor: 500, // base ms, delay is backoffFactor * 2^attempt (default: 500)
});

import {
	SpidraClient,
	SpidraError,
	SpidraAuthenticationError,
	SpidraInsufficientCreditsError,
	SpidraValidationError,
	SpidraRateLimitError,
	SpidraServerError,
	SpidraJobError,
	SpidraTimeoutError,
} from 'spidra';

try {
	await spidra.scrape({
		urls: [{ url: 'https://spidra.io' }],
		prompt: '...',
	});
} catch (err) {
	if (err instanceof SpidraAuthenticationError) {
		console.error('Invalid or missing API key'); // 401
	} else if (err instanceof SpidraInsufficientCreditsError) {
		console.error('Out of credits, top up your account'); // 403
	} else if (err instanceof SpidraValidationError) {
		console.error('Bad request:', err.errors); // 422, one entry per problem
	} else if (err instanceof SpidraRateLimitError) {
		// 429, the error tells you exactly how long to wait
		console.error(`Rate limited: ${err.remaining}/${err.limit} left, retry in ${err.retryAfterMs}ms`);
	} else if (err instanceof SpidraJobError) {
		// the job itself failed or was cancelled (not a transport error)
		console.error(`Job ${err.jobId} ${err.jobStatus}: ${err.message}`);
	} else if (err instanceof SpidraTimeoutError) {
		// your poll timeout elapsed, the job is still running server-side
		console.error(`Still running after ${err.timeoutMs}ms, check ${err.jobId} later`);
	} else if (err instanceof SpidraServerError) {
		console.error('Server error, already retried automatically'); // 5xx
	} else if (err instanceof SpidraError) {
		console.error(`API error ${err.status}: ${err.message}`);
	}
}

import { verifySpidraWebhook } from 'spidra';

// Express example, verify against the RAW body, not the parsed JSON
app.post('/webhooks/spidra', express.raw({ type: 'application/json' }), async (req, res) => {
	const valid = await verifySpidraWebhook(
		req.body,
		req.header('x-spidra-signature'),
		process.env.SPIDRA_WEBHOOK_SECRET!,
	);
	if (!valid) return res.status(401).end();

	const event = JSON.parse(req.body.toString());
	if (event.event === 'crawl.page') {
		console.log('New page:', event.page.url);
	}
	res.status(200).end();
});

import { generateText, tool } from 'ai';
import { anthropic } from '@ai-sdk/anthropic';
import { SpidraClient } from 'spidra';
import { z } from 'zod';

const spidra = new SpidraClient({ apiKey: process.env.SPIDRA_API_KEY });

const result = await generateText({
	model: anthropic('claude-opus-4-6'),
	maxSteps: 5,
	tools: {
		scrapeUrl: tool({
			description: 'Fetch and extract structured data from a URL',
			parameters: z.object({
				url: z.string().describe('The URL to scrape'),
				prompt: z.string().describe('What data to extract'),
			}),
			execute: async ({ url, prompt }) => {
				const result = await spidra.scrape({ urls: [{ url }], prompt });
				return JSON.stringify(result.content);
			},
		}),
	},
	prompt: 'What are the top 3 trending repositories on GitHub today?',
});

console.log(result.text);