curl --request POST \
--url https://api.spidra.io/api/crawl \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"baseUrl": "https://books.toscrape.com/catalogue/category/books/mystery_3/index.html",
"crawlInstruction": "Individual book pages",
"schema": {
"type": "object",
"properties": {
"title": {
"type": "string"
},
"price": {
"type": "number"
},
"in_stock": {
"type": "boolean"
}
}
},
"maxPages": 3
}
'import requests
url = "https://api.spidra.io/api/crawl"
payload = {
"baseUrl": "https://books.toscrape.com/catalogue/category/books/mystery_3/index.html",
"crawlInstruction": "Individual book pages",
"schema": {
"type": "object",
"properties": {
"title": { "type": "string" },
"price": { "type": "number" },
"in_stock": { "type": "boolean" }
}
},
"maxPages": 3
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
baseUrl: 'https://books.toscrape.com/catalogue/category/books/mystery_3/index.html',
crawlInstruction: 'Individual book pages',
schema: {
type: 'object',
properties: {title: {type: 'string'}, price: {type: 'number'}, in_stock: {type: 'boolean'}}
},
maxPages: 3
})
};
fetch('https://api.spidra.io/api/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.spidra.io/api/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'baseUrl' => 'https://books.toscrape.com/catalogue/category/books/mystery_3/index.html',
'crawlInstruction' => 'Individual book pages',
'schema' => [
'type' => 'object',
'properties' => [
'title' => [
'type' => 'string'
],
'price' => [
'type' => 'number'
],
'in_stock' => [
'type' => 'boolean'
]
]
],
'maxPages' => 3
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.spidra.io/api/crawl"
payload := strings.NewReader("{\n \"baseUrl\": \"https://books.toscrape.com/catalogue/category/books/mystery_3/index.html\",\n \"crawlInstruction\": \"Individual book pages\",\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"title\": {\n \"type\": \"string\"\n },\n \"price\": {\n \"type\": \"number\"\n },\n \"in_stock\": {\n \"type\": \"boolean\"\n }\n }\n },\n \"maxPages\": 3\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.spidra.io/api/crawl")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"baseUrl\": \"https://books.toscrape.com/catalogue/category/books/mystery_3/index.html\",\n \"crawlInstruction\": \"Individual book pages\",\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"title\": {\n \"type\": \"string\"\n },\n \"price\": {\n \"type\": \"number\"\n },\n \"in_stock\": {\n \"type\": \"boolean\"\n }\n }\n },\n \"maxPages\": 3\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.spidra.io/api/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"baseUrl\": \"https://books.toscrape.com/catalogue/category/books/mystery_3/index.html\",\n \"crawlInstruction\": \"Individual book pages\",\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"title\": {\n \"type\": \"string\"\n },\n \"price\": {\n \"type\": \"number\"\n },\n \"in_stock\": {\n \"type\": \"boolean\"\n }\n }\n },\n \"maxPages\": 3\n}"
response = http.request(request)
puts response.read_body{
"status": "queued",
"jobId": "e303fb87-846e-4a30-8aa5-bfca457a5357",
"message": "Crawl job queued. Poll /api/crawl/e303fb87-846e-4a30-8aa5-bfca457a5357 for results."
}Crawl
Starts a crawl that begins at one URL, follows links toward the pages you describe in crawlInstruction, and returns a jobId. Poll Get a crawl until status is completed, failed or cancelled. Guide: Crawl.
maxPages counts only the pages returned. Pages read just to find them are free and don’t count. A crawl stays under the start URL’s path unless crawlEntireDomain is true. With a schema or transformInstruction, each page’s data is what the AI extracted. With neither, it’s the page as Markdown and no AI runs.
Credits are held for every page the crawl may return. If your balance can’t cover maxPages, the crawl starts anyway with maxPages lowered to what it covers, and the response says so. If you send the exact same request again within 5 seconds, you get the first job’s jobId back with deduplicated: true.
With webhookUrl, Spidra sends a POST for each page (crawl.page), and one when the crawl finishes (crawl.completed) or fails (crawl.failed). Each request times out after 5 seconds and isn’t retried, so treat them as notifications and poll for the full result. See Webhooks.
Credits: 1 per page returned, plus AI credits for the tokens used with a schema or transformInstruction, plus 5 per CAPTCHA solved. A crawl that returns no pages costs nothing. See How credits work.
Rate limits: 60 requests a minute, shared with extract and page retries, and up to 5 unfinished crawls at a time.
curl --request POST \
--url https://api.spidra.io/api/crawl \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"baseUrl": "https://books.toscrape.com/catalogue/category/books/mystery_3/index.html",
"crawlInstruction": "Individual book pages",
"schema": {
"type": "object",
"properties": {
"title": {
"type": "string"
},
"price": {
"type": "number"
},
"in_stock": {
"type": "boolean"
}
}
},
"maxPages": 3
}
'import requests
url = "https://api.spidra.io/api/crawl"
payload = {
"baseUrl": "https://books.toscrape.com/catalogue/category/books/mystery_3/index.html",
"crawlInstruction": "Individual book pages",
"schema": {
"type": "object",
"properties": {
"title": { "type": "string" },
"price": { "type": "number" },
"in_stock": { "type": "boolean" }
}
},
"maxPages": 3
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
baseUrl: 'https://books.toscrape.com/catalogue/category/books/mystery_3/index.html',
crawlInstruction: 'Individual book pages',
schema: {
type: 'object',
properties: {title: {type: 'string'}, price: {type: 'number'}, in_stock: {type: 'boolean'}}
},
maxPages: 3
})
};
fetch('https://api.spidra.io/api/crawl', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.spidra.io/api/crawl",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'baseUrl' => 'https://books.toscrape.com/catalogue/category/books/mystery_3/index.html',
'crawlInstruction' => 'Individual book pages',
'schema' => [
'type' => 'object',
'properties' => [
'title' => [
'type' => 'string'
],
'price' => [
'type' => 'number'
],
'in_stock' => [
'type' => 'boolean'
]
]
],
'maxPages' => 3
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.spidra.io/api/crawl"
payload := strings.NewReader("{\n \"baseUrl\": \"https://books.toscrape.com/catalogue/category/books/mystery_3/index.html\",\n \"crawlInstruction\": \"Individual book pages\",\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"title\": {\n \"type\": \"string\"\n },\n \"price\": {\n \"type\": \"number\"\n },\n \"in_stock\": {\n \"type\": \"boolean\"\n }\n }\n },\n \"maxPages\": 3\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.spidra.io/api/crawl")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"baseUrl\": \"https://books.toscrape.com/catalogue/category/books/mystery_3/index.html\",\n \"crawlInstruction\": \"Individual book pages\",\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"title\": {\n \"type\": \"string\"\n },\n \"price\": {\n \"type\": \"number\"\n },\n \"in_stock\": {\n \"type\": \"boolean\"\n }\n }\n },\n \"maxPages\": 3\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.spidra.io/api/crawl")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"baseUrl\": \"https://books.toscrape.com/catalogue/category/books/mystery_3/index.html\",\n \"crawlInstruction\": \"Individual book pages\",\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"title\": {\n \"type\": \"string\"\n },\n \"price\": {\n \"type\": \"number\"\n },\n \"in_stock\": {\n \"type\": \"boolean\"\n }\n }\n },\n \"maxPages\": 3\n}"
response = http.request(request)
puts response.read_body{
"status": "queued",
"jobId": "e303fb87-846e-4a30-8aa5-bfca457a5357",
"message": "Crawl job queued. Poll /api/crawl/e303fb87-846e-4a30-8aa5-bfca457a5357 for results."
}Authorizations
Body
Which pages to return, in plain English, such as "Individual book pages". The report shows how it was understood.
A JSON Schema for each page's data. Takes precedence over transformInstruction for the shape.
Turns on Stealth Mode. Billed by bandwidth, not credits. Not needed when you set proxyCountry. false keeps the proxy off, even with a country. When Stealth Mode can't be used because your proxy bandwidth ran out, the request runs without it and warnings says so.
The country to read pages from, as a two-letter code such as us, gb or de, or eu or asia. Setting a country turns Stealth Mode on (billed by bandwidth), and the response's warnings says so, unless useProxy is false: then the country isn't used, and warnings says that instead. global means any country.
"gb"
Cookies from your own signed-in browser session, to crawl pages that need an account. Never stored. See Logged-in pages.
An http or https URL to notify for each page and when the crawl ends. Private and internal addresses are rejected. See Webhooks.
Response
The crawl was queued. Poll GET /crawl/{jobId} for the result.
Was this page helpful?

