List crawls
curl --request GET \
--url https://api.spidra.io/api/crawl/history \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.spidra.io/api/crawl/history"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.spidra.io/api/crawl/history', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.spidra.io/api/crawl/history",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.spidra.io/api/crawl/history"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.spidra.io/api/crawl/history")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.spidra.io/api/crawl/history")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"jobs": [
{
"id": "1b4dc02b-f146-404e-9fd2-3e444f1e01cc",
"user_uuid": "4aeefb9f-30c3-41fb-886a-10245923dc31",
"source_job_id": "e303fb87-846e-4a30-8aa5-bfca457a5357",
"base_url": "https://books.toscrape.com/catalogue/category/books/mystery_3/index.html",
"crawl_instruction": null,
"transform_instruction": "Return the book's title and whether it is in stock",
"output_schema": null,
"max_pages": 3,
"include_paths": null,
"exclude_paths": null,
"max_depth": null,
"allow_subdomains": 0,
"crawl_entire_domain": 0,
"ignore_query_params": 0,
"webhook_url": null,
"status": "completed",
"input_tokens": 3367,
"output_tokens": 35,
"credits_used": "2.00",
"created_at": "2026-10-25T03:16:33.000Z",
"updated_at": "2026-10-25T03:16:35.000Z",
"use_proxy": 0,
"proxy_billing": "none",
"proxy_bandwidth_mb": null,
"pages_crawled": 3,
"site_title": "Sharp Objects | Books to Scrape - Sandbox"
},
{
"id": "e303fb87-846e-4a30-8aa5-bfca457a5357",
"user_uuid": "4aeefb9f-30c3-41fb-886a-10245923dc31",
"source_job_id": null,
"base_url": "https://books.toscrape.com/catalogue/category/books/mystery_3/index.html",
"crawl_instruction": "Individual book pages",
"transform_instruction": null,
"output_schema": {
"type": "object",
"properties": {
"price": {
"type": "number"
},
"title": {
"type": "string"
},
"in_stock": {
"type": "boolean"
}
}
},
"max_pages": 3,
"include_paths": null,
"exclude_paths": null,
"max_depth": null,
"allow_subdomains": 0,
"crawl_entire_domain": 0,
"ignore_query_params": 0,
"webhook_url": null,
"status": "completed",
"input_tokens": 4376,
"output_tokens": 59,
"credits_used": "6.00",
"created_at": "2026-10-25T03:14:45.000Z",
"updated_at": "2026-10-25T03:16:38.000Z",
"use_proxy": 1,
"proxy_billing": "free_auto",
"proxy_bandwidth_mb": "1.8537",
"pages_crawled": 3,
"site_title": "Sharp Objects | Books to Scrape - Sandbox"
}
],
"total": 361,
"page": 1,
"totalPages": 181
}Crawl
List crawls
Lists your crawls and extractions, newest first, with their settings, page counts and cost. An extraction has source_job_id set to the crawl it came from. Crawls are kept, so this includes ones whose GET /crawl/{jobId} has expired.
This response uses snake_case, and credits_used is a decimal string.
GET
/
crawl
/
history
List crawls
curl --request GET \
--url https://api.spidra.io/api/crawl/history \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.spidra.io/api/crawl/history"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.spidra.io/api/crawl/history', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.spidra.io/api/crawl/history",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.spidra.io/api/crawl/history"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.spidra.io/api/crawl/history")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.spidra.io/api/crawl/history")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"jobs": [
{
"id": "1b4dc02b-f146-404e-9fd2-3e444f1e01cc",
"user_uuid": "4aeefb9f-30c3-41fb-886a-10245923dc31",
"source_job_id": "e303fb87-846e-4a30-8aa5-bfca457a5357",
"base_url": "https://books.toscrape.com/catalogue/category/books/mystery_3/index.html",
"crawl_instruction": null,
"transform_instruction": "Return the book's title and whether it is in stock",
"output_schema": null,
"max_pages": 3,
"include_paths": null,
"exclude_paths": null,
"max_depth": null,
"allow_subdomains": 0,
"crawl_entire_domain": 0,
"ignore_query_params": 0,
"webhook_url": null,
"status": "completed",
"input_tokens": 3367,
"output_tokens": 35,
"credits_used": "2.00",
"created_at": "2026-10-25T03:16:33.000Z",
"updated_at": "2026-10-25T03:16:35.000Z",
"use_proxy": 0,
"proxy_billing": "none",
"proxy_bandwidth_mb": null,
"pages_crawled": 3,
"site_title": "Sharp Objects | Books to Scrape - Sandbox"
},
{
"id": "e303fb87-846e-4a30-8aa5-bfca457a5357",
"user_uuid": "4aeefb9f-30c3-41fb-886a-10245923dc31",
"source_job_id": null,
"base_url": "https://books.toscrape.com/catalogue/category/books/mystery_3/index.html",
"crawl_instruction": "Individual book pages",
"transform_instruction": null,
"output_schema": {
"type": "object",
"properties": {
"price": {
"type": "number"
},
"title": {
"type": "string"
},
"in_stock": {
"type": "boolean"
}
}
},
"max_pages": 3,
"include_paths": null,
"exclude_paths": null,
"max_depth": null,
"allow_subdomains": 0,
"crawl_entire_domain": 0,
"ignore_query_params": 0,
"webhook_url": null,
"status": "completed",
"input_tokens": 4376,
"output_tokens": 59,
"credits_used": "6.00",
"created_at": "2026-10-25T03:14:45.000Z",
"updated_at": "2026-10-25T03:16:38.000Z",
"use_proxy": 1,
"proxy_billing": "free_auto",
"proxy_bandwidth_mb": "1.8537",
"pages_crawled": 3,
"site_title": "Sharp Objects | Books to Scrape - Sandbox"
}
],
"total": 361,
"page": 1,
"totalPages": 181
}Authorizations
BearerAuthApiKeyAuth
Query Parameters
Was this page helpful?

