List Crawl History
curl --request GET \
--url https://api.spidra.io/api/crawl/history \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.spidra.io/api/crawl/history"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.spidra.io/api/crawl/history', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.spidra.io/api/crawl/history",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.spidra.io/api/crawl/history"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.spidra.io/api/crawl/history")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.spidra.io/api/crawl/history")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"jobs": [
{
"id": "abc-123",
"base_url": "https://example.com",
"status": "completed",
"max_pages": 10,
"pages_crawled": 8,
"created_at": "2025-12-17T15:00:00Z",
"credits_used": 25
}
],
"total": 15,
"page": 1,
"totalPages": 2
}{
"status": "error",
"message": "Access token invalid or expired"
}Crawl Endpoints
List Crawl History
Retrieve a paginated list of your past crawl jobs, including status, page counts, and credit usage.
GET
/
crawl
/
history
List Crawl History
curl --request GET \
--url https://api.spidra.io/api/crawl/history \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.spidra.io/api/crawl/history"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.spidra.io/api/crawl/history', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.spidra.io/api/crawl/history",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.spidra.io/api/crawl/history"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.spidra.io/api/crawl/history")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.spidra.io/api/crawl/history")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"jobs": [
{
"id": "abc-123",
"base_url": "https://example.com",
"status": "completed",
"max_pages": 10,
"pages_crawled": 8,
"created_at": "2025-12-17T15:00:00Z",
"credits_used": 25
}
],
"total": 15,
"page": 1,
"totalPages": 2
}{
"status": "error",
"message": "Access token invalid or expired"
}Use this endpoint to browse all the crawl jobs your account has submitted. Each record shows the base URL, how many pages were crawled, the current job status, and how many credits were consumed. This is useful for building dashboards, auditing your usage, or picking up a job ID you want to work with in a follow-up request.
Pagination
The response is paginated. Use thepage and limit query parameters to navigate through your history.
# First page (default: 10 results)
curl https://api.spidra.io/api/crawl/history \
-H "Authorization: Bearer YOUR_API_KEY"
# Second page with 25 results per page
curl "https://api.spidra.io/api/crawl/history?page=2&limit=25" \
-H "Authorization: Bearer YOUR_API_KEY"
import requests
# First page
response = requests.get(
"https://api.spidra.io/api/crawl/history",
headers={"Authorization": "Bearer YOUR_API_KEY"}
)
# Second page with 25 results per page
response = requests.get(
"https://api.spidra.io/api/crawl/history",
headers={"Authorization": "Bearer YOUR_API_KEY"},
params={"page": 2, "limit": 25}
)
// First page
const response = await fetch("https://api.spidra.io/api/crawl/history", {
headers: { Authorization: "Bearer YOUR_API_KEY" }
});
// Second page with 25 results per page
const response = await fetch(
"https://api.spidra.io/api/crawl/history?page=2&limit=25",
{ headers: { Authorization: "Bearer YOUR_API_KEY" } }
);
Response Fields
| Field | Type | Description |
|---|---|---|
jobs | array | List of crawl job records for this page |
jobs[].id | string | Unique job ID. Use this to call other endpoints like GET /crawl//pages |
jobs[].base_url | string | The starting URL that was crawled |
jobs[].status | string | Job status: waiting, active, completed, or failed |
jobs[].max_pages | integer | The maximum number of pages requested |
jobs[].pages_crawled | integer | Actual number of pages successfully crawled |
jobs[].created_at | string | ISO 8601 timestamp when the job was created |
jobs[].credits_used | number | Credits charged to your account for this job |
total | integer | Total number of crawl jobs in your account |
page | integer | The current page number |
totalPages | integer | Total number of pages available |
Example Response
{
"jobs": [
{
"id": "abc-123",
"base_url": "https://example.com/blog",
"status": "completed",
"max_pages": 10,
"pages_crawled": 8,
"created_at": "2025-12-17T15:00:00Z",
"credits_used": 25
},
{
"id": "def-456",
"base_url": "https://store.example.com/products",
"status": "failed",
"max_pages": 20,
"pages_crawled": 3,
"created_at": "2025-12-16T09:30:00Z",
"credits_used": 8
}
],
"total": 34,
"page": 1,
"totalPages": 4
}
To get the full extracted data for a completed job, call GET /crawl//pages using the
id from this response.Authorizations
BearerAuthApiKeyAuth
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Query Parameters
Required range:
x >= 1Required range:
1 <= x <= 100Was this page helpful?
⌘I

