Get Batch Status
curl --request GET \
--url https://api.spidra.io/api/batch/scrape/{batchId} \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.spidra.io/api/batch/scrape/{batchId}"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.spidra.io/api/batch/scrape/{batchId}', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.spidra.io/api/batch/scrape/{batchId}",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.spidra.io/api/batch/scrape/{batchId}"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.spidra.io/api/batch/scrape/{batchId}")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.spidra.io/api/batch/scrape/{batchId}")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"status": "pending",
"totalUrls": 123,
"completedCount": 123,
"failedCount": 123,
"items": [
{
"uuid": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"url": "<string>",
"jobId": "<string>",
"status": "pending",
"result": "<unknown>",
"error": "<string>",
"creditsUsed": 123,
"startedAt": "2023-11-07T05:31:56Z",
"finishedAt": "2023-11-07T05:31:56Z",
"screenshotUrl": "<string>"
}
],
"createdAt": "2023-11-07T05:31:56Z",
"finishedAt": "2023-11-07T05:31:56Z"
}{
"status": "error",
"message": "<string>"
}{
"status": "error",
"message": "Batch scrape job not found.",
"code": "NOT_FOUND"
}Scrape Endpoints
Get Batch Status
Poll for batch progress and retrieve per-item results
GET
/
batch
/
scrape
/
{batchId}
Get Batch Status
curl --request GET \
--url https://api.spidra.io/api/batch/scrape/{batchId} \
--header 'Authorization: Bearer <token>'import requests
url = "https://api.spidra.io/api/batch/scrape/{batchId}"
headers = {"Authorization": "Bearer <token>"}
response = requests.get(url, headers=headers)
print(response.text)const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};
fetch('https://api.spidra.io/api/batch/scrape/{batchId}', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.spidra.io/api/batch/scrape/{batchId}",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://api.spidra.io/api/batch/scrape/{batchId}"
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.get("https://api.spidra.io/api/batch/scrape/{batchId}")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.spidra.io/api/batch/scrape/{batchId}")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"status": "pending",
"totalUrls": 123,
"completedCount": 123,
"failedCount": 123,
"items": [
{
"uuid": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"url": "<string>",
"jobId": "<string>",
"status": "pending",
"result": "<unknown>",
"error": "<string>",
"creditsUsed": 123,
"startedAt": "2023-11-07T05:31:56Z",
"finishedAt": "2023-11-07T05:31:56Z",
"screenshotUrl": "<string>"
}
],
"createdAt": "2023-11-07T05:31:56Z",
"finishedAt": "2023-11-07T05:31:56Z"
}{
"status": "error",
"message": "<string>"
}{
"status": "error",
"message": "Batch scrape job not found.",
"code": "NOT_FOUND"
}Polling Pattern
Poll this endpoint every 2–5 seconds after submitting a batch. Oncestatus is completed, failed, or cancelled, stop polling (the batch will not change further).
async function pollBatch(batchId) {
while (true) {
const res = await fetch(
`https://api.spidra.io/api/batch/scrape/${batchId}`,
{ headers: { Authorization: "Bearer YOUR_API_KEY" } }
);
const data = await res.json();
console.log(`${data.completedCount}/${data.totalUrls} complete, ${data.failedCount} failed`);
if (["completed", "failed", "cancelled"].includes(data.status)) {
return data;
}
await new Promise((r) => setTimeout(r, 3000));
}
}
import time, requests
def poll_batch(batch_id):
headers = {"Authorization": "Bearer YOUR_API_KEY"}
while True:
data = requests.get(
f"https://api.spidra.io/api/batch/scrape/{batch_id}",
headers=headers,
).json()
print(f"{data['completedCount']}/{data['totalUrls']} complete, {data['failedCount']} failed")
if data["status"] in ("completed", "failed", "cancelled"):
return data
time.sleep(3)
curl https://api.spidra.io/api/batch/scrape/YOUR_BATCH_ID \
-H "Authorization: Bearer YOUR_API_KEY"
Batch Status Values
| Status | Meaning |
|---|---|
pending | Queued — no items have started yet |
running | At least one item is being processed |
completed | All items reached a terminal state. Check failedCount for partial failures |
failed | The batch failed unexpectedly |
cancelled | Cancelled via DELETE /api/batch/scrape/{batchId} |
completed does not guarantee every URL succeeded. A batch is marked completed when all items have a terminal status (completed or failed). Always inspect failedCount and individual item statuses to detect partial failures.Response
Example — batch in progress:{
"status": "running",
"totalUrls": 5,
"completedCount": 3,
"failedCount": 0,
"items": [...],
"createdAt": "2024-01-15T10:00:00Z",
"finishedAt": null
}
{
"status": "completed",
"totalUrls": 5,
"completedCount": 4,
"failedCount": 1,
"items": [...],
"createdAt": "2024-01-15T10:00:00Z",
"finishedAt": "2024-01-15T10:00:42Z"
}
Top-Level Fields
| Field | Type | Description |
|---|---|---|
status | string | Batch-level status: pending, running, completed, failed, or cancelled |
totalUrls | number | Total number of URLs in the batch |
completedCount | number | Items that finished successfully |
failedCount | number | Items that encountered an error |
items | array | Per-item results (see below) |
createdAt | string | ISO 8601 timestamp when the batch was submitted |
finishedAt | string | null | ISO 8601 timestamp when the batch reached a terminal state, or null if still running |
Per-Item Fields
Each entry initems represents one URL:
{
"uuid": "a1b2c3d4-0000-0000-0000-000000000000",
"url": "https://example.com/product/42",
"jobId": "bull-worker-job-id",
"status": "completed",
"result": {
"name": "Widget Pro",
"price": 49.99,
"available": true
},
"error": null,
"creditsUsed": 3,
"startedAt": "2024-01-15T10:00:05Z",
"finishedAt": "2024-01-15T10:00:11Z",
"screenshotUrl": null
}
| Field | Type | Description |
|---|---|---|
uuid | string | Unique ID for this batch item |
url | string | The URL that was processed |
jobId | string | null | Internal worker job ID. null if the item is still pending |
status | string | Item-level status: pending, running, completed, or failed |
result | any | null | Extracted content. Object if output: "json", string if "markdown". null until completed |
error | string | null | Error description if status is failed, otherwise null |
creditsUsed | number | Credits consumed by this item. 0 for failed or cancelled items |
startedAt | string | null | ISO 8601 timestamp when the worker started this item |
finishedAt | string | null | ISO 8601 timestamp when this item completed or failed |
screenshotUrl | string | null | S3 URL for the screenshot, or null if screenshots were not requested |
Handling Partial Failures
WhencompletedCount + failedCount === totalUrls but some items failed, retry them without re-running the whole batch:
const data = await pollBatch(batchId);
if (data.failedCount > 0) {
const retry = await fetch(
`https://api.spidra.io/api/batch/scrape/${batchId}/retry`,
{ method: "POST", headers: { Authorization: "Bearer YOUR_API_KEY" } }
);
const { retriedCount } = await retry.json();
console.log(`Retrying ${retriedCount} failed items...`);
}
Errors
| Code | Reason |
|---|---|
401 | Missing API key authentication header |
403 | Invalid or expired API key |
404 | No batch found with this ID, or it belongs to a different user |
Retry Failed Batch Scrapes
Re-queue only the failed items
Cancel a Batch
Stop processing and refund credits
Authorizations
BearerAuthApiKeyAuth
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Path Parameters
The batch ID returned by POST /batch/scrape
Was this page helpful?
⌘I

