Skip to main content
GET
/
batch
/
scrape
/
{batchId}
Get Batch Status
curl --request GET \
  --url https://api.spidra.io/api/batch/scrape/{batchId} \
  --header 'Authorization: Bearer <token>'
import requests

url = "https://api.spidra.io/api/batch/scrape/{batchId}"

headers = {"Authorization": "Bearer <token>"}

response = requests.get(url, headers=headers)

print(response.text)
const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://api.spidra.io/api/batch/scrape/{batchId}', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));
<?php

$curl = curl_init();

curl_setopt_array($curl, [
CURLOPT_URL => "https://api.spidra.io/api/batch/scrape/{batchId}",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}
package main

import (
"fmt"
"net/http"
"io"
)

func main() {

url := "https://api.spidra.io/api/batch/scrape/{batchId}"

req, _ := http.NewRequest("GET", url, nil)

req.Header.Add("Authorization", "Bearer <token>")

res, _ := http.DefaultClient.Do(req)

defer res.Body.Close()
body, _ := io.ReadAll(res.Body)

fmt.Println(string(body))

}
HttpResponse<String> response = Unirest.get("https://api.spidra.io/api/batch/scrape/{batchId}")
.header("Authorization", "Bearer <token>")
.asString();
require 'uri'
require 'net/http'

url = URI("https://api.spidra.io/api/batch/scrape/{batchId}")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body
{
  "totalUrls": 123,
  "completedCount": 123,
  "failedCount": 123,
  "items": [
    {
      "uuid": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
      "url": "<string>",
      "jobId": "<string>",
      "result": "<unknown>",
      "error": "<string>",
      "creditsUsed": 123,
      "startedAt": "2023-11-07T05:31:56Z",
      "finishedAt": "2023-11-07T05:31:56Z",
      "screenshotUrl": "<string>"
    }
  ],
  "createdAt": "2023-11-07T05:31:56Z",
  "finishedAt": "2023-11-07T05:31:56Z"
}
{
"status": "error",
"message": "<string>"
}
{
"status": "error",
"message": "Batch scrape job not found.",
"code": "NOT_FOUND"
}

Polling Pattern

Poll this endpoint every 2–5 seconds after submitting a batch. Once status is completed, failed, or cancelled, stop polling (the batch will not change further).
async function pollBatch(batchId) {
  while (true) {
    const res = await fetch(
      `https://api.spidra.io/api/batch/scrape/${batchId}`,
      { headers: { Authorization: "Bearer YOUR_API_KEY" } }
    );
    const data = await res.json();

    console.log(`${data.completedCount}/${data.totalUrls} complete, ${data.failedCount} failed`);

    if (["completed", "failed", "cancelled"].includes(data.status)) {
      return data;
    }

    await new Promise((r) => setTimeout(r, 3000));
  }
}
import time, requests

def poll_batch(batch_id):
    headers = {"Authorization": "Bearer YOUR_API_KEY"}
    while True:
        data = requests.get(
            f"https://api.spidra.io/api/batch/scrape/{batch_id}",
            headers=headers,
        ).json()

        print(f"{data['completedCount']}/{data['totalUrls']} complete, {data['failedCount']} failed")

        if data["status"] in ("completed", "failed", "cancelled"):
            return data

        time.sleep(3)
curl https://api.spidra.io/api/batch/scrape/YOUR_BATCH_ID \
  -H "Authorization: Bearer YOUR_API_KEY"

Batch Status Values

StatusMeaning
pendingQueued — no items have started yet
runningAt least one item is being processed
completedAll items reached a terminal state. Check failedCount for partial failures
failedThe batch failed unexpectedly
cancelledCancelled via DELETE /api/batch/scrape/{batchId}
completed does not guarantee every URL succeeded. A batch is marked completed when all items have a terminal status (completed or failed). Always inspect failedCount and individual item statuses to detect partial failures.

Response

Example — batch in progress:
{
  "status": "running",
  "totalUrls": 5,
  "completedCount": 3,
  "failedCount": 0,
  "items": [...],
  "createdAt": "2024-01-15T10:00:00Z",
  "finishedAt": null
}
Example — batch completed:
{
  "status": "completed",
  "totalUrls": 5,
  "completedCount": 4,
  "failedCount": 1,
  "items": [...],
  "createdAt": "2024-01-15T10:00:00Z",
  "finishedAt": "2024-01-15T10:00:42Z"
}

Top-Level Fields

FieldTypeDescription
statusstringBatch-level status: pending, running, completed, failed, or cancelled
totalUrlsnumberTotal number of URLs in the batch
completedCountnumberItems that finished successfully
failedCountnumberItems that encountered an error
itemsarrayPer-item results (see below)
createdAtstringISO 8601 timestamp when the batch was submitted
finishedAtstring | nullISO 8601 timestamp when the batch reached a terminal state, or null if still running

Per-Item Fields

Each entry in items represents one URL:
{
  "uuid": "a1b2c3d4-0000-0000-0000-000000000000",
  "url": "https://example.com/product/42",
  "jobId": "bull-worker-job-id",
  "status": "completed",
  "result": {
    "name": "Widget Pro",
    "price": 49.99,
    "available": true
  },
  "error": null,
  "creditsUsed": 3,
  "startedAt": "2024-01-15T10:00:05Z",
  "finishedAt": "2024-01-15T10:00:11Z",
  "screenshotUrl": null
}
FieldTypeDescription
uuidstringUnique ID for this batch item
urlstringThe URL that was processed
jobIdstring | nullInternal worker job ID. null if the item is still pending
statusstringItem-level status: pending, running, completed, or failed
resultany | nullExtracted content. Object if output: "json", string if "markdown". null until completed
errorstring | nullError description if status is failed, otherwise null
creditsUsednumberCredits consumed by this item. 0 for failed or cancelled items
startedAtstring | nullISO 8601 timestamp when the worker started this item
finishedAtstring | nullISO 8601 timestamp when this item completed or failed
screenshotUrlstring | nullS3 URL for the screenshot, or null if screenshots were not requested

Handling Partial Failures

When completedCount + failedCount === totalUrls but some items failed, retry them without re-running the whole batch:
const data = await pollBatch(batchId);

if (data.failedCount > 0) {
  const retry = await fetch(
    `https://api.spidra.io/api/batch/scrape/${batchId}/retry`,
    { method: "POST", headers: { Authorization: "Bearer YOUR_API_KEY" } }
  );
  const { retriedCount } = await retry.json();
  console.log(`Retrying ${retriedCount} failed items...`);
}

Errors

CodeReason
401Missing API key authentication header
403Invalid or expired API key
404No batch found with this ID, or it belongs to a different user

Retry Failed Batch Scrapes

Re-queue only the failed items

Cancel a Batch

Stop processing and refund credits

Authorizations

Authorization
string
header
required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Path Parameters

batchId
string<uuid>
required

The batch ID returned by POST /batch/scrape

Response

Batch status and per-item results

status
enum<string>
Available options:
pending,
running,
completed,
failed,
cancelled
totalUrls
integer
completedCount
integer
failedCount
integer
items
object[]
createdAt
string<date-time>
finishedAt
string<date-time> | null