Skip to main content
GET
/
crawl
/
{jobId}
Get Crawl Job Status
curl --request GET \
  --url https://api.spidra.io/api/crawl/{jobId} \
  --header 'Authorization: Bearer <token>'
import requests

url = "https://api.spidra.io/api/crawl/{jobId}"

headers = {"Authorization": "Bearer <token>"}

response = requests.get(url, headers=headers)

print(response.text)
const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://api.spidra.io/api/crawl/{jobId}', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));
<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.spidra.io/api/crawl/{jobId}",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}
package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.spidra.io/api/crawl/{jobId}"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "Bearer <token>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}
HttpResponse<String> response = Unirest.get("https://api.spidra.io/api/crawl/{jobId}")
  .header("Authorization", "Bearer <token>")
  .asString();
require 'uri'
require 'net/http'

url = URI("https://api.spidra.io/api/crawl/{jobId}")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body
{
  "status": "completed",
  "progress": {
    "message": "Crawl complete",
    "pagesCrawled": 5,
    "maxPages": 10
  },
  "result": [
    {
      "url": "https://example.com/blog/post-1",
      "title": "First Post",
      "status": "success",
      "data": {
        "title": "First Post",
        "author": "John",
        "date": "2025-01-01"
      },
      "html": "https://storage.spidra.io/signed/crawl/abc-123/page1.html?...",
      "markdown": "https://storage.spidra.io/signed/crawl/abc-123/page1.md?..."
    }
  ],
  "error": null
}
{
  "status": "error",
  "message": "You do not have permission to access this job"
}
{
  "status": "error",
  "message": "Crawl job not found"
}
Poll this endpoint after submitting a crawl job. While the job is running it returns progress information. Once the job completes, the result array contains one entry per successfully crawled page. Poll every 2–5 seconds. Stop when status is completed, failed, or cancelled.

​
Example Request

curl https://api.spidra.io/api/crawl/abc-123 \
  -H "Authorization: Bearer YOUR_API_KEY"

import requests, time

while True:
    r = requests.get(
        "https://api.spidra.io/api/crawl/abc-123",
        headers={"Authorization": "Bearer YOUR_API_KEY"}
    ).json()
    if r["status"] in ("completed", "failed", "cancelled"):
        break
    time.sleep(3)

let data;
do {
  await new Promise(r => setTimeout(r, 3000));
  const res = await fetch("https://api.spidra.io/api/crawl/abc-123", {
    headers: { Authorization: "Bearer YOUR_API_KEY" }
  });
  data = await res.json();
} while (!["completed", "failed", "cancelled"].includes(data.status));

​
Response Fields

FieldTypeDescription
statusstringwaiting, active, completed, failed, or cancelled.
progressobjectCurrent progress message and page counts. Present while the job is running.
progress.messagestringA description of what the crawler is doing right now.
progress.pagesCrawledintegerNumber of pages processed so far.
progress.maxPagesintegerThe page limit for this job.
resultarray or nullThe full page result set. Populated once status is completed. null while the job is still running.
result[].urlstringThe URL of this page.
result[].titlestringPage title.
result[].statusstringsuccess or failed.
result[].dataanyExtracted content. Contains AI-extracted data when a transformInstruction or schema was provided. Contains the raw page markdown when neither was set.
result[].htmlstring or nullSigned URL to the raw HTML snapshot (valid for 1 hour).
result[].markdownstring or nullSigned URL to the markdown version of this page (valid for 1 hour).
errorstring or nullError message if status is failed.

​
Example Responses

While running: 
{
  "status": "active",
  "progress": {
    "message": "Crawling page 3 of 10",
    "pagesCrawled": 3,
    "maxPages": 10
  },
  "result": null,
  "error": null
}
Completed: 
{
  "status": "completed",
  "progress": {
    "message": "Crawl complete",
    "pagesCrawled": 10,
    "maxPages": 10
  },
  "result": [
    {
      "url": "https://example.com/blog/post-1",
      "title": "First Post",
      "status": "success",
      "data": {
        "title": "First Post",
        "author": "John",
        "date": "2025-01-01"
      },
      "html": "https://storage.spidra.io/signed/...",
      "markdown": "https://storage.spidra.io/signed/..."
    }
  ],
  "error": null
}
Unlike the scrape endpoint, the result field here is an array — one object per page crawled. For richer metadata on each page (page IDs, error messages, per-page timestamps), use GET /crawl//pages after the job completes.

Authorizations

​
Authorization
string
header
required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Path Parameters

​
jobId
string
required

The job ID returned from POST /crawl

Response

Job status and results

​
status
enum<string>
Available options:
waiting,
active,
completed,
failed,
cancelled
​
progress
object
​
result
object[] | null
​
error
string | null