Get Crawl Job Details

curl --request GET \
  --url https://api.spidra.io/api/crawl/job/{jobId} \
  --header 'Authorization: Bearer <token>'

import requests

url = "https://api.spidra.io/api/crawl/job/{jobId}"

headers = {"Authorization": "Bearer <token>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://api.spidra.io/api/crawl/job/{jobId}', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.spidra.io/api/crawl/job/{jobId}",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.spidra.io/api/crawl/job/{jobId}"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "Bearer <token>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.get("https://api.spidra.io/api/crawl/job/{jobId}")
  .header("Authorization", "Bearer <token>")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.spidra.io/api/crawl/job/{jobId}")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body

{
  "id": "abc-123",
  "base_url": "https://example.com/blog",
  "crawl_instruction": "Crawl all blog post pages",
  "transform_instruction": "Extract title, author, and publish date",
  "output_schema": null,
  "max_pages": 10,
  "max_depth": 2,
  "include_paths": [
    "/blog/*"
  ],
  "exclude_paths": [
    "/blog/tag/*"
  ],
  "allow_subdomains": false,
  "crawl_entire_domain": false,
  "ignore_query_params": true,
  "webhook_url": null,
  "status": "completed",
  "pages_crawled": 8,
  "input_tokens": 14820,
  "output_tokens": 3210,
  "credits_used": 25,
  "created_at": "2025-12-17T15:00:00Z",
  "updated_at": "2025-12-17T15:03:42Z"
}

{
  "status": "error",
  "message": "Access token invalid or expired"
}

{
  "status": "error",
  "message": "Crawl job not found"
}

GET

crawl

job

{jobId}

Get Crawl Job Details

curl --request GET \
  --url https://api.spidra.io/api/crawl/job/{jobId} \
  --header 'Authorization: Bearer <token>'

import requests

url = "https://api.spidra.io/api/crawl/job/{jobId}"

headers = {"Authorization": "Bearer <token>"}

response = requests.get(url, headers=headers)

print(response.text)

const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://api.spidra.io/api/crawl/job/{jobId}', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.spidra.io/api/crawl/job/{jobId}",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.spidra.io/api/crawl/job/{jobId}"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "Bearer <token>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

HttpResponse<String> response = Unirest.get("https://api.spidra.io/api/crawl/job/{jobId}")
  .header("Authorization", "Bearer <token>")
  .asString();

require 'uri'
require 'net/http'

url = URI("https://api.spidra.io/api/crawl/job/{jobId}")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body

{
  "id": "abc-123",
  "base_url": "https://example.com/blog",
  "crawl_instruction": "Crawl all blog post pages",
  "transform_instruction": "Extract title, author, and publish date",
  "output_schema": null,
  "max_pages": 10,
  "max_depth": 2,
  "include_paths": [
    "/blog/*"
  ],
  "exclude_paths": [
    "/blog/tag/*"
  ],
  "allow_subdomains": false,
  "crawl_entire_domain": false,
  "ignore_query_params": true,
  "webhook_url": null,
  "status": "completed",
  "pages_crawled": 8,
  "input_tokens": 14820,
  "output_tokens": 3210,
  "credits_used": 25,
  "created_at": "2025-12-17T15:00:00Z",
  "updated_at": "2025-12-17T15:03:42Z"
}

{
  "status": "error",
  "message": "Access token invalid or expired"
}

{
  "status": "error",
  "message": "Crawl job not found"
}

Returns the complete record for a crawl job: the configuration you submitted, current status, token usage, and credit cost. It does not return the extracted page data. For the actual content from each page, use GET /crawl//pages.

When to Use This Endpoint

Inspect the instructions and options that were used for a job
Check token and credit costs for accounting or reporting
Confirm job configuration before re-running extraction with POST /crawl//extract

Example Request

curl https://api.spidra.io/api/crawl/job/abc-123 \
  -H "Authorization: Bearer YOUR_API_KEY"

import requests

response = requests.get(
    "https://api.spidra.io/api/crawl/job/abc-123",
    headers={"Authorization": "Bearer YOUR_API_KEY"}
)

const response = await fetch("https://api.spidra.io/api/crawl/job/abc-123", {
  headers: { Authorization: "Bearer YOUR_API_KEY" }
});

Response Fields

Job configuration

Field	Type	Description
`id`	string	Unique job identifier.
`base_url`	string	The starting URL that was crawled.
`crawl_instruction`	string	The instruction used to decide which pages to follow.
`transform_instruction`	string or null	The prompt used to extract data from each page. `null` if extraction used a schema or if no extraction was configured.
`output_schema`	object or null	The JSON Schema used for structured extraction. `null` if a transform instruction was used instead, or if no extraction was configured.
`max_pages`	integer	The page limit set for this job.
`max_depth`	integer or null	The link depth limit. `null` if not set (unlimited).
`include_paths`	string[] or null	Path patterns that were included. `null` if not set.
`exclude_paths`	string[] or null	Path patterns that were excluded. `null` if not set.
`allow_subdomains`	boolean	Whether subdomain links were followed.
`crawl_entire_domain`	boolean	Whether all paths on the root domain were eligible.
`ignore_query_params`	boolean	Whether URLs differing only by query string were deduplicated.
`webhook_url`	string or null	The webhook URL that received per-page events, if configured.

Status and stats

Field	Type	Description
`status`	string	`waiting`, `active`, `completed`, `failed`, or `cancelled`.
`pages_crawled`	integer	Number of pages successfully processed.
`input_tokens`	number or null	Total input tokens consumed by AI across all pages. `null` when no AI extraction was used.
`output_tokens`	number or null	Total output tokens generated by AI across all pages. `null` when no AI extraction was used.
`credits_used`	number or null	Total credits charged for this job.
`created_at`	string	ISO 8601 timestamp when the job was created.
`updated_at`	string	ISO 8601 timestamp of the last status change.

Example Response

{
  "id": "abc-123",
  "base_url": "https://example.com/blog",
  "crawl_instruction": "Crawl all blog post pages",
  "transform_instruction": "Extract title, author, publish date, and the first 200 words of the body",
  "output_schema": null,
  "max_pages": 10,
  "max_depth": 2,
  "include_paths": ["/blog/*"],
  "exclude_paths": ["/blog/tag/*"],
  "allow_subdomains": false,
  "crawl_entire_domain": false,
  "ignore_query_params": true,
  "webhook_url": null,
  "status": "completed",
  "pages_crawled": 8,
  "input_tokens": 14820,
  "output_tokens": 3210,
  "credits_used": 25,
  "created_at": "2025-12-17T15:00:00Z",
  "updated_at": "2025-12-17T15:03:42Z"
}

For the actual extracted content from each page, call GET /crawl//pages.

Authorizations

Authorization

string

header

required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Path Parameters

jobId

string

required

Response

Job details

string

base_url

string

crawl_instruction

string

transform_instruction

string | null

output_schema

object | null

The JSON Schema used for structured extraction, if one was provided.

max_pages

integer

max_depth

integer | null

include_paths

string[] | null

exclude_paths

string[] | null

allow_subdomains

boolean

crawl_entire_domain

boolean

ignore_query_params

boolean

webhook_url

string | null

pages_crawled

integer

status

enum<string>

Available options:

waiting,

active,

completed,

failed,

cancelled

created_at

string<date-time>

updated_at

string<date-time>

input_tokens

number | null

output_tokens

number | null

credits_used

number | null

Download Crawl Results List Crawl History

​When to Use This Endpoint

​Example Request

​Response Fields

​Job configuration

​Status and stats

​Example Response

Authorizations

Path Parameters

Response

When to Use This Endpoint

Example Request

Response Fields

Job configuration

Status and stats

Example Response