Get Crawl Job Details

curl --request GET \
  --url https://api.spidra.io/api/crawl/job/{jobId} \
  --header 'Authorization: Bearer <token>'

const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://api.spidra.io/api/crawl/job/{jobId}', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.spidra.io/api/crawl/job/{jobId}",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.spidra.io/api/crawl/job/{jobId}"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "Bearer <token>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

require 'uri'
require 'net/http'

url = URI("https://api.spidra.io/api/crawl/job/{jobId}")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body

{
  "id": "abc-123",
  "base_url": "https://example.com/blog",
  "crawl_instruction": "Crawl all blog post pages",
  "transform_instruction": "Extract title, author, and publish date",
  "output_schema": null,
  "max_pages": 10,
  "max_depth": 2,
  "include_paths": [
    "/blog/*"
  ],
  "exclude_paths": [
    "/blog/tag/*"
  ],
  "allow_subdomains": false,
  "crawl_entire_domain": false,
  "ignore_query_params": true,
  "webhook_url": null,
  "status": "completed",
  "pages_crawled": 8,
  "input_tokens": 14820,
  "output_tokens": 3210,
  "credits_used": 25,
  "created_at": "2025-12-17T15:00:00Z",
  "updated_at": "2025-12-17T15:03:42Z"
}

{
  "status": "error",
  "message": "Access token invalid or expired"
}

{
  "status": "error",
  "message": "Crawl job not found"
}

GET

crawl

job

{jobId}

Get Crawl Job Details

curl --request GET \
  --url https://api.spidra.io/api/crawl/job/{jobId} \
  --header 'Authorization: Bearer <token>'

const options = {method: 'GET', headers: {Authorization: 'Bearer <token>'}};

fetch('https://api.spidra.io/api/crawl/job/{jobId}', options)
  .then(res => res.json())
  .then(res => console.log(res))
  .catch(err => console.error(err));

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.spidra.io/api/crawl/job/{jobId}",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}

package main

import (
	"fmt"
	"net/http"
	"io"
)

func main() {

	url := "https://api.spidra.io/api/crawl/job/{jobId}"

	req, _ := http.NewRequest("GET", url, nil)

	req.Header.Add("Authorization", "Bearer <token>")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}

require 'uri'
require 'net/http'

url = URI("https://api.spidra.io/api/crawl/job/{jobId}")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Get.new(url)
request["Authorization"] = 'Bearer <token>'

response = http.request(request)
puts response.read_body

{
  "id": "abc-123",
  "base_url": "https://example.com/blog",
  "crawl_instruction": "Crawl all blog post pages",
  "transform_instruction": "Extract title, author, and publish date",
  "output_schema": null,
  "max_pages": 10,
  "max_depth": 2,
  "include_paths": [
    "/blog/*"
  ],
  "exclude_paths": [
    "/blog/tag/*"
  ],
  "allow_subdomains": false,
  "crawl_entire_domain": false,
  "ignore_query_params": true,
  "webhook_url": null,
  "status": "completed",
  "pages_crawled": 8,
  "input_tokens": 14820,
  "output_tokens": 3210,
  "credits_used": 25,
  "created_at": "2025-12-17T15:00:00Z",
  "updated_at": "2025-12-17T15:03:42Z"
}

{
  "status": "error",
  "message": "Access token invalid or expired"
}

{
  "status": "error",
  "message": "Crawl job not found"
}

Returns the complete record for a crawl job: the configuration you submitted, current status, token usage, and credit cost. It does not return the extracted page data. For the actual content from each page, use GET /crawl//pages.

When to Use This Endpoint

Inspect the instructions and options that were used for a job
Check token and credit costs for accounting or reporting
Confirm job configuration before re-running extraction with POST /crawl//extract

Example Request

curl https://api.spidra.io/api/crawl/job/abc-123 \
  -H "Authorization: Bearer YOUR_API_KEY"

import requests

response = requests.get(
    "https://api.spidra.io/api/crawl/job/abc-123",
    headers={"Authorization": "Bearer YOUR_API_KEY"}
)

const response = await fetch("https://api.spidra.io/api/crawl/job/abc-123", {
  headers: { Authorization: "Bearer YOUR_API_KEY" }
});

Response Fields

Job configuration

Field	Type	Description
`id`	string	Unique job identifier.
`base_url`	string	The starting URL that was crawled.
`crawl_instruction`	string	The instruction used to decide which pages to follow.
`transform_instruction`	string or null	The prompt used to extract data from each page. `null` if extraction used a schema or if no extraction was configured.
`output_schema`	object or null	The JSON Schema used for structured extraction. `null` if a transform instruction was used instead, or if no extraction was configured.
`max_pages`	integer	The page limit set for this job.
`max_depth`	integer or null	The link depth limit. `null` if not set (unlimited).
`include_paths`	string[] or null	Path patterns that were included. `null` if not set.
`exclude_paths`	string[] or null	Path patterns that were excluded. `null` if not set.
`allow_subdomains`	boolean	Whether subdomain links were followed.
`crawl_entire_domain`	boolean	Whether all paths on the root domain were eligible.
`ignore_query_params`	boolean	Whether URLs differing only by query string were deduplicated.
`webhook_url`	string or null	The webhook URL that received per-page events, if configured.

Status and stats

Field	Type	Description
`status`	string	`waiting`, `active`, `completed`, `failed`, or `cancelled`.
`pages_crawled`	integer	Number of pages successfully processed.
`input_tokens`	number or null	Total input tokens consumed by AI across all pages. `null` when no AI extraction was used.
`output_tokens`	number or null	Total output tokens generated by AI across all pages. `null` when no AI extraction was used.
`credits_used`	number or null	Total credits charged for this job.
`created_at`	string	ISO 8601 timestamp when the job was created.
`updated_at`	string	ISO 8601 timestamp of the last status change.

Example Response

{
  "id": "abc-123",
  "base_url": "https://example.com/blog",
  "crawl_instruction": "Crawl all blog post pages",
  "transform_instruction": "Extract title, author, publish date, and the first 200 words of the body",
  "output_schema": null,
  "max_pages": 10,
  "max_depth": 2,
  "include_paths": ["/blog/*"],
  "exclude_paths": ["/blog/tag/*"],
  "allow_subdomains": false,
  "crawl_entire_domain": false,
  "ignore_query_params": true,
  "webhook_url": null,
  "status": "completed",
  "pages_crawled": 8,
  "input_tokens": 14820,
  "output_tokens": 3210,
  "credits_used": 25,
  "created_at": "2025-12-17T15:00:00Z",
  "updated_at": "2025-12-17T15:03:42Z"
}

For the actual extracted content from each page, call GET /crawl//pages.

Authorizations

Authorization

string

header

required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Path Parameters

jobId

string

required

Response

Job details

string

base_url

string

crawl_instruction

string

transform_instruction

string | null

output_schema

object | null

The JSON Schema used for structured extraction, if one was provided.

max_pages

integer

max_depth

integer | null

include_paths

string[] | null

exclude_paths

string[] | null

allow_subdomains

boolean

crawl_entire_domain

boolean

ignore_query_params

boolean

webhook_url

string | null

pages_crawled

integer

status

enum<string>

Available options:

waiting,

active,

completed,

failed,

cancelled

created_at

string<date-time>

updated_at

string<date-time>

input_tokens

number | null

output_tokens

number | null

credits_used

number | null

Using the API

Scrape Endpoints

Logs

Crawl Endpoints

Account

Get Crawl Job Details

When to Use This Endpoint

Example Request

Response Fields

Job configuration

Status and stats

Example Response

Authorizations

Path Parameters

Response

​When to Use This Endpoint

​Example Request

​Response Fields

​Job configuration

​Status and stats

​Example Response

Authorizations

Path Parameters

Response

When to Use This Endpoint

Example Request

Response Fields

Job configuration

Status and stats

Example Response