curl --request POST \
--url https://api.spidra.io/api/map \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "https://docs.python.org/3/",
"search": "asyncio",
"limit": 20
}
'import requests
url = "https://api.spidra.io/api/map"
payload = {
"url": "https://docs.python.org/3/",
"search": "asyncio",
"limit": 20
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({url: 'https://docs.python.org/3/', search: 'asyncio', limit: 20})
};
fetch('https://api.spidra.io/api/map', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.spidra.io/api/map",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => 'https://docs.python.org/3/',
'search' => 'asyncio',
'limit' => 20
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.spidra.io/api/map"
payload := strings.NewReader("{\n \"url\": \"https://docs.python.org/3/\",\n \"search\": \"asyncio\",\n \"limit\": 20\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.spidra.io/api/map")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"https://docs.python.org/3/\",\n \"search\": \"asyncio\",\n \"limit\": 20\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.spidra.io/api/map")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"https://docs.python.org/3/\",\n \"search\": \"asyncio\",\n \"limit\": 20\n}"
response = http.request(request)
puts response.read_body{
"status": "success",
"links": [
{
"url": "https://docs.python.org/3/library/asyncio.html",
"title": "asyncio"
},
{
"url": "https://docs.python.org/3/library/asyncio-threading.html",
"title": "asyncio and free-threaded Python"
},
{
"url": "https://docs.python.org/3/howto/a-conceptual-overview-of-asyncio.html",
"title": "A conceptual overview of asyncio"
}
],
"total": 21,
"scanned": 594,
"sources": {
"sitemap": 7,
"pages": 587
},
"creditsUsed": 1,
"durationMs": 4914
}{
"status": "error",
"message": "Access token invalid or expired"
}{
"status": "error",
"message": "You have exceeded your monthly credit limit."
}{
"status": "error",
"message": "<string>"
}{
"status": "error",
"message": "Too many map requests. Please slow down and try again shortly."
}{
"status": "error",
"message": "The site took too long to map. Try again, or use sitemap: \"only\".",
"creditsUsed": 0
}Map a Site
List the page URLs of a website without scraping them, optionally ranked by a search. Answers in the same request for 1 credit.
curl --request POST \
--url https://api.spidra.io/api/map \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"url": "https://docs.python.org/3/",
"search": "asyncio",
"limit": 20
}
'import requests
url = "https://api.spidra.io/api/map"
payload = {
"url": "https://docs.python.org/3/",
"search": "asyncio",
"limit": 20
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({url: 'https://docs.python.org/3/', search: 'asyncio', limit: 20})
};
fetch('https://api.spidra.io/api/map', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.spidra.io/api/map",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'url' => 'https://docs.python.org/3/',
'search' => 'asyncio',
'limit' => 20
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.spidra.io/api/map"
payload := strings.NewReader("{\n \"url\": \"https://docs.python.org/3/\",\n \"search\": \"asyncio\",\n \"limit\": 20\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.spidra.io/api/map")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"url\": \"https://docs.python.org/3/\",\n \"search\": \"asyncio\",\n \"limit\": 20\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.spidra.io/api/map")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"url\": \"https://docs.python.org/3/\",\n \"search\": \"asyncio\",\n \"limit\": 20\n}"
response = http.request(request)
puts response.read_body{
"status": "success",
"links": [
{
"url": "https://docs.python.org/3/library/asyncio.html",
"title": "asyncio"
},
{
"url": "https://docs.python.org/3/library/asyncio-threading.html",
"title": "asyncio and free-threaded Python"
},
{
"url": "https://docs.python.org/3/howto/a-conceptual-overview-of-asyncio.html",
"title": "A conceptual overview of asyncio"
}
],
"total": 21,
"scanned": 594,
"sources": {
"sitemap": 7,
"pages": 587
},
"creditsUsed": 1,
"durationMs": 4914
}{
"status": "error",
"message": "Access token invalid or expired"
}{
"status": "error",
"message": "You have exceeded your monthly credit limit."
}{
"status": "error",
"message": "<string>"
}{
"status": "error",
"message": "Too many map requests. Please slow down and try again shortly."
}{
"status": "error",
"message": "The site took too long to map. Try again, or use sitemap: \"only\".",
"creditsUsed": 0
}Example Request
curl -X POST https://api.spidra.io/api/map \
-H "Authorization: Bearer YOUR_API_KEY" \
-H "Content-Type: application/json" \
-d '{
"url": "https://docs.python.org/3/",
"search": "asyncio",
"limit": 20
}'
import { SpidraClient } from "spidra";
const spidra = new SpidraClient({ apiKey: process.env.SPIDRA_API_KEY });
const { links, total, scanned } = await spidra.map({
url: "https://docs.python.org/3/",
search: "asyncio",
limit: 20,
});
from spidra import Spidra
client = Spidra(api_key="YOUR_API_KEY")
result = client.map("https://docs.python.org/3/", search="asyncio", limit=20)
for link in result.links:
print(link.url, link.title)
Response
A real response, shortened to the first three links:{
"status": "success",
"links": [
{ "url": "https://docs.python.org/3/library/asyncio.html", "title": "asyncio" },
{ "url": "https://docs.python.org/3/library/asyncio-threading.html", "title": "asyncio and free-threaded Python" },
{ "url": "https://docs.python.org/3/howto/a-conceptual-overview-of-asyncio.html", "title": "A conceptual overview of asyncio" }
],
"total": 21,
"scanned": 594,
"sources": { "sitemap": 7, "pages": 587 },
"creditsUsed": 1,
"durationMs": 4914
}
| Field | Description |
|---|---|
links | The URLs, best matches first when you pass search. title is the link text the URL was found with, when it came from a page. URLs from the sitemap only have no title. |
total | How many URLs matched the search. Without a search, every URL found. |
scanned | Every URL found on the site, before the search filter. |
sources | How many URLs came from the sitemap and from page links. |
creditsUsed | 1, or 0 when nothing was found. |
message | Present when no URLs were found, saying why. |
How Map finds URLs
- The sitemap. Spidra reads the
Sitemaplines in the site’s robots.txt, then/sitemap.xmland/sitemap_index.xml, follows sitemap indexes and reads gzipped sitemaps. - The start page. Its links, with their text.
- More pages. Spidra reads up to 40 more pages for links the start page doesn’t show (12 when the sitemap already lists 1,000 URLs or more). With a
search, it reads pages about your search first, because they usually link to the rest of that topic. Without one, it reads index, contents and section pages first.
includeSubdomains), duplicates and links that perform actions such as log out are dropped, and current pages are listed before other versions and translations of them.
Tips
- Use
sitemap: "only"for speed on sites with a complete sitemap. It skips reading pages. - Use
sitemap: "skip"when a sitemap is out of date. - Search in the site’s language.
searchmatches the words in URLs and link text, soBezirkfinds pages on a German site thatdistrictwouldn’t. - No URLs found? The site may block automated visits or build its links with scripts. A crawl with Stealth Mode reads pages in a real browser.
Authorizations
Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Body
The website to map.
Only return URLs related to these words, best matches first. Matches the URL and the link text, in any language.
500Most URLs returned.
1 <= x <= 5000'include' reads the sitemap and the links on the site's pages, 'only' reads the sitemap only (fastest), 'skip' reads links only.
include, only, skip Include URLs on subdomains of the site.
Treat URLs that differ only by query string as one.
Leave out URLs the site's robots.txt disallows for crawlers.
Session cookies, for pages behind a login.
Response
The URLs found. When nothing is found, links is empty, creditsUsed is 0 and message says why.
success Show child attributes
Show child attributes
URLs that matched the search (all URLs found when there is no search).
All URLs found on the site, before the search filter.
How many URLs came from the sitemap and from page links.
Show child attributes
Show child attributes
Was this page helpful?

