Build a RAG knowledge base from a docs site

job = client.crawl(
    "https://docs.python.org/3/tutorial/",
    crawl_instruction="Every page of the tutorial",
    max_pages=10,
)

# docs_to_chunks.py
import json
import os
import re

from spidra import Spidra

client = Spidra(api_key=os.environ["SPIDRA_API_KEY"])

# 1. Crawl the docs section as Markdown (no extraction, so no AI)
job = client.crawl(
    "https://docs.python.org/3/tutorial/",
    crawl_instruction="Every page of the tutorial",
    max_pages=10,
)
print(f"{len(job.result)} pages")

# 2. Split each page into chunks at its headings, skipping menus
def is_menu(section):
    lines = [l.strip() for l in section.splitlines() if l.strip()]
    short_bullets = sum(1 for l in lines if l.startswith(("* ", "- ")) and len(l) < 80)
    return not lines or short_bullets / len(lines) > 0.5

def chunks(markdown, max_chars=2000):
    sections = re.split(r"\n(?=#{1,3} )", markdown)
    for section in sections:
        if is_menu(section):
            continue
        for start in range(0, len(section), max_chars):
            piece = section[start:start + max_chars].strip()
            if len(piece) > 200:
                yield piece

# 3. Save one JSON line per chunk, with its source
count = 0
with open("chunks.jsonl", "w") as out:
    for page in job.result:
        for i, text in enumerate(chunks(page.data)):
            out.write(json.dumps({"url": page.url, "title": page.title, "chunk": i, "text": text}) + "\n")
            count += 1
print(f"{count} chunks")

10 pages
167 chunks

{"url": "https://docs.python.org/3/tutorial/appetite.html", "title": "1. Whetting Your Appetite — Python 3.15.1 documentation", "chunk": 0, "text": "# 1\\. Whetting Your Appetite\n\nIf you do much work on computers, eventually you find that there’s some task you’d like to automate. For example, you may wish to perform a search-and-replace over a large number of text files..."}