Handle failed pages and retries

def submit(urls, attempts=5):
    """Start a batch, backing off if Spidra says to slow down or is busy."""
    for attempt in range(attempts):
        try:
            return client.start_batch_scrape(urls, output="markdown")
        except (SpidraRateLimitError, SpidraServerError) as err:
            wait = 2 ** attempt
            print(f"{err.status}: retrying in {wait}s")
            time.sleep(wait)
    raise RuntimeError("Spidra stayed busy; try again later")

# retries.py
import os
import time

from spidra import Spidra, SpidraRateLimitError, SpidraServerError

client = Spidra(api_key=os.environ["SPIDRA_API_KEY"])

urls = [
    "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html",
    "https://books.toscrape.com/catalogue/this-page-does-not-exist_000/index.html",
    "https://quotes.toscrape.com/",
]


def submit(urls, attempts=5):
    """Start a batch, backing off if Spidra says to slow down or is busy."""
    for attempt in range(attempts):
        try:
            return client.start_batch_scrape(urls, output="markdown")
        except (SpidraRateLimitError, SpidraServerError) as err:
            wait = 2 ** attempt
            print(f"{err.status}: retrying in {wait}s")
            time.sleep(wait)
    raise RuntimeError("Spidra stayed busy; try again later")


def wait_for(batch_id):
    while True:
        time.sleep(3)
        batch = client.get_batch_scrape(batch_id)
        if batch.status in ("completed", "cancelled"):
            return batch


batch_id = submit(urls).batch_id
batch = wait_for(batch_id)
print(f"{batch.completed_count} done, {batch.failed_count} failed")

for item in batch.items:
    if item.status == "failed":
        print(f"  {item.url}\n  {item.error}")

# Retry once: this fixes pages that failed for a passing reason (a timeout, a busy site)
if batch.failed_count:
    print(f"Retrying {client.retry_batch_scrape(batch_id)} item(s)")
    batch = wait_for(batch_id)
    print(f"{batch.completed_count} done, {batch.failed_count} failed")

2 done, 1 failed
  https://books.toscrape.com/catalogue/this-page-does-not-exist_000/index.html
  Failed to scrape https://books.toscrape.com/catalogue/this-page-does-not-exist_000/index.html: The page was not delivered: the page answered HTTP 404. Nothing was charged. Check the URL.
Retrying 1 item(s)
2 done, 1 failed