NestJS

npm install spidra-nestjs spidra

import { Module } from "@nestjs/common";
import { SpidraModule } from "spidra-nestjs";

@Module({
  imports: [
    SpidraModule.forRoot({
      apiKey: process.env.SPIDRA_API_KEY,
      baseUrl: process.env.SPIDRA_BASE_URL,
    }),
  ],
})
export class AppModule {}

import { Module } from "@nestjs/common";
import { ConfigModule, ConfigService } from "@nestjs/config";
import { SpidraModule } from "spidra-nestjs";

@Module({
  imports: [
    ConfigModule.forRoot(),
    SpidraModule.forRootAsync({
      imports: [ConfigModule],
      useFactory: (config: ConfigService) => ({
        apiKey: config.get<string>("SPIDRA_API_KEY"),
        baseUrl: config.get<string>("SPIDRA_BASE_URL"),
      }),
      inject: [ConfigService],
    }),
  ],
})
export class AppModule {}

import { Injectable } from "@nestjs/common";
import { SpidraService } from "spidra-nestjs";

@Injectable()
export class ScraperService {
  constructor(private readonly spidra: SpidraService) {}

  async scrapeProduct(url: string) {
    return await this.spidra.scrape({
      urls: [{ url }],
      prompt: "Extract product title, price, and availability",
      output: "json",
    });
  }
}

import { Controller, Post, Body, UseGuards } from "@nestjs/common";
import { SpidraWebhookGuard } from "spidra-nestjs";

@Controller("webhooks")
export class WebhookController {
  @Post("spidra")
  @UseGuards(SpidraWebhookGuard)
  handleWebhook(@Body() event: any) {
    console.log("Verified Spidra webhook received:", event.event);

    if (event.event === "crawl.completed") {
      console.log(`Crawl job ${event.jobId} finished!`);
    }

    return { received: true };
  }
}

@Injectable()
export class AuthService {
  constructor(private readonly spidra: SpidraService) {}

  async scrapeAuthenticatedPage(url: string) {
    return await this.spidra.scrape({
      urls: [
        {
          url,
          actions: [
            { type: "type", selector: "#user", value: "admin" },
            { type: "type", selector: "#pass", value: "secret" },
            { type: "click", selector: "button[type='submit']" },
            { type: "waitFor", selector: ".dashboard" },
          ],
        },
      ],
      prompt: "Extract dashboard metrics",
      output: "json",
      useProxy: true,
      proxyCountry: "us",
    });
  }
}

@Injectable()
export class CrawlerService {
  constructor(private readonly spidra: SpidraService) {}

  async crawlDocumentation(domainUrl: string) {
    return await this.spidra.crawl({
      baseUrl: domainUrl,
      crawlInstruction: "Follow guides and documentation pages only",
      transformInstruction: "Extract page title and summary",
      maxPages: 30,
      includePaths: ["/docs/*"],
    });
  }
}

@Injectable()
export class SearchService {
  constructor(private readonly spidra: SpidraService) {}

  async searchLiveWeb(query: string) {
    return await this.spidra.search({
      query,
      sources: ["web", "news"],
      limit: 10,
    });
  }
}

@Injectable()
export class BatchService {
  constructor(private readonly spidra: SpidraService) {}

  async scrapeMultiple(urls: string[]) {
    const batch = await this.spidra.startBatchScrape({
      urls,
      prompt: "Extract title and meta description",
    });

    const watcher = this.spidra.client.watchBatch(batch.batchId);
    watcher.on("item", (item) => {
      console.log(`Item completed: ${item.url}`);
    });

    return batch;
  }
}

@Injectable()
export class MappingService {
  constructor(private readonly spidra: SpidraService) {}

  async mapDomain(url: string) {
    return await this.spidra.map({
      url,
      limit: 100,
    });
  }
}

import { readFile } from "fs/promises";

@Injectable()
export class DocumentService {
  constructor(private readonly spidra: SpidraService) {}

  async parsePdf(filePath: string) {
    const bytes = await readFile(filePath);
    return await this.spidra.parse({
      file: bytes,
      fileName: "report.pdf",
    }, {
      prompt: "Extract invoice total, date, and vendor",
    });
  }
}

@Injectable()
export class LeadService {
  constructor(private readonly spidra: SpidraService) {}

  async getLeads(query: string, area: string) {
    return await this.spidra.runTemplate("google-maps-leads", {
      query,
      areas: [area],
    });
  }

  async setupMonitor(targetUrl: string, webhookUrl: string) {
    return await this.spidra.createMonitor({
      name: "Price Watch",
      sources: [{ kind: "urls", urls: [targetUrl] }],
      schedule: { every: "1d", at: "08:00" },
      alertOn: "page",
      goal: "Alert on any price change",
      notify: { webhook: { url: webhookUrl } },
    });
  }
}

import {
  SpidraAuthenticationError,
  SpidraRateLimitError,
  SpidraPaymentRequiredError,
  SpidraError,
} from "spidra-nestjs";

try {
  await this.spidra.scrape({ urls: [{ url: "https://example.com" }] });
} catch (err) {
  if (err instanceof SpidraAuthenticationError) {
    // 401 Unauthorized
  } else if (err instanceof SpidraRateLimitError) {
    // 429 Rate Limited
  } else if (err instanceof SpidraPaymentRequiredError) {
    // 402 Credits Exhausted
  } else if (err instanceof SpidraError) {
    // 5xx / API error
  }
}