@radzor/web-scraper — LLM & AI Component | Radzor Registry

@radzor/web-scraper

Web page scraping with CSS selector extraction, pagination support, and rate limiting. Fetches HTML content and extracts structured data using selectors.

Data & Databasev0.1.0typescriptpythonServerscraperwebhtmlcss-selectorextractioncrawlby Radzor

Install

View source on GitHub →

$npx radzor@latest add web-scraper

⚠ Constraints: Respects robots.txt conventions. Use responsibly — excessive scraping may violate ToS. Server-side only.

Inputs

Name	Type	Default	Description
userAgent	string	RadzorBot/1.0	User-Agent string for requests.
timeout	number	30000	Request timeout in milliseconds.
rateLimit	number	1000	Minimum delay between requests in milliseconds.

userAgentstring

User-Agent string for requests.

default: RadzorBot/1.0

timeoutnumber

Request timeout in milliseconds.

default: 30000

rateLimitnumber

Minimum delay between requests in milliseconds.

Outputs

Name	Type	Description
scrapeResult	ScrapeResult	Extracted data with selected elements and raw HTML. .urlstring .statusnumber .dataRecord<string, string[]> .htmlstring

scrapeResultScrapeResult

Extracted data with selected elements and raw HTML.

.urlstring

.statusnumber

.dataRecord<string, string[]>

.htmlstring

radzor.manifest.json

{
  "llm": {
    "constraints": "Respects robots.txt conventions. Use responsibly — excessive scraping may violate ToS. Server-side only.",
    "usageExamples": "llm/examples.md",
    "integrationPrompt": "llm/integration.md"
  },
  "name": "@radzor/web-scraper",
  "tags": [
    "scraper",
    "web",
    "html",
    "css-selector",
    "extraction",
    "crawl"
  ],
  "events": [
    {
      "name": "onPageFetched",
      "payload": {
        "url": "string",
        "size": "number",
        "status": "number"
      },
      "description": "Fired when a page is fetched."
    },
    {
      "name": "onError",
      "payload": {
        "url": "string",
        "code": "string",
        "message": "string"
      },
      "description": "Fired on fetch or parse errors."
    }
  ],
  "inputs": [
    {
      "name": "userAgent",
      "type": "string",
      "default": "RadzorBot/1.0",
      "required": false,
      "description": "User-Agent string for requests."
    },
    {
      "name": "timeout",
      "type": "number",
      "default": 30000,
      "required": false,
      "description": "Request timeout in milliseconds."
    },
    {
      "name": "rateLimit",
      "type": "number",
      "default": 1000,
      "required": false,
      "description": "Minimum delay between requests in milliseconds."
    }
  ],
  "radzor": "1.0.0",
  "actions": [
    {
      "name": "scrape",
      "params": [
        {
          "name": "url",
          "type": "string",
          "description": "URL to scrape."
        },
        {
          "name": "selectors",
          "type": "Record<string, string>",
          "description": "Named CSS selectors to extract (e.g. { title: 'h1', prices: '.price' })."
        }
      ],
      "returns": "Promise<ScrapeResult>",
      "description": "Fetch a page and extract data using CSS selectors."
    },
    {
      "name": "fetchHtml",
      "params": [
        {
          "name": "url",
          "type": "string",
          "description": "URL to fetch."
        }
      ],
      "returns": "Promise<string>",
      "description": "Fetch raw HTML content from a URL."
    }
  ],
  "outputs": [
    {
      "name": "scrapeResult",
      "type": "ScrapeResult",
      "fields": [
        {
          "name": "url",
          "type": "string"
        },
        {
          "name": "status",
          "type": "number"
        },
        {
          "name": "data",
          "type": "Record<string, string[]>"
        },
        {
          "name": "html",
          "type": "string"
        }
      ],
      "description": "Extracted data with selected elements and raw HTML."
    }
  ],
  "runtime": "server",
  "version": "0.1.0",
  "category": "data",
  "languages": [
    "typescript",
    "python"
  ],
  "description": "Web page scraping with CSS selector extraction, pagination support, and rate limiting. Fetches HTML content and extracts structured data using selectors.",
  "dependencies": {
    "radzor": [],
    "packages": {}
  },
  "composability": {
    "connectsTo": [
      {
        "output": "scrapeResult",
        "mapField": "html",
        "compatibleWith": [
          "@radzor/search-index.action.index.documents",
          "@radzor/llm-completion.action.complete.prompt",
          "@radzor/embeddings-store.action.add.text",
          "@radzor/structured-output.action.extract.text"
        ]
      },
      {
        "output": "scrapeResult",
        "mapField": "data",
        "description": "Export scraped data to CSV or generate PDF from HTML.",
        "compatibleWith": [
          "@radzor/csv-export.action.generate.data",
          "@radzor/pdf-generate.action.fromHtml.html"
        ]
      }
    ]
  }
}

@radzor/web-scraper

Install

Inputs

Outputs

Actions

Events

Composability

radzor.manifest.json

Version History