Crawl API

1from crawlingo import Crawl
2
3results = (
4    Crawl("https://docs.example.com")
5    .follow("a[href^='/docs']")    # CSS selector for links to follow
6    .limit(100)                      # Max pages to crawl
7    .depth(3)                        # Max link depth
8    .concurrency(5)                  # Concurrent requests
9    .delay(1.0)                      # Delay between requests (seconds)
10    .field("title", "h1")
11    .field("content", "main p")
12    .build()
13)
14
15print(f"Crawled {len(results)} pages")
16results.to_json("crawl_output.json")
17results.to_parquet("crawl_output.parquet")

1import { Crawl } from 'crawlingo';
2
3const results = await new Crawl('https://docs.example.com')
4  .follow('a[href^="/docs"]')
5  .limit(100)
6  .depth(3)
7  .concurrency(5)
8  .delay(1.0)
9  .field('title', 'h1')
10  .run();
11
12console.log(`Crawled ${results.length} pages`);

1from crawlingo import Session, Crawl
2
3with Session() as session:
4    session.rate_limit(5.0)  # 5 req/s per host
5    session.proxy_pool(["http://proxy1:8080", "http://proxy2:8080"])
6
7    results = (
8        Crawl("https://example.com", session=session)
9        .follow("a")
10        .limit(100)
11        .delay(0.5)
12        .build()
13    )