Production-ready, battle-tested code recipes for common web scraping and crawling scenarios.
1import crawlingo23session = crawlingo.Session()4dataset = crawlingo.Dataset("https://example.com/products/item-1", session)56dataset.field("title", ".product-detail h1")7dataset.field("price", ".product-detail .price", extract_type="price")8dataset.field("sku", ".product-detail [data-sku]", extract_type="attr:data-sku")910result = dataset.build()11print(result.to_dict())
1import crawlingo23config = crawlingo.PaginationConfig.next_link("a.pagination-next")4crawl = crawlingo.Crawl("https://example.com/blog", crawlingo.Session())5crawl.with_pagination(config).field("title", "article h2")6results = crawl.build()
1import crawlingo23config = crawlingo.PaginationConfig.page_number("https://example.com/list?page={page}", start_page=1, max_pages=10)4crawl = crawlingo.Crawl("https://example.com/list", crawlingo.Session())5crawl.with_pagination(config).field("title", ".item-name")6results = crawl.build()
1import crawlingo23session = crawlingo.Session()45# 1. Bearer Token Auth6session.bearer_auth("my_secret_token_123")78# 2. HTTP Basic Auth9session.basic_auth("user", "pass")1011# 3. Custom API Keys via Headers12session.headers({"X-API-Key": "my-secret-key"})1314# 4. Session Cookies15session.cookies({"session_token": "abcde12345"})
1import crawlingo23session = crawlingo.Session()4watcher = crawlingo.Watch("https://example.com/stock-ticker", session)56watcher.field("price", ".ticker-value", extract_type="price")7watcher.interval(10) # poll every 10 seconds89def on_change(event):10 print(f"Price updated from {event.old_value} to {event.new_value}!")1112watcher.on_change(on_change)
