Rust SDK

1use crawlingo::{Page, Session, Dataset, DatasetField, ExtractionType};
2use std::sync::Arc;
3
4#[tokio::main]
5async fn main() -> anyhow::Result<()> {
6    // 1. Create a thread-safe Session
7    let session = Arc::new(
8        Session::new()
9            .set_auto_match(true)
10            .set_fetcher_tier("stealthy")
11            .set_rate_limit(5.0)
12    );
13
14    // 2. Fetch a single page
15    let page = Page::new("https://example.com", session.clone()).await?;
16    println!("Title: {:?}", page.title());
17    println!("Status: {}", page.status());
18
19    // 3. Compile a Dataset from a listing page
20    let result = Dataset::new("https://shop.example.com", session.clone())
21        .with_field(DatasetField::new("title", "h1"))
22        .with_field(
23            DatasetField::new("price", ".price")
24                .with_extract_type(ExtractionType::Price)
25                .with_default("0.00")
26        )
27        .build_async()
28        .await?;
29
30    // Print values
31    println!("{:#?}", result.fields);
32
33    // Export dataset
34    result.to_parquet("listings.parquet")?;
35    result.to_json_file("listings.json")?;
36
37    Ok(())
38}