Dataset API

1from crawlingo import Dataset
2
3dataset = (
4    Dataset("https://example.com/product/1")
5    .auto_match(True)
6    .field("title", "h1.product-title")
7    .field("price", "span.price-value", extraction_type="price")
8    .field("description", "p.description")
9    .field("availability", ".stock-status", default="unknown")
10    .field("image_url", "img.main", extraction_type="url")
11    .build()
12)
13
14# Access results
15print(dataset.to_dict())
16
17# Export formats
18dataset.to_json("output.json")
19dataset.to_csv("output.csv")
20dataset.to_parquet("output.parquet")

1import { Dataset } from 'crawlingo';
2
3const dataset = new Dataset('https://example.com/product/1')
4  .autoMatch(true)
5  .field('title', 'h1.product-title')
6  .field('price', 'span.price-value', { extractionType: 'price' });
7
8const result = await dataset.build();
9console.log(result.toDict());
10
11await result.toJson('output.json');
12await result.toCsv('output.csv');

1.field(
2    name: str,           # Field name in output
3    selector: str,        # CSS / XPath / Regex selector
4    selector_type="css",  # "css" | "xpath" | "regex" | "text"
5    extraction_type=None, # "text" | "price" | "datetime" | "url" | "datalink_*"
6    default=None          # Fallback value if selector finds nothing
7)

1dataset = Dataset("https://example.com")
2dataset.field("title", "h1")
3
4stream = dataset.build_many_streamed(
5    urls=["https://example.com/a", "https://example.com/b", "https://example.com/c"],
6    concurrency=10
7)
8
9for record in stream:
10    print(record.fields)