Node.js SDK

1import { Page, Session, Dataset, Crawl, Watch } from 'crawlingo';
2
3// ── Single page ─────────────────────────────────────────────
4const page = await Page.create("https://example.com");
5console.log(page.title());   // "Example Domain"
6console.log(page.status);    // 200
7
8// CSS selectors
9const h1 = page.css("h1").first()?.text;
10const links = [...page.css("a")].map(el => el.attr("href"));
11
12// XPath
13const paragraphs = [...page.xpath("//p")].map(el => el.text);
14
15// ── Session with stealth mode ────────────────────────────────
16const session = new Session()
17  .autoMatch(true)
18  .fetcherTier("stealthy")
19  .browserProfile("chrome")
20  .rateLimit(5);
21
22const stealthPage = await Page.create("https://protected.com", { session });
23
24// ── Dataset ─────────────────────────────────────────────────
25const result = await new Dataset("https://shop.example.com", session)
26  .field("title",  "h1")
27  .field("price",  ".price")
28  .field("rating", ".rating")
29  .build();
30
31console.log(result.toDict());
32await result.toJson("data.json");
33await result.toParquet("data.parquet");
34
35// ── Crawl ────────────────────────────────────────────────────
36const results = await new Crawl("https://docs.example.com", session)
37  .follow("a")
38  .limit(100)
39  .depth(3)
40  .concurrency(8)
41  .field("title",   "h1")
42  .field("content", "article")
43  .run();
44
45Dataset.saveJson(results.map(r => r.toDict()), "crawl.json");
46
47// ── Watch ────────────────────────────────────────────────────
48const watcher = new Watch("https://shop.example.com/p/1", session)
49  .field("price", ".price")
50  .interval(300);
51
52watcher.run((err, e) => {
53  if (err) return;
54  console.log(`${e.field}: ${e.oldValue} → ${e.newValue}`);
55});

1export interface WatchChangeEvent {
2  url:        string;
3  field:      string;
4  changeType: string;
5  oldValue?:  string;
6  newValue?:  string;
7}
8
9export interface DatasetResult {
10  toDict():               Record<string, string>;
11  toJson(path: string):    Promise<void>;
12  toCsv(path: string):     Promise<void>;
13  toParquet(path: string): Promise<void>;
14}
15
16export interface Element {
17  readonly text: string;
18  readonly html: string;
19  attr(name: string): string | null;
20}
21
22export interface ElementList extends Iterable<Element> {
23  first(): Element | null;
24  at(i: number): Element | null;
25  readonly text: string[];
26  readonly html: string[];
27  attr(name: string): (string | null)[];
28  readonly length: number;
29}