Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/browser/cdp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { request as httpsRequest } from 'node:https';
import type { BrowserCookie, IPage, ScreenshotOptions, SnapshotOptions, WaitOptions } from '../types.js';
import type { IBrowserFactory } from '../runtime.js';
import { wrapForEval } from './utils.js';
import { normalizeWaitTimeoutMs } from './wait.js';
import { generateSnapshotJs, scrollToRefJs, getFormStateJs } from './dom-snapshot.js';
import { generateStealthJs } from './stealth.js';
import {
Expand Down Expand Up @@ -267,12 +268,12 @@ class CDPPage implements IPage {
return;
}
if (options.selector) {
const timeout = (options.timeout ?? 10) * 1000;
const timeout = normalizeWaitTimeoutMs(options.timeout, 10);
await this.evaluate(waitForSelectorJs(options.selector, timeout));
return;
}
if (options.text) {
const timeout = (options.timeout ?? 30) * 1000;
const timeout = normalizeWaitTimeoutMs(options.timeout, 30);
await this.evaluate(waitForTextJs(options.text, timeout));
}
}
Expand Down
5 changes: 3 additions & 2 deletions src/browser/page.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { formatSnapshot } from '../snapshotFormatter.js';
import type { BrowserCookie, IPage, ScreenshotOptions, SnapshotOptions, WaitOptions } from '../types.js';
import { sendCommand } from './daemon-client.js';
import { wrapForEval } from './utils.js';
import { normalizeWaitTimeoutMs } from './wait.js';
import { saveBase64ToFile } from '../utils.js';
import { generateSnapshotJs, scrollToRefJs, getFormStateJs } from './dom-snapshot.js';
import { generateStealthJs } from './stealth.js';
Expand Down Expand Up @@ -239,13 +240,13 @@ export class Page implements IPage {
return;
}
if (options.selector) {
const timeout = (options.timeout ?? 10) * 1000;
const timeout = normalizeWaitTimeoutMs(options.timeout, 10);
const code = waitForSelectorJs(options.selector, timeout);
await sendCommand('exec', { code, ...this._cmdOpts() });
return;
}
if (options.text) {
const timeout = (options.timeout ?? 30) * 1000;
const timeout = normalizeWaitTimeoutMs(options.timeout, 30);
const code = waitForTextJs(options.text, timeout);
await sendCommand('exec', { code, ...this._cmdOpts() });
}
Expand Down
23 changes: 23 additions & 0 deletions src/browser/wait.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import { describe, expect, it } from 'vitest';

import { normalizeWaitTimeoutMs } from './wait.js';

describe('normalizeWaitTimeoutMs', () => {
it('uses the default timeout when no timeout is provided', () => {
expect(normalizeWaitTimeoutMs(undefined, 10)).toBe(10_000);
});

it('treats sub-1000 values as seconds', () => {
expect(normalizeWaitTimeoutMs(3, 10)).toBe(3_000);
expect(normalizeWaitTimeoutMs(0.5, 10)).toBe(500);
});

it('treats large values as milliseconds for backward compatibility', () => {
expect(normalizeWaitTimeoutMs(10_000, 10)).toBe(10_000);
});

it('clamps non-positive values to zero', () => {
expect(normalizeWaitTimeoutMs(0, 10)).toBe(0);
expect(normalizeWaitTimeoutMs(-2, 10)).toBe(0);
});
});
17 changes: 17 additions & 0 deletions src/browser/wait.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
export function normalizeWaitTimeoutMs(timeout: number | undefined, defaultSeconds: number): number {
if (typeof timeout !== 'number' || !Number.isFinite(timeout)) {
return defaultSeconds * 1000;
}

if (timeout <= 0) {
return 0;
}

// Older adapters occasionally passed milliseconds here even though
// wait({ timeout }) is documented in seconds. Preserve those callers.
if (timeout >= 1000) {
return Math.round(timeout);
}

return Math.round(timeout * 1000);
}
41 changes: 41 additions & 0 deletions src/clis/36kr/article.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
import { beforeAll, describe, expect, it, vi } from 'vitest';

import { getRegistry } from '../../registry.js';

beforeAll(async () => {
await import('./article.js');
});

describe('36kr/article', () => {
it('waits for capture before scraping the rendered article', async () => {
const command = getRegistry().get('36kr/article');
expect(command?.func).toBeTypeOf('function');

const page = {
installInterceptor: vi.fn().mockResolvedValue(undefined),
goto: vi.fn().mockResolvedValue(undefined),
waitForCapture: vi.fn().mockResolvedValue(undefined),
wait: vi.fn().mockResolvedValue(undefined),
evaluate: vi.fn().mockResolvedValue({
title: 'OpenCLI ships faster 36kr article scraping',
author: 'opencli',
date: '2026-03-31',
body: 'A useful article body',
}),
} as any;

const result = await command!.func!(page, { id: 'https://www.36kr.com/p/321654987' });

expect(page.installInterceptor).toHaveBeenCalledWith('36kr.com/api');
expect(page.goto).toHaveBeenCalledWith('https://www.36kr.com/p/321654987');
expect(page.waitForCapture).toHaveBeenCalledWith(6);
expect(page.wait).toHaveBeenCalledWith({ selector: '.article-title, h1', timeout: 3 });
expect(result).toEqual([
{ field: 'title', value: 'OpenCLI ships faster 36kr article scraping' },
{ field: 'author', value: 'opencli' },
{ field: 'date', value: '2026-03-31' },
{ field: 'url', value: 'https://36kr.com/p/321654987' },
{ field: 'body', value: 'A useful article body' },
]);
});
});
3 changes: 2 additions & 1 deletion src/clis/36kr/article.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ cli({

await page.installInterceptor('36kr.com/api');
await page.goto(`https://www.36kr.com/p/${articleId}`);
await page.wait(5);
await page.waitForCapture(6);
await page.wait({ selector: '.article-title, h1', timeout: 3 });

const data: any = await page.evaluate(`
(() => {
Expand Down
2 changes: 1 addition & 1 deletion src/clis/sinafinance/rolling-news.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ cli({
columns: ['column', 'title', 'date', 'url'],
func: async (page, _args) => {
await page.goto(`https://finance.sina.com.cn/roll/#pageid=384&lid=2519`);
await page.wait({ selector: '.d_list_txt li', timeout: 10000 });
await page.wait({ selector: '.d_list_txt li', timeout: 10 });

const payload = await page.evaluate(`
(() => {
Expand Down
2 changes: 1 addition & 1 deletion src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ export interface WaitOptions {
text?: string;
selector?: string; // wait until document.querySelector(selector) matches
time?: number;
timeout?: number;
timeout?: number; // seconds by default; values >= 1000 are treated as ms for backward compatibility
}

export interface ScreenshotOptions {
Expand Down