Update: crawlPage API allows crawling pages with cookies

Author: coder-hxl

README.md

@@ -64,11 +64,13 @@ The crawlPage API internally uses the [puppeteer](https://github.com/puppeteer/p
   - [RequestConfig](#RequestConfig)
   - [IntervalTime](#IntervalTime)
   - [XCrawlBaseConfig](#XCrawlBaseConfig)
-  - [CrawlPageConfig](#CrawlPageConfig)
   - [CrawlBaseConfigV1](#CrawlBaseConfigV1)
+  - [CrawlBaseConfigV2](#CrawlBaseConfigV2)
+  - [CrawlPageConfig](#CrawlPageConfig)
   - [CrawlDataConfig](#CrawlDataConfig)
   - [CrawlFileConfig](#CrawlFileConfig)
   - [StartPollingConfig](#StartPollingConfig)
+  - [XCrawlInstance](#XCrawlInstance)
   - [CrawlResCommonV1](#CrawlResCommonV1)
   - [CrawlResCommonArrV1](#CrawlResCommonArrV1)
   - [CrawlPage](#CrawlPage-1)
@@ -747,34 +749,42 @@ interface XCrawlBaseConfig {
 }
 ```
 
-### CrawlPageConfig
+### CrawlBaseConfigV1
 
 ```ts
-type CrawlPageConfig = string | RequestConfigObjectV1
+interface CrawlBaseConfigV1 extends RequestConfigObjectV1 {
+  cookies?: string | Protocol.Network.CookieParam | Protocol.Network.CookieParam[] // The Protocol is from the puppeteer library
+}
 ```
 
-### CrawlBaseConfigV1
+### CrawlBaseConfigV2
 
 ```ts
-interface CrawlBaseConfigV1 {
+interface CrawlBaseConfigV2 {
   requestConfig: RequestConfig | RequestConfig[]
   intervalTime?: IntervalTime
 }
 ```
 
+### CrawlPageConfig
+
+```ts
+type CrawlPageConfig = string | CrawlBaseConfigV1
+```
+
 ### CrawlDataConfig
 
 ```ts
-interface CrawlDataConfig extends CrawlBaseConfigV1 {}
+interface CrawlDataConfig extends CrawlBaseConfigV2 {}
 ```
 
 ### CrawlFileConfig
 
 ```ts
-interface CrawlFileConfig extends CrawlBaseConfigV1 {
+interface CrawlFileConfig extends CrawlBaseConfigV2 {
   fileConfig: {
     storeDir: string // Store folder
-    extension?: string // Filename extension
+    extension?: string // filename extension
   }
 }
 ```
@@ -821,7 +831,7 @@ interface XCrawlInstance {
 interface CrawlResCommonV1<T> {
   id: number
   statusCode: number | undefined
-  headers: IncomingHttpHeaders // nodejs: http type
+  headers: IncomingHttpHeaders // The http is from the nodejs library
   data: T
 }
 ```
@@ -836,10 +846,10 @@ type CrawlResCommonArrV1<T> = CrawlResCommonV1<T>[]
 
 ```ts
 interface CrawlPage {
-  httpResponse: HTTPResponse | null // The type of HTTPResponse in the puppeteer library
-  browser: Browser // The Browser type of the puppeteer library
-  page: Page // The Page type of the puppeteer library
-  jsdom: JSDOM // jsdom type of the JSDOM library
+  httpResponse: HTTPResponse | null // The HTTPResponse is from the puppeteer library
+  browser: Browser // The Browser is from the puppeteer library
+  page: Page // The Page is from the puppeteer library
+  jsdom: JSDOM // The JSDOM is from the jsdom library
 }
 ```
 

docs/cn.md

@@ -64,11 +64,13 @@ crawlPage API 内部使用 [puppeteer](https://github.com/puppeteer/puppeteer) 
   - [RequestConfig](#RequestConfig)
   - [IntervalTime](#IntervalTime)
   - [XCrawlBaseConfig](#XCrawlBaseConfig)
-  - [CrawlPageConfig](#CrawlPageConfig)
   - [CrawlBaseConfigV1](#CrawlBaseConfigV1)
+  - [CrawlBaseConfigV2](#CrawlBaseConfigV2)
+  - [CrawlPageConfig](#CrawlPageConfig)
   - [CrawlDataConfig](#CrawlDataConfig)
   - [CrawlFileConfig](#CrawlFileConfig)
   - [StartPollingConfig](#StartPollingConfig)
+  - [XCrawlInstance](#XCrawlInstance)
   - [CrawlResCommonV1](#CrawlResCommonV1)
   - [CrawlResCommonArrV1](#CrawlResCommonArrV1)
   - [CrawlPage](#CrawlPage-1)
@@ -740,31 +742,39 @@ interface XCrawlBaseConfig {
 }
 ```
 
-### CrawlPageConfig
+### CrawlBaseConfigV1
 
 ```ts
-type CrawlPageConfig = string | RequestConfigObjectV1
+interface CrawlBaseConfigV1 extends RequestConfigObjectV1 {
+  cookies?: string | Protocol.Network.CookieParam | Protocol.Network.CookieParam[] // Protocol 来自 puppeteer 库
+}
 ```
 
-### CrawlBaseConfigV1
+### CrawlBaseConfigV2
 
 ```ts
-interface CrawlBaseConfigV1 {
+interface CrawlBaseConfigV2 {
   requestConfig: RequestConfig | RequestConfig[]
   intervalTime?: IntervalTime
 }
 ```
 
+### CrawlPageConfig
+
+```ts
+type CrawlPageConfig = string | CrawlBaseConfigV1
+```
+
 ### CrawlDataConfig
 
 ```ts
-interface CrawlDataConfig extends CrawlBaseConfigV1 {}
+interface CrawlDataConfig extends CrawlBaseConfigV2 {}
 ```
 
 ### CrawlFileConfig
 
 ```ts
-interface CrawlFileConfig extends CrawlBaseConfigV1 {
+interface CrawlFileConfig extends CrawlBaseConfigV2 {
   fileConfig: {
     storeDir: string // 存放文件夹
     extension?: string // 文件扩展名

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "x-crawl",
-  "version": "3.2.13",
+  "version": "3.3.0",
   "author": "coderHXL",
   "description": "x-crawl is a flexible nodejs crawler library.",
   "license": "MIT",

publish/package.json

@@ -1,6 +1,6 @@
 {
   "name": "x-crawl",
-  "version": "3.2.13",
+  "version": "3.3.0",
   "author": "coderHXL",
   "description": "x-crawl is a flexible nodejs crawler library.",
   "license": "MIT",

src/api.ts

@@ -2,7 +2,7 @@ import fs from 'node:fs'
 import { writeFile } from 'node:fs/promises'
 import path from 'node:path'
 import { JSDOM } from 'jsdom'
-import puppeteer, { Browser } from 'puppeteer'
+import puppeteer, { Browser, Protocol } from 'puppeteer'
 
 import { batchRequest, syncBatchRequest } from './request'
 import { quickSort } from './sort'
@@ -18,6 +18,7 @@ import {
 } from './utils'
 
 import {
+  Cookies,
   CrawlDataConfig,
   CrawlFileConfig,
   CrawlPage,
@@ -93,6 +94,36 @@ async function useBatchRequestByMode(
   }
 }
 
+function parseCrawlPageCookies(
+  url: string,
+  cookies: Cookies
+): Protocol.Network.CookieParam[] {
+  const cookiesArr: Protocol.Network.CookieParam[] = []
+
+  if (typeof cookies === 'string') {
+    cookies.split('; ').forEach((item) => {
+      const cookie = item.split('=')
+      cookiesArr.push({ name: cookie[0], value: cookie[1], url })
+    })
+  } else if (Array.isArray(cookies)) {
+    cookies.forEach((cookie) => {
+      if (!cookie.url) {
+        cookie.url = url
+      }
+
+      cookiesArr.push(cookie)
+    })
+  } else if (typeof cookies === 'object' && cookies) {
+    if (!cookies.url) {
+      cookies.url = url
+    }
+
+    cookiesArr.push(cookies)
+  }
+
+  return cookiesArr
+}
+
 export function createCrawlPage(baseConfig: LoaderXCrawlBaseConfig) {
   let browser: Browser | null = null
   let createBrowserState: Promise<void> | null = null
@@ -143,6 +174,12 @@ export function createCrawlPage(baseConfig: LoaderXCrawlBaseConfig) {
       await page.setExtraHTTPHeaders(Headers as any as Record<string, string>)
     }
 
+    if (requestConfig.cookies) {
+      await page.setCookie(
+        ...parseCrawlPageCookies(requestConfig.url, requestConfig.cookies)
+      )
+    }
+
     let httpResponse = null
     try {
       httpResponse = await page!.goto(requestConfig.url, {

src/types/api.ts

@@ -1,5 +1,5 @@
 import { IncomingHttpHeaders } from 'node:http'
-import { Browser, HTTPResponse, Page } from 'puppeteer'
+import { Browser, HTTPResponse, Page, Protocol } from 'puppeteer'
 import { JSDOM } from 'jsdom'
 
 import { RequestConfigObjectV1, RequestConfigObjectV2 } from './request'
@@ -23,7 +23,7 @@ export type MergeConfigRawConfig = {
 }
 
 export type MergeConfigV1 = {
-  requestConfig: RequestConfigObjectV1[]
+  requestConfig: CrawlBaseConfigV1[]
   intervalTime?: IntervalTime
 }
 
@@ -36,16 +36,25 @@ export type MergeConfigV2<T extends AnyObject> = MapTypeObject<
 }
 
 /* API Config */
-export type CrawlPageConfig = string | RequestConfigObjectV1
+export type Cookies =
+  | string
+  | Protocol.Network.CookieParam
+  | Protocol.Network.CookieParam[]
+
+export interface CrawlBaseConfigV1 extends RequestConfigObjectV1 {
+  cookies?: Cookies
+}
 
-export interface CrawlBaseConfigV1 {
+export interface CrawlBaseConfigV2 {
   requestConfig: RequestConfig | RequestConfig[]
   intervalTime?: IntervalTime
 }
 
-export interface CrawlDataConfig extends CrawlBaseConfigV1 {}
+export type CrawlPageConfig = string | CrawlBaseConfigV1
+
+export interface CrawlDataConfig extends CrawlBaseConfigV2 {}
 
-export interface CrawlFileConfig extends CrawlBaseConfigV1 {
+export interface CrawlFileConfig extends CrawlBaseConfigV2 {
   fileConfig: {
     storeDir: string
     extension?: string