chore(bidi): add support for fetching request bodies

Author: hbenl

packages/playwright-core/src/server/bidi/bidiBrowser.ts

@@ -74,7 +74,7 @@ export class BidiBrowser extends Browser {
     });
 
     await browser._browserSession.send('network.addDataCollector', {
-      dataTypes: [bidi.Network.DataType.Response],
+      dataTypes: [bidi.Network.DataType.Request, bidi.Network.DataType.Response],
       maxEncodedDataSize: 20_000_000, // same default as in CDP: https://source.chromium.org/chromium/chromium/src/+/main:third_party/blink/renderer/core/inspector/inspector_network_agent.cc;l=134;drc=4128411589187a396829a827f59a655bed876aa7
     });
 

packages/playwright-core/src/server/bidi/bidiNetworkManager.ts

@@ -79,7 +79,8 @@ export class BidiNetworkManager {
         route = new BidiRouteImpl(this._session, param.request.request);
       }
     }
-    const request = new BidiRequest(frame, redirectedFrom, param, route);
+    const getRequestBody = param.request.bodySize ? () => getNetworkData(this._session, param.request, bidi.Network.DataType.Request) : null;
+    const request = new BidiRequest(frame, redirectedFrom, param, getRequestBody, route);
     this._requests.set(request._id, request);
     this._page.frameManager.requestStarted(request.request, route);
   }
@@ -88,11 +89,7 @@ export class BidiNetworkManager {
     const request = this._requests.get(params.request.request);
     if (!request)
       return;
-    const getResponseBody = async () => {
-      const { bytes } = await this._session.send('network.getData', { request: params.request.request, dataType: bidi.Network.DataType.Response });
-      const encoding = bytes.type === 'base64' ? 'base64' : 'utf8';
-      return Buffer.from(bytes.value, encoding);
-    };
+    const getResponseBody = () => getNetworkData(this._session, params.request, bidi.Network.DataType.Response);
     const timings = params.request.timings;
     const startTime = timings.requestTime;
     function relativeToStart(time: number): number {
@@ -236,14 +233,12 @@ class BidiRequest {
   // store the first and only Route in the chain (if any).
   _originalRequestRoute: BidiRouteImpl | undefined;
 
-  constructor(frame: frames.Frame, redirectedFrom: BidiRequest | null, payload: bidi.Network.BeforeRequestSentParameters, route: BidiRouteImpl | undefined) {
+  constructor(frame: frames.Frame, redirectedFrom: BidiRequest | null, payload: bidi.Network.BeforeRequestSentParameters, getRequestBody: (() => Promise<Buffer>) | null, route: BidiRouteImpl | undefined) {
     this._id = payload.request.request;
     if (redirectedFrom)
       redirectedFrom._redirectedTo = this;
-    // TODO: missing in the spec?
-    const postDataBuffer = null;
     this.request = new network.Request(frame._page.browserContext, frame, null, redirectedFrom ? redirectedFrom.request : null, payload.navigation ?? undefined, payload.request.url,
-        resourceTypeFromBidi(payload.request.destination, payload.request.initiatorType, payload.initiator?.type), payload.request.method, postDataBuffer, fromBidiHeaders(payload.request.headers));
+        resourceTypeFromBidi(payload.request.destination, payload.request.initiatorType, payload.initiator?.type), payload.request.method, null, fromBidiHeaders(payload.request.headers), getRequestBody);
     // "raw" headers are the same as "provisional" headers in Bidi.
     this.request.setRawRequestHeaders(null);
     this.request._setBodySize(payload.request.bodySize || 0);
@@ -390,3 +385,9 @@ function resourceTypeFromBidi(requestDestination: string, requestInitiatorType:
     default: return 'other';
   }
 }
+
+async function getNetworkData(session: BidiSession, request: bidi.Network.RequestData, dataType: bidi.Network.DataType) {
+  const { bytes } = await session.send('network.getData', { request: request.request, dataType });
+  const encoding = bytes.type === 'base64' ? 'base64' : 'utf8';
+  return Buffer.from(bytes.value, encoding);
+}

packages/playwright-core/src/server/bidi/third_party/bidiProtocolCore.ts

@@ -1383,6 +1383,7 @@ export namespace Network {
 }
 export namespace Network {
   export const enum DataType {
+    Request = 'request',
     Response = 'response',
   }
 }

packages/playwright-core/src/server/dispatchers/networkDispatchers.ts

@@ -67,7 +67,7 @@ export class RequestDispatcher extends Dispatcher<Request, channels.RequestChann
   }
 
   async body(params: channels.RequestBodyParams, progress: Progress): Promise<channels.RequestBodyResult> {
-    const postData = this._object.postDataBuffer();
+    const postData = await this._object.body();
     return { body: postData === null ? undefined : postData };
   }
 

packages/playwright-core/src/server/network.ts

@@ -105,6 +105,7 @@ export class Request extends SdkObject {
   private _resourceType: string;
   private _method: string;
   private _postData: Buffer | null;
+  private _bodyCallback: (() => Promise<Buffer>) | null;
   readonly _headers: HeadersArray;
   private _headersMap = new Map<string, string>();
   readonly _frame: frames.Frame | null = null;
@@ -122,7 +123,7 @@ export class Request extends SdkObject {
   };
 
   constructor(context: contexts.BrowserContext, frame: frames.Frame | null, serviceWorker: pages.Worker | null, redirectedFrom: Request | null, documentId: string | undefined,
-    url: string, resourceType: string, method: string, postData: Buffer | null, headers: HeadersArray) {
+    url: string, resourceType: string, method: string, postData: Buffer | null, headers: HeadersArray, bodyCallback: (() => Promise<Buffer>) | null = null) {
     super(frame || context, 'request');
     assert(!url.startsWith('data:'), 'Data urls should not fire requests');
     this._context = context;
@@ -136,6 +137,7 @@ export class Request extends SdkObject {
     this._resourceType = resourceType;
     this._method = method;
     this._postData = postData;
+    this._bodyCallback = bodyCallback;
     this._headers = headers;
     this._updateHeadersMap();
     this._isFavicon = url.endsWith('/favicon.ico') || !!redirectedFrom?._isFavicon;
@@ -173,6 +175,15 @@ export class Request extends SdkObject {
     return this._overrides?.method || this._method;
   }
 
+  async body(): Promise<Buffer | null> {
+    if (this._overrides?.postData)
+      return this._overrides?.postData;
+    if (this._bodyCallback)
+      return await this._bodyCallback();
+    else
+      return this._postData;
+  }
+
   postDataBuffer(): Buffer | null {
     return this._overrides?.postData || this._postData;
   }

tests/library/browsercontext-har.spec.ts

@@ -342,7 +342,7 @@ it('should record overridden requests to har', async ({ contextFactory, server }
   await page1.route('**/echo_redir', async route => {
     await route.fallback({
       url: server.PREFIX + '/echo',
-      postData: +route.request().postData() + 10,
+      postData: +(await route.request().body()) + 10,
     });
   });
   expect(await page1.evaluate(fetchFunction, { path: '/echo_redir', body: '1' })).toBe('11');

tests/library/browsercontext-route.spec.ts

@@ -21,13 +21,13 @@ import type { Route } from '@playwright/test';
 it('should intercept', async ({ browser, server }) => {
   const context = await browser.newContext();
   let intercepted = false;
-  await context.route('**/empty.html', route => {
+  await context.route('**/empty.html', async route => {
     intercepted = true;
     const request = route.request();
     expect(request.url()).toContain('empty.html');
     expect(request.headers()['user-agent']).toBeTruthy();
     expect(request.method()).toBe('GET');
-    expect(request.postData()).toBe(null);
+    expect(await request.body()).toBe(null);
     expect(request.isNavigationRequest()).toBe(true);
     expect(request.resourceType()).toBe('document');
     expect(request.frame() === page.mainFrame()).toBe(true);

tests/page/network-post-data.spec.ts

@@ -16,7 +16,9 @@
 
 import { test as it, expect } from './pageTest';
 
-it('should return correct postData buffer for utf-8 body', async ({ page, server }) => {
+it.skip(({ channel }) => channel?.startsWith('bidi-chrom') || channel?.startsWith('moz-firefox'), 'request.postData is not supported with BiDi');
+
+it('should return correct postData buffer for utf-8 body', async ({ page, server, channel }) => {
   await page.goto(server.EMPTY_PAGE);
   const value = 'baẞ';
   const [request] = await Promise.all([
@@ -34,7 +36,7 @@ it('should return correct postData buffer for utf-8 body', async ({ page, server
   expect(request.postDataJSON()).toBe(value);
 });
 
-it('should return post data w/o content-type @smoke', async ({ page, server }) => {
+it('should return post data w/o content-type @smoke', async ({ page, server, channel }) => {
   await page.goto(server.EMPTY_PAGE);
   const [request] = await Promise.all([
     page.waitForRequest('**'),
@@ -50,7 +52,7 @@ it('should return post data w/o content-type @smoke', async ({ page, server }) =
   expect(request.postDataJSON()).toEqual({ value: 42 });
 });
 
-it('should throw on invalid JSON in post data', async ({ page, server }) => {
+it('should throw on invalid JSON in post data', async ({ page, server, channel }) => {
   await page.goto(server.EMPTY_PAGE);
   const [request] = await Promise.all([
     page.waitForRequest('**'),
@@ -71,7 +73,7 @@ it('should throw on invalid JSON in post data', async ({ page, server }) => {
   expect(error.message).toContain('POST data is not a valid JSON object: <not a json>');
 });
 
-it('should return post data for PUT requests', async ({ page, server }) => {
+it('should return post data for PUT requests', async ({ page, server, channel }) => {
   await page.goto(server.EMPTY_PAGE);
   const [request] = await Promise.all([
     page.waitForRequest('**'),
@@ -86,7 +88,7 @@ it('should return post data for PUT requests', async ({ page, server }) => {
   expect(request.postDataJSON()).toEqual({ value: 42 });
 });
 
-it('should get post data for file/blob', async ({ page, server, browserName }) => {
+it('should get post data for file/blob', async ({ page, server, browserName, channel }) => {
   it.fail(browserName === 'webkit' || browserName === 'chromium');
   await page.goto(server.EMPTY_PAGE);
   const [request] = await Promise.all([
@@ -106,7 +108,7 @@ it('should get post data for file/blob', async ({ page, server, browserName }) =
   expect(request.postData()).toBe('file-contents');
 });
 
-it('should get post data for navigator.sendBeacon api calls', async ({ page, server, browserName }) => {
+it('should get post data for navigator.sendBeacon api calls', async ({ page, server, browserName, channel }) => {
   it.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/12231' });
   it.fail(browserName === 'chromium', 'postData is empty');
   it.fail(browserName === 'webkit', 'postData is empty');

tests/page/page-network-request.spec.ts

@@ -214,7 +214,7 @@ it('should return postData', async ({ page, server }) => {
   page.on('request', r => request = r);
   await page.evaluate(() => fetch('./post', { method: 'POST', body: JSON.stringify({ foo: 'bar' }) }));
   expect(request).toBeTruthy();
-  expect(request.postData()).toBe('{"foo":"bar"}');
+  expect(await request.body()).toBe('{"foo":"bar"}');
 });
 
 it('should work with binary post data', async ({ page, server }) => {
@@ -226,7 +226,7 @@ it('should work with binary post data', async ({ page, server }) => {
     await fetch('./post', { method: 'POST', body: new Uint8Array(Array.from(Array(256).keys())) });
   });
   expect(request).toBeTruthy();
-  const buffer = request.postDataBuffer();
+  const buffer = await request.bodyBuffer();
   expect(buffer.length).toBe(256);
   for (let i = 0; i < 256; ++i)
     expect(buffer[i]).toBe(i);
@@ -242,7 +242,7 @@ it('should work with binary post data and interception', async ({ page, server }
     await fetch('./post', { method: 'POST', body: new Uint8Array(Array.from(Array(256).keys())) });
   });
   expect(request).toBeTruthy();
-  const buffer = request.postDataBuffer();
+  const buffer = await request.bodyBuffer();
   expect(buffer.length).toBe(256);
   for (let i = 0; i < 256; ++i)
     expect(buffer[i]).toBe(i);
@@ -255,12 +255,12 @@ it('should override post data content type', async ({ page, server }) => {
     request = req;
     res.end();
   });
-  await page.route('**/post', (route, request) => {
+  await page.route('**/post', async (route, request) => {
     const headers = request.headers();
     headers['content-type'] = 'application/x-www-form-urlencoded; charset=UTF-8';
     void route.continue({
       headers,
-      postData: request.postData()
+      postData: await request.body()
     });
   });
   await page.evaluate(async () => {
@@ -270,9 +270,9 @@ it('should override post data content type', async ({ page, server }) => {
   expect(request.headers['content-type']).toBe('application/x-www-form-urlencoded; charset=UTF-8');
 });
 
-it('should get |undefined| with postData() when there is no post data', async ({ page, server }) => {
+it('should get |undefined| with body() when there is no post data', async ({ page, server }) => {
   const response = await page.goto(server.EMPTY_PAGE);
-  expect(response.request().postData()).toBe(null);
+  expect(await response.request().body()).toBe(null);
 });
 
 it('should parse the json post data', async ({ page, server }) => {
@@ -282,7 +282,7 @@ it('should parse the json post data', async ({ page, server }) => {
   page.on('request', r => request = r);
   await page.evaluate(() => fetch('./post', { method: 'POST', body: JSON.stringify({ foo: 'bar' }) }));
   expect(request).toBeTruthy();
-  expect(request.postDataJSON()).toEqual({ 'foo': 'bar' });
+  expect(await request.bodyJSON()).toEqual({ 'foo': 'bar' });
 });
 
 it('should parse the data if content-type is application/x-www-form-urlencoded', async ({ page, server }) => {
@@ -293,7 +293,7 @@ it('should parse the data if content-type is application/x-www-form-urlencoded',
   await page.setContent(`<form method='POST' action='/post'><input type='text' name='foo' value='bar'><input type='number' name='baz' value='123'><input type='submit'></form>`);
   await page.click('input[type=submit]');
   expect(request).toBeTruthy();
-  expect(request.postDataJSON()).toEqual({ 'foo': 'bar', 'baz': '123' });
+  expect(await request.bodyJSON()).toEqual({ 'foo': 'bar', 'baz': '123' });
 });
 
 it('should parse the data if content-type is application/x-www-form-urlencoded; charset=UTF-8', async ({ page, server }) => {
@@ -307,12 +307,12 @@ it('should parse the data if content-type is application/x-www-form-urlencoded;
     },
     body: 'foo=bar&baz=123'
   }));
-  expect((await requestPromise).postDataJSON()).toEqual({ 'foo': 'bar', 'baz': '123' });
+  expect(await (await requestPromise).bodyJSON()).toEqual({ 'foo': 'bar', 'baz': '123' });
 });
 
-it('should get |undefined| with postDataJSON() when there is no post data', async ({ page, server }) => {
+it('should get |undefined| with bodyJSON() when there is no post data', async ({ page, server }) => {
   const response = await page.goto(server.EMPTY_PAGE);
-  expect(response.request().postDataJSON()).toBe(null);
+  expect(await response.request().bodyJSON()).toBe(null);
 });
 
 it('should return multipart/form-data', async ({ page, server, browserName, browserMajorVersion }) => {
@@ -337,7 +337,7 @@ it('should return multipart/form-data', async ({ page, server, browserName, brow
   expect(contentType).toMatch(re);
   const b = contentType.match(re)[1]!;
   const expected = `--${b}\r\nContent-Disposition: form-data; name=\"name1\"\r\n\r\nvalue1\r\n--${b}\r\nContent-Disposition: form-data; name=\"file\"; filename=\"foo.txt\"\r\nContent-Type: application/octet-stream\r\n\r\nfile-value\r\n--${b}\r\nContent-Disposition: form-data; name=\"name2\"\r\n\r\nvalue2\r\n--${b}\r\nContent-Disposition: form-data; name=\"name2\"\r\n\r\nanother-value2\r\n--${b}--\r\n`;
-  expect(request.postDataBuffer().toString('utf8')).toEqual(expected);
+  expect((await request.bodyBuffer()).toString('utf8')).toEqual(expected);
 });
 
 it('should return event source', async ({ page, server }) => {

tests/page/page-request-intercept.spec.ts

@@ -191,7 +191,7 @@ it('should intercept multipart/form-data request body', async ({ page, server, a
     page.click('input[type=submit]', { noWaitAfter: true })
   ]);
   expect(request.method()).toBe('POST');
-  expect(request.postData()).toContain(fs.readFileSync(filePath, 'utf8'));
+  expect(await request.body()).toContain(fs.readFileSync(filePath, 'utf8'));
 });
 
 it('should fulfill intercepted response using alias', async ({ page, server, isElectron, electronMajorVersion, isAndroid }) => {
@@ -307,7 +307,7 @@ it('request.postData is not null when fetching FormData with a Blob', {
   const postDataPromise = new Promise<string>(resolve => resolvePostData = resolve);
   await page.route(server.PREFIX + '/upload', async (route, request) => {
     expect(request.method()).toBe('POST');
-    resolvePostData(await request.postData());
+    resolvePostData(await request.body());
     await route.fulfill({
       status: 200,
       body: 'ok',