feat: page.requests()

docs/src/api/class-page.md

@@ -3137,6 +3137,13 @@ return value resolves to `[]`.
 * since: v1.9
 
 
+## async method: Page.requests
+* since: v1.56
+- returns: <[Array]<[Request]>>
+
+Returns up to 100 last network request from this page. See [`event: Page.request`] for more details.
+
+
 ## async method: Page.addLocatorHandler
 * since: v1.42
 

packages/playwright-client/types/types.d.ts

@@ -3926,6 +3926,12 @@ export interface Page {
    */
   requestGC(): Promise<void>;
 
+  /**
+   * Returns up to 100 last network request from this page. See
+   * [page.on('request')](https://playwright.dev/docs/api/class-page#page-event-request) for more details.
+   */
+  requests(): Promise<Array<Request>>;
+
   /**
    * Routing provides the capability to modify network requests that are made by a page.
    *

packages/playwright-core/src/client/page.ts

@@ -29,7 +29,7 @@ import { Frame, verifyLoadState } from './frame';
 import { HarRouter } from './harRouter';
 import { Keyboard, Mouse, Touchscreen } from './input';
 import { JSHandle, assertMaxArguments, parseResult, serializeArgument } from './jsHandle';
-import { Response, Route, RouteHandler, WebSocket,  WebSocketRoute, WebSocketRouteHandler, validateHeaders } from './network';
+import { Request, Response, Route, RouteHandler, WebSocket,  WebSocketRoute, WebSocketRouteHandler, validateHeaders } from './network';
 import { Video } from './video';
 import { Waiter } from './waiter';
 import { Worker } from './worker';
@@ -48,7 +48,7 @@ import type { Clock } from './clock';
 import type { APIRequestContext } from './fetch';
 import type { WaitForNavigationOptions } from './frame';
 import type { FrameLocator, Locator, LocatorOptions } from './locator';
-import type { Request, RouteHandlerCallback, WebSocketRouteHandlerCallback } from './network';
+import type { RouteHandlerCallback, WebSocketRouteHandlerCallback } from './network';
 import type { FilePayload, Headers, LifecycleEvent, SelectOption, SelectOptionOptions, Size, TimeoutOptions, WaitForEventOptions, WaitForFunctionOptions } from './types';
 import type * as structs from '../../types/structs';
 import type * as api from '../../types/types';
@@ -804,6 +804,11 @@ export class Page extends ChannelOwner<channels.PageChannel> implements api.Page
     return await this._mainFrame.waitForFunction(pageFunction, arg, options);
   }
 
+  async requests() {
+    const { requests } = await this._channel.requests();
+    return requests.map(request => Request.from(request));
+  }
+
   workers(): Worker[] {
     return [...this._workers];
   }

packages/playwright-core/src/protocol/validator.ts

@@ -1481,6 +1481,10 @@ scheme.PagePdfParams = tObject({
 scheme.PagePdfResult = tObject({
   pdf: tBinary,
 });
+scheme.PageRequestsParams = tOptional(tObject({}));
+scheme.PageRequestsResult = tObject({
+  requests: tArray(tChannel(['Request'])),
+});
 scheme.PageSnapshotForAIParams = tObject({
   timeout: tFloat,
 });

packages/playwright-core/src/server/browserContext.ts

@@ -62,6 +62,7 @@ export abstract class BrowserContext extends SdkObject {
     RequestAborted: 'requestaborted',
     RequestFulfilled: 'requestfulfilled',
     RequestContinued: 'requestcontinued',
+    RequestCollected: 'requestcollected',
     BeforeClose: 'beforeclose',
     VideoStarted: 'videostarted',
     RecorderEvent: 'recorderevent',

packages/playwright-core/src/server/dispatchers/browserContextDispatcher.ts

@@ -154,6 +154,7 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
       if (!redirectFromDispatcher && !this._shouldDispatchNetworkEvent(request, 'request') && !request.isNavigationRequest())
         return;
       const requestDispatcher = RequestDispatcher.from(this, request);
+      requestDispatcher.reportedThroughEvent = true;
       this._dispatchEvent('request', {
         request: requestDispatcher,
         page: PageDispatcher.fromNullable(this, request.frame()?._page.initializedOrUndefined())
@@ -190,6 +191,11 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
         page: PageDispatcher.fromNullable(this, request.frame()?._page.initializedOrUndefined()),
       });
     });
+    this.addObjectListener(BrowserContext.Events.RequestCollected, (request: Request) => {
+      const requestDispatcher = this.connection.existingDispatcher<RequestDispatcher>(request);
+      if (requestDispatcher && !requestDispatcher.reportedThroughEvent)
+        requestDispatcher._dispose('gc');
+    });
     this.addObjectListener(BrowserContext.Events.RecorderEvent, ({ event, data, page, code }: { event: 'actionAdded' | 'actionUpdated' | 'signalAdded', data: any, page: Page, code: string }) => {
       this._dispatchEvent('recorderEvent', { event, data, code, page: PageDispatcher.from(this, page) });
     });

packages/playwright-core/src/server/dispatchers/networkDispatchers.ts

@@ -32,6 +32,7 @@ import type { Progress } from '@protocol/progress';
 export class RequestDispatcher extends Dispatcher<Request, channels.RequestChannel, BrowserContextDispatcher | PageDispatcher | FrameDispatcher> implements channels.RequestChannel {
   _type_Request: boolean;
   private _browserContextDispatcher: BrowserContextDispatcher;
+  reportedThroughEvent = false;
 
   static from(scope: BrowserContextDispatcher, request: Request): RequestDispatcher {
     const result = scope.connection.existingDispatcher<RequestDispatcher>(request);
@@ -48,9 +49,9 @@ export class RequestDispatcher extends Dispatcher<Request, channels.RequestChann
     const frame = request.frame();
     const page = request.frame()?._page;
     const pageDispatcher = page ? scope.connection.existingDispatcher<PageDispatcher>(page) : null;
-    const frameDispatcher = frame ? FrameDispatcher.from(scope, frame) : null;
+    const frameDispatcher = FrameDispatcher.fromNullable(scope, frame);
     super(pageDispatcher || frameDispatcher || scope, request, 'Request', {
-      frame: FrameDispatcher.fromNullable(scope, request.frame()),
+      frame: frameDispatcher,
       serviceWorker: WorkerDispatcher.fromNullable(scope, request.serviceWorker()),
       url: request.url(),
       resourceType: request.resourceType(),

packages/playwright-core/src/server/dispatchers/pageDispatcher.ts

@@ -339,6 +339,10 @@ export class PageDispatcher extends Dispatcher<Page, channels.PageChannel, Brows
     return { pdf: buffer };
   }
 
+  async requests(params: channels.PageRequestsParams, progress: Progress): Promise<channels.PageRequestsResult> {
+    return { requests: this._page.networkRequests().map(request => RequestDispatcher.from(this.parentScope(), request)) };
+  }
+
   async snapshotForAI(params: channels.PageSnapshotForAIParams, progress: Progress): Promise<channels.PageSnapshotForAIResult> {
     return { snapshot: await this._page.snapshotForAI(progress) };
   }

packages/playwright-core/src/server/frames.ts

@@ -302,6 +302,7 @@ export class FrameManager {
       route?.abort('aborted').catch(() => {});
       return;
     }
+    this._page.addNetworkRequest(request);
     this._page.emitOnContext(BrowserContext.Events.Request, request);
     if (route)
       new network.Route(request, route).handle([...this._page.requestInterceptors, ...this._page.browserContext.requestInterceptors]);

packages/playwright-core/src/server/page.ts

@@ -166,6 +166,7 @@ export class Page extends SdkObject {
   private _locatorHandlers = new Map<number, { selector: string, noWaitAfter?: boolean, resolved?: ManualPromise<void> }>();
   private _lastLocatorHandlerUid = 0;
   private _locatorHandlerRunningCounter = 0;
+  private _networkRequests: network.Request[] = [];
 
   // Aiming at 25 fps by default - each frame is 40ms, but we give some slack with 35ms.
   // When throttling for tracing, 200ms between frames, except for 10 frames around the action.
@@ -350,6 +351,16 @@ export class Page extends SdkObject {
     return this._extraHTTPHeaders;
   }
 
+  addNetworkRequest(request: network.Request) {
+    this._networkRequests.push(request);
+    for (const collected of ensureArrayLimit(this._networkRequests, 100))
+      this.emitOnContext(BrowserContext.Events.RequestCollected, collected);
+  }
+
+  networkRequests() {
+    return this._networkRequests;
+  }
+
   async onBindingCalled(payload: string, context: dom.FrameExecutionContext) {
     if (this._closedState === 'closed')
       return;
@@ -1078,7 +1089,8 @@ async function snapshotFrameForAI(progress: Progress, frame: frames.Frame, frame
   return result;
 }
 
-function ensureArrayLimit(array: any[], limit: number) {
+function ensureArrayLimit<T>(array: T[], limit: number): T[] {
   if (array.length > limit)
-    array.splice(0, limit / 10);
+    return array.splice(0, limit / 10);
+  return [];
 }

packages/playwright-core/src/utils/isomorphic/protocolMetainfo.ts

@@ -131,6 +131,7 @@ export const methodMetainfo = new Map<string, { internal?: boolean, title?: stri
   ['Page.accessibilitySnapshot', { title: 'Accessibility snapshot', group: 'getter', }],
   ['Page.pageErrors', { title: 'Get page errors', group: 'getter', }],
   ['Page.pdf', { title: 'PDF', }],
+  ['Page.requests', { title: 'Get network requests', group: 'getter', }],
   ['Page.snapshotForAI', { internal: true, }],
   ['Page.startJSCoverage', { title: 'Start JS coverage', group: 'configuration', }],
   ['Page.stopJSCoverage', { title: 'Stop JS coverage', group: 'configuration', }],

packages/playwright-core/types/types.d.ts

@@ -3926,6 +3926,12 @@ export interface Page {
    */
   requestGC(): Promise<void>;
 
+  /**
+   * Returns up to 100 last network request from this page. See
+   * [page.on('request')](https://playwright.dev/docs/api/class-page#page-event-request) for more details.
+   */
+  requests(): Promise<Array<Request>>;
+
   /**
    * Routing provides the capability to modify network requests that are made by a page.
    *

packages/protocol/src/channels.d.ts

@@ -2121,6 +2121,7 @@ export interface PageChannel extends PageEventTarget, EventTargetChannel {
   accessibilitySnapshot(params: PageAccessibilitySnapshotParams, progress?: Progress): Promise<PageAccessibilitySnapshotResult>;
   pageErrors(params?: PagePageErrorsParams, progress?: Progress): Promise<PagePageErrorsResult>;
   pdf(params: PagePdfParams, progress?: Progress): Promise<PagePdfResult>;
+  requests(params?: PageRequestsParams, progress?: Progress): Promise<PageRequestsResult>;
   snapshotForAI(params: PageSnapshotForAIParams, progress?: Progress): Promise<PageSnapshotForAIResult>;
   startJSCoverage(params: PageStartJSCoverageParams, progress?: Progress): Promise<PageStartJSCoverageResult>;
   stopJSCoverage(params?: PageStopJSCoverageParams, progress?: Progress): Promise<PageStopJSCoverageResult>;
@@ -2569,6 +2570,11 @@ export type PagePdfOptions = {
 export type PagePdfResult = {
   pdf: Binary,
 };
+export type PageRequestsParams = {};
+export type PageRequestsOptions = {};
+export type PageRequestsResult = {
+  requests: RequestChannel[],
+};
 export type PageSnapshotForAIParams = {
   timeout: number,
 };

packages/protocol/src/protocol.yml

@@ -1998,6 +1998,14 @@ Page:
       returns:
         pdf: binary
 
+    requests:
+      title: Get network requests
+      group: getter
+      returns:
+        requests:
+          type: array
+          items: Request
+
     snapshotForAI:
       internal: true
       parameters:

tests/page/page-event-request.spec.ts

@@ -376,3 +376,45 @@ it('should not expose preflight OPTIONS request with network interception', {
     `POST ${server.CROSS_PROCESS_PREFIX}/cors`,
   ]);
 });
+
+it('should return last requests', async ({ page, server }) => {
+  await page.goto(server.PREFIX + '/title.html');
+  for (let i = 0; i < 200; ++i)
+    server.setRoute('/fetch?' + i, (req, res) => res.end('url:' + server.PREFIX + req.url));
+
+  // #0 is the navigation request, so start with #1.
+  for (let i = 1; i < 50; ++i)
+    await page.evaluate(url => fetch(url), server.PREFIX + '/fetch?' + i);
+  const first50Requests = await page.requests();
+  const firstReponse = await first50Requests[1].response();
+  expect(await firstReponse.text()).toBe('url:' + server.PREFIX + '/fetch?1');
+
+  page.on('request', () => {});
+  for (let i = 50; i < 100; ++i)
+    await page.evaluate(url => fetch(url), server.PREFIX + '/fetch?' + i);
+  const first100Requests = await page.requests();
+
+  for (let i = 100; i < 200; ++i)
+    await page.evaluate(url => fetch(url), server.PREFIX + '/fetch?' + i);
+  const last100Requests = await page.requests();
+
+  // Last 100 requests are fully functional.
+  const received = await Promise.all(last100Requests.map(async request => {
+    const response = await request.response();
+    return { text: await response.text(), url: request.url() };
+  }));
+  const expected = [];
+  for (let i = 100; i < 200; ++i) {
+    const url = server.PREFIX + '/fetch?' + i;
+    expected.push({ url, text: 'url:' + url });
+  }
+  expect(received).toEqual(expected);
+
+  // First 50 requests were collected.
+  const error = await first50Requests[1].response().catch(e => e);
+  expect(error.message).toContain('request.response: The object has been collected to prevent unbounded heap growth.');
+
+  // Second 50 requests are functional, because they were reported through the event and not collected.
+  const reponse50 = await first100Requests[50].response();
+  expect(await reponse50.text()).toBe('url:' + server.PREFIX + '/fetch?50');
+});

tests/stress/heap.spec.ts

@@ -83,6 +83,18 @@ test('should not leak dispatchers after closing page', async ({ context, server
   expect(await queryObjectCount(require('../../packages/playwright-core/lib/client/network').Response)).toBe(0);
 });
 
+test('should not leak requests over 100', async ({ context, server }) => {
+  const page = await context.newPage();
+  await page.goto(server.PREFIX + '/title.html');
+  for (let i = 0; i < 100; ++i)
+    await page.evaluate(url => fetch(url), server.EMPTY_PAGE);
+  await page.requests();
+  for (let i = 0; i < 200; ++i)
+    await page.evaluate(url => fetch(url), server.EMPTY_PAGE);
+  await page.requests();
+  expect(await queryObjectCount(require('../../packages/playwright-core/lib/server/dispatchers/networkDispatchers').RequestDispatcher)).toBeLessThanOrEqual(100);
+});
+
 test.describe(() => {
   test.beforeEach(() => {
     require('../../packages/playwright-core/lib/server/dispatchers/dispatcher').setMaxDispatchersForTest(100);