feat(fetch): happy eyeballs

packages/playwright-core/src/client/fetch.ts

@@ -195,6 +195,9 @@ export class APIRequestContext extends ChannelOwner<channels.APIRequestContextCh
       }
       if (postDataBuffer === undefined && jsonData === undefined && formData === undefined && multipartData === undefined)
         postDataBuffer = options.request?.postDataBuffer() || undefined;
+      const fixtures = {
+        __testHookLookup: (options as any).__testHookLookup
+      };
       const result = await this._channel.fetch({
         url,
         params,
@@ -208,6 +211,7 @@ export class APIRequestContext extends ChannelOwner<channels.APIRequestContextCh
         failOnStatusCode: options.failOnStatusCode,
         ignoreHTTPSErrors: options.ignoreHTTPSErrors,
         maxRedirects: maxRedirects,
+        ...fixtures
       });
       return new APIResponse(this, result.response);
     });

packages/playwright-core/src/server/fetch.ts

@@ -14,20 +14,23 @@
  * limitations under the License.
  */
 
+import type * as channels from '@protocol/channels';
 import * as http from 'http';
 import * as https from 'https';
+import * as dns from 'dns';
 import type { Readable, TransformCallback } from 'stream';
 import { pipeline, Transform } from 'stream';
 import url from 'url';
 import zlib from 'zlib';
 import type { HTTPCredentials } from '../../types/types';
-import type * as channels from '@protocol/channels';
 import { TimeoutSettings } from '../common/timeoutSettings';
 import { getUserAgent } from '../common/userAgent';
 import { assert, createGuid, monotonicTime } from '../utils';
+import { HttpsProxyAgent, SocksProxyAgent } from '../utilsBundle';
 import { BrowserContext } from './browserContext';
 import { CookieStore, domainMatches } from './cookieStore';
 import { MultipartFormData } from './formData';
+import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent } from './happy-eyeballs';
 import type { CallMetadata } from './instrumentation';
 import { SdkObject } from './instrumentation';
 import type { Playwright } from './playwright';
@@ -36,7 +39,6 @@ import { ProgressController } from './progress';
 import { Tracing } from './trace/recorder/tracing';
 import type * as types from './types';
 import type { HeadersArray, ProxySettings } from './types';
-import { HttpsProxyAgent, SocksProxyAgent } from '../utilsBundle';
 
 type FetchRequestOptions = {
   userAgent: string;
@@ -67,6 +69,12 @@ export type APIRequestFinishedEvent = {
   body?: Buffer;
 };
 
+export type SendRequestOptions = https.RequestOptions & {
+  maxRedirects: number,
+  deadline: number,
+  __testHookLookup?: (hostname: string) => dns.LookupAddress[]
+};
+
 export abstract class APIRequestContext extends SdkObject {
   static Events = {
     Dispose: 'dispose',
@@ -159,13 +167,14 @@ export abstract class APIRequestContext extends SdkObject {
     const timeout = defaults.timeoutSettings.timeout(params);
     const deadline = timeout && (monotonicTime() + timeout);
 
-    const options: https.RequestOptions & { maxRedirects: number, deadline: number } = {
+    const options: SendRequestOptions = {
       method,
       headers,
       agent,
       maxRedirects: params.maxRedirects === 0 ? -1 : params.maxRedirects === undefined ? 20 : params.maxRedirects,
       timeout,
-      deadline
+      deadline,
+      __testHookLookup: (params as any).__testHookLookup,
     };
     // rejectUnauthorized = undefined is treated as true in node 12.
     if (params.ignoreHTTPSErrors || defaults.ignoreHTTPSErrors)
@@ -228,7 +237,7 @@ export abstract class APIRequestContext extends SdkObject {
     }
   }
 
-  private async _sendRequest(progress: Progress, url: URL, options: https.RequestOptions & { maxRedirects: number, deadline: number }, postData?: Buffer): Promise<Omit<channels.APIResponse, 'fetchUid'> & { body: Buffer }>{
+  private async _sendRequest(progress: Progress, url: URL, options: SendRequestOptions, postData?: Buffer): Promise<Omit<channels.APIResponse, 'fetchUid'> & { body: Buffer }>{
     await this._updateRequestCookieHeader(url, options);
 
     const requestCookies = (options.headers!['cookie'] as (string | undefined))?.split(';').map(p => {
@@ -247,7 +256,10 @@ export abstract class APIRequestContext extends SdkObject {
     return new Promise((fulfill, reject) => {
       const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest)
         = (url.protocol === 'https:' ? https : http).request;
-      const request = requestConstructor(url, options, async response => {
+      // If we have a proxy agent already, do not override it.
+      const agent = options.agent || (url.protocol === 'https:' ? httpsHappyEyeballsAgent : httpHappyEyeballsAgent);
+      const requestOptions = { ...options, agent };
+      const request = requestConstructor(url, requestOptions as any, async response => {
         const notifyRequestFinished = (body?: Buffer) => {
           const requestFinishedEvent: APIRequestFinishedEvent = {
             requestEvent,
@@ -292,13 +304,14 @@ export abstract class APIRequestContext extends SdkObject {
             delete headers[`content-type`];
           }
 
-          const redirectOptions: https.RequestOptions & { maxRedirects: number, deadline: number } = {
+          const redirectOptions: SendRequestOptions = {
             method,
             headers,
             agent: options.agent,
             maxRedirects: options.maxRedirects - 1,
             timeout: options.timeout,
-            deadline: options.deadline
+            deadline: options.deadline,
+            __testHookLookup: options.__testHookLookup,
           };
           // rejectUnauthorized = undefined is treated as true in node 12.
           if (options.rejectUnauthorized === false)

packages/playwright-core/src/server/happy-eyeballs.ts

@@ -0,0 +1,123 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as dns from 'dns';
+import * as http from 'http';
+import * as https from 'https';
+import * as net from 'net';
+import * as tls from 'tls';
+import { ManualPromise } from '../utils/manualPromise';
+import { SendRequestOptions } from './fetch';
+
+// Implementation(partial) of Happy Eyeballs 2 algorithm described in
+// https://www.rfc-editor.org/rfc/rfc8305
+
+// Same as in Chromium (https://source.chromium.org/chromium/chromium/src/+/5666ff4f5077a7e2f72902f3a95f5d553ea0d88d:net/socket/transport_connect_job.cc;l=102)
+const connectionAttemptDelayMs = 300;
+
+class HttpHappyEyeballsAgent extends http.Agent {
+  createConnection(options: http.ClientRequestArgs, oncreate?: (err: Error | null, socket?: net.Socket) => void): net.Socket | undefined {
+    // There is no ambiguity in case of IP address.
+    if (net.isIP(options.hostname!))
+      return net.createConnection(options as net.NetConnectOpts);
+    createConnectionAsync(options, oncreate).catch(err => oncreate?.(err));
+  }
+}
+
+class HttpsHappyEyeballsAgent extends https.Agent {
+  createConnection(options: http.ClientRequestArgs, oncreate?: (err: Error | null, socket?: net.Socket) => void): net.Socket | undefined {
+    // There is no ambiguity in case of IP address.
+    if (net.isIP(options.hostname!))
+      return tls.connect(options as tls.ConnectionOptions);
+    createConnectionAsync(options, oncreate).catch(err => oncreate?.(err));
+  }
+}
+
+export const httpsHappyEyeballsAgent = new HttpsHappyEyeballsAgent();
+export const httpHappyEyeballsAgent = new HttpHappyEyeballsAgent();
+
+async function createConnectionAsync(options: http.ClientRequestArgs, oncreate?: (err: Error | null, socket?: net.Socket) => void) {
+  const lookup = (options as SendRequestOptions).__testHookLookup || lookupAddresses;
+  const addresses = await lookup(options.hostname!);
+  const sockets: net.Socket[] = [];
+  let firstError;
+  let errorCount = 0;
+  const handleError = (err: Error) => {
+    ++errorCount;
+    firstError ??= err;
+    if (errorCount === addresses.length)
+      oncreate?.(err);
+  };
+
+  const connected = new ManualPromise();
+  for (const { address } of addresses) {
+    const socket = options.protocol === 'https:' ?
+      tls.connect({
+        ...(options as tls.ConnectionOptions),
+        port: options.port as number,
+        host: address,
+        servername: options.hostname || undefined }) :
+      net.createConnection({
+        ...options,
+        port: options.port as number,
+        host: address });
+
+    socket.on('connect', () => {
+      connected.resolve();
+      oncreate?.(null, socket);
+      // TODO: Cache the result?
+      // Close other outstanding sockets.
+      for (const s of sockets) {
+        if (s !== socket)
+          s.destroy();
+      }
+    });
+    socket.on('timeout', () => {
+      handleError(new Error('Connection timeout'));
+      socket.destroy();
+    });
+    socket.on('error', handleError)
+    sockets.push(socket);
+    await Promise.race([
+      connected,
+      new Promise(f => setTimeout(f, connectionAttemptDelayMs))
+    ]);
+    if (connected.isDone())
+      break;
+  }
+}
+
+async function lookupAddresses(hostname: string): Promise<dns.LookupAddress[]> {
+  const addresses = await dns.promises.lookup(hostname, { all: true, family: 0, verbatim: true });
+  let firstFamily = addresses.filter(({family}) => family === 6);
+  let secondFamily = addresses.filter(({family}) => family === 4);
+  // Make sure first address in the list is the same as in the original order.
+  if (firstFamily.length && firstFamily[0] !== addresses[0]) {
+    const tmp = firstFamily;
+    firstFamily = secondFamily;
+    secondFamily = tmp;
+  }
+  const result = [];
+  // Alternate ipv6 and ipv4 addreses.
+  for (let i = 0; i < Math.max(firstFamily.length, secondFamily.length); i++) {
+    if (firstFamily[i])
+      result.push(firstFamily[i]);
+    if (secondFamily[i])
+      result.push(secondFamily[i]);
+  }
+  return result;
+}
+

tests/library/browsercontext-fetch.spec.ts

@@ -18,31 +18,21 @@ import formidable from 'formidable';
 import http from 'http';
 import zlib from 'zlib';
 import fs from 'fs';
+import dns from 'dns';
 import { pipeline } from 'stream';
 import { contextTest as it, expect } from '../config/browserTest';
 import { suppressCertificateWarning } from '../config/utils';
 
 it.skip(({ mode }) => mode !== 'default');
 
-let prevAgent: http.Agent;
-it.beforeAll(() => {
-  prevAgent = http.globalAgent;
-  http.globalAgent = new http.Agent({
-    // @ts-expect-error
-    lookup: (hostname, options, callback) => {
-      if (hostname === 'localhost' || hostname.endsWith('playwright.dev'))
-        callback(null, '127.0.0.1', 4);
-      else
-        throw new Error(`Failed to resolve hostname: ${hostname}`);
-    }
-  });
-});
+const __testHookLookup = (hostname: string): dns.LookupAddress[] => {
+  if (hostname === 'localhost' || hostname.endsWith('playwright.dev'))
+    return [{ address: '127.0.0.1', family: 4 }];
+  else
+    throw new Error(`Failed to resolve hostname: ${hostname}`);
+};
 
-it.afterAll(() => {
-  http.globalAgent = prevAgent;
-});
-
-it('get should work @smoke', async ({ context, server }) => {
+it('get should work @smoke', async ({ context, server, mode }) => {
   const response = await context.request.get(server.PREFIX + '/simple.json');
   expect(response.url()).toBe(server.PREFIX + '/simple.json');
   expect(response.status()).toBe(200);
@@ -123,7 +113,9 @@ it('should add session cookies to request', async ({ context, server }) => {
   }]);
   const [req] = await Promise.all([
     server.waitForRequest('/simple.json'),
-    context.request.get(`http://www.my.playwright.dev:${server.PORT}/simple.json`),
+    context.request.get(`http://www.my.playwright.dev:${server.PORT}/simple.json`, {
+      __testHookLookup
+    } as any),
   ]);
   expect(req.headers.cookie).toEqual('username=John Doe');
 });
@@ -176,8 +168,9 @@ it('should not add context cookie if cookie header passed as a parameter', async
     context.request.get(`http://www.my.playwright.dev:${server.PORT}/empty.html`, {
       headers: {
         'Cookie': 'foo=bar'
-      }
-    }),
+      },
+      __testHookLookup
+    } as any),
   ]);
   expect(req.headers.cookie).toEqual('foo=bar');
 });
@@ -197,7 +190,7 @@ it('should follow redirects', async ({ context, server }) => {
   }]);
   const [req, response] = await Promise.all([
     server.waitForRequest('/simple.json'),
-    context.request.get(`http://www.my.playwright.dev:${server.PORT}/redirect1`),
+    context.request.get(`http://www.my.playwright.dev:${server.PORT}/redirect1`, { __testHookLookup } as any),
   ]);
   expect(req.headers.cookie).toEqual('username=John Doe');
   expect(response.url()).toBe(`http://www.my.playwright.dev:${server.PORT}/simple.json`);