[build-tools] Start independent device session setup in parallel

CHANGELOG.md

@@ -10,6 +10,7 @@ This is the log of notable changes to EAS CLI and related packages.
 
 - [eas-cli] Allow workflow submissions to skip loading app config when project and application identifiers are supplied. ([#4471](https://github.com/expo/eas-cli/pull/4471) by [@sjchmiela](https://github.com/sjchmiela))
 - [build-tools] Record Android device sessions and upload the screen recording, including the partial recording a killed Device Hub leaves behind. ([#4418](https://github.com/expo/eas-cli/pull/4418) by [@vonovak](https://github.com/vonovak))
+- [build-tools] Start the Android device session ffmpeg install during the emulator boot, and start the agent-device daemon and the session host in parallel. ([#4490](https://github.com/expo/eas-cli/pull/4490) by [@szdziedzic](https://github.com/szdziedzic))
 
 ### 🐛 Bug fixes
 

packages/build-tools/src/steps/functions/__tests__/startAgentDeviceRemoteSession-orchestration.test.ts

@@ -193,6 +193,69 @@ describe('createStartAgentDeviceRemoteSessionBuildFunction orchestration', () =>
     expect(spawnDetached).not.toHaveBeenCalled();
   });
 
+  it('starts the session host without waiting for the agent-device daemon', async () => {
+    const logger = { info: jest.fn(), warn: jest.fn() };
+    const sessionHost = await jest.mocked(startDeviceSessionHostAsync).getMockImplementation()!(
+      ctx,
+      {} as never
+    );
+    let markHostStarted!: () => void;
+    const hostStarted = new Promise<void>(resolve => {
+      markHostStarted = resolve;
+    });
+    jest.mocked(startDeviceSessionHostAsync).mockImplementation(async () => {
+      markHostStarted();
+      return sessionHost;
+    });
+    // The daemon credentials only appear after the session host started. A sequential
+    // startup would wait here forever.
+    jest.mocked(waitForFileAsync).mockImplementation(async () => {
+      await hostStarted;
+      return { port: 5678, token: 'daemon-token' };
+    });
+
+    await runAsync(logger, BuildRuntimePlatform.DARWIN);
+
+    expect(uploadRemoteSessionConfigAsync).toHaveBeenCalledWith(
+      expect.objectContaining({
+        remoteConfig: expect.objectContaining({
+          agentDeviceRemoteSessionUrl: 'https://agent-device-abc.tunnel.example.com',
+          agentDeviceRemoteSessionToken: 'daemon-token',
+          webPreviewUrl: 'https://expo.dev/simulator-preview/preview-id',
+        }),
+      })
+    );
+  });
+
+  it('stops the daemon and its tunnel when the session host fails to start', async () => {
+    const logger = { info: jest.fn(), warn: jest.fn() };
+    jest
+      .mocked(startDeviceSessionHostAsync)
+      .mockRejectedValue(new Error('serve-sim did not start'));
+
+    await expect(runAsync(logger, BuildRuntimePlatform.DARWIN)).rejects.toThrow(
+      'serve-sim did not start'
+    );
+
+    expect(uploadRemoteSessionConfigAsync).not.toHaveBeenCalled();
+    expect(mockDaemonStopAsync).toHaveBeenCalledTimes(1);
+    expect(mockTunnelStopAsync).toHaveBeenCalledTimes(1);
+  });
+
+  it('stops the session host when the agent-device daemon fails to start', async () => {
+    const logger = { info: jest.fn(), warn: jest.fn() };
+    jest.mocked(waitForFileAsync).mockRejectedValue(new Error('no daemon credentials'));
+
+    await expect(runAsync(logger, BuildRuntimePlatform.DARWIN)).rejects.toThrow(
+      'no daemon credentials'
+    );
+
+    expect(uploadRemoteSessionConfigAsync).not.toHaveBeenCalled();
+    expect(startNgrokTunnelAsync).not.toHaveBeenCalled();
+    expect(mockPreviewStopAsync).toHaveBeenCalledTimes(1);
+    expect(mockDaemonStopAsync).toHaveBeenCalledTimes(1);
+  });
+
   it('declares the launch inputs', () => {
     const buildFunction = createStartAgentDeviceRemoteSessionBuildFunction(ctx);
     const globalCtx = createGlobalContextMock();

packages/build-tools/src/steps/functions/__tests__/startAndroidEmulator.test.ts

@@ -8,6 +8,7 @@ import {
   AndroidVirtualDeviceName,
 } from '../../../utils/AndroidEmulatorUtils';
 import { retryAsync } from '../../../utils/retry';
+import { ensureFfmpegInstalledOnceAsync } from '../../utils/remoteDeviceRunSession';
 import { createStartAndroidEmulatorBuildFunction } from '../startAndroidEmulator';
 
 jest.mock('@expo/turtle-spawn', () => ({
@@ -19,6 +20,10 @@ jest.mock('../../../utils/retry', () => ({
   retryAsync: jest.fn(),
 }));
 
+jest.mock('../../utils/remoteDeviceRunSession', () => ({
+  ensureFfmpegInstalledOnceAsync: jest.fn(),
+}));
+
 jest.mock('../../../utils/AndroidEmulatorUtils', () => ({
   AndroidEmulatorUtils: {
     defaultSystemImagePackage: 'system-images;android-30;default;x86_64',
@@ -284,6 +289,33 @@ describe(createStartAndroidEmulatorBuildFunction, () => {
     );
   });
 
+  it('installs ffmpeg in the background during the boot of a device run session', async () => {
+    // Never settles: the boot must not wait for the install.
+    jest.mocked(ensureFfmpegInstalledOnceAsync).mockReturnValue(new Promise(() => {}));
+    const step = createStep(undefined, { DEVICE_RUN_SESSION_ID: 'device-run-session-id' });
+
+    await step.executeAsync();
+
+    expect(ensureFfmpegInstalledOnceAsync).toHaveBeenCalledTimes(1);
+    expect(ensureFfmpegInstalledOnceAsync).toHaveBeenCalledWith(
+      expect.objectContaining({
+        env: expect.objectContaining({ DEVICE_RUN_SESSION_ID: 'device-run-session-id' }),
+      })
+    );
+    expect(jest.mocked(ensureFfmpegInstalledOnceAsync).mock.invocationCallOrder[0]).toBeLessThan(
+      mockedAndroidUtils.startAsync.mock.invocationCallOrder[0]
+    );
+    expect(mockedAndroidUtils.waitForReadyAsync).toHaveBeenCalled();
+  });
+
+  it('does not install ffmpeg outside a device run session', async () => {
+    const step = createStep();
+
+    await step.executeAsync();
+
+    expect(ensureFfmpegInstalledOnceAsync).not.toHaveBeenCalled();
+  });
+
   it('skips animation scale adjustments when opt out env var is disabled', async () => {
     const step = createStep(undefined, {
       ANDROID_EMULATOR_ADJUST_ANIMATION_SCALE: 'false',

packages/build-tools/src/steps/functions/startAgentDeviceRemoteSession.ts

@@ -117,31 +117,36 @@ export function createStartAgentDeviceRemoteSessionBuildFunction(
         await selectXcodeDeveloperDirectoryAsync({ env, logger });
       }
 
-      logger.info('Launching agent-device daemon.');
-      const daemonProcess = await startAgentDeviceDaemonAsync({ packageVersion, env, logger });
-
-      logger.info(`Waiting for daemon credentials at ${DAEMON_JSON_PATH}.`);
-      const { port: daemonPort, token: daemonToken } = await waitForDaemonInfoAsync({
-        daemonProcess,
-      });
-      logger.info(`Daemon is listening on port ${daemonPort}; loaded auth token.`);
-
-      const agentDeviceTunnel = await startNgrokTunnelAsync({
-        port: daemonPort,
-        subdomainPrefix: 'agent-device',
-        baseDomain: ngrokTunnelDomain,
-        authtoken: ngrokAuthtoken,
-        logger,
-      });
-      const agentDeviceRemoteSessionUrl = agentDeviceTunnel.url;
-      logger.info(`Tunnel is ready at ${agentDeviceRemoteSessionUrl}.`);
-
+      let daemonProcess: DetachedProcessHandle | undefined;
+      let agentDeviceTunnel: Awaited<ReturnType<typeof startNgrokTunnelAsync>> | undefined;
       let sessionHost: DeviceSessionHost | undefined;
       let eventCollection:
         | Awaited<ReturnType<typeof startAgentDeviceEventCollectionAsync>>
         | undefined;
       let sessionFailed = false;
-      try {
+
+      // The daemon (with its tunnel) and the session host (with the web preview) do not
+      // depend on each other, so they start in parallel. Each chain stores what it
+      // started, so the teardown below can stop it even when the other chain failed.
+      const agentDeviceStartup = (async () => {
+        logger.info('Launching agent-device daemon.');
+        daemonProcess = await startAgentDeviceDaemonAsync({ packageVersion, env, logger });
+
+        logger.info(`Waiting for daemon credentials at ${DAEMON_JSON_PATH}.`);
+        const daemonInfo = await waitForDaemonInfoAsync({ daemonProcess });
+        logger.info(`Daemon is listening on port ${daemonInfo.port}; loaded auth token.`);
+
+        agentDeviceTunnel = await startNgrokTunnelAsync({
+          port: daemonInfo.port,
+          subdomainPrefix: 'agent-device',
+          baseDomain: ngrokTunnelDomain,
+          authtoken: ngrokAuthtoken,
+          logger,
+        });
+        logger.info(`Tunnel is ready at ${agentDeviceTunnel.url}.`);
+        return { ...daemonInfo, remoteSessionUrl: agentDeviceTunnel.url };
+      })();
+      const sessionHostStartup = (async () => {
         const launchDescription = describeServeSimLaunch(launch);
         if (launchDescription) {
           logger.info(launchDescription);
@@ -159,6 +164,14 @@ export function createStartAgentDeviceRemoteSessionBuildFunction(
         logger.info(
           `Web preview URL: ${webPreview.previewPageUrl} (server: ${webPreview.apiUrl}).`
         );
+        return webPreview;
+      })();
+
+      try {
+        const [
+          { port: daemonPort, token: daemonToken, remoteSessionUrl: agentDeviceRemoteSessionUrl },
+          webPreview,
+        ] = await Promise.all([agentDeviceStartup, sessionHostStartup]);
 
         await uploadRemoteSessionConfigWithLocalEgressAsync({
           env,
@@ -206,26 +219,31 @@ export function createStartAgentDeviceRemoteSessionBuildFunction(
         sessionFailed = true;
         throw error;
       } finally {
+        // Promise.all rejects on the first failure while the other chain can still be
+        // starting. Wait for both, so the teardown sees everything that was started.
+        await Promise.allSettled([agentDeviceStartup, sessionHostStartup]);
+        const startedDaemon = daemonProcess;
         await finishRemoteSessionAsync({
           logger,
           sessionFailed,
           teardown: [
-            ['agent-device tunnel', agentDeviceTunnel.stopAsync()],
+            ['agent-device tunnel', agentDeviceTunnel?.stopAsync()],
             [
               'agent-device daemon',
-              (async () => {
-                try {
-                  if (eventCollection) {
-                    await stopAgentDeviceEventCollectionSafelyAsync({
-                      eventCollection,
-                      deviceRunSessionId,
-                      logger,
-                    });
+              startedDaemon &&
+                (async () => {
+                  try {
+                    if (eventCollection) {
+                      await stopAgentDeviceEventCollectionSafelyAsync({
+                        eventCollection,
+                        deviceRunSessionId,
+                        logger,
+                      });
+                    }
+                  } finally {
+                    await startedDaemon.stopAsync();
                   }
-                } finally {
-                  await daemonProcess.stopAsync();
-                }
-              })(),
+                })(),
             ],
             ['session host', sessionHost?.finishAsync()],
           ],

packages/build-tools/src/steps/functions/startAndroidEmulator.ts

@@ -18,6 +18,7 @@ import {
   AndroidVirtualDeviceName,
 } from '../../utils/AndroidEmulatorUtils';
 import { retryAsync } from '../../utils/retry';
+import { ensureFfmpegInstalledOnceAsync } from '../utils/remoteDeviceRunSession';
 
 const ANDROID_STARTUP_ATTEMPT_TIMEOUT_MS = [60_000, 120_000, 180_000];
 const ANDROID_STARTUP_RETRIES_COUNT = ANDROID_STARTUP_ATTEMPT_TIMEOUT_MS.length - 1;
@@ -74,12 +75,25 @@ export function createStartAndroidEmulatorBuildFunction(): BuildFunction {
         required: true,
       }),
     ],
-    fn: async ({ logger }, { inputs, outputs, env }) => {
+    fn: async ({ logger, global }, { inputs, outputs, env }) => {
       const logcatDirectory = await fs.promises.mkdtemp(
         path.join(os.tmpdir(), 'eas-android-emulator-logcat-')
       );
       outputs.logcat_directory.set(logcatDirectory);
 
+      // A device run session installs ffmpeg for expo-device-hub after the boot, which
+      // took about 25 s with apt. It does not need the emulator, so start it now and let
+      // it run during the boot; the session host then waits for the same install.
+      // It never rejects: failures are logged and the session continues without it.
+      if (env.DEVICE_RUN_SESSION_ID) {
+        logger.info('Installing ffmpeg for the device session in the background.');
+        void ensureFfmpegInstalledOnceAsync({
+          runtimePlatform: global.runtimePlatform,
+          env,
+          logger,
+        });
+      }
+
       if (env.EAS_NO_EMULATOR_HOST_SUPPORT_CHECK !== '1') {
         await assertAndroidEmulatorHostSupportAsync({ env });
       } else {