Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions apps/desktop/src/app/DesktopLifecycle.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ function makeDesktopWindowLayer(
showConnectingSplash: Effect.void,
handleBackendReady: () => Effect.void,
handleBackendNotReady: Effect.void,
handleBackendFailed: () => Effect.void,
flushMainWindowBounds: input.flushMainWindowBounds ?? Effect.void,
prepareCaptureReveal: Effect.void,
dispatchMenuAction: () => Effect.void,
Expand Down
114 changes: 114 additions & 0 deletions apps/desktop/src/backend/DesktopBackendManager.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ interface MakeInstanceInput {
readonly backendOutputLog?: Partial<DesktopObservability.DesktopBackendOutputLogShape>;
readonly onReady?: Effect.Effect<void>;
readonly onShutdown?: Effect.Effect<void>;
readonly onFailed?: (reason: string) => Effect.Effect<void>;
readonly onPreflightFailed?: (
failure: DesktopBackendManager.PreflightFailure,
) => Effect.Effect<boolean>;
Expand Down Expand Up @@ -184,13 +185,126 @@ function makeTestInstance(input: MakeInstanceInput) {
configResolve: input.configResolve ?? Effect.succeed(input.config ?? baseConfig),
...(input.onReady ? { onReady: () => input.onReady! } : {}),
...(input.onShutdown ? { onShutdown: () => input.onShutdown! } : {}),
...(input.onFailed ? { onFailed: input.onFailed } : {}),
...(input.onPreflightFailed ? { onPreflightFailed: input.onPreflightFailed } : {}),
});

return instance.pipe(Effect.provide(servicesLayer));
}

describe("DesktopBackendManager", () => {
for (const [becomesReady, preflightRetries] of [
[false, 0],
[true, 0],
[false, 4],
[true, 4],
] as const) {
it.effect(
`stops after five crashes (ready: ${becomesReady}, preflight retries: ${preflightRetries})`,
() =>
Effect.scoped(
Effect.gen(function* () {
const persisted = yield* Queue.unbounded<string>();
const ready = yield* Queue.unbounded<void>();
const failures: string[] = [];
let starts = 0;
let resolves = 0;
const instance = yield* makeTestInstance({
configResolve: Effect.sync(() =>
resolves++ < preflightRetries
? {
...baseConfig,
preflightFailure: Option.some({ reason: "WSL is starting", fatal: false }),
}
: baseConfig,
),
spawnerLayer: Layer.succeed(
ChildProcessSpawner.ChildProcessSpawner,
ChildProcessSpawner.make(() =>
Effect.sync(() => {
starts += 1;
return makeProcess({
exitCode: (becomesReady ? Queue.take(ready) : Effect.void).pipe(
Effect.as(ChildProcessSpawner.ExitCode(1)),
),
});
}),
),
),
httpClientLayer: becomesReady
? healthyHttpClientLayer
: httpClientLayer(() => Effect.never),
onReady: Queue.offer(ready, undefined).pipe(Effect.asVoid),
backendOutputLog: {
persistFailure: ({ details }) =>
Queue.offer(persisted, details).pipe(Effect.asVoid),
},
onFailed: (reason) =>
Effect.sync(() => {
failures.push(reason);
}),
});
yield* instance.start;
if (preflightRetries > 0) yield* TestClock.adjust(7_500);
yield* Queue.take(persisted);
yield* TestClock.adjust(0);
assert.deepEqual(failures, []);
for (const delay of [500, 1_000, 2_000, 4_000]) {
yield* TestClock.adjust(preflightRetries > 0 ? 10_000 : delay);
yield* Queue.take(persisted);
}
yield* TestClock.adjust(10_000);
assert.deepEqual(failures, ["code=1"]);
assert.equal(starts, 5);
assert.equal((yield* instance.snapshot).desiredRunning, false);
assert.equal((yield* instance.snapshot).restartScheduled, false);
}).pipe(Effect.provide(TestClock.layer())),
),
);
}

it.effect("resets the primary crash allowance after a minute of readiness", () =>
Effect.scoped(
Effect.gen(function* () {
const ready = yield* Queue.unbounded<void>();
const exits = yield* Queue.unbounded<void>();
const persisted = yield* Queue.unbounded<void>();
const failures: string[] = [];
const instance = yield* makeTestInstance({
spawnerLayer: Layer.succeed(
ChildProcessSpawner.ChildProcessSpawner,
ChildProcessSpawner.make(() =>
Effect.succeed(
makeProcess({
exitCode: Queue.take(exits).pipe(Effect.as(ChildProcessSpawner.ExitCode(1))),
}),
),
),
),
onReady: Queue.offer(ready, undefined).pipe(Effect.asVoid),
onFailed: (reason) =>
Effect.sync(() => {
failures.push(reason);
}),
backendOutputLog: {
persistFailure: () => Queue.offer(persisted, undefined).pipe(Effect.asVoid),
},
});
yield* instance.start;
for (const [index, delay] of [500, 1_000, 500, 1_000, 2_000].entries()) {
yield* Queue.take(ready);
if (index === 2) yield* TestClock.adjust(60_000);
yield* Queue.offer(exits, undefined);
yield* Queue.take(persisted);
yield* TestClock.adjust(delay);
}
yield* Queue.take(ready);
assert.deepEqual(failures, []);
assert.equal((yield* instance.snapshot).ready, true);
}).pipe(Effect.provide(TestClock.layer())),
),
);

it.effect("spawns the backend with fd3 bootstrap and fd4 telemetry", () =>
Effect.scoped(
Effect.gen(function* () {
Expand Down
40 changes: 37 additions & 3 deletions apps/desktop/src/backend/DesktopBackendManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@

import * as Brand from "effect/Brand";
import * as Cause from "effect/Cause";
import * as Clock from "effect/Clock";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
Expand Down Expand Up @@ -56,6 +57,8 @@ import * as DesktopWslEnvironment from "../wsl/DesktopWslEnvironment.ts";

const INITIAL_RESTART_DELAY = Duration.millis(500);
const MAX_RESTART_DELAY = Duration.seconds(10);
const MAX_CRASH_ATTEMPTS = 5;
Comment thread
macroscopeapp[bot] marked this conversation as resolved.
const STABLE_BACKEND_UPTIME_MS = 60_000;
// After this many consecutive fatal preflight failures, stop the silent
// restart loop and surface the reason via onPreflightFailed. Transient
// failures may instead provide their own larger retryLimit when they should
Expand Down Expand Up @@ -295,6 +298,8 @@ export interface BackendInstanceSpec {
// between "fired onReady" and "currentConfig already advanced".
readonly onReady?: (httpBaseUrl: URL) => Effect.Effect<void>;
readonly onShutdown?: () => Effect.Effect<void>;
// Opts this instance into bounded crash recovery and reports the terminal failure.
readonly onFailed?: (reason: string) => Effect.Effect<void>;
// Fired once when a fatal or bounded preflight failure has exhausted its
// retries. Returns true when the callback changed configuration and the
// manager should resolve once more; false stops the failed instance.
Expand All @@ -316,6 +321,7 @@ interface BackendManagerState {
readonly config: Option.Option<DesktopBackendStartConfig>;
readonly active: Option.Option<ActiveBackendRun>;
readonly restartAttempt: number;
readonly crashAttempt: number;
// Consecutive bounded/fatal preflight failures, reset on a clean or
// unbounded-transient preflight. restartAttempt counts all restarts.
readonly preflightFailureAttempt: number;
Expand All @@ -329,6 +335,7 @@ const initialState: BackendManagerState = {
config: Option.none(),
active: Option.none(),
restartAttempt: 0,
crashAttempt: 0,
preflightFailureAttempt: 0,
restartFiber: Option.none(),
nextRunId: 1,
Expand Down Expand Up @@ -735,6 +742,8 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
ready: false,
config: Option.some(config.value),
preflightFailureAttempt: resetFatalPreflightCounter ? 0 : latest.preflightFailureAttempt,
restartAttempt: current.desiredRunning ? latest.restartAttempt : 0,
crashAttempt: current.desiredRunning ? latest.crashAttempt : 0,
}));

const preflightFailure = config.value.preflightFailure;
Expand Down Expand Up @@ -805,7 +814,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
);

if (!entryExists) {
yield* scheduleRestart(`missing server entry at ${config.value.entryPath}`);
yield* scheduleRestart(`missing server entry at ${config.value.entryPath}`, true);
return;
}

Expand All @@ -826,11 +835,13 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
},
]);

let readyAt: number | undefined;
const finalizeRun = Effect.fn("desktop.backendInstance.finalizeRun")(function* (
reason: string,
) {
yield* mutex.withPermits(1)(
Effect.gen(function* () {
const now = yield* Clock.currentTimeMillis;
const { isCurrentRun, nextState, pid, exitObserved, stopRequested, wasReady } =
yield* Ref.modify(
state,
Expand Down Expand Up @@ -864,6 +875,14 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (

const next = {
...latest,
restartAttempt:
readyAt !== undefined && now - readyAt >= STABLE_BACKEND_UPTIME_MS
? 0
: latest.restartAttempt,
crashAttempt:
readyAt !== undefined && now - readyAt >= STABLE_BACKEND_UPTIME_MS
? 0
: latest.crashAttempt,
active: Option.none<ActiveBackendRun>(),
ready: false,
};
Expand Down Expand Up @@ -898,7 +917,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
}

if (isCurrentRun && nextState.desiredRunning) {
yield* scheduleRestart(reason);
yield* scheduleRestart(reason, true);
}
}),
);
Expand All @@ -924,6 +943,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
exitObserved: true,
})),
onReady: Effect.fn("desktop.backendInstance.onReady")(function* () {
readyAt = yield* Clock.currentTimeMillis;
const isCurrentRun = yield* Ref.modify(state, (latest) => {
const activeRun = Option.getOrUndefined(latest.active);
if (activeRun?.id !== runId) {
Expand All @@ -934,7 +954,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
true,
{
...latest,
restartAttempt: 0,
restartAttempt: spec.onFailed ? latest.restartAttempt : 0,
ready: true,
},
] as const;
Expand Down Expand Up @@ -987,7 +1007,20 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (

const scheduleRestart = Effect.fn("desktop.backendInstance.scheduleRestart")(function* (
reason: string,
terminalFailure = false,
) {
const current = yield* Ref.get(state);
if (
terminalFailure &&
current.desiredRunning &&
spec.onFailed &&
current.crashAttempt >= MAX_CRASH_ATTEMPTS - 1
) {
yield* Ref.update(state, (latest) => ({ ...latest, desiredRunning: false, ready: false }));
// The dialog can quit the app, whose shutdown acquires this instance's mutex.
yield* Effect.forkIn(spec.onFailed(reason), parentScope);
return;
}
const scheduled = yield* Ref.modify(state, (latest) => {
if (!latest.desiredRunning || Option.isSome(latest.restartFiber)) {
return [Option.none<Duration.Duration>(), latest] as const;
Expand All @@ -999,6 +1032,7 @@ export const makeBackendInstance = Effect.fn("makeBackendInstance")(function* (
{
...latest,
restartAttempt: latest.restartAttempt + 1,
crashAttempt: latest.crashAttempt + (terminalFailure ? 1 : 0),
},
] as const;
});
Expand Down
1 change: 1 addition & 0 deletions apps/desktop/src/backend/DesktopBackendPool.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ function makePoolLayer(
showConnectingSplash: Effect.void,
handleBackendReady: () => Effect.void,
handleBackendNotReady: Effect.void,
handleBackendFailed: () => Effect.void,
flushMainWindowBounds: Effect.void,
prepareCaptureReveal: Effect.void,
dispatchMenuAction: () => Effect.die("unexpected menu action"),
Expand Down
4 changes: 2 additions & 2 deletions apps/desktop/src/backend/DesktopBackendPool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -286,8 +286,7 @@ export const layer = Layer.effect(
label: configuration.resolvePrimaryLabel,
configResolve: configuration.resolvePrimary,
// Window creation errors propagating out of handleBackendReady must
// not block the readiness callback (that would prevent restartAttempt
// from being reset), so we absorb them here. The window service only
// not block the readiness callback, so we absorb them here. The window service only
// logs on success, so log the failure here before swallowing it —
// otherwise a post-readiness window-open failure vanishes silently and
// is near-impossible to diagnose in production.
Expand All @@ -300,6 +299,7 @@ export const layer = Layer.effect(
),
),
onShutdown: () => desktopWindow.handleBackendNotReady,
onFailed: desktopWindow.handleBackendFailed,
onPreflightFailed: handlePrimaryPreflightFailure,
});

Expand Down
8 changes: 6 additions & 2 deletions apps/desktop/src/electron/ElectronDialog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ export class ElectronDialog extends Context.Service<
) => Effect.Effect<readonly string[], ElectronDialogPickFilesError>;
readonly showMessageBox: (
options: Electron.MessageBoxOptions,
owner?: Electron.BrowserWindow,
) => Effect.Effect<Electron.MessageBoxReturnValue, ElectronDialogShowMessageBoxError>;
readonly showErrorBox: (title: string, content: string) => Effect.Effect<void>;
}
Expand Down Expand Up @@ -164,9 +165,12 @@ export const make = ElectronDialog.of({
});
return result.canceled ? [] : result.filePaths;
}),
showMessageBox: (options) =>
showMessageBox: (options, owner) =>
Effect.tryPromise({
try: () => Electron.dialog.showMessageBox(options),
try: () =>
owner && !owner.isDestroyed()
? Electron.dialog.showMessageBox(owner, options)
: Electron.dialog.showMessageBox(options),
catch: (cause) =>
new ElectronDialogShowMessageBoxError({
type: options.type ?? null,
Expand Down
1 change: 1 addition & 0 deletions apps/desktop/src/window/DesktopApplicationMenu.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ const makeDesktopWindowLayer = (selectedAction: Deferred.Deferred<string>) =>
showConnectingSplash: Effect.void,
handleBackendReady: () => Effect.void,
handleBackendNotReady: Effect.void,
handleBackendFailed: () => Effect.void,
flushMainWindowBounds: Effect.void,
prepareCaptureReveal: Effect.void,
dispatchMenuAction: (action) => Deferred.succeed(selectedAction, action).pipe(Effect.asVoid),
Expand Down
Loading
Loading