fix(server): provider settings saved during startup are no longer lost

apps/server/src/provider/ProviderInstanceRegistryHydration.ts

@@ -69,25 +69,28 @@ export const deriveProviderInstanceConfigMap = (
   return merged as ProviderInstanceConfigMap;
 };
 
-const layerSettingsWatcher = Layer.effectDiscard(
-  Effect.gen(function* () {
-    const mutator = yield* ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator;
-    const serverSettings = yield* Settings.ServerSettingsService;
-    const settingsChanges = yield* serverSettings.subscribeChanges;
-    yield* settingsChanges.pipe(
-      Stream.runForEach((next) =>
-        mutator
-          .reconcile(deriveProviderInstanceConfigMap(next))
-          .pipe(
+const layerSettingsWatcher = (settingsChanges: Stream.Stream<ServerSettings>) =>
+  Layer.effectDiscard(
+    Effect.gen(function* () {
+      const mutator = yield* ProviderInstanceRegistryMutator.ProviderInstanceRegistryMutator;
+      const serverSettings = yield* Settings.ServerSettingsService;
+      yield* settingsChanges.pipe(
+        Stream.runForEach(() =>
+          // The subscription predates the seed read, so a queued change can be
+          // older than the seed. A failed current read must leave the registry intact.
+          serverSettings.getSettings.pipe(
+            Effect.flatMap((current) =>
+              mutator.reconcile(deriveProviderInstanceConfigMap(current)),
+            ),
             Effect.catchCause((cause) =>
               Effect.logError("ProviderInstanceRegistry reconcile failed", cause),
             ),
           ),
-      ),
-      Effect.forkScoped,
-    );
-  }),
-);
+        ),
+        Effect.forkScoped,
+      );
+    }),
+  );
 
 export const layer: Layer.Layer<
   ProviderInstanceRegistry.ProviderInstanceRegistry,
@@ -96,6 +99,8 @@ export const layer: Layer.Layer<
 > = Layer.unwrap(
   Effect.gen(function* () {
     const serverSettings = yield* Settings.ServerSettingsService;
+    // Subscribe before the seed read so saves during registry construction are queued.
+    const settingsChanges = yield* serverSettings.subscribeChanges;
     const initialSettings: ServerSettings | undefined = yield* serverSettings.getSettings.pipe(
       Effect.orElseSucceed(() => undefined),
     );
@@ -113,7 +118,7 @@ export const layer: Layer.Layer<
       Layer.provide(ProviderHostLive.layer),
     );
 
-    return layerSettingsWatcher.pipe(Layer.provideMerge(layerMutable));
+    return layerSettingsWatcher(settingsChanges).pipe(Layer.provideMerge(layerMutable));
   }),
 ) as Layer.Layer<
   ProviderInstanceRegistry.ProviderInstanceRegistry,

apps/server/src/provider/ProviderRegistry.test.ts

@@ -28,6 +28,7 @@ import {
   ProviderDriverKind,
   ProviderInstanceId,
   ServerSettings,
+  ServerSettingsError,
   ServerProvider,
   type ServerProviderSlashCommand,
   type ServerSettings as ContractServerSettings,
@@ -3414,6 +3415,149 @@ it.layer(
       }),
     );
 
+    const startupSettings = (instanceId?: string) =>
+      decodeServerSettings({
+        ...encodedDefaultServerSettings,
+        providerInstances: {
+          ...disabledDefaultSlots(
+            "codex",
+            "claudeAgent",
+            "cursor",
+            "grok",
+            "opencode",
+            "antigravity",
+            "pi",
+            "muse",
+          ),
+          ...(instanceId ? { [instanceId]: { driver: "ghostDriver" } } : {}),
+        },
+      });
+
+    const hydrateDuringSettingsChange = Effect.fnUntraced(function* (input: {
+      seed: ContractServerSettings;
+      saved: ContractServerSettings;
+      current: ContractServerSettings;
+      failCurrentRead?: boolean;
+    }) {
+      const changes = yield* PubSub.unbounded<ContractServerSettings | undefined>();
+      const consume = yield* Deferred.make<void>();
+      const drained: Array<Deferred.Deferred<void>> = [];
+      let readSeed = true;
+      let failCurrentRead = false;
+      const settings = yield* makeMutableServerSettingsService(input.current);
+      const serverSettings = {
+        ...settings,
+        getSettings: Effect.suspend(() => {
+          if (!readSeed) {
+            readSeed = true;
+            return PubSub.publish(changes, input.saved).pipe(Effect.as(input.seed));
+          }
+          return failCurrentRead
+            ? Effect.fail(
+                new ServerSettingsError({
+                  settingsPath: "<memory>",
+                  operation: "read-secret",
+                  cause: new Error("secret materialization failed"),
+                }),
+              )
+            : settings.getSettings;
+        }),
+        streamChanges: Stream.empty,
+        subscribeChanges: Effect.gen(function* () {
+          const subscription = yield* PubSub.subscribe(changes);
+          const done = yield* Deferred.make<void>();
+          drained.push(done);
+          // The terminal marker acknowledges preceding reconciles, including no-op/failed reads.
+          return Stream.fromEffectRepeat(
+            Deferred.await(consume).pipe(Effect.andThen(PubSub.take(subscription))),
+          ).pipe(
+            Stream.takeWhile((next): next is ContractServerSettings => next !== undefined),
+            Stream.ensuring(Deferred.succeed(done, undefined)),
+          );
+        }),
+      } satisfies ServerSettingsModule.ServerSettingsService["Service"];
+      const dependencies = yield* Layer.build(
+        Layer.mergeAll(
+          AntigravityInstallation.AntigravityInstallation.layer,
+          ServerSecretStore.layer,
+          OpenCodeRuntime.layer.pipe(Layer.provide(OpenCodeServerLedger.layerTest)),
+        ).pipe(
+          Layer.provideMerge(
+            Layer.succeed(ServerSettingsModule.ServerSettingsService, serverSettings),
+          ),
+          Layer.provideMerge(
+            ServerConfig.layerTest(process.cwd(), { prefix: "t3-provider-registry-" }),
+          ),
+          Layer.provideMerge(layerTestHttpClient),
+          Layer.provideMerge(
+            Layer.succeed(
+              ProviderEventLoggers.ProviderEventLoggers,
+              ProviderEventLoggers.NoOpProviderEventLoggers,
+            ),
+          ),
+          Layer.provideMerge(ProviderLatestVersions.layer),
+          Layer.provideMerge(McpProviderSessions.layer),
+          Layer.provideMerge(ResetCreditCoordinator.layerTest),
+          Layer.provideMerge(layerBackgroundPolicyAlwaysRun),
+        ),
+      );
+      readSeed = false;
+      const services = yield* Layer.build(ProviderInstanceRegistryHydration.layer).pipe(
+        Effect.provide(dependencies),
+      );
+      failCurrentRead = input.failCurrentRead ?? false;
+      yield* PubSub.publish(changes, undefined);
+      yield* Deferred.succeed(consume, undefined);
+      yield* Effect.forEach(drained, Deferred.await);
+      const registry = yield* ProviderInstanceRegistry.ProviderInstanceRegistry.pipe(
+        Effect.provide(services),
+      );
+      return (yield* registry.listUnavailable).map(
+        (provider): string | undefined => provider.instanceId,
+      );
+    });
+
+    it.effect("applies a settings change saved while the provider registry starts", () =>
+      Effect.gen(function* () {
+        const saved = startupSettings("ghost_late");
+        const unavailable = yield* hydrateDuringSettingsChange({
+          seed: startupSettings(),
+          saved,
+          current: saved,
+        });
+        assert.include(unavailable, "ghost_late");
+      }),
+    );
+
+    it.effect("does not apply a queued settings change older than the registry seed", () =>
+      Effect.gen(function* () {
+        const seed = startupSettings("ghost_current");
+        const unavailable = yield* hydrateDuringSettingsChange({
+          seed,
+          saved: startupSettings("ghost_stale"),
+          current: seed,
+        });
+        assert.include(unavailable, "ghost_current");
+        assert.notInclude(unavailable, "ghost_stale");
+      }),
+    );
+
+    it.effect(
+      "retains the newer registry seed when a queued stale change cannot read current settings",
+      () =>
+        Effect.gen(function* () {
+          const seed = startupSettings("ghost_current");
+          const unavailable = yield* hydrateDuringSettingsChange({
+            seed,
+            saved: startupSettings("ghost_stale"),
+            current: seed,
+            failCurrentRead: true,
+          });
+          assert.include(unavailable, "ghost_current");
+          assert.notInclude(unavailable, "ghost_stale");
+        }),
+    );
+
     it.effect(
       "keeps Cursor disabled and skips provider probing when settings use their defaults",
       () =>