CBG-5435: do not let a join resurrect an ended background process

[torcolvin]

CBG-5435: do not let a join resurrect an ended background process

Fixes:

1. node1 POST /db/_resync?action=start
2. node2 and node3 started the join process
3. node1 POST /db/_resync?action=stop to node1. This moves the shared status document moves to "stopped".
4. node3 was still in the middle of its join, and its status write put that document back to "running".
5. node2 and node3 saw "running" again, rejoined a job that was already over, and kept reading and rewriting documents.

A node joining a multi-node background process wrote its status to the cluster unconditionally. A join admitted against a running status document could therefore land after the process had been stopped everywhere, putting the cluster back to running and drawing the other nodes into a process that was over.

The status write for a join now refuses when the document no longer shows a running process, and the joining node ends the run it launched instead.

Pre-review checklist

• Logging sensitive data? Make sure it's tagged (e.g. base.UD(docID), base.MD(dbName))
• Updated relevant information in the API specifications (such as endpoint descriptions, schemas, ...) in docs/api

Integration Tests

• https://jenkins.sgwdev.com/job/SyncGatewayIntegration-Pipeline/39/

[Copilot]

🟡 Changes recommended

An in-flight status callback can still restore the serialized running state after join rejection.

Get a fresh assessment by requesting another Copilot review.

Pull request overview

Prevents late multi-node joins from reviving completed background processes.

Changes:

• Adds a join-specific status update mode with terminal-state rejection.
• Terminates rejected local joins.
• Adds concurrent stop/join regression coverage.

File summaries

File	Description
db/background_mgr.go	Adds join-state validation and termination handling.
db/background_mgr_test.go	Enables and expands join/stop race tests.

Review details

• Files reviewed: 2/2 changed files
• Comments generated: 1
• Review effort level: Balanced

---

💡 Add a code-review agent skill or configure MCP servers for context-aware, tailored reviews. Learn more in the docs.

[factory-droid]

Droid finished @torcolvin's task —— View job

---

Review summary: The join-resurrection fix looks solid and the new tests cover the key races. One remaining gap is that multi-node Start claims can still overwrite a cluster-wide stop-in-progress state because start-mode status writes skip validating the current status doc.

[bbrks]

This CAS op is only matching Running, so when a local Stop happens when we're in Init() (from the cluster) the state can already be stopping and nothing moves it on to stopped.

It seems to leave the manager stuck in a stopping state, and every subsequent Start after just gets a 503 / BGTCompletionMaxWait hangs?

I have a repro test and I think the correct way to fix this is to push this complexity down into a helper behind the statusLock directly rather than CAS?

Suggested change

	// The cluster ended the process while this node was joining it, so end the run rather than
	// report this node running. A cluster still stopping leaves this node stopped, because there is
	// no Process.Run here to carry it the rest of the way.
	endState := stateErr.state
	if endState == BackgroundProcessStateStopping {
	endState = BackgroundProcessStateStopped
	}
	b.compareAndSwapRunState(BackgroundProcessStateRunning, endState)
	// The cluster ended the process while this node was joining it, so end the run rather than report this node running.
	b.endRunWithoutProcess(stateErr.state)

// endRunWithoutProcess records a terminal run state for a run that never launched Process.Run, so nothing else
// will carry the state the rest of the way. clusterState is the state the cluster ended on. A stop - the cluster's,
// or one that raced this node's claim - lands on stopped, since there is no run left to finish that transition.
func (b *BackgroundManager[O]) endRunWithoutProcess(clusterState BackgroundProcessState) {
	b.statusLock.Lock()
	defer b.statusLock.Unlock()
	switch b.status.State {
	case BackgroundProcessStateRunning:
		if clusterState == BackgroundProcessStateStopping {
			b.status.State = BackgroundProcessStateStopped
				return
			}
			b.status.State = clusterState
	case BackgroundProcessStateStopping:
		b.status.State = BackgroundProcessStateStopped
	}
}

// TestBackgroundManagerRefusedJoinAfterLocalStop covers a Join that is refused because the cluster ended the process
// while this node was joining it, with a Stop of this node landing in the same window. There is no Process.Run to
// carry the local run state on from stopping, so the refusal has to leave the node in a terminal state itself.
func TestBackgroundManagerRefusedJoinAfterLocalStop(t *testing.T) {
	testBucket := base.GetTestBucket(t)
	ctx := base.TestCtx(t)
	defer testBucket.Close(ctx)

	clusterAwareOptions := &ClusterAwareBackgroundManagerOptions{
		metadataStore: testBucket.DefaultDataStore(ctx),
		metaKeys:      base.NewMetadataKeys("test-join-local-stop"),
		processSuffix: "join-local-stop",
		multiNode:     true,
	}
	timeout := sgtest.GetBackgroundManagerStatusTransitionTimeout(t)

	runningNode := &BackgroundManager[MockProcessOptions]{
		name:                "join-local-stop-running",
		Process:             &MockProcess{},
		clusterAwareOptions: clusterAwareOptions,
	}
	require.NoError(t, runningNode.Start(ctx, MockProcessOptions{}))
	require.EventuallyWithT(t, func(c *assert.CollectT) {
		state, err := runningNode.getClusterStatusState(ctx)
		assert.NoError(c, err)
		assert.Equal(c, BackgroundProcessStateRunning, state)
	}, timeout, 10*time.Millisecond)

	process := &gatedInitProcess{initEntered: make(chan struct{}), releaseInit: make(chan struct{})}
	joiningNode := &BackgroundManager[MockProcessOptions]{
		name:                "join-local-stop-joining",
		Process:             process,
		clusterAwareOptions: clusterAwareOptions,
	}
	var joinErr error
	joined := make(chan struct{})
	go func() {
		defer close(joined)
		joinErr = joiningNode.Join(ctx)
	}()

	// The joining node is admitted against a running status document, and then held before it claims the run.
	base.RequireChanClosed(t, process.initEntered, "joining node did not reach Init")

	// A stop reaches the joining node itself while it is still inside Init, so its run state is already stopping
	// by the time the claim is refused.
	require.NoError(t, joiningNode.Stop(ctx))
	require.Equal(t, BackgroundProcessStateStopping, joiningNode.GetRunState())

	close(process.releaseInit)
	base.RequireChanClosed(t, joined, "Join did not return")
	require.NoError(t, joinErr)

	require.EventuallyWithT(t, func(c *assert.CollectT) {
		assert.True(c, joiningNode.GetRunState().isTerminal(), "run state stuck at %q", joiningNode.GetRunState())
	}, timeout, 10*time.Millisecond)
}

[bbrks]

Sorry for pasting Claude review output directly - this is the same symptom of resync getting stuck in a stopping state and no recovery though which is bad enough that I don't feel like this should be merged as-is.

If you want to just revisit background manager more generally with this and the other PR #8805 that's fine too

---

[high] refused join leaves the cluster status doc stuck at stopping

db/background_mgr.go:368

What breaks: The refused-join branch settles the local run state and returns, but never publishes a terminal status to the cluster. No Process.Run exists on that node to do it either (the claim now happens before Run is launched). If that same node's own Stop wrote "stopping" to the cluster doc, nothing ever carries it to "stopped" — the doc is stuck at stopping forever, and markStart then refuses every later Start with errBackgroundManagerStatusAlreadyStopping (503 Process currently stopping. Wait until stopped to retry). Resync is bricked for that database until the doc is edited by hand.

Trigger:

1. node1 Start → cluster doc running.
2. node2 Join, admitted against running, still inside Process.Init.
3. node1 Stop → cluster doc stopped.
4. node2 Stop — stopProcess writes Update-mode status with proposedState == stopping, so the new guard (proposedState == Running) does not fire and the doc goes back to stopping.
5. node2's Init returns; the join claim is refused (bucketState == stopping), node2 settles locally to stopped, writes nothing. Doc stays stopping.

func TestReproRefusedJoinLeavesClusterStopping(t *testing.T) {
      testBucket := base.GetTestBucket(t)
      ctx := base.TestCtx(t)
      defer testBucket.Close(ctx)

      clusterAwareOptions := &ClusterAwareBackgroundManagerOptions{
              metadataStore: testBucket.DefaultDataStore(ctx),
              metaKeys:      base.NewMetadataKeys("test-stuck-stopping"),
              processSuffix: "stuck-stopping",
              multiNode:     true,
      }
      timeout := sgtest.GetBackgroundManagerStatusTransitionTimeout(t)

      runningNode := &BackgroundManager[MockProcessOptions]{
              name: "stuck-stopping-running", Process: &MockProcess{}, clusterAwareOptions: clusterAwareOptions,
      }
      require.NoError(t, runningNode.Start(ctx, MockProcessOptions{}))
      require.EventuallyWithT(t, func(c *assert.CollectT) {
              state, err := runningNode.getClusterStatusState(ctx)
              assert.NoError(c, err)
              assert.Equal(c, BackgroundProcessStateRunning, state)
      }, timeout, 10*time.Millisecond)

      process := &gatedInitProcess{initEntered: make(chan struct{}), releaseInit: make(chan struct{})}
      joiningNode := &BackgroundManager[MockProcessOptions]{
              name: "stuck-stopping-joining", Process: process, clusterAwareOptions: clusterAwareOptions,
      }
      joined := make(chan struct{})
      var joinErr error
      go func() { defer close(joined); joinErr = joiningNode.Join(ctx) }()
      base.RequireChanClosed(t, process.initEntered, "joining node did not reach Init")

      require.NoError(t, runningNode.Stop(ctx))
      require.EventuallyWithT(t, func(c *assert.CollectT) {
              state, err := runningNode.getClusterStatusState(ctx)
              assert.NoError(c, err)
              assert.Equal(c, BackgroundProcessStateStopped, state)
      }, timeout, 10*time.Millisecond)

      // A stop reaches the joining node itself, which writes "stopping" over the stopped cluster doc.
      require.NoError(t, joiningNode.Stop(ctx))
      state, err := joiningNode.getClusterStatusState(ctx)
      require.NoError(t, err)
      require.Equal(t, BackgroundProcessStateStopping, state, "precondition: the local stop put the cluster doc back to stopping")

      close(process.releaseInit)
      base.RequireChanClosed(t, joined, "Join did not return")
      require.NoError(t, joinErr)

      // Nothing is left running anywhere, so the cluster doc has to settle on a terminal state.
      require.EventuallyWithT(t, func(c *assert.CollectT) {
              state, err := joiningNode.getClusterStatusState(ctx)
              assert.NoError(c, err)
              assert.True(c, state.isTerminal(), "cluster status doc stuck at %q", state)
      }, timeout, 10*time.Millisecond)

      // And a fresh Start must be admitted rather than refused as "currently stopping".
      nextNode := &BackgroundManager[MockProcessOptions]{
              name: "stuck-stopping-next", Process: &MockProcess{}, clusterAwareOptions: clusterAwareOptions,
      }
      require.NoError(t, nextNode.Start(ctx, MockProcessOptions{}))
      require.NoError(t, nextNode.Stop(ctx))
}