Fix YouTube WebSub topic and back off subscription failures

src/statichost/StaticHost/Live/LiveEndpoints.cs

@@ -334,6 +334,8 @@ private static async Task<IResult> YouTubeVerify(
             return Results.NotFound();
         }
 
+        logger.LogInformation("YouTube WebSub subscription verified for {Topic}; granted lease {LeaseSeconds}s.",
+            topic, leaseSeconds);
         return Results.Text(challenge, "text/plain");
     }
 

src/statichost/StaticHost/Live/LiveStatusServiceCollectionExtensions.cs

@@ -12,7 +12,7 @@ public static class LiveStatusServiceCollectionExtensions
     /// Registers Redis-backed canonical state, a per-process
     /// <see cref="LiveStatusBroadcaster"/>, named
     /// <see cref="HttpClient"/> instances (<c>"twitch"</c>, <c>"twitch-id"</c>,
-    /// <c>"youtube"</c>, <c>"youtube-pubsub"</c>) with standard resilience, the
+    /// <c>"youtube"</c>, <c>"youtube-pubsub"</c>), the
     /// Twitch token provider + Helix client, the YouTube Data + WebSub client,
     /// and the two <see cref="BackgroundService"/> workers
     /// (<see cref="TwitchEventSubService"/> and <see cref="YouTubeWebSubService"/>).
@@ -49,8 +49,9 @@ public static TBuilder AddLiveStatus<TBuilder>(this TBuilder builder)
             .AddStandardResilienceHandler();
         builder.Services.AddHttpClient(YouTubeClient.HttpClientName)
             .AddStandardResilienceHandler();
-        builder.Services.AddHttpClient(YouTubeClient.PubSubHttpClientName)
-            .AddStandardResilienceHandler();
+        // Subscription POST retries are scheduled in Redis, not inside the HTTP request.
+        builder.Services.AddHttpClient(YouTubeClient.PubSubHttpClientName,
+            client => client.Timeout = TimeSpan.FromSeconds(30));
 
         builder.Services.AddSingleton<TwitchAppTokenProvider>();
         builder.Services.AddSingleton<ITwitchClient, TwitchClient>();

src/statichost/StaticHost/Live/README.md

@@ -33,9 +33,35 @@ Two `BackgroundService` workers keep the state current:
 | `YouTubeWebSubService`    | PubSubHubbub `videos.xml` push   | `videos.list` every 2 min while live; `search.list` every 30 min while idle |
 
 Webhook signature and parsing logic remains separate and unit-testable.
-Outgoing HTTP is performed by named, resilient
-`HttpClient` instances (`twitch`, `twitch-id`, `youtube`,
-`youtube-pubsub`) registered with `AddStandardResilienceHandler`.
+The `twitch`, `twitch-id`, and `youtube` named HTTP clients use
+`AddStandardResilienceHandler`. The `youtube-pubsub` client instead uses a
+30-second timeout without in-request retries. Redis schedules subscription
+retries separately from live-status polling.
+
+### YouTube subscription failures
+
+Both subscription requests and verification use Google's documented topic:
+`https://www.youtube.com/feeds/videos.xml?channel_id=CHANNEL_ID`.
+
+Failed subscription requests back off on a 2, 5, 15, 30, then 60-minute schedule,
+with each delay jittered between 90% and 100% of that duration. The worker
+retries on its first normal tick at or after the persisted deadline. Failure
+counts and deadlines are shared in Redis so restarts or leader changes do not
+restart the retry sequence. Successful verification resets the backoff;
+acceptance of the subscription POST alone does not. A channel change starts a
+fresh sequence.
+
+Each failed attempt logs one concise warning for expected HTTP errors or
+timeouts, including the failure count and next retry deadline. Exception
+details and hub error response bodies are available at Debug; unexpected
+exceptions remain Error-level with their stack traces. Successful verification
+is logged at Information. Shutdown or leadership cancellation does not count
+as an upstream failure.
+
+The live and idle polling intervals remain unchanged during subscription
+backoff. Google documents feed notifications for uploads and video
+title/description updates, not guaranteed broadcast start/stop events, so
+fallback polling is still necessary.
 
 ## Configuration
 
@@ -345,8 +371,11 @@ project or `Aspire.Dev.slnx` trigger that job.
 - Existing SSE connections retain the last local snapshot during a temporary
   Redis interruption. New snapshot and SSE requests require canonical Redis
   state.
-- Provider loops log failures and retry on their configured intervals.
-- Standard resilience pipeline on every named `HttpClient`.
+- Provider loops log failures and retry on their configured intervals, with
+  separate shared backoff for failed YouTube subscription requests.
+- Standard resilience pipelines protect Data API and Twitch requests;
+  YouTube subscription POSTs use the bounded timeout and shared retry schedule
+  described above.
 - Reconciliation timers are the safety net for missed individual webhooks.
 - SSE heartbeats every 15 s defeat proxy idle-timeouts; the client uses
   exponential backoff with a `visibilitychange`-aware reconnect.

src/statichost/StaticHost/Live/YouTube/RedisYouTubeWebSubSubscriptionState.cs

@@ -25,14 +25,19 @@ internal sealed class RedisYouTubeWebSubSubscriptionState(
             },
             cancellationToken);
 
-    public async ValueTask MarkRequestFailedAsync(
+    public async ValueTask<YouTubeWebSubRetryState?> MarkRequestFailedAsync(
         YouTubeWebSubSubscriptionRequest request,
         CancellationToken cancellationToken = default)
     {
-        _ = await UpdateAsync(
-            current => new StateTransition<bool>(
-                YouTubeWebSubSubscriptionTransitions.MarkRequestFailed(current, request),
-                true),
+        var failedAt = timeProvider.GetUtcNow();
+        return await UpdateAsync(
+            current =>
+            {
+                var next = YouTubeWebSubSubscriptionTransitions.MarkRequestFailed(current, request, failedAt);
+                return new StateTransition<YouTubeWebSubRetryState?>(
+                    next,
+                    next == current ? null : next.Retry);
+            },
             cancellationToken).ConfigureAwait(false);
     }
 

src/statichost/StaticHost/Live/YouTube/YouTubeClient.cs

@@ -114,8 +114,8 @@ public async Task SubscribeAsync(
         var c = httpFactory.CreateClient(PubSubHttpClientName);
         c.BaseAddress ??= new Uri("https://pubsubhubbub.appspot.com/");
 
-        var topic = $"https://www.youtube.com/xml/feeds/videos.xml?channel_id={channelId}";
-        var form = new FormUrlEncodedContent(
+        var topic = YouTubeWebSubSubscriptionTransitions.TopicFor(channelId);
+        using var form = new FormUrlEncodedContent(
         [
             new KeyValuePair<string,string>("hub.callback", callbackUrl),
             new KeyValuePair<string,string>("hub.topic", topic),
@@ -130,7 +130,7 @@ public async Task SubscribeAsync(
         if (!response.IsSuccessStatusCode)
         {
             var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
-            logger.LogWarning("YouTube WebSub subscribe failed: {Status} {Body}", response.StatusCode, body);
+            logger.LogDebug("YouTube WebSub subscribe failed: {Status} {Body}", response.StatusCode, body);
             response.EnsureSuccessStatusCode();
         }
     }

src/statichost/StaticHost/Live/YouTube/YouTubeWebSubService.cs

@@ -1,4 +1,5 @@
 using Microsoft.Extensions.Options;
+using Polly.Timeout;
 
 namespace StaticHost.Live.YouTube;
 
@@ -128,17 +129,32 @@ await client.SubscribeAsync(
             }
             catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
             {
-                await _subscriptions.MarkRequestFailedAsync(
-                    request,
-                    CancellationToken.None).ConfigureAwait(false);
+                // Let the short send reservation expire on shutdown or leadership loss.
                 throw;
             }
             catch (Exception ex)
             {
-                await _subscriptions.MarkRequestFailedAsync(
+                var retry = await _subscriptions.MarkRequestFailedAsync(
                     request,
                     CancellationToken.None).ConfigureAwait(false);
-                logger.LogWarning(ex, "YouTube WebSub subscribe failed; will retry next tick.");
+                if (ex is HttpRequestException or OperationCanceledException or TimeoutRejectedException)
+                {
+                    if (retry is not null)
+                    {
+                        logger.LogWarning(
+                            "YouTube WebSub subscribe failed ({FailureType}, HTTP {StatusCode}); attempt {FailureCount}. " +
+                            "Next subscription attempt no earlier than {RetryAt}. Live-status polling continues.",
+                            ex.GetType().Name,
+                            (ex as HttpRequestException)?.StatusCode,
+                            retry.FailureCount,
+                            retry.RetryAt);
+                    }
+                    logger.LogDebug(ex, "YouTube WebSub subscription request failure details.");
+                }
+                else
+                {
+                    logger.LogError(ex, "Unexpected YouTube WebSub subscription failure.");
+                }
             }
 
             if (requestSent)

src/statichost/StaticHost/Live/YouTube/YouTubeWebSubSubscriptionState.cs

@@ -15,7 +15,7 @@ public interface IYouTubeWebSubSubscriptionState
         DateTimeOffset now,
         CancellationToken cancellationToken = default);
 
-    ValueTask MarkRequestFailedAsync(
+    ValueTask<YouTubeWebSubRetryState?> MarkRequestFailedAsync(
         YouTubeWebSubSubscriptionRequest request,
         CancellationToken cancellationToken = default);
 
@@ -41,12 +41,19 @@ public sealed record YouTubeWebSubSubscriptionRequest(
     DateTimeOffset RequestedAt,
     DateTimeOffset? SentAt = null);
 
+public sealed record YouTubeWebSubRetryState(
+    string Topic,
+    int FailureCount,
+    DateTimeOffset RetryAt);
+
 internal sealed record YouTubeWebSubSubscriptionData(
     YouTubeWebSubSubscriptionRequest? Pending,
     string? ActiveTopic,
     DateTimeOffset RenewAt,
     YouTubeWebSubConfirmation? RecentConfirmation = null)
 {
+    public YouTubeWebSubRetryState? Retry { get; init; }
+
     public static YouTubeWebSubSubscriptionData Empty { get; } = new(
         Pending: null,
         ActiveTopic: null,
@@ -79,6 +86,14 @@ internal static class YouTubeWebSubSubscriptionTransitions
         out YouTubeWebSubSubscriptionData next)
     {
         var topic = TopicFor(channelId);
+        if (current.Retry is { } retry &&
+            string.Equals(retry.Topic, topic, StringComparison.Ordinal) &&
+            now < retry.RetryAt)
+        {
+            next = current;
+            return null;
+        }
+
         var pending = current.Pending;
 
         if (pending is not null)
@@ -110,14 +125,41 @@ internal static class YouTubeWebSubSubscriptionTransitions
             Convert.ToHexStringLower(RandomNumberGenerator.GetBytes(32)),
             now);
 
-        next = current with { Pending = request, RecentConfirmation = null };
+        next = current with
+        {
+            Pending = request,
+            RecentConfirmation = null,
+            Retry = string.Equals(current.Retry?.Topic, topic, StringComparison.Ordinal) ? current.Retry : null,
+        };
         return request;
     }
 
     public static YouTubeWebSubSubscriptionData MarkRequestFailed(
         YouTubeWebSubSubscriptionData current,
-        YouTubeWebSubSubscriptionRequest request) =>
-        RequestsMatch(current.Pending, request) ? current with { Pending = null } : current;
+        YouTubeWebSubSubscriptionRequest request,
+        DateTimeOffset failedAt)
+    {
+        if (!RequestsMatch(current.Pending, request))
+        {
+            return current;
+        }
+
+        var failures = (int)Math.Min((long)(current.Retry?.FailureCount ?? 0) + 1, int.MaxValue);
+        var minutes = failures switch
+        {
+            1 => 2,
+            2 => 5,
+            3 => 15,
+            4 => 30,
+            _ => 60,
+        };
+        var delay = TimeSpan.FromMinutes(minutes * (0.9 + Random.Shared.NextDouble() * 0.1));
+        return current with
+        {
+            Pending = null,
+            Retry = new YouTubeWebSubRetryState(request.Topic, failures, failedAt.Add(delay)),
+        };
+    }
 
     public static YouTubeWebSubSubscriptionData MarkRequestSent(
         YouTubeWebSubSubscriptionData current,
@@ -185,8 +227,8 @@ public static bool HasValidConfirmationShape(
         !string.IsNullOrEmpty(verifyToken) &&
         leaseSeconds > 0;
 
-    private static string TopicFor(string channelId) =>
-        $"https://www.youtube.com/xml/feeds/videos.xml?channel_id={channelId}";
+    internal static string TopicFor(string channelId) =>
+        $"https://www.youtube.com/feeds/videos.xml?channel_id={channelId}";
 
     private static bool RequestsMatch(
         YouTubeWebSubSubscriptionRequest? current,

tests/StaticHost.Tests/Live/InMemoryLiveStatusInfrastructure.cs

@@ -112,7 +112,7 @@ public ValueTask DisposeAsync()
     }
 }
 
-internal sealed class YouTubeWebSubSubscriptionState : IYouTubeWebSubSubscriptionState
+internal sealed class YouTubeWebSubSubscriptionState(TimeProvider? timeProvider = null) : IYouTubeWebSubSubscriptionState
 {
     private readonly Lock _gate = new();
     private YouTubeWebSubSubscriptionData _state = YouTubeWebSubSubscriptionData.Empty;
@@ -135,18 +135,19 @@ internal sealed class YouTubeWebSubSubscriptionState : IYouTubeWebSubSubscriptio
         }
     }
 
-    public ValueTask MarkRequestFailedAsync(
+    public ValueTask<YouTubeWebSubRetryState?> MarkRequestFailedAsync(
         YouTubeWebSubSubscriptionRequest request,
         CancellationToken cancellationToken = default)
     {
         cancellationToken.ThrowIfCancellationRequested();
 
         lock (_gate)
         {
-            _state = YouTubeWebSubSubscriptionTransitions.MarkRequestFailed(_state, request);
+            var previous = _state;
+            _state = YouTubeWebSubSubscriptionTransitions.MarkRequestFailed(
+                _state, request, (timeProvider ?? TimeProvider.System).GetUtcNow());
+            return ValueTask.FromResult(_state == previous ? null : _state.Retry);
         }
-
-        return ValueTask.CompletedTask;
     }
 
     public ValueTask MarkRequestSentAsync(

tests/StaticHost.Tests/Live/RedisYouTubeIntegrationTests.cs

@@ -222,23 +222,24 @@ public async Task WebSubReplacement_RejectsAbandonedOwnersLateWritesAndVerificat
     }
 
     [Fact]
-    public async Task WebSubFailedSend_AllowsAnotherWorkerToRetryWithoutClearingItsReservation()
+    public async Task WebSubFailedSend_SharesBackoffWithoutClearingAnotherWorkersReservation()
     {
         var now = DateTimeOffset.UtcNow;
         var first = SubscriptionState(Redis.First);
         var second = SubscriptionState(Redis.Second);
         var failed = Assert.IsType<YouTubeWebSubSubscriptionRequest>(
             await first.TryBeginSubscriptionAsync("channel-123", now));
-        await first.MarkRequestFailedAsync(failed);
+        var failure = Assert.IsType<YouTubeWebSubRetryState>(await first.MarkRequestFailedAsync(failed));
+        Assert.Null(await second.TryBeginSubscriptionAsync("channel-123", failure.RetryAt.AddTicks(-1)));
         var retry = Assert.IsType<YouTubeWebSubSubscriptionRequest>(
-            await second.TryBeginSubscriptionAsync("channel-123", now.AddSeconds(1)));
-        await second.MarkRequestSentAsync(retry, now.AddSeconds(2));
+            await second.TryBeginSubscriptionAsync("channel-123", failure.RetryAt));
+        await second.MarkRequestSentAsync(retry, failure.RetryAt.AddSeconds(1));
 
         await first.MarkRequestFailedAsync(failed);
         await first.MarkRequestSentAsync(failed, now.AddMinutes(1));
-        Assert.Null(await first.TryBeginSubscriptionAsync("channel-123", now.AddMinutes(9)));
+        Assert.Null(await first.TryBeginSubscriptionAsync("channel-123", failure.RetryAt.AddMinutes(9)));
         Assert.True(await first.TryConfirmSubscriptionAsync(
-            "subscribe", retry.Topic, retry.VerifyToken, 3_600, now.AddMinutes(9)));
+            "subscribe", retry.Topic, retry.VerifyToken, 3_600, failure.RetryAt.AddMinutes(9)));
     }
 
     private static RedisYouTubeWebSubSubscriptionState SubscriptionState(IConnectionMultiplexer connection) =>

tests/StaticHost.Tests/Live/YouTubeClientTests.cs

@@ -1,3 +1,5 @@
+using Microsoft.Extensions.Hosting;
+
 namespace StaticHost.Tests.Live;
 
 public sealed class YouTubeClientTests
@@ -117,7 +119,27 @@ await client.SubscribeAsync(
         Assert.Contains("hub.secret=webhook-secret", request.Content, StringComparison.Ordinal);
         Assert.Contains("hub.lease_seconds=432000", request.Content, StringComparison.Ordinal);
         Assert.Contains("hub.callback=https%3A%2F%2Fexample.com%2Fapi%2Flive%2Fyoutube%2Fwebhook", request.Content, StringComparison.Ordinal);
-        Assert.Contains("hub.topic=https%3A%2F%2Fwww.youtube.com%2Fxml%2Ffeeds%2Fvideos.xml%3Fchannel_id%3Dchannel-123", request.Content, StringComparison.Ordinal);
+        Assert.Contains("hub.topic=https%3A%2F%2Fwww.youtube.com%2Ffeeds%2Fvideos.xml%3Fchannel_id%3Dchannel-123", request.Content, StringComparison.Ordinal);
+    }
+
+    [Fact]
+    public async Task PubSubHttpClient_HasBoundedTimeoutAndDoesNotRetryPosts()
+    {
+        var builder = Host.CreateApplicationBuilder();
+        builder.AddLiveStatus();
+        var handler = new RecordingHttpMessageHandler(_ => new HttpResponseMessage(HttpStatusCode.ServiceUnavailable));
+        builder.Services.AddHttpClient(YouTubeClient.PubSubHttpClientName)
+            .ConfigurePrimaryHttpMessageHandler(() => handler);
+        using var host = builder.Build();
+        var client = host.Services.GetRequiredService<IHttpClientFactory>()
+            .CreateClient(YouTubeClient.PubSubHttpClientName);
+        using var form = new FormUrlEncodedContent([]);
+
+        using var response = await client.PostAsync("https://pubsubhubbub.appspot.com/subscribe", form);
+
+        Assert.Equal(TimeSpan.FromSeconds(30), client.Timeout);
+        Assert.Equal(HttpStatusCode.ServiceUnavailable, response.StatusCode);
+        Assert.Single(handler.Requests);
     }
 
     private static YouTubeClient CreateClient(