diff --git a/posthog_flutter/lib/src/core/file_storage.dart b/posthog_flutter/lib/src/core/file_storage.dart index 8c550457..58d3691b 100644 --- a/posthog_flutter/lib/src/core/file_storage.dart +++ b/posthog_flutter/lib/src/core/file_storage.dart @@ -562,6 +562,27 @@ class FileEventQueue { selected.add(id); bytes += size; } + return _read(selected).then((events) { + // An empty peek ends the flush, which must not leave the events behind + // the unreadable ones unsent. + return events.isEmpty && selected.isNotEmpty + ? _peekWritten(count, maxBytes, skipped: {...skipped, ...selected}) + : events; + }); + } + + /// Returns the events with the given [ids], in that order, or null when + /// one of them is no longer queued or cannot be read now. + Future?> peekIds(List ids) { + final queued = _ids.toSet(); + if (!ids.every(queued.contains)) return SynchronousFuture(null); + return _read(ids) + .then((events) => events.length == ids.length ? events : null); + } + + /// The [selected] events still queued, read by the worker unless they are + /// kept in memory. + Future> _read(List selected) { final toRead = [ for (final id in selected) if (!_unwritten.containsKey(id)) id, @@ -569,16 +590,9 @@ class FileEventQueue { final read = toRead.isEmpty ? SynchronousFuture(const []) : _job(_worker.read([for (final id in toRead) _pathOf(id)])); - return read.then((read) { - final events = _collect(selected, { - for (var i = 0; i < toRead.length; i++) toRead[i]: read[i], - }); - // An empty peek ends the flush, which must not leave the events behind - // the unreadable ones unsent. - return events.isEmpty && selected.isNotEmpty - ? _peekWritten(count, maxBytes, skipped: {...skipped, ...selected}) - : events; - }); + return read.then((read) => _collect(selected, { + for (var i = 0; i < toRead.length; i++) toRead[i]: read[i], + })); } /// The [selected] events still queued, from memory or from [read]. diff --git a/posthog_flutter/lib/src/core/posthog_core_stateless.dart b/posthog_flutter/lib/src/core/posthog_core_stateless.dart index 1c5b255b..7ad6b62d 100644 --- a/posthog_flutter/lib/src/core/posthog_core_stateless.dart +++ b/posthog_flutter/lib/src/core/posthog_core_stateless.dart @@ -705,8 +705,7 @@ abstract class PostHogCoreStateless { // Peeked again after every request, so events queued in the meantime // go out with the same flush. The queue sizes the batch before // reading it, so events past the batch are not decoded. - final batch = - await queue.peek(lane.maxBatchSize, maxBytes: captureV1MaxBatchBytes); + final batch = await _nextBatch(lane); if (_closed) break; if (batch.isEmpty) { // The next peek waits for the event still being written. @@ -718,6 +717,20 @@ abstract class PostHogCoreStateless { } } + /// The events of the request waiting to be retried, alone, so that the + /// retry keeps its request id and `created_at`; otherwise the next batch + /// from the front of the queue. Once one of the events to retry is gone, + /// e.g. dropped to make room, the request starts over as a new one. + Future> _nextBatch(_CaptureLane lane) async { + final retryIds = lane.retryIds; + lane.retryIds = null; + if (retryIds != null) { + final retry = await lane.queue.peekIds(retryIds); + if (retry != null) return retry; + } + return lane.queue.peek(lane.maxBatchSize, maxBytes: captureV1MaxBatchBytes); + } + /// Sends [batch] to capture v1 and removes from the queue the events that /// are done: delivered, or dropped for good. Emits `flush` with the /// delivered ones right away, so a long flush doesn't hold on to the @@ -766,6 +779,7 @@ abstract class PostHogCoreStateless { // until the next capture (forever in an idle app after an offline // failure). if (_isTransientCaptureError(e)) { + lane.retryIds = [for (final queued in batch) queued.id]; _backOff(lane, e is PostHogFetchHttpError ? e.retryAfter : null); } else { queue.remove([for (final queued in batch) queued.id]); @@ -793,6 +807,10 @@ abstract class PostHogCoreStateless { return true; } + lane.retryIds = [ + for (final queued in batch) + if (retryUuids.contains(queued.event['uuid'])) queued.id, + ]; // Counts only: uuids and server details can identify users. logger.info('Capture asked to retry ${batch.length - delivered.length} ' 'of ${batch.length} events.'); @@ -1143,6 +1161,10 @@ class _CaptureLane { /// retried. Kept in memory: after a restart the retry is a new request. CaptureV1Identity? pendingIdentity; + /// The queue ids of the events of [pendingIdentity], which the next + /// request sends alone, before any newer event. + List? retryIds; + /// Flushes in a row whose batch failed with a transient error or had /// events the server asked to retry. Each one pauses the queue for longer, /// see [captureRetryDelay], so a failure that lasts, like a missing diff --git a/posthog_flutter/test/core/capture_v1_test.dart b/posthog_flutter/test/core/capture_v1_test.dart index ecc97a4c..eb5c1362 100644 --- a/posthog_flutter/test/core/capture_v1_test.dart +++ b/posthog_flutter/test/core/capture_v1_test.dart @@ -483,28 +483,67 @@ void main() { }); }); - test('starts a new request when new events join a retried batch', () { + for (final (name, firstResponse) in [ + ( + 'a partial retry', + (PostHogRequest request) => PostHogResponse.results( + request, (name) => name == 'retried' ? 'retry' : null), + ), + ( + 'a failed request', + (PostHogRequest _) => const PostHogResponse(HttpStatus.badGateway), + ), + ]) { + test('resends $name alone, then sends newer events as a new request', () { + final api = InProcessPostHogApi(); + final storage = tempStorage(); + api.respond = (request) => api.batchRequests.length == 1 + ? firstResponse(request) + : PostHogResponse.results(request, (_) => 'ok'); + fakeAsync((async) { + final client = testClient(api, storage: storage); + client.capture('retried'); + client.flush().catchError((Object _) {}); + async.elapse(Duration.zero); + client.capture('joined'); + async.elapse(const Duration(seconds: 1)); + + final [first, retry, next] = api.batchRequests; + expect(retry.eventNames, ['retried']); + expect(retry.headers['posthog-request-id'], + first.headers['posthog-request-id']); + expect(retry.headers['posthog-attempt'], '2'); + expect(retry.body['created_at'], first.body['created_at']); + expect(next.eventNames, ['joined']); + expect(next.headers['posthog-request-id'], + isNot(first.headers['posthog-request-id'])); + expect(next.headers['posthog-attempt'], '1'); + expect(getQueue(storage), isEmpty); + }); + }); + } + + test('starts a new request once an event to retry is dropped', () { final api = InProcessPostHogApi(); final storage = tempStorage(); api.respond = (request) => api.batchRequests.length == 1 - ? PostHogResponse.results( - request, (name) => name == 'retried' ? 'retry' : null, - headers: {'Retry-After': '5'}) + ? const PostHogResponse(HttpStatus.badGateway) : PostHogResponse.results(request, (_) => 'ok'); fakeAsync((async) { - final client = testClient(api, storage: storage); - client.capture('retried'); - client.flush(); + final client = testClient(api, + config: testConfig(maxQueueSize: 2), storage: storage); + client.capture('dropped'); + client.capture('kept'); + client.flush().catchError((Object _) {}); async.elapse(Duration.zero); client.capture('joined'); - async.elapse(const Duration(seconds: 5)); + async.elapse(const Duration(seconds: 1)); final [first, second] = api.batchRequests; - expect(second.eventNames, ['retried', 'joined']); + expect(second.eventNames, ['kept', 'joined']); expect(second.headers['posthog-request-id'], isNot(first.headers['posthog-request-id'])); expect(second.headers['posthog-attempt'], '1'); - expect(getQueue(storage), isEmpty); }); }); diff --git a/posthog_flutter/test/core/queue_flush_test.dart b/posthog_flutter/test/core/queue_flush_test.dart index d839fa77..194f6212 100644 --- a/posthog_flutter/test/core/queue_flush_test.dart +++ b/posthog_flutter/test/core/queue_flush_test.dart @@ -417,17 +417,17 @@ void main() { expect(queuedEvents(storage), ['first', 'second']); async.elapse(const Duration(seconds: 1)); - expect(api.batchRequests, hasLength(2)); - expect(api.batchRequests.last.eventNames, ['first', 'second']); - expect(api.batchRequests.last.events.first['uuid'], original['uuid']); - expect(api.batchRequests.last.events.first['timestamp'], - original['timestamp']); + final [_, retry, next] = api.batchRequests; + expect(retry.eventNames, ['first']); + expect(retry.events.single['uuid'], original['uuid']); + expect(retry.events.single['timestamp'], original['timestamp']); + expect(next.eventNames, ['second']); expect(getQueue(storage), isEmpty); client.capture('third'); client.capture('fourth'); async.elapse(Duration.zero); - expect(api.batchRequests, hasLength(3), + expect(api.batchRequests, hasLength(4), reason: 'the previous pause does not apply after a success'); }); }); @@ -522,10 +522,11 @@ void main() { async.elapse(const Duration(seconds: 19)); expect(api.batchRequests, hasLength(1)); async.elapse(const Duration(seconds: 1)); - final events = api.batchRequests.last.events; - expect(events.map((event) => event['event']), ['before', 'after']); - expect(events[0]['distinct_id'], previousId); - expect(events[1]['distinct_id'], nextId); + final [_, before, after] = api.batchRequests; + expect(before.eventNames, ['before']); + expect(before.events.single['distinct_id'], previousId); + expect(after.eventNames, ['after']); + expect(after.events.single['distinct_id'], nextId); }); }); @@ -546,8 +547,9 @@ void main() { client.flush(); client.flush(); async.elapse(Duration.zero); - expect(api.batchRequests, hasLength(2)); - expect(api.batchRequests.last.eventNames, ['first', 'second']); + final [_, retry, next] = api.batchRequests; + expect(retry.eventNames, ['first']); + expect(next.eventNames, ['second']); }); });