Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 24 additions & 10 deletions posthog_flutter/lib/src/core/file_storage.dart
Original file line number Diff line number Diff line change
Expand Up @@ -562,23 +562,37 @@ class FileEventQueue {
selected.add(id);
bytes += size;
}
return _read(selected).then((events) {
// An empty peek ends the flush, which must not leave the events behind
// the unreadable ones unsent.
return events.isEmpty && selected.isNotEmpty
? _peekWritten(count, maxBytes, skipped: {...skipped, ...selected})
: events;
});
}

/// Returns the events with the given [ids], in that order, or null when
/// one of them is no longer queued or cannot be read now.
Future<List<PostHogQueuedEvent>?> peekIds(List<String> ids) {
final queued = _ids.toSet();
if (!ids.every(queued.contains)) return SynchronousFuture(null);
return _read(ids)
.then((events) => events.length == ids.length ? events : null);
}

/// The [selected] events still queued, read by the worker unless they are
/// kept in memory.
Future<List<PostHogQueuedEvent>> _read(List<String> selected) {
final toRead = [
for (final id in selected)
if (!_unwritten.containsKey(id)) id,
];
final read = toRead.isEmpty
? SynchronousFuture(const <QueueReadResult>[])
: _job(_worker.read([for (final id in toRead) _pathOf(id)]));
return read.then((read) {
final events = _collect(selected, {
for (var i = 0; i < toRead.length; i++) toRead[i]: read[i],
});
// An empty peek ends the flush, which must not leave the events behind
// the unreadable ones unsent.
return events.isEmpty && selected.isNotEmpty
? _peekWritten(count, maxBytes, skipped: {...skipped, ...selected})
: events;
});
return read.then((read) => _collect(selected, {
for (var i = 0; i < toRead.length; i++) toRead[i]: read[i],
}));
}

/// The [selected] events still queued, from memory or from [read].
Expand Down
26 changes: 24 additions & 2 deletions posthog_flutter/lib/src/core/posthog_core_stateless.dart
Original file line number Diff line number Diff line change
Expand Up @@ -705,8 +705,7 @@ abstract class PostHogCoreStateless {
// Peeked again after every request, so events queued in the meantime
// go out with the same flush. The queue sizes the batch before
// reading it, so events past the batch are not decoded.
final batch =
await queue.peek(lane.maxBatchSize, maxBytes: captureV1MaxBatchBytes);
final batch = await _nextBatch(lane);
if (_closed) break;
if (batch.isEmpty) {
// The next peek waits for the event still being written.
Expand All @@ -718,6 +717,20 @@ abstract class PostHogCoreStateless {
}
}

/// The events of the request waiting to be retried, alone, so that the
/// retry keeps its request id and `created_at`; otherwise the next batch
/// from the front of the queue. Once one of the events to retry is gone,
/// e.g. dropped to make room, the request starts over as a new one.
Future<List<PostHogQueuedEvent>> _nextBatch(_CaptureLane lane) async {
final retryIds = lane.retryIds;
lane.retryIds = null;
if (retryIds != null) {
final retry = await lane.queue.peekIds(retryIds);
if (retry != null) return retry;
}
return lane.queue.peek(lane.maxBatchSize, maxBytes: captureV1MaxBatchBytes);
}

/// Sends [batch] to capture v1 and removes from the queue the events that
/// are done: delivered, or dropped for good. Emits `flush` with the
/// delivered ones right away, so a long flush doesn't hold on to the
Expand Down Expand Up @@ -766,6 +779,7 @@ abstract class PostHogCoreStateless {
// until the next capture (forever in an idle app after an offline
// failure).
if (_isTransientCaptureError(e)) {
lane.retryIds = [for (final queued in batch) queued.id];
_backOff(lane, e is PostHogFetchHttpError ? e.retryAfter : null);
} else {
queue.remove([for (final queued in batch) queued.id]);
Expand Down Expand Up @@ -793,6 +807,10 @@ abstract class PostHogCoreStateless {
return true;
}

lane.retryIds = [
for (final queued in batch)
if (retryUuids.contains(queued.event['uuid'])) queued.id,
];
// Counts only: uuids and server details can identify users.
logger.info('Capture asked to retry ${batch.length - delivered.length} '
'of ${batch.length} events.');
Expand Down Expand Up @@ -1143,6 +1161,10 @@ class _CaptureLane {
/// retried. Kept in memory: after a restart the retry is a new request.
CaptureV1Identity? pendingIdentity;

/// The queue ids of the events of [pendingIdentity], which the next
/// request sends alone, before any newer event.
List<String>? retryIds;

/// Flushes in a row whose batch failed with a transient error or had
/// events the server asked to retry. Each one pauses the queue for longer,
/// see [captureRetryDelay], so a failure that lasts, like a missing
Expand Down
59 changes: 49 additions & 10 deletions posthog_flutter/test/core/capture_v1_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -483,28 +483,67 @@ void main() {
});
});

test('starts a new request when new events join a retried batch', () {
for (final (name, firstResponse) in [
(
'a partial retry',
(PostHogRequest request) => PostHogResponse.results(
request, (name) => name == 'retried' ? 'retry' : null),
),
(
'a failed request',
(PostHogRequest _) => const PostHogResponse(HttpStatus.badGateway),
),
]) {
test('resends $name alone, then sends newer events as a new request', () {
final api = InProcessPostHogApi();
final storage = tempStorage();
api.respond = (request) => api.batchRequests.length == 1
? firstResponse(request)
: PostHogResponse.results(request, (_) => 'ok');
fakeAsync((async) {
final client = testClient(api, storage: storage);
client.capture('retried');
client.flush().catchError((Object _) {});
async.elapse(Duration.zero);
client.capture('joined');
async.elapse(const Duration(seconds: 1));

final [first, retry, next] = api.batchRequests;
expect(retry.eventNames, ['retried']);
expect(retry.headers['posthog-request-id'],
first.headers['posthog-request-id']);
expect(retry.headers['posthog-attempt'], '2');
expect(retry.body['created_at'], first.body['created_at']);
expect(next.eventNames, ['joined']);
expect(next.headers['posthog-request-id'],
isNot(first.headers['posthog-request-id']));
expect(next.headers['posthog-attempt'], '1');
expect(getQueue(storage), isEmpty);
});
});
}

test('starts a new request once an event to retry is dropped', () {
final api = InProcessPostHogApi();
final storage = tempStorage();
api.respond = (request) => api.batchRequests.length == 1
? PostHogResponse.results(
request, (name) => name == 'retried' ? 'retry' : null,
headers: {'Retry-After': '5'})
? const PostHogResponse(HttpStatus.badGateway)
: PostHogResponse.results(request, (_) => 'ok');
fakeAsync((async) {
final client = testClient(api, storage: storage);
client.capture('retried');
client.flush();
final client = testClient(api,
config: testConfig(maxQueueSize: 2), storage: storage);
client.capture('dropped');
client.capture('kept');
client.flush().catchError((Object _) {});
async.elapse(Duration.zero);
client.capture('joined');
async.elapse(const Duration(seconds: 5));
async.elapse(const Duration(seconds: 1));

final [first, second] = api.batchRequests;
expect(second.eventNames, ['retried', 'joined']);
expect(second.eventNames, ['kept', 'joined']);
expect(second.headers['posthog-request-id'],
isNot(first.headers['posthog-request-id']));
expect(second.headers['posthog-attempt'], '1');
expect(getQueue(storage), isEmpty);
});
});

Expand Down
26 changes: 14 additions & 12 deletions posthog_flutter/test/core/queue_flush_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -417,17 +417,17 @@ void main() {
expect(queuedEvents(storage), ['first', 'second']);

async.elapse(const Duration(seconds: 1));
expect(api.batchRequests, hasLength(2));
expect(api.batchRequests.last.eventNames, ['first', 'second']);
expect(api.batchRequests.last.events.first['uuid'], original['uuid']);
expect(api.batchRequests.last.events.first['timestamp'],
original['timestamp']);
final [_, retry, next] = api.batchRequests;
expect(retry.eventNames, ['first']);
expect(retry.events.single['uuid'], original['uuid']);
expect(retry.events.single['timestamp'], original['timestamp']);
expect(next.eventNames, ['second']);
expect(getQueue(storage), isEmpty);

client.capture('third');
client.capture('fourth');
async.elapse(Duration.zero);
expect(api.batchRequests, hasLength(3),
expect(api.batchRequests, hasLength(4),
reason: 'the previous pause does not apply after a success');
});
});
Expand Down Expand Up @@ -522,10 +522,11 @@ void main() {
async.elapse(const Duration(seconds: 19));
expect(api.batchRequests, hasLength(1));
async.elapse(const Duration(seconds: 1));
final events = api.batchRequests.last.events;
expect(events.map((event) => event['event']), ['before', 'after']);
expect(events[0]['distinct_id'], previousId);
expect(events[1]['distinct_id'], nextId);
final [_, before, after] = api.batchRequests;
expect(before.eventNames, ['before']);
expect(before.events.single['distinct_id'], previousId);
expect(after.eventNames, ['after']);
expect(after.events.single['distinct_id'], nextId);
});
});

Expand All @@ -546,8 +547,9 @@ void main() {
client.flush();
client.flush();
async.elapse(Duration.zero);
expect(api.batchRequests, hasLength(2));
expect(api.batchRequests.last.eventNames, ['first', 'second']);
final [_, retry, next] = api.batchRequests;
expect(retry.eventNames, ['first']);
expect(next.eventNames, ['second']);
});
});

Expand Down
Loading