diff --git a/.changeset/quiet-hashes-wait.md b/.changeset/quiet-hashes-wait.md new file mode 100644 index 000000000..8b00eeaef --- /dev/null +++ b/.changeset/quiet-hashes-wait.md @@ -0,0 +1,5 @@ +--- +"posthog": patch +--- + +Store the person-properties dedup hash only after the `$set` or `$identify` event is queued, and only when the event as sent still processes the person. An update that `beforeSend` dropped or made personless, or one made while opted out, is no longer skipped when it is repeated. diff --git a/posthog/src/main/java/com/posthog/PostHog.kt b/posthog/src/main/java/com/posthog/PostHog.kt index c742a9fa9..2c190c72c 100644 --- a/posthog/src/main/java/com/posthog/PostHog.kt +++ b/posthog/src/main/java/com/posthog/PostHog.kt @@ -52,6 +52,10 @@ import java.util.concurrent.Executors private const val PUSH_NOTIFICATION_OPENED_EVENT = "\$push_notification_opened" private const val MAX_DEBUG_ERROR_LENGTH = 500 + +// Stands in for the hash of a person-properties update that cannot be serialized: such an update +// is always sent and never cached, as no real hash is empty. +private const val UNHASHABLE_PERSON_PROPERTIES = "" private const val REPLAY_DEBUG_PROPERTIES_INTERVAL_MILLIS = 30_000L // Kept on every event because the app reads them per event; the rest of the replay debug bundle is throttled. @@ -967,6 +971,23 @@ public class PostHog private constructor( userPropertiesSetOnce: Map?, groups: Map?, timestamp: Date?, + ) { + captureEvent(event, distinctId, properties, userProperties, userPropertiesSetOnce, groups, timestamp) + } + + /** + * [personPropertiesHash], when set, is recorded or cleared once the event is queued, from the + * event as sent; see [updatePersonPropertiesHashForQueuedEvent]. + */ + private fun captureEvent( + event: String, + distinctId: String?, + properties: Map?, + userProperties: Map?, + userPropertiesSetOnce: Map?, + groups: Map?, + timestamp: Date?, + personPropertiesHash: String? = null, ) { // Tracked on this call rather than derived from the event, so it survives beforeSend renaming // or replacing the event. @@ -1079,6 +1100,9 @@ public class PostHog private constructor( if (claimedReplayDebugBundle) { commitReplayDebugPropertiesWindow() } + if (personPropertiesHash != null) { + updatePersonPropertiesHashForQueuedEvent(event, postHogEvent, personPropertiesHash) + } } // Notify surveys integration about the event surveysHandler?.onEvent(event, mergedProperties) @@ -1646,12 +1670,17 @@ public class PostHog private constructor( if (shouldIdentify || shouldTransitionToIdentified) config.notifyIntegrationsChanged() if (shouldIdentify) { - capture( + captureEvent( PostHogEventName.IDENTIFY.event, distinctId = distinctId, properties = props, userProperties = userProperties, userPropertiesSetOnce = userPropertiesSetOnce, + groups = null, + timestamp = null, + personPropertiesHash = + getPersonPropertiesHash(distinctId, userProperties, userPropertiesSetOnce) + ?: UNHASHABLE_PERSON_PROPERTIES, ) if (config?.reuseAnonymousId != true) { @@ -1686,37 +1715,44 @@ public class PostHog private constructor( setPersonPropertiesForFlagsIfNeeded(userProperties, userPropertiesSetOnce) - capture( + // The transition event must fire even when an identical property call was cached + // earlier, so it skips the duplicate check and only records the hash once queued. + captureEvent( PostHogEventName.SET.event, distinctId = distinctId, + properties = null, userProperties = userProperties ?: emptyMap(), userPropertiesSetOnce = userPropertiesSetOnce ?: emptyMap(), + groups = null, + timestamp = null, + personPropertiesHash = + getPersonPropertiesHash(distinctId, userProperties, userPropertiesSetOnce) + ?: UNHASHABLE_PERSON_PROPERTIES, ) - // The transition event must fire even when an identical property call was cached - // earlier; cache only after capture so deduplication cannot suppress it. - synchronized(cachedPersonPropertiesLock) { - cachedPersonPropertiesHash = getPersonPropertiesHash(distinctId, userProperties, userPropertiesSetOnce) - } - // The identified state itself is not part of the flags request; reload only when the // caller supplied properties that can affect flag evaluation. if ((userProperties?.isNotEmpty() == true || userPropertiesSetOnce?.isNotEmpty() == true) && reloadFeatureFlags) { reloadFeatureFlags(config?.onFeatureFlags) } } else if (!hasDifferentDistinctId && (userProperties?.isNotEmpty() == true || userPropertiesSetOnce?.isNotEmpty() == true)) { - if (shouldCapturePersonPropertiesEvent( + val hash = + personPropertiesHashIfChanged( distinctId, userProperties, userPropertiesSetOnce, "A duplicate identify call was made with the same properties. The \$set event has been ignored.", ) - ) { - capture( + if (hash != null) { + captureEvent( PostHogEventName.SET.event, distinctId = distinctId, + properties = null, userProperties = userProperties, userPropertiesSetOnce = userPropertiesSetOnce, + groups = null, + timestamp = null, + personPropertiesHash = hash, ) } // Note we don't reload flags on property changes as these get processed async @@ -1743,15 +1779,13 @@ public class PostHog private constructor( val currentDistinctId = this.distinctId - if (!shouldCapturePersonPropertiesEvent( + val hash = + personPropertiesHashIfChanged( currentDistinctId, userPropertiesToSet, userPropertiesToSetOnce, "A duplicate setPersonProperties call was made with the same properties. It has been ignored.", - ) - ) { - return - } + ) ?: return // Update person properties for flags (setOnce properties are applied first, then set properties override) val allProperties = mutableMapOf() @@ -1762,11 +1796,15 @@ public class PostHog private constructor( } // Send the $set event - capture( + captureEvent( PostHogEventName.SET.event, distinctId = currentDistinctId, + properties = null, userProperties = userPropertiesToSet, userPropertiesSetOnce = userPropertiesToSetOnce, + groups = null, + timestamp = null, + personPropertiesHash = hash, ) } @@ -1779,42 +1817,84 @@ public class PostHog private constructor( distinctId: String, userPropertiesToSet: Map?, userPropertiesToSetOnce: Map?, - ): String { - // Sort keys recursively to ensure deterministic hashing regardless of map insertion order - val sortedSet = userPropertiesToSet?.let { sortMapRecursively(it) } - val sortedSetOnce = userPropertiesToSetOnce?.let { sortMapRecursively(it) } - - // Create a consistent JSON-like string representation similar to JS SDK's jsonStringify - val hashData = - sortedMapOf( - "distinct_id" to distinctId, - "userPropertiesToSet" to sortedSet, - "userPropertiesToSetOnce" to sortedSetOnce, - ) - return config?.serializer?.serializeObject(hashData) ?: hashData.toString() + ): String? { + // Gson throws on values such as Double.NaN, and deeply nested properties can overflow the + // recursive sort. A string fallback could equate different values (1 and "1"), so an update + // that cannot be hashed has no hash and is never deduplicated. + return try { + // Sort keys recursively to ensure deterministic hashing regardless of map insertion order. + // An empty map hashes like a missing one, as both send no properties. + val sortedSet = userPropertiesToSet?.takeIf { it.isNotEmpty() }?.let { sortMapRecursively(it) } + val sortedSetOnce = userPropertiesToSetOnce?.takeIf { it.isNotEmpty() }?.let { sortMapRecursively(it) } + + // Create a consistent JSON-like string representation similar to JS SDK's jsonStringify + val hashData = + sortedMapOf( + "distinct_id" to distinctId, + "userPropertiesToSet" to sortedSet, + "userPropertiesToSetOnce" to sortedSetOnce, + ) + config?.serializer?.serializeObject(hashData) + } catch (e: Throwable) { + config?.logger?.log("Failed to hash person properties for deduplication: $e.") + null + } } /** - * Checks if person properties have changed by comparing hash values. - * Updates the cached hash if different and returns true if the event should be captured. - * Returns false if the hash matches (duplicate call), logging the provided message. + * Returns the hash of this person-properties update, or null when it matches the last update + * that was queued (a duplicate call), logging the provided message. The hash is not stored here: + * [updatePersonPropertiesHashForQueuedEvent] stores it once the event is queued. */ - private fun shouldCapturePersonPropertiesEvent( + private fun personPropertiesHashIfChanged( distinctId: String, userPropertiesToSet: Map?, userPropertiesToSetOnce: Map?, duplicateLogMessage: String, - ): Boolean { - val hash = getPersonPropertiesHash(distinctId, userPropertiesToSet, userPropertiesToSetOnce) + ): String? { + val hash = + getPersonPropertiesHash(distinctId, userPropertiesToSet, userPropertiesToSetOnce) + ?: return UNHASHABLE_PERSON_PROPERTIES synchronized(cachedPersonPropertiesLock) { if (cachedPersonPropertiesHash == hash) { config?.logger?.log(duplicateLogMessage) - return false + return null } - cachedPersonPropertiesHash = hash } - return true + return hash + } + + /** + * Runs once [queuedEvent] is queued; a dropped event leaves the hash alone. The event as sent, + * after `beforeSend`, decides: an update that keeps its name, its distinct id and its `$set` and + * `$set_once` payload, and whose `$process_person_profile` is missing or the Boolean `true`, surely + * processes the person, so its hash is recorded. Anything else may not have sent these properties + * to the person, so the hash is cleared and a later identical update is sent. + */ + private fun updatePersonPropertiesHashForQueuedEvent( + capturedEvent: String, + queuedEvent: PostHogEvent, + hash: String, + ) { + val processPersonProfile = queuedEvent.properties?.get("\$process_person_profile") + val surelyProcessesPerson = + hash != UNHASHABLE_PERSON_PROPERTIES && + queuedEvent.event == capturedEvent && + (processPersonProfile == null || processPersonProfile == true) && + sentPersonPropertiesHash(queuedEvent) == hash + synchronized(cachedPersonPropertiesLock) { + cachedPersonPropertiesHash = if (surelyProcessesPerson) hash else null + } + } + + @Suppress("UNCHECKED_CAST") + private fun sentPersonPropertiesHash(queuedEvent: PostHogEvent): String? { + val properties = queuedEvent.properties + val set = properties?.get("\$set") + val setOnce = properties?.get("\$set_once") + if ((set != null && set !is Map<*, *>) || (setOnce != null && setOnce !is Map<*, *>)) return null + return getPersonPropertiesHash(queuedEvent.distinctId, set as Map?, setOnce as Map?) } private fun hasPersonProcessing(): Boolean { diff --git a/posthog/src/test/java/com/posthog/PostHogTest.kt b/posthog/src/test/java/com/posthog/PostHogTest.kt index 49b11a5e3..a76646c4c 100644 --- a/posthog/src/test/java/com/posthog/PostHogTest.kt +++ b/posthog/src/test/java/com/posthog/PostHogTest.kt @@ -5197,6 +5197,150 @@ internal class PostHogTest { sut.close() } + @Test + fun `setPersonProperties resends an update that beforeSend dropped`() { + val http = mockHttp() + var setEvents = 0 + val sut = + getSut(http.url("/").toString(), preloadFeatureFlags = false, beforeSend = { + if (it.event == "\$set" && ++setEvents == 1) null else it + }) + + sut.setPersonProperties(mapOf("email" to "user@example.com")) + sut.setPersonProperties(mapOf("email" to "user@example.com")) + queueExecutor.shutdownAndAwaitTermination() + + assertEquals(1, http.requestCount) + + sut.close() + } + + @Test + fun `setPersonProperties resends an update that beforeSend made personless`() { + val http = mockHttp(total = 2) + var setEvents = 0 + val sut = + getSut(http.url("/").toString(), preloadFeatureFlags = false, beforeSend = { + if (it.event == "\$set" && ++setEvents == 1) it.properties!!["\$process_person_profile"] = false + it + }) + + sut.setPersonProperties(mapOf("email" to "user@example.com")) + sut.setPersonProperties(mapOf("email" to "user@example.com")) + + queueExecutor.shutdownAndAwaitTermination() + + assertEquals(2, http.requestCount) + val flags = + List(2) { + serializer.deserialize(http.takeRequest().body.unGzip().reader())!! + .batch.single().properties!!["\$process_person_profile"] + } + assertEquals(listOf(false, true), flags) + + sut.close() + } + + @Test + fun `setPersonProperties sends an update made while opted out after opting in`() { + val http = mockHttp() + val sut = getSut(http.url("/").toString(), preloadFeatureFlags = false, optOut = true) + + sut.setPersonProperties(mapOf("email" to "user@example.com")) + sut.optIn() + sut.setPersonProperties(mapOf("email" to "user@example.com")) + queueExecutor.shutdownAndAwaitTermination() + + assertEquals(1, http.requestCount) + + sut.close() + } + + @Test + fun `identify skips a set that repeats the properties its identify event sent`() { + val http = mockHttp() + val sut = getSut(http.url("/").toString(), preloadFeatureFlags = false, reloadFeatureFlags = false, flushAt = 2) + + sut.identify(DISTINCT_ID, userProperties = userProps) + sut.identify(DISTINCT_ID, userProperties = userProps) + sut.setPersonProperties(mapOf("plan" to "pro")) + + val events = capturedEvents(http) + assertEquals(listOf("\$identify", "\$set"), events.map { it.event }) + assertEquals(mapOf("plan" to "pro"), events[1].properties!!["\$set"]) + + sut.close() + } + + @Test + fun `identify resends set properties that beforeSend removed from the identify event`() { + val http = mockHttp() + val sut = + getSut( + http.url("/").toString(), + preloadFeatureFlags = false, + reloadFeatureFlags = false, + flushAt = 2, + beforeSend = { + if (it.event == "\$identify") it.properties!!.remove("\$set") + it + }, + ) + + sut.identify(DISTINCT_ID, userProperties = userProps) + sut.setPersonProperties(userProps) + sut.flush() + + val events = capturedEvents(http) + assertEquals(listOf("\$identify", "\$set"), events.map { it.event }) + assertEquals(userProps, events[1].properties!!["\$set"]) + + sut.close() + } + + @Test + fun `identify with a person property the hash cannot serialize still identifies`() { + val http = mockHttp() + val sut = getSut(http.url("/").toString(), preloadFeatureFlags = false, reloadFeatureFlags = false) + + sut.identify(DISTINCT_ID, userProperties = mapOf("score" to Double.NaN)) + + assertEquals(DISTINCT_ID, sut.distinctId()) + + sut.close() + } + + @Test + fun `identify with person properties too deeply nested to hash still identifies`() { + val http = mockHttp() + val sut = getSut(http.url("/").toString(), preloadFeatureFlags = false, reloadFeatureFlags = false) + var nested: Map = mapOf("leaf" to 1) + repeat(100_000) { nested = mapOf("child" to nested) } + + sut.identify(DISTINCT_ID, userProperties = nested) + + assertEquals(DISTINCT_ID, sut.distinctId()) + + sut.close() + } + + @Test + fun `setPersonProperties sends every update the hash cannot serialize`() { + val http = mockHttp() + val sut = + getSut(http.url("/").toString(), preloadFeatureFlags = false, reloadFeatureFlags = false, flushAt = 2) + + sut.setPersonProperties(mapOf("score" to Double.NaN, "plan" to 1)) + sut.setPersonProperties(mapOf("score" to Double.NaN, "plan" to "1")) + sut.flush() + + val events = capturedEvents(http) + assertEquals(listOf("\$set", "\$set"), events.map { it.event }) + assertEquals("1", events[1].properties!!["\$set"].let { it as Map<*, *> }["plan"]) + + sut.close() + } + @Test fun `setPersonProperties deduplicates calls with same setOnce properties in different order`() { val http = mockHttp()