Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/quiet-hashes-wait.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"posthog": patch
---

Store the person-properties dedup hash only after the `$set` or `$identify` event is queued, and only when the event as sent still processes the person. An update that `beforeSend` dropped or made personless, or one made while opted out, is no longer skipped when it is repeated.
158 changes: 119 additions & 39 deletions posthog/src/main/java/com/posthog/PostHog.kt
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ import java.util.concurrent.Executors

private const val PUSH_NOTIFICATION_OPENED_EVENT = "\$push_notification_opened"
private const val MAX_DEBUG_ERROR_LENGTH = 500

// Stands in for the hash of a person-properties update that cannot be serialized: such an update
// is always sent and never cached, as no real hash is empty.
private const val UNHASHABLE_PERSON_PROPERTIES = ""
private const val REPLAY_DEBUG_PROPERTIES_INTERVAL_MILLIS = 30_000L

// Kept on every event because the app reads them per event; the rest of the replay debug bundle is throttled.
Expand Down Expand Up @@ -967,6 +971,23 @@ public class PostHog private constructor(
userPropertiesSetOnce: Map<String, Any>?,
groups: Map<String, String>?,
timestamp: Date?,
) {
captureEvent(event, distinctId, properties, userProperties, userPropertiesSetOnce, groups, timestamp)
}

/**
* [personPropertiesHash], when set, is recorded or cleared once the event is queued, from the
* event as sent; see [updatePersonPropertiesHashForQueuedEvent].
*/
private fun captureEvent(
event: String,
distinctId: String?,
properties: Map<String, Any>?,
userProperties: Map<String, Any>?,
userPropertiesSetOnce: Map<String, Any>?,
groups: Map<String, String>?,
timestamp: Date?,
personPropertiesHash: String? = null,
) {
// Tracked on this call rather than derived from the event, so it survives beforeSend renaming
// or replacing the event.
Expand Down Expand Up @@ -1079,6 +1100,9 @@ public class PostHog private constructor(
if (claimedReplayDebugBundle) {
commitReplayDebugPropertiesWindow()
}
if (personPropertiesHash != null) {
updatePersonPropertiesHashForQueuedEvent(event, postHogEvent, personPropertiesHash)
}
}
// Notify surveys integration about the event
surveysHandler?.onEvent(event, mergedProperties)
Expand Down Expand Up @@ -1646,12 +1670,17 @@ public class PostHog private constructor(
if (shouldIdentify || shouldTransitionToIdentified) config.notifyIntegrationsChanged()

if (shouldIdentify) {
capture(
captureEvent(
PostHogEventName.IDENTIFY.event,
distinctId = distinctId,
properties = props,
userProperties = userProperties,
userPropertiesSetOnce = userPropertiesSetOnce,
groups = null,
timestamp = null,
personPropertiesHash =
getPersonPropertiesHash(distinctId, userProperties, userPropertiesSetOnce)
?: UNHASHABLE_PERSON_PROPERTIES,
)

if (config?.reuseAnonymousId != true) {
Expand Down Expand Up @@ -1686,37 +1715,44 @@ public class PostHog private constructor(

setPersonPropertiesForFlagsIfNeeded(userProperties, userPropertiesSetOnce)

capture(
// The transition event must fire even when an identical property call was cached
// earlier, so it skips the duplicate check and only records the hash once queued.
captureEvent(
PostHogEventName.SET.event,
distinctId = distinctId,
properties = null,
userProperties = userProperties ?: emptyMap(),
userPropertiesSetOnce = userPropertiesSetOnce ?: emptyMap(),
groups = null,
timestamp = null,
personPropertiesHash =
getPersonPropertiesHash(distinctId, userProperties, userPropertiesSetOnce)
?: UNHASHABLE_PERSON_PROPERTIES,
)

// The transition event must fire even when an identical property call was cached
// earlier; cache only after capture so deduplication cannot suppress it.
synchronized(cachedPersonPropertiesLock) {
cachedPersonPropertiesHash = getPersonPropertiesHash(distinctId, userProperties, userPropertiesSetOnce)
}

// The identified state itself is not part of the flags request; reload only when the
// caller supplied properties that can affect flag evaluation.
if ((userProperties?.isNotEmpty() == true || userPropertiesSetOnce?.isNotEmpty() == true) && reloadFeatureFlags) {
reloadFeatureFlags(config?.onFeatureFlags)
}
} else if (!hasDifferentDistinctId && (userProperties?.isNotEmpty() == true || userPropertiesSetOnce?.isNotEmpty() == true)) {
if (shouldCapturePersonPropertiesEvent(
val hash =
personPropertiesHashIfChanged(
distinctId,
userProperties,
userPropertiesSetOnce,
"A duplicate identify call was made with the same properties. The \$set event has been ignored.",
)
) {
capture(
if (hash != null) {
captureEvent(
PostHogEventName.SET.event,
distinctId = distinctId,
properties = null,
userProperties = userProperties,
userPropertiesSetOnce = userPropertiesSetOnce,
groups = null,
timestamp = null,
personPropertiesHash = hash,
)
}
// Note we don't reload flags on property changes as these get processed async
Expand All @@ -1743,15 +1779,13 @@ public class PostHog private constructor(

val currentDistinctId = this.distinctId

if (!shouldCapturePersonPropertiesEvent(
val hash =
personPropertiesHashIfChanged(
currentDistinctId,
userPropertiesToSet,
userPropertiesToSetOnce,
"A duplicate setPersonProperties call was made with the same properties. It has been ignored.",
)
) {
return
}
) ?: return

// Update person properties for flags (setOnce properties are applied first, then set properties override)
val allProperties = mutableMapOf<String, Any>()
Expand All @@ -1762,11 +1796,15 @@ public class PostHog private constructor(
}

// Send the $set event
capture(
captureEvent(
PostHogEventName.SET.event,
distinctId = currentDistinctId,
properties = null,
userProperties = userPropertiesToSet,
userPropertiesSetOnce = userPropertiesToSetOnce,
groups = null,
timestamp = null,
personPropertiesHash = hash,
)
}

Expand All @@ -1779,42 +1817,84 @@ public class PostHog private constructor(
distinctId: String,
userPropertiesToSet: Map<String, Any>?,
userPropertiesToSetOnce: Map<String, Any>?,
): String {
// Sort keys recursively to ensure deterministic hashing regardless of map insertion order
val sortedSet = userPropertiesToSet?.let { sortMapRecursively(it) }
val sortedSetOnce = userPropertiesToSetOnce?.let { sortMapRecursively(it) }

// Create a consistent JSON-like string representation similar to JS SDK's jsonStringify
val hashData =
sortedMapOf(
"distinct_id" to distinctId,
"userPropertiesToSet" to sortedSet,
"userPropertiesToSetOnce" to sortedSetOnce,
)
return config?.serializer?.serializeObject(hashData) ?: hashData.toString()
): String? {
// Gson throws on values such as Double.NaN, and deeply nested properties can overflow the
// recursive sort. A string fallback could equate different values (1 and "1"), so an update
// that cannot be hashed has no hash and is never deduplicated.
return try {
// Sort keys recursively to ensure deterministic hashing regardless of map insertion order.
// An empty map hashes like a missing one, as both send no properties.
val sortedSet = userPropertiesToSet?.takeIf { it.isNotEmpty() }?.let { sortMapRecursively(it) }
val sortedSetOnce = userPropertiesToSetOnce?.takeIf { it.isNotEmpty() }?.let { sortMapRecursively(it) }

// Create a consistent JSON-like string representation similar to JS SDK's jsonStringify
val hashData =
sortedMapOf(
"distinct_id" to distinctId,
"userPropertiesToSet" to sortedSet,
"userPropertiesToSetOnce" to sortedSetOnce,
)
config?.serializer?.serializeObject(hashData)
} catch (e: Throwable) {
config?.logger?.log("Failed to hash person properties for deduplication: $e.")
null
}
}

/**
* Checks if person properties have changed by comparing hash values.
* Updates the cached hash if different and returns true if the event should be captured.
* Returns false if the hash matches (duplicate call), logging the provided message.
* Returns the hash of this person-properties update, or null when it matches the last update
* that was queued (a duplicate call), logging the provided message. The hash is not stored here:
* [updatePersonPropertiesHashForQueuedEvent] stores it once the event is queued.
*/
private fun shouldCapturePersonPropertiesEvent(
private fun personPropertiesHashIfChanged(
distinctId: String,
userPropertiesToSet: Map<String, Any>?,
userPropertiesToSetOnce: Map<String, Any>?,
duplicateLogMessage: String,
): Boolean {
val hash = getPersonPropertiesHash(distinctId, userPropertiesToSet, userPropertiesToSetOnce)
): String? {
val hash =
getPersonPropertiesHash(distinctId, userPropertiesToSet, userPropertiesToSetOnce)
?: return UNHASHABLE_PERSON_PROPERTIES

synchronized(cachedPersonPropertiesLock) {
if (cachedPersonPropertiesHash == hash) {
config?.logger?.log(duplicateLogMessage)
return false
return null
}
cachedPersonPropertiesHash = hash
}
return true
return hash
}

/**
* Runs once [queuedEvent] is queued; a dropped event leaves the hash alone. The event as sent,
* after `beforeSend`, decides: an update that keeps its name, its distinct id and its `$set` and
* `$set_once` payload, and whose `$process_person_profile` is missing or the Boolean `true`, surely
* processes the person, so its hash is recorded. Anything else may not have sent these properties
* to the person, so the hash is cleared and a later identical update is sent.
*/
private fun updatePersonPropertiesHashForQueuedEvent(
capturedEvent: String,
queuedEvent: PostHogEvent,
hash: String,
) {
val processPersonProfile = queuedEvent.properties?.get("\$process_person_profile")
val surelyProcessesPerson =
hash != UNHASHABLE_PERSON_PROPERTIES &&
queuedEvent.event == capturedEvent &&
(processPersonProfile == null || processPersonProfile == true) &&
sentPersonPropertiesHash(queuedEvent) == hash
synchronized(cachedPersonPropertiesLock) {
cachedPersonPropertiesHash = if (surelyProcessesPerson) hash else null
Comment thread
turnipdabeets marked this conversation as resolved.
Resolved
}
}

@Suppress("UNCHECKED_CAST")
private fun sentPersonPropertiesHash(queuedEvent: PostHogEvent): String? {
val properties = queuedEvent.properties
val set = properties?.get("\$set")
val setOnce = properties?.get("\$set_once")
if ((set != null && set !is Map<*, *>) || (setOnce != null && setOnce !is Map<*, *>)) return null
return getPersonPropertiesHash(queuedEvent.distinctId, set as Map<String, Any>?, setOnce as Map<String, Any>?)
}

private fun hasPersonProcessing(): Boolean {
Expand Down
Loading
Loading