diff --git a/Cargo.lock b/Cargo.lock index d36bbaff719..bc83ec975f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1053,15 +1053,20 @@ dependencies = [ name = "buzz-db" version = "0.1.0" dependencies = [ + "async-trait", "buzz-auth", "buzz-core", "buzz-datastore-tracing", + "buzz-media", "chrono", + "dashmap", "hex", + "http", "metrics", "metrics-util", "nostr 0.44.7", "rand 0.10.1", + "rust-s3", "serde", "serde_json", "sha2 0.11.0", diff --git a/Cargo.toml b/Cargo.toml index c228a754da4..4072b9cfa38 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ tokio-util = { version = "0.7", features = ["rt", "codec"] } # HTTP + WebSocket axum = { version = "0.8", features = ["ws", "macros"] } +http = "1" tower = { version = "0.5", features = ["timeout", "util", "limit"] } tower-http = { version = "0.6", features = ["trace", "cors", "compression-gzip", "limit", "timeout", "fs"] } diff --git a/crates/buzz-db/Cargo.toml b/crates/buzz-db/Cargo.toml index 816e02642fc..742d9615024 100644 --- a/crates/buzz-db/Cargo.toml +++ b/crates/buzz-db/Cargo.toml @@ -11,6 +11,10 @@ description = "Postgres event store and data access layer for Buzz" buzz-auth = { workspace = true } buzz-core = { workspace = true } buzz-datastore-tracing = { workspace = true } +async-trait = "0.1" +http = { workspace = true } +buzz-media = { workspace = true } +dashmap = { workspace = true } sqlx = { workspace = true } tokio = { workspace = true } serde = { workspace = true } @@ -24,6 +28,10 @@ thiserror = { workspace = true } nostr = { workspace = true } rand = { workspace = true } metrics = { workspace = true } +s3 = { version = "0.37", package = "rust-s3", default-features = false, features = ["tokio-rustls-tls", "fail-on-err", "tags"] } + +[features] +test-utils = [] [dev-dependencies] tokio = { workspace = true } diff --git a/crates/buzz-db/src/authorization_events.rs b/crates/buzz-db/src/authorization_events.rs index 1fa5e3fd1c0..42f2812fb5d 100644 --- a/crates/buzz-db/src/authorization_events.rs +++ b/crates/buzz-db/src/authorization_events.rs @@ -12,7 +12,7 @@ use buzz_auth::{ AuthorizationEventCapacityPolicy, FinalizedAuthContext, PreparedAuthorization, ProofTransport, VerifiedFederatedAssertion, VerifiedNostrProof, }; -use buzz_core::{CanonicalCurrentBindingEvidence, CommunityId}; +use buzz_core::CommunityId; use chrono::{DateTime, Utc}; use sha2::{Digest, Sha256}; use sqlx::{Postgres, Row, Transaction}; @@ -94,8 +94,6 @@ pub enum AuthorizationOperationKind { ProtectedMutation = 11, /// Invalidation advance. Invalidation = 12, - /// Client-status revision. - StatusRevision = 13, } /// Persisted result classification for one operation receipt. @@ -207,10 +205,6 @@ pub enum AuthorizationEventKind { ProtectedAllowed = 10, /// Protected operation denied. ProtectedDenied = 11, - /// Current-binding status published. - StatusPublished = 12, - /// Current-binding status withdrawn. - StatusWithdrawn = 13, /// Invalidation generation advanced. InvalidationAdvanced = 14, } @@ -229,8 +223,6 @@ impl AuthorizationEventKind { 9 => Ok(Self::OperatorDenied), 10 => Ok(Self::ProtectedAllowed), 11 => Ok(Self::ProtectedDenied), - 12 => Ok(Self::StatusPublished), - 13 => Ok(Self::StatusWithdrawn), 14 => Ok(Self::InvalidationAdvanced), _ => Err(DbError::InvalidData( "authorization event kind is invalid".to_owned(), @@ -549,6 +541,16 @@ impl AuthorizationEventActor { pub const fn authority_loss_target(&self) -> Option { self.authority_loss_target } + + #[cfg(test)] + pub(crate) fn test_direct(community_id: CommunityId) -> Self { + local_actor( + AuthorizationActorKind::Direct, + community_id, + &[b"protected-version-test"], + None, + ) + } } const fn proof_transport_code(transport: ProofTransport) -> i16 { @@ -777,132 +779,6 @@ fn local_actor( } } -/// Mint status-publication attribution only after the complete evidence tuple -/// has been rechecked inside the allocation transaction. -pub async fn resolve_current_binding_event_actor_tx( - transaction: &mut Transaction<'_, Postgres>, - evidence: &CanonicalCurrentBindingEvidence, -) -> Result { - let mut object_key_digest = Sha256::new(); - object_key_digest.update(b"buzz:client-binding-status-authority:v1"); - object_key_digest.update((16_u64).to_be_bytes()); - object_key_digest.update(evidence.authorization_domain().as_uuid().as_bytes()); - object_key_digest.update((32_u64).to_be_bytes()); - object_key_digest.update(evidence.event_author_pubkey().to_bytes()); - let object_key: [u8; 32] = object_key_digest.finalize().into(); - sqlx::query( - "SELECT 1 FROM identity_bindings b \ - JOIN identity_enrollment_policies p \ - ON p.community_id=b.community_id AND p.policy_revision=b.policy_revision \ - JOIN authorization_invalidation_domains d ON d.community_id=b.community_id \ - JOIN authorization_authority_epochs a \ - ON a.community_id=b.community_id AND a.object_kind=7 AND a.object_key=$10 \ - WHERE b.community_id=$1 AND b.binding_id=$2 AND b.binding_version=$3 \ - AND b.event_author_pubkey=$4 AND b.policy_revision=$5 \ - AND d.current_generation=$6 AND a.authority_epoch=$7 AND a.fence=$8 \ - AND b.binding_state=1 AND $9 > clock_timestamp() \ - AND $11 <= clock_timestamp() \ - AND (b.expires_at IS NULL OR b.expires_at > clock_timestamp()) \ - AND p.effective_at <= clock_timestamp() \ - AND (p.expires_at IS NULL OR p.expires_at > clock_timestamp()) \ - FOR SHARE OF b,p,d,a", - ) - .bind(evidence.authorization_domain().as_uuid()) - .bind(evidence.binding_id()) - .bind(i64::try_from(evidence.binding_version()).map_err(|_| { - DbError::InvalidData("authorization binding version is out of range".to_owned()) - })?) - .bind(evidence.event_author_pubkey().to_bytes().as_slice()) - .bind(i64::try_from(evidence.policy_revision()).map_err(|_| { - DbError::InvalidData("authorization policy revision is out of range".to_owned()) - })?) - .bind( - i64::try_from(evidence.invalidation_generation()).map_err(|_| { - DbError::InvalidData("authorization invalidation generation is out of range".to_owned()) - })?, - ) - .bind(i64::try_from(evidence.authority_epoch()).map_err(|_| { - DbError::InvalidData("authorization authority epoch is out of range".to_owned()) - })?) - .bind(evidence.fence().as_bytes().as_slice()) - .bind(evidence.fresh_until()) - .bind(object_key.as_slice()) - .bind(evidence.observed_at()) - .fetch_optional(&mut **transaction) - .await? - .ok_or_else(|| DbError::NotFound("current binding event actor".to_owned()))?; - current_binding_event_actor(evidence) -} - -/// Derive redaction-safe attribution from already rechecked binding evidence. -pub fn current_binding_event_actor( - evidence: &CanonicalCurrentBindingEvidence, -) -> Result { - Ok(local_actor( - AuthorizationActorKind::Direct, - evidence.authorization_domain(), - &[ - evidence.binding_id().as_bytes(), - &evidence.binding_version().to_be_bytes(), - &evidence.event_author_pubkey().to_bytes(), - ], - None, - )) -} - -/// Mint withdrawal attribution only from the exact current durable status -/// receipt that is about to be superseded. -pub async fn resolve_status_withdrawal_event_actor_tx( - transaction: &mut Transaction<'_, Postgres>, - community_id: CommunityId, - event_author_pubkey: [u8; 32], - supersedes_revision: u64, -) -> Result { - let current = sqlx::query( - "SELECT revision,disposition FROM client_status_revisions \ - WHERE community_id=$1 AND event_author_pubkey=$2 \ - ORDER BY revision DESC LIMIT 1 FOR UPDATE", - ) - .bind(community_id.as_uuid()) - .bind(event_author_pubkey.as_slice()) - .fetch_optional(&mut **transaction) - .await?; - let Some(current) = current else { - return Err(DbError::NotFound( - "current status withdrawal actor".to_owned(), - )); - }; - if current.try_get::("disposition")? != 1 - || u64::try_from(current.try_get::("revision")?).map_err(|_| { - DbError::InvalidData("authorization status revision is invalid".to_owned()) - })? != supersedes_revision - { - return Err(DbError::InvalidData( - "current status withdrawal actor changed".to_owned(), - )); - } - status_withdrawal_event_actor(community_id, event_author_pubkey, supersedes_revision) -} - -/// Derive withdrawal attribution from one exact durable status revision. -pub fn status_withdrawal_event_actor( - community_id: CommunityId, - event_author_pubkey: [u8; 32], - supersedes_revision: u64, -) -> Result { - if community_id.as_uuid().is_nil() || supersedes_revision == 0 { - return Err(DbError::InvalidData( - "authorization status withdrawal actor is invalid".to_owned(), - )); - } - Ok(local_actor( - AuthorizationActorKind::Direct, - community_id, - &[&event_author_pubkey, &supersedes_revision.to_be_bytes()], - None, - )) -} - fn append_hash_optional(digest: &mut Sha256, value: Option<&[u8]>) { match value { Some(value) => { @@ -1022,8 +898,6 @@ fn valid_event_semantics( && !matches!(reason, Reason::Current | Reason::Replay | Reason::Withdrawn) } Kind::ProtectedAllowed => outcome == Outcome::Allowed && reason == Reason::Current, - Kind::StatusPublished => outcome == Outcome::Allowed && reason == Reason::Current, - Kind::StatusWithdrawn => outcome == Outcome::Withdrawn && reason == Reason::Withdrawn, Kind::InvalidationAdvanced => outcome == Outcome::Allowed && reason == Reason::Invalidated, } } @@ -1887,12 +1761,6 @@ mod tests { Uuid::from_u128(34), ) }; - assert!(base( - AuthorizationEventKind::StatusPublished, - AuthorizationEventOutcome::Denied, - AuthorizationReasonCode::Withdrawn, - ) - .is_err()); assert!(base( AuthorizationEventKind::InvalidationAdvanced, AuthorizationEventOutcome::Allowed, @@ -1906,4 +1774,15 @@ mod tests { ) .is_ok()); } + + #[test] + fn retired_status_codes_are_not_canonical_authority() { + assert!(AuthorizationEventKind::from_database(12).is_err()); + assert!(AuthorizationEventKind::from_database(13).is_err()); + assert_eq!( + AuthorizationEventKind::from_database(14).unwrap(), + AuthorizationEventKind::InvalidationAdvanced, + ); + assert_eq!(AuthorizationOperationKind::Invalidation as i16, 12); + } } diff --git a/crates/buzz-db/src/authorization_invalidation.rs b/crates/buzz-db/src/authorization_invalidation.rs new file mode 100644 index 00000000000..41a4c0839b7 --- /dev/null +++ b/crates/buzz-db/src/authorization_invalidation.rs @@ -0,0 +1,1693 @@ +//! Durable provider-free authorization invalidation state. +//! +//! PostgreSQL is authoritative. Callers capture the exact domain generation +//! and dependency floors before evaluation, then compare them again at the +//! final allow fence. Local lifecycle and delegation authority changes enter +//! through the same typed selector set; there is no provider event vocabulary. + +use std::{collections::BTreeSet, fmt}; + +use buzz_core::CommunityId; +use sha2::{Digest, Sha256}; +use sqlx::{ + postgres::{PgListener, PgPoolOptions}, + Postgres, Row, Transaction, +}; +use thiserror::Error; +use uuid::Uuid; + +use crate::{ + authorization_events::{ + record_authorization_event_tx, record_authorization_operation_receipt_tx, + AuthorizationEventActor, AuthorizationEventKind, AuthorizationEventOutcome, + AuthorizationEventWriteError, AuthorizationOperationKind, AuthorizationOperationOutcome, + AuthorizationOperationReceipt, AuthorizationReasonCode, AuthorizationReceiptWrite, + NewAuthorizationEvent, + }, + authorization_version::{ + authorization_version_delegated_relationship_component_key, + authorization_version_invalidation_generation_component_key, load_manifest_connection, + record_authorization_operation_version_delta_tx, AuthorizationAuthorityEpochAdvance, + AuthorizationAuthorityObjectEvidence, AuthorizationOperationVersionDelta, + AuthorizationOperationVersionDeltaManifest, AuthorizationProtectedObjectKind, + AuthorizationVersionComponentKind, ProtectedPublicationDependency, + }, + Db, DbError, Result, +}; + +/// Maximum exact selectors accepted by one invalidation operation. +pub const MAX_AUTHORIZATION_INVALIDATION_SELECTORS: usize = 64; + +const AUTHORIZATION_INVALIDATION_CHANNEL: &str = "buzz_authorization_invalidation_v1"; + +/// Committed authorization dependency change delivered to live observers. +#[derive(Clone, PartialEq, Eq)] +pub enum AuthorizationInvalidationNotice { + /// The canonical domain invalidation generation advanced. + DomainAdvanced { + /// Server-resolved authorization domain. + authorization_domain: CommunityId, + /// Exact committed generation. + generation: u64, + }, + /// One protected publication authority epoch advanced. + ProtectedPublicationAdvanced { + /// Opaque exact protected-object dependency. + dependency: ProtectedPublicationDependency, + /// Exact committed authority epoch. + authority_epoch: u64, + }, +} + +impl fmt::Debug for AuthorizationInvalidationNotice { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::DomainAdvanced { generation, .. } => formatter + .debug_struct("AuthorizationInvalidationNotice::DomainAdvanced") + .field("generation", generation) + .field("authorization_domain", &"[REDACTED]") + .finish(), + Self::ProtectedPublicationAdvanced { + authority_epoch, .. + } => formatter + .debug_struct("AuthorizationInvalidationNotice::ProtectedPublicationAdvanced") + .field("authority_epoch", authority_epoch) + .field("dependency", &"[REDACTED]") + .finish(), + } + } +} + +/// Dedicated independent PostgreSQL subscription for live authorization loss. +pub struct AuthorizationInvalidationSubscription { + listener: PgListener, + failed: bool, +} + +impl AuthorizationInvalidationSubscription { + /// Receive one committed typed notice. + /// + /// A lost listener connection is returned as an error instead of being + /// transparently reconnected across a notification gap. + pub async fn recv(&mut self) -> Result { + if self.failed { + return Err(DbError::InvalidData( + "authorization invalidation listener is unhealthy".to_owned(), + )); + } + let notification = match self.listener.try_recv().await { + Ok(Some(notification)) => notification, + Ok(None) => { + self.failed = true; + return Err(DbError::InvalidData( + "authorization invalidation listener was lost".to_owned(), + )); + } + Err(error) => { + self.failed = true; + return Err(error.into()); + } + }; + if notification.channel() != AUTHORIZATION_INVALIDATION_CHANNEL { + self.failed = true; + return Err(DbError::InvalidData( + "authorization invalidation channel is invalid".to_owned(), + )); + } + match parse_authorization_invalidation_notice(notification.payload()) { + Ok(notice) => Ok(notice), + Err(error) => { + self.failed = true; + Err(error) + } + } + } +} + +impl fmt::Debug for AuthorizationInvalidationSubscription { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("AuthorizationInvalidationSubscription([REDACTED])") + } +} + +/// Closed invalidation selector classes from migration 0030. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(i16)] +pub enum AuthorizationInvalidationSelectorKind { + /// Exact local principal fingerprint. + Principal = 1, + /// Exact Nostr key. + NostrKey = 2, + /// Exact binding with invalid-through binding version. + Binding = 3, + /// Exact server-issued session target. + Session = 4, + /// Entire authorization domain. + Domain = 5, + /// Exact local configuration revision. + ConfigurationRevision = 6, + /// Exact delegated relationship with invalid-through revision. + DelegatedRelationship = 7, +} + +impl AuthorizationInvalidationSelectorKind { + fn from_database(value: i16) -> Result { + match value { + 1 => Ok(Self::Principal), + 2 => Ok(Self::NostrKey), + 3 => Ok(Self::Binding), + 4 => Ok(Self::Session), + 5 => Ok(Self::Domain), + 6 => Ok(Self::ConfigurationRevision), + 7 => Ok(Self::DelegatedRelationship), + _ => Err(DbError::InvalidData( + "authorization invalidation selector kind is invalid".to_owned(), + )), + } + } +} + +/// Exact server-issued session target resistant to UUID reuse. +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct AuthorizationSessionTarget { + session_id: Uuid, + issuance_fence: Uuid, +} + +impl AuthorizationSessionTarget { + /// Construct a non-reusable session target. + pub fn new(session_id: Uuid, issuance_fence: Uuid) -> Result { + if session_id.is_nil() || issuance_fence.is_nil() { + return Err(DbError::InvalidData( + "authorization session target is invalid".to_owned(), + )); + } + Ok(Self { + session_id, + issuance_fence, + }) + } +} + +impl fmt::Debug for AuthorizationSessionTarget { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("AuthorizationSessionTarget([REDACTED])") + } +} + +/// Typed local invalidation dependency. +#[derive(Clone, PartialEq, Eq)] +pub enum AuthorizationInvalidationSelector { + /// Already-derived exact principal fingerprint. + Principal([u8; 32]), + /// Exact Nostr key. + NostrKey([u8; 32]), + /// Exact binding generation floor. + Binding { + /// Stable binding ID. + binding_id: Uuid, + /// Highest invalid binding version. + invalid_through: u64, + }, + /// Exact server-issued session. + Session(AuthorizationSessionTarget), + /// Entire domain. + Domain, + /// Exact positive local configuration revision. + ConfigurationRevision(u64), + /// Exact delegated relationship revision floor. + DelegatedRelationship { + /// Stable verifier-defined relationship identity. + relationship_id: Uuid, + /// Highest invalid relationship revision. + invalid_through: u64, + }, +} + +impl AuthorizationInvalidationSelector { + /// Validate a principal fingerprint selector. + pub fn principal(fingerprint: [u8; 32]) -> Result { + nonzero(fingerprint, "principal fingerprint")?; + Ok(Self::Principal(fingerprint)) + } + + /// Validate an exact Nostr-key selector. + pub fn nostr_key(key: [u8; 32]) -> Result { + nonzero(key, "Nostr key")?; + Ok(Self::NostrKey(key)) + } + + /// Validate a binding invalid-through selector. + pub fn binding(binding_id: Uuid, invalid_through: u64) -> Result { + if binding_id.is_nil() || invalid_through == 0 || invalid_through > i64::MAX as u64 { + return Err(DbError::InvalidData( + "authorization binding invalidation selector is invalid".to_owned(), + )); + } + Ok(Self::Binding { + binding_id, + invalid_through, + }) + } + + /// Select an exact session issuance. + pub const fn session(target: AuthorizationSessionTarget) -> Self { + Self::Session(target) + } + + /// Select the entire server-resolved domain. + pub const fn domain() -> Self { + Self::Domain + } + + /// Validate a local configuration revision selector. + pub fn configuration_revision(revision: u64) -> Result { + if revision == 0 || revision > i64::MAX as u64 { + return Err(DbError::InvalidData( + "authorization configuration revision is invalid".to_owned(), + )); + } + Ok(Self::ConfigurationRevision(revision)) + } + + /// Validate an exact delegated-relationship revision selector. + pub fn delegated_relationship(relationship_id: Uuid, invalid_through: u64) -> Result { + if relationship_id.is_nil() || invalid_through == 0 || invalid_through > i64::MAX as u64 { + return Err(DbError::InvalidData( + "authorization delegated relationship selector is invalid".to_owned(), + )); + } + Ok(Self::DelegatedRelationship { + relationship_id, + invalid_through, + }) + } + + /// Closed selector class. + pub const fn kind(&self) -> AuthorizationInvalidationSelectorKind { + match self { + Self::Principal(_) => AuthorizationInvalidationSelectorKind::Principal, + Self::NostrKey(_) => AuthorizationInvalidationSelectorKind::NostrKey, + Self::Binding { .. } => AuthorizationInvalidationSelectorKind::Binding, + Self::Session(_) => AuthorizationInvalidationSelectorKind::Session, + Self::Domain => AuthorizationInvalidationSelectorKind::Domain, + Self::ConfigurationRevision(_) => { + AuthorizationInvalidationSelectorKind::ConfigurationRevision + } + Self::DelegatedRelationship { .. } => { + AuthorizationInvalidationSelectorKind::DelegatedRelationship + } + } + } + + /// Domain-separated selector fingerprint. + pub fn fingerprint(&self, community_id: CommunityId) -> [u8; 32] { + if let Self::DelegatedRelationship { + relationship_id, .. + } = self + { + // Migration 0030 intentionally reuses selector_fingerprint as the + // restore component coordinate for delegated relationships. All + // readers and writers must therefore store this one canonical key. + return authorization_version_delegated_relationship_component_key( + community_id, + *relationship_id, + ); + } + let mut digest = Sha256::new(); + framed(&mut digest, b"buzz:authorization-invalidation-selector:v1"); + framed(&mut digest, community_id.as_uuid().as_bytes()); + framed(&mut digest, &(self.kind() as i16).to_be_bytes()); + match self { + Self::Principal(value) | Self::NostrKey(value) => framed(&mut digest, value), + Self::Binding { binding_id, .. } => framed(&mut digest, binding_id.as_bytes()), + Self::Session(target) => { + framed(&mut digest, target.session_id.as_bytes()); + framed(&mut digest, target.issuance_fence.as_bytes()); + } + Self::Domain => {} + Self::ConfigurationRevision(revision) => { + framed(&mut digest, &revision.to_be_bytes()); + } + Self::DelegatedRelationship { .. } => { + unreachable!("delegated relationship coordinates return before selector encoding") + } + } + digest.finalize().into() + } + + fn binding_floor(&self) -> Option { + match self { + Self::Binding { + invalid_through, .. + } => Some(*invalid_through), + _ => None, + } + } + + fn relationship_floor(&self) -> Option { + match self { + Self::DelegatedRelationship { + invalid_through, .. + } => Some(*invalid_through), + _ => None, + } + } +} + +impl fmt::Debug for AuthorizationInvalidationSelector { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("AuthorizationInvalidationSelector") + .field("kind", &self.kind()) + .field("value", &"[REDACTED]") + .finish() + } +} + +/// One stored selector floor at an exact generation. +#[derive(Clone, PartialEq, Eq)] +pub struct AuthorizationInvalidationFloor { + /// Closed selector class. + pub kind: AuthorizationInvalidationSelectorKind, + /// Opaque selector fingerprint. + pub fingerprint: [u8; 32], + /// Domain generation that last advanced this floor. + pub generation: u64, + /// Highest invalid binding version for a binding selector. + pub binding_version_floor: Option, + /// Highest invalid relationship revision for a relationship selector. + pub relationship_revision_floor: Option, +} + +impl fmt::Debug for AuthorizationInvalidationFloor { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("AuthorizationInvalidationFloor") + .field("kind", &self.kind) + .field("fingerprint", &"[REDACTED]") + .field("generation", &"[REDACTED]") + .field("binding_version_floor", &"[REDACTED]") + .field("relationship_revision_floor", &"[REDACTED]") + .finish() + } +} + +/// Exact invalidation state captured for an authorization evaluation. +#[derive(Clone, PartialEq, Eq)] +pub struct AuthorizationInvalidationSnapshot { + community_id: CommunityId, + generation: u64, + floors: Vec, +} + +impl AuthorizationInvalidationSnapshot { + /// Server-resolved domain. + pub const fn community_id(&self) -> CommunityId { + self.community_id + } + + /// Current durable domain generation. + pub const fn generation(&self) -> u64 { + self.generation + } + + /// Exact requested floors, sorted by kind and fingerprint. + pub fn floors(&self) -> &[AuthorizationInvalidationFloor] { + &self.floors + } + + /// Require byte-for-byte current state at a final fence. + pub fn accepts_exact_recheck(&self, current: &Self) -> bool { + self == current + } +} + +impl fmt::Debug for AuthorizationInvalidationSnapshot { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("AuthorizationInvalidationSnapshot") + .field("community_id", &"[REDACTED]") + .field("generation", &"[REDACTED]") + .field("floor_count", &self.floors.len()) + .finish() + } +} + +/// Complete caller input for one durable invalidation operation. +#[derive(Clone)] +pub(crate) struct AuthorizationInvalidationRequest { + /// Server-resolved domain. + pub(crate) community_id: CommunityId, + /// Idempotency operation ID. + pub(crate) operation_id: Uuid, + /// Exact semantic request fingerprint. + pub(crate) request_fingerprint: [u8; 32], + /// Actor derived from origin-sealed or database-rechecked authority. + pub(crate) actor: AuthorizationEventActor, + /// Optional pseudonymous subject fingerprint. + pub(crate) subject_fingerprint: Option<[u8; 32]>, + /// Correlation ID. + pub(crate) correlation_id: Uuid, + /// Exact attempt ID for canonical evidence replay. + pub(crate) attempt_id: Uuid, + /// Stable canonical event ID. + pub(crate) event_id: Uuid, + /// Non-empty exact selectors. + pub(crate) selectors: Vec, +} + +impl fmt::Debug for AuthorizationInvalidationRequest { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("AuthorizationInvalidationRequest") + .field("community_id", &"[REDACTED]") + .field("operation_id", &"[REDACTED]") + .field("selector_count", &self.selectors.len()) + .finish() + } +} + +/// Result of one atomic invalidation application. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct AuthorizationInvalidationApplied { + /// Durable domain generation after this operation. + pub(crate) generation: u64, + /// Whether the exact operation was already committed. + pub(crate) replay: bool, +} + +/// Exact generation advance returned to a caller-owned lifecycle transaction. +/// +/// The contained delta is opaque outside `buzz-db`, preventing relay callers +/// from fabricating restore attribution. +pub(crate) struct AuthorizationInvalidationAdvance { + generation: u64, + authority_objects: Vec, + deltas: Vec, +} + +impl AuthorizationInvalidationAdvance { + /// New durable domain generation. + pub(crate) const fn generation(&self) -> u64 { + self.generation + } + + /// Exact object evidence refenced before invalidation locking. + #[allow(dead_code)] // Consumed by canonical admission-loss integration. + pub(crate) fn authority_objects(&self) -> &[AuthorizationAuthorityObjectEvidence] { + &self.authority_objects + } + + /// Consume all exact database-owned restore deltas. + pub(crate) fn into_deltas(self) -> Vec { + self.deltas + } +} + +impl fmt::Debug for AuthorizationInvalidationAdvance { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("AuthorizationInvalidationAdvance([REDACTED])") + } +} + +/// Advance invalidation for one exact local admission loss inside the +/// lifecycle transaction that owns the receipt, history, audit, and manifest. +#[allow(dead_code)] // Consumed by lifecycle integration. +pub(crate) async fn apply_admission_loss_invalidation_tx( + transaction: &mut Transaction<'_, Postgres>, + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + selectors: &[AuthorizationInvalidationSelector], + authority_advance: AuthorizationAuthorityEpochAdvance, +) -> Result { + validate_selectors(community_id, selectors)?; + if operation_id.is_nil() + || request_fingerprint == [0; 32] + || !authority_advance.matches_operation(community_id, operation_id, request_fingerprint) + || !loss_target_matches_selectors(authority_advance.loss_target(), selectors) + { + return Err(DbError::InvalidData( + "authorization admission-loss invalidation is incomplete".to_owned(), + )); + } + let (authority_objects, mut authority_deltas) = authority_advance.into_parts(); + let before = lock_invalidation_generation_tx(transaction, community_id).await?; + let invalidation_advance = advance_locked_invalidation_tx( + transaction, + community_id, + operation_id, + request_fingerprint, + selectors, + before, + ) + .await?; + authority_deltas.extend(invalidation_advance.deltas); + Ok(AuthorizationInvalidationAdvance { + generation: invalidation_advance.generation, + authority_objects, + deltas: authority_deltas, + }) +} + +/// Closed application failure preserving audit-unavailable classification. +#[derive(Debug, Error)] +pub(crate) enum AuthorizationInvalidationApplyError { + /// Canonical evidence could not be persisted; no invalidation committed. + #[error("authorization invalidation audit is unavailable")] + AuditUnavailable, + /// Invalid input or PostgreSQL failure. + #[error(transparent)] + Database(#[from] DbError), +} + +impl Db { + /// Install a ready listener on a dedicated connection independent of the + /// writer pool's capacity. + pub async fn subscribe_authorization_invalidations( + &self, + ) -> Result { + let connect_options = self.pool.connect_options(); + let listener_pool = PgPoolOptions::new() + .min_connections(1) + .max_connections(1) + .connect_with(connect_options.as_ref().clone()) + .await?; + let mut listener = PgListener::connect_with(&listener_pool).await?; + listener.eager_reconnect(false); + listener.listen(AUTHORIZATION_INVALIDATION_CHANNEL).await?; + Ok(AuthorizationInvalidationSubscription { + listener, + failed: false, + }) + } + + /// Atomically advance one domain generation, selector floors, canonical + /// receipt/event, and exact operation version manifest. + #[allow(dead_code)] // Called by the verified mutation adapter. + pub(crate) async fn apply_authorization_invalidation( + &self, + request: AuthorizationInvalidationRequest, + ) -> std::result::Result + { + validate_request(&request)?; + let mut transaction = self.pool.begin().await.map_err(DbError::from)?; + + if sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM authorization_operation_receipts \ + WHERE community_id=$1 AND operation_id=$2)", + ) + .bind(request.community_id.as_uuid()) + .bind(request.operation_id) + .fetch_one(&mut *transaction) + .await + .map_err(DbError::from)? + { + let manifest = load_manifest_connection( + &mut transaction, + request.community_id, + request.operation_id, + ) + .await? + .ok_or_else(|| { + DbError::InvalidData( + "authorization invalidation replay lacks exact attribution".to_owned(), + ) + })?; + let exact_component = invalidation_manifest_matches_request(&manifest, &request); + if !exact_component { + return Err(DbError::InvalidData( + "authorization invalidation replay attribution is incomplete".to_owned(), + ) + .into()); + } + let generation = manifest + .components() + .iter() + .find(|component| { + component.component_kind() + == AuthorizationVersionComponentKind::InvalidationGeneration + }) + .map(AuthorizationOperationVersionDelta::after_version) + .ok_or_else(|| { + DbError::InvalidData( + "authorization invalidation replay generation is missing".to_owned(), + ) + })?; + let result_digest = invalidation_result_digest( + request.community_id, + request.operation_id, + request.request_fingerprint, + generation, + &request.selectors, + ); + let receipt = AuthorizationOperationReceipt::new( + request.community_id, + request.operation_id, + request.request_fingerprint, + AuthorizationOperationKind::Invalidation, + request.actor.clone(), + AuthorizationOperationOutcome::Applied, + result_digest, + )?; + if record_authorization_operation_receipt_tx(&mut transaction, &receipt).await? + != AuthorizationReceiptWrite::ExactReplay + { + return Err(DbError::InvalidData( + "authorization invalidation replay receipt is incomplete".to_owned(), + ) + .into()); + } + let event = NewAuthorizationEvent::new( + request.community_id, + request.event_id, + AuthorizationEventKind::InvalidationAdvanced, + AuthorizationEventOutcome::Allowed, + AuthorizationReasonCode::Invalidated, + request.actor.clone(), + request.subject_fingerprint, + request.operation_id, + Some(request.request_fingerprint), + request.correlation_id, + request.attempt_id, + )?; + if !matches!( + record_authorization_event_tx(&mut transaction, &event).await, + Ok(AuthorizationReceiptWrite::ExactReplay) + ) { + return Err(DbError::InvalidData( + "authorization invalidation replay event is incomplete".to_owned(), + ) + .into()); + } + transaction.rollback().await.map_err(DbError::from)?; + return Ok(AuthorizationInvalidationApplied { + generation, + replay: true, + }); + } + + let authority_advance = crate::authorization_version::advance_admission_loss_authority_tx( + &mut transaction, + request.community_id, + request.operation_id, + request.request_fingerprint, + &request.actor, + ) + .await?; + let advance = apply_admission_loss_invalidation_tx( + &mut transaction, + request.community_id, + request.operation_id, + request.request_fingerprint, + &request.selectors, + authority_advance, + ) + .await?; + let generation = advance.generation(); + + let result_digest = invalidation_result_digest( + request.community_id, + request.operation_id, + request.request_fingerprint, + generation, + &request.selectors, + ); + let receipt = AuthorizationOperationReceipt::new( + request.community_id, + request.operation_id, + request.request_fingerprint, + AuthorizationOperationKind::Invalidation, + request.actor.clone(), + AuthorizationOperationOutcome::Applied, + result_digest, + )?; + let receipt_write = + record_authorization_operation_receipt_tx(&mut transaction, &receipt).await?; + if receipt_write != AuthorizationReceiptWrite::Inserted { + return Err(DbError::InvalidData( + "authorization invalidation receipt raced with another writer".to_owned(), + ) + .into()); + } + + record_authorization_operation_version_delta_tx( + &mut transaction, + request.community_id, + request.operation_id, + request.request_fingerprint, + advance.into_deltas(), + ) + .await?; + + let event = NewAuthorizationEvent::new( + request.community_id, + request.event_id, + AuthorizationEventKind::InvalidationAdvanced, + AuthorizationEventOutcome::Allowed, + AuthorizationReasonCode::Invalidated, + request.actor.clone(), + request.subject_fingerprint, + request.operation_id, + Some(request.request_fingerprint), + request.correlation_id, + request.attempt_id, + )?; + match record_authorization_event_tx(&mut transaction, &event).await { + Ok(AuthorizationReceiptWrite::Inserted) => {} + Ok(AuthorizationReceiptWrite::ExactReplay) => { + return Err(DbError::InvalidData( + "authorization invalidation event raced with another writer".to_owned(), + ) + .into()); + } + Err(AuthorizationEventWriteError::CapacityUnavailable) => { + transaction.rollback().await.map_err(DbError::from)?; + latch_invalidation_audit_failure( + self, + request.community_id, + crate::authorization_events::AuthorizationAuditFailureCode::CapacityExhausted, + ) + .await; + return Err(AuthorizationInvalidationApplyError::AuditUnavailable); + } + Err(AuthorizationEventWriteError::Database(_)) => { + transaction.rollback().await.map_err(DbError::from)?; + latch_invalidation_audit_failure( + self, + request.community_id, + crate::authorization_events::AuthorizationAuditFailureCode::StorageUnavailable, + ) + .await; + return Err(AuthorizationInvalidationApplyError::AuditUnavailable); + } + } + + transaction.commit().await.map_err(DbError::from)?; + Ok(AuthorizationInvalidationApplied { + generation, + replay: false, + }) + } + + /// Capture exact current invalidation state for a bounded selector set. + pub async fn authorization_invalidation_snapshot( + &self, + community_id: CommunityId, + selectors: &[AuthorizationInvalidationSelector], + ) -> Result { + validate_selectors(community_id, selectors)?; + let mut transaction = self.pool.begin().await?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY") + .execute(&mut *transaction) + .await?; + let generation: i64 = sqlx::query_scalar( + "SELECT current_generation FROM authorization_invalidation_domains WHERE community_id=$1", + ) + .bind(community_id.as_uuid()) + .fetch_optional(&mut *transaction) + .await? + .ok_or_else(|| { + DbError::NotFound("authorization invalidation domain is not activated".to_owned()) + })?; + let mut floors = Vec::new(); + for selector in selectors { + let fingerprint = selector.fingerprint(community_id); + if let Some(row) = sqlx::query( + "SELECT selector_kind,floor_generation,binding_version_floor, \ + relationship_revision_floor \ + FROM authorization_invalidation_floors \ + WHERE community_id=$1 AND selector_kind=$2 AND selector_fingerprint=$3", + ) + .bind(community_id.as_uuid()) + .bind(selector.kind() as i16) + .bind(fingerprint.as_slice()) + .fetch_optional(&mut *transaction) + .await? + { + floors.push(AuthorizationInvalidationFloor { + kind: AuthorizationInvalidationSelectorKind::from_database( + row.try_get("selector_kind")?, + )?, + fingerprint, + generation: database_version(row.try_get("floor_generation")?)?, + binding_version_floor: row + .try_get::, _>("binding_version_floor")? + .map(database_version) + .transpose()?, + relationship_revision_floor: row + .try_get::, _>("relationship_revision_floor")? + .map(database_version) + .transpose()?, + }); + } + } + transaction.commit().await?; + floors.sort_by_key(|floor| (floor.kind, floor.fingerprint)); + Ok(AuthorizationInvalidationSnapshot { + community_id, + generation: database_version(generation)?, + floors, + }) + } +} + +async fn lock_invalidation_generation_tx( + transaction: &mut Transaction<'_, Postgres>, + community_id: CommunityId, +) -> Result { + let generation: i64 = sqlx::query_scalar( + "SELECT current_generation FROM authorization_invalidation_domains \ + WHERE community_id=$1 FOR UPDATE", + ) + .bind(community_id.as_uuid()) + .fetch_optional(&mut **transaction) + .await? + .ok_or_else(|| { + DbError::NotFound("authorization invalidation domain is not activated".to_owned()) + })?; + database_version(generation) +} + +async fn advance_locked_invalidation_tx( + transaction: &mut Transaction<'_, Postgres>, + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + selectors: &[AuthorizationInvalidationSelector], + before: u64, +) -> Result { + let generation = before.checked_add(1).ok_or_else(|| { + DbError::InvalidData("authorization invalidation generation exhausted".to_owned()) + })?; + let updated = sqlx::query( + "UPDATE authorization_invalidation_domains \ + SET current_generation=$2,updated_at=clock_timestamp() \ + WHERE community_id=$1 AND current_generation=$3", + ) + .bind(community_id.as_uuid()) + .bind(to_database_version(generation)?) + .bind(to_database_version(before)?) + .execute(&mut **transaction) + .await?; + if updated.rows_affected() != 1 { + return Err(DbError::InvalidData( + "authorization invalidation generation changed concurrently".to_owned(), + )); + } + + let mut deltas = vec![AuthorizationOperationVersionDelta::new( + AuthorizationVersionComponentKind::InvalidationGeneration, + authorization_version_invalidation_generation_component_key(community_id), + before, + generation, + )?]; + for selector in selectors { + let selector_fingerprint = selector.fingerprint(community_id); + let prior_relationship_floor = if selector.relationship_floor().is_some() { + sqlx::query_scalar::<_, Option>( + "SELECT relationship_revision_floor FROM authorization_invalidation_floors \ + WHERE community_id=$1 AND selector_kind=7 AND selector_fingerprint=$2 FOR UPDATE", + ) + .bind(community_id.as_uuid()) + .bind(selector_fingerprint.as_slice()) + .fetch_optional(&mut **transaction) + .await? + .flatten() + .map(database_version) + .transpose()? + .unwrap_or(0) + } else { + 0 + }; + sqlx::query( + "INSERT INTO authorization_invalidation_floors \ + (community_id,selector_kind,selector_fingerprint,floor_generation, \ + binding_version_floor,relationship_revision_floor,operation_id,request_fingerprint) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8) \ + ON CONFLICT (community_id,selector_kind,selector_fingerprint) DO UPDATE SET \ + floor_generation=GREATEST(authorization_invalidation_floors.floor_generation, \ + EXCLUDED.floor_generation), \ + binding_version_floor=CASE WHEN EXCLUDED.binding_version_floor IS NULL \ + THEN authorization_invalidation_floors.binding_version_floor \ + ELSE GREATEST(authorization_invalidation_floors.binding_version_floor, \ + EXCLUDED.binding_version_floor) END, \ + relationship_revision_floor=CASE WHEN EXCLUDED.relationship_revision_floor IS NULL \ + THEN authorization_invalidation_floors.relationship_revision_floor \ + ELSE GREATEST(authorization_invalidation_floors.relationship_revision_floor, \ + EXCLUDED.relationship_revision_floor) END, \ + operation_id=EXCLUDED.operation_id,request_fingerprint=EXCLUDED.request_fingerprint, \ + updated_at=clock_timestamp()", + ) + .bind(community_id.as_uuid()) + .bind(selector.kind() as i16) + .bind(selector_fingerprint.as_slice()) + .bind(to_database_version(generation)?) + .bind(selector.binding_floor().map(to_database_version).transpose()?) + .bind(selector.relationship_floor().map(to_database_version).transpose()?) + .bind(operation_id) + .bind(request_fingerprint.as_slice()) + .execute(&mut **transaction) + .await?; + if let Some(relationship_floor) = selector.relationship_floor() { + if relationship_floor > prior_relationship_floor { + deltas.push(AuthorizationOperationVersionDelta::new( + AuthorizationVersionComponentKind::DelegatedRelationship, + selector_fingerprint, + prior_relationship_floor, + relationship_floor, + )?); + } + } + } + + notify_authorization_change_tx( + transaction, + &format!("D:{}:{generation}", community_id.as_uuid()), + ) + .await?; + + Ok(AuthorizationInvalidationAdvance { + generation, + authority_objects: Vec::new(), + deltas, + }) +} + +pub(crate) async fn notify_protected_publication_advance_tx( + transaction: &mut Transaction<'_, Postgres>, + dependency: &ProtectedPublicationDependency, + authority_epoch: u64, +) -> Result<()> { + if authority_epoch == 0 { + return Err(DbError::InvalidData( + "authorization publication notice epoch is invalid".to_owned(), + )); + } + notify_authorization_change_tx( + transaction, + &format!( + "P:{}:{}:{}:{authority_epoch}", + dependency.authorization_domain().as_uuid(), + dependency.object_kind() as i16, + hex::encode(dependency.object_key()) + ), + ) + .await +} + +async fn notify_authorization_change_tx( + transaction: &mut Transaction<'_, Postgres>, + payload: &str, +) -> Result<()> { + sqlx::query("SELECT pg_notify($1,$2)") + .bind(AUTHORIZATION_INVALIDATION_CHANNEL) + .bind(payload) + .execute(&mut **transaction) + .await?; + Ok(()) +} + +fn parse_authorization_invalidation_notice( + payload: &str, +) -> Result { + let fields: Vec<_> = payload.split(':').collect(); + match fields.as_slice() { + ["D", domain, generation] => { + let authorization_domain = + CommunityId::from_uuid(Uuid::parse_str(domain).map_err(|_| { + DbError::InvalidData( + "authorization invalidation notice domain is invalid".to_owned(), + ) + })?); + let generation = generation.parse::().map_err(|_| { + DbError::InvalidData( + "authorization invalidation notice generation is invalid".to_owned(), + ) + })?; + if authorization_domain.as_uuid().is_nil() || generation == 0 { + return Err(DbError::InvalidData( + "authorization invalidation notice is invalid".to_owned(), + )); + } + Ok(AuthorizationInvalidationNotice::DomainAdvanced { + authorization_domain, + generation, + }) + } + ["P", domain, kind, key, authority_epoch] => { + let authorization_domain = + CommunityId::from_uuid(Uuid::parse_str(domain).map_err(|_| { + DbError::InvalidData( + "authorization publication notice domain is invalid".to_owned(), + ) + })?); + let object_kind = kind + .parse::() + .map_err(|_| { + DbError::InvalidData( + "authorization publication notice kind is invalid".to_owned(), + ) + }) + .and_then(AuthorizationProtectedObjectKind::from_database)?; + let key = hex::decode(key).map_err(|_| { + DbError::InvalidData("authorization publication notice key is invalid".to_owned()) + })?; + let object_key: [u8; 32] = key.try_into().map_err(|_| { + DbError::InvalidData("authorization publication notice key is invalid".to_owned()) + })?; + let authority_epoch = authority_epoch.parse::().map_err(|_| { + DbError::InvalidData("authorization publication notice epoch is invalid".to_owned()) + })?; + if authority_epoch == 0 { + return Err(DbError::InvalidData( + "authorization publication notice epoch is invalid".to_owned(), + )); + } + Ok( + AuthorizationInvalidationNotice::ProtectedPublicationAdvanced { + dependency: ProtectedPublicationDependency::from_database_parts( + authorization_domain, + object_kind, + object_key, + )?, + authority_epoch, + }, + ) + } + _ => Err(DbError::InvalidData( + "authorization invalidation notice is malformed".to_owned(), + )), + } +} + +fn invalidation_manifest_matches_request( + manifest: &AuthorizationOperationVersionDeltaManifest, + request: &AuthorizationInvalidationRequest, +) -> bool { + if manifest.request_fingerprint() != request.request_fingerprint { + return false; + } + let invalidation_key = + authorization_version_invalidation_generation_component_key(request.community_id); + let delegated: BTreeSet<_> = request + .selectors + .iter() + .filter_map(|selector| match selector { + AuthorizationInvalidationSelector::DelegatedRelationship { + relationship_id, + invalid_through, + } => Some(( + authorization_version_delegated_relationship_component_key( + request.community_id, + *relationship_id, + ), + *invalid_through, + )), + _ => None, + }) + .collect(); + let mut invalidation_count = 0_usize; + for component in manifest.components() { + match component.component_kind() { + AuthorizationVersionComponentKind::InvalidationGeneration => { + invalidation_count += 1; + if component.component_key() != invalidation_key + || component.before_version().checked_add(1) != Some(component.after_version()) + { + return false; + } + } + AuthorizationVersionComponentKind::DelegatedRelationship => { + if !delegated.contains(&(component.component_key(), component.after_version())) { + return false; + } + } + AuthorizationVersionComponentKind::AuthorityEpoch => {} + _ => return false, + } + } + invalidation_count == 1 +} + +fn validate_request(request: &AuthorizationInvalidationRequest) -> Result<()> { + if request.community_id.as_uuid().is_nil() + || request.operation_id.is_nil() + || request.request_fingerprint == [0; 32] + || !request.actor.is_authenticated() + || !request.actor.is_bound_to(request.community_id) + || !actor_matches_selectors(&request.actor, &request.selectors) + || request.correlation_id.is_nil() + || request.attempt_id.is_nil() + || request.event_id.is_nil() + { + return Err(DbError::InvalidData( + "authorization invalidation request is invalid".to_owned(), + )); + } + validate_selectors(request.community_id, &request.selectors) +} + +fn actor_matches_selectors( + actor: &AuthorizationEventActor, + selectors: &[AuthorizationInvalidationSelector], +) -> bool { + actor + .authority_loss_target() + .is_some_and(|target| loss_target_matches_selectors(target, selectors)) +} + +fn loss_target_matches_selectors( + target: crate::authorization_events::AuthorizationAuthorityLossTarget, + selectors: &[AuthorizationInvalidationSelector], +) -> bool { + if selectors.len() != 1 { + return false; + } + match target { + crate::authorization_events::AuthorizationAuthorityLossTarget::Binding( + binding_id, + binding_version, + ) => selectors.iter().any(|selector| { + matches!(selector, AuthorizationInvalidationSelector::Binding { + binding_id: candidate_id, + invalid_through, + } if *candidate_id == binding_id && *invalid_through == binding_version) + }), + crate::authorization_events::AuthorizationAuthorityLossTarget::Policy(policy_revision) => { + selectors.iter().any(|selector| { + matches!(selector, AuthorizationInvalidationSelector::ConfigurationRevision( + revision + ) if *revision == policy_revision) + }) + } + crate::authorization_events::AuthorizationAuthorityLossTarget::DelegatedRelationship( + relationship_id, + relationship_revision, + ) => selectors.iter().any(|selector| { + matches!(selector, AuthorizationInvalidationSelector::DelegatedRelationship { + relationship_id: candidate_id, + invalid_through, + } if *candidate_id == relationship_id && *invalid_through == relationship_revision) + }), + } +} + +async fn latch_invalidation_audit_failure( + db: &Db, + community_id: CommunityId, + failure: crate::authorization_events::AuthorizationAuditFailureCode, +) { + if let Err(error) = db + .latch_authorization_event_failure(community_id, failure) + .await + { + tracing::error!(error = %error, "failed to durably latch invalidation audit health"); + } +} + +fn validate_selectors( + community_id: CommunityId, + selectors: &[AuthorizationInvalidationSelector], +) -> Result<()> { + if community_id.as_uuid().is_nil() + || selectors.is_empty() + || selectors.len() > MAX_AUTHORIZATION_INVALIDATION_SELECTORS + { + return Err(DbError::InvalidData( + "authorization invalidation selector set is invalid".to_owned(), + )); + } + let unique: BTreeSet<_> = selectors + .iter() + .map(|selector| (selector.kind(), selector.fingerprint(community_id))) + .collect(); + if unique.len() != selectors.len() { + return Err(DbError::InvalidData( + "authorization invalidation selector set contains duplicates".to_owned(), + )); + } + Ok(()) +} + +fn invalidation_result_digest( + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + generation: u64, + selectors: &[AuthorizationInvalidationSelector], +) -> [u8; 32] { + let mut coordinates: Vec<_> = selectors + .iter() + .map(|selector| { + ( + selector.kind(), + selector.fingerprint(community_id), + selector.binding_floor(), + selector.relationship_floor(), + ) + }) + .collect(); + coordinates.sort(); + let mut digest = Sha256::new(); + framed(&mut digest, b"buzz:authorization-invalidation-result:v1"); + framed(&mut digest, community_id.as_uuid().as_bytes()); + framed(&mut digest, operation_id.as_bytes()); + framed(&mut digest, &request_fingerprint); + framed(&mut digest, &generation.to_be_bytes()); + for (kind, fingerprint, binding_floor, relationship_floor) in coordinates { + framed(&mut digest, &(kind as i16).to_be_bytes()); + framed(&mut digest, &fingerprint); + framed( + &mut digest, + &binding_floor.unwrap_or_default().to_be_bytes(), + ); + framed( + &mut digest, + &relationship_floor.unwrap_or_default().to_be_bytes(), + ); + } + digest.finalize().into() +} + +fn nonzero(value: [u8; 32], name: &str) -> Result<()> { + if value == [0; 32] { + return Err(DbError::InvalidData(format!( + "authorization {name} must not be zero" + ))); + } + Ok(()) +} + +fn framed(digest: &mut Sha256, value: &[u8]) { + digest.update((value.len() as u64).to_be_bytes()); + digest.update(value); +} + +fn to_database_version(value: u64) -> Result { + i64::try_from(value) + .map_err(|_| DbError::InvalidData("authorization version exceeds BIGINT".to_owned())) +} + +fn database_version(value: i64) -> Result { + u64::try_from(value) + .map_err(|_| DbError::InvalidData("authorization version is negative".to_owned())) +} + +#[cfg(test)] +mod tests { + use super::*; + use buzz_auth::AuthorizationEventCapacityPolicy; + use sqlx::PgPool; + + use crate::authorization_events::{ + resolve_local_admission_loss_actor_tx, LocalAdmissionLossCause, + }; + + fn domain() -> CommunityId { + CommunityId::from_uuid(Uuid::from_u128(1)) + } + + fn loopback_test_database_url() -> String { + format!( + "{}://{}:{}@{}:{}/{}", + "postgres", "buzz", "buzz_dev", "localhost", 5432, "buzz" + ) + } + + #[test] + fn selector_coordinates_are_closed_and_redacted() { + let binding = AuthorizationInvalidationSelector::binding(Uuid::from_u128(2), 7) + .expect("valid binding selector"); + let relationship = + AuthorizationInvalidationSelector::delegated_relationship(Uuid::from_u128(2), 7) + .expect("valid relationship selector"); + assert_ne!( + binding.fingerprint(domain()), + relationship.fingerprint(domain()) + ); + assert_eq!( + relationship.fingerprint(domain()), + authorization_version_delegated_relationship_component_key( + domain(), + Uuid::from_u128(2) + ) + ); + assert!(!format!("{binding:?}").contains(&Uuid::from_u128(2).to_string())); + assert!(AuthorizationInvalidationSelector::binding(Uuid::nil(), 7).is_err()); + assert!( + AuthorizationInvalidationSelector::delegated_relationship(Uuid::from_u128(2), 0) + .is_err() + ); + } + + #[test] + fn selected_domain_and_duplicate_sets_fail_closed() { + let selector = AuthorizationInvalidationSelector::domain(); + assert!(validate_selectors(domain(), std::slice::from_ref(&selector)).is_ok()); + assert!(validate_selectors(domain(), &[selector.clone(), selector]).is_err()); + assert!(validate_selectors( + CommunityId::from_uuid(Uuid::nil()), + &[AuthorizationInvalidationSelector::domain()] + ) + .is_err()); + } + + #[test] + fn snapshot_requires_generation_and_floors_to_match_exactly() { + let original = AuthorizationInvalidationSnapshot { + community_id: domain(), + generation: 7, + floors: Vec::new(), + }; + assert!(original.accepts_exact_recheck(&original)); + let changed = AuthorizationInvalidationSnapshot { + community_id: domain(), + generation: 8, + floors: Vec::new(), + }; + assert!(!original.accepts_exact_recheck(&changed)); + } + + #[test] + fn live_notice_payloads_are_typed_bounded_and_redacted() { + let domain = domain(); + let domain_notice = + parse_authorization_invalidation_notice(&format!("D:{}:7", domain.as_uuid())) + .expect("valid domain notice"); + assert!(matches!( + domain_notice, + AuthorizationInvalidationNotice::DomainAdvanced { + authorization_domain, + generation: 7, + } if authorization_domain == domain + )); + let object_notice = parse_authorization_invalidation_notice(&format!( + "P:{}:3:{}:9", + domain.as_uuid(), + hex::encode([8_u8; 32]) + )) + .expect("valid publication notice"); + assert!(matches!( + &object_notice, + AuthorizationInvalidationNotice::ProtectedPublicationAdvanced { + authority_epoch: 9, + .. + } + )); + assert!(!format!("{object_notice:?}").contains(&hex::encode([8_u8; 32]))); + for malformed in [ + "D:nil:1", + "D:00000000-0000-0000-0000-000000000000:1", + "D:00000000-0000-0000-0000-000000000001:0", + "P:00000000-0000-0000-0000-000000000001:3:00:1", + "P:00000000-0000-0000-0000-000000000001:99:0000000000000000000000000000000000000000000000000000000000000000:1", + ] { + assert!(parse_authorization_invalidation_notice(malformed).is_err()); + } + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn replay_rejects_missing_event_and_incomplete_manifest() { + let admin_url = + std::env::var("TEST_DATABASE_URL").unwrap_or_else(|_| loopback_test_database_url()); + let admin = PgPool::connect(&admin_url).await.expect("connect admin"); + let name = format!("s5_invalidation_replay_{}", Uuid::new_v4().simple()); + sqlx::query(sqlx::AssertSqlSafe(format!("CREATE DATABASE {name}"))) + .execute(&admin) + .await + .expect("create scratch database"); + let split = admin_url.rfind('/').expect("database URL path"); + let scratch_url = format!("{}/{}", &admin_url[..split], name); + let pool = PgPool::connect(&scratch_url) + .await + .expect("connect scratch database"); + crate::migration::run_migrations(&pool) + .await + .expect("migrate scratch database"); + + let community_uuid = Uuid::new_v4(); + let community = CommunityId::from_uuid(community_uuid); + sqlx::query("INSERT INTO communities (id,host) VALUES ($1,$2)") + .bind(community_uuid) + .bind(format!("invalidation-{}.example", community_uuid.simple())) + .execute(&pool) + .await + .expect("insert community"); + sqlx::query( + "INSERT INTO identity_enrollment_policies \ + (community_id,policy_revision,enrollment_mode,policy_digest,effective_at) \ + VALUES ($1,1,1,$2,clock_timestamp() - INTERVAL '1 second')", + ) + .bind(community_uuid) + .bind(vec![1_u8; 32]) + .execute(&pool) + .await + .expect("insert policy"); + sqlx::query( + "INSERT INTO authorization_invalidation_domains (community_id,current_generation) \ + VALUES ($1,0)", + ) + .bind(community_uuid) + .execute(&pool) + .await + .expect("activate invalidation"); + let db = Db::from_pool(pool.clone()); + db.install_authorization_event_capacity( + community, + AuthorizationEventCapacityPolicy::new(100, 1 << 20, 16 << 10).expect("valid capacity"), + ) + .await + .expect("install capacity"); + let actor = { + let mut transaction = pool.begin().await.expect("begin actor recheck"); + let actor = resolve_local_admission_loss_actor_tx( + &mut transaction, + community, + LocalAdmissionLossCause::Policy { policy_revision: 1 }, + ) + .await + .expect("resolve local policy actor"); + transaction.commit().await.expect("commit actor recheck"); + actor + }; + let mut live_notices = db + .subscribe_authorization_invalidations() + .await + .expect("install ready independent listener"); + + let first = invalidation_request(community, actor.clone(), 10); + let mut mismatched = first.clone(); + mismatched.operation_id = Uuid::from_u128(9_999); + mismatched.selectors = vec![AuthorizationInvalidationSelector::domain()]; + assert!(db + .apply_authorization_invalidation(mismatched) + .await + .is_err()); + let applied = db + .apply_authorization_invalidation(first.clone()) + .await + .expect("apply first invalidation"); + assert!(!applied.replay); + assert!(matches!( + live_notices.recv().await.expect("receive first generation"), + AuthorizationInvalidationNotice::DomainAdvanced { + authorization_domain, + generation: 1, + } if authorization_domain == community + )); + assert!( + db.apply_authorization_invalidation(first.clone()) + .await + .expect("exact replay") + .replay + ); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), live_notices.recv()) + .await + .is_err(), + "exact replay emits no live notice" + ); + + let relationship_id = Uuid::new_v4(); + let relationship_selector = + AuthorizationInvalidationSelector::delegated_relationship(relationship_id, 7) + .expect("valid delegated relationship selector"); + let relationship_operation = Uuid::new_v4(); + let relationship_request = [71_u8; 32]; + let mut relationship_tx = pool.begin().await.expect("begin relationship floor"); + let relationship_before = lock_invalidation_generation_tx(&mut relationship_tx, community) + .await + .expect("lock relationship generation"); + let relationship_advance = advance_locked_invalidation_tx( + &mut relationship_tx, + community, + relationship_operation, + relationship_request, + std::slice::from_ref(&relationship_selector), + relationship_before, + ) + .await + .expect("advance relationship floor"); + let relationship_receipt = AuthorizationOperationReceipt::new( + community, + relationship_operation, + relationship_request, + AuthorizationOperationKind::Invalidation, + actor.clone(), + AuthorizationOperationOutcome::Applied, + [72_u8; 32], + ) + .expect("relationship receipt"); + record_authorization_operation_receipt_tx(&mut relationship_tx, &relationship_receipt) + .await + .expect("record relationship receipt"); + record_authorization_operation_version_delta_tx( + &mut relationship_tx, + community, + relationship_operation, + relationship_request, + relationship_advance.into_deltas(), + ) + .await + .expect("record relationship manifest"); + relationship_tx + .commit() + .await + .expect("commit relationship floor"); + assert!(matches!( + live_notices + .recv() + .await + .expect("receive relationship generation"), + AuthorizationInvalidationNotice::DomainAdvanced { + authorization_domain, + generation: 2, + } if authorization_domain == community + )); + let relationship_manifest = db + .authorization_operation_version_delta( + community, + relationship_operation, + relationship_request, + ) + .await + .expect("load relationship manifest"); + assert!(relationship_manifest.components().iter().any(|component| { + component.component_kind() == AuthorizationVersionComponentKind::DelegatedRelationship + && component.component_key() + == authorization_version_delegated_relationship_component_key( + community, + relationship_id, + ) + && component.before_version() == 0 + && component.after_version() == 7 + })); + let floors = db + .authorization_version_component_floors(community) + .await + .expect("load relationship floor"); + assert!(floors.iter().any(|floor| { + floor.component_kind == AuthorizationVersionComponentKind::DelegatedRelationship + && floor.component_key + == authorization_version_delegated_relationship_component_key( + community, + relationship_id, + ) + && floor.version == 7 + })); + + let mut corrupt = pool.acquire().await.expect("corrupt connection"); + sqlx::query("SET session_replication_role=replica") + .execute(&mut *corrupt) + .await + .expect("disable immutable triggers"); + sqlx::query("DELETE FROM authorization_events WHERE community_id=$1 AND operation_id=$2") + .bind(community_uuid) + .bind(first.operation_id) + .execute(&mut *corrupt) + .await + .expect("remove replay event"); + assert!(db.apply_authorization_invalidation(first).await.is_err()); + + let second = invalidation_request(community, actor.clone(), 20); + db.apply_authorization_invalidation(second.clone()) + .await + .expect("apply second invalidation"); + sqlx::query( + "DELETE FROM authorization_operation_version_deltas \ + WHERE community_id=$1 AND operation_id=$2", + ) + .bind(community_uuid) + .bind(second.operation_id) + .execute(&mut *corrupt) + .await + .expect("remove replay delta"); + assert!(db.apply_authorization_invalidation(second).await.is_err()); + + let third = invalidation_request(community, actor.clone(), 30); + db.apply_authorization_invalidation(third.clone()) + .await + .expect("apply third invalidation"); + sqlx::query( + "UPDATE authorization_operation_receipts SET result_digest=$3 \ + WHERE community_id=$1 AND operation_id=$2", + ) + .bind(community_uuid) + .bind(third.operation_id) + .bind(vec![99_u8; 32]) + .execute(&mut *corrupt) + .await + .expect("corrupt replay receipt"); + assert!(db.apply_authorization_invalidation(third).await.is_err()); + + let fourth = invalidation_request(community, actor, 40); + db.apply_authorization_invalidation(fourth.clone()) + .await + .expect("apply fourth invalidation"); + sqlx::query( + "UPDATE authorization_events SET canonical_envelope=$3 \ + WHERE community_id=$1 AND operation_id=$2", + ) + .bind(community_uuid) + .bind(fourth.operation_id) + .bind(vec![98_u8; 32]) + .execute(&mut *corrupt) + .await + .expect("corrupt replay envelope"); + assert!(db.apply_authorization_invalidation(fourth).await.is_err()); + sqlx::query("SET session_replication_role=origin") + .execute(&mut *corrupt) + .await + .expect("restore immutable triggers"); + drop(corrupt); + + pool.close().await; + let _ = sqlx::query(sqlx::AssertSqlSafe(format!( + "DROP DATABASE IF EXISTS {name} WITH (FORCE)" + ))) + .execute(&admin) + .await; + } + + fn invalidation_request( + community_id: CommunityId, + actor: AuthorizationEventActor, + tag: u8, + ) -> AuthorizationInvalidationRequest { + AuthorizationInvalidationRequest { + community_id, + operation_id: Uuid::from_u128(u128::from(tag) + 1_000), + request_fingerprint: [tag; 32], + actor, + subject_fingerprint: None, + correlation_id: Uuid::from_u128(u128::from(tag) + 2_000), + attempt_id: Uuid::from_u128(u128::from(tag) + 3_000), + event_id: Uuid::from_u128(u128::from(tag) + 4_000), + selectors: vec![AuthorizationInvalidationSelector::configuration_revision(1) + .expect("valid policy selector")], + } + } +} diff --git a/crates/buzz-db/src/authorization_resolver.rs b/crates/buzz-db/src/authorization_resolver.rs new file mode 100644 index 00000000000..d9968f05508 --- /dev/null +++ b/crates/buzz-db/src/authorization_resolver.rs @@ -0,0 +1,455 @@ +//! PostgreSQL-backed local binding resolution. +//! +//! All existing-binding and status paths are observation-only. They never +//! enroll, consume lifecycle selectors, append history, or refresh timestamps. + +use std::fmt; + +use buzz_auth::{ + ActiveLocalBinding, BindingResolutionRequest, CurrentBindingStatusEvidenceRequest, + LocalAuthorizationPolicy, LocalBindingResolution, LocalBindingResolver, + LocalBindingResolverCapability, +}; +use buzz_core::{AuthorizationLeaseFence, CanonicalCurrentBindingEvidence, CommunityId}; +use chrono::{DateTime, Utc}; +use nostr::PublicKey; +use sha2::{Digest, Sha256}; +use sqlx::Row; +use thiserror::Error; + +use crate::{ + authorization_version::{protected_object_key, AuthorizationProtectedObjectKind}, + Db, DbError, +}; + +/// Concrete provider-free PostgreSQL local binding resolver. +#[derive(Clone)] +pub struct PostgresLocalBindingResolver { + db: Db, +} + +impl PostgresLocalBindingResolver { + /// Bind the resolver to the authoritative writer database. + pub fn new(db: Db) -> Self { + Self { db } + } + + /// Borrow the underlying database handle for same-authority composition. + pub const fn db(&self) -> &Db { + &self.db + } +} + +impl fmt::Debug for PostgresLocalBindingResolver { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("PostgresLocalBindingResolver([REDACTED])") + } +} + +/// Fail-closed local resolver errors. +#[derive(Debug, Error)] +pub enum AuthorizationResolverError { + /// PostgreSQL failed or returned malformed authoritative state. + #[error(transparent)] + Database(#[from] DbError), + /// No exact current binding exists. + #[error("current local binding is unavailable")] + BindingUnavailable, + /// A frozen cross-lane trusted constructor or mutation seam is not installed. + #[error("local binding resolver contract is unavailable")] + ContractUnavailable, + /// Delegated protected transport has no reviewed positive authority source. + #[error("delegated protected authorization is unavailable")] + DelegationAuthorityUnavailable, + /// The exact local policy or protected authority snapshot is unavailable. + #[error("local authorization policy is unavailable")] + PolicyUnavailable, +} + +impl PostgresLocalBindingResolver { + /// Resolve the exact direct-route policy and protected-object fence from + /// one repeatable-read PostgreSQL snapshot. + pub async fn protected_publication_policy( + &self, + request: &BindingResolutionRequest, + object_kind: AuthorizationProtectedObjectKind, + maximum_lease: chrono::Duration, + ) -> std::result::Result<(LocalAuthorizationPolicy, DateTime), AuthorizationResolverError> + { + if maximum_lease <= chrono::Duration::zero() { + return Err(AuthorizationResolverError::PolicyUnavailable); + } + let (assertion, proof, capability) = match request { + BindingResolutionRequest::Direct { + assertion, + proof, + capability, + } => (assertion, proof, *capability), + BindingResolutionRequest::Delegated { .. } => { + return Err(AuthorizationResolverError::DelegationAuthorityUnavailable); + } + BindingResolutionRequest::Enrollment { .. } => { + return Err(AuthorizationResolverError::ContractUnavailable); + } + }; + if assertion.authorization_domain() != proof.authorization_domain() + || !object_kind.admits_read(capability) + { + return Err(AuthorizationResolverError::PolicyUnavailable); + } + + let domain = assertion.authorization_domain(); + let principal = assertion.principal_storage_key(); + let object_key = protected_object_key(domain, object_kind, *proof.target_fingerprint()); + let mut transaction = self.db.pool.begin().await.map_err(DbError::from)?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY") + .execute(&mut *transaction) + .await + .map_err(DbError::from)?; + let row = sqlx::query( + "SELECT binding.policy_revision,policy.expires_at AS policy_expires_at, \ + invalidation.current_generation,clock_timestamp() AS authoritative_now, \ + epoch.authority_epoch AS epoch_authority_epoch,epoch.fence AS epoch_fence, \ + protected.authority_epoch AS protected_authority_epoch, \ + protected.fence AS protected_fence \ + FROM identity_bindings binding \ + JOIN identity_enrollment_policies policy \ + ON policy.community_id=binding.community_id \ + AND policy.policy_revision=binding.policy_revision \ + JOIN authorization_invalidation_domains invalidation \ + ON invalidation.community_id=binding.community_id \ + LEFT JOIN authorization_authority_epochs epoch \ + ON epoch.community_id=binding.community_id AND epoch.object_kind=$5 \ + AND epoch.object_key=$6 \ + LEFT JOIN protected_object_authority protected \ + ON protected.community_id=binding.community_id AND protected.object_kind=$5 \ + AND protected.object_key=$6 \ + WHERE binding.community_id=$1 AND binding.issuer=$2 AND binding.subject=$3 \ + AND binding.event_author_pubkey=$4 AND binding.binding_state=1 \ + AND (binding.expires_at IS NULL OR binding.expires_at > clock_timestamp()) \ + AND policy.effective_at <= clock_timestamp() \ + AND (policy.expires_at IS NULL OR policy.expires_at > clock_timestamp())", + ) + .bind(domain.as_uuid()) + .bind(principal.issuer()) + .bind(principal.subject()) + .bind(proof.actor_pubkey().to_bytes().as_slice()) + .bind(object_kind as i16) + .bind(object_key.as_slice()) + .fetch_optional(&mut *transaction) + .await + .map_err(DbError::from)?; + transaction.commit().await.map_err(DbError::from)?; + let row = row.ok_or(AuthorizationResolverError::PolicyUnavailable)?; + let now: DateTime = row.try_get("authoritative_now").map_err(DbError::from)?; + let policy_expires_at: Option> = + row.try_get("policy_expires_at").map_err(DbError::from)?; + let configured_expires_at = now + .checked_add_signed(maximum_lease) + .ok_or(AuthorizationResolverError::PolicyUnavailable)?; + let expires_at = policy_expires_at + .map(|expiry| expiry.min(configured_expires_at)) + .unwrap_or(configured_expires_at); + if expires_at <= now { + return Err(AuthorizationResolverError::PolicyUnavailable); + } + + let lease_id = uuid::Uuid::new_v4(); + let epoch = row + .try_get::, _>("epoch_authority_epoch") + .map_err(DbError::from)? + .map(|value| database_u64(value, "authority epoch")) + .transpose()?; + let epoch_fence = row + .try_get::>, _>("epoch_fence") + .map_err(DbError::from)? + .map(|value| parsed_fence(value, "authority fence")) + .transpose()?; + let protected_epoch = row + .try_get::, _>("protected_authority_epoch") + .map_err(DbError::from)? + .map(|value| database_u64(value, "protected authority epoch")) + .transpose()?; + let protected_fence = row + .try_get::>, _>("protected_fence") + .map_err(DbError::from)? + .map(|value| parsed_fence(value, "protected authority fence")) + .transpose()?; + let (authority_epoch, fence) = match (epoch, epoch_fence, protected_epoch, protected_fence) + { + (None, None, None, None) => ( + 1, + initial_publication_fence(domain, object_kind, object_key, lease_id)?, + ), + (Some(epoch), Some(epoch_fence), Some(protected_epoch), Some(protected_fence)) + if epoch == protected_epoch && epoch_fence == protected_fence => + { + (epoch, epoch_fence) + } + _ => { + return Err(AuthorizationResolverError::Database(DbError::InvalidData( + "protected publication authority rows disagree".to_owned(), + ))) + } + }; + let policy_revision = database_u64( + row.try_get("policy_revision").map_err(DbError::from)?, + "policy revision", + )?; + let generation = database_u64( + row.try_get("current_generation").map_err(DbError::from)?, + "invalidation generation", + )?; + let policy = LocalAuthorizationPolicy::from_database( + domain, + lease_id, + policy_revision, + generation, + authority_epoch, + fence, + capability, + expires_at, + None, + None, + ) + .ok_or_else(|| { + AuthorizationResolverError::Database(DbError::InvalidData( + "protected publication policy row is invalid".to_owned(), + )) + })?; + Ok((policy, now)) + } +} + +impl LocalBindingResolver for PostgresLocalBindingResolver { + type Error = AuthorizationResolverError; + + fn capability(&self) -> LocalBindingResolverCapability { + LocalBindingResolverCapability::DirectAndDelegatedOwnerBound + } + + async fn resolve<'a>( + &'a self, + request: &'a BindingResolutionRequest, + ) -> std::result::Result { + match request { + BindingResolutionRequest::Direct { + assertion, proof, .. + } => { + let key = assertion.principal_storage_key(); + let row = read_active_binding( + &self.db, + assertion.authorization_domain(), + key.issuer(), + key.subject(), + proof.actor_pubkey(), + ) + .await? + .ok_or(AuthorizationResolverError::BindingUnavailable)?; + let binding = ActiveLocalBinding::from_storage( + assertion.authorization_domain(), + assertion.principal_for_storage(), + row.binding_id, + row.binding_version, + row.event_author_pubkey, + row.expires_at, + ) + .ok_or_else(|| { + AuthorizationResolverError::Database(DbError::InvalidData( + "active local binding row is invalid".to_owned(), + )) + })?; + Ok(LocalBindingResolution::direct(assertion.clone(), binding)) + } + BindingResolutionRequest::Delegated { + delegation, proof, .. + } => { + if proof.actor_pubkey() != delegation.delegate_pubkey() { + return Err(AuthorizationResolverError::BindingUnavailable); + } + let row = read_active_binding_by_author( + &self.db, + delegation.authorization_domain(), + delegation.owner_pubkey(), + ) + .await? + .ok_or(AuthorizationResolverError::BindingUnavailable)?; + let binding = ActiveLocalBinding::from_storage_parts( + delegation.authorization_domain(), + row.issuer, + row.subject, + row.binding_id, + row.binding_version, + row.event_author_pubkey, + row.expires_at, + ) + .ok_or_else(|| { + AuthorizationResolverError::Database(DbError::InvalidData( + "delegated owner binding row is invalid".to_owned(), + )) + })?; + Ok(LocalBindingResolution::delegated( + delegation.clone(), + binding, + )) + } + BindingResolutionRequest::Enrollment { .. } => { + Err(AuthorizationResolverError::ContractUnavailable) + } + } + } + + async fn current_status_evidence<'a>( + &'a self, + request: &'a CurrentBindingStatusEvidenceRequest, + ) -> std::result::Result { + let _ = request; + Err(AuthorizationResolverError::ContractUnavailable) + } + + async fn recheck_current_status_evidence<'a>( + &'a self, + evidence: &'a CanonicalCurrentBindingEvidence, + ) -> std::result::Result { + let _ = evidence; + Err(AuthorizationResolverError::ContractUnavailable) + } +} + +#[derive(Clone)] +struct ActiveBindingRow { + issuer: String, + subject: String, + binding_id: uuid::Uuid, + binding_version: u64, + event_author_pubkey: PublicKey, + expires_at: Option>, +} + +async fn read_active_binding( + db: &Db, + community_id: CommunityId, + issuer: &str, + subject: &str, + event_author_pubkey: PublicKey, +) -> std::result::Result, AuthorizationResolverError> { + let row = sqlx::query( + "SELECT issuer,subject,binding_id,binding_version,event_author_pubkey,expires_at \ + FROM identity_bindings \ + WHERE community_id=$1 AND issuer=$2 AND subject=$3 AND event_author_pubkey=$4 \ + AND binding_state=1 AND (expires_at IS NULL OR expires_at > clock_timestamp())", + ) + .bind(community_id.as_uuid()) + .bind(issuer) + .bind(subject) + .bind(event_author_pubkey.to_bytes().as_slice()) + .fetch_optional(&db.pool) + .await + .map_err(DbError::from)?; + row.map(parse_active_binding).transpose() +} + +async fn read_active_binding_by_author( + db: &Db, + community_id: CommunityId, + event_author_pubkey: PublicKey, +) -> std::result::Result, AuthorizationResolverError> { + let row = sqlx::query( + "SELECT issuer,subject,binding_id,binding_version,event_author_pubkey,expires_at \ + FROM identity_bindings \ + WHERE community_id=$1 AND event_author_pubkey=$2 AND binding_state=1 \ + AND (expires_at IS NULL OR expires_at > clock_timestamp())", + ) + .bind(community_id.as_uuid()) + .bind(event_author_pubkey.to_bytes().as_slice()) + .fetch_optional(&db.pool) + .await + .map_err(DbError::from)?; + row.map(parse_active_binding).transpose() +} + +fn parse_active_binding( + row: sqlx::postgres::PgRow, +) -> std::result::Result { + let binding_version: i64 = row.try_get("binding_version").map_err(DbError::from)?; + let event_author: Vec = row.try_get("event_author_pubkey").map_err(DbError::from)?; + Ok(ActiveBindingRow { + issuer: row.try_get("issuer").map_err(DbError::from)?, + subject: row.try_get("subject").map_err(DbError::from)?, + binding_id: row.try_get("binding_id").map_err(DbError::from)?, + binding_version: u64::try_from(binding_version) + .map_err(|_| DbError::InvalidData("active binding version is invalid".to_owned()))?, + event_author_pubkey: PublicKey::from_slice(&event_author) + .map_err(|_| DbError::InvalidData("active binding author key is invalid".to_owned()))?, + expires_at: row.try_get("expires_at").map_err(DbError::from)?, + }) +} + +fn bytes32(value: Vec, name: &str) -> crate::Result<[u8; 32]> { + value + .try_into() + .map_err(|_| DbError::InvalidData(format!("authorization {name} is malformed"))) +} + +fn database_u64(value: i64, name: &str) -> crate::Result { + u64::try_from(value) + .map_err(|_| DbError::InvalidData(format!("authorization {name} is negative"))) +} + +fn parsed_fence(value: Vec, name: &str) -> crate::Result { + AuthorizationLeaseFence::from_bytes(bytes32(value, name)?) + .map_err(|_| DbError::InvalidData(format!("authorization {name} is invalid"))) +} + +fn initial_publication_fence( + community_id: CommunityId, + object_kind: AuthorizationProtectedObjectKind, + object_key: [u8; 32], + lease_id: uuid::Uuid, +) -> crate::Result { + let mut digest = Sha256::new(); + for field in [ + b"buzz:protected-publication-initial-fence:v1".as_slice(), + community_id.as_uuid().as_bytes(), + &(object_kind as i16).to_be_bytes(), + object_key.as_slice(), + lease_id.as_bytes(), + ] { + digest.update((field.len() as u64).to_be_bytes()); + digest.update(field); + } + AuthorizationLeaseFence::from_bytes(digest.finalize().into()) + .map_err(|_| DbError::InvalidData("authorization initial fence is invalid".to_owned())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn concrete_resolver_is_redacted_and_declares_owner_bound_delegation() { + assert_eq!( + PostgresLocalBindingResolver { + db: lazy_debug_test_db() + } + .capability(), + LocalBindingResolverCapability::DirectAndDelegatedOwnerBound + ); + assert!(!format!( + "{:?}", + PostgresLocalBindingResolver { + db: lazy_debug_test_db() + } + ) + .contains("postgres")); + } + + fn lazy_debug_test_db() -> Db { + // The production crate forbids unsafe code. Build a lazy pool through + // the public configuration helper rather than fabricating a handle. + let pool = sqlx::postgres::PgPoolOptions::new() + .connect_lazy("postgresql://localhost/buzz_debug_test") + .expect("static URL is valid"); + Db::from_pool(pool) + } +} diff --git a/crates/buzz-db/src/authorization_restore.rs b/crates/buzz-db/src/authorization_restore.rs new file mode 100644 index 00000000000..64bdcb690b7 --- /dev/null +++ b/crates/buzz-db/src/authorization_restore.rs @@ -0,0 +1,5726 @@ +//! Operation-bound restore witness for protected authorization mutations. +//! +//! The external store contains fixed, immutable, bounded CAS shards of +//! owner-leased operation intents and one independently CAS-guarded record per +//! exact authority component. It never stores or compares a domain-global +//! version vector, and unrelated component advances never serialize on one +//! domain object. + +use std::{ + collections::{BTreeMap, HashMap}, + fmt, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; + +use crate::{ + authorization_version::{ + AuthorizationOperationFence, AuthorizationOperationFenceAcquireError, + AuthorizationOperationVersionDeltaManifest, AuthorizationVersionComponentFloor, + AuthorizationVersionComponentKind, ProtectedPublicationCommit, ProtectedPublicationError, + ProtectedPublicationRequest, ProtectedPublicationRestoreIdentity, + }, + Db, DbError, +}; +use async_trait::async_trait; +use buzz_core::CommunityId; +use dashmap::DashSet; +use s3::{creds::Credentials, error::S3Error, Bucket, Region}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use sqlx::PgPool; +use thiserror::Error; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; +use uuid::Uuid; + +const FORMAT_VERSION: u8 = 1; +const OPERATION_SHARD_FORMAT_VERSION: u8 = 1; +const OPERATION_SHARD_MAPPING_VERSION: u8 = 1; +const MAX_RECORD_BYTES: u64 = 64 * 1024; +const DEFAULT_LEASE: Duration = Duration::from_secs(30); +const MAX_LEASE: Duration = Duration::from_secs(300); +const MAX_CAS_ATTEMPTS: usize = 16; +const MAX_DOMAIN_FLOORS: usize = 100_000; +const PRODUCTION_OPERATION_SHARD_COUNT: u16 = 256; +const PRODUCTION_OPERATION_SHARD_CAPACITY: u16 = 64; +const PRODUCTION_TERMINAL_RETENTION: Duration = Duration::from_secs(24 * 60 * 60); +const MIN_TERMINAL_RETENTION: Duration = Duration::from_secs(5 * 60); +const MAX_TERMINAL_RETENTION: Duration = Duration::from_secs(30 * 24 * 60 * 60); +const MAX_OPERATION_SHARDS: u16 = 4_096; +const MAX_OPERATION_SHARD_CAPACITY: u16 = 128; +const CAS_RETRY_DELAY: Duration = Duration::from_millis(10); +const CAUSAL_LAG_BOUND: Duration = Duration::from_millis(150); +const OPERATION_FENCE_WAIT: Duration = Duration::from_millis(150); +const STORE_CAS_TIMEOUT: Duration = Duration::from_secs(5); +/// Fixed production capacity for detached primary sessions that may own an +/// operation fence. Changing this is a versioned production-layout change. +pub const PRODUCTION_OPERATION_FENCE_CONNECTIONS: u32 = 16; + +/// Immutable per-domain external operation-store policy. +#[derive(Clone, Copy, PartialEq, Eq)] +struct OperationRestoreRetentionPolicy { + shard_count: u16, + entries_per_shard: u16, + terminal_retention: Duration, +} + +impl OperationRestoreRetentionPolicy { + /// Construct a bounded policy that is sealed into the domain bootstrap. + #[cfg(test)] + fn new( + shard_count: u16, + entries_per_shard: u16, + terminal_retention: Duration, + ) -> Result { + let policy = Self { + shard_count, + entries_per_shard, + terminal_retention, + }; + policy.validate()?; + Ok(policy) + } + + fn production_default() -> Self { + Self { + shard_count: PRODUCTION_OPERATION_SHARD_COUNT, + entries_per_shard: PRODUCTION_OPERATION_SHARD_CAPACITY, + terminal_retention: PRODUCTION_TERMINAL_RETENTION, + } + } + + fn validate(self) -> Result<(), OperationRestoreError> { + if self.shard_count == 0 + || self.shard_count > MAX_OPERATION_SHARDS + || !self.shard_count.is_power_of_two() + || self.entries_per_shard == 0 + || self.entries_per_shard > MAX_OPERATION_SHARD_CAPACITY + || self.terminal_retention < MIN_TERMINAL_RETENTION + || self.terminal_retention > MAX_TERMINAL_RETENTION + { + return Err(OperationRestoreError::InvalidInput); + } + Ok(()) + } + + fn retention_millis(self) -> Result { + u64::try_from(self.terminal_retention.as_millis()) + .map_err(|_| OperationRestoreError::InvalidInput) + } + + fn layout_digest(self) -> [u8; 32] { + let mut digest = Sha256::new(); + digest.update(b"buzz:nip-fi-operation-restore-layout:v1"); + digest.update([OPERATION_SHARD_MAPPING_VERSION]); + digest.update(self.shard_count.to_be_bytes()); + digest.update(self.entries_per_shard.to_be_bytes()); + digest.update(self.retention_millis().unwrap_or(u64::MAX).to_be_bytes()); + digest.update(operation_shard_record_format_hash()); + digest.finalize().into() + } +} + +impl fmt::Debug for OperationRestoreRetentionPolicy { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OperationRestoreRetentionPolicy") + .field("shard_count", &self.shard_count) + .field("entries_per_shard", &self.entries_per_shard) + .field("terminal_retention", &"[REDACTED]") + .finish() + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +struct OperationIdentity { + domain: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], +} + +impl OperationIdentity { + fn validate(self) -> Result { + if self.domain.as_uuid().is_nil() + || self.operation_id.is_nil() + || self.request_fingerprint == [0; 32] + { + return Err(OperationRestoreError::InvalidInput); + } + Ok(self) + } +} + +impl fmt::Debug for OperationIdentity { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("OperationIdentity([REDACTED])") + } +} + +struct OperationFenceGuard { + capability: Option, + _permit: Option, + #[cfg(any(test, feature = "test-utils"))] + key: i64, + #[cfg(any(test, feature = "test-utils"))] + test_fences: Option>>, +} + +impl OperationFenceGuard { + fn capability_mut( + &mut self, + ) -> Result<&mut AuthorizationOperationFence, OperationRestoreError> { + self.capability + .as_mut() + .ok_or(OperationRestoreError::DatabaseUnavailable) + } + + async fn unlock(mut self) -> Result<(), OperationRestoreError> { + #[cfg(any(test, feature = "test-utils"))] + if let Some(fences) = self.test_fences.take() { + fences.remove(&self.key); + return Ok(()); + } + let Some(capability) = self.capability.take() else { + return Err(OperationRestoreError::DatabaseUnavailable); + }; + capability + .release(OPERATION_FENCE_WAIT) + .await + .map_err(|_| OperationRestoreError::DatabaseUnavailable) + } +} + +impl Drop for OperationFenceGuard { + fn drop(&mut self) { + #[cfg(any(test, feature = "test-utils"))] + if let Some(fences) = &self.test_fences { + fences.remove(&self.key); + } + // A detached PgConnection is deliberately dropped rather than ever + // returning a possibly session-locked connection to a pool. + let _ = self.capability.take(); + } +} + +impl fmt::Debug for OperationFenceGuard { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("OperationFenceGuard([REDACTED])") + } +} + +struct FencedRestoreIntent { + restore: Option, + fence: OperationFenceGuard, + replay: Option, +} + +impl FencedRestoreIntent { + fn capability_mut( + &mut self, + ) -> Result<&mut AuthorizationOperationFence, OperationRestoreError> { + self.fence.capability_mut() + } +} + +/// Fail-closed combined publication/database/restore result. +#[derive(Debug, Error)] +pub enum RestoredProtectedPublicationError { + /// The canonical publication transaction denied or failed. + #[error(transparent)] + Publication(#[from] ProtectedPublicationError), + /// The external operation witness failed. + #[error(transparent)] + Restore(#[from] OperationRestoreError), +} + +/// Crate-internal consumed owner capability. Production callers receive only +/// an operation-specific sealed wrapper. +struct OperationRestoreIntent { + identity: OperationIdentity, + owner_token: Uuid, + shard_id: u16, +} + +impl fmt::Debug for OperationRestoreIntent { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("OperationRestoreIntent([REDACTED])") + } +} + +/// Internal result of reserving an exact external operation intent. +enum OperationRestoreBegin { + /// The caller exclusively owns the bounded pending intent. + Acquired(OperationRestoreIntent), + /// The exact operation and manifest were already witnessed. + ExactReplay(OperationRestoreCommit), +} + +impl fmt::Debug for OperationRestoreBegin { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Acquired(_) => formatter.write_str("OperationRestoreBegin::Acquired([REDACTED])"), + Self::ExactReplay(_) => { + formatter.write_str("OperationRestoreBegin::ExactReplay([REDACTED])") + } + } + } +} + +/// Internal durable witness result for one exact PostgreSQL operation manifest. +#[derive(Clone, Copy, PartialEq, Eq)] +struct OperationRestoreCommit { + manifest_digest: [u8; 32], + replay: bool, + causally_superseded: bool, +} + +/// Sealed reconciliation result for one protected-publication operation. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct ProtectedPublicationRestoreReconciliation { + replay: bool, + causally_superseded: bool, +} + +impl ProtectedPublicationRestoreReconciliation { + /// Whether the exact external witness was already terminal. + pub const fn replay(&self) -> bool { + self.replay + } + + /// Whether a later, DB-proven operation owns the current component floor. + pub const fn causally_superseded(&self) -> bool { + self.causally_superseded + } +} + +impl fmt::Debug for ProtectedPublicationRestoreReconciliation { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ProtectedPublicationRestoreReconciliation") + .field("replay", &self.replay) + .field("causally_superseded", &self.causally_superseded) + .finish() + } +} + +impl From for ProtectedPublicationRestoreReconciliation { + fn from(commit: OperationRestoreCommit) -> Self { + Self { + replay: commit.replay, + causally_superseded: commit.causally_superseded, + } + } +} + +impl fmt::Debug for OperationRestoreCommit { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OperationRestoreCommit") + .field("manifest_digest", &"[REDACTED]") + .field("replay", &self.replay) + .field("causally_superseded", &self.causally_superseded) + .finish() + } +} + +#[derive(Clone)] +struct OperationManifest { + identity: OperationIdentity, + manifest_digest: [u8; 32], + components: Vec, +} + +#[derive(Clone)] +struct ManifestComponent { + kind: AuthorizationVersionComponentKind, + key: [u8; 32], + before: u64, + after: u64, + digest: [u8; 32], +} + +impl From for OperationManifest { + fn from(manifest: AuthorizationOperationVersionDeltaManifest) -> Self { + Self { + identity: OperationIdentity { + domain: manifest.community_id(), + operation_id: manifest.operation_id(), + request_fingerprint: manifest.request_fingerprint(), + }, + manifest_digest: manifest.manifest_digest(), + components: manifest + .components() + .iter() + .map(|component| ManifestComponent { + kind: component.component_kind(), + key: component.component_key(), + before: component.before_version(), + after: component.after_version(), + digest: component.component_digest(), + }) + .collect(), + } + } +} + +#[async_trait] +trait ManifestSource: Send + Sync { + async fn authoritative_now_ms(&self) -> Result; + + async fn manifest( + &self, + identity: OperationIdentity, + ) -> Result; + + async fn floors( + &self, + domain: CommunityId, + ) -> Result, ManifestReadError>; + + async fn predecessors( + &self, + manifest: &OperationManifest, + ) -> Result, ManifestReadError>; + + async fn prove_supersession( + &self, + pending: &OperationManifest, + component: &ManifestComponent, + floor: &FloorRecord, + ) -> Result<(), ManifestReadError>; +} + +struct PostgresManifestSource(Db); + +#[async_trait] +impl ManifestSource for PostgresManifestSource { + async fn authoritative_now_ms(&self) -> Result { + self.0 + .authorization_restore_clock_millis() + .await + .map_err(|_| ManifestReadError::Unavailable) + } + + async fn manifest( + &self, + identity: OperationIdentity, + ) -> Result { + self.0 + .authorization_operation_version_delta( + identity.domain, + identity.operation_id, + identity.request_fingerprint, + ) + .await + .map(OperationManifest::from) + .map_err(|error| match error { + DbError::NotFound(_) => ManifestReadError::Missing, + DbError::InvalidData(_) => ManifestReadError::Invalid, + _ => ManifestReadError::Unavailable, + }) + } + + async fn floors( + &self, + domain: CommunityId, + ) -> Result, ManifestReadError> { + self.0 + .authorization_version_component_floors(domain) + .await + .map_err(|_| ManifestReadError::Unavailable) + } + + async fn predecessors( + &self, + manifest: &OperationManifest, + ) -> Result, ManifestReadError> { + self.0 + .authorization_operation_version_predecessors( + manifest.identity.domain, + &manifest + .components + .iter() + .map(|component| (component.kind, component.key, component.before)) + .collect::>(), + ) + .await + .map(|manifests| manifests.into_iter().map(OperationManifest::from).collect()) + .map_err(|error| match error { + DbError::InvalidData(_) | DbError::NotFound(_) => ManifestReadError::Invalid, + _ => ManifestReadError::Unavailable, + }) + } + + async fn prove_supersession( + &self, + pending: &OperationManifest, + component: &ManifestComponent, + floor: &FloorRecord, + ) -> Result<(), ManifestReadError> { + let provenance = floor.provenance().map_err(|_| ManifestReadError::Invalid)?; + self.0 + .authorization_version_proves_component_supersession( + pending.identity.domain, + component.kind, + component.key, + component.after, + floor.version, + provenance.operation_id, + provenance.request_fingerprint, + provenance.component_digest, + provenance.manifest_digest, + ) + .await + .map_err(|error| match error { + DbError::InvalidData(_) | DbError::NotFound(_) => ManifestReadError::Invalid, + _ => ManifestReadError::Unavailable, + }) + } +} + +enum ManifestReadError { + Missing, + Invalid, + Unavailable, +} + +struct StoredObject { + etag: String, + body: Vec, +} + +enum StoreCas { + Won, + Lost, +} + +#[async_trait] +trait RestoreStore: Send + Sync { + async fn load(&self, key: &str) -> Result, ()>; + async fn list(&self, prefix: &str) -> Result, ()>; + async fn compare_and_swap( + &self, + key: &str, + expected_etag: Option<&str>, + body: &[u8], + ) -> Result; +} + +#[cfg(any(test, feature = "test-utils"))] +#[derive(Default)] +struct TestRestoreStore(std::sync::Mutex)>>); + +#[cfg(any(test, feature = "test-utils"))] +#[async_trait] +impl RestoreStore for TestRestoreStore { + async fn load(&self, key: &str) -> Result, ()> { + Ok(self + .0 + .lock() + .map_err(|_| ())? + .get(key) + .map(|(version, body)| StoredObject { + etag: version.to_string(), + body: body.clone(), + })) + } + + async fn list(&self, prefix: &str) -> Result, ()> { + Ok(self + .0 + .lock() + .map_err(|_| ())? + .keys() + .filter(|key| key.starts_with(prefix)) + .cloned() + .collect()) + } + + async fn compare_and_swap( + &self, + key: &str, + expected_etag: Option<&str>, + body: &[u8], + ) -> Result { + let mut records = self.0.lock().map_err(|_| ())?; + let current = records.get(key).map(|(version, _)| version.to_string()); + if current.as_deref() != expected_etag { + return Ok(StoreCas::Lost); + } + let next = records.get(key).map_or(1, |(version, _)| version + 1); + records.insert(key.to_owned(), (next, body.to_vec())); + Ok(StoreCas::Won) + } +} + +struct S3RestoreStore { + bucket: Arc, +} + +impl S3RestoreStore { + fn new(config: &buzz_media::MediaConfig) -> Result { + let region = Region::Custom { + region: config.s3_region.clone(), + endpoint: config.s3_endpoint.clone(), + }; + let credentials = match ( + config.s3_access_key.is_empty(), + config.s3_secret_key.is_empty(), + ) { + (false, false) => Credentials::new( + Some(&config.s3_access_key), + Some(&config.s3_secret_key), + None, + None, + None, + ), + (true, true) => Credentials::default(), + _ => return Err(()), + } + .map_err(|_| ())?; + let bucket = Bucket::new(&config.s3_bucket, region, credentials).map_err(|_| ())?; + let bucket = match config.s3_addressing_style { + buzz_media::config::S3AddressingStyle::Path => bucket.with_path_style(), + buzz_media::config::S3AddressingStyle::Virtual => bucket, + }; + Ok(Self { + bucket: Arc::from(bucket), + }) + } +} + +#[async_trait] +impl RestoreStore for S3RestoreStore { + async fn load(&self, key: &str) -> Result, ()> { + let (head, status) = match self.bucket.head_object(key).await { + Ok(result) => result, + Err(S3Error::HttpFailWithBody(404, _)) => return Ok(None), + Err(_) => return Err(()), + }; + if status == 404 { + return Ok(None); + } + if !(200..300).contains(&status) + || head + .content_length + .is_some_and(|length| u64::try_from(length).unwrap_or(u64::MAX) > MAX_RECORD_BYTES) + { + return Err(()); + } + match self.bucket.get_object(key).await { + Ok(response) => { + let headers = response.headers(); + let etag = headers + .get("etag") + .or_else(|| headers.get("ETag")) + .cloned() + .ok_or(())?; + let body = response.to_vec(); + if u64::try_from(body.len()).unwrap_or(u64::MAX) > MAX_RECORD_BYTES { + return Err(()); + } + Ok(Some(StoredObject { etag, body })) + } + Err(S3Error::HttpFailWithBody(404, _)) => Ok(None), + Err(_) => Err(()), + } + } + + async fn list(&self, prefix: &str) -> Result, ()> { + let pages = self + .bucket + .list(prefix.to_owned(), None) + .await + .map_err(|_| ())?; + let mut keys = Vec::new(); + for page in pages { + for object in page.contents { + if keys.len() >= MAX_DOMAIN_FLOORS { + return Err(()); + } + keys.push(object.key); + } + } + Ok(keys) + } + + async fn compare_and_swap( + &self, + key: &str, + expected_etag: Option<&str>, + body: &[u8], + ) -> Result { + if u64::try_from(body.len()).unwrap_or(u64::MAX) > MAX_RECORD_BYTES { + return Err(()); + } + let mut headers = http::HeaderMap::new(); + match expected_etag { + Some(etag) => { + headers.insert(http::header::IF_MATCH, etag.parse().map_err(|_| ())?); + } + None => { + headers.insert(http::header::IF_NONE_MATCH, "*".parse().map_err(|_| ())?); + } + } + match self + .bucket + .put_object_with_content_type_and_headers(key, body, "application/json", Some(headers)) + .await + { + Ok(response) if (200..300).contains(&response.status_code()) => Ok(StoreCas::Won), + Err(S3Error::HttpFailWithBody(412, _)) => Ok(StoreCas::Lost), + _ => Err(()), + } + } +} + +struct RestoreInner { + writer_db: Db, + source: Arc, + store: Arc, + bootstrap_ids: Arc>, + ready_domains: Arc>, + lease: Duration, + retention: OperationRestoreRetentionPolicy, + fence_pool: PgPool, + fence_slots: Arc, + #[cfg(any(test, feature = "test-utils"))] + test_fences: Option>>, + healthy: AtomicBool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ComponentReconciliation { + Exact, + Applied, + CausallySuperseded, +} + +/// Cloneable operation-bound restore witness runtime. +/// +/// Raw owner intents are intentionally not part of the public surface: +/// +/// ```compile_fail +/// use buzz_db::authorization_restore::{OperationRestoreIntent, OperationRestoreCommit}; +/// ``` +/// +/// The lock-owning session capability is likewise crate-private: +/// +/// ```compile_fail +/// use buzz_db::authorization_version::AuthorizationOperationFence; +/// ``` +#[derive(Clone)] +pub struct OperationRestoreRuntime(Arc); + +impl OperationRestoreRuntime { + /// Construct with the canonical writer, a separately created witness-read + /// handle, and the external CAS store. + pub fn new( + writer_db: Db, + witness_read_db: Db, + fence_pool: PgPool, + store_configuration: buzz_media::MediaConfig, + domains: impl IntoIterator, + lease: Duration, + ) -> Result { + Self::new_with_retention_policy( + writer_db, + witness_read_db, + fence_pool, + store_configuration, + domains, + lease, + OperationRestoreRetentionPolicy::production_default(), + ) + } + + fn new_with_retention_policy( + writer_db: Db, + witness_read_db: Db, + fence_pool: PgPool, + store_configuration: buzz_media::MediaConfig, + domains: impl IntoIterator, + lease: Duration, + retention: OperationRestoreRetentionPolicy, + ) -> Result { + if lease.is_zero() || lease > MAX_LEASE { + return Err(OperationRestoreError::InvalidInput); + } + retention.validate()?; + let mut bootstrap_ids = HashMap::new(); + for (domain, bootstrap) in domains { + if bootstrap_ids.insert(domain, bootstrap).is_some() { + return Err(OperationRestoreError::InvalidBootstrap); + } + } + if bootstrap_ids.is_empty() + || bootstrap_ids + .iter() + .any(|(domain, bootstrap)| domain.as_uuid().is_nil() || bootstrap.is_nil()) + { + return Err(OperationRestoreError::InvalidBootstrap); + } + let store = S3RestoreStore::new(&store_configuration) + .map_err(|_| OperationRestoreError::InvalidInput)?; + Ok(Self(Arc::new(RestoreInner { + writer_db, + source: Arc::new(PostgresManifestSource(witness_read_db)), + store: Arc::new(store), + bootstrap_ids: Arc::new(bootstrap_ids), + ready_domains: Arc::new(DashSet::new()), + lease, + retention, + fence_pool, + fence_slots: Arc::new(Semaphore::new( + PRODUCTION_OPERATION_FENCE_CONNECTIONS as usize, + )), + #[cfg(any(test, feature = "test-utils"))] + test_fences: None, + healthy: AtomicBool::new(true), + }))) + } + + /// Construct with the default bounded owner lease. + pub fn with_default_lease( + writer_db: Db, + witness_read_db: Db, + fence_pool: PgPool, + store_configuration: buzz_media::MediaConfig, + domains: impl IntoIterator, + ) -> Result { + Self::new( + writer_db, + witness_read_db, + fence_pool, + store_configuration, + domains, + DEFAULT_LEASE, + ) + } + + #[cfg(any(test, feature = "test-utils"))] + #[doc(hidden)] + pub fn for_tests( + witness_read_db: Db, + domains: impl IntoIterator, + ) -> Self { + let bootstrap_ids: HashMap<_, _> = domains.into_iter().collect(); + let retention = OperationRestoreRetentionPolicy::production_default(); + let store = TestRestoreStore::default(); + { + let mut records = store.0.lock().expect("test restore store"); + for (domain, bootstrap_id) in &bootstrap_ids { + for shard_id in 0..retention.shard_count { + records.insert( + operation_shard_key(*domain, shard_id), + ( + 1, + serde_json::to_vec(&OperationShardRecord::empty( + *domain, + *bootstrap_id, + shard_id, + retention, + )) + .expect("encode test restore shard"), + ), + ); + } + } + } + Self(Arc::new(RestoreInner { + writer_db: witness_read_db.clone(), + source: Arc::new(PostgresManifestSource(witness_read_db)), + store: Arc::new(store), + bootstrap_ids: Arc::new(bootstrap_ids), + ready_domains: Arc::new(DashSet::new()), + lease: DEFAULT_LEASE, + retention, + fence_pool: sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect_lazy("postgresql://127.0.0.1:1/unreachable") + .expect("parse test fence database URL"), + fence_slots: Arc::new(Semaphore::new(1)), + test_fences: Some(Arc::new(DashSet::new())), + healthy: AtomicBool::new(true), + })) + } + + #[cfg(any(test, feature = "test-utils"))] + #[doc(hidden)] + pub fn force_unhealthy_for_test(&self) { + self.latch_unhealthy(); + } + + #[cfg(any(test, feature = "test-utils"))] + #[doc(hidden)] + pub fn mark_domain_ready_for_test(&self, domain: CommunityId) { + self.0.ready_domains.insert(domain); + } + + /// Explicitly provision a first immutable bootstrap and exact component baselines. + /// Normal production startup calls [`Self::verify_domain`] and never calls this method. + pub async fn provision_domain(&self, domain: CommunityId) -> Result<(), OperationRestoreError> { + self.require_healthy()?; + let bootstrap_id = self.bootstrap_id(domain)?; + let floors = self.read_database_floors(domain).await?; + for floor in floors { + self.provision_floor(domain, floor).await?; + } + self.validate_worst_case_shard_size(domain, bootstrap_id)?; + for shard_id in 0..self.0.retention.shard_count { + self.provision_operation_shard(domain, bootstrap_id, shard_id) + .await?; + } + let marker_key = bootstrap_key(domain); + let marker = BootstrapRecord::new(domain, bootstrap_id, self.0.retention); + match self.store_record(&marker_key, None, &marker).await? { + StoreCas::Won => {} + StoreCas::Lost => { + let Some((_, existing)) = self.load_bootstrap(&marker_key).await? else { + self.latch_unhealthy(); + return Err(OperationRestoreError::StoreUnavailable); + }; + self.validate_bootstrap(&existing, domain, bootstrap_id)?; + } + } + self.0.ready_domains.insert(domain); + Ok(()) + } + + /// Verify a previously provisioned bootstrap. Missing state is never synthesized. + pub async fn verify_domain(&self, domain: CommunityId) -> Result<(), OperationRestoreError> { + self.require_healthy()?; + let bootstrap_id = self.bootstrap_id(domain)?; + let marker_key = bootstrap_key(domain); + let Some((_, marker)) = self.load_bootstrap(&marker_key).await? else { + self.latch_unhealthy(); + return Err(OperationRestoreError::MissingBootstrap); + }; + self.validate_bootstrap(&marker, domain, bootstrap_id)?; + self.validate_worst_case_shard_size(domain, bootstrap_id)?; + for shard_id in 0..self.0.retention.shard_count { + let key = operation_shard_key(domain, shard_id); + let Some((_, shard)) = self.load_operation_shard(&key).await? else { + self.latch_unhealthy(); + return Err(OperationRestoreError::MissingOperationShard); + }; + self.validate_operation_shard(&shard, domain, bootstrap_id, shard_id)?; + } + // A process may die after PostgreSQL commits its exact manifest but + // before the external floors and operation record are advanced. The + // operation namespace is therefore reconciled before comparing the + // external component floors with the current database snapshot. + self.reconcile_startup_operations(domain).await?; + let mut database_floors = self + .read_database_floors(domain) + .await? + .into_iter() + .map(|floor| ((floor.component_kind, floor.component_key), floor.version)) + .collect::>(); + let prefix = floor_prefix(domain); + let floor_keys = self.0.store.list(&prefix).await.map_err(|_| { + self.latch_unhealthy(); + OperationRestoreError::StoreUnavailable + })?; + if floor_keys.len() > MAX_DOMAIN_FLOORS { + self.latch_unhealthy(); + return Err(OperationRestoreError::StoreUnavailable); + } + for stored_key in floor_keys { + let Some((_, existing)) = self.load_floor(&stored_key).await? else { + self.latch_unhealthy(); + return Err(OperationRestoreError::MissingFloor); + }; + let kind = component_kind_from_i16(existing.component_kind)?; + let component_key = decode32(&existing.component_key)?; + if stored_key != floor_key(domain, kind, component_key) { + self.latch_unhealthy(); + return Err(OperationRestoreError::WrongComponentKey); + } + if let Err(error) = + existing.validate_coordinate(domain, bootstrap_id, kind, component_key) + { + self.latch_unhealthy(); + return Err(error); + } + let database_version = database_floors.remove(&(kind, component_key)).unwrap_or(0); + if existing.version != database_version { + self.latch_unhealthy(); + return Err(if existing.version > database_version { + OperationRestoreError::BackwardFloor + } else { + OperationRestoreError::StaleFloor + }); + } + } + if !database_floors.is_empty() { + self.latch_unhealthy(); + return Err(OperationRestoreError::MissingFloor); + } + self.0.ready_domains.insert(domain); + Ok(()) + } + + async fn reconcile_startup_operations( + &self, + domain: CommunityId, + ) -> Result<(), OperationRestoreError> { + let bootstrap_id = self.bootstrap_id(domain)?; + loop { + let now = self.authoritative_now_ms().await?; + let mut saw_pending = false; + let mut saw_live_pending = false; + let mut progressed = false; + let mut retryable = None; + + // Scan fixed shard IDs directly. At most one detached primary + // session is retained at a time, so startup can reconcile more + // operations than the bounded fence-pool size without + // self-saturating that pool. + for shard_id in 0..self.0.retention.shard_count { + let key = operation_shard_key(domain, shard_id); + let shard = self + .compact_collectible_terminals(domain, bootstrap_id, shard_id, &key, now) + .await?; + for (operation, record) in shard.entries { + if record.state != IntentState::Pending { + continue; + } + saw_pending = true; + let identity = + self.identity_from_shard_entry(domain, shard_id, &operation, &record)?; + let fence = match self.acquire_operation_fence(identity).await { + Ok(fence) => fence, + Err(error) if error.is_healthy_retryable() => { + retryable.get_or_insert(error); + continue; + } + Err(error) => return Err(error), + }; + match self.0.source.manifest(identity).await { + Ok(manifest) if manifest.identity == identity => { + if self.manifest_ready_for_startup(&manifest).await? { + let result = self + .reconcile_pending_manifest(shard_id, &record, manifest) + .await; + match result { + Ok(_) => fence + .unlock() + .await + .inspect_err(|_| self.latch_unhealthy())?, + Err(error) => { + drop(fence); + return Err(error); + } + } + progressed = true; + } else { + drop(fence); + } + } + Ok(_) | Err(ManifestReadError::Invalid) => { + drop(fence); + self.latch_unhealthy(); + return Err(OperationRestoreError::InvalidAttribution); + } + Err(ManifestReadError::Missing) if now >= record.expires_at_ms => { + let intent = OperationRestoreIntent { + identity, + owner_token: record.owner_token, + shard_id, + }; + let result = self.abort_owned_intent(&intent).await; + match result { + Ok(()) => fence + .unlock() + .await + .inspect_err(|_| self.latch_unhealthy())?, + Err(error) => { + drop(fence); + return Err(error); + } + } + progressed = true; + } + Err(ManifestReadError::Missing) => { + saw_live_pending = true; + drop(fence); + } + Err(ManifestReadError::Unavailable) => { + drop(fence); + self.latch_unhealthy(); + return Err(OperationRestoreError::DatabaseUnavailable); + } + } + } + } + + if !saw_pending { + return Ok(()); + } + if progressed { + continue; + } + if let Some(error) = retryable { + return Err(error); + } + if saw_live_pending { + return Err(OperationRestoreError::OperationInProgress); + } + self.latch_unhealthy(); + return Err(OperationRestoreError::StaleFloor); + } + } + + async fn compact_collectible_terminals( + &self, + domain: CommunityId, + bootstrap_id: Uuid, + shard_id: u16, + key: &str, + now: u64, + ) -> Result { + for _ in 0..MAX_CAS_ATTEMPTS { + let Some((etag, shard)) = self.load_operation_shard(key).await? else { + self.latch_unhealthy(); + return Err(OperationRestoreError::MissingOperationShard); + }; + self.validate_operation_shard(&shard, domain, bootstrap_id, shard_id)?; + let mut next = shard.clone(); + next.entries.retain(|_, record| !record.is_collectible(now)); + if next.entries.len() == shard.entries.len() { + return Ok(shard); + } + if matches!( + self.store_record(key, Some(&etag), &next).await?, + StoreCas::Won + ) { + return Ok(next); + } + } + Err(OperationRestoreError::OperationContention) + } + + async fn manifest_ready_for_startup( + &self, + manifest: &OperationManifest, + ) -> Result { + let bootstrap_id = self.bootstrap_id(manifest.identity.domain)?; + for component in &manifest.components { + if component.key == [0; 32] + || component.after <= component.before + || component.after > i64::MAX as u64 + { + self.latch_unhealthy(); + return Err(OperationRestoreError::InvalidAttribution); + } + let key = floor_key(manifest.identity.domain, component.kind, component.key); + let Some((_, floor)) = self.load_floor(&key).await? else { + if component.before == 0 { + continue; + } + return Ok(false); + }; + if let Err(error) = floor.validate_coordinate( + manifest.identity.domain, + bootstrap_id, + component.kind, + component.key, + ) { + self.latch_unhealthy(); + return Err(error); + } + if floor.exact_advance(manifest, component)? || floor.version == component.before { + continue; + } + if floor.version < component.before { + return Ok(false); + } + if floor.version > component.after { + self.prove_supersession(manifest, component, &floor).await?; + continue; + } + self.latch_unhealthy(); + return Err(OperationRestoreError::InvalidAttribution); + } + Ok(true) + } + + async fn acquire_operation_fence( + &self, + identity: OperationIdentity, + ) -> Result { + self.acquire_operation_fence_with_timeout(identity, OPERATION_FENCE_WAIT) + .await + } + + async fn acquire_operation_fence_with_timeout( + &self, + identity: OperationIdentity, + timeout: Duration, + ) -> Result { + if timeout.is_zero() { + return Err(OperationRestoreError::OperationFenceTimeout); + } + #[cfg(any(test, feature = "test-utils"))] + if let Some(fences) = &self.0.test_fences { + let key = operation_fence_test_key(identity); + if !fences.insert(key) { + // Model a bounded PostgreSQL lock probe so tests prove the + // shard-wide deadline, not merely an immediate in-memory + // collision fast path. + tokio::time::sleep(Duration::from_millis(20).min(timeout)).await; + return Err(OperationRestoreError::OperationInProgress); + } + return Ok(OperationFenceGuard { + key, + capability: None, + _permit: None, + test_fences: Some(fences.clone()), + }); + } + + let permit = self + .0 + .fence_slots + .clone() + .try_acquire_owned() + .map_err(|_| OperationRestoreError::OperationFenceSaturated)?; + let capability = match AuthorizationOperationFence::acquire_for_operation_restore( + &self.0.fence_pool, + identity.domain, + identity.operation_id, + identity.request_fingerprint, + timeout, + ) + .await + { + Ok(Some(capability)) => capability, + Ok(None) | Err(AuthorizationOperationFenceAcquireError::LockTimedOut) => { + return Err(OperationRestoreError::OperationFenceTimeout) + } + Err(AuthorizationOperationFenceAcquireError::PoolSaturated) => { + return Err(OperationRestoreError::OperationFenceSaturated) + } + Err(AuthorizationOperationFenceAcquireError::Database(_)) => { + self.latch_unhealthy(); + return Err(OperationRestoreError::DatabaseUnavailable); + } + }; + Ok(OperationFenceGuard { + capability: Some(capability), + _permit: Some(permit), + #[cfg(any(test, feature = "test-utils"))] + key: operation_fence_test_key(identity), + #[cfg(any(test, feature = "test-utils"))] + test_fences: None, + }) + } + + /// Test/internal reservation primitive. Production callers use a sealed + /// operation-specific wrapper that acquires the PostgreSQL session fence. + #[cfg(test)] + async fn begin( + &self, + domain: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + ) -> Result { + self.require_healthy()?; + let identity = OperationIdentity { + domain, + operation_id, + request_fingerprint, + } + .validate()?; + self.require_domain_ready(domain)?; + let bootstrap_id = self.bootstrap_id(domain)?; + let shard_id = operation_shard_id(identity, bootstrap_id, self.0.retention.shard_count); + self.maintain_begin_shard(domain, shard_id).await?; + self.reserve_identity(identity, shard_id).await + } + + async fn begin_fenced( + &self, + identity: OperationIdentity, + ) -> Result { + self.require_healthy()?; + let identity = identity.validate()?; + self.require_domain_ready(identity.domain)?; + let bootstrap_id = self.bootstrap_id(identity.domain)?; + let shard_id = operation_shard_id(identity, bootstrap_id, self.0.retention.shard_count); + self.maintain_begin_shard(identity.domain, shard_id).await?; + let fence = self.acquire_operation_fence(identity).await?; + match self.reserve_identity(identity, shard_id).await { + Ok(OperationRestoreBegin::Acquired(intent)) => Ok(FencedRestoreIntent { + restore: Some(intent), + fence, + replay: None, + }), + Ok(OperationRestoreBegin::ExactReplay(replay)) => Ok(FencedRestoreIntent { + restore: None, + fence, + replay: Some(replay), + }), + Err(error) => { + drop(fence); + Err(error) + } + } + } + + async fn reserve_identity( + &self, + identity: OperationIdentity, + shard_id: u16, + ) -> Result { + let domain = identity.domain; + let operation_id = identity.operation_id; + let bootstrap_id = self.bootstrap_id(domain)?; + let key = operation_shard_key(domain, shard_id); + for _ in 0..MAX_CAS_ATTEMPTS { + let Some((etag, mut shard)) = self.load_operation_shard(&key).await? else { + self.latch_unhealthy(); + return Err(OperationRestoreError::MissingOperationShard); + }; + self.validate_operation_shard(&shard, domain, bootstrap_id, shard_id)?; + let operation_key = operation_id.to_string(); + if let Some(record) = shard.entries.get(&operation_key) { + self.validate_intent_record(record, identity)?; + return match record.state { + IntentState::Committed => Ok(OperationRestoreBegin::ExactReplay( + self.validate_committed(identity, record).await?, + )), + IntentState::Aborted => Err(OperationRestoreError::OperationAborted), + IntentState::Pending => Err(OperationRestoreError::OperationInProgress), + }; + } + if shard.entries.len() >= usize::from(self.0.retention.entries_per_shard) { + return Err(OperationRestoreError::OperationCapacityExceeded); + } + let now = self.authoritative_now_ms().await?; + let owner_token = Uuid::new_v4(); + shard.entries.insert( + operation_key, + IntentRecord::pending( + identity, + owner_token, + bootstrap_id, + now, + lease_expiry(now, self.0.lease)?, + ), + ); + if matches!( + self.store_record(&key, Some(&etag), &shard).await?, + StoreCas::Won + ) { + return Ok(OperationRestoreBegin::Acquired(OperationRestoreIntent { + identity, + owner_token, + shard_id, + })); + } + } + Err(OperationRestoreError::OperationContention) + } + + async fn begin_protected_publication( + &self, + identity: ProtectedPublicationRestoreIdentity, + ) -> Result { + self.begin_fenced(OperationIdentity { + domain: identity.authorization_domain(), + operation_id: identity.operation_id(), + request_fingerprint: identity.request_fingerprint(), + }) + .await + } + + /// Execute one sealed staged publication on its lock-owning primary + /// session. The session is never returned to a pool. + pub async fn commit_protected_publication( + &self, + request: &ProtectedPublicationRequest, + ) -> Result { + let mut fenced = self + .begin_protected_publication(request.restore_identity()) + .await?; + let result = self + .0 + .writer_db + .commit_staged_protected_publication_fenced(fenced.capability_mut()?, request) + .await; + match result { + Ok(commit) => { + self.commit_fenced(fenced).await?; + Ok(commit) + } + Err( + error @ (ProtectedPublicationError::Conflict + | ProtectedPublicationError::StaleAuthorization + | ProtectedPublicationError::AuditUnavailable), + ) => { + self.abort_fenced(fenced).await?; + Err(error.into()) + } + Err(error @ ProtectedPublicationError::Database(_)) => { + self.abandon_ambiguous(fenced); + Err(error.into()) + } + } + } + + async fn commit_fenced( + &self, + mut fenced: FencedRestoreIntent, + ) -> Result<(), OperationRestoreError> { + if let Some(intent) = fenced.restore.take() { + self.commit(intent).await? + } else { + fenced + .replay + .take() + .ok_or(OperationRestoreError::InvalidAttribution)? + }; + if let Err(error) = fenced.fence.unlock().await { + self.latch_unhealthy(); + return Err(error); + } + Ok(()) + } + + async fn abort_fenced( + &self, + mut fenced: FencedRestoreIntent, + ) -> Result<(), OperationRestoreError> { + if let Some(intent) = fenced.restore.take() { + self.abort(intent).await?; + } + if let Err(error) = fenced.fence.unlock().await { + self.latch_unhealthy(); + return Err(error); + } + Ok(()) + } + + fn abandon_ambiguous(&self, fenced: FencedRestoreIntent) { + self.latch_unhealthy(); + drop(fenced); + } + + /// Resolve the exact manifest through the independent read handle and advance its floors. + async fn commit( + &self, + intent: OperationRestoreIntent, + ) -> Result { + self.require_healthy()?; + let record = self.load_owned_intent(&intent).await?; + if record.state == IntentState::Committed { + return self.validate_committed(intent.identity, &record).await; + } + match self.0.source.manifest(intent.identity).await { + Ok(manifest) if manifest.identity == intent.identity => { + self.reconcile_pending_manifest(intent.shard_id, &record, manifest) + .await + } + Ok(_) | Err(ManifestReadError::Invalid) => { + self.latch_unhealthy(); + Err(OperationRestoreError::InvalidAttribution) + } + Err(ManifestReadError::Missing) => { + self.abort_owned_intent(&intent).await?; + self.latch_unhealthy(); + Err(OperationRestoreError::MissingAttribution) + } + Err(ManifestReadError::Unavailable) => { + self.latch_unhealthy(); + Err(OperationRestoreError::DatabaseUnavailable) + } + } + } + + /// Consume an uncommitted owner lease on a normal caller cancellation/error path. + async fn abort(&self, intent: OperationRestoreIntent) -> Result<(), OperationRestoreError> { + self.require_healthy()?; + self.abort_owned_intent(&intent).await + } + + /// Reconcile a lost response/crash from the exact receipt-bound PostgreSQL manifest only. + async fn reconcile( + &self, + domain: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + ) -> Result { + self.require_healthy()?; + let identity = OperationIdentity { + domain, + operation_id, + request_fingerprint, + } + .validate()?; + self.require_domain_ready(domain)?; + let bootstrap_id = self.bootstrap_id(domain)?; + let shard_id = operation_shard_id(identity, bootstrap_id, self.0.retention.shard_count); + let key = operation_shard_key(domain, shard_id); + let Some((_, shard)) = self.load_operation_shard(&key).await? else { + self.latch_unhealthy(); + return Err(OperationRestoreError::MissingOperationShard); + }; + self.validate_operation_shard(&shard, domain, bootstrap_id, shard_id)?; + let Some(record) = shard.entries.get(&operation_id.to_string()) else { + return Err(OperationRestoreError::MissingIntent); + }; + self.validate_intent_record(record, identity)?; + match record.state { + IntentState::Committed => self.validate_committed(identity, record).await, + IntentState::Aborted => Err(OperationRestoreError::OperationAborted), + IntentState::Pending => match self.0.source.manifest(identity).await { + Ok(manifest) if manifest.identity == identity => { + self.reconcile_pending_manifest(shard_id, record, manifest) + .await + } + Ok(_) | Err(ManifestReadError::Invalid) => { + self.latch_unhealthy(); + Err(OperationRestoreError::InvalidAttribution) + } + Err(ManifestReadError::Missing) => Err(OperationRestoreError::MissingAttribution), + Err(ManifestReadError::Unavailable) => { + self.latch_unhealthy(); + Err(OperationRestoreError::DatabaseUnavailable) + } + }, + } + } + + /// Reconcile the exact sealed protected publication request after an + /// ambiguous database outcome. + pub async fn reconcile_protected_publication( + &self, + identity: ProtectedPublicationRestoreIdentity, + ) -> Result { + let operation_identity = OperationIdentity { + domain: identity.authorization_domain(), + operation_id: identity.operation_id(), + request_fingerprint: identity.request_fingerprint(), + } + .validate()?; + let fence = self.acquire_operation_fence(operation_identity).await?; + let result = self + .reconcile( + identity.authorization_domain(), + identity.operation_id(), + identity.request_fingerprint(), + ) + .await; + match result { + Ok(committed) => { + fence + .unlock() + .await + .inspect_err(|_| self.latch_unhealthy())?; + Ok(committed.into()) + } + Err( + error @ (OperationRestoreError::MissingAttribution + | OperationRestoreError::MissingIntent + | OperationRestoreError::OperationAborted), + ) => { + fence + .unlock() + .await + .inspect_err(|_| self.latch_unhealthy())?; + Err(error) + } + Err(error) => { + drop(fence); + Err(error) + } + } + } + + /// Sticky storage/attribution health. + pub fn is_healthy(&self) -> bool { + self.0.healthy.load(Ordering::Acquire) + } + + fn owner_mismatch(&self) -> Result { + self.latch_unhealthy(); + Err(OperationRestoreError::OwnerMismatch) + } + + async fn reconcile_pending_manifest( + &self, + shard_id: u16, + operation_record: &IntentRecord, + manifest: OperationManifest, + ) -> Result { + let identity = manifest.identity; + self.validate_intent_record(operation_record, identity)?; + let mut causally_superseded = false; + for component in &manifest.components { + causally_superseded |= matches!( + self.apply_component(&manifest, component).await?, + ComponentReconciliation::CausallySuperseded + ); + } + let terminal_at = self.authoritative_now_ms().await?; + let retain_until = self.terminal_retain_until(terminal_at)?; + let key = operation_shard_key(identity.domain, shard_id); + let bootstrap_id = self.bootstrap_id(identity.domain)?; + for _ in 0..MAX_CAS_ATTEMPTS { + let Some((etag, mut shard)) = self.load_operation_shard(&key).await? else { + self.latch_unhealthy(); + return Err(OperationRestoreError::MissingOperationShard); + }; + self.validate_operation_shard(&shard, identity.domain, bootstrap_id, shard_id)?; + let Some(current) = shard.entries.get(&identity.operation_id.to_string()) else { + self.latch_unhealthy(); + return Err(OperationRestoreError::MissingIntent); + }; + self.validate_intent_record(current, identity)?; + match current.state { + IntentState::Committed => return self.validate_committed(identity, current).await, + IntentState::Aborted => return Err(OperationRestoreError::OperationAborted), + IntentState::Pending if current.owner_token != operation_record.owner_token => { + return self.owner_mismatch() + } + IntentState::Pending => {} + } + shard.entries.insert( + identity.operation_id.to_string(), + current.committed( + manifest.manifest_digest, + terminal_at, + retain_until, + causally_superseded, + ), + ); + if matches!( + self.store_record(&key, Some(&etag), &shard).await?, + StoreCas::Won + ) { + return Ok(OperationRestoreCommit { + manifest_digest: manifest.manifest_digest, + replay: false, + causally_superseded, + }); + } + } + Err(OperationRestoreError::OperationContention) + } + + async fn validate_committed( + &self, + identity: OperationIdentity, + record: &IntentRecord, + ) -> Result { + let recorded_digest = decode32( + record + .manifest_digest + .as_deref() + .ok_or(OperationRestoreError::InvalidAttribution)?, + )?; + let manifest = match self.0.source.manifest(identity).await { + Ok(manifest) => manifest, + Err(ManifestReadError::Missing) => { + self.latch_unhealthy(); + return Err(OperationRestoreError::MissingAttribution); + } + Err(ManifestReadError::Invalid) => { + self.latch_unhealthy(); + return Err(OperationRestoreError::InvalidAttribution); + } + Err(ManifestReadError::Unavailable) => { + self.latch_unhealthy(); + return Err(OperationRestoreError::DatabaseUnavailable); + } + }; + if manifest.identity != identity || manifest.manifest_digest != recorded_digest { + self.latch_unhealthy(); + return Err(OperationRestoreError::InvalidAttribution); + } + let bootstrap_id = self.bootstrap_id(identity.domain)?; + let mut causally_superseded = false; + for component in &manifest.components { + let key = floor_key(identity.domain, component.kind, component.key); + let Some((_, floor)) = self.load_floor(&key).await? else { + self.latch_unhealthy(); + return Err(OperationRestoreError::MissingFloor); + }; + if let Err(error) = floor.validate_coordinate( + identity.domain, + bootstrap_id, + component.kind, + component.key, + ) { + self.latch_unhealthy(); + return Err(error); + } + if floor.version < component.after { + self.latch_unhealthy(); + return Err(OperationRestoreError::BackwardFloor); + } + if floor.version == component.after { + if !floor.exact_advance(&manifest, component)? { + self.latch_unhealthy(); + return Err(OperationRestoreError::InvalidAttribution); + } + } else { + self.prove_supersession(&manifest, component, &floor) + .await?; + causally_superseded = true; + } + } + if record.causally_superseded && !causally_superseded { + self.latch_unhealthy(); + return Err(OperationRestoreError::InvalidAttribution); + } + Ok(OperationRestoreCommit { + manifest_digest: recorded_digest, + replay: true, + causally_superseded, + }) + } + + async fn maintain_begin_shard( + &self, + domain: CommunityId, + shard_id: u16, + ) -> Result<(), OperationRestoreError> { + let deadline = tokio::time::Instant::now() + CAUSAL_LAG_BOUND; + match tokio::time::timeout( + CAUSAL_LAG_BOUND, + self.maintain_begin_shard_until(domain, shard_id, deadline), + ) + .await + { + Ok(result) => result, + Err(_) => Err(OperationRestoreError::CausalLag), + } + } + + async fn maintain_begin_shard_until( + &self, + domain: CommunityId, + shard_id: u16, + deadline: tokio::time::Instant, + ) -> Result<(), OperationRestoreError> { + let bootstrap_id = self.bootstrap_id(domain)?; + let key = operation_shard_key(domain, shard_id); + let max_passes = usize::from(self.0.retention.entries_per_shard).saturating_mul(2); + for _ in 0..max_passes { + if tokio::time::Instant::now() >= deadline { + return Err(OperationRestoreError::CausalLag); + } + let now = self.authoritative_now_ms().await?; + let shard = self + .compact_collectible_terminals(domain, bootstrap_id, shard_id, &key, now) + .await?; + let mut causal_lag = false; + for (operation, record) in &shard.entries { + if record.state != IntentState::Pending || now < record.expires_at_ms { + continue; + } + let identity = + self.identity_from_shard_entry(domain, shard_id, operation, record)?; + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + if remaining.is_zero() { + return Err(OperationRestoreError::CausalLag); + } + let fence = match self + .acquire_operation_fence_with_timeout(identity, remaining) + .await + { + Ok(fence) => fence, + Err( + OperationRestoreError::OperationInProgress + | OperationRestoreError::OperationFenceTimeout + | OperationRestoreError::OperationFenceSaturated, + ) => { + causal_lag = true; + continue; + } + Err(error) => return Err(error), + }; + match self.0.source.manifest(identity).await { + Ok(manifest) if manifest.identity == identity => { + if !self.manifest_ready_for_startup(&manifest).await? { + drop(fence); + let _progressed = self + .resolve_manifest_predecessors(&manifest, deadline) + .await?; + // A predecessor transition always requires a + // reload of this shard before it can be declared + // ready; an unresolved predecessor remains + // bounded causal lag. + causal_lag = true; + continue; + } + let result = self + .reconcile_pending_manifest(shard_id, record, manifest) + .await; + match result { + Ok(_) => fence + .unlock() + .await + .inspect_err(|_| self.latch_unhealthy())?, + Err(error) => { + drop(fence); + return Err(error); + } + } + // Reload after every terminal transition; no stale + // eligibility or ETag is carried across a CAS. + continue; + } + Ok(_) | Err(ManifestReadError::Invalid) => { + drop(fence); + self.latch_unhealthy(); + return Err(OperationRestoreError::InvalidAttribution); + } + Err(ManifestReadError::Missing) => { + let intent = OperationRestoreIntent { + identity, + owner_token: record.owner_token, + shard_id, + }; + let result = self.abort_owned_intent(&intent).await; + match result { + Ok(()) => fence + .unlock() + .await + .inspect_err(|_| self.latch_unhealthy())?, + Err(error) => { + drop(fence); + return Err(error); + } + } + continue; + } + Err(ManifestReadError::Unavailable) => { + drop(fence); + self.latch_unhealthy(); + return Err(OperationRestoreError::DatabaseUnavailable); + } + } + } + if !causal_lag { + return Ok(()); + } + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + if remaining.is_zero() { + return Err(OperationRestoreError::CausalLag); + } + tokio::time::sleep(CAS_RETRY_DELAY.min(remaining)).await; + } + Err(OperationRestoreError::OperationContention) + } + + async fn resolve_manifest_predecessors( + &self, + successor: &OperationManifest, + deadline: tokio::time::Instant, + ) -> Result { + let predecessors = self + .0 + .source + .predecessors(successor) + .await + .map_err(|error| { + self.latch_unhealthy(); + match error { + ManifestReadError::Missing | ManifestReadError::Invalid => { + OperationRestoreError::InvalidAttribution + } + ManifestReadError::Unavailable => OperationRestoreError::DatabaseUnavailable, + } + })?; + let mut pending = BTreeMap::new(); + let mut discovered = std::collections::BTreeSet::new(); + for predecessor in predecessors { + let coordinate = ( + predecessor.identity.operation_id, + predecessor.identity.request_fingerprint, + ); + discovered.insert(coordinate); + pending.insert(coordinate, predecessor); + } + let mut progressed = false; + while !pending.is_empty() { + if tokio::time::Instant::now() >= deadline { + return Ok(progressed); + } + let coordinates: Vec<_> = pending.keys().copied().collect(); + let mut changed = false; + for coordinate in coordinates { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + if remaining.is_zero() { + return Ok(progressed); + } + let predecessor = pending + .get(&coordinate) + .cloned() + .ok_or(OperationRestoreError::InvalidAttribution)?; + let identity = predecessor.identity; + let bootstrap_id = self.bootstrap_id(identity.domain)?; + let shard_id = + operation_shard_id(identity, bootstrap_id, self.0.retention.shard_count); + let key = operation_shard_key(identity.domain, shard_id); + let Some((_, shard)) = self.load_operation_shard(&key).await? else { + self.latch_unhealthy(); + return Err(OperationRestoreError::MissingOperationShard); + }; + self.validate_operation_shard(&shard, identity.domain, bootstrap_id, shard_id)?; + let Some(record) = shard.entries.get(&identity.operation_id.to_string()) else { + pending.remove(&coordinate); + changed = true; + continue; + }; + self.validate_intent_record(record, identity)?; + if record.state != IntentState::Pending { + pending.remove(&coordinate); + changed = true; + continue; + } + let fence = match self + .acquire_operation_fence_with_timeout(identity, remaining) + .await + { + Ok(fence) => fence, + Err(error) if error.is_healthy_retryable() => continue, + Err(error) => return Err(error), + }; + let current = match self.0.source.manifest(identity).await { + Ok(manifest) if manifest.identity == identity => manifest, + Ok(_) | Err(ManifestReadError::Invalid) => { + drop(fence); + self.latch_unhealthy(); + return Err(OperationRestoreError::InvalidAttribution); + } + Err(ManifestReadError::Missing) => { + drop(fence); + pending.remove(&coordinate); + changed = true; + continue; + } + Err(ManifestReadError::Unavailable) => { + drop(fence); + self.latch_unhealthy(); + return Err(OperationRestoreError::DatabaseUnavailable); + } + }; + if !self.manifest_ready_for_startup(¤t).await? { + drop(fence); + let earlier = self + .0 + .source + .predecessors(¤t) + .await + .map_err(|error| { + self.latch_unhealthy(); + match error { + ManifestReadError::Missing | ManifestReadError::Invalid => { + OperationRestoreError::InvalidAttribution + } + ManifestReadError::Unavailable => { + OperationRestoreError::DatabaseUnavailable + } + } + })?; + for earlier in earlier { + let earlier_coordinate = ( + earlier.identity.operation_id, + earlier.identity.request_fingerprint, + ); + if discovered.insert(earlier_coordinate) { + pending.insert(earlier_coordinate, earlier); + changed = true; + } + } + continue; + } + let result = self + .reconcile_pending_manifest(shard_id, record, current) + .await; + match result { + Ok(_) => fence + .unlock() + .await + .inspect_err(|_| self.latch_unhealthy())?, + Err(error) => { + drop(fence); + return Err(error); + } + } + pending.remove(&coordinate); + progressed = true; + changed = true; + } + if !changed { + return Ok(progressed); + } + } + Ok(progressed) + } + + async fn load_owned_intent( + &self, + intent: &OperationRestoreIntent, + ) -> Result { + let key = operation_shard_key(intent.identity.domain, intent.shard_id); + let bootstrap_id = self.bootstrap_id(intent.identity.domain)?; + let Some((_, shard)) = self.load_operation_shard(&key).await? else { + self.latch_unhealthy(); + return Err(OperationRestoreError::MissingOperationShard); + }; + self.validate_operation_shard( + &shard, + intent.identity.domain, + bootstrap_id, + intent.shard_id, + )?; + let Some(record) = shard.entries.get(&intent.identity.operation_id.to_string()) else { + return self.owner_mismatch(); + }; + self.validate_intent_record(record, intent.identity)?; + if record.owner_token != intent.owner_token { + return self.owner_mismatch(); + } + Ok(record.clone()) + } + + async fn abort_owned_intent( + &self, + intent: &OperationRestoreIntent, + ) -> Result<(), OperationRestoreError> { + let key = operation_shard_key(intent.identity.domain, intent.shard_id); + let bootstrap_id = self.bootstrap_id(intent.identity.domain)?; + for _ in 0..MAX_CAS_ATTEMPTS { + let Some((etag, mut shard)) = self.load_operation_shard(&key).await? else { + self.latch_unhealthy(); + return Err(OperationRestoreError::MissingOperationShard); + }; + self.validate_operation_shard( + &shard, + intent.identity.domain, + bootstrap_id, + intent.shard_id, + )?; + let Some(record) = shard.entries.get(&intent.identity.operation_id.to_string()) else { + return self.owner_mismatch(); + }; + self.validate_intent_record(record, intent.identity)?; + match record.state { + IntentState::Committed => return Err(OperationRestoreError::AlreadyCommitted), + IntentState::Aborted if record.owner_token == intent.owner_token => return Ok(()), + IntentState::Aborted => return self.owner_mismatch(), + IntentState::Pending if record.owner_token != intent.owner_token => { + return self.owner_mismatch() + } + IntentState::Pending => {} + } + let now = self.authoritative_now_ms().await?; + let aborted = record.aborted(now, self.terminal_retain_until(now)?); + shard + .entries + .insert(intent.identity.operation_id.to_string(), aborted); + if matches!( + self.store_record(&key, Some(&etag), &shard).await?, + StoreCas::Won + ) { + return Ok(()); + } + } + Err(OperationRestoreError::OperationContention) + } + + async fn provision_floor( + &self, + domain: CommunityId, + floor: AuthorizationVersionComponentFloor, + ) -> Result<(), OperationRestoreError> { + let bootstrap_id = self.bootstrap_id(domain)?; + if floor.component_key == [0; 32] || floor.version > i64::MAX as u64 { + self.latch_unhealthy(); + return Err(OperationRestoreError::InvalidAttribution); + } + let key = floor_key(domain, floor.component_kind, floor.component_key); + let expected = FloorRecord::provisioned(domain, bootstrap_id, &floor); + for _ in 0..MAX_CAS_ATTEMPTS { + match self.load_floor(&key).await? { + None => { + if matches!( + self.store_record(&key, None, &expected).await?, + StoreCas::Won + ) { + return Ok(()); + } + } + Some((_, existing)) => { + if let Err(error) = existing.validate_coordinate( + domain, + bootstrap_id, + floor.component_kind, + floor.component_key, + ) { + self.latch_unhealthy(); + return Err(error); + } + if existing.version == floor.version { + return Ok(()); + } + self.latch_unhealthy(); + return Err(if existing.version > floor.version { + OperationRestoreError::BackwardFloor + } else { + OperationRestoreError::StaleFloor + }); + } + } + } + self.latch_unhealthy(); + Err(OperationRestoreError::StoreUnavailable) + } + + fn validate_worst_case_shard_size( + &self, + domain: CommunityId, + bootstrap_id: Uuid, + ) -> Result<(), OperationRestoreError> { + let mut shard = OperationShardRecord::empty(domain, bootstrap_id, 0, self.0.retention); + let retention_ms = self.0.retention.retention_millis()?; + let terminal_at = u64::MAX + .checked_sub(retention_ms) + .ok_or(OperationRestoreError::InvalidInput)?; + let issued_at = terminal_at + .checked_sub(2) + .ok_or(OperationRestoreError::InvalidInput)?; + for index in 0..self.0.retention.entries_per_shard { + let operation_id = Uuid::from_u128(u128::from(index) + 1); + let identity = OperationIdentity { + domain, + operation_id, + request_fingerprint: [u8::try_from(index % 254 + 1).unwrap_or(1); 32], + }; + let record = IntentRecord::pending( + identity, + Uuid::from_u128(u128::from(index) + 10_000), + bootstrap_id, + issued_at, + issued_at + 1, + ) + .committed([7; 32], terminal_at, u64::MAX, true); + shard.entries.insert(operation_id.to_string(), record); + } + let encoded = + serde_json::to_vec(&shard).map_err(|_| OperationRestoreError::InvalidAttribution)?; + if u64::try_from(encoded.len()).unwrap_or(u64::MAX) > MAX_RECORD_BYTES { + return Err(OperationRestoreError::InvalidInput); + } + Ok(()) + } + + async fn provision_operation_shard( + &self, + domain: CommunityId, + bootstrap_id: Uuid, + shard_id: u16, + ) -> Result<(), OperationRestoreError> { + let key = operation_shard_key(domain, shard_id); + let expected = + OperationShardRecord::empty(domain, bootstrap_id, shard_id, self.0.retention); + for _ in 0..MAX_CAS_ATTEMPTS { + match self.load_operation_shard(&key).await? { + None => { + if matches!( + self.store_record(&key, None, &expected).await?, + StoreCas::Won + ) { + return Ok(()); + } + } + Some((_, existing)) => { + self.validate_operation_shard(&existing, domain, bootstrap_id, shard_id)?; + return Ok(()); + } + } + } + Err(OperationRestoreError::OperationContention) + } + + fn validate_operation_shard( + &self, + shard: &OperationShardRecord, + domain: CommunityId, + bootstrap_id: Uuid, + shard_id: u16, + ) -> Result<(), OperationRestoreError> { + if shard.format != OPERATION_SHARD_FORMAT_VERSION + || shard.mapping_version != OPERATION_SHARD_MAPPING_VERSION + || shard.domain != domain.as_uuid().to_string() + || shard.bootstrap_id != bootstrap_id + || shard.bootstrap_id.is_nil() + || shard.shard_id != shard_id + || decode32(&shard.layout_digest)? != self.0.retention.layout_digest() + || shard.entries.len() > usize::from(self.0.retention.entries_per_shard) + { + self.latch_unhealthy(); + return Err(OperationRestoreError::InvalidOperationShard); + } + for (operation, record) in &shard.entries { + let operation_id = Uuid::parse_str(operation).map_err(|_| { + self.latch_unhealthy(); + OperationRestoreError::InvalidOperationShard + })?; + let request_fingerprint = decode32(&record.request_fingerprint).map_err(|_| { + self.latch_unhealthy(); + OperationRestoreError::InvalidOperationShard + })?; + let identity = OperationIdentity { + domain, + operation_id, + request_fingerprint, + } + .validate() + .map_err(|_| { + self.latch_unhealthy(); + OperationRestoreError::InvalidOperationShard + })?; + if operation_shard_id(identity, bootstrap_id, self.0.retention.shard_count) != shard_id + { + self.latch_unhealthy(); + return Err(OperationRestoreError::InvalidOperationShard); + } + self.validate_intent_record(record, identity)?; + } + Ok(()) + } + + async fn apply_component( + &self, + manifest: &OperationManifest, + component: &ManifestComponent, + ) -> Result { + let bootstrap_id = self.bootstrap_id(manifest.identity.domain)?; + if component.key == [0; 32] + || component.after <= component.before + || component.after > i64::MAX as u64 + { + self.latch_unhealthy(); + return Err(OperationRestoreError::InvalidAttribution); + } + let key = floor_key(manifest.identity.domain, component.kind, component.key); + let started = tokio::time::Instant::now(); + for attempt in 0..MAX_CAS_ATTEMPTS { + match self.load_floor(&key).await? { + None if component.before == 0 => { + let record = FloorRecord::advanced(bootstrap_id, manifest, component); + if matches!(self.store_record(&key, None, &record).await?, StoreCas::Won) { + return Ok(ComponentReconciliation::Applied); + } + } + None => { + if attempt + 1 < MAX_CAS_ATTEMPTS && started.elapsed() < CAUSAL_LAG_BOUND { + tokio::time::sleep(CAS_RETRY_DELAY).await; + continue; + } + let database_version = self + .0 + .source + .floors(manifest.identity.domain) + .await + .map_err(|error| { + self.latch_unhealthy(); + match error { + ManifestReadError::Unavailable => { + OperationRestoreError::DatabaseUnavailable + } + ManifestReadError::Missing | ManifestReadError::Invalid => { + OperationRestoreError::InvalidAttribution + } + } + })? + .into_iter() + .find(|floor| { + floor.component_kind == component.kind + && floor.component_key == component.key + }) + .map_or(0, |floor| floor.version); + if database_version >= component.after { + return Err(OperationRestoreError::CausalLag); + } + self.latch_unhealthy(); + return Err(OperationRestoreError::MissingFloor); + } + Some((etag, record)) => { + if let Err(error) = record.validate_coordinate( + manifest.identity.domain, + bootstrap_id, + component.kind, + component.key, + ) { + self.latch_unhealthy(); + return Err(error); + } + if record.exact_advance(manifest, component)? { + return Ok(ComponentReconciliation::Exact); + } + if record.version == component.before { + let advanced = FloorRecord::advanced(bootstrap_id, manifest, component); + if matches!( + self.store_record(&key, Some(&etag), &advanced).await?, + StoreCas::Won + ) { + return Ok(ComponentReconciliation::Applied); + } + continue; + } + if record.version > component.after { + self.prove_supersession(manifest, component, &record) + .await?; + return Ok(ComponentReconciliation::CausallySuperseded); + } + if record.version < component.before { + if attempt + 1 < MAX_CAS_ATTEMPTS && started.elapsed() < CAUSAL_LAG_BOUND { + tokio::time::sleep(CAS_RETRY_DELAY).await; + continue; + } + let database_version = self + .0 + .source + .floors(manifest.identity.domain) + .await + .map_err(|error| { + self.latch_unhealthy(); + match error { + ManifestReadError::Unavailable => { + OperationRestoreError::DatabaseUnavailable + } + ManifestReadError::Missing | ManifestReadError::Invalid => { + OperationRestoreError::InvalidAttribution + } + } + })? + .into_iter() + .find(|floor| { + floor.component_kind == component.kind + && floor.component_key == component.key + }) + .map_or(0, |floor| floor.version); + if database_version >= component.after { + return Err(OperationRestoreError::CausalLag); + } + self.latch_unhealthy(); + return Err(OperationRestoreError::StaleFloor); + } + self.latch_unhealthy(); + return Err(OperationRestoreError::StaleFloor); + } + } + } + Err(OperationRestoreError::OperationContention) + } + + async fn prove_supersession( + &self, + manifest: &OperationManifest, + component: &ManifestComponent, + floor: &FloorRecord, + ) -> Result<(), OperationRestoreError> { + self.0 + .source + .prove_supersession(manifest, component, floor) + .await + .map_err(|error| { + self.latch_unhealthy(); + match error { + ManifestReadError::Unavailable => OperationRestoreError::DatabaseUnavailable, + ManifestReadError::Missing | ManifestReadError::Invalid => { + OperationRestoreError::InvalidAttribution + } + } + }) + } + + async fn load_operation_shard( + &self, + key: &str, + ) -> Result, OperationRestoreError> { + self.load_record(key).await + } + + async fn load_bootstrap( + &self, + key: &str, + ) -> Result, OperationRestoreError> { + self.load_record(key).await + } + + async fn load_floor( + &self, + key: &str, + ) -> Result, OperationRestoreError> { + self.load_record(key).await + } + + async fn load_record Deserialize<'de>>( + &self, + key: &str, + ) -> Result, OperationRestoreError> { + let loaded = self.0.store.load(key).await.map_err(|_| { + self.latch_unhealthy(); + OperationRestoreError::StoreUnavailable + })?; + loaded + .map(|loaded| { + if u64::try_from(loaded.body.len()).unwrap_or(u64::MAX) > MAX_RECORD_BYTES { + self.latch_unhealthy(); + return Err(OperationRestoreError::StoreUnavailable); + } + serde_json::from_slice(&loaded.body) + .map(|record| (loaded.etag, record)) + .map_err(|_| { + self.latch_unhealthy(); + OperationRestoreError::InvalidAttribution + }) + }) + .transpose() + } + + async fn store_record( + &self, + key: &str, + expected_etag: Option<&str>, + record: &T, + ) -> Result { + let body = serde_json::to_vec(record).map_err(|_| { + self.latch_unhealthy(); + OperationRestoreError::InvalidAttribution + })?; + if u64::try_from(body.len()).unwrap_or(u64::MAX) > MAX_RECORD_BYTES { + self.latch_unhealthy(); + return Err(OperationRestoreError::StoreUnavailable); + } + tokio::time::timeout( + STORE_CAS_TIMEOUT, + self.0.store.compare_and_swap(key, expected_etag, &body), + ) + .await + .map_err(|_| { + self.latch_unhealthy(); + OperationRestoreError::StoreUnavailable + })? + .map_err(|_| { + self.latch_unhealthy(); + OperationRestoreError::StoreUnavailable + }) + } + + fn require_healthy(&self) -> Result<(), OperationRestoreError> { + if self.is_healthy() { + Ok(()) + } else { + Err(OperationRestoreError::Unhealthy) + } + } + + fn bootstrap_id(&self, domain: CommunityId) -> Result { + self.0 + .bootstrap_ids + .get(&domain) + .copied() + .ok_or(OperationRestoreError::DomainNotConfigured) + } + + fn require_domain_ready(&self, domain: CommunityId) -> Result<(), OperationRestoreError> { + if self.0.ready_domains.contains(&domain) { + Ok(()) + } else { + Err(OperationRestoreError::MissingBootstrap) + } + } + + async fn read_database_floors( + &self, + domain: CommunityId, + ) -> Result, OperationRestoreError> { + self.0.source.floors(domain).await.map_err(|error| { + self.latch_unhealthy(); + match error { + ManifestReadError::Missing | ManifestReadError::Invalid => { + OperationRestoreError::InvalidAttribution + } + ManifestReadError::Unavailable => OperationRestoreError::DatabaseUnavailable, + } + }) + } + + async fn authoritative_now_ms(&self) -> Result { + self.0.source.authoritative_now_ms().await.map_err(|error| { + self.latch_unhealthy(); + match error { + ManifestReadError::Unavailable => OperationRestoreError::DatabaseUnavailable, + ManifestReadError::Missing | ManifestReadError::Invalid => { + OperationRestoreError::InvalidAttribution + } + } + }) + } + + fn terminal_retain_until(&self, terminal_at_ms: u64) -> Result { + terminal_at_ms + .checked_add(self.0.retention.retention_millis()?) + .ok_or(OperationRestoreError::InvalidAttribution) + } + + fn validate_bootstrap( + &self, + marker: &BootstrapRecord, + domain: CommunityId, + bootstrap_id: Uuid, + ) -> Result<(), OperationRestoreError> { + if marker.format == FORMAT_VERSION + && marker.domain == domain.as_uuid().to_string() + && marker.bootstrap_id == bootstrap_id + && !marker.bootstrap_id.is_nil() + && marker.operation_mapping_version == OPERATION_SHARD_MAPPING_VERSION + && marker.operation_shard_count == self.0.retention.shard_count + && marker.operation_entries_per_shard == self.0.retention.entries_per_shard + && marker.terminal_retention_ms == self.0.retention.retention_millis()? + && decode32(&marker.operation_record_format_hash)? + == operation_shard_record_format_hash() + && decode32(&marker.operation_layout_digest)? == self.0.retention.layout_digest() + { + Ok(()) + } else { + self.latch_unhealthy(); + Err(OperationRestoreError::InvalidBootstrap) + } + } + + fn validate_intent_record( + &self, + record: &IntentRecord, + identity: OperationIdentity, + ) -> Result<(), OperationRestoreError> { + let bootstrap_id = self.bootstrap_id(identity.domain)?; + let validation = record + .validate_identity(identity, bootstrap_id) + .and_then(|()| { + if let Some((terminal_at, retain_until)) = + record.terminal_at_ms.zip(record.retain_until_ms) + { + if retain_until.checked_sub(terminal_at) + != Some(self.0.retention.retention_millis()?) + { + return Err(OperationRestoreError::InvalidAttribution); + } + } + Ok(()) + }); + match validation { + Err(OperationRestoreError::InvalidAttribution) => { + self.latch_unhealthy(); + Err(OperationRestoreError::InvalidAttribution) + } + other => other, + } + } + + fn identity_from_shard_entry( + &self, + domain: CommunityId, + shard_id: u16, + operation: &str, + record: &IntentRecord, + ) -> Result { + let operation_id = Uuid::parse_str(&record.operation_id).map_err(|_| { + self.latch_unhealthy(); + OperationRestoreError::InvalidAttribution + })?; + let request_fingerprint = decode32(&record.request_fingerprint).inspect_err(|_| { + self.latch_unhealthy(); + })?; + let identity = OperationIdentity { + domain, + operation_id, + request_fingerprint, + } + .validate() + .inspect_err(|_| { + self.latch_unhealthy(); + })?; + if identity.operation_id.to_string() != operation + || operation_shard_id( + identity, + self.bootstrap_id(domain)?, + self.0.retention.shard_count, + ) != shard_id + { + self.latch_unhealthy(); + return Err(OperationRestoreError::InvalidAttribution); + } + self.validate_intent_record(record, identity)?; + Ok(identity) + } + + fn latch_unhealthy(&self) { + self.0.healthy.store(false, Ordering::Release); + } +} + +impl fmt::Debug for OperationRestoreRuntime { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OperationRestoreRuntime") + .field("healthy", &self.is_healthy()) + .field("coordinates", &"[REDACTED]") + .finish() + } +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +enum IntentState { + Pending, + Committed, + Aborted, +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct BootstrapRecord { + format: u8, + domain: String, + bootstrap_id: Uuid, + operation_mapping_version: u8, + operation_shard_count: u16, + operation_entries_per_shard: u16, + terminal_retention_ms: u64, + operation_record_format_hash: String, + operation_layout_digest: String, +} + +impl BootstrapRecord { + fn new( + domain: CommunityId, + bootstrap_id: Uuid, + retention: OperationRestoreRetentionPolicy, + ) -> Self { + Self { + format: FORMAT_VERSION, + domain: domain.as_uuid().to_string(), + bootstrap_id, + operation_mapping_version: OPERATION_SHARD_MAPPING_VERSION, + operation_shard_count: retention.shard_count, + operation_entries_per_shard: retention.entries_per_shard, + terminal_retention_ms: retention.retention_millis().unwrap_or(u64::MAX), + operation_record_format_hash: hex::encode(operation_shard_record_format_hash()), + operation_layout_digest: hex::encode(retention.layout_digest()), + } + } +} + +impl fmt::Debug for BootstrapRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("BootstrapRecord([REDACTED])") + } +} + +#[derive(Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct IntentRecord { + format: u8, + domain: String, + operation_id: String, + request_fingerprint: String, + bootstrap_id: Uuid, + owner_token: Uuid, + issued_at_ms: u64, + expires_at_ms: u64, + state: IntentState, + manifest_digest: Option, + terminal_at_ms: Option, + retain_until_ms: Option, + causally_superseded: bool, +} + +impl IntentRecord { + fn pending( + identity: OperationIdentity, + owner_token: Uuid, + bootstrap_id: Uuid, + issued_at_ms: u64, + expires_at_ms: u64, + ) -> Self { + Self { + format: FORMAT_VERSION, + domain: identity.domain.as_uuid().to_string(), + operation_id: identity.operation_id.to_string(), + request_fingerprint: hex::encode(identity.request_fingerprint), + bootstrap_id, + owner_token, + issued_at_ms, + expires_at_ms, + state: IntentState::Pending, + manifest_digest: None, + terminal_at_ms: None, + retain_until_ms: None, + causally_superseded: false, + } + } + + fn validate_identity( + &self, + expected: OperationIdentity, + bootstrap_id: Uuid, + ) -> Result<(), OperationRestoreError> { + let terminal_shape_valid = self.terminal_at_ms.zip(self.retain_until_ms).is_some_and( + |(terminal_at, retain_until)| { + terminal_at >= self.issued_at_ms && retain_until > terminal_at + }, + ); + let state_shape_valid = match self.state { + IntentState::Pending => { + self.manifest_digest.is_none() + && self.terminal_at_ms.is_none() + && self.retain_until_ms.is_none() + && !self.causally_superseded + } + IntentState::Aborted => { + self.manifest_digest.is_none() && terminal_shape_valid && !self.causally_superseded + } + IntentState::Committed => { + self.manifest_digest + .as_deref() + .is_some_and(|digest| decode32(digest).is_ok()) + && terminal_shape_valid + } + }; + if self.format != FORMAT_VERSION + || self.domain != expected.domain.as_uuid().to_string() + || self.operation_id != expected.operation_id.to_string() + || self.bootstrap_id != bootstrap_id + || self.bootstrap_id.is_nil() + || self.owner_token.is_nil() + || self.issued_at_ms == 0 + || self.expires_at_ms <= self.issued_at_ms + || self.expires_at_ms - self.issued_at_ms + > u64::try_from(MAX_LEASE.as_millis()).unwrap_or(u64::MAX) + || !state_shape_valid + { + return Err(OperationRestoreError::InvalidAttribution); + } + if decode32(&self.request_fingerprint)? != expected.request_fingerprint { + return Err(OperationRestoreError::OperationIdentityConflict); + } + Ok(()) + } + + fn committed( + &self, + manifest_digest: [u8; 32], + terminal_at_ms: u64, + retain_until_ms: u64, + causally_superseded: bool, + ) -> Self { + Self { + format: self.format, + domain: self.domain.clone(), + operation_id: self.operation_id.clone(), + request_fingerprint: self.request_fingerprint.clone(), + bootstrap_id: self.bootstrap_id, + owner_token: self.owner_token, + issued_at_ms: self.issued_at_ms, + expires_at_ms: self.expires_at_ms, + state: IntentState::Committed, + manifest_digest: Some(hex::encode(manifest_digest)), + terminal_at_ms: Some(terminal_at_ms), + retain_until_ms: Some(retain_until_ms), + causally_superseded, + } + } + + fn aborted(&self, terminal_at_ms: u64, retain_until_ms: u64) -> Self { + Self { + format: self.format, + domain: self.domain.clone(), + operation_id: self.operation_id.clone(), + request_fingerprint: self.request_fingerprint.clone(), + bootstrap_id: self.bootstrap_id, + owner_token: self.owner_token, + issued_at_ms: self.issued_at_ms, + expires_at_ms: self.expires_at_ms, + state: IntentState::Aborted, + manifest_digest: None, + terminal_at_ms: Some(terminal_at_ms), + retain_until_ms: Some(retain_until_ms), + causally_superseded: false, + } + } + + fn is_collectible(&self, authoritative_now_ms: u64) -> bool { + matches!(self.state, IntentState::Committed | IntentState::Aborted) + && self + .retain_until_ms + .is_some_and(|retain_until| authoritative_now_ms >= retain_until) + } +} + +impl fmt::Debug for IntentRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("IntentRecord([REDACTED])") + } +} + +#[derive(Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct OperationShardRecord { + format: u8, + mapping_version: u8, + domain: String, + bootstrap_id: Uuid, + layout_digest: String, + shard_id: u16, + entries: BTreeMap, +} + +impl OperationShardRecord { + fn empty( + domain: CommunityId, + bootstrap_id: Uuid, + shard_id: u16, + retention: OperationRestoreRetentionPolicy, + ) -> Self { + Self { + format: OPERATION_SHARD_FORMAT_VERSION, + mapping_version: OPERATION_SHARD_MAPPING_VERSION, + domain: domain.as_uuid().to_string(), + bootstrap_id, + layout_digest: hex::encode(retention.layout_digest()), + shard_id, + entries: BTreeMap::new(), + } + } +} + +impl fmt::Debug for OperationShardRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OperationShardRecord") + .field("shard_id", &self.shard_id) + .field("entry_count", &self.entries.len()) + .field("coordinates", &"[REDACTED]") + .finish() + } +} + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +struct FloorRecord { + format: u8, + domain: String, + bootstrap_id: Uuid, + component_kind: i16, + component_key: String, + version: u64, + operation_id: Option, + request_fingerprint: Option, + component_digest: Option, + manifest_digest: Option, +} + +struct FloorProvenance { + operation_id: Uuid, + request_fingerprint: [u8; 32], + component_digest: [u8; 32], + manifest_digest: [u8; 32], +} + +impl FloorRecord { + fn provisioned( + domain: CommunityId, + bootstrap_id: Uuid, + floor: &AuthorizationVersionComponentFloor, + ) -> Self { + Self { + format: FORMAT_VERSION, + domain: domain.as_uuid().to_string(), + bootstrap_id, + component_kind: floor.component_kind as i16, + component_key: hex::encode(floor.component_key), + version: floor.version, + operation_id: None, + request_fingerprint: None, + component_digest: None, + manifest_digest: None, + } + } + + fn advanced( + bootstrap_id: Uuid, + manifest: &OperationManifest, + component: &ManifestComponent, + ) -> Self { + Self { + format: FORMAT_VERSION, + domain: manifest.identity.domain.as_uuid().to_string(), + bootstrap_id, + component_kind: component.kind as i16, + component_key: hex::encode(component.key), + version: component.after, + operation_id: Some(manifest.identity.operation_id.to_string()), + request_fingerprint: Some(hex::encode(manifest.identity.request_fingerprint)), + component_digest: Some(hex::encode(component.digest)), + manifest_digest: Some(hex::encode(manifest.manifest_digest)), + } + } + + fn validate_coordinate( + &self, + domain: CommunityId, + bootstrap_id: Uuid, + kind: AuthorizationVersionComponentKind, + key: [u8; 32], + ) -> Result<(), OperationRestoreError> { + let provenance_valid = match ( + self.operation_id.as_deref(), + self.request_fingerprint.as_deref(), + self.component_digest.as_deref(), + self.manifest_digest.as_deref(), + ) { + (None, None, None, None) => true, + (Some(operation), Some(request), Some(component), Some(manifest)) => { + Uuid::parse_str(operation).is_ok_and(|operation| !operation.is_nil()) + && decode32(request).is_ok() + && decode32(component).is_ok() + && decode32(manifest).is_ok() + } + _ => false, + }; + if self.format == FORMAT_VERSION + && self.domain == domain.as_uuid().to_string() + && self.bootstrap_id == bootstrap_id + && !self.bootstrap_id.is_nil() + && self.component_kind == kind as i16 + && decode32(&self.component_key)? == key + && self.version <= i64::MAX as u64 + && provenance_valid + { + Ok(()) + } else { + Err(OperationRestoreError::WrongComponentKey) + } + } + + fn exact_advance( + &self, + manifest: &OperationManifest, + component: &ManifestComponent, + ) -> Result { + if self.version != component.after { + return Ok(false); + } + Ok(self.operation_id.as_deref() + == Some(manifest.identity.operation_id.to_string().as_str()) + && self.request_fingerprint.as_deref() + == Some(hex::encode(manifest.identity.request_fingerprint).as_str()) + && self.component_digest.as_deref() == Some(hex::encode(component.digest).as_str()) + && self.manifest_digest.as_deref() + == Some(hex::encode(manifest.manifest_digest).as_str())) + } + + fn provenance(&self) -> Result { + let operation_id = Uuid::parse_str( + self.operation_id + .as_deref() + .ok_or(OperationRestoreError::InvalidAttribution)?, + ) + .map_err(|_| OperationRestoreError::InvalidAttribution)?; + if operation_id.is_nil() { + return Err(OperationRestoreError::InvalidAttribution); + } + Ok(FloorProvenance { + operation_id, + request_fingerprint: decode32( + self.request_fingerprint + .as_deref() + .ok_or(OperationRestoreError::InvalidAttribution)?, + )?, + component_digest: decode32( + self.component_digest + .as_deref() + .ok_or(OperationRestoreError::InvalidAttribution)?, + )?, + manifest_digest: decode32( + self.manifest_digest + .as_deref() + .ok_or(OperationRestoreError::InvalidAttribution)?, + )?, + }) + } +} + +impl fmt::Debug for FloorRecord { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("FloorRecord([REDACTED])") + } +} + +fn operation_shard_key(domain: CommunityId, shard_id: u16) -> String { + format!( + "nip-fi/restore/v1/operation-shards/{}/{shard_id:04x}", + domain.as_uuid() + ) +} + +#[cfg(any(test, feature = "test-utils"))] +fn operation_fence_test_key(identity: OperationIdentity) -> i64 { + let mut digest = Sha256::new(); + digest.update(b"buzz:nip-fi-operation-session-fence:v1"); + digest.update(identity.domain.as_uuid().as_bytes()); + digest.update(identity.operation_id.as_bytes()); + digest.update(identity.request_fingerprint); + let digest: [u8; 32] = digest.finalize().into(); + i64::from_be_bytes(digest[..8].try_into().expect("eight-byte fence key")) +} + +fn operation_shard_id(identity: OperationIdentity, bootstrap_id: Uuid, shard_count: u16) -> u16 { + let mut digest = Sha256::new(); + digest.update(b"buzz:nip-fi-operation-restore-shard:v1"); + digest.update(bootstrap_id.as_bytes()); + digest.update(identity.domain.as_uuid().as_bytes()); + digest.update(identity.operation_id.as_bytes()); + let bytes: [u8; 32] = digest.finalize().into(); + u16::from_be_bytes([bytes[0], bytes[1]]) & (shard_count - 1) +} + +fn operation_shard_record_format_hash() -> [u8; 32] { + let mut digest = Sha256::new(); + digest.update( + b"buzz:nip-fi-operation-shard-record:v1|format|mapping_version|domain|bootstrap_id|layout_digest|shard_id|entries|operation_id|request_fingerprint|owner_token|issued_at_ms|expires_at_ms|state|manifest_digest|terminal_at_ms|retain_until_ms|causally_superseded", + ); + digest.finalize().into() +} + +fn bootstrap_key(domain: CommunityId) -> String { + format!("nip-fi/restore/v1/bootstrap/{}", domain.as_uuid()) +} + +fn floor_key( + domain: CommunityId, + kind: AuthorizationVersionComponentKind, + key: [u8; 32], +) -> String { + format!( + "nip-fi/restore/v1/floors/{}/{}/{}", + domain.as_uuid(), + kind as i16, + hex::encode(key) + ) +} + +fn floor_prefix(domain: CommunityId) -> String { + format!("nip-fi/restore/v1/floors/{}/", domain.as_uuid()) +} + +fn component_kind_from_i16( + value: i16, +) -> Result { + match value { + 1 => Ok(AuthorizationVersionComponentKind::Binding), + 2 => Ok(AuthorizationVersionComponentKind::Policy), + 3 => Ok(AuthorizationVersionComponentKind::InvalidationGeneration), + 4 => Ok(AuthorizationVersionComponentKind::AuthorityEpoch), + 6 => Ok(AuthorizationVersionComponentKind::DelegatedRelationship), + 7 => Ok(AuthorizationVersionComponentKind::LifecycleSelector), + _ => Err(OperationRestoreError::InvalidAttribution), + } +} + +fn lease_expiry(now_ms: u64, lease: Duration) -> Result { + let lease_ms = + u64::try_from(lease.as_millis()).map_err(|_| OperationRestoreError::InvalidInput)?; + now_ms + .checked_add(lease_ms) + .ok_or(OperationRestoreError::InvalidInput) +} + +fn decode32(value: &str) -> Result<[u8; 32], OperationRestoreError> { + let decoded = hex::decode(value).map_err(|_| OperationRestoreError::InvalidAttribution)?; + decoded + .try_into() + .map_err(|_| OperationRestoreError::InvalidAttribution) +} + +/// Closed fail-closed restore witness failures. Coordinates are never rendered. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum OperationRestoreError { + /// Caller or deployment input is malformed. + #[error("operation restore input is invalid")] + InvalidInput, + /// Domain is not part of the immutable configured restore set. + #[error("operation restore domain is not configured")] + DomainNotConfigured, + /// Configured immutable bootstrap identity is malformed or mismatched. + #[error("operation restore bootstrap is invalid")] + InvalidBootstrap, + /// Explicit bootstrap provisioning has not completed. + #[error("operation restore bootstrap is missing")] + MissingBootstrap, + /// One immutable provisioned operation shard is missing. + #[error("operation restore shard is missing")] + MissingOperationShard, + /// An operation shard is malformed or bound to another immutable layout. + #[error("operation restore shard is invalid")] + InvalidOperationShard, + /// The exact mapped shard has no capacity after bounded collection. + #[error("operation restore shard capacity is exhausted")] + OperationCapacityExceeded, + /// Bounded healthy CAS contention prevented reservation or terminalization. + #[error("operation restore shard is contended")] + OperationContention, + /// The fixed detached-session capacity is currently saturated. + #[error("operation restore fence capacity is saturated")] + OperationFenceSaturated, + /// The exact operation advisory lock was not acquired within its bound. + #[error("operation restore fence is contended")] + OperationFenceTimeout, + /// A causal predecessor did not become externally visible within the + /// bounded non-sticky reconciliation window. + #[error("operation restore causal predecessor is pending")] + CausalLag, + /// The exact operation key is bound to different immutable identity. + #[error("operation restore identity conflicts")] + OperationIdentityConflict, + /// Another live owner holds the operation intent. + #[error("operation restore is already in progress")] + OperationInProgress, + /// The owner token does not match the pending operation. + #[error("operation restore owner does not match")] + OwnerMismatch, + /// The operation was explicitly aborted. + #[error("operation restore was aborted")] + OperationAborted, + /// The operation was already committed and cannot be aborted. + #[error("operation restore is already committed")] + AlreadyCommitted, + /// No external intent exists for reconciliation. + #[error("operation restore intent is missing")] + MissingIntent, + /// PostgreSQL has no exact operation-bound manifest. + #[error("operation restore attribution is missing")] + MissingAttribution, + /// Manifest or external record data is malformed/incomplete. + #[error("operation restore attribution is invalid")] + InvalidAttribution, + /// An existing component record is bound to another exact coordinate. + #[error("operation restore component key conflicts")] + WrongComponentKey, + /// A nonzero first advance lacks a provisioned baseline. + #[error("operation restore component floor is missing")] + MissingFloor, + /// External state does not contain the manifest's exact before value. + #[error("operation restore component floor is stale")] + StaleFloor, + /// PostgreSQL is behind a previously witnessed external floor. + #[error("operation restore detected a backward component floor")] + BackwardFloor, + /// Independent PostgreSQL witness reads failed. + #[error("operation restore database is unavailable")] + DatabaseUnavailable, + /// External CAS/read failed or exhausted its bounded retries. + #[error("operation restore store is unavailable")] + StoreUnavailable, + /// Sticky health was previously lost. + #[error("operation restore runtime is unhealthy")] + Unhealthy, +} + +impl OperationRestoreError { + /// Ordinary bounded contention/capacity outcomes do not poison runtime + /// health and may be retried with a fresh server operation. + pub const fn is_healthy_retryable(&self) -> bool { + matches!( + self, + Self::OperationCapacityExceeded + | Self::OperationContention + | Self::OperationFenceSaturated + | Self::OperationFenceTimeout + | Self::OperationInProgress + | Self::CausalLag + ) + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::HashMap, + sync::{atomic::AtomicU64, Mutex}, + }; + + use super::*; + use sqlx::postgres::PgPoolOptions; + + #[derive(Default)] + struct MemoryStore { + records: Mutex)>>, + fail: AtomicBool, + forced_cas_losses: AtomicU64, + floor_barrier: Option>, + } + + #[async_trait] + impl RestoreStore for MemoryStore { + async fn load(&self, key: &str) -> Result, ()> { + if self.fail.load(Ordering::Acquire) { + return Err(()); + } + Ok(self + .records + .lock() + .expect("records") + .get(key) + .map(|(version, body)| StoredObject { + etag: version.to_string(), + body: body.clone(), + })) + } + + async fn list(&self, prefix: &str) -> Result, ()> { + if self.fail.load(Ordering::Acquire) { + return Err(()); + } + Ok(self + .records + .lock() + .expect("records") + .keys() + .filter(|key| key.starts_with(prefix)) + .cloned() + .collect()) + } + + async fn compare_and_swap( + &self, + key: &str, + expected_etag: Option<&str>, + body: &[u8], + ) -> Result { + if self.fail.load(Ordering::Acquire) { + return Err(()); + } + if self + .forced_cas_losses + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + { + return Ok(StoreCas::Lost); + } + if key.contains("/floors/") { + if let Some(barrier) = &self.floor_barrier { + barrier.wait().await; + } + } + let mut records = self.records.lock().expect("records"); + let current = records.get(key).map(|(version, _)| version.to_string()); + if current.as_deref() != expected_etag { + return Ok(StoreCas::Lost); + } + let next = records.get(key).map_or(1, |(version, _)| version + 1); + records.insert(key.to_owned(), (next, body.to_vec())); + Ok(StoreCas::Won) + } + } + + struct MemorySource { + manifests: Mutex>, + floors: Mutex>, + authoritative_now_ms: AtomicU64, + valid_supersession: AtomicBool, + } + + #[async_trait] + impl ManifestSource for MemorySource { + async fn authoritative_now_ms(&self) -> Result { + Ok(self.authoritative_now_ms.load(Ordering::Acquire)) + } + + async fn manifest( + &self, + identity: OperationIdentity, + ) -> Result { + self.manifests + .lock() + .expect("manifests") + .get(&(identity.operation_id, identity.request_fingerprint)) + .cloned() + .ok_or(ManifestReadError::Missing) + } + + async fn floors( + &self, + _domain: CommunityId, + ) -> Result, ManifestReadError> { + Ok(self.floors.lock().expect("floors").clone()) + } + + async fn predecessors( + &self, + manifest: &OperationManifest, + ) -> Result, ManifestReadError> { + let manifests = self.manifests.lock().expect("manifests"); + let mut predecessors = BTreeMap::new(); + for candidate in manifests.values() { + if candidate.identity == manifest.identity { + continue; + } + if candidate.components.iter().any(|candidate_component| { + manifest.components.iter().any(|component| { + candidate_component.kind == component.kind + && candidate_component.key == component.key + && candidate_component.after == component.before + }) + }) { + predecessors.insert( + ( + candidate.identity.operation_id, + candidate.identity.request_fingerprint, + ), + candidate.clone(), + ); + } + } + Ok(predecessors.into_values().collect()) + } + + async fn prove_supersession( + &self, + _pending: &OperationManifest, + _component: &ManifestComponent, + _floor: &FloorRecord, + ) -> Result<(), ManifestReadError> { + if self.valid_supersession.load(Ordering::Acquire) { + Ok(()) + } else { + Err(ManifestReadError::Invalid) + } + } + } + + fn test_retention() -> OperationRestoreRetentionPolicy { + OperationRestoreRetentionPolicy::new(4, 8, MIN_TERMINAL_RETENTION) + .expect("valid test retention") + } + + fn runtime( + source: Arc, + store: Arc, + domain: CommunityId, + ) -> OperationRestoreRuntime { + runtime_with_lease(source, store, domain, Duration::from_secs(30)) + } + + fn runtime_with_lease( + source: Arc, + store: Arc, + domain: CommunityId, + lease: Duration, + ) -> OperationRestoreRuntime { + runtime_with_policy(source, store, domain, lease, test_retention()) + } + + fn runtime_with_policy( + source: Arc, + store: Arc, + domain: CommunityId, + lease: Duration, + retention: OperationRestoreRetentionPolicy, + ) -> OperationRestoreRuntime { + let bootstrap_id = Uuid::new_v4(); + { + let mut records = store.records.lock().expect("records"); + for shard_id in 0..retention.shard_count { + records.insert( + operation_shard_key(domain, shard_id), + ( + 1, + serde_json::to_vec(&OperationShardRecord::empty( + domain, + bootstrap_id, + shard_id, + retention, + )) + .expect("encode shard"), + ), + ); + } + } + let ready_domains = Arc::new(DashSet::new()); + ready_domains.insert(domain); + OperationRestoreRuntime(Arc::new(RestoreInner { + writer_db: test_db(), + source, + store, + bootstrap_ids: Arc::new(HashMap::from([(domain, bootstrap_id)])), + ready_domains, + lease, + retention, + fence_pool: test_fence_pool(), + fence_slots: Arc::new(Semaphore::new(1)), + test_fences: Some(Arc::new(DashSet::new())), + healthy: AtomicBool::new(true), + })) + } + + fn unready_runtime( + source: Arc, + store: Arc, + domain: CommunityId, + bootstrap_id: Uuid, + ) -> OperationRestoreRuntime { + unready_runtime_with_policy(source, store, domain, bootstrap_id, test_retention()) + } + + fn unready_runtime_with_policy( + source: Arc, + store: Arc, + domain: CommunityId, + bootstrap_id: Uuid, + retention: OperationRestoreRetentionPolicy, + ) -> OperationRestoreRuntime { + OperationRestoreRuntime(Arc::new(RestoreInner { + writer_db: test_db(), + source, + store, + bootstrap_ids: Arc::new(HashMap::from([(domain, bootstrap_id)])), + ready_domains: Arc::new(DashSet::new()), + lease: Duration::from_secs(30), + retention, + fence_pool: test_fence_pool(), + fence_slots: Arc::new(Semaphore::new(1)), + test_fences: Some(Arc::new(DashSet::new())), + healthy: AtomicBool::new(true), + })) + } + + fn test_fence_pool() -> PgPool { + PgPoolOptions::new() + .max_connections(1) + .connect_lazy("postgresql://127.0.0.1:1/unreachable") + .expect("test fence URL") + } + + fn test_db() -> Db { + Db::from_pool(test_fence_pool()) + } + + fn manifest( + domain: CommunityId, + operation_id: Uuid, + request: [u8; 32], + key: [u8; 32], + before: u64, + after: u64, + ) -> OperationManifest { + manifest_for_kind( + domain, + operation_id, + request, + AuthorizationVersionComponentKind::Binding, + key, + before, + after, + ) + } + + fn manifest_for_kind( + domain: CommunityId, + operation_id: Uuid, + request: [u8; 32], + kind: AuthorizationVersionComponentKind, + key: [u8; 32], + before: u64, + after: u64, + ) -> OperationManifest { + OperationManifest { + identity: OperationIdentity { + domain, + operation_id, + request_fingerprint: request, + }, + manifest_digest: [8; 32], + components: vec![ManifestComponent { + kind, + key, + before, + after, + digest: [9; 32], + }], + } + } + + fn fixture() -> ( + OperationRestoreRuntime, + Arc, + Arc, + CommunityId, + ) { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(false), + }); + let store = Arc::new(MemoryStore::default()); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + ( + runtime(source.clone(), store.clone(), domain), + source, + store, + domain, + ) + } + + #[tokio::test] + async fn exact_replay_is_operation_bound_and_wrong_request_conflicts() { + let (runtime, source, _, domain) = fixture(); + let operation = Uuid::new_v4(); + let request = [1; 32]; + source.manifests.lock().expect("manifests").insert( + (operation, request), + manifest(domain, operation, request, [2; 32], 0, 1), + ); + let OperationRestoreBegin::Acquired(intent) = runtime + .begin(domain, operation, request) + .await + .expect("begin") + else { + panic!("first begin acquired"); + }; + assert!(!runtime.commit(intent).await.expect("commit").replay); + assert!(matches!( + runtime + .begin(domain, operation, request) + .await + .expect("replay"), + OperationRestoreBegin::ExactReplay(_) + )); + assert!(matches!( + runtime.begin(domain, operation, [3; 32]).await, + Err(OperationRestoreError::OperationIdentityConflict) + )); + } + + #[tokio::test] + async fn startup_never_synthesizes_a_missing_or_mismatched_bootstrap() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(false), + }); + let store = Arc::new(MemoryStore::default()); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let missing = unready_runtime(source.clone(), store.clone(), domain, Uuid::new_v4()); + assert_eq!( + missing.verify_domain(domain).await, + Err(OperationRestoreError::MissingBootstrap) + ); + assert!(!missing.is_healthy()); + + let bootstrap_id = Uuid::new_v4(); + let provisioner = unready_runtime(source.clone(), store.clone(), domain, bootstrap_id); + provisioner + .provision_domain(domain) + .await + .expect("explicit bootstrap provisioning"); + let mismatched = unready_runtime(source, store, domain, Uuid::new_v4()); + assert_eq!( + mismatched.verify_domain(domain).await, + Err(OperationRestoreError::InvalidBootstrap) + ); + assert!(!mismatched.is_healthy()); + } + + #[tokio::test] + async fn startup_detects_a_component_removed_by_database_restore() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(vec![AuthorizationVersionComponentFloor { + component_kind: AuthorizationVersionComponentKind::Binding, + component_key: [4; 32], + version: 7, + }]), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(false), + }); + let store = Arc::new(MemoryStore::default()); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let bootstrap_id = Uuid::new_v4(); + let provisioner = unready_runtime(source.clone(), store.clone(), domain, bootstrap_id); + provisioner + .provision_domain(domain) + .await + .expect("provision exact component"); + source.floors.lock().expect("floors").clear(); + let verifier = unready_runtime(source, store, domain, bootstrap_id); + assert_eq!( + verifier.verify_domain(domain).await, + Err(OperationRestoreError::BackwardFloor) + ); + assert!(!verifier.is_healthy()); + } + + #[tokio::test] + async fn startup_reconciles_pending_postgres_commit_before_floor_comparison() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(false), + }); + let store = Arc::new(MemoryStore::default()); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let bootstrap_id = Uuid::new_v4(); + let original = unready_runtime(source.clone(), store.clone(), domain, bootstrap_id); + original + .provision_domain(domain) + .await + .expect("provision initial absent baseline"); + let operation = Uuid::new_v4(); + let request = [1; 32]; + let component_key = [2; 32]; + let OperationRestoreBegin::Acquired(_lost_owner) = original + .begin(domain, operation, request) + .await + .expect("persist pending owner before writer transaction") + else { + panic!("acquired"); + }; + + // Model a durable PostgreSQL commit followed by process death before + // restore.commit(): the exact manifest and current DB floor advanced, + // while the external intent is still Pending and its floor is absent. + source.manifests.lock().expect("manifests").insert( + (operation, request), + manifest(domain, operation, request, component_key, 0, 1), + ); + *source.floors.lock().expect("floors") = vec![AuthorizationVersionComponentFloor { + component_kind: AuthorizationVersionComponentKind::Binding, + component_key, + version: 1, + }]; + + let restarted = unready_runtime(source, store, domain, bootstrap_id); + restarted + .verify_domain(domain) + .await + .expect("restart reconciles exact pending manifest first"); + assert!(matches!( + restarted.begin(domain, operation, request).await, + Ok(OperationRestoreBegin::ExactReplay(OperationRestoreCommit { + replay: true, + .. + })) + )); + assert!(restarted.is_healthy()); + } + + #[tokio::test] + async fn startup_preserves_live_missing_owner_and_aborts_expired_missing_owner() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(false), + }); + let store = Arc::new(MemoryStore::default()); + let live_domain = CommunityId::from_uuid(Uuid::new_v4()); + let live_bootstrap = Uuid::new_v4(); + let live = unready_runtime(source.clone(), store.clone(), live_domain, live_bootstrap); + live.provision_domain(live_domain) + .await + .expect("provision live domain"); + live.begin(live_domain, Uuid::new_v4(), [1; 32]) + .await + .expect("acquire live pending owner"); + let live_restart = + unready_runtime(source.clone(), store.clone(), live_domain, live_bootstrap); + assert_eq!( + live_restart.verify_domain(live_domain).await, + Err(OperationRestoreError::OperationInProgress) + ); + assert!(live_restart.is_healthy()); + + let expired_domain = CommunityId::from_uuid(Uuid::new_v4()); + let expired_bootstrap = Uuid::new_v4(); + let expired_ready = Arc::new(DashSet::new()); + let expired = OperationRestoreRuntime(Arc::new(RestoreInner { + writer_db: test_db(), + source: source.clone(), + store: store.clone(), + bootstrap_ids: Arc::new(HashMap::from([(expired_domain, expired_bootstrap)])), + ready_domains: expired_ready, + lease: Duration::from_millis(1), + retention: test_retention(), + fence_pool: test_fence_pool(), + fence_slots: Arc::new(Semaphore::new(1)), + test_fences: Some(Arc::new(DashSet::new())), + healthy: AtomicBool::new(true), + })); + expired + .provision_domain(expired_domain) + .await + .expect("provision expired domain"); + let operation = Uuid::new_v4(); + let request = [2; 32]; + expired + .begin(expired_domain, operation, request) + .await + .expect("acquire expiring owner"); + source + .authoritative_now_ms + .store(1_000_002, Ordering::Release); + let expired_restart = OperationRestoreRuntime(Arc::new(RestoreInner { + writer_db: test_db(), + source, + store, + bootstrap_ids: Arc::new(HashMap::from([(expired_domain, expired_bootstrap)])), + ready_domains: Arc::new(DashSet::new()), + lease: Duration::from_millis(1), + retention: test_retention(), + fence_pool: test_fence_pool(), + fence_slots: Arc::new(Semaphore::new(1)), + test_fences: Some(Arc::new(DashSet::new())), + healthy: AtomicBool::new(true), + })); + expired_restart + .verify_domain(expired_domain) + .await + .expect("expired missing attribution is consumed as abort"); + assert_eq!( + expired_restart + .reconcile(expired_domain, operation, request) + .await, + Err(OperationRestoreError::OperationAborted) + ); + assert!(expired_restart.is_healthy()); + } + + #[tokio::test] + async fn startup_rejects_malformed_operation_records_and_latches_health() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(false), + }); + let store = Arc::new(MemoryStore::default()); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let bootstrap_id = Uuid::new_v4(); + let provisioner = unready_runtime(source.clone(), store.clone(), domain, bootstrap_id); + provisioner + .provision_domain(domain) + .await + .expect("provision bootstrap"); + store.records.lock().expect("records").insert( + operation_shard_key(domain, 0), + (1, br#"{"unexpected":"record"}"#.to_vec()), + ); + let restarted = unready_runtime(source, store, domain, bootstrap_id); + assert_eq!( + restarted.verify_domain(domain).await, + Err(OperationRestoreError::InvalidAttribution) + ); + assert!(!restarted.is_healthy()); + } + + #[tokio::test] + async fn immutable_shards_reject_missing_remapped_and_oversized_state() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(false), + }); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let bootstrap_id = Uuid::new_v4(); + + let missing_store = Arc::new(MemoryStore::default()); + let provisioner = + unready_runtime(source.clone(), missing_store.clone(), domain, bootstrap_id); + provisioner + .provision_domain(domain) + .await + .expect("provision every immutable shard"); + missing_store + .records + .lock() + .expect("records") + .remove(&operation_shard_key(domain, 2)); + let missing = unready_runtime(source.clone(), missing_store, domain, bootstrap_id); + assert_eq!( + missing.verify_domain(domain).await, + Err(OperationRestoreError::MissingOperationShard) + ); + assert!(!missing.is_healthy()); + + let remap_store = Arc::new(MemoryStore::default()); + let original = unready_runtime(source.clone(), remap_store.clone(), domain, bootstrap_id); + original + .provision_domain(domain) + .await + .expect("provision original layout"); + let remapped_policy = OperationRestoreRetentionPolicy::new(8, 8, MIN_TERMINAL_RETENTION) + .expect("valid alternative layout"); + let remapped = unready_runtime_with_policy( + source.clone(), + remap_store, + domain, + bootstrap_id, + remapped_policy, + ); + assert_eq!( + remapped.verify_domain(domain).await, + Err(OperationRestoreError::InvalidBootstrap) + ); + assert!(!remapped.is_healthy()); + + let oversized_store = Arc::new(MemoryStore::default()); + let oversized_provisioner = unready_runtime( + source.clone(), + oversized_store.clone(), + domain, + bootstrap_id, + ); + oversized_provisioner + .provision_domain(domain) + .await + .expect("provision bounded records"); + oversized_store.records.lock().expect("records").insert( + operation_shard_key(domain, 0), + ( + 1, + vec![b'x'; usize::try_from(MAX_RECORD_BYTES + 1).expect("bounded")], + ), + ); + let oversized = unready_runtime(source, oversized_store, domain, bootstrap_id); + assert_eq!( + oversized.verify_domain(domain).await, + Err(OperationRestoreError::StoreUnavailable) + ); + assert!(!oversized.is_healthy()); + } + + #[tokio::test] + async fn hot_shard_capacity_and_cas_contention_are_healthy_pre_db_denials() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(false), + }); + let policy = OperationRestoreRetentionPolicy::new(1, 1, MIN_TERMINAL_RETENTION) + .expect("one-slot policy"); + let store = Arc::new(MemoryStore::default()); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let runtime = runtime_with_policy( + source.clone(), + store.clone(), + domain, + Duration::from_secs(30), + policy, + ); + let first_operation = Uuid::new_v4(); + let OperationRestoreBegin::Acquired(first) = runtime + .begin(domain, first_operation, [1; 32]) + .await + .expect("reserve only slot") + else { + panic!("acquired") + }; + assert!(matches!( + runtime.begin(domain, Uuid::new_v4(), [2; 32]).await, + Err(OperationRestoreError::OperationCapacityExceeded) + )); + assert!(runtime.is_healthy()); + assert!(source.manifests.lock().expect("manifests").is_empty()); + assert!(store + .records + .lock() + .expect("records") + .keys() + .all(|key| !key.contains("/floors/"))); + + runtime.abort(first).await.expect("terminalize first slot"); + assert!(matches!( + runtime.begin(domain, Uuid::new_v4(), [3; 32]).await, + Err(OperationRestoreError::OperationCapacityExceeded) + )); + source.authoritative_now_ms.store( + 1_000_000 + policy.retention_millis().expect("retention"), + Ordering::Release, + ); + store.forced_cas_losses.store(1, Ordering::Release); + assert!(matches!( + runtime + .begin(domain, Uuid::new_v4(), [4; 32]) + .await + .expect("reload after lost compaction CAS"), + OperationRestoreBegin::Acquired(_) + )); + assert!(runtime.is_healthy()); + + let contention_store = Arc::new(MemoryStore::default()); + let contention_domain = CommunityId::from_uuid(Uuid::new_v4()); + let contention = runtime_with_policy( + source, + contention_store.clone(), + contention_domain, + Duration::from_secs(30), + policy, + ); + contention_store + .forced_cas_losses + .store(MAX_CAS_ATTEMPTS as u64, Ordering::Release); + assert!(matches!( + contention + .begin(contention_domain, Uuid::new_v4(), [5; 32]) + .await, + Err(OperationRestoreError::OperationContention) + )); + assert!(contention.is_healthy()); + + let concurrent_source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(2_000_000), + valid_supersession: AtomicBool::new(false), + }); + let concurrent_store = Arc::new(MemoryStore::default()); + let concurrent_domain = CommunityId::from_uuid(Uuid::new_v4()); + let two_shards = OperationRestoreRetentionPolicy::new(2, 1, MIN_TERMINAL_RETENTION) + .expect("two one-slot shards"); + let concurrent = runtime_with_policy( + concurrent_source, + concurrent_store, + concurrent_domain, + Duration::from_secs(30), + two_shards, + ); + let bootstrap = concurrent + .bootstrap_id(concurrent_domain) + .expect("bootstrap"); + let first = Uuid::new_v4(); + let first_shard = operation_shard_id( + OperationIdentity { + domain: concurrent_domain, + operation_id: first, + request_fingerprint: [6; 32], + }, + bootstrap, + two_shards.shard_count, + ); + let (same_shard, free_shard) = loop { + let same = Uuid::new_v4(); + let free = Uuid::new_v4(); + let same_id = OperationIdentity { + domain: concurrent_domain, + operation_id: same, + request_fingerprint: [7; 32], + }; + let free_id = OperationIdentity { + domain: concurrent_domain, + operation_id: free, + request_fingerprint: [8; 32], + }; + if operation_shard_id(same_id, bootstrap, two_shards.shard_count) == first_shard + && operation_shard_id(free_id, bootstrap, two_shards.shard_count) != first_shard + { + break (same, free); + } + }; + let first_begin = concurrent.begin(concurrent_domain, first, [6; 32]); + let same_begin = concurrent.begin(concurrent_domain, same_shard, [7; 32]); + let (first_result, same_result) = tokio::join!(first_begin, same_begin); + assert!(matches!( + (&first_result, &same_result), + ( + Ok(OperationRestoreBegin::Acquired(_)), + Err(OperationRestoreError::OperationCapacityExceeded) + ) | ( + Err(OperationRestoreError::OperationCapacityExceeded), + Ok(OperationRestoreBegin::Acquired(_)) + ) + )); + assert!(matches!( + concurrent + .begin(concurrent_domain, free_shard, [8; 32]) + .await + .expect("independently mapped free shard remains available"), + OperationRestoreBegin::Acquired(_) + )); + assert!(concurrent.is_healthy()); + } + + #[tokio::test] + async fn startup_resolves_reverse_shard_multistep_chain_without_supersession_guessing() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(true), + }); + let store = Arc::new(MemoryStore::default()); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let bootstrap_id = Uuid::new_v4(); + let original = unready_runtime(source.clone(), store.clone(), domain, bootstrap_id); + original + .provision_domain(domain) + .await + .expect("provision domain"); + + let mut by_shard = BTreeMap::new(); + while by_shard.len() < 3 { + let operation = Uuid::new_v4(); + let identity = OperationIdentity { + domain, + operation_id: operation, + request_fingerprint: [u8::try_from(by_shard.len() + 1).expect("small"); 32], + }; + by_shard + .entry(operation_shard_id( + identity, + bootstrap_id, + test_retention().shard_count, + )) + .or_insert(operation); + } + let operations: Vec<_> = by_shard.into_values().rev().collect(); + let key = [44; 32]; + for (index, operation) in operations.iter().copied().enumerate() { + let request = [u8::try_from(index + 1).expect("small"); 32]; + assert!(matches!( + original + .begin(domain, operation, request) + .await + .expect("reserve pending chain member"), + OperationRestoreBegin::Acquired(_) + )); + source.manifests.lock().expect("manifests").insert( + (operation, request), + manifest( + domain, + operation, + request, + key, + index as u64, + index as u64 + 1, + ), + ); + } + *source.floors.lock().expect("floors") = vec![AuthorizationVersionComponentFloor { + component_kind: AuthorizationVersionComponentKind::Binding, + component_key: key, + version: 3, + }]; + let restarted = unready_runtime(source, store, domain, bootstrap_id); + restarted + .verify_domain(domain) + .await + .expect("causal passes reconcile reverse shard scan order"); + for (index, operation) in operations.iter().copied().enumerate() { + let request = [u8::try_from(index + 1).expect("small"); 32]; + assert!(matches!( + restarted.begin(domain, operation, request).await, + Ok(OperationRestoreBegin::ExactReplay(_)) + )); + } + } + + #[tokio::test] + async fn startup_resolves_reverse_same_shard_predecessor_order() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(true), + }); + let store = Arc::new(MemoryStore::default()); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let bootstrap = Uuid::new_v4(); + let policy = OperationRestoreRetentionPolicy::new(1, 8, MIN_TERMINAL_RETENTION) + .expect("single shard policy"); + let original = + unready_runtime_with_policy(source.clone(), store.clone(), domain, bootstrap, policy); + original.provision_domain(domain).await.expect("provision"); + let successor = Uuid::from_u128(1); + let predecessor = Uuid::from_u128(2); + let key = [61_u8; 32]; + original + .begin(domain, successor, [1_u8; 32]) + .await + .expect("reserve lexically first successor"); + original + .begin(domain, predecessor, [2_u8; 32]) + .await + .expect("reserve predecessor"); + source.manifests.lock().expect("manifests").extend([ + ( + (successor, [1_u8; 32]), + manifest(domain, successor, [1_u8; 32], key, 1, 2), + ), + ( + (predecessor, [2_u8; 32]), + manifest(domain, predecessor, [2_u8; 32], key, 0, 1), + ), + ]); + *source.floors.lock().expect("floors") = vec![AuthorizationVersionComponentFloor { + component_kind: AuthorizationVersionComponentKind::Binding, + component_key: key, + version: 2, + }]; + let restarted = unready_runtime_with_policy(source, store, domain, bootstrap, policy); + restarted + .verify_domain(domain) + .await + .expect("fixed-point scan finds predecessor after successor"); + assert!(restarted.is_healthy()); + } + + #[tokio::test] + async fn begin_recovers_a_three_operation_chain_across_three_shards() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(true), + }); + let store = Arc::new(MemoryStore::default()); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let policy = OperationRestoreRetentionPolicy::new(8, 8, MIN_TERMINAL_RETENTION) + .expect("multi-shard policy"); + let runtime = runtime_with_policy( + source.clone(), + store.clone(), + domain, + Duration::from_millis(1), + policy, + ); + let bootstrap = runtime.bootstrap_id(domain).expect("bootstrap"); + let request_a = [66_u8; 32]; + let request_b = [67_u8; 32]; + let request_c = [68_u8; 32]; + let operation_a = Uuid::from_u128(1); + let shard_a = operation_shard_id( + OperationIdentity { + domain, + operation_id: operation_a, + request_fingerprint: request_a, + }, + bootstrap, + policy.shard_count, + ); + let mut nonce = 2_u128; + let operation_b = loop { + let candidate = Uuid::from_u128(nonce); + nonce += 1; + let shard = operation_shard_id( + OperationIdentity { + domain, + operation_id: candidate, + request_fingerprint: request_b, + }, + bootstrap, + policy.shard_count, + ); + if shard != shard_a { + break candidate; + } + }; + let shard_b = operation_shard_id( + OperationIdentity { + domain, + operation_id: operation_b, + request_fingerprint: request_b, + }, + bootstrap, + policy.shard_count, + ); + let operation_c = loop { + let candidate = Uuid::from_u128(nonce); + nonce += 1; + let shard = operation_shard_id( + OperationIdentity { + domain, + operation_id: candidate, + request_fingerprint: request_c, + }, + bootstrap, + policy.shard_count, + ); + if shard != shard_a && shard != shard_b { + break candidate; + } + }; + let shard_c = operation_shard_id( + OperationIdentity { + domain, + operation_id: operation_c, + request_fingerprint: request_c, + }, + bootstrap, + policy.shard_count, + ); + runtime + .begin(domain, operation_a, request_a) + .await + .expect("reserve predecessor"); + runtime + .begin(domain, operation_b, request_b) + .await + .expect("reserve successor"); + runtime + .begin(domain, operation_c, request_c) + .await + .expect("reserve final successor"); + let key = [69_u8; 32]; + source.manifests.lock().expect("manifests").extend([ + ( + (operation_a, request_a), + manifest(domain, operation_a, request_a, key, 0, 1), + ), + ( + (operation_b, request_b), + manifest(domain, operation_b, request_b, key, 1, 2), + ), + ( + (operation_c, request_c), + manifest(domain, operation_c, request_c, key, 2, 3), + ), + ]); + source + .authoritative_now_ms + .store(1_000_002, Ordering::Release); + + let successor = runtime.begin(domain, operation_c, request_c).await; + assert!( + matches!(successor, Ok(OperationRestoreBegin::ExactReplay(_))), + "unexpected successor result: {successor:?}" + ); + assert!(matches!( + runtime.begin(domain, operation_b, request_b).await, + Ok(OperationRestoreBegin::ExactReplay(_)) + )); + assert!(matches!( + runtime.begin(domain, operation_a, request_a).await, + Ok(OperationRestoreBegin::ExactReplay(_)) + )); + let records = store.records.lock().expect("records"); + for (operation, shard) in [ + (operation_a, shard_a), + (operation_b, shard_b), + (operation_c, shard_c), + ] { + let encoded = &records + .get(&operation_shard_key(domain, shard)) + .expect("operation shard") + .1; + let shard: OperationShardRecord = + serde_json::from_slice(encoded).expect("decode operation shard"); + assert_eq!( + shard + .entries + .get(&operation.to_string()) + .expect("intent") + .state, + IntentState::Committed + ); + } + let encoded = &records + .get(&floor_key( + domain, + AuthorizationVersionComponentKind::Binding, + key, + )) + .expect("component floor") + .1; + let floor: FloorRecord = serde_json::from_slice(encoded).expect("decode floor"); + assert_eq!(floor.version, 3); + assert_eq!(floor.operation_id, Some(operation_c.to_string())); + drop(records); + assert!(runtime.is_healthy()); + } + + #[tokio::test] + async fn expired_writer_fence_blocks_reaper_without_poisoning_health() { + let (runtime, source, _, domain) = fixture(); + let identity = OperationIdentity { + domain, + operation_id: Uuid::new_v4(), + request_fingerprint: [62_u8; 32], + }; + let fenced = runtime + .begin_fenced(identity) + .await + .expect("writer owns fence before Pending"); + source.authoritative_now_ms.store( + 1_000_000 + u64::try_from(DEFAULT_LEASE.as_millis()).expect("lease") + 1, + Ordering::Release, + ); + let shard = operation_shard_id( + identity, + runtime.bootstrap_id(domain).expect("bootstrap"), + runtime.0.retention.shard_count, + ); + let started = tokio::time::Instant::now(); + assert_eq!( + runtime.maintain_begin_shard(domain, shard).await, + Err(OperationRestoreError::CausalLag) + ); + assert!(started.elapsed() >= Duration::from_millis(140)); + assert!(runtime.is_healthy()); + drop(fenced); + let reacquired = runtime + .acquire_operation_fence(identity) + .await + .expect("cancelled writer drop releases the fence"); + reacquired.unlock().await.expect("explicit test unlock"); + } + + #[tokio::test] + async fn many_contended_fences_obey_one_shard_deadline() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(false), + }); + let store = Arc::new(MemoryStore::default()); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let policy = OperationRestoreRetentionPolicy::new(1, 64, MIN_TERMINAL_RETENTION) + .expect("one full production-sized shard"); + let runtime = runtime_with_policy( + source.clone(), + store, + domain, + Duration::from_millis(1), + policy, + ); + let fences = runtime.0.test_fences.as_ref().expect("test fences"); + for nonce in 1..=u128::from(policy.entries_per_shard) { + let identity = OperationIdentity { + domain, + operation_id: Uuid::from_u128(nonce), + request_fingerprint: [65_u8; 32], + }; + runtime + .begin(domain, identity.operation_id, identity.request_fingerprint) + .await + .expect("reserve pending contender"); + assert!(fences.insert(operation_fence_test_key(identity))); + } + source + .authoritative_now_ms + .store(1_000_002, Ordering::Release); + + let started = tokio::time::Instant::now(); + assert_eq!( + runtime.maintain_begin_shard(domain, 0).await, + Err(OperationRestoreError::CausalLag) + ); + assert!(started.elapsed() >= Duration::from_millis(120)); + assert!(started.elapsed() < Duration::from_millis(300)); + assert!(runtime.is_healthy()); + } + + #[tokio::test] + async fn fixed_production_shard_saturation_is_healthy_and_pre_database() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(false), + }); + let store = Arc::new(MemoryStore::default()); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let policy = OperationRestoreRetentionPolicy::production_default(); + let runtime = runtime_with_policy(source.clone(), store, domain, DEFAULT_LEASE, policy); + let bootstrap = runtime.bootstrap_id(domain).expect("bootstrap"); + let target_shard = 0; + let mut filled = 0_u16; + let mut nonce = 1_u128; + while filled < policy.entries_per_shard { + let operation = Uuid::from_u128(nonce); + nonce += 1; + let identity = OperationIdentity { + domain, + operation_id: operation, + request_fingerprint: [63_u8; 32], + }; + if operation_shard_id(identity, bootstrap, policy.shard_count) != target_shard { + continue; + } + runtime + .begin(domain, operation, identity.request_fingerprint) + .await + .expect("fill exact production shard"); + filled += 1; + } + let saturated = loop { + let operation = Uuid::from_u128(nonce); + nonce += 1; + let identity = OperationIdentity { + domain, + operation_id: operation, + request_fingerprint: [64_u8; 32], + }; + if operation_shard_id(identity, bootstrap, policy.shard_count) == target_shard { + break identity; + } + }; + assert!(matches!( + runtime + .begin( + domain, + saturated.operation_id, + saturated.request_fingerprint, + ) + .await, + Err(OperationRestoreError::OperationCapacityExceeded) + )); + assert!(runtime.is_healthy()); + assert!(source.manifests.lock().expect("manifests").is_empty()); + } + + #[tokio::test] + async fn pending_operation_terminalizes_only_after_authoritative_supersession_proof() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(true), + }); + let store = Arc::new(MemoryStore::default()); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let bootstrap_id = Uuid::new_v4(); + let original = unready_runtime(source.clone(), store.clone(), domain, bootstrap_id); + original + .provision_domain(domain) + .await + .expect("provision domain"); + let pending_operation = Uuid::new_v4(); + let pending_request = [61; 32]; + let key = [62; 32]; + assert!(matches!( + original + .begin(domain, pending_operation, pending_request) + .await + .expect("reserve pending operation"), + OperationRestoreBegin::Acquired(_) + )); + source.manifests.lock().expect("manifests").insert( + (pending_operation, pending_request), + manifest(domain, pending_operation, pending_request, key, 0, 1), + ); + let latest = manifest(domain, Uuid::new_v4(), [63; 32], key, 2, 3); + let latest_component = latest.components[0].clone(); + store.records.lock().expect("records").insert( + floor_key(domain, AuthorizationVersionComponentKind::Binding, key), + ( + 1, + serde_json::to_vec(&FloorRecord::advanced( + bootstrap_id, + &latest, + &latest_component, + )) + .expect("encode proven latest floor"), + ), + ); + *source.floors.lock().expect("floors") = vec![AuthorizationVersionComponentFloor { + component_kind: AuthorizationVersionComponentKind::Binding, + component_key: key, + version: 3, + }]; + + let restarted = unready_runtime(source, store, domain, bootstrap_id); + restarted + .verify_domain(domain) + .await + .expect("complete provenance proves supersession"); + let OperationRestoreBegin::ExactReplay(replay) = restarted + .begin(domain, pending_operation, pending_request) + .await + .expect("replay superseded operation") + else { + panic!("exact replay") + }; + assert!(replay.causally_superseded); + } + + #[tokio::test] + async fn exact_replay_fails_closed_when_postgres_attribution_disappears() { + let (runtime, source, _, domain) = fixture(); + let operation = Uuid::new_v4(); + let request = [1; 32]; + source.manifests.lock().expect("manifests").insert( + (operation, request), + manifest(domain, operation, request, [2; 32], 0, 1), + ); + let OperationRestoreBegin::Acquired(intent) = runtime + .begin(domain, operation, request) + .await + .expect("begin") + else { + panic!("acquired"); + }; + runtime.commit(intent).await.expect("commit"); + source + .manifests + .lock() + .expect("manifests") + .remove(&(operation, request)); + assert!(matches!( + runtime.begin(domain, operation, request).await, + Err(OperationRestoreError::MissingAttribution) + )); + assert!(!runtime.is_healthy()); + } + + #[tokio::test] + async fn exact_replay_revalidates_its_external_component_floor() { + let (runtime, source, store, domain) = fixture(); + let operation = Uuid::new_v4(); + let request = [1; 32]; + let component_key = [2; 32]; + source.manifests.lock().expect("manifests").insert( + (operation, request), + manifest(domain, operation, request, component_key, 0, 1), + ); + let OperationRestoreBegin::Acquired(intent) = runtime + .begin(domain, operation, request) + .await + .expect("begin") + else { + panic!("acquired"); + }; + runtime.commit(intent).await.expect("commit"); + store.records.lock().expect("records").remove(&floor_key( + domain, + AuthorizationVersionComponentKind::Binding, + component_key, + )); + + assert!(matches!( + runtime.begin(domain, operation, request).await, + Err(OperationRestoreError::MissingFloor) + )); + assert!(!runtime.is_healthy()); + } + + #[tokio::test] + async fn lost_response_reconcile_commits_and_consumes_the_original_owner() { + let (runtime, source, _, domain) = fixture(); + let operation = Uuid::new_v4(); + let request = [1; 32]; + source.manifests.lock().expect("manifests").insert( + (operation, request), + manifest(domain, operation, request, [2; 32], 0, 1), + ); + let OperationRestoreBegin::Acquired(intent) = runtime + .begin(domain, operation, request) + .await + .expect("begin") + else { + panic!("acquired"); + }; + let reconciled = runtime + .reconcile(domain, operation, request) + .await + .expect("reconcile exact committed PostgreSQL operation"); + assert!(!reconciled.replay); + assert_eq!( + runtime.abort(intent).await, + Err(OperationRestoreError::AlreadyCommitted) + ); + assert!(matches!( + runtime + .reconcile(domain, operation, request) + .await + .expect("exact reconcile replay"), + OperationRestoreCommit { replay: true, .. } + )); + } + + #[tokio::test] + async fn missing_nonzero_floor_and_backward_floor_fail_closed() { + let (runtime, source, _, domain) = fixture(); + let operation = Uuid::new_v4(); + let request = [1; 32]; + source.manifests.lock().expect("manifests").insert( + (operation, request), + manifest(domain, operation, request, [2; 32], 7, 8), + ); + let OperationRestoreBegin::Acquired(intent) = runtime + .begin(domain, operation, request) + .await + .expect("begin") + else { + panic!("acquired"); + }; + assert_eq!( + runtime.commit(intent).await, + Err(OperationRestoreError::MissingFloor) + ); + assert!(!runtime.is_healthy()); + + let (runtime, source, _, domain) = fixture(); + *source.floors.lock().expect("floors") = vec![AuthorizationVersionComponentFloor { + component_kind: AuthorizationVersionComponentKind::Binding, + component_key: [2; 32], + version: 9, + }]; + runtime.provision_domain(domain).await.expect("provision"); + source.floors.lock().expect("floors")[0].version = 8; + assert_eq!( + runtime.provision_domain(domain).await, + Err(OperationRestoreError::BackwardFloor) + ); + assert!(!runtime.is_healthy()); + } + + #[tokio::test] + async fn every_component_kind_anchors_first_advance_to_absent_zero() { + let kinds = [ + AuthorizationVersionComponentKind::Binding, + AuthorizationVersionComponentKind::Policy, + AuthorizationVersionComponentKind::InvalidationGeneration, + AuthorizationVersionComponentKind::AuthorityEpoch, + AuthorizationVersionComponentKind::DelegatedRelationship, + AuthorizationVersionComponentKind::LifecycleSelector, + ]; + for (index, kind) in kinds.into_iter().enumerate() { + let (runtime, source, _, domain) = fixture(); + let operation = Uuid::new_v4(); + let mut request = [1; 32]; + request[0] = u8::try_from(index + 1).expect("bounded kind index"); + let mut key = [2; 32]; + key[0] = request[0]; + source.manifests.lock().expect("manifests").insert( + (operation, request), + manifest_for_kind(domain, operation, request, kind, key, 1, 2), + ); + let OperationRestoreBegin::Acquired(intent) = runtime + .begin(domain, operation, request) + .await + .expect("begin gap attempt") + else { + panic!("acquired"); + }; + assert_eq!( + runtime.commit(intent).await, + Err(OperationRestoreError::MissingFloor), + "{kind:?} accepted an unprovisioned nonzero first floor" + ); + + let (runtime, source, _, domain) = fixture(); + let operation = Uuid::new_v4(); + source.manifests.lock().expect("manifests").insert( + (operation, request), + manifest_for_kind(domain, operation, request, kind, key, 0, 1), + ); + let OperationRestoreBegin::Acquired(intent) = runtime + .begin(domain, operation, request) + .await + .expect("begin exact absent baseline") + else { + panic!("acquired"); + }; + assert!(runtime.commit(intent).await.is_ok(), "{kind:?}"); + } + } + + #[tokio::test] + async fn wrong_key_record_is_rejected_and_health_is_sticky() { + let (runtime, source, store, domain) = fixture(); + let operation = Uuid::new_v4(); + let request = [1; 32]; + let key = [2; 32]; + source.manifests.lock().expect("manifests").insert( + (operation, request), + manifest(domain, operation, request, key, 0, 1), + ); + let wrong = FloorRecord { + component_key: hex::encode([3; 32]), + ..FloorRecord::advanced( + runtime.bootstrap_id(domain).expect("configured bootstrap"), + source + .manifests + .lock() + .expect("manifests") + .get(&(operation, request)) + .expect("manifest"), + &ManifestComponent { + kind: AuthorizationVersionComponentKind::Binding, + key, + before: 0, + after: 1, + digest: [9; 32], + }, + ) + }; + store.records.lock().expect("records").insert( + floor_key(domain, AuthorizationVersionComponentKind::Binding, key), + (1, serde_json::to_vec(&wrong).expect("encode")), + ); + let OperationRestoreBegin::Acquired(intent) = runtime + .begin(domain, operation, request) + .await + .expect("begin") + else { + panic!("acquired"); + }; + assert_eq!( + runtime.commit(intent).await, + Err(OperationRestoreError::WrongComponentKey) + ); + assert!(!runtime.is_healthy()); + assert!(matches!( + runtime.begin(domain, Uuid::new_v4(), [4; 32]).await, + Err(OperationRestoreError::Unhealthy) + )); + } + + #[tokio::test] + async fn abort_is_owner_checked_and_missing_manifest_is_consumed() { + let (runtime, _, _, domain) = fixture(); + let operation = Uuid::new_v4(); + let request = [1; 32]; + let OperationRestoreBegin::Acquired(intent) = runtime + .begin(domain, operation, request) + .await + .expect("begin") + else { + panic!("acquired"); + }; + let wrong_owner = OperationRestoreIntent { + identity: intent.identity, + owner_token: Uuid::new_v4(), + shard_id: intent.shard_id, + }; + assert_eq!( + runtime.abort(wrong_owner).await, + Err(OperationRestoreError::OwnerMismatch) + ); + assert!(!runtime.is_healthy()); + assert_eq!( + runtime.abort(intent).await, + Err(OperationRestoreError::Unhealthy) + ); + + let (runtime, _, _, domain) = fixture(); + let operation = Uuid::new_v4(); + let OperationRestoreBegin::Acquired(intent) = runtime + .begin(domain, operation, request) + .await + .expect("begin") + else { + panic!("acquired"); + }; + assert_eq!( + runtime.commit(intent).await, + Err(OperationRestoreError::MissingAttribution) + ); + assert!(!runtime.is_healthy()); + } + + #[tokio::test] + async fn expired_missing_intent_is_retained_then_reacquired_after_pg_retention() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(false), + }); + let store = Arc::new(MemoryStore::default()); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let runtime = runtime_with_lease(source.clone(), store, domain, Duration::from_millis(1)); + let operation = Uuid::new_v4(); + let request = [1; 32]; + let OperationRestoreBegin::Acquired(first) = runtime + .begin(domain, operation, request) + .await + .expect("first owner") + else { + panic!("acquired"); + }; + source + .authoritative_now_ms + .store(1_000_002, Ordering::Release); + assert!(matches!( + runtime.begin(domain, operation, request).await, + Err(OperationRestoreError::OperationAborted) + )); + source.authoritative_now_ms.store( + 1_000_002 + test_retention().retention_millis().expect("retention"), + Ordering::Release, + ); + let OperationRestoreBegin::Acquired(second) = runtime + .begin(domain, operation, request) + .await + .expect("replacement owner after terminal retention") + else { + panic!("reacquired"); + }; + assert_ne!(first.owner_token, second.owner_token); + assert_eq!( + runtime.abort(first).await, + Err(OperationRestoreError::OwnerMismatch) + ); + assert!(!runtime.is_healthy()); + assert_eq!( + runtime.abort(second).await, + Err(OperationRestoreError::Unhealthy) + ); + } + + #[test] + fn restore_debug_surfaces_are_redacted() { + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let operation_id = Uuid::new_v4(); + let owner_token = Uuid::new_v4(); + let intent = OperationRestoreIntent { + identity: OperationIdentity { + domain, + operation_id, + request_fingerprint: [7; 32], + }, + owner_token, + shard_id: 0, + }; + let rendered = format!("{intent:?}"); + assert!(!rendered.contains(&domain.as_uuid().to_string())); + assert!(!rendered.contains(&operation_id.to_string())); + assert!(!rendered.contains(&owner_token.to_string())); + assert!(!rendered.contains(&hex::encode([7; 32]))); + } + + #[tokio::test] + async fn unrelated_operations_advance_independent_components_concurrently() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(false), + }); + let store = Arc::new(MemoryStore { + records: Mutex::new(HashMap::new()), + fail: AtomicBool::new(false), + forced_cas_losses: AtomicU64::new(0), + floor_barrier: Some(Arc::new(tokio::sync::Barrier::new(2))), + }); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let runtime = runtime(source.clone(), store, domain); + let first_operation = Uuid::new_v4(); + let second_operation = Uuid::new_v4(); + let first_request = [1; 32]; + let second_request = [2; 32]; + source.manifests.lock().expect("manifests").extend([ + ( + (first_operation, first_request), + manifest(domain, first_operation, first_request, [3; 32], 0, 1), + ), + ( + (second_operation, second_request), + manifest(domain, second_operation, second_request, [4; 32], 0, 1), + ), + ]); + let OperationRestoreBegin::Acquired(first) = runtime + .begin(domain, first_operation, first_request) + .await + .expect("first begin") + else { + panic!("first acquired"); + }; + let OperationRestoreBegin::Acquired(second) = runtime + .begin(domain, second_operation, second_request) + .await + .expect("second begin") + else { + panic!("second acquired"); + }; + let (first_result, second_result) = tokio::time::timeout(Duration::from_secs(1), async { + tokio::join!(runtime.commit(first), runtime.commit(second)) + }) + .await + .expect("unrelated components reach their independent CAS without a domain mutex"); + assert!(first_result.is_ok()); + assert!(second_result.is_ok()); + assert!(runtime.is_healthy()); + } + + #[tokio::test] + async fn later_same_component_commit_waits_for_earlier_witness_floor() { + let (runtime, source, _, domain) = fixture(); + let key = [3; 32]; + let first_operation = Uuid::new_v4(); + let second_operation = Uuid::new_v4(); + let first_request = [1; 32]; + let second_request = [2; 32]; + source.manifests.lock().expect("manifests").extend([ + ( + (first_operation, first_request), + manifest(domain, first_operation, first_request, key, 0, 1), + ), + ( + (second_operation, second_request), + manifest(domain, second_operation, second_request, key, 1, 2), + ), + ]); + *source.floors.lock().expect("floors") = vec![AuthorizationVersionComponentFloor { + component_kind: AuthorizationVersionComponentKind::Binding, + component_key: key, + version: 2, + }]; + let OperationRestoreBegin::Acquired(first) = runtime + .begin(domain, first_operation, first_request) + .await + .expect("first begin") + else { + panic!("first acquired"); + }; + let OperationRestoreBegin::Acquired(second) = runtime + .begin(domain, second_operation, second_request) + .await + .expect("second begin") + else { + panic!("second acquired"); + }; + let (second_result, first_result) = tokio::time::timeout(Duration::from_secs(1), async { + tokio::join!(runtime.commit(second), async { + tokio::time::sleep(Duration::from_millis(25)).await; + runtime.commit(first).await + }) + }) + .await + .expect("bounded same-component reconciliation"); + assert!(first_result.is_ok()); + assert!(second_result.is_ok()); + assert!(runtime.is_healthy()); + } + + #[tokio::test] + async fn stale_gap_is_not_laundered_by_a_later_operation() { + let (runtime, source, _, domain) = fixture(); + *source.floors.lock().expect("floors") = vec![AuthorizationVersionComponentFloor { + component_kind: AuthorizationVersionComponentKind::Binding, + component_key: [2; 32], + version: 7, + }]; + runtime.provision_domain(domain).await.expect("provision"); + let operation = Uuid::new_v4(); + let request = [1; 32]; + source.manifests.lock().expect("manifests").insert( + (operation, request), + manifest(domain, operation, request, [2; 32], 8, 9), + ); + let OperationRestoreBegin::Acquired(intent) = runtime + .begin(domain, operation, request) + .await + .expect("begin") + else { + panic!("acquired"); + }; + assert_eq!( + runtime.commit(intent).await, + Err(OperationRestoreError::StaleFloor) + ); + assert!(!runtime.is_healthy()); + } + + #[tokio::test] + async fn causal_lag_past_the_bound_is_healthy_and_retains_pending() { + let (runtime, source, _, domain) = fixture(); + let key = [27_u8; 32]; + *source.floors.lock().expect("floors") = vec![AuthorizationVersionComponentFloor { + component_kind: AuthorizationVersionComponentKind::Binding, + component_key: key, + version: 7, + }]; + runtime.provision_domain(domain).await.expect("provision"); + let operation = Uuid::new_v4(); + let request = [28_u8; 32]; + source.manifests.lock().expect("manifests").insert( + (operation, request), + manifest(domain, operation, request, key, 8, 9), + ); + source.floors.lock().expect("floors")[0].version = 9; + let OperationRestoreBegin::Acquired(intent) = runtime + .begin(domain, operation, request) + .await + .expect("begin") + else { + panic!("acquired"); + }; + let started = tokio::time::Instant::now(); + assert_eq!( + runtime.commit(intent).await, + Err(OperationRestoreError::CausalLag) + ); + assert!(started.elapsed() >= Duration::from_millis(140)); + assert!(runtime.is_healthy()); + assert!(matches!( + runtime.begin(domain, operation, request).await, + Err(OperationRestoreError::OperationInProgress) + )); + } + + #[tokio::test] + async fn production_shard_capacity_fits_maximum_width_terminal_records() { + let source = Arc::new(MemorySource { + manifests: Mutex::new(HashMap::new()), + floors: Mutex::new(Vec::new()), + authoritative_now_ms: AtomicU64::new(1_000_000), + valid_supersession: AtomicBool::new(false), + }); + let store = Arc::new(MemoryStore::default()); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let runtime = runtime_with_policy( + source, + store, + domain, + DEFAULT_LEASE, + OperationRestoreRetentionPolicy::production_default(), + ); + let bootstrap = runtime.bootstrap_id(domain).expect("bootstrap"); + assert_eq!(runtime.0.retention.shard_count, 256); + assert_eq!(runtime.0.retention.entries_per_shard, 64); + assert_eq!( + runtime.0.retention.terminal_retention, + Duration::from_secs(24 * 60 * 60) + ); + runtime + .validate_worst_case_shard_size(domain, bootstrap) + .expect("maximum-width production shard remains below 64 KiB"); + } + + #[tokio::test] + async fn external_store_failure_is_sticky_and_fail_closed() { + let (runtime, _, store, domain) = fixture(); + store.fail.store(true, Ordering::Release); + assert!(matches!( + runtime.begin(domain, Uuid::new_v4(), [1; 32]).await, + Err(OperationRestoreError::StoreUnavailable) + )); + store.fail.store(false, Ordering::Release); + assert!(matches!( + runtime.begin(domain, Uuid::new_v4(), [2; 32]).await, + Err(OperationRestoreError::Unhealthy) + )); + } + + #[tokio::test] + async fn ambiguous_database_unavailable_keeps_the_exact_intent_pending() { + struct UnavailableManifestSource; + + #[async_trait] + impl ManifestSource for UnavailableManifestSource { + async fn authoritative_now_ms(&self) -> Result { + Ok(1_000_000) + } + + async fn manifest( + &self, + _identity: OperationIdentity, + ) -> Result { + Err(ManifestReadError::Unavailable) + } + + async fn floors( + &self, + _domain: CommunityId, + ) -> Result, ManifestReadError> { + Ok(Vec::new()) + } + + async fn predecessors( + &self, + _manifest: &OperationManifest, + ) -> Result, ManifestReadError> { + Err(ManifestReadError::Unavailable) + } + + async fn prove_supersession( + &self, + _pending: &OperationManifest, + _component: &ManifestComponent, + _floor: &FloorRecord, + ) -> Result<(), ManifestReadError> { + Err(ManifestReadError::Unavailable) + } + } + + let domain = CommunityId::from_uuid(Uuid::new_v4()); + let store = Arc::new(MemoryStore::default()); + let bootstrap_id = Uuid::new_v4(); + let retention = test_retention(); + for shard_id in 0..retention.shard_count { + store.records.lock().expect("records").insert( + operation_shard_key(domain, shard_id), + ( + 1, + serde_json::to_vec(&OperationShardRecord::empty( + domain, + bootstrap_id, + shard_id, + retention, + )) + .expect("encode shard"), + ), + ); + } + let ready = Arc::new(DashSet::new()); + ready.insert(domain); + let runtime = OperationRestoreRuntime(Arc::new(RestoreInner { + writer_db: test_db(), + source: Arc::new(UnavailableManifestSource), + store: store.clone(), + bootstrap_ids: Arc::new(HashMap::from([(domain, bootstrap_id)])), + ready_domains: ready, + lease: DEFAULT_LEASE, + retention, + fence_pool: test_fence_pool(), + fence_slots: Arc::new(Semaphore::new(1)), + test_fences: Some(Arc::new(DashSet::new())), + healthy: AtomicBool::new(true), + })); + let operation = Uuid::new_v4(); + let request = [1_u8; 32]; + let OperationRestoreBegin::Acquired(_intent) = runtime + .begin(domain, operation, request) + .await + .expect("acquire exact intent") + else { + panic!("acquired"); + }; + assert_eq!( + runtime.reconcile(domain, operation, request).await, + Err(OperationRestoreError::DatabaseUnavailable) + ); + let identity = OperationIdentity { + domain, + operation_id: operation, + request_fingerprint: request, + }; + let shard_id = operation_shard_id(identity, bootstrap_id, retention.shard_count); + let stored = store + .records + .lock() + .expect("records") + .get(&operation_shard_key(domain, shard_id)) + .expect("pending record remains") + .1 + .clone(); + let shard: OperationShardRecord = serde_json::from_slice(&stored).expect("decode shard"); + let record = shard + .entries + .get(&operation.to_string()) + .expect("pending record remains"); + assert_eq!(record.state, IntentState::Pending); + assert!(!runtime.is_healthy()); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn postgres_witness_read_completes_while_size_one_writer_pool_is_held() { + use sqlx::postgres::PgPoolOptions; + + let admin_url = std::env::var("TEST_DATABASE_URL") + .unwrap_or_else(|_| crate::authorization_version::loopback_test_database_url()); + let admin = sqlx::PgPool::connect(&admin_url) + .await + .expect("connect admin"); + let name = format!("s5_restore_pool_{}", Uuid::new_v4().simple()); + sqlx::query(sqlx::AssertSqlSafe(format!("CREATE DATABASE {name}"))) + .execute(&admin) + .await + .expect("create scratch database"); + let split = admin_url.rfind('/').expect("database URL path"); + let url = format!("{}/{}", &admin_url[..split], name); + let writer = PgPoolOptions::new() + .max_connections(1) + .connect(&url) + .await + .expect("writer pool"); + crate::migration::run_migrations(&writer) + .await + .expect("migrate"); + let witness = PgPoolOptions::new() + .max_connections(1) + .connect(&url) + .await + .expect("independent witness pool"); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + sqlx::query("INSERT INTO communities (id,host) VALUES ($1,$2)") + .bind(domain.as_uuid()) + .bind("restore-pool.example") + .execute(&writer) + .await + .expect("community"); + let held_writer = writer.acquire().await.expect("hold sole writer connection"); + let runtime = OperationRestoreRuntime(Arc::new(RestoreInner { + writer_db: Db::from_pool(writer.clone()), + source: Arc::new(PostgresManifestSource(Db::from_pool(witness.clone()))), + store: Arc::new(MemoryStore::default()), + bootstrap_ids: Arc::new(HashMap::from([(domain, Uuid::new_v4())])), + ready_domains: Arc::new(DashSet::new()), + lease: Duration::from_millis(1), + retention: test_retention(), + fence_pool: test_fence_pool(), + fence_slots: Arc::new(Semaphore::new(1)), + test_fences: Some(Arc::new(DashSet::new())), + healthy: AtomicBool::new(true), + })); + tokio::time::timeout(Duration::from_secs(2), runtime.provision_domain(domain)) + .await + .expect("witness must not wait for writer pool") + .expect("provision exact floors"); + let operation = Uuid::new_v4(); + assert!(matches!( + runtime + .begin(domain, operation, [91_u8; 32]) + .await + .expect("reserve with PostgreSQL clock"), + OperationRestoreBegin::Acquired(_) + )); + tokio::time::sleep(Duration::from_millis(10)).await; + assert!(matches!( + runtime.begin(domain, operation, [91_u8; 32]).await, + Err(OperationRestoreError::OperationAborted) + )); + assert!(runtime.is_healthy()); + drop(held_writer); + writer.close().await; + witness.close().await; + sqlx::query(sqlx::AssertSqlSafe(format!("DROP DATABASE {name}"))) + .execute(&admin) + .await + .expect("drop scratch database"); + admin.close().await; + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn postgres_restart_reconciles_crash_after_publication_commit() { + use crate::authorization_version::{ + protected_publication_request_for_restore_test, ProtectedPublicationDisposition, + ProtectedPublicationError, ProtectedPublicationOperationIdentity, + }; + use buzz_auth::AuthorizationEventCapacityPolicy; + use buzz_core::AuthorizationLeaseFence; + use nostr::Keys; + use sqlx::postgres::PgPoolOptions; + + let admin_url = std::env::var("TEST_DATABASE_URL") + .unwrap_or_else(|_| crate::authorization_version::loopback_test_database_url()); + let admin = sqlx::PgPool::connect(&admin_url) + .await + .expect("connect admin"); + let name = format!("s5_restore_restart_{}", Uuid::new_v4().simple()); + sqlx::query(sqlx::AssertSqlSafe(format!("CREATE DATABASE {name}"))) + .execute(&admin) + .await + .expect("create scratch database"); + let split = admin_url.rfind('/').expect("database URL path"); + let url = format!("{}/{}", &admin_url[..split], name); + let writer = PgPoolOptions::new() + .max_connections(3) + .connect(&url) + .await + .expect("writer pool"); + crate::migration::run_migrations(&writer) + .await + .expect("migrate"); + let witness = PgPoolOptions::new() + .max_connections(1) + .connect(&url) + .await + .expect("independent witness pool"); + let fence_probe = PgPoolOptions::new() + .max_connections(1) + .connect(&url) + .await + .expect("independent fence probe pool"); + let blocker_pool = PgPoolOptions::new() + .max_connections(1) + .connect(&url) + .await + .expect("independent blocker pool"); + let domain = CommunityId::from_uuid(Uuid::new_v4()); + sqlx::query("INSERT INTO communities (id,host) VALUES ($1,$2)") + .bind(domain.as_uuid()) + .bind(format!( + "restore-restart-{}.example", + domain.as_uuid().simple() + )) + .execute(&writer) + .await + .expect("community"); + sqlx::query( + "INSERT INTO identity_enrollment_policies \ + (community_id,policy_revision,enrollment_mode,policy_digest,effective_at) \ + VALUES ($1,1,1,$2,clock_timestamp() - INTERVAL '1 second')", + ) + .bind(domain.as_uuid()) + .bind(vec![1_u8; 32]) + .execute(&writer) + .await + .expect("policy"); + let actor = Keys::generate(); + let binding_id = Uuid::new_v4(); + let mut seed = writer.acquire().await.expect("seed binding connection"); + sqlx::query("SET session_replication_role=replica") + .execute(&mut *seed) + .await + .expect("disable binding triggers"); + let binding_version: i64 = sqlx::query_scalar( + "INSERT INTO identity_bindings \ + (community_id,binding_id,issuer,subject,principal_fingerprint,event_author_pubkey, \ + binding_state,lifecycle_revision,binding_provenance,policy_revision, \ + enrollment_evidence_digest,birth_history_id,creation_operation_id, \ + creation_request_fingerprint) \ + VALUES ($1,$2,'https://issuer.example','opaque-123',$3,$4,1,1,1,1,$5,$6,$7,$8) \ + RETURNING binding_version", + ) + .bind(domain.as_uuid()) + .bind(binding_id) + .bind(vec![2_u8; 32]) + .bind(actor.public_key().to_bytes().as_slice()) + .bind(vec![3_u8; 32]) + .bind(Uuid::new_v4()) + .bind(Uuid::new_v4()) + .bind(vec![4_u8; 32]) + .fetch_one(&mut *seed) + .await + .expect("active binding"); + sqlx::query("SET session_replication_role=origin") + .execute(&mut *seed) + .await + .expect("restore binding triggers"); + drop(seed); + sqlx::query( + "INSERT INTO authorization_invalidation_domains (community_id,current_generation) \ + VALUES ($1,0)", + ) + .bind(domain.as_uuid()) + .execute(&writer) + .await + .expect("activate invalidation"); + let db = Db::from_pool(writer.clone()); + db.install_authorization_event_capacity( + domain, + AuthorizationEventCapacityPolicy::new(100, 1 << 20, 16 << 10).expect("capacity"), + ) + .await + .expect("install capacity"); + + let operation = ProtectedPublicationOperationIdentity::new( + Uuid::new_v4(), + Uuid::new_v4(), + Uuid::new_v4(), + ) + .expect("operation identity"); + let (coordinate, request) = protected_publication_request_for_restore_test( + domain, + actor.public_key().to_bytes(), + binding_id, + u64::try_from(binding_version).expect("binding version"), + AuthorizationLeaseFence::from_bytes([7_u8; 32]).expect("fence"), + 0, + 1, + operation, + [8_u8; 32], + None, + ) + .expect("sealed publication request"); + let request = Arc::new(request); + let restore_identity = request.restore_identity(); + + let store = Arc::new(MemoryStore::default()); + let bootstrap_id = Uuid::new_v4(); + let original = OperationRestoreRuntime(Arc::new(RestoreInner { + writer_db: db.clone(), + source: Arc::new(PostgresManifestSource(Db::from_pool(witness.clone()))), + store: store.clone(), + bootstrap_ids: Arc::new(HashMap::from([(domain, bootstrap_id)])), + ready_domains: Arc::new(DashSet::new()), + lease: Duration::from_millis(25), + retention: test_retention(), + fence_pool: writer.clone(), + fence_slots: Arc::new(Semaphore::new(1)), + test_fences: None, + healthy: AtomicBool::new(true), + })); + original + .provision_domain(domain) + .await + .expect("provision exact DB baseline"); + let mut crash_fenced = original + .begin_protected_publication(restore_identity) + .await + .expect("begin restore before writer transaction"); + let (_, saturated_request) = protected_publication_request_for_restore_test( + domain, + actor.public_key().to_bytes(), + binding_id, + u64::try_from(binding_version).expect("binding version"), + AuthorizationLeaseFence::from_bytes([7_u8; 32]).expect("fence"), + 0, + 1, + ProtectedPublicationOperationIdentity::new( + Uuid::new_v4(), + Uuid::new_v4(), + Uuid::new_v4(), + ) + .expect("saturated operation"), + [18_u8; 32], + None, + ) + .expect("sealed saturated publication request"); + let saturated_identity = saturated_request.restore_identity(); + assert!(matches!( + original + .begin_protected_publication(saturated_identity) + .await, + Err(OperationRestoreError::OperationFenceSaturated) + )); + assert!(original.is_healthy()); + let mut blocker = blocker_pool + .begin() + .await + .expect("begin blocking transaction"); + let blocker_pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&mut *blocker) + .await + .expect("blocking backend pid"); + sqlx::query( + "SELECT 1 FROM identity_bindings \ + WHERE community_id=$1 AND binding_id=$2 FOR UPDATE", + ) + .bind(domain.as_uuid()) + .bind(binding_id) + .fetch_one(&mut *blocker) + .await + .expect("hold binding writer lock"); + let writer_db = db.clone(); + let writer_request = request.clone(); + let writer_task = tokio::spawn(async move { + let result = writer_db + .commit_staged_protected_publication_fenced( + crash_fenced.capability_mut().expect("fenced capability"), + &writer_request, + ) + .await; + (result, crash_fenced) + }); + tokio::time::timeout(Duration::from_secs(1), async { + loop { + let writer_pid: Option = sqlx::query_scalar( + "SELECT pid FROM pg_stat_activity \ + WHERE datname=current_database() AND wait_event_type='Lock' \ + AND query LIKE '%FROM identity_bindings%' \ + AND $1 = ANY(pg_blocking_pids(pid)) LIMIT 1", + ) + .bind(blocker_pid) + .fetch_optional(&witness) + .await + .expect("inspect blocked canonical writer"); + if writer_pid.is_some() { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("canonical writer reaches the blocked binding re-fence"); + let crash_shard = operation_shard_id( + OperationIdentity { + domain: restore_identity.authorization_domain(), + operation_id: restore_identity.operation_id(), + request_fingerprint: restore_identity.request_fingerprint(), + }, + bootstrap_id, + original.0.retention.shard_count, + ); + let shard_key = operation_shard_key(domain, crash_shard); + let expires_at_ms = { + let records = store.records.lock().expect("records"); + let shard: OperationShardRecord = + serde_json::from_slice(&records.get(&shard_key).expect("operation shard").1) + .expect("decode operation shard"); + shard + .entries + .get(&restore_identity.operation_id().to_string()) + .expect("pending writer intent") + .expires_at_ms + }; + tokio::time::timeout(Duration::from_secs(1), async { + loop { + let now_ms: i64 = sqlx::query_scalar( + "SELECT floor(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::bigint", + ) + .fetch_one(&witness) + .await + .expect("sample PostgreSQL clock"); + if u64::try_from(now_ms).expect("nonnegative PostgreSQL time") >= expires_at_ms { + break; + } + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("Pending lease expires by PostgreSQL time"); + let reaper = OperationRestoreRuntime(Arc::new(RestoreInner { + writer_db: original.0.writer_db.clone(), + source: original.0.source.clone(), + store: original.0.store.clone(), + bootstrap_ids: original.0.bootstrap_ids.clone(), + ready_domains: original.0.ready_domains.clone(), + lease: original.0.lease, + retention: original.0.retention, + fence_pool: fence_probe.clone(), + fence_slots: Arc::new(Semaphore::new(1)), + test_fences: None, + healthy: AtomicBool::new(true), + })); + let reaper_result = reaper.maintain_begin_shard(domain, crash_shard).await; + assert!( + matches!(reaper_result, Err(OperationRestoreError::CausalLag)), + "reaper must retain the live writer intent, got {reaper_result:?}" + ); + assert!(reaper.is_healthy()); + let operation_identity = OperationIdentity { + domain: restore_identity.authorization_domain(), + operation_id: restore_identity.operation_id(), + request_fingerprint: restore_identity.request_fingerprint(), + }; + let exact_contention = reaper + .acquire_operation_fence_with_timeout(operation_identity, OPERATION_FENCE_WAIT) + .await; + assert!( + matches!( + exact_contention, + Err(OperationRestoreError::OperationFenceTimeout) + ), + "reaper must contend on the exact live session fence, got {exact_contention:?}" + ); + let shard: OperationShardRecord = serde_json::from_slice( + &store + .records + .lock() + .expect("records") + .get(&shard_key) + .expect("operation shard") + .1, + ) + .expect("decode operation shard"); + assert_eq!( + shard + .entries + .get(&restore_identity.operation_id().to_string()) + .expect("pending writer intent") + .state, + IntentState::Pending + ); + for table in [ + "authorization_operation_receipts", + "authorization_operation_version_delta_manifests", + ] { + let count: i64 = sqlx::query_scalar(sqlx::AssertSqlSafe(format!( + "SELECT count(*) FROM {table} WHERE community_id=$1 AND operation_id=$2" + ))) + .bind(domain.as_uuid()) + .bind(restore_identity.operation_id()) + .fetch_one(&witness) + .await + .expect("inspect precommit attribution"); + assert_eq!( + count, 0, + "{table} must remain empty while writer is blocked" + ); + } + let publication_count: i64 = sqlx::query_scalar( + "SELECT count(*) FROM protected_object_authority WHERE community_id=$1", + ) + .bind(domain.as_uuid()) + .fetch_one(&witness) + .await + .expect("inspect precommit publication"); + assert_eq!(publication_count, 0); + blocker + .rollback() + .await + .expect("release binding writer lock"); + let (committed, crash_fenced) = tokio::time::timeout(Duration::from_secs(2), writer_task) + .await + .expect("canonical writer completes after blocker release") + .expect("join fenced writer"); + let committed = committed.expect("commit publication before simulated process death"); + assert_eq!( + committed.disposition(), + ProtectedPublicationDisposition::Applied + ); + drop(crash_fenced); + assert_eq!( + original + .reconcile_protected_publication(saturated_identity) + .await, + Err(OperationRestoreError::MissingIntent) + ); + + let (_, contended_request) = protected_publication_request_for_restore_test( + domain, + actor.public_key().to_bytes(), + binding_id, + u64::try_from(binding_version).expect("binding version"), + AuthorizationLeaseFence::from_bytes([7_u8; 32]).expect("fence"), + 0, + 1, + ProtectedPublicationOperationIdentity::new( + Uuid::new_v4(), + Uuid::new_v4(), + Uuid::new_v4(), + ) + .expect("contended operation"), + [19_u8; 32], + None, + ) + .expect("sealed contended publication request"); + let contended_identity = contended_request.restore_identity(); + let held = original + .begin_protected_publication(contended_identity) + .await + .expect("first runtime holds exact operation fence"); + let contender = OperationRestoreRuntime(Arc::new(RestoreInner { + writer_db: original.0.writer_db.clone(), + source: original.0.source.clone(), + store: original.0.store.clone(), + bootstrap_ids: original.0.bootstrap_ids.clone(), + ready_domains: original.0.ready_domains.clone(), + lease: original.0.lease, + retention: original.0.retention, + fence_pool: fence_probe, + fence_slots: Arc::new(Semaphore::new(1)), + test_fences: None, + healthy: AtomicBool::new(true), + })); + let contention = contender + .begin_protected_publication(contended_identity) + .await; + assert!( + matches!( + &contention, + Err(OperationRestoreError::OperationFenceTimeout) + ), + "same-operation contender must observe the held session fence, got {:?}", + contention.as_ref().err() + ); + assert!(contender.is_healthy()); + original + .abort_fenced(held) + .await + .expect("abort held intent"); + assert_eq!( + original + .reconcile_protected_publication(contended_identity) + .await, + Err(OperationRestoreError::OperationAborted) + ); + + let restarted = OperationRestoreRuntime(Arc::new(RestoreInner { + writer_db: db.clone(), + source: Arc::new(PostgresManifestSource(Db::from_pool(witness.clone()))), + store, + bootstrap_ids: Arc::new(HashMap::from([(domain, bootstrap_id)])), + ready_domains: Arc::new(DashSet::new()), + lease: Duration::from_millis(25), + retention: test_retention(), + fence_pool: writer.clone(), + fence_slots: Arc::new(Semaphore::new(1)), + test_fences: None, + healthy: AtomicBool::new(true), + })); + restarted + .verify_domain(domain) + .await + .expect("restart reconciles pending publication before floor comparison"); + let replay = restarted + .commit_protected_publication(&request) + .await + .expect("reconstruct exact committed witness"); + assert_eq!( + replay.disposition(), + ProtectedPublicationDisposition::ExactReplay + ); + assert_eq!(replay.result_digest(), committed.result_digest()); + + let (_, conflict_request) = protected_publication_request_for_restore_test( + domain, + actor.public_key().to_bytes(), + binding_id, + u64::try_from(binding_version).expect("binding version"), + AuthorizationLeaseFence::from_bytes([7_u8; 32]).expect("fence"), + 0, + 1, + ProtectedPublicationOperationIdentity::new( + Uuid::new_v4(), + Uuid::new_v4(), + Uuid::new_v4(), + ) + .expect("conflict operation"), + [9_u8; 32], + Some([10_u8; 32]), + ) + .expect("sealed conflicting later publication"); + let conflict_identity = conflict_request.restore_identity(); + assert!(matches!( + restarted + .commit_protected_publication(&conflict_request) + .await, + Err(RestoredProtectedPublicationError::Publication( + ProtectedPublicationError::Conflict + )) + )); + assert_eq!( + db.protected_publication_witness(&coordinate) + .await + .expect("load unchanged witness") + .expect("publication remains") + .result_digest(), + committed.result_digest() + ); + assert_eq!( + restarted + .reconcile_protected_publication(conflict_identity) + .await, + Err(OperationRestoreError::OperationAborted) + ); + + writer.close().await; + witness.close().await; + blocker_pool.close().await; + sqlx::query(sqlx::AssertSqlSafe(format!("DROP DATABASE {name}"))) + .execute(&admin) + .await + .expect("drop scratch database"); + admin.close().await; + } +} diff --git a/crates/buzz-db/src/authorization_version.rs b/crates/buzz-db/src/authorization_version.rs new file mode 100644 index 00000000000..6ec4e42458b --- /dev/null +++ b/crates/buzz-db/src/authorization_version.rs @@ -0,0 +1,2087 @@ +//! Exact, operation-bound authorization version attribution. +//! +//! Every mutation records the complete set of authority components it advanced +//! under the same operation receipt. Restore witnessing consumes only this +//! manifest; there is deliberately no domain-global vector fallback. + +mod protected; + +pub(crate) use protected::protected_object_key; +#[cfg(test)] +pub(crate) use protected::protected_publication_request_for_restore_test; + +pub use protected::{ + ProtectedPublicationCommit, ProtectedPublicationCoordinate, ProtectedPublicationDependency, + ProtectedPublicationDisposition, ProtectedPublicationError, + ProtectedPublicationOperationIdentity, ProtectedPublicationRequest, + ProtectedPublicationRestoreIdentity, ProtectedPublicationValidity, ProtectedPublicationWitness, +}; + +use std::{collections::BTreeSet, fmt, time::Duration}; + +use buzz_auth::RouteCapability; +use buzz_core::{AuthorizationLeaseFence, CommunityId}; +use sha2::{Digest, Sha256}; +use sqlx::{postgres::PgConnection, Connection, PgPool, Postgres, Row, Transaction}; +use thiserror::Error; +use uuid::Uuid; + +use crate::{ + authorization_events::{AuthorizationAuthorityLossTarget, AuthorizationEventActor}, + Db, DbError, Result, +}; + +const MAX_COMPONENTS: usize = 1_024; +const MAX_SUPERSESSION_STEPS: usize = 1_024; +const MAX_DATABASE_VERSION: u64 = i64::MAX as u64; + +#[cfg(test)] +pub(crate) fn loopback_test_database_url() -> String { + format!( + "{}://{}:{}@{}:{}/{}", + "postgres", "buzz", "buzz_dev", "localhost", 5432, "buzz" + ) +} + +fn authorization_operation_fence_key( + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], +) -> i64 { + let mut digest = Sha256::new(); + digest.update(b"buzz:nip-fi-operation-session-fence:v1"); + digest.update(community_id.as_uuid().as_bytes()); + digest.update(operation_id.as_bytes()); + digest.update(request_fingerprint); + let digest: [u8; 32] = digest.finalize().into(); + let mut fence_key = [0_u8; 8]; + fence_key.copy_from_slice(&digest[..8]); + i64::from_be_bytes(fence_key) +} + +/// Opaque lock-owning primary session issued only inside operation restore. +/// +/// Canonical writers accept this capability rather than a raw PostgreSQL +/// connection. Its exact operation identity and session lock cannot be +/// changed or inspected by callers. +/// +pub(crate) struct AuthorizationOperationFence { + connection: Option, + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + lock_key: i64, +} + +/// Bounded failures while operation restore acquires its opaque session. +#[derive(Debug, Error)] +pub(crate) enum AuthorizationOperationFenceAcquireError { + /// No dedicated primary connection became available within the bound. + #[error("authorization operation fence pool is saturated")] + PoolSaturated, + /// The advisory-lock statement did not complete within the bound. + #[error("authorization operation fence lock timed out")] + LockTimedOut, + /// PostgreSQL failed while acquiring the dedicated session. + #[error(transparent)] + Database(#[from] sqlx::Error), +} + +impl AuthorizationOperationFence { + pub(crate) async fn acquire_for_operation_restore( + pool: &PgPool, + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + timeout: Duration, + ) -> std::result::Result, AuthorizationOperationFenceAcquireError> { + if community_id.as_uuid().is_nil() + || operation_id.is_nil() + || request_fingerprint == [0; 32] + { + return Err(AuthorizationOperationFenceAcquireError::Database( + sqlx::Error::Protocol( + "authorization operation fence identity is invalid".to_owned(), + ), + )); + } + let deadline = tokio::time::Instant::now() + timeout; + let pooled = tokio::time::timeout(timeout, pool.acquire()) + .await + .map_err(|_| AuthorizationOperationFenceAcquireError::PoolSaturated)??; + let mut connection = pooled.detach(); + let lock_key = + authorization_operation_fence_key(community_id, operation_id, request_fingerprint); + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + if remaining.is_zero() { + drop(connection); + return Err(AuthorizationOperationFenceAcquireError::LockTimedOut); + } + let acquired = tokio::time::timeout( + remaining, + sqlx::query_scalar::<_, bool>("SELECT pg_try_advisory_lock($1)") + .bind(lock_key) + .fetch_one(&mut connection), + ) + .await + .map_err(|_| AuthorizationOperationFenceAcquireError::LockTimedOut)??; + if !acquired { + drop(connection); + return Ok(None); + } + Ok(Some(Self { + connection: Some(connection), + community_id, + operation_id, + request_fingerprint, + lock_key, + })) + } + + pub(crate) fn connection_for( + &mut self, + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + ) -> Result<&mut PgConnection> { + if self.community_id != community_id + || self.operation_id != operation_id + || self.request_fingerprint != request_fingerprint + { + return Err(DbError::InvalidData( + "authorization operation fence identity mismatch".to_owned(), + )); + } + self.connection.as_mut().ok_or_else(|| { + DbError::InvalidData("authorization operation fence is consumed".to_owned()) + }) + } + + pub(crate) async fn release(mut self, timeout: Duration) -> Result<()> { + let Some(mut connection) = self.connection.take() else { + return Err(DbError::InvalidData( + "authorization operation fence is consumed".to_owned(), + )); + }; + let unlocked = tokio::time::timeout( + timeout, + sqlx::query_scalar::<_, bool>("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .fetch_one(&mut connection), + ) + .await + .map_err(|_| DbError::InvalidData("authorization fence unlock timed out".to_owned()))??; + if !unlocked { + return Err(DbError::InvalidData( + "authorization operation fence was not held".to_owned(), + )); + } + tokio::time::timeout(timeout, connection.close()) + .await + .map_err(|_| { + DbError::InvalidData("authorization fence close timed out".to_owned()) + })??; + Ok(()) + } +} + +impl fmt::Debug for AuthorizationOperationFence { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("AuthorizationOperationFence([REDACTED])") + } +} + +/// Closed component classes persisted by migration 0030. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(i16)] +pub enum AuthorizationVersionComponentKind { + /// Immutable local binding generation. + Binding = 1, + /// Local enrollment-policy revision. + Policy = 2, + /// Domain invalidation generation. + InvalidationGeneration = 3, + /// Protected-object authority epoch. + AuthorityEpoch = 4, + /// Verified delegated-relationship revision. + DelegatedRelationship = 6, + /// Immutable lifecycle-selector fact generation. + LifecycleSelector = 7, +} + +/// Closed protected-object classes persisted by migration 0030. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[repr(i16)] +pub enum AuthorizationProtectedObjectKind { + /// Whole authorization domain. + Domain = 1, + /// Relay channel. + Channel = 2, + /// Git repository. + Repository = 3, + /// Immutable media object. + Media = 4, + /// Moderation target. + ModerationTarget = 5, + /// Audio session. + AudioSession = 6, +} + +impl AuthorizationProtectedObjectKind { + /// Whether this object class admits the exact read capability. + pub fn admits_read(self, capability: RouteCapability) -> bool { + match self { + Self::Repository => matches!( + capability, + RouteCapability::ReposRead + | RouteCapability::ReposWrite + | RouteCapability::GitRead + | RouteCapability::GitWrite + | RouteCapability::GitStream + ), + Self::Media => matches!( + capability, + RouteCapability::MediaRead | RouteCapability::MediaWrite + ), + Self::ModerationTarget => capability == RouteCapability::Moderation, + Self::Domain | Self::Channel | Self::AudioSession => false, + } + } + + /// Whether this object class admits the exact mutation capability. + pub fn admits_mutation(self, capability: RouteCapability) -> bool { + match self { + Self::Repository => matches!( + capability, + RouteCapability::ReposWrite | RouteCapability::GitWrite + ), + Self::Media => capability == RouteCapability::MediaWrite, + Self::ModerationTarget => capability == RouteCapability::Moderation, + Self::Domain | Self::Channel | Self::AudioSession => false, + } + } + + pub(crate) fn from_database(value: i16) -> Result { + match value { + 1 => Ok(Self::Domain), + 2 => Ok(Self::Channel), + 3 => Ok(Self::Repository), + 4 => Ok(Self::Media), + 5 => Ok(Self::ModerationTarget), + 6 => Ok(Self::AudioSession), + _ => Err(DbError::InvalidData( + "authorization protected object kind is invalid".to_owned(), + )), + } + } +} + +impl AuthorizationVersionComponentKind { + fn from_database(value: i16) -> Result { + match value { + 1 => Ok(Self::Binding), + 2 => Ok(Self::Policy), + 3 => Ok(Self::InvalidationGeneration), + 4 => Ok(Self::AuthorityEpoch), + 6 => Ok(Self::DelegatedRelationship), + 7 => Ok(Self::LifecycleSelector), + _ => Err(DbError::InvalidData( + "authorization version component kind is invalid".to_owned(), + )), + } + } +} + +/// One strict before-to-after advance for an exact authority coordinate. +#[derive(Clone, PartialEq, Eq)] +pub struct AuthorizationOperationVersionDelta { + component_kind: AuthorizationVersionComponentKind, + component_key: [u8; 32], + before_version: u64, + after_version: u64, + component_digest: [u8; 32], +} + +impl AuthorizationOperationVersionDelta { + /// Construct a strict, database-representable version advance. + pub(crate) fn new( + component_kind: AuthorizationVersionComponentKind, + component_key: [u8; 32], + before_version: u64, + after_version: u64, + ) -> Result { + if component_key == [0; 32] + || before_version > MAX_DATABASE_VERSION + || after_version > MAX_DATABASE_VERSION + || after_version <= before_version + { + return Err(DbError::InvalidData( + "authorization version delta is invalid".to_owned(), + )); + } + let component_digest = + component_digest(component_kind, component_key, before_version, after_version); + Ok(Self { + component_kind, + component_key, + before_version, + after_version, + component_digest, + }) + } + + /// Closed component class. + pub const fn component_kind(&self) -> AuthorizationVersionComponentKind { + self.component_kind + } + + /// Opaque exact component coordinate. + pub const fn component_key(&self) -> [u8; 32] { + self.component_key + } + + /// Version observed before the operation. + pub const fn before_version(&self) -> u64 { + self.before_version + } + + /// Version committed by the operation. + pub const fn after_version(&self) -> u64 { + self.after_version + } + + /// Digest binding the complete component advance. + pub const fn component_digest(&self) -> [u8; 32] { + self.component_digest + } +} + +impl fmt::Debug for AuthorizationOperationVersionDelta { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("AuthorizationOperationVersionDelta") + .field("component_kind", &self.component_kind) + .field("component_key", &"[REDACTED]") + .field("before_version", &"[REDACTED]") + .field("after_version", &"[REDACTED]") + .field("component_digest", &"[REDACTED]") + .finish() + } +} + +/// Canonical exact-operation version manifest loaded from PostgreSQL. +#[derive(Clone, PartialEq, Eq)] +pub struct AuthorizationOperationVersionDeltaManifest { + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + before_digest: [u8; 32], + after_digest: [u8; 32], + manifest_digest: [u8; 32], + components: Vec, +} + +impl AuthorizationOperationVersionDeltaManifest { + /// Server-resolved authorization domain. + pub const fn community_id(&self) -> CommunityId { + self.community_id + } + + /// Exact operation receipt identity. + pub const fn operation_id(&self) -> Uuid { + self.operation_id + } + + /// Exact request fingerprint bound to the operation receipt. + pub const fn request_fingerprint(&self) -> [u8; 32] { + self.request_fingerprint + } + + /// Digest of the complete before-state projection. + pub const fn before_digest(&self) -> [u8; 32] { + self.before_digest + } + + /// Digest of the complete after-state projection. + pub const fn after_digest(&self) -> [u8; 32] { + self.after_digest + } + + /// Digest binding operation identity and every component. + pub const fn manifest_digest(&self) -> [u8; 32] { + self.manifest_digest + } + + /// Strictly sorted, unique component advances. + pub fn components(&self) -> &[AuthorizationOperationVersionDelta] { + &self.components + } +} + +impl fmt::Debug for AuthorizationOperationVersionDeltaManifest { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("AuthorizationOperationVersionDeltaManifest") + .field("community_id", &"[REDACTED]") + .field("operation_id", &"[REDACTED]") + .field("request_fingerprint", &"[REDACTED]") + .field("before_digest", &"[REDACTED]") + .field("after_digest", &"[REDACTED]") + .field("manifest_digest", &"[REDACTED]") + .field("component_count", &self.components.len()) + .finish() + } +} + +/// One authoritative component baseline used to provision an external witness. +#[derive(Clone, PartialEq, Eq)] +pub struct AuthorizationVersionComponentFloor { + /// Closed component class. + pub component_kind: AuthorizationVersionComponentKind, + /// Opaque exact component coordinate. + pub component_key: [u8; 32], + /// Current authoritative version; an absent resource has baseline zero. + pub version: u64, +} + +/// Exact protected-object authority coordinate advanced after a local +/// admission loss. Values are opaque and Debug-redacted outside this module. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct AuthorizationAuthorityObjectEvidence { + object_kind: AuthorizationProtectedObjectKind, + object_key: [u8; 32], + authority_epoch: u64, + fence: AuthorizationLeaseFence, +} + +impl AuthorizationAuthorityObjectEvidence { + /// Closed migration-0030 object kind and canonical object key. + #[allow(dead_code)] // Consumed by canonical admission integration. + pub(crate) const fn coordinate(&self) -> (AuthorizationProtectedObjectKind, [u8; 32]) { + (self.object_kind, self.object_key) + } + + /// New exact epoch and nonzero observable fence. + #[allow(dead_code)] // Consumed by canonical admission integration. + pub(crate) const fn epoch_and_fence(&self) -> (u64, AuthorizationLeaseFence) { + (self.authority_epoch, self.fence) + } +} + +impl fmt::Debug for AuthorizationAuthorityObjectEvidence { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("AuthorizationAuthorityObjectEvidence([REDACTED])") + } +} + +/// Opaque exact set of protected-authority advances for one admission loss. +#[allow(dead_code)] // Consumed by lifecycle integration. +pub(crate) struct AuthorizationAuthorityEpochAdvance { + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + loss_target: AuthorizationAuthorityLossTarget, + objects: Vec, + deltas: Vec, +} + +impl AuthorizationAuthorityEpochAdvance { + /// Exact object coordinates re-fenced in canonical order. + #[allow(dead_code)] // Consumed by canonical admission integration. + pub(crate) fn objects(&self) -> &[AuthorizationAuthorityObjectEvidence] { + &self.objects + } + + /// Consume the exact database-owned AuthorityEpoch deltas for the complete + /// operation manifest. + #[allow(dead_code)] // Consumed by canonical admission integration. + pub(crate) fn matches_operation( + &self, + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + ) -> bool { + self.community_id == community_id + && self.operation_id == operation_id + && self.request_fingerprint == request_fingerprint + } + + /// Database-rechecked admission-loss coordinate that authorized this + /// authority-first advance. + pub(crate) const fn loss_target(&self) -> AuthorizationAuthorityLossTarget { + self.loss_target + } + + /// Consume the authority evidence and its exact DB-owned deltas. + pub(crate) fn into_parts( + self, + ) -> ( + Vec, + Vec, + ) { + (self.objects, self.deltas) + } +} + +impl fmt::Debug for AuthorizationAuthorityEpochAdvance { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("AuthorizationAuthorityEpochAdvance") + .field("object_count", &self.objects.len()) + .finish() + } +} + +impl fmt::Debug for AuthorizationVersionComponentFloor { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("AuthorizationVersionComponentFloor") + .field("component_kind", &self.component_kind) + .field("component_key", &"[REDACTED]") + .field("version", &"[REDACTED]") + .finish() + } +} + +/// Derive an opaque, length-framed component coordinate. +pub fn authorization_version_component_key(namespace: &[u8], parts: &[&[u8]]) -> [u8; 32] { + let mut digest = Sha256::new(); + framed(&mut digest, b"buzz:authorization-version-component:v1"); + framed(&mut digest, namespace); + for part in parts { + framed(&mut digest, part); + } + digest.finalize().into() +} + +/// Canonical binding-generation component coordinate. +pub fn authorization_version_binding_component_key( + community_id: CommunityId, + binding_id: Uuid, +) -> [u8; 32] { + authorization_version_component_key( + b"binding", + &[community_id.as_uuid().as_bytes(), binding_id.as_bytes()], + ) +} + +/// Canonical domain policy-revision component coordinate. +pub fn authorization_version_policy_component_key(community_id: CommunityId) -> [u8; 32] { + authorization_version_component_key(b"policy", &[community_id.as_uuid().as_bytes()]) +} + +/// Canonical domain invalidation-generation component coordinate. +pub fn authorization_version_invalidation_generation_component_key( + community_id: CommunityId, +) -> [u8; 32] { + authorization_version_component_key(b"invalidation", &[community_id.as_uuid().as_bytes()]) +} + +/// Canonical protected-object authority-epoch component coordinate. +pub fn authorization_version_authority_epoch_component_key( + community_id: CommunityId, + object_kind: AuthorizationProtectedObjectKind, + object_key: [u8; 32], +) -> [u8; 32] { + authorization_version_component_key( + b"authority-epoch", + &[ + community_id.as_uuid().as_bytes(), + &(object_kind as i16).to_be_bytes(), + &object_key, + ], + ) +} + +/// Canonical delegated-relationship component coordinate. +pub fn authorization_version_delegated_relationship_component_key( + community_id: CommunityId, + relationship_id: Uuid, +) -> [u8; 32] { + authorization_version_component_key( + b"delegated-relationship", + &[ + community_id.as_uuid().as_bytes(), + relationship_id.as_bytes(), + ], + ) +} + +/// Canonical lifecycle-selector component coordinate. +pub fn authorization_version_lifecycle_selector_component_key( + community_id: CommunityId, + selector_id: Uuid, +) -> [u8; 32] { + authorization_version_component_key( + b"lifecycle-selector", + &[community_id.as_uuid().as_bytes(), selector_id.as_bytes()], + ) +} + +/// Advance every protected-object authority row affected by one exact, +/// database-rechecked local admission loss. +/// +/// Lock order is all matching `authorization_authority_epochs` rows in +/// canonical `(object_kind, object_key)` order, followed by their coupled +/// `protected_object_authority` rows. No matching protected object is an +/// explicit valid outcome and returns empty evidence/deltas. Existing rows +/// must advance together with a new nonzero fence; partial state fails closed. +#[allow(dead_code)] // Consumed by admission-loss repair integration. +pub(crate) async fn advance_admission_loss_authority_tx( + transaction: &mut Transaction<'_, Postgres>, + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + actor: &AuthorizationEventActor, +) -> Result { + if community_id.as_uuid().is_nil() || operation_id.is_nil() || request_fingerprint == [0; 32] { + return Err(DbError::InvalidData( + "authorization authority advance identity is invalid".to_owned(), + )); + } + if !actor.is_bound_to(community_id) { + return Err(DbError::InvalidData( + "authorization authority actor domain does not match".to_owned(), + )); + } + let target = actor.authority_loss_target().ok_or_else(|| { + DbError::InvalidData( + "authorization authority advance lacks a rechecked loss cause".to_owned(), + ) + })?; + let ( + target_kind, + binding_id, + binding_version, + policy_revision, + relationship_id, + relationship_revision, + ) = match target { + AuthorizationAuthorityLossTarget::Binding(binding_id, binding_version) => ( + 1_i16, + Some(binding_id), + Some(to_database_version(binding_version)?), + None, + None, + None, + ), + AuthorizationAuthorityLossTarget::Policy(policy_revision) => ( + 2_i16, + None, + None, + Some(to_database_version(policy_revision)?), + None, + None, + ), + AuthorizationAuthorityLossTarget::DelegatedRelationship( + relationship_id, + relationship_revision, + ) => ( + 3_i16, + None, + None, + None, + Some(relationship_id), + Some(to_database_version(relationship_revision)?), + ), + }; + + let rows = sqlx::query( + "SELECT epoch.object_kind,epoch.object_key,epoch.authority_epoch,epoch.fence \ + FROM authorization_authority_epochs epoch \ + WHERE epoch.community_id=$1 AND EXISTS ( \ + SELECT 1 FROM protected_object_authority protected \ + WHERE protected.community_id=epoch.community_id \ + AND protected.object_kind=epoch.object_kind AND protected.object_key=epoch.object_key \ + AND (($2=1 AND protected.binding_id=$3 AND protected.binding_version=$4) \ + OR ($2=2 AND protected.policy_revision=$5) \ + OR ($2=3 AND protected.delegated_relationship_id=$6 \ + AND protected.delegated_relationship_revision=$7))) \ + ORDER BY epoch.object_kind,epoch.object_key LIMIT 1025 FOR UPDATE OF epoch", + ) + .bind(community_id.as_uuid()) + .bind(target_kind) + .bind(binding_id) + .bind(binding_version) + .bind(policy_revision) + .bind(relationship_id) + .bind(relationship_revision) + .fetch_all(&mut **transaction) + .await?; + if rows.len() > MAX_COMPONENTS { + return Err(DbError::InvalidData( + "authorization authority advance exceeds manifest capacity".to_owned(), + )); + } + + let mut objects = Vec::with_capacity(rows.len()); + let mut deltas = Vec::with_capacity(rows.len()); + for row in rows { + let object_kind = + AuthorizationProtectedObjectKind::from_database(row.try_get("object_kind")?)?; + let object_key = bytes32(row.try_get("object_key")?, "authority object key")?; + let before = from_database_version(row.try_get("authority_epoch")?)?; + let old_fence = authorization_lease_fence(row.try_get("fence")?, "authority fence")?; + let after = before.checked_add(1).ok_or_else(|| { + DbError::InvalidData("authorization authority epoch exhausted".to_owned()) + })?; + let fence = next_authority_fence( + community_id, + operation_id, + request_fingerprint, + object_kind, + object_key, + before, + old_fence, + )?; + + let locked = sqlx::query( + "SELECT 1 FROM protected_object_authority \ + WHERE community_id=$1 AND object_kind=$2 AND object_key=$3 \ + AND authority_epoch=$4 AND fence=$5 \ + AND (($6=1 AND binding_id=$7 AND binding_version=$8) \ + OR ($6=2 AND policy_revision=$9) \ + OR ($6=3 AND delegated_relationship_id=$10 \ + AND delegated_relationship_revision=$11)) FOR UPDATE", + ) + .bind(community_id.as_uuid()) + .bind(object_kind as i16) + .bind(object_key.as_slice()) + .bind(to_database_version(before)?) + .bind(old_fence.as_bytes().as_slice()) + .bind(target_kind) + .bind(binding_id) + .bind(binding_version) + .bind(policy_revision) + .bind(relationship_id) + .bind(relationship_revision) + .fetch_optional(&mut **transaction) + .await?; + if locked.is_none() { + return Err(DbError::InvalidData( + "authorization protected authority changed during refence".to_owned(), + )); + } + + let epoch_updated = sqlx::query( + "UPDATE authorization_authority_epochs SET authority_epoch=$6,fence=$7, \ + operation_id=$8,request_fingerprint=$9, \ + updated_at=GREATEST(clock_timestamp(),updated_at + INTERVAL '1 microsecond') \ + WHERE community_id=$1 AND object_kind=$2 AND object_key=$3 \ + AND authority_epoch=$4 AND fence=$5", + ) + .bind(community_id.as_uuid()) + .bind(object_kind as i16) + .bind(object_key.as_slice()) + .bind(to_database_version(before)?) + .bind(old_fence.as_bytes().as_slice()) + .bind(to_database_version(after)?) + .bind(fence.as_bytes().as_slice()) + .bind(operation_id) + .bind(request_fingerprint.as_slice()) + .execute(&mut **transaction) + .await?; + if epoch_updated.rows_affected() != 1 { + return Err(DbError::InvalidData( + "authorization authority epoch did not advance exactly once".to_owned(), + )); + } + + let protected_updated = sqlx::query( + "WITH next_time AS (SELECT GREATEST(clock_timestamp(),issued_at + \ + INTERVAL '1 microsecond') AS value FROM protected_object_authority \ + WHERE community_id=$1 AND object_kind=$2 AND object_key=$3) \ + UPDATE protected_object_authority protected SET authority_epoch=$6,fence=$7, \ + operation_id=$8,request_fingerprint=$9,issued_at=next_time.value, \ + expires_at=next_time.value + INTERVAL '1 microsecond' \ + FROM next_time \ + WHERE protected.community_id=$1 AND protected.object_kind=$2 \ + AND protected.object_key=$3 AND protected.authority_epoch=$4 \ + AND protected.fence=$5", + ) + .bind(community_id.as_uuid()) + .bind(object_kind as i16) + .bind(object_key.as_slice()) + .bind(to_database_version(before)?) + .bind(old_fence.as_bytes().as_slice()) + .bind(to_database_version(after)?) + .bind(fence.as_bytes().as_slice()) + .bind(operation_id) + .bind(request_fingerprint.as_slice()) + .execute(&mut **transaction) + .await?; + if protected_updated.rows_affected() != 1 { + return Err(DbError::InvalidData( + "authorization protected authority did not advance exactly once".to_owned(), + )); + } + + objects.push(AuthorizationAuthorityObjectEvidence { + object_kind, + object_key, + authority_epoch: after, + fence, + }); + deltas.push(AuthorizationOperationVersionDelta::new( + AuthorizationVersionComponentKind::AuthorityEpoch, + authorization_version_authority_epoch_component_key( + community_id, + object_kind, + object_key, + ), + before, + after, + )?); + } + Ok(AuthorizationAuthorityEpochAdvance { + community_id, + operation_id, + request_fingerprint, + loss_target: target, + objects, + deltas, + }) +} + +fn next_authority_fence( + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + object_kind: AuthorizationProtectedObjectKind, + object_key: [u8; 32], + before: u64, + old_fence: AuthorizationLeaseFence, +) -> Result { + let mut digest = Sha256::new(); + framed(&mut digest, b"buzz:authorization-authority-refence:v1"); + framed(&mut digest, community_id.as_uuid().as_bytes()); + framed(&mut digest, operation_id.as_bytes()); + framed(&mut digest, &request_fingerprint); + framed(&mut digest, &(object_kind as i16).to_be_bytes()); + framed(&mut digest, &object_key); + framed(&mut digest, &before.to_be_bytes()); + framed(&mut digest, old_fence.as_bytes()); + let fence: [u8; 32] = digest.finalize().into(); + if fence == *old_fence.as_bytes() { + return Err(DbError::InvalidData( + "authorization authority fence did not advance".to_owned(), + )); + } + AuthorizationLeaseFence::from_bytes(fence) + .map_err(|_| DbError::InvalidData("authorization authority fence is invalid".to_owned())) +} + +/// Record a complete operation manifest inside the caller-owned transaction. +/// +/// The generic operation receipt may be inserted before or after this call; +/// migration 0030's deferred foreign key validates the pair at commit. +pub(crate) async fn record_authorization_operation_version_delta_tx( + transaction: &mut Transaction<'_, Postgres>, + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + components: Vec, +) -> Result { + let candidate = build_manifest(community_id, operation_id, request_fingerprint, components)?; + + if let Some(existing) = + load_manifest_connection(transaction, community_id, operation_id).await? + { + if existing == candidate { + return Ok(existing); + } + return Err(DbError::InvalidData( + "authorization operation version attribution conflicts with prior receipt".to_owned(), + )); + } + + sqlx::query( + "INSERT INTO authorization_operation_version_delta_manifests \ + (community_id, operation_id, request_fingerprint, component_count, \ + before_digest, after_digest, manifest_digest) \ + VALUES ($1,$2,$3,$4,$5,$6,$7)", + ) + .bind(community_id.as_uuid()) + .bind(operation_id) + .bind(request_fingerprint.as_slice()) + .bind(i32::try_from(candidate.components.len()).map_err(|_| { + DbError::InvalidData("authorization version component count is invalid".to_owned()) + })?) + .bind(candidate.before_digest.as_slice()) + .bind(candidate.after_digest.as_slice()) + .bind(candidate.manifest_digest.as_slice()) + .execute(&mut **transaction) + .await?; + + for component in &candidate.components { + sqlx::query( + "INSERT INTO authorization_operation_version_deltas \ + (community_id, operation_id, component_kind, component_key, before_version, \ + after_version, component_digest) VALUES ($1,$2,$3,$4,$5,$6,$7)", + ) + .bind(community_id.as_uuid()) + .bind(operation_id) + .bind(component.component_kind as i16) + .bind(component.component_key.as_slice()) + .bind(to_database_version(component.before_version)?) + .bind(to_database_version(component.after_version)?) + .bind(component.component_digest.as_slice()) + .execute(&mut **transaction) + .await?; + } + + Ok(candidate) +} + +impl Db { + /// Read PostgreSQL's authoritative wall clock for external restore leases. + pub async fn authorization_restore_clock_millis(&self) -> Result { + let value: i64 = sqlx::query_scalar( + "SELECT FLOOR(EXTRACT(EPOCH FROM clock_timestamp()) * 1000)::BIGINT", + ) + .fetch_one(&self.pool) + .await?; + u64::try_from(value) + .map_err(|_| DbError::InvalidData("authorization restore clock is invalid".to_owned())) + } + + /// Load one exact operation manifest; missing or malformed attribution fails closed. + pub async fn authorization_operation_version_delta( + &self, + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + ) -> Result { + let mut connection = self.pool.acquire().await?; + let manifest = load_manifest_connection(&mut connection, community_id, operation_id) + .await? + .ok_or_else(|| { + DbError::NotFound("authorization operation version attribution".to_owned()) + })?; + if manifest.request_fingerprint != request_fingerprint { + return Err(DbError::InvalidData( + "authorization operation version attribution identity mismatch".to_owned(), + )); + } + Ok(manifest) + } + + /// Resolve the immediate canonical operation predecessors of a pending + /// manifest. This lets bounded restore maintenance find a predecessor in + /// another immutable operation shard without scanning unrelated locks. + pub async fn authorization_operation_version_predecessors( + &self, + community_id: CommunityId, + components: &[(AuthorizationVersionComponentKind, [u8; 32], u64)], + ) -> Result> { + if community_id.as_uuid().is_nil() || components.len() > MAX_COMPONENTS { + return Err(DbError::InvalidData( + "authorization predecessor coordinates are invalid".to_owned(), + )); + } + let mut connection = self.pool.acquire().await?.detach(); + let mut identities = BTreeSet::new(); + for (kind, key, before) in components { + if *key == [0; 32] || *before > MAX_DATABASE_VERSION { + return Err(DbError::InvalidData( + "authorization predecessor component is invalid".to_owned(), + )); + } + if *before == 0 { + continue; + } + let rows = sqlx::query( + "SELECT d.operation_id,m.request_fingerprint \ + FROM authorization_operation_version_deltas d \ + JOIN authorization_operation_version_delta_manifests m \ + ON m.community_id=d.community_id AND m.operation_id=d.operation_id \ + WHERE d.community_id=$1 AND d.component_kind=$2 AND d.component_key=$3 \ + AND d.after_version=$4 \ + ORDER BY d.operation_id LIMIT 2", + ) + .bind(community_id.as_uuid()) + .bind(*kind as i16) + .bind(key.as_slice()) + .bind(to_database_version(*before)?) + .fetch_all(&mut connection) + .await?; + if rows.len() > 1 { + return Err(DbError::InvalidData( + "authorization predecessor branches are ambiguous".to_owned(), + )); + } + if let Some(row) = rows.first() { + identities.insert(( + row.try_get::("operation_id")?, + bytes32(row.try_get("request_fingerprint")?, "request fingerprint")?, + )); + } + } + + let mut predecessors = Vec::with_capacity(identities.len()); + for (operation_id, request_fingerprint) in identities { + let manifest = load_manifest_connection(&mut connection, community_id, operation_id) + .await? + .ok_or_else(|| { + DbError::InvalidData("authorization predecessor manifest is missing".to_owned()) + })?; + if manifest.request_fingerprint != request_fingerprint { + return Err(DbError::InvalidData( + "authorization predecessor identity mismatches".to_owned(), + )); + } + predecessors.push(manifest); + } + connection.close().await?; + Ok(predecessors) + } + + /// Prove that a higher external component floor causally supersedes an + /// exact pending operation component. + /// + /// Every immutable PostgreSQL delta in the interval is loaded through the + /// canonical manifest validator. Gaps, overlaps, duplicate branches, or a + /// latest provenance mismatch fail closed. + #[allow(clippy::too_many_arguments)] + pub async fn authorization_version_proves_component_supersession( + &self, + community_id: CommunityId, + component_kind: AuthorizationVersionComponentKind, + component_key: [u8; 32], + pending_after: u64, + floor_version: u64, + floor_operation_id: Uuid, + floor_request_fingerprint: [u8; 32], + floor_component_digest: [u8; 32], + floor_manifest_digest: [u8; 32], + ) -> Result<()> { + if community_id.as_uuid().is_nil() + || component_key == [0; 32] + || pending_after == 0 + || floor_version <= pending_after + || floor_version > MAX_DATABASE_VERSION + || floor_operation_id.is_nil() + || floor_request_fingerprint == [0; 32] + || floor_component_digest == [0; 32] + || floor_manifest_digest == [0; 32] + { + return Err(DbError::InvalidData( + "authorization supersession coordinates are invalid".to_owned(), + )); + } + + let before_floor = self + .authorization_version_component_floors(community_id) + .await? + .into_iter() + .find(|floor| { + floor.component_kind == component_kind && floor.component_key == component_key + }) + .map(|floor| floor.version) + .unwrap_or(0); + if before_floor != floor_version { + return Err(DbError::InvalidData( + "authorization supersession floor is not current".to_owned(), + )); + } + + let row_limit = i64::try_from(MAX_SUPERSESSION_STEPS + 1).map_err(|_| { + DbError::InvalidData("authorization supersession step bound is invalid".to_owned()) + })?; + let rows = sqlx::query( + "SELECT d.operation_id,m.request_fingerprint,m.manifest_digest,\ + d.before_version,d.after_version,d.component_digest \ + FROM authorization_operation_version_deltas d \ + JOIN authorization_operation_version_delta_manifests m \ + ON m.community_id=d.community_id AND m.operation_id=d.operation_id \ + WHERE d.community_id=$1 AND d.component_kind=$2 AND d.component_key=$3 \ + AND d.after_version>$4 AND d.after_version<=$5 \ + ORDER BY d.before_version,d.after_version,d.operation_id \ + LIMIT $6", + ) + .bind(community_id.as_uuid()) + .bind(component_kind as i16) + .bind(component_key.as_slice()) + .bind(i64::try_from(pending_after).map_err(|_| { + DbError::InvalidData("authorization supersession version overflow".to_owned()) + })?) + .bind(i64::try_from(floor_version).map_err(|_| { + DbError::InvalidData("authorization supersession version overflow".to_owned()) + })?) + .bind(row_limit) + .fetch_all(&self.pool) + .await?; + if rows.is_empty() || rows.len() > MAX_SUPERSESSION_STEPS { + return Err(DbError::InvalidData( + "authorization supersession chain is missing or unbounded".to_owned(), + )); + } + + let mut expected_before = pending_after; + let mut latest = None; + for row in rows { + let operation_id: Uuid = row.try_get("operation_id")?; + let request_fingerprint = + bytes32(row.try_get("request_fingerprint")?, "request fingerprint")?; + let manifest_digest = bytes32(row.try_get("manifest_digest")?, "manifest digest")?; + let before = from_database_version(row.try_get("before_version")?)?; + let after = from_database_version(row.try_get("after_version")?)?; + let component_digest = bytes32(row.try_get("component_digest")?, "component digest")?; + if before != expected_before || after <= before { + return Err(DbError::InvalidData( + "authorization supersession chain has a gap or overlap".to_owned(), + )); + } + let manifest = self + .authorization_operation_version_delta( + community_id, + operation_id, + request_fingerprint, + ) + .await?; + if manifest.manifest_digest != manifest_digest + || !manifest.components.iter().any(|component| { + component.component_kind == component_kind + && component.component_key == component_key + && component.before_version == before + && component.after_version == after + && component.component_digest == component_digest + }) + { + return Err(DbError::InvalidData( + "authorization supersession manifest provenance mismatches".to_owned(), + )); + } + expected_before = after; + latest = Some(( + operation_id, + request_fingerprint, + component_digest, + manifest_digest, + )); + } + if expected_before != floor_version + || latest + != Some(( + floor_operation_id, + floor_request_fingerprint, + floor_component_digest, + floor_manifest_digest, + )) + { + return Err(DbError::InvalidData( + "authorization supersession latest provenance mismatches".to_owned(), + )); + } + + let after_floor = self + .authorization_version_component_floors(community_id) + .await? + .into_iter() + .find(|floor| { + floor.component_kind == component_kind && floor.component_key == component_key + }) + .map(|floor| floor.version) + .unwrap_or(0); + if after_floor != floor_version { + return Err(DbError::InvalidData( + "authorization supersession floor changed during proof".to_owned(), + )); + } + Ok(()) + } + + /// Read every current component floor for witness provisioning. + /// + /// Scalar policy and invalidation coordinates are returned even when their + /// backing row is absent, using the frozen absent baseline zero. + pub async fn authorization_version_component_floors( + &self, + community_id: CommunityId, + ) -> Result> { + let mut transaction = self.pool.begin().await?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY") + .execute(&mut *transaction) + .await?; + let mut floors = Vec::new(); + + let binding_rows = sqlx::query( + "SELECT binding_id, binding_version FROM identity_bindings \ + WHERE community_id=$1 ORDER BY binding_id", + ) + .bind(community_id.as_uuid()) + .fetch_all(&mut *transaction) + .await?; + for row in binding_rows { + let binding_id: Uuid = row.try_get("binding_id")?; + floors.push(AuthorizationVersionComponentFloor { + component_kind: AuthorizationVersionComponentKind::Binding, + component_key: authorization_version_binding_component_key( + community_id, + binding_id, + ), + version: from_database_version(row.try_get("binding_version")?)?, + }); + } + + let policy_version: Option = sqlx::query_scalar( + "SELECT MAX(policy_revision) FROM identity_enrollment_policies WHERE community_id=$1", + ) + .bind(community_id.as_uuid()) + .fetch_one(&mut *transaction) + .await?; + floors.push(AuthorizationVersionComponentFloor { + component_kind: AuthorizationVersionComponentKind::Policy, + component_key: authorization_version_policy_component_key(community_id), + version: policy_version + .map(from_database_version) + .transpose()? + .unwrap_or(0), + }); + + let invalidation: Option = sqlx::query_scalar( + "SELECT current_generation FROM authorization_invalidation_domains WHERE community_id=$1", + ) + .bind(community_id.as_uuid()) + .fetch_optional(&mut *transaction) + .await?; + floors.push(AuthorizationVersionComponentFloor { + component_kind: AuthorizationVersionComponentKind::InvalidationGeneration, + component_key: authorization_version_invalidation_generation_component_key( + community_id, + ), + version: invalidation + .map(from_database_version) + .transpose()? + .unwrap_or(0), + }); + + let authority_rows = sqlx::query( + "SELECT object_kind, object_key, authority_epoch \ + FROM authorization_authority_epochs WHERE community_id=$1 \ + ORDER BY object_kind, object_key", + ) + .bind(community_id.as_uuid()) + .fetch_all(&mut *transaction) + .await?; + for row in authority_rows { + let object_key = bytes32(row.try_get("object_key")?, "authority object key")?; + let object_kind = + AuthorizationProtectedObjectKind::from_database(row.try_get("object_kind")?)?; + floors.push(AuthorizationVersionComponentFloor { + component_kind: AuthorizationVersionComponentKind::AuthorityEpoch, + component_key: authorization_version_authority_epoch_component_key( + community_id, + object_kind, + object_key, + ), + version: from_database_version(row.try_get("authority_epoch")?)?, + }); + } + + let relationship_rows = sqlx::query( + "SELECT selector_fingerprint, relationship_revision_floor \ + FROM authorization_invalidation_floors \ + WHERE community_id=$1 AND selector_kind=7 \ + ORDER BY selector_fingerprint", + ) + .bind(community_id.as_uuid()) + .fetch_all(&mut *transaction) + .await?; + for row in relationship_rows { + floors.push(AuthorizationVersionComponentFloor { + component_kind: AuthorizationVersionComponentKind::DelegatedRelationship, + component_key: bytes32( + row.try_get("selector_fingerprint")?, + "delegated relationship key", + )?, + version: from_database_version(row.try_get("relationship_revision_floor")?)?, + }); + } + + let selector_rows = sqlx::query( + "SELECT selector_id, fact_generation FROM identity_lifecycle_selectors \ + WHERE community_id=$1 ORDER BY selector_id", + ) + .bind(community_id.as_uuid()) + .fetch_all(&mut *transaction) + .await?; + for row in selector_rows { + let selector_id: Uuid = row.try_get("selector_id")?; + floors.push(AuthorizationVersionComponentFloor { + component_kind: AuthorizationVersionComponentKind::LifecycleSelector, + component_key: authorization_version_lifecycle_selector_component_key( + community_id, + selector_id, + ), + version: from_database_version(row.try_get("fact_generation")?)?, + }); + } + + transaction.commit().await?; + floors.sort_by_key(|floor| (floor.component_kind, floor.component_key)); + if floors.windows(2).any(|window| { + window[0].component_kind == window[1].component_kind + && window[0].component_key == window[1].component_key + }) { + return Err(DbError::InvalidData( + "authorization component floors are not unique".to_owned(), + )); + } + Ok(floors) + } +} + +fn build_manifest( + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + mut components: Vec, +) -> Result { + if community_id.as_uuid().is_nil() + || operation_id.is_nil() + || request_fingerprint == [0; 32] + || components.len() > MAX_COMPONENTS + { + return Err(DbError::InvalidData( + "authorization operation version manifest identity is invalid".to_owned(), + )); + } + components.sort_by_key(|component| (component.component_kind, component.component_key)); + let coordinates: BTreeSet<_> = components + .iter() + .map(|component| (component.component_kind, component.component_key)) + .collect(); + if coordinates.len() != components.len() { + return Err(DbError::InvalidData( + "authorization operation version manifest has duplicate components".to_owned(), + )); + } + + let before_digest = state_digest( + b"before", + community_id, + operation_id, + request_fingerprint, + &components, + false, + ); + let after_digest = state_digest( + b"after", + community_id, + operation_id, + request_fingerprint, + &components, + true, + ); + let mut digest = Sha256::new(); + framed(&mut digest, b"buzz:authorization-operation-manifest:v1"); + framed(&mut digest, community_id.as_uuid().as_bytes()); + framed(&mut digest, operation_id.as_bytes()); + framed(&mut digest, &request_fingerprint); + framed(&mut digest, &before_digest); + framed(&mut digest, &after_digest); + framed(&mut digest, &(components.len() as u64).to_be_bytes()); + for component in &components { + framed(&mut digest, &component.component_digest); + } + Ok(AuthorizationOperationVersionDeltaManifest { + community_id, + operation_id, + request_fingerprint, + before_digest, + after_digest, + manifest_digest: digest.finalize().into(), + components, + }) +} + +pub(crate) async fn load_manifest_connection( + connection: &mut PgConnection, + community_id: CommunityId, + operation_id: Uuid, +) -> Result> { + let Some(row) = sqlx::query( + "SELECT request_fingerprint, component_count, before_digest, after_digest, manifest_digest \ + FROM authorization_operation_version_delta_manifests \ + WHERE community_id=$1 AND operation_id=$2", + ) + .bind(community_id.as_uuid()) + .bind(operation_id) + .fetch_optional(&mut *connection) + .await? + else { + return Ok(None); + }; + + let request_fingerprint = bytes32(row.try_get("request_fingerprint")?, "request fingerprint")?; + let component_count: i32 = row.try_get("component_count")?; + if !(0..=MAX_COMPONENTS as i32).contains(&component_count) { + return Err(DbError::InvalidData( + "authorization operation component count is invalid".to_owned(), + )); + } + let component_rows = sqlx::query( + "SELECT component_kind, component_key, before_version, after_version, component_digest \ + FROM authorization_operation_version_deltas \ + WHERE community_id=$1 AND operation_id=$2 \ + ORDER BY component_kind, component_key", + ) + .bind(community_id.as_uuid()) + .bind(operation_id) + .fetch_all(&mut *connection) + .await?; + if component_rows.len() != component_count as usize { + return Err(DbError::InvalidData( + "authorization operation manifest is incomplete".to_owned(), + )); + } + let mut components = Vec::with_capacity(component_rows.len()); + for component_row in component_rows { + let kind = AuthorizationVersionComponentKind::from_database( + component_row.try_get("component_kind")?, + )?; + let key = bytes32(component_row.try_get("component_key")?, "component key")?; + let before = from_database_version(component_row.try_get("before_version")?)?; + let after = from_database_version(component_row.try_get("after_version")?)?; + let component = AuthorizationOperationVersionDelta::new(kind, key, before, after)?; + let stored_digest = bytes32( + component_row.try_get("component_digest")?, + "component digest", + )?; + if component.component_digest != stored_digest { + return Err(DbError::InvalidData( + "authorization operation component digest mismatch".to_owned(), + )); + } + components.push(component); + } + let rebuilt = build_manifest(community_id, operation_id, request_fingerprint, components)?; + if rebuilt.before_digest != bytes32(row.try_get("before_digest")?, "before digest")? + || rebuilt.after_digest != bytes32(row.try_get("after_digest")?, "after digest")? + || rebuilt.manifest_digest != bytes32(row.try_get("manifest_digest")?, "manifest digest")? + { + return Err(DbError::InvalidData( + "authorization operation manifest digest mismatch".to_owned(), + )); + } + Ok(Some(rebuilt)) +} + +fn component_digest( + kind: AuthorizationVersionComponentKind, + key: [u8; 32], + before: u64, + after: u64, +) -> [u8; 32] { + let mut digest = Sha256::new(); + framed(&mut digest, b"buzz:authorization-version-delta:v1"); + framed(&mut digest, &(kind as i16).to_be_bytes()); + framed(&mut digest, &key); + framed(&mut digest, &before.to_be_bytes()); + framed(&mut digest, &after.to_be_bytes()); + digest.finalize().into() +} + +fn state_digest( + label: &[u8], + community_id: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], + components: &[AuthorizationOperationVersionDelta], + after: bool, +) -> [u8; 32] { + let mut digest = Sha256::new(); + framed(&mut digest, b"buzz:authorization-version-state:v1"); + framed(&mut digest, label); + framed(&mut digest, community_id.as_uuid().as_bytes()); + framed(&mut digest, operation_id.as_bytes()); + framed(&mut digest, &request_fingerprint); + framed(&mut digest, &(components.len() as u64).to_be_bytes()); + for component in components { + framed( + &mut digest, + &(component.component_kind as i16).to_be_bytes(), + ); + framed(&mut digest, &component.component_key); + framed( + &mut digest, + &if after { + component.after_version + } else { + component.before_version + } + .to_be_bytes(), + ); + } + digest.finalize().into() +} + +fn framed(digest: &mut Sha256, value: &[u8]) { + digest.update((value.len() as u64).to_be_bytes()); + digest.update(value); +} + +fn bytes32(value: Vec, field: &str) -> Result<[u8; 32]> { + value.try_into().map_err(|_| { + DbError::InvalidData(format!( + "authorization {field} must contain exactly 32 bytes" + )) + }) +} + +fn authorization_lease_fence(value: Vec, field: &str) -> Result { + AuthorizationLeaseFence::from_bytes(bytes32(value, field)?) + .map_err(|_| DbError::InvalidData(format!("authorization {field} must be nonzero"))) +} + +fn to_database_version(value: u64) -> Result { + i64::try_from(value) + .map_err(|_| DbError::InvalidData("authorization version exceeds BIGINT".to_owned())) +} + +fn from_database_version(value: i64) -> Result { + u64::try_from(value) + .map_err(|_| DbError::InvalidData("authorization version is negative".to_owned())) +} + +#[cfg(test)] +mod tests { + use super::*; + use sqlx::{Acquire, PgPool}; + + use crate::authorization_events::{ + record_authorization_operation_receipt_tx, resolve_local_admission_loss_actor_tx, + AuthorizationOperationKind, AuthorizationOperationOutcome, AuthorizationOperationReceipt, + LocalAdmissionLossCause, + }; + + fn domain() -> CommunityId { + CommunityId::from_uuid(Uuid::from_u128(1)) + } + + #[test] + fn deltas_are_strict_bounded_and_redacted() { + let key = authorization_version_policy_component_key(domain()); + let delta = AuthorizationOperationVersionDelta::new( + AuthorizationVersionComponentKind::Policy, + key, + 7, + 9, + ) + .expect("strict non-unit advance"); + assert_eq!(delta.before_version(), 7); + assert_eq!(delta.after_version(), 9); + assert!(!format!("{delta:?}").contains(&hex::encode(key))); + assert!(AuthorizationOperationVersionDelta::new( + AuthorizationVersionComponentKind::Policy, + key, + 7, + 7, + ) + .is_err()); + assert!(AuthorizationOperationVersionDelta::new( + AuthorizationVersionComponentKind::Policy, + [0; 32], + 7, + 8, + ) + .is_err()); + } + + #[test] + fn manifests_are_canonical_and_empty_is_explicit() { + let operation = Uuid::from_u128(2); + let request = [3; 32]; + let first = AuthorizationOperationVersionDelta::new( + AuthorizationVersionComponentKind::Policy, + authorization_version_policy_component_key(domain()), + 1, + 2, + ) + .expect("policy delta"); + let second = AuthorizationOperationVersionDelta::new( + AuthorizationVersionComponentKind::InvalidationGeneration, + authorization_version_invalidation_generation_component_key(domain()), + 4, + 8, + ) + .expect("generation delta"); + let forward = build_manifest( + domain(), + operation, + request, + vec![first.clone(), second.clone()], + ) + .expect("forward manifest"); + let reverse = build_manifest(domain(), operation, request, vec![second, first]) + .expect("reverse manifest"); + assert_eq!(forward, reverse); + + let empty = build_manifest(domain(), operation, request, Vec::new()) + .expect("explicit empty manifest"); + assert!(empty.components().is_empty()); + assert_ne!(empty.manifest_digest(), [0; 32]); + } + + #[test] + fn component_namespaces_and_coordinates_do_not_alias() { + let binding = authorization_version_binding_component_key(domain(), Uuid::from_u128(9)); + let selector = + authorization_version_lifecycle_selector_component_key(domain(), Uuid::from_u128(9)); + let relation = authorization_version_delegated_relationship_component_key( + domain(), + Uuid::from_u128(9), + ); + assert_ne!(binding, selector); + assert_ne!(binding, relation); + assert_ne!(selector, relation); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn component_floor_provisioning_is_one_repeatable_read_snapshot() { + let admin_url = + std::env::var("TEST_DATABASE_URL").unwrap_or_else(|_| loopback_test_database_url()); + let admin = PgPool::connect(&admin_url).await.expect("connect admin"); + let name = format!("s5_floor_snapshot_{}", Uuid::new_v4().simple()); + sqlx::query(sqlx::AssertSqlSafe(format!("CREATE DATABASE {name}"))) + .execute(&admin) + .await + .expect("create scratch database"); + let split = admin_url.rfind('/').expect("database URL path"); + let scratch_url = format!("{}/{}", &admin_url[..split], name); + let pool = PgPool::connect(&scratch_url) + .await + .expect("connect scratch database"); + crate::migration::run_migrations(&pool) + .await + .expect("migrate scratch database"); + + let community = Uuid::new_v4(); + sqlx::query("INSERT INTO communities (id,host) VALUES ($1,$2)") + .bind(community) + .bind(format!("floor-{}.example", community.simple())) + .execute(&pool) + .await + .expect("insert community"); + let mut seed = pool.acquire().await.expect("seed connection"); + sqlx::query("SET session_replication_role=replica") + .execute(&mut *seed) + .await + .expect("disable seed triggers"); + insert_policy(&mut seed, community, 1).await; + insert_binding(&mut seed, community, 1).await; + sqlx::query("SET session_replication_role=origin") + .execute(&mut *seed) + .await + .expect("restore seed triggers"); + drop(seed); + + let mut blocker = pool.acquire().await.expect("blocker connection"); + sqlx::query("SET session_replication_role=replica") + .execute(&mut *blocker) + .await + .expect("disable blocker triggers"); + let mut blocker_tx = blocker.begin().await.expect("begin blocker"); + sqlx::query("LOCK TABLE identity_enrollment_policies IN ACCESS EXCLUSIVE MODE") + .execute(&mut *blocker_tx) + .await + .expect("lock policy table"); + + let db = Db::from_pool(pool.clone()); + let domain = CommunityId::from_uuid(community); + let reader = tokio::spawn(async move { + db.authorization_version_component_floors(domain) + .await + .expect("read component floors") + }); + tokio::time::sleep(std::time::Duration::from_millis(150)).await; + insert_policy(&mut blocker_tx, community, 2).await; + insert_binding(&mut blocker_tx, community, 2).await; + blocker_tx + .commit() + .await + .expect("commit concurrent advance"); + let floors = reader.await.expect("join floor reader"); + + let binding_count = floors + .iter() + .filter(|floor| floor.component_kind == AuthorizationVersionComponentKind::Binding) + .count() as u64; + let policy_version = floors + .iter() + .find(|floor| floor.component_kind == AuthorizationVersionComponentKind::Policy) + .expect("policy floor") + .version; + assert_eq!( + binding_count, policy_version, + "all floor classes must come from one snapshot" + ); + + let mut absent_tx = pool.begin().await.expect("begin absent authority check"); + let absent_actor = resolve_local_admission_loss_actor_tx( + &mut absent_tx, + domain, + LocalAdmissionLossCause::Policy { policy_revision: 1 }, + ) + .await + .expect("resolve absent policy actor"); + let absent = advance_admission_loss_authority_tx( + &mut absent_tx, + domain, + Uuid::new_v4(), + [50_u8; 32], + &absent_actor, + ) + .await + .expect("absence is an explicit valid result"); + assert!(absent.objects().is_empty()); + assert!(absent.into_parts().1.is_empty()); + absent_tx.rollback().await.expect("rollback absent check"); + + // Exercise the coupled AuthorityEpoch + protected-object refence seam + // on the same fresh schema. Seed the prior immutable authority through + // the test-only replication role, then require one exact atomic advance. + let old_operation = Uuid::new_v4(); + let new_operation = Uuid::new_v4(); + let object_key = [51_u8; 32]; + sqlx::query( + "INSERT INTO authorization_operation_receipts \ + (community_id,operation_id,request_fingerprint,operation_kind,actor_fingerprint, \ + outcome_code,result_digest) VALUES ($1,$2,$3,11,$4,1,$5)", + ) + .bind(community) + .bind(old_operation) + .bind(vec![52_u8; 32]) + .bind(vec![53_u8; 32]) + .bind(vec![54_u8; 32]) + .execute(&mut *blocker) + .await + .expect("seed prior authority receipt"); + sqlx::query( + "INSERT INTO authorization_authority_epochs \ + (community_id,object_kind,object_key,authority_epoch,fence,operation_id,request_fingerprint) \ + VALUES ($1,2,$2,1,$3,$4,$5)", + ) + .bind(community) + .bind(object_key.as_slice()) + .bind(vec![55_u8; 32]) + .bind(old_operation) + .bind(vec![52_u8; 32]) + .execute(&mut *blocker) + .await + .expect("seed authority epoch"); + sqlx::query( + "INSERT INTO protected_object_authority \ + (community_id,object_kind,object_key,capability,actor_pubkey,binding_id,binding_version, \ + policy_revision,invalidation_generation,authority_epoch,fence,issued_at,expires_at, \ + operation_id,request_fingerprint) \ + VALUES ($1,2,$2,1,$3,$4,2,2,0,1,$5,clock_timestamp()-INTERVAL '1 second', \ + clock_timestamp()+INTERVAL '1 hour',$6,$7)", + ) + .bind(community) + .bind(object_key.as_slice()) + .bind(vec![12_u8; 32]) + .bind(Uuid::from_u128(102)) + .bind(vec![55_u8; 32]) + .bind(old_operation) + .bind(vec![52_u8; 32]) + .execute(&mut *blocker) + .await + .expect("seed protected authority"); + sqlx::query("SET session_replication_role=origin") + .execute(&mut *blocker) + .await + .expect("restore blocker triggers"); + drop(blocker); + + let request_fingerprint = [61_u8; 32]; + let mut authority_tx = pool.begin().await.expect("begin authority advance"); + let actor = resolve_local_admission_loss_actor_tx( + &mut authority_tx, + domain, + LocalAdmissionLossCause::Policy { policy_revision: 2 }, + ) + .await + .expect("resolve policy actor"); + let advance = advance_admission_loss_authority_tx( + &mut authority_tx, + domain, + new_operation, + request_fingerprint, + &actor, + ) + .await + .expect("advance coupled authority"); + assert_eq!(advance.objects().len(), 1); + assert_eq!( + advance.objects()[0].coordinate(), + (AuthorizationProtectedObjectKind::Channel, object_key) + ); + assert_eq!(advance.objects()[0].epoch_and_fence().0, 2); + let (_, deltas) = advance.into_parts(); + let receipt = AuthorizationOperationReceipt::new( + domain, + new_operation, + request_fingerprint, + // The seam itself is operation-kind neutral; use kind 11 here so + // This focused test does not need to fabricate lifecycle history. + // The lifecycle integration test supplies the kind-9 history row. + AuthorizationOperationKind::ProtectedMutation, + actor, + AuthorizationOperationOutcome::Applied, + [62_u8; 32], + ) + .expect("authority receipt"); + record_authorization_operation_receipt_tx(&mut authority_tx, &receipt) + .await + .expect("record authority receipt"); + record_authorization_operation_version_delta_tx( + &mut authority_tx, + domain, + new_operation, + request_fingerprint, + deltas, + ) + .await + .expect("record authority delta"); + authority_tx + .commit() + .await + .expect("commit authority advance"); + let coupled: (i64, Vec, i64, Vec) = sqlx::query_as( + "SELECT epoch.authority_epoch,epoch.fence,protected.authority_epoch,protected.fence \ + FROM authorization_authority_epochs epoch JOIN protected_object_authority protected \ + USING (community_id,object_kind,object_key) \ + WHERE epoch.community_id=$1 AND epoch.object_kind=2 AND epoch.object_key=$2", + ) + .bind(community) + .bind(object_key.as_slice()) + .fetch_one(&pool) + .await + .expect("read coupled authority"); + assert_eq!(coupled.0, 2); + assert_eq!(coupled.2, 2); + assert_eq!(coupled.1, coupled.3); + + pool.close().await; + let _ = sqlx::query(sqlx::AssertSqlSafe(format!( + "DROP DATABASE IF EXISTS {name} WITH (FORCE)" + ))) + .execute(&admin) + .await; + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn supersession_proof_requires_one_complete_contiguous_manifest_chain() { + let admin_url = + std::env::var("TEST_DATABASE_URL").unwrap_or_else(|_| loopback_test_database_url()); + let admin = PgPool::connect(&admin_url).await.expect("connect admin"); + let name = format!("s5_supersession_{}", Uuid::new_v4().simple()); + sqlx::query(sqlx::AssertSqlSafe(format!("CREATE DATABASE {name}"))) + .execute(&admin) + .await + .expect("create scratch database"); + let split = admin_url.rfind('/').expect("database URL path"); + let scratch_url = format!("{}/{}", &admin_url[..split], name); + let pool = PgPool::connect(&scratch_url) + .await + .expect("connect scratch database"); + crate::migration::run_migrations(&pool) + .await + .expect("migrate scratch database"); + let community = Uuid::new_v4(); + sqlx::query("INSERT INTO communities (id,host) VALUES ($1,$2)") + .bind(community) + .bind(format!("supersession-{}.example", community.simple())) + .execute(&pool) + .await + .expect("insert community"); + let mut seed = pool.acquire().await.expect("seed connection"); + sqlx::query("SET session_replication_role=replica") + .execute(&mut *seed) + .await + .expect("disable seed triggers"); + for revision in 1..=3 { + insert_policy(&mut seed, community, revision).await; + } + sqlx::query("SET session_replication_role=origin") + .execute(&mut *seed) + .await + .expect("restore seed triggers"); + drop(seed); + + let domain = CommunityId::from_uuid(community); + let key = authorization_version_policy_component_key(domain); + let operations = [Uuid::new_v4(), Uuid::new_v4()]; + let requests = [[21_u8; 32], [22_u8; 32]]; + for (index, (operation, request)) in operations + .iter() + .copied() + .zip(requests.iter().copied()) + .enumerate() + { + let mut transaction = pool.begin().await.expect("begin causal operation"); + let actor = resolve_local_admission_loss_actor_tx( + &mut transaction, + domain, + LocalAdmissionLossCause::Policy { policy_revision: 3 }, + ) + .await + .expect("resolve policy actor"); + let delta = AuthorizationOperationVersionDelta::new( + AuthorizationVersionComponentKind::Policy, + key, + index as u64 + 1, + index as u64 + 2, + ) + .expect("strict causal delta"); + record_authorization_operation_version_delta_tx( + &mut transaction, + domain, + operation, + request, + vec![delta], + ) + .await + .expect("record causal manifest"); + let receipt = AuthorizationOperationReceipt::new( + domain, + operation, + request, + AuthorizationOperationKind::ProtectedMutation, + actor, + AuthorizationOperationOutcome::Applied, + [u8::try_from(31 + index).expect("small"); 32], + ) + .expect("causal receipt"); + record_authorization_operation_receipt_tx(&mut transaction, &receipt) + .await + .expect("record causal receipt"); + transaction.commit().await.expect("commit causal operation"); + } + + let db = Db::from_pool(pool.clone()); + let latest = db + .authorization_operation_version_delta(domain, operations[1], requests[1]) + .await + .expect("load latest manifest"); + let latest_component = latest.components().first().expect("latest component"); + db.authorization_version_proves_component_supersession( + domain, + AuthorizationVersionComponentKind::Policy, + key, + 1, + 3, + operations[1], + requests[1], + latest_component.component_digest(), + latest.manifest_digest(), + ) + .await + .expect("two-step exact provenance proves supersession"); + + let branch_operation = Uuid::new_v4(); + let branch_request = [23_u8; 32]; + let mut branch = pool.begin().await.expect("begin overlapping branch"); + let branch_actor = resolve_local_admission_loss_actor_tx( + &mut branch, + domain, + LocalAdmissionLossCause::Policy { policy_revision: 3 }, + ) + .await + .expect("resolve branch actor"); + record_authorization_operation_version_delta_tx( + &mut branch, + domain, + branch_operation, + branch_request, + vec![AuthorizationOperationVersionDelta::new( + AuthorizationVersionComponentKind::Policy, + key, + 1, + 3, + ) + .expect("overlapping branch delta")], + ) + .await + .expect("record overlapping manifest"); + let branch_receipt = AuthorizationOperationReceipt::new( + domain, + branch_operation, + branch_request, + AuthorizationOperationKind::ProtectedMutation, + branch_actor, + AuthorizationOperationOutcome::Applied, + [33_u8; 32], + ) + .expect("branch receipt"); + record_authorization_operation_receipt_tx(&mut branch, &branch_receipt) + .await + .expect("record branch receipt"); + branch.commit().await.expect("commit overlapping branch"); + assert!(db + .authorization_version_proves_component_supersession( + domain, + AuthorizationVersionComponentKind::Policy, + key, + 1, + 3, + operations[1], + requests[1], + latest_component.component_digest(), + latest.manifest_digest(), + ) + .await + .is_err()); + + pool.close().await; + let _ = sqlx::query(sqlx::AssertSqlSafe(format!( + "DROP DATABASE IF EXISTS {name} WITH (FORCE)" + ))) + .execute(&admin) + .await; + admin.close().await; + } + + async fn insert_policy(connection: &mut PgConnection, community: Uuid, revision: i64) { + sqlx::query( + "INSERT INTO identity_enrollment_policies \ + (community_id,policy_revision,enrollment_mode,policy_digest,effective_at) \ + VALUES ($1,$2,1,$3,clock_timestamp() - INTERVAL '1 second')", + ) + .bind(community) + .bind(revision) + .bind(vec![revision as u8; 32]) + .execute(&mut *connection) + .await + .expect("insert policy"); + } + + async fn insert_binding(connection: &mut PgConnection, community: Uuid, ordinal: u8) { + sqlx::query( + "INSERT INTO identity_bindings \ + (community_id,binding_id,issuer,subject,principal_fingerprint,event_author_pubkey, \ + binding_state,lifecycle_revision,binding_provenance,policy_revision, \ + enrollment_evidence_digest,birth_history_id,creation_operation_id, \ + creation_request_fingerprint) \ + VALUES ($1,$2,$3,$4,$5,$6,1,1,1,$7,$8,$9,$10,$11)", + ) + .bind(community) + .bind(Uuid::from_u128(u128::from(ordinal) + 100)) + .bind(format!("issuer-{ordinal}")) + .bind(format!("subject-{ordinal}")) + .bind(vec![ordinal; 32]) + .bind(vec![ordinal.saturating_add(10); 32]) + .bind(i64::from(ordinal)) + .bind(vec![ordinal.saturating_add(20); 32]) + .bind(Uuid::from_u128(u128::from(ordinal) + 200)) + .bind(Uuid::from_u128(u128::from(ordinal) + 300)) + .bind(vec![ordinal.saturating_add(30); 32]) + .execute(&mut *connection) + .await + .expect("insert binding"); + } +} diff --git a/crates/buzz-db/src/authorization_version/protected.rs b/crates/buzz-db/src/authorization_version/protected.rs new file mode 100644 index 00000000000..5af294f0d38 --- /dev/null +++ b/crates/buzz-db/src/authorization_version/protected.rs @@ -0,0 +1,2549 @@ +//! High-level DB-authoritative publication for protected immutable content. +//! +//! Content bytes are staged under their immutable digest before this API is +//! called. PostgreSQL then owns visibility: one transaction re-fences the +//! sealed route, advances the exact protected-object authority, and records +//! the canonical receipt, event, and version manifest. External pointers are +//! caches and must not be exposed without a final witness recheck. + +use std::fmt; + +use buzz_auth::{ + AuthorizationLeaseDependencySnapshot, FinalizedAuthContext, ProofTransport, RouteCapability, +}; +use buzz_core::{AuthorizationLeaseFence, CommunityId}; +use chrono::{DateTime, Duration, Utc}; +use sha2::{Digest, Sha256}; +use sqlx::{Postgres, Row, Transaction}; +use thiserror::Error; +use uuid::Uuid; + +use super::{ + authorization_lease_fence, authorization_version_authority_epoch_component_key, bytes32, + framed, from_database_version, load_manifest_connection, next_authority_fence, + record_authorization_operation_version_delta_tx, to_database_version, + AuthorizationOperationFence, AuthorizationOperationVersionDelta, + AuthorizationProtectedObjectKind, AuthorizationVersionComponentKind, +}; +use crate::{ + authorization_events::{ + record_authorization_event_tx, record_authorization_operation_receipt_tx, + AuthorizationAuditFailureCode, AuthorizationEventActor, AuthorizationEventKind, + AuthorizationEventOutcome, AuthorizationEventWriteError, AuthorizationOperationKind, + AuthorizationOperationOutcome, AuthorizationOperationReceipt, AuthorizationReasonCode, + AuthorizationReceiptWrite, NewAuthorizationEvent, + }, + authorization_invalidation::AuthorizationInvalidationSelector, + Db, DbError, Result, +}; + +/// Canonical protected object derived from an origin-sealed route. +#[derive(Clone, PartialEq, Eq)] +pub struct ProtectedPublicationCoordinate { + community_id: CommunityId, + object_kind: AuthorizationProtectedObjectKind, + object_key: [u8; 32], + target_fingerprint: [u8; 32], +} + +impl ProtectedPublicationCoordinate { + /// Derive the canonical repository coordinate for a Git/repository route. + pub fn repository(context: &FinalizedAuthContext) -> Result { + Self::from_context(context, AuthorizationProtectedObjectKind::Repository) + } + + /// Derive the canonical immutable-media coordinate for a media route. + pub fn media(context: &FinalizedAuthContext) -> Result { + Self::from_context(context, AuthorizationProtectedObjectKind::Media) + } + + /// Derive the canonical moderation-target coordinate. + pub fn moderation_target(context: &FinalizedAuthContext) -> Result { + Self::from_context(context, AuthorizationProtectedObjectKind::ModerationTarget) + } + + fn from_context( + context: &FinalizedAuthContext, + object_kind: AuthorizationProtectedObjectKind, + ) -> Result { + if !capability_reads_object(context.capability(), object_kind) { + return Err(DbError::InvalidData( + "authorization route cannot address this protected object".to_owned(), + )); + } + let (_, target_fingerprint, _) = context.lease().request_binding(); + if *target_fingerprint == [0; 32] { + return Err(DbError::InvalidData( + "authorization protected target is invalid".to_owned(), + )); + } + let object_key = protected_object_key( + context.authorization_domain(), + object_kind, + *target_fingerprint, + ); + Ok(Self { + community_id: context.authorization_domain(), + object_kind, + object_key, + target_fingerprint: *target_fingerprint, + }) + } + + /// Server-resolved authorization domain. + pub const fn authorization_domain(&self) -> CommunityId { + self.community_id + } + + /// Closed protected-object class. + pub const fn object_kind(&self) -> AuthorizationProtectedObjectKind { + self.object_kind + } +} + +impl fmt::Debug for ProtectedPublicationCoordinate { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ProtectedPublicationCoordinate") + .field("object_kind", &self.object_kind) + .field("coordinate", &"[REDACTED]") + .finish() + } +} + +/// Server-generated identities for one publication attempt. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct ProtectedPublicationOperationIdentity { + operation_id: Uuid, + event_id: Uuid, + attempt_id: Uuid, +} + +/// Read-only restore coordinates sealed by a validated publication request. +/// +/// Callers cannot construct or alter these coordinates; operation restore +/// consumes the exact request fingerprint computed by the database boundary. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct ProtectedPublicationRestoreIdentity { + authorization_domain: CommunityId, + operation_id: Uuid, + request_fingerprint: [u8; 32], +} + +impl ProtectedPublicationRestoreIdentity { + /// Exact server-resolved authorization domain. + pub const fn authorization_domain(self) -> CommunityId { + self.authorization_domain + } + + /// Exact server-generated operation receipt identity. + pub const fn operation_id(self) -> Uuid { + self.operation_id + } + + /// Canonical fingerprint of the complete sealed publication request. + pub const fn request_fingerprint(self) -> [u8; 32] { + self.request_fingerprint + } +} + +impl fmt::Debug for ProtectedPublicationRestoreIdentity { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("ProtectedPublicationRestoreIdentity([REDACTED])") + } +} + +impl ProtectedPublicationOperationIdentity { + /// Validate separately generated operation, event, and attempt IDs. + pub fn new(operation_id: Uuid, event_id: Uuid, attempt_id: Uuid) -> Result { + if operation_id.is_nil() || event_id.is_nil() || attempt_id.is_nil() { + return Err(DbError::InvalidData( + "authorization protected publication identity is invalid".to_owned(), + )); + } + Ok(Self { + operation_id, + event_id, + attempt_id, + }) + } + + /// Stable operation receipt identity. + pub const fn operation_id(&self) -> Uuid { + self.operation_id + } +} + +impl fmt::Debug for ProtectedPublicationOperationIdentity { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("ProtectedPublicationOperationIdentity([REDACTED])") + } +} + +/// Sealed current DB publication authority returned to immutable fetchers. +#[derive(Clone)] +pub struct ProtectedPublicationWitness { + coordinate: ProtectedPublicationCoordinate, + dependency: ProtectedPublicationDependency, + result_digest: [u8; 32], + authority_epoch: u64, + invalidation_generation: u64, + fence: AuthorizationLeaseFence, + authoritative_now: DateTime, + expires_at: DateTime, + operation_id: Uuid, + request_fingerprint: [u8; 32], +} + +impl ProtectedPublicationWitness { + /// Exact immutable content digest committed by the canonical receipt. + pub const fn result_digest(&self) -> [u8; 32] { + self.result_digest + } + + /// Current protected-object authority epoch. + pub const fn authority_epoch(&self) -> u64 { + self.authority_epoch + } + + /// Current nonzero observable fence. + pub const fn fence(&self) -> AuthorizationLeaseFence { + self.fence + } + + /// Domain generation included in the joined dependency snapshot. + pub const fn invalidation_generation(&self) -> u64 { + self.invalidation_generation + } + + /// PostgreSQL time sampled by the query that sealed this snapshot. + pub const fn authoritative_now(&self) -> DateTime { + self.authoritative_now + } + + /// Persisted exclusive lease expiry for this publication authority. + pub const fn expires_at(&self) -> DateTime { + self.expires_at + } + + /// Canonical protected object this witness seals. + pub const fn coordinate(&self) -> &ProtectedPublicationCoordinate { + &self.coordinate + } + + /// Opaque dependency coordinate used by the live observer. + pub const fn dependency(&self) -> &ProtectedPublicationDependency { + &self.dependency + } +} + +impl PartialEq for ProtectedPublicationWitness { + fn eq(&self, other: &Self) -> bool { + self.coordinate == other.coordinate + && self.dependency == other.dependency + && self.result_digest == other.result_digest + && self.authority_epoch == other.authority_epoch + && self.invalidation_generation == other.invalidation_generation + && self.fence == other.fence + && self.expires_at == other.expires_at + && self.operation_id == other.operation_id + && self.request_fingerprint == other.request_fingerprint + } +} + +/// Opaque exact protected-object dependency coordinate. +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct ProtectedPublicationDependency { + community_id: CommunityId, + object_kind: AuthorizationProtectedObjectKind, + object_key: [u8; 32], +} + +impl ProtectedPublicationDependency { + fn from_coordinate(coordinate: &ProtectedPublicationCoordinate) -> Self { + Self { + community_id: coordinate.community_id, + object_kind: coordinate.object_kind, + object_key: coordinate.object_key, + } + } + + pub(crate) fn from_database_parts( + community_id: CommunityId, + object_kind: AuthorizationProtectedObjectKind, + object_key: [u8; 32], + ) -> Result { + if community_id.as_uuid().is_nil() || object_key == [0; 32] { + return Err(DbError::InvalidData( + "authorization publication dependency is invalid".to_owned(), + )); + } + Ok(Self { + community_id, + object_kind, + object_key, + }) + } + + /// Server-resolved authorization domain. + pub const fn authorization_domain(&self) -> CommunityId { + self.community_id + } + + pub(crate) const fn object_kind(&self) -> AuthorizationProtectedObjectKind { + self.object_kind + } + + pub(crate) const fn object_key(&self) -> [u8; 32] { + self.object_key + } +} + +impl fmt::Debug for ProtectedPublicationDependency { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("ProtectedPublicationDependency([REDACTED])") + } +} + +/// PostgreSQL time and exact persisted expiry returned by a final recheck. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct ProtectedPublicationValidity { + authoritative_now: DateTime, + expires_at: DateTime, +} + +impl ProtectedPublicationValidity { + /// PostgreSQL time sampled during the final recheck. + pub const fn authoritative_now(self) -> DateTime { + self.authoritative_now + } + + /// Persisted exclusive authority expiry. + pub const fn expires_at(self) -> DateTime { + self.expires_at + } +} + +impl fmt::Debug for ProtectedPublicationValidity { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("ProtectedPublicationValidity([REDACTED])") + } +} + +impl Eq for ProtectedPublicationWitness {} + +impl fmt::Debug for ProtectedPublicationWitness { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ProtectedPublicationWitness") + .field("object_kind", &self.coordinate.object_kind) + .field("authority", &"[REDACTED]") + .field("result_digest", &"[REDACTED]") + .finish() + } +} + +/// One staged immutable publication and its exact expected parent. +pub struct ProtectedPublicationRequest { + coordinate: ProtectedPublicationCoordinate, + operation: ProtectedPublicationOperationIdentity, + authority: ProtectedPublicationAuthority, + staged_digest: [u8; 32], + expected_parent_result_digest: Option<[u8; 32]>, + request_fingerprint: [u8; 32], +} + +impl ProtectedPublicationRequest { + /// Bind a nonzero staged digest and optional expected parent to one sealed route. + pub fn new( + context: &FinalizedAuthContext, + coordinate: ProtectedPublicationCoordinate, + operation: ProtectedPublicationOperationIdentity, + staged_digest: [u8; 32], + expected_parent_result_digest: Option<[u8; 32]>, + ) -> Result { + if staged_digest == [0; 32] + || expected_parent_result_digest == Some([0; 32]) + || coordinate.community_id != context.authorization_domain() + || !capability_mutates_object(context.capability(), coordinate.object_kind) + { + return Err(DbError::InvalidData( + "authorization protected publication request is invalid".to_owned(), + )); + } + let authority = ProtectedPublicationAuthority::from_context(context)?; + if authority.target_fingerprint != coordinate.target_fingerprint { + return Err(DbError::InvalidData( + "authorization protected publication target changed".to_owned(), + )); + } + let request_fingerprint = publication_request_fingerprint( + &coordinate, + operation, + &authority, + staged_digest, + expected_parent_result_digest, + ); + Ok(Self { + coordinate, + operation, + authority, + staged_digest, + expected_parent_result_digest, + request_fingerprint, + }) + } + + /// Exact read-only operation restore identity sealed with this request. + pub const fn restore_identity(&self) -> ProtectedPublicationRestoreIdentity { + ProtectedPublicationRestoreIdentity { + authorization_domain: self.coordinate.community_id, + operation_id: self.operation.operation_id, + request_fingerprint: self.request_fingerprint, + } + } +} + +impl fmt::Debug for ProtectedPublicationRequest { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ProtectedPublicationRequest") + .field("object_kind", &self.coordinate.object_kind) + .field("operation", &self.operation) + .field("payload", &"[REDACTED]") + .finish() + } +} + +/// Whether a protected publication created state or exactly replayed it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProtectedPublicationDisposition { + /// New publication authority was committed. + Applied, + /// The complete prior receipt, event, manifest, and authority replayed. + ExactReplay, +} + +/// Canonical committed result of a staged publication. +#[derive(Clone, PartialEq, Eq)] +pub struct ProtectedPublicationCommit { + disposition: ProtectedPublicationDisposition, + witness: ProtectedPublicationWitness, +} + +impl ProtectedPublicationCommit { + /// Applied versus exact replay. + pub const fn disposition(&self) -> ProtectedPublicationDisposition { + self.disposition + } + + /// Exact immutable digest committed by the canonical receipt. + pub const fn result_digest(&self) -> [u8; 32] { + self.witness.result_digest + } + + /// Current read witness for the committed publication. + pub const fn witness(&self) -> &ProtectedPublicationWitness { + &self.witness + } + + /// Consume the result and retain its read witness. + pub fn into_witness(self) -> ProtectedPublicationWitness { + self.witness + } +} + +impl fmt::Debug for ProtectedPublicationCommit { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("ProtectedPublicationCommit") + .field("disposition", &self.disposition) + .field("witness", &self.witness) + .finish() + } +} + +/// Fail-closed publication outcome. +#[derive(Debug, Error)] +pub enum ProtectedPublicationError { + /// The expected parent or operation intent no longer matches. + #[error("protected publication conflicts with current authority")] + Conflict, + /// The sealed route or final dependency fence is stale. + #[error("protected publication authorization is stale")] + StaleAuthorization, + /// Canonical durable authorization evidence could not be recorded. + #[error("protected publication audit is unavailable")] + AuditUnavailable, + /// PostgreSQL contract or storage failure outside the canonical event write. + #[error(transparent)] + Database(#[from] DbError), +} + +#[derive(Clone)] +struct ProtectedPublicationAuthority { + community_id: CommunityId, + correlation_id: Uuid, + actor: AuthorizationEventActor, + capability: RouteCapability, + actor_pubkey: [u8; 32], + owner_pubkey: Option<[u8; 32]>, + binding_id: Uuid, + binding_version: u64, + relationship_id: Option, + relationship_revision: Option, + delegation_conditions_fingerprint: Option<[u8; 32]>, + source_request_fingerprint: [u8; 32], + target_fingerprint: [u8; 32], + transport: ProofTransport, + transport_context_fingerprint: [u8; 32], + policy_revision: u64, + invalidation_generation: u64, + authority_epoch: u64, + fence: AuthorizationLeaseFence, + issued_at: DateTime, + expires_at: DateTime, +} + +impl ProtectedPublicationAuthority { + fn from_context(context: &FinalizedAuthContext) -> Result { + let actor = AuthorizationEventActor::from_auth_context(context)?; + let snapshot = context.lease().dependency_snapshot(); + Self::from_snapshot(context.correlation_id(), actor, snapshot) + } + + fn from_snapshot( + correlation_id: Uuid, + actor: AuthorizationEventActor, + snapshot: AuthorizationLeaseDependencySnapshot, + ) -> Result { + let (_, community_id) = snapshot.identity(); + let (capability, actor_pubkey, owner_pubkey) = snapshot.authority(); + let (binding_id, binding_version) = snapshot.binding(); + let (source_request_fingerprint, target_fingerprint, transport, transport_context) = + snapshot.request_binding(); + let (policy_revision, invalidation_generation, authority_epoch) = + snapshot.dependency_versions(); + let (issued_at, expires_at) = snapshot.time_bounds(); + let delegated = snapshot.delegated_relationship(); + if correlation_id.is_nil() + || community_id.as_uuid().is_nil() + || binding_id.is_nil() + || binding_version == 0 + || policy_revision == 0 + || authority_epoch == 0 + || *source_request_fingerprint == [0; 32] + || *target_fingerprint == [0; 32] + || *transport_context == [0; 32] + || issued_at >= expires_at + || !actor.is_bound_to(community_id) + || owner_pubkey.is_some() != delegated.is_some() + { + return Err(DbError::InvalidData( + "authorization protected publication authority is invalid".to_owned(), + )); + } + let (relationship_id, relationship_revision, delegation_conditions_fingerprint) = + match delegated { + Some((id, revision, conditions)) + if !id.is_nil() && revision > 0 && *conditions != [0; 32] => + { + (Some(id), Some(revision), Some(*conditions)) + } + Some(_) => { + return Err(DbError::InvalidData( + "authorization delegated publication authority is invalid".to_owned(), + )); + } + None => (None, None, None), + }; + Ok(Self { + community_id, + correlation_id, + actor, + capability, + actor_pubkey: actor_pubkey.to_bytes(), + owner_pubkey: owner_pubkey.map(|key| key.to_bytes()), + binding_id, + binding_version, + relationship_id, + relationship_revision, + delegation_conditions_fingerprint, + source_request_fingerprint: *source_request_fingerprint, + target_fingerprint: *target_fingerprint, + transport, + transport_context_fingerprint: *transport_context, + policy_revision, + invalidation_generation, + authority_epoch, + fence: snapshot.fence(), + issued_at, + expires_at, + }) + } +} + +impl fmt::Debug for ProtectedPublicationAuthority { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("ProtectedPublicationAuthority([REDACTED])") + } +} + +#[cfg(test)] +#[allow(clippy::too_many_arguments)] +pub(crate) fn protected_publication_request_for_restore_test( + community_id: CommunityId, + actor_pubkey: [u8; 32], + binding_id: Uuid, + binding_version: u64, + fence: AuthorizationLeaseFence, + invalidation_generation: u64, + authority_epoch: u64, + operation: ProtectedPublicationOperationIdentity, + staged_digest: [u8; 32], + expected_parent_result_digest: Option<[u8; 32]>, +) -> Result<(ProtectedPublicationCoordinate, ProtectedPublicationRequest)> { + let target_fingerprint = [21_u8; 32]; + let coordinate = ProtectedPublicationCoordinate { + community_id, + object_kind: AuthorizationProtectedObjectKind::Repository, + object_key: protected_object_key( + community_id, + AuthorizationProtectedObjectKind::Repository, + target_fingerprint, + ), + target_fingerprint, + }; + let now = Utc::now(); + let authority = ProtectedPublicationAuthority { + community_id, + correlation_id: Uuid::new_v4(), + actor: AuthorizationEventActor::test_direct(community_id), + capability: RouteCapability::GitWrite, + actor_pubkey, + owner_pubkey: None, + binding_id, + binding_version, + relationship_id: None, + relationship_revision: None, + delegation_conditions_fingerprint: None, + source_request_fingerprint: [22_u8; 32], + target_fingerprint, + transport: ProofTransport::Nip42, + transport_context_fingerprint: [23_u8; 32], + policy_revision: 1, + invalidation_generation, + authority_epoch, + fence, + issued_at: now, + expires_at: now + Duration::minutes(5), + }; + let request_fingerprint = publication_request_fingerprint( + &coordinate, + operation, + &authority, + staged_digest, + expected_parent_result_digest, + ); + Ok(( + coordinate.clone(), + ProtectedPublicationRequest { + coordinate, + operation, + authority, + staged_digest, + expected_parent_result_digest, + request_fingerprint, + }, + )) +} + +struct LockedPublicationState { + current: Option, + prior_issued_at: Option>, +} + +enum PublicationTxError { + Conflict, + Stale, + Audit(AuthorizationAuditFailureCode), + Database(DbError), +} + +impl From for PublicationTxError { + fn from(value: DbError) -> Self { + Self::Database(value) + } +} + +impl From for PublicationTxError { + fn from(value: sqlx::Error) -> Self { + Self::Database(value.into()) + } +} + +impl Db { + /// Commit on the exact physical session whose restore fence was acquired + /// before the external Pending reservation. + pub(crate) async fn commit_staged_protected_publication_fenced( + &self, + fence: &mut AuthorizationOperationFence, + request: &ProtectedPublicationRequest, + ) -> std::result::Result { + let identity = request.restore_identity(); + let connection = fence + .connection_for( + identity.authorization_domain(), + identity.operation_id(), + identity.request_fingerprint(), + ) + .map_err(ProtectedPublicationError::Database)?; + let community_id = request.coordinate.community_id; + let mut transaction = Transaction::begin(&mut *connection, None) + .await + .map_err(DbError::from)?; + let result = commit_publication_tx(&mut transaction, request).await; + match result { + Ok(commit) => { + if let Err(error) = transaction.commit().await { + let _ = self + .latch_authorization_event_failure( + community_id, + AuthorizationAuditFailureCode::StorageUnavailable, + ) + .await; + return Err(ProtectedPublicationError::Database(error.into())); + } + Ok(commit) + } + Err(error) => { + let _ = transaction.rollback().await; + match error { + PublicationTxError::Conflict => Err(ProtectedPublicationError::Conflict), + PublicationTxError::Stale => Err(ProtectedPublicationError::StaleAuthorization), + PublicationTxError::Audit(failure) => { + let _ = self + .latch_authorization_event_failure(community_id, failure) + .await; + Err(ProtectedPublicationError::AuditUnavailable) + } + PublicationTxError::Database(error) => { + Err(ProtectedPublicationError::Database(error)) + } + } + } + } + } + + /// Resolve the current canonical publication witness for one sealed coordinate. + pub async fn protected_publication_witness( + &self, + coordinate: &ProtectedPublicationCoordinate, + ) -> Result> { + load_witness_pool(self, coordinate).await + } + + /// Final post-fetch recheck immediately before any refs, headers, ranges, or bytes. + pub async fn recheck_protected_publication_witness( + &self, + witness: &ProtectedPublicationWitness, + ) -> Result { + Ok(self + .recheck_protected_publication_validity(witness) + .await? + .is_some()) + } + + /// Final joined recheck with the PostgreSQL time sample needed for a + /// conservative monotonic expiry deadline. + pub async fn recheck_protected_publication_validity( + &self, + witness: &ProtectedPublicationWitness, + ) -> Result> { + match load_witness_pool(self, &witness.coordinate).await? { + Some(current) if current == *witness => Ok(Some(ProtectedPublicationValidity { + authoritative_now: current.authoritative_now, + expires_at: current.expires_at, + })), + _ => Ok(None), + } + } + + /// Recheck a freshly finalized route against one atomic PostgreSQL snapshot. + /// + /// This is the last allow fence before transport code may construct + /// protected access. Publication commit performs the same checks again + /// under its writer locks. + pub async fn recheck_protected_publication_authorization( + &self, + context: &FinalizedAuthContext, + coordinate: &ProtectedPublicationCoordinate, + ) -> Result { + let authority = ProtectedPublicationAuthority::from_context(context)?; + if authority.community_id != coordinate.community_id + || authority.target_fingerprint != coordinate.target_fingerprint + || !capability_reads_object(authority.capability, coordinate.object_kind) + { + return Ok(false); + } + + let mut transaction = self.pool.begin().await?; + sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") + .execute(&mut *transaction) + .await?; + match refence_publication_lease(&mut transaction, &authority).await { + Ok(()) => {} + Err(PublicationTxError::Stale) => { + transaction.rollback().await?; + return Ok(false); + } + Err(PublicationTxError::Conflict | PublicationTxError::Audit(_)) => { + transaction.rollback().await?; + return Ok(false); + } + Err(PublicationTxError::Database(error)) => { + transaction.rollback().await?; + return Err(error); + } + } + + let row = sqlx::query( + "SELECT epoch.authority_epoch AS epoch_authority_epoch, \ + epoch.fence AS epoch_fence, \ + protected.authority_epoch AS protected_authority_epoch, \ + protected.fence AS protected_fence \ + FROM (VALUES (1)) AS seed(value) \ + LEFT JOIN authorization_authority_epochs epoch \ + ON epoch.community_id=$1 AND epoch.object_kind=$2 AND epoch.object_key=$3 \ + LEFT JOIN protected_object_authority protected \ + ON protected.community_id=$1 AND protected.object_kind=$2 \ + AND protected.object_key=$3", + ) + .bind(coordinate.community_id.as_uuid()) + .bind(coordinate.object_kind as i16) + .bind(coordinate.object_key.as_slice()) + .fetch_one(&mut *transaction) + .await?; + transaction.commit().await?; + + let epoch = row + .try_get::, _>("epoch_authority_epoch")? + .map(from_database_version) + .transpose()?; + let epoch_fence = row + .try_get::>, _>("epoch_fence")? + .map(|value| authorization_lease_fence(value, "authority fence")) + .transpose()?; + let protected = row + .try_get::, _>("protected_authority_epoch")? + .map(from_database_version) + .transpose()?; + let protected_fence = row + .try_get::>, _>("protected_fence")? + .map(|value| authorization_lease_fence(value, "protected fence")) + .transpose()?; + Ok(match (epoch, epoch_fence, protected, protected_fence) { + (None, None, None, None) => authority.authority_epoch == 1, + (Some(epoch), Some(epoch_fence), Some(protected), Some(protected_fence)) => { + epoch == authority.authority_epoch + && protected == authority.authority_epoch + && epoch_fence == authority.fence + && protected_fence == authority.fence + } + _ => false, + }) + } + + /// Atomically make one already-staged immutable publication authoritative. + pub async fn commit_staged_protected_publication( + &self, + _request: ProtectedPublicationRequest, + ) -> std::result::Result { + #[cfg(test)] + { + return self + .commit_staged_protected_publication_unfenced_for_test(&_request) + .await; + } + #[cfg(not(test))] + Err(ProtectedPublicationError::Database(DbError::InvalidData( + "protected publication requires an operation restore fence".to_owned(), + ))) + } + + /// Borrowed publication commit/replay entrypoint for operation restore. + /// + /// Retaining the sealed request lets a caller reconcile an ambiguous + /// database outcome and then invoke the exact PostgreSQL replay path to + /// reconstruct the canonical committed witness and result digest. + pub async fn commit_staged_protected_publication_ref( + &self, + _request: &ProtectedPublicationRequest, + ) -> std::result::Result { + #[cfg(test)] + { + return self + .commit_staged_protected_publication_unfenced_for_test(_request) + .await; + } + #[cfg(not(test))] + Err(ProtectedPublicationError::Database(DbError::InvalidData( + "protected publication requires an operation restore fence".to_owned(), + ))) + } + + #[cfg(test)] + async fn commit_staged_protected_publication_unfenced_for_test( + &self, + request: &ProtectedPublicationRequest, + ) -> std::result::Result { + let community_id = request.coordinate.community_id; + let mut transaction = self.pool.begin().await.map_err(DbError::from)?; + let result = commit_publication_tx(&mut transaction, request).await; + match result { + Ok(commit) => { + if let Err(error) = transaction.commit().await { + let _ = self + .latch_authorization_event_failure( + community_id, + AuthorizationAuditFailureCode::StorageUnavailable, + ) + .await; + return Err(ProtectedPublicationError::Database(error.into())); + } + Ok(commit) + } + Err(error) => { + let _ = transaction.rollback().await; + match error { + PublicationTxError::Conflict => Err(ProtectedPublicationError::Conflict), + PublicationTxError::Stale => Err(ProtectedPublicationError::StaleAuthorization), + PublicationTxError::Audit(failure) => { + let _ = self + .latch_authorization_event_failure(community_id, failure) + .await; + Err(ProtectedPublicationError::AuditUnavailable) + } + PublicationTxError::Database(error) => { + Err(ProtectedPublicationError::Database(error)) + } + } + } + } + } +} + +async fn commit_publication_tx( + transaction: &mut Transaction<'_, Postgres>, + request: &ProtectedPublicationRequest, +) -> std::result::Result { + acquire_publication_lock(transaction, &request.coordinate).await?; + + let receipt = publication_receipt(request)?; + let event = publication_event(request)?; + if record_authorization_operation_receipt_tx(transaction, &receipt).await? + == AuthorizationReceiptWrite::ExactReplay + { + let locked = lock_publication_state(transaction, &request.coordinate).await?; + return validate_publication_replay(transaction, request, &event, locked.current).await; + } + + let locked = lock_publication_state(transaction, &request.coordinate).await?; + refence_publication_lease(transaction, &request.authority).await?; + if locked + .current + .as_ref() + .map(ProtectedPublicationWitness::result_digest) + != request.expected_parent_result_digest + { + return Err(PublicationTxError::Conflict); + } + + let before_epoch = locked + .current + .as_ref() + .map_or(0, ProtectedPublicationWitness::authority_epoch); + if before_epoch == 0 && request.authority.authority_epoch != 1 { + return Err(PublicationTxError::Stale); + } + if before_epoch > 0 + && (request.authority.authority_epoch != before_epoch + || locked + .current + .as_ref() + .map(ProtectedPublicationWitness::fence) + != Some(request.authority.fence)) + { + return Err(PublicationTxError::Stale); + } + let after_epoch = if before_epoch == 0 { + 1 + } else { + before_epoch.checked_add(1).ok_or_else(|| { + PublicationTxError::Database(DbError::InvalidData( + "authorization protected publication epoch exhausted".to_owned(), + )) + })? + }; + let next_fence = if before_epoch == 0 { + request.authority.fence + } else { + next_authority_fence( + request.coordinate.community_id, + request.operation.operation_id, + request.request_fingerprint, + request.coordinate.object_kind, + request.coordinate.object_key, + before_epoch, + request.authority.fence, + )? + }; + let now: DateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut **transaction) + .await?; + if !lease_valid_at(&request.authority, now) { + return Err(PublicationTxError::Stale); + } + let issued_at = locked.prior_issued_at.map_or(now, |prior| { + std::cmp::max(now, prior + Duration::microseconds(1)) + }); + if request.authority.expires_at <= issued_at { + return Err(PublicationTxError::Stale); + } + + write_publication_authority( + transaction, + request, + before_epoch, + after_epoch, + next_fence, + issued_at, + ) + .await?; + let delta = AuthorizationOperationVersionDelta::new( + AuthorizationVersionComponentKind::AuthorityEpoch, + authorization_version_authority_epoch_component_key( + request.coordinate.community_id, + request.coordinate.object_kind, + request.coordinate.object_key, + ), + before_epoch, + after_epoch, + )?; + record_authorization_operation_version_delta_tx( + transaction, + request.coordinate.community_id, + request.operation.operation_id, + request.request_fingerprint, + vec![delta], + ) + .await?; + match record_authorization_event_tx(transaction, &event).await { + Ok(AuthorizationReceiptWrite::Inserted) => {} + Ok(AuthorizationReceiptWrite::ExactReplay) => { + return Err(PublicationTxError::Conflict); + } + Err(AuthorizationEventWriteError::CapacityUnavailable) => { + return Err(PublicationTxError::Audit( + AuthorizationAuditFailureCode::CapacityExhausted, + )); + } + Err(AuthorizationEventWriteError::Database(_)) => { + return Err(PublicationTxError::Audit( + AuthorizationAuditFailureCode::StorageUnavailable, + )); + } + } + crate::authorization_invalidation::notify_protected_publication_advance_tx( + transaction, + &ProtectedPublicationDependency::from_coordinate(&request.coordinate), + after_epoch, + ) + .await?; + let witness = ProtectedPublicationWitness { + coordinate: request.coordinate.clone(), + dependency: ProtectedPublicationDependency::from_coordinate(&request.coordinate), + result_digest: request.staged_digest, + authority_epoch: after_epoch, + invalidation_generation: request.authority.invalidation_generation, + fence: next_fence, + authoritative_now: issued_at, + expires_at: request.authority.expires_at, + operation_id: request.operation.operation_id, + request_fingerprint: request.request_fingerprint, + }; + Ok(ProtectedPublicationCommit { + disposition: ProtectedPublicationDisposition::Applied, + witness, + }) +} + +async fn validate_publication_replay( + transaction: &mut Transaction<'_, Postgres>, + request: &ProtectedPublicationRequest, + event: &NewAuthorizationEvent, + current: Option, +) -> std::result::Result { + match record_authorization_event_tx(transaction, event).await { + Ok(AuthorizationReceiptWrite::ExactReplay) => {} + Ok(AuthorizationReceiptWrite::Inserted) => return Err(PublicationTxError::Conflict), + Err(AuthorizationEventWriteError::CapacityUnavailable) => { + return Err(PublicationTxError::Audit( + AuthorizationAuditFailureCode::CapacityExhausted, + )); + } + Err(AuthorizationEventWriteError::Database(error)) => { + return Err(PublicationTxError::Database(error)); + } + } + let manifest = load_manifest_connection( + transaction, + request.coordinate.community_id, + request.operation.operation_id, + ) + .await? + .ok_or_else(|| { + PublicationTxError::Database(DbError::InvalidData( + "authorization protected publication replay lacks a manifest".to_owned(), + )) + })?; + if manifest.request_fingerprint() != request.request_fingerprint + || manifest.components().len() != 1 + { + return Err(PublicationTxError::Conflict); + } + let component = &manifest.components()[0]; + let expected_key = authorization_version_authority_epoch_component_key( + request.coordinate.community_id, + request.coordinate.object_kind, + request.coordinate.object_key, + ); + if component.component_kind() != AuthorizationVersionComponentKind::AuthorityEpoch + || component.component_key() != expected_key + { + return Err(PublicationTxError::Conflict); + } + let witness = current.ok_or(PublicationTxError::Conflict)?; + if witness.operation_id != request.operation.operation_id + || witness.request_fingerprint != request.request_fingerprint + || witness.result_digest != request.staged_digest + || witness.authority_epoch != component.after_version() + { + return Err(PublicationTxError::Conflict); + } + Ok(ProtectedPublicationCommit { + disposition: ProtectedPublicationDisposition::ExactReplay, + witness, + }) +} + +fn publication_receipt( + request: &ProtectedPublicationRequest, +) -> Result { + AuthorizationOperationReceipt::new( + request.coordinate.community_id, + request.operation.operation_id, + request.request_fingerprint, + AuthorizationOperationKind::ProtectedMutation, + request.authority.actor.clone(), + AuthorizationOperationOutcome::Applied, + request.staged_digest, + ) +} + +fn publication_event(request: &ProtectedPublicationRequest) -> Result { + NewAuthorizationEvent::new( + request.coordinate.community_id, + request.operation.event_id, + AuthorizationEventKind::ProtectedAllowed, + AuthorizationEventOutcome::Allowed, + AuthorizationReasonCode::Current, + request.authority.actor.clone(), + Some(request.coordinate.object_key), + request.operation.operation_id, + Some(request.request_fingerprint), + request.authority.correlation_id, + request.operation.attempt_id, + ) +} + +async fn acquire_publication_lock( + transaction: &mut Transaction<'_, Postgres>, + coordinate: &ProtectedPublicationCoordinate, +) -> Result<()> { + let mut bytes = [0_u8; 8]; + bytes.copy_from_slice(&coordinate.object_key[..8]); + let mut domain_bytes = [0_u8; 8]; + domain_bytes.copy_from_slice(&coordinate.community_id.as_uuid().as_bytes()[..8]); + let key = i64::from_be_bytes(bytes) + ^ i64::from(coordinate.object_kind as i16) + ^ i64::from_be_bytes(domain_bytes); + sqlx::query("SELECT pg_advisory_xact_lock($1)") + .bind(key) + .execute(&mut **transaction) + .await?; + Ok(()) +} + +async fn lock_publication_state( + transaction: &mut Transaction<'_, Postgres>, + coordinate: &ProtectedPublicationCoordinate, +) -> Result { + let epoch = sqlx::query( + "SELECT authority_epoch,fence,operation_id,request_fingerprint \ + FROM authorization_authority_epochs \ + WHERE community_id=$1 AND object_kind=$2 AND object_key=$3 FOR UPDATE", + ) + .bind(coordinate.community_id.as_uuid()) + .bind(coordinate.object_kind as i16) + .bind(coordinate.object_key.as_slice()) + .fetch_optional(&mut **transaction) + .await?; + let protected = sqlx::query( + "SELECT protected.authority_epoch,protected.fence,protected.operation_id, \ + protected.request_fingerprint,protected.issued_at,protected.expires_at, \ + protected.invalidation_generation,clock_timestamp() AS database_now, \ + receipt.operation_kind,receipt.outcome_code,receipt.actor_fingerprint, \ + receipt.result_digest \ + FROM protected_object_authority protected \ + JOIN authorization_operation_receipts receipt \ + ON receipt.community_id=protected.community_id \ + AND receipt.operation_id=protected.operation_id \ + AND receipt.request_fingerprint=protected.request_fingerprint \ + WHERE protected.community_id=$1 AND protected.object_kind=$2 \ + AND protected.object_key=$3 FOR UPDATE OF protected", + ) + .bind(coordinate.community_id.as_uuid()) + .bind(coordinate.object_kind as i16) + .bind(coordinate.object_key.as_slice()) + .fetch_optional(&mut **transaction) + .await?; + match (epoch, protected) { + (None, None) => Ok(LockedPublicationState { + current: None, + prior_issued_at: None, + }), + (Some(epoch), Some(protected)) => { + let witness = witness_from_rows(coordinate, &epoch, &protected)?; + Ok(LockedPublicationState { + current: Some(witness), + prior_issued_at: Some(protected.try_get("issued_at")?), + }) + } + _ => Err(DbError::InvalidData( + "authorization protected publication authority is partial".to_owned(), + )), + } +} + +async fn refence_publication_lease( + transaction: &mut Transaction<'_, Postgres>, + authority: &ProtectedPublicationAuthority, +) -> std::result::Result<(), PublicationTxError> { + let now: DateTime = sqlx::query_scalar("SELECT clock_timestamp()") + .fetch_one(&mut **transaction) + .await?; + if !lease_valid_at(authority, now) { + return Err(PublicationTxError::Stale); + } + let expected_author = authority.owner_pubkey.unwrap_or(authority.actor_pubkey); + let binding = sqlx::query( + "SELECT event_author_pubkey FROM identity_bindings \ + WHERE community_id=$1 AND binding_id=$2 AND binding_version=$3 \ + AND binding_state=1 AND (expires_at IS NULL OR expires_at > $4) FOR SHARE", + ) + .bind(authority.community_id.as_uuid()) + .bind(authority.binding_id) + .bind(to_database_version(authority.binding_version)?) + .bind(now) + .fetch_optional(&mut **transaction) + .await?; + let Some(binding) = binding else { + return Err(PublicationTxError::Stale); + }; + if bytes32(binding.try_get("event_author_pubkey")?, "binding author")? != expected_author { + return Err(PublicationTxError::Stale); + } + let policy_exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM identity_enrollment_policies \ + WHERE community_id=$1 AND policy_revision=$2 AND effective_at <= $3 \ + AND (expires_at IS NULL OR expires_at > $3))", + ) + .bind(authority.community_id.as_uuid()) + .bind(to_database_version(authority.policy_revision)?) + .bind(now) + .fetch_one(&mut **transaction) + .await?; + if !policy_exists { + return Err(PublicationTxError::Stale); + } + let generation: Option = sqlx::query_scalar( + "SELECT current_generation FROM authorization_invalidation_domains \ + WHERE community_id=$1 FOR SHARE", + ) + .bind(authority.community_id.as_uuid()) + .fetch_optional(&mut **transaction) + .await?; + if generation.map(from_database_version).transpose()? != Some(authority.invalidation_generation) + { + return Err(PublicationTxError::Stale); + } + let binding_selector = AuthorizationInvalidationSelector::binding( + authority.binding_id, + authority.binding_version, + )?; + let binding_floor: Option = sqlx::query_scalar( + "SELECT binding_version_floor FROM authorization_invalidation_floors \ + WHERE community_id=$1 AND selector_kind=3 AND selector_fingerprint=$2 FOR SHARE", + ) + .bind(authority.community_id.as_uuid()) + .bind( + binding_selector + .fingerprint(authority.community_id) + .as_slice(), + ) + .fetch_optional(&mut **transaction) + .await? + .flatten(); + if binding_floor + .map(from_database_version) + .transpose()? + .is_some_and(|floor| floor >= authority.binding_version) + { + return Err(PublicationTxError::Stale); + } + match ( + authority.relationship_id, + authority.relationship_revision, + authority.delegation_conditions_fingerprint, + ) { + (Some(relationship_id), Some(revision), Some(_)) => { + let selector = AuthorizationInvalidationSelector::delegated_relationship( + relationship_id, + revision, + )?; + let floor: Option = sqlx::query_scalar( + "SELECT relationship_revision_floor FROM authorization_invalidation_floors \ + WHERE community_id=$1 AND selector_kind=7 AND selector_fingerprint=$2 FOR SHARE", + ) + .bind(authority.community_id.as_uuid()) + .bind(selector.fingerprint(authority.community_id).as_slice()) + .fetch_optional(&mut **transaction) + .await? + .flatten(); + if floor + .map(from_database_version) + .transpose()? + .is_some_and(|floor| floor >= revision) + { + return Err(PublicationTxError::Stale); + } + } + (None, None, None) => {} + _ => { + return Err(PublicationTxError::Database(DbError::InvalidData( + "authorization delegated publication authority is partial".to_owned(), + ))); + } + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +async fn write_publication_authority( + transaction: &mut Transaction<'_, Postgres>, + request: &ProtectedPublicationRequest, + before_epoch: u64, + after_epoch: u64, + fence: AuthorizationLeaseFence, + issued_at: DateTime, +) -> Result<()> { + if before_epoch == 0 { + sqlx::query( + "INSERT INTO authorization_authority_epochs \ + (community_id,object_kind,object_key,authority_epoch,fence,operation_id, \ + request_fingerprint) VALUES ($1,$2,$3,$4,$5,$6,$7)", + ) + .bind(request.coordinate.community_id.as_uuid()) + .bind(request.coordinate.object_kind as i16) + .bind(request.coordinate.object_key.as_slice()) + .bind(to_database_version(after_epoch)?) + .bind(fence.as_bytes().as_slice()) + .bind(request.operation.operation_id) + .bind(request.request_fingerprint.as_slice()) + .execute(&mut **transaction) + .await?; + } else { + let updated = sqlx::query( + "UPDATE authorization_authority_epochs SET authority_epoch=$6,fence=$7, \ + operation_id=$8,request_fingerprint=$9, \ + updated_at=GREATEST(clock_timestamp(),updated_at + INTERVAL '1 microsecond') \ + WHERE community_id=$1 AND object_kind=$2 AND object_key=$3 \ + AND authority_epoch=$4 AND fence=$5", + ) + .bind(request.coordinate.community_id.as_uuid()) + .bind(request.coordinate.object_kind as i16) + .bind(request.coordinate.object_key.as_slice()) + .bind(to_database_version(before_epoch)?) + .bind(request.authority.fence.as_bytes().as_slice()) + .bind(to_database_version(after_epoch)?) + .bind(fence.as_bytes().as_slice()) + .bind(request.operation.operation_id) + .bind(request.request_fingerprint.as_slice()) + .execute(&mut **transaction) + .await?; + if updated.rows_affected() != 1 { + return Err(DbError::InvalidData( + "authorization protected publication epoch changed".to_owned(), + )); + } + } + + let capability = capability_code(request.authority.capability); + let relationship_revision = request + .authority + .relationship_revision + .map(to_database_version) + .transpose()?; + if before_epoch == 0 { + sqlx::query( + "INSERT INTO protected_object_authority \ + (community_id,object_kind,object_key,capability,actor_pubkey,owner_pubkey, \ + binding_id,binding_version,delegated_relationship_id, \ + delegated_relationship_revision,delegation_conditions_fingerprint, \ + policy_revision,invalidation_generation,authority_epoch,fence,issued_at,expires_at, \ + operation_id,request_fingerprint) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19)", + ) + .bind(request.coordinate.community_id.as_uuid()) + .bind(request.coordinate.object_kind as i16) + .bind(request.coordinate.object_key.as_slice()) + .bind(capability) + .bind(request.authority.actor_pubkey.as_slice()) + .bind(request.authority.owner_pubkey.map(|value| value.to_vec())) + .bind(request.authority.binding_id) + .bind(to_database_version(request.authority.binding_version)?) + .bind(request.authority.relationship_id) + .bind(relationship_revision) + .bind( + request + .authority + .delegation_conditions_fingerprint + .map(|value| value.to_vec()), + ) + .bind(to_database_version(request.authority.policy_revision)?) + .bind(to_database_version( + request.authority.invalidation_generation, + )?) + .bind(to_database_version(after_epoch)?) + .bind(fence.as_bytes().as_slice()) + .bind(issued_at) + .bind(request.authority.expires_at) + .bind(request.operation.operation_id) + .bind(request.request_fingerprint.as_slice()) + .execute(&mut **transaction) + .await?; + } else { + let updated = sqlx::query( + "UPDATE protected_object_authority SET capability=$6,actor_pubkey=$7,owner_pubkey=$8, \ + binding_id=$9,binding_version=$10,delegated_relationship_id=$11, \ + delegated_relationship_revision=$12,delegation_conditions_fingerprint=$13, \ + policy_revision=$14,invalidation_generation=$15,authority_epoch=$16,fence=$17, \ + issued_at=$18,expires_at=$19,operation_id=$20,request_fingerprint=$21 \ + WHERE community_id=$1 AND object_kind=$2 AND object_key=$3 \ + AND authority_epoch=$4 AND fence=$5", + ) + .bind(request.coordinate.community_id.as_uuid()) + .bind(request.coordinate.object_kind as i16) + .bind(request.coordinate.object_key.as_slice()) + .bind(to_database_version(before_epoch)?) + .bind(request.authority.fence.as_bytes().as_slice()) + .bind(capability) + .bind(request.authority.actor_pubkey.as_slice()) + .bind(request.authority.owner_pubkey.map(|value| value.to_vec())) + .bind(request.authority.binding_id) + .bind(to_database_version(request.authority.binding_version)?) + .bind(request.authority.relationship_id) + .bind(relationship_revision) + .bind( + request + .authority + .delegation_conditions_fingerprint + .map(|value| value.to_vec()), + ) + .bind(to_database_version(request.authority.policy_revision)?) + .bind(to_database_version( + request.authority.invalidation_generation, + )?) + .bind(to_database_version(after_epoch)?) + .bind(fence.as_bytes().as_slice()) + .bind(issued_at) + .bind(request.authority.expires_at) + .bind(request.operation.operation_id) + .bind(request.request_fingerprint.as_slice()) + .execute(&mut **transaction) + .await?; + if updated.rows_affected() != 1 { + return Err(DbError::InvalidData( + "authorization protected publication authority changed".to_owned(), + )); + } + } + Ok(()) +} + +async fn load_witness_pool( + db: &Db, + coordinate: &ProtectedPublicationCoordinate, +) -> Result> { + let rows = sqlx::query( + "WITH observed AS MATERIALIZED (SELECT clock_timestamp() AS database_now) \ + SELECT epoch.authority_epoch AS epoch_authority_epoch,epoch.fence AS epoch_fence, \ + epoch.operation_id AS epoch_operation_id, \ + epoch.request_fingerprint AS epoch_request_fingerprint, \ + protected.authority_epoch,protected.fence,protected.operation_id, \ + protected.request_fingerprint,protected.issued_at,receipt.operation_kind, \ + receipt.outcome_code,receipt.actor_fingerprint,receipt.result_digest, \ + protected.invalidation_generation,protected.expires_at, \ + observed.database_now, \ + (SELECT invalidation.current_generation \ + FROM authorization_invalidation_domains invalidation \ + WHERE invalidation.community_id=protected.community_id) \ + AS current_invalidation_generation, \ + EXISTS(SELECT 1 FROM identity_bindings binding \ + WHERE binding.community_id=protected.community_id \ + AND binding.binding_id=protected.binding_id \ + AND binding.binding_version=protected.binding_version \ + AND binding.binding_state=1 \ + AND (binding.expires_at IS NULL \ + OR binding.expires_at > observed.database_now) \ + AND binding.event_author_pubkey= \ + COALESCE(protected.owner_pubkey,protected.actor_pubkey)) AS binding_current, \ + EXISTS(SELECT 1 FROM identity_enrollment_policies policy \ + WHERE policy.community_id=protected.community_id \ + AND policy.policy_revision=protected.policy_revision \ + AND policy.effective_at <= observed.database_now \ + AND (policy.expires_at IS NULL \ + OR policy.expires_at > observed.database_now)) \ + AS policy_current \ + FROM authorization_authority_epochs epoch \ + FULL JOIN protected_object_authority protected \ + ON protected.community_id=epoch.community_id AND protected.object_kind=epoch.object_kind \ + AND protected.object_key=epoch.object_key \ + LEFT JOIN authorization_operation_receipts receipt \ + ON receipt.community_id=protected.community_id \ + AND receipt.operation_id=protected.operation_id \ + AND receipt.request_fingerprint=protected.request_fingerprint \ + CROSS JOIN observed \ + WHERE COALESCE(epoch.community_id,protected.community_id)=$1 \ + AND COALESCE(epoch.object_kind,protected.object_kind)=$2 \ + AND COALESCE(epoch.object_key,protected.object_key)=$3", + ) + .bind(coordinate.community_id.as_uuid()) + .bind(coordinate.object_kind as i16) + .bind(coordinate.object_key.as_slice()) + .fetch_all(&db.pool) + .await?; + match rows.len() { + 0 => Ok(None), + 1 => witness_from_joined_row(coordinate, &rows[0]), + _ => Err(DbError::InvalidData( + "authorization protected publication witness is not unique".to_owned(), + )), + } +} + +fn witness_from_rows( + coordinate: &ProtectedPublicationCoordinate, + epoch: &sqlx::postgres::PgRow, + protected: &sqlx::postgres::PgRow, +) -> Result { + let authority_epoch = from_database_version(epoch.try_get("authority_epoch")?)?; + let fence = authorization_lease_fence(epoch.try_get("fence")?, "authority fence")?; + let operation_id: Uuid = epoch.try_get("operation_id")?; + let request_fingerprint = bytes32( + epoch.try_get("request_fingerprint")?, + "authority request fingerprint", + )?; + if from_database_version(protected.try_get("authority_epoch")?)? != authority_epoch + || authorization_lease_fence(protected.try_get("fence")?, "protected fence")? != fence + || protected.try_get::("operation_id")? != operation_id + || bytes32( + protected.try_get("request_fingerprint")?, + "protected request fingerprint", + )? != request_fingerprint + || protected.try_get::("operation_kind")? + != AuthorizationOperationKind::ProtectedMutation as i16 + || protected.try_get::("outcome_code")? + != AuthorizationOperationOutcome::Applied as i16 + || bytes32(protected.try_get("actor_fingerprint")?, "receipt actor")? == [0; 32] + { + return Err(DbError::InvalidData( + "authorization protected publication witness does not match".to_owned(), + )); + } + let result_digest = bytes32(protected.try_get("result_digest")?, "publication digest")?; + if result_digest == [0; 32] { + return Err(DbError::InvalidData( + "authorization protected publication digest is invalid".to_owned(), + )); + } + let invalidation_generation = + from_database_version(protected.try_get("invalidation_generation")?)?; + let authoritative_now: DateTime = protected.try_get("database_now")?; + let expires_at: DateTime = protected.try_get("expires_at")?; + Ok(ProtectedPublicationWitness { + coordinate: coordinate.clone(), + dependency: ProtectedPublicationDependency::from_coordinate(coordinate), + result_digest, + authority_epoch, + invalidation_generation, + fence, + authoritative_now, + expires_at, + operation_id, + request_fingerprint, + }) +} + +fn witness_from_joined_row( + coordinate: &ProtectedPublicationCoordinate, + row: &sqlx::postgres::PgRow, +) -> Result> { + let authority_epoch: Option = row.try_get("epoch_authority_epoch")?; + let protected_epoch: Option = row.try_get("authority_epoch")?; + let (Some(authority_epoch), Some(protected_epoch)) = (authority_epoch, protected_epoch) else { + return Err(DbError::InvalidData( + "authorization protected publication witness is partial".to_owned(), + )); + }; + let epoch_fence: Option> = row.try_get("epoch_fence")?; + let protected_fence: Option> = row.try_get("fence")?; + let epoch_operation: Option = row.try_get("epoch_operation_id")?; + let protected_operation: Option = row.try_get("operation_id")?; + let epoch_request: Option> = row.try_get("epoch_request_fingerprint")?; + let protected_request: Option> = row.try_get("request_fingerprint")?; + let operation_kind: Option = row.try_get("operation_kind")?; + let outcome_code: Option = row.try_get("outcome_code")?; + let actor_fingerprint: Option> = row.try_get("actor_fingerprint")?; + let result_digest: Option> = row.try_get("result_digest")?; + let protected_generation: Option = row.try_get("invalidation_generation")?; + let current_generation: Option = row.try_get("current_invalidation_generation")?; + let expires_at: Option> = row.try_get("expires_at")?; + let authoritative_now: DateTime = row.try_get("database_now")?; + let binding_current: bool = row.try_get("binding_current")?; + let policy_current: bool = row.try_get("policy_current")?; + let authority_epoch = from_database_version(authority_epoch)?; + if from_database_version(protected_epoch)? != authority_epoch + || epoch_fence.is_none() + || epoch_fence != protected_fence + || epoch_operation.is_none() + || epoch_operation != protected_operation + || epoch_request.is_none() + || epoch_request != protected_request + || operation_kind != Some(AuthorizationOperationKind::ProtectedMutation as i16) + || outcome_code != Some(AuthorizationOperationOutcome::Applied as i16) + { + return Err(DbError::InvalidData( + "authorization protected publication witness does not match".to_owned(), + )); + } + let fence = authorization_lease_fence( + epoch_fence.ok_or_else(|| { + DbError::InvalidData("authorization publication fence is absent".to_owned()) + })?, + "publication fence", + )?; + let operation_id = epoch_operation.ok_or_else(|| { + DbError::InvalidData("authorization publication operation is absent".to_owned()) + })?; + let request_fingerprint = bytes32( + epoch_request.ok_or_else(|| { + DbError::InvalidData("authorization publication request is absent".to_owned()) + })?, + "publication request fingerprint", + )?; + if bytes32( + actor_fingerprint.ok_or_else(|| { + DbError::InvalidData("authorization publication actor is absent".to_owned()) + })?, + "publication actor fingerprint", + )? == [0; 32] + { + return Err(DbError::InvalidData( + "authorization publication actor is invalid".to_owned(), + )); + } + let result_digest = bytes32( + result_digest.ok_or_else(|| { + DbError::InvalidData("authorization publication digest is absent".to_owned()) + })?, + "publication result digest", + )?; + if result_digest == [0; 32] { + return Err(DbError::InvalidData( + "authorization publication digest is invalid".to_owned(), + )); + } + let invalidation_generation = protected_generation + .map(from_database_version) + .transpose()? + .ok_or_else(|| { + DbError::InvalidData( + "authorization publication invalidation generation is absent".to_owned(), + ) + })?; + let current_generation = current_generation.map(from_database_version).transpose()?; + let expires_at = expires_at.ok_or_else(|| { + DbError::InvalidData("authorization publication expiry is absent".to_owned()) + })?; + if current_generation != Some(invalidation_generation) + || !binding_current + || !policy_current + || authoritative_now >= expires_at + { + return Ok(None); + } + Ok(Some(ProtectedPublicationWitness { + coordinate: coordinate.clone(), + dependency: ProtectedPublicationDependency::from_coordinate(coordinate), + result_digest, + authority_epoch, + invalidation_generation, + fence, + authoritative_now, + expires_at, + operation_id, + request_fingerprint, + })) +} + +fn lease_valid_at(authority: &ProtectedPublicationAuthority, now: DateTime) -> bool { + now >= authority.issued_at && now < authority.expires_at +} + +pub(crate) fn protected_object_key( + community_id: CommunityId, + object_kind: AuthorizationProtectedObjectKind, + target_fingerprint: [u8; 32], +) -> [u8; 32] { + let mut digest = Sha256::new(); + framed(&mut digest, b"buzz:protected-publication-object:v1"); + framed(&mut digest, community_id.as_uuid().as_bytes()); + framed(&mut digest, &(object_kind as i16).to_be_bytes()); + framed(&mut digest, &target_fingerprint); + digest.finalize().into() +} + +fn publication_request_fingerprint( + coordinate: &ProtectedPublicationCoordinate, + operation: ProtectedPublicationOperationIdentity, + authority: &ProtectedPublicationAuthority, + staged_digest: [u8; 32], + expected_parent_result_digest: Option<[u8; 32]>, +) -> [u8; 32] { + let mut digest = Sha256::new(); + framed(&mut digest, b"buzz:protected-publication-request:v1"); + framed(&mut digest, coordinate.community_id.as_uuid().as_bytes()); + framed(&mut digest, &(coordinate.object_kind as i16).to_be_bytes()); + framed(&mut digest, &coordinate.object_key); + framed(&mut digest, operation.operation_id.as_bytes()); + framed(&mut digest, operation.event_id.as_bytes()); + framed(&mut digest, operation.attempt_id.as_bytes()); + framed(&mut digest, authority.correlation_id.as_bytes()); + framed(&mut digest, &authority.source_request_fingerprint); + framed(&mut digest, &staged_digest); + framed( + &mut digest, + &capability_code(authority.capability).to_be_bytes(), + ); + framed( + &mut digest, + &(proof_transport_code(authority.transport)).to_be_bytes(), + ); + framed(&mut digest, &authority.transport_context_fingerprint); + framed(&mut digest, &authority.actor_pubkey); + match authority.owner_pubkey { + Some(owner) => { + framed(&mut digest, &[1]); + framed(&mut digest, &owner); + } + None => framed(&mut digest, &[0]), + } + framed(&mut digest, authority.binding_id.as_bytes()); + framed(&mut digest, &authority.binding_version.to_be_bytes()); + match ( + authority.relationship_id, + authority.relationship_revision, + authority.delegation_conditions_fingerprint, + ) { + (Some(relationship_id), Some(revision), Some(conditions)) => { + framed(&mut digest, &[1]); + framed(&mut digest, relationship_id.as_bytes()); + framed(&mut digest, &revision.to_be_bytes()); + framed(&mut digest, &conditions); + } + _ => framed(&mut digest, &[0]), + } + framed(&mut digest, &authority.policy_revision.to_be_bytes()); + framed( + &mut digest, + &authority.invalidation_generation.to_be_bytes(), + ); + framed(&mut digest, &authority.authority_epoch.to_be_bytes()); + framed(&mut digest, authority.fence.as_bytes()); + framed(&mut digest, &authority.issued_at.timestamp().to_be_bytes()); + framed( + &mut digest, + &authority.issued_at.timestamp_subsec_nanos().to_be_bytes(), + ); + framed(&mut digest, &authority.expires_at.timestamp().to_be_bytes()); + framed( + &mut digest, + &authority.expires_at.timestamp_subsec_nanos().to_be_bytes(), + ); + match expected_parent_result_digest { + Some(parent_digest) => { + framed(&mut digest, &[1]); + framed(&mut digest, &parent_digest); + } + None => framed(&mut digest, &[0]), + } + digest.finalize().into() +} + +fn capability_reads_object( + capability: RouteCapability, + object_kind: AuthorizationProtectedObjectKind, +) -> bool { + object_kind.admits_read(capability) +} + +fn capability_mutates_object( + capability: RouteCapability, + object_kind: AuthorizationProtectedObjectKind, +) -> bool { + object_kind.admits_mutation(capability) +} + +fn capability_code(capability: RouteCapability) -> i16 { + match capability { + RouteCapability::MessagesRead => 1, + RouteCapability::MessagesWrite => 2, + RouteCapability::ChannelsRead => 3, + RouteCapability::ChannelsWrite => 4, + RouteCapability::AdminChannels => 5, + RouteCapability::UsersRead => 6, + RouteCapability::UsersWrite => 7, + RouteCapability::AdminUsers => 8, + RouteCapability::JobsRead => 9, + RouteCapability::JobsWrite => 10, + RouteCapability::SubscriptionsRead => 11, + RouteCapability::SubscriptionsWrite => 12, + RouteCapability::FilesRead => 13, + RouteCapability::FilesWrite => 14, + RouteCapability::ReposRead => 15, + RouteCapability::ReposWrite => 16, + RouteCapability::GitRead => 17, + RouteCapability::GitWrite => 18, + RouteCapability::GitStream => 19, + RouteCapability::MediaRead => 20, + RouteCapability::MediaWrite => 21, + RouteCapability::Moderation => 22, + RouteCapability::AudioJoin => 23, + RouteCapability::AudioMedia => 24, + RouteCapability::Discovery => 25, + RouteCapability::BindingStatus => 26, + RouteCapability::Enrollment => 27, + RouteCapability::InviteMint => 28, + RouteCapability::InviteClaim => 29, + } +} + +fn proof_transport_code(transport: ProofTransport) -> i16 { + match transport { + ProofTransport::Nip42 => 1, + ProofTransport::Nip98 => 2, + ProofTransport::GitSmartHttpSession => 3, + ProofTransport::Blossom => 4, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use buzz_auth::AuthorizationEventCapacityPolicy; + use sqlx::PgPool; + + fn domain() -> CommunityId { + CommunityId::from_uuid(Uuid::from_u128(1)) + } + + fn coordinate(kind: AuthorizationProtectedObjectKind) -> ProtectedPublicationCoordinate { + let target_fingerprint = [7_u8; 32]; + ProtectedPublicationCoordinate { + community_id: domain(), + object_kind: kind, + object_key: protected_object_key(domain(), kind, target_fingerprint), + target_fingerprint, + } + } + + fn authority(capability: RouteCapability) -> ProtectedPublicationAuthority { + ProtectedPublicationAuthority { + community_id: domain(), + correlation_id: Uuid::from_u128(2), + actor: AuthorizationEventActor::test_direct(domain()), + capability, + actor_pubkey: [3; 32], + owner_pubkey: None, + binding_id: Uuid::from_u128(4), + binding_version: 5, + relationship_id: None, + relationship_revision: None, + delegation_conditions_fingerprint: None, + source_request_fingerprint: [6; 32], + target_fingerprint: [7; 32], + transport: ProofTransport::Nip42, + transport_context_fingerprint: [8; 32], + policy_revision: 9, + invalidation_generation: 10, + authority_epoch: 11, + fence: AuthorizationLeaseFence::from_bytes([12; 32]).expect("valid fence"), + issued_at: DateTime::::from_timestamp(1_000, 0).expect("valid time"), + expires_at: DateTime::::from_timestamp(2_000, 0).expect("valid time"), + } + } + + #[test] + fn protected_coordinates_are_kind_separated_and_redacted() { + let repository = coordinate(AuthorizationProtectedObjectKind::Repository); + let media = coordinate(AuthorizationProtectedObjectKind::Media); + assert_ne!(repository.object_key, media.object_key); + assert!(!format!("{repository:?}").contains(&hex::encode(repository.object_key))); + } + + #[test] + fn publication_fingerprint_binds_parent_and_operation() { + let coordinate = coordinate(AuthorizationProtectedObjectKind::Repository); + let authority = authority(RouteCapability::GitWrite); + let operation = ProtectedPublicationOperationIdentity::new( + Uuid::from_u128(20), + Uuid::from_u128(21), + Uuid::from_u128(22), + ) + .expect("valid identity"); + let absent = + publication_request_fingerprint(&coordinate, operation, &authority, [23; 32], None); + let parent = ProtectedPublicationWitness { + coordinate: coordinate.clone(), + dependency: ProtectedPublicationDependency::from_coordinate(&coordinate), + result_digest: [24; 32], + authority_epoch: 11, + invalidation_generation: authority.invalidation_generation, + fence: authority.fence, + authoritative_now: authority.issued_at, + expires_at: authority.expires_at, + operation_id: Uuid::from_u128(25), + request_fingerprint: [26; 32], + }; + assert_ne!( + absent, + publication_request_fingerprint( + &coordinate, + operation, + &authority, + [23; 32], + Some(parent.result_digest), + ) + ); + let changed_event = ProtectedPublicationOperationIdentity::new( + operation.operation_id, + Uuid::from_u128(99), + operation.attempt_id, + ) + .expect("valid changed event identity"); + assert_ne!( + absent, + publication_request_fingerprint(&coordinate, changed_event, &authority, [23; 32], None,) + ); + } + + #[test] + fn publication_restore_identity_is_sealed_exact_and_redacted() { + let coordinate = coordinate(AuthorizationProtectedObjectKind::Repository); + let authority = authority(RouteCapability::GitWrite); + let operation = ProtectedPublicationOperationIdentity::new( + Uuid::from_u128(20), + Uuid::from_u128(21), + Uuid::from_u128(22), + ) + .expect("valid identity"); + let request_fingerprint = + publication_request_fingerprint(&coordinate, operation, &authority, [23; 32], None); + let request = ProtectedPublicationRequest { + coordinate, + operation, + authority, + staged_digest: [23; 32], + expected_parent_result_digest: None, + request_fingerprint, + }; + let identity = request.restore_identity(); + assert_eq!(identity.authorization_domain(), domain()); + assert_eq!(identity.operation_id(), operation.operation_id()); + assert_eq!(identity.request_fingerprint(), request_fingerprint); + let rendered = format!("{identity:?}"); + assert!(!rendered.contains(&operation.operation_id().to_string())); + assert!(!rendered.contains(&hex::encode(request_fingerprint))); + } + + #[test] + fn write_capabilities_are_closed_by_object_kind() { + assert!(capability_mutates_object( + RouteCapability::GitWrite, + AuthorizationProtectedObjectKind::Repository + )); + assert!(!capability_mutates_object( + RouteCapability::GitRead, + AuthorizationProtectedObjectKind::Repository + )); + assert!(!capability_mutates_object( + RouteCapability::MediaWrite, + AuthorizationProtectedObjectKind::Repository + )); + assert!(capability_mutates_object( + RouteCapability::MediaWrite, + AuthorizationProtectedObjectKind::Media + )); + assert_eq!(proof_transport_code(ProofTransport::Nip42), 1); + assert_eq!(proof_transport_code(ProofTransport::Nip98), 2); + assert_eq!(proof_transport_code(ProofTransport::GitSmartHttpSession), 3); + assert_eq!(proof_transport_code(ProofTransport::Blossom), 4); + } + + fn request( + coordinate: ProtectedPublicationCoordinate, + operation: ProtectedPublicationOperationIdentity, + authority: ProtectedPublicationAuthority, + staged_digest: [u8; 32], + expected_parent_result_digest: Option<[u8; 32]>, + ) -> ProtectedPublicationRequest { + let request_fingerprint = publication_request_fingerprint( + &coordinate, + operation, + &authority, + staged_digest, + expected_parent_result_digest, + ); + ProtectedPublicationRequest { + coordinate, + operation, + authority, + staged_digest, + expected_parent_result_digest, + request_fingerprint, + } + } + + async fn overwrite_publication_expiry_for_test( + pool: &PgPool, + community_id: Uuid, + object_kind: AuthorizationProtectedObjectKind, + object_key: &[u8; 32], + offset_microseconds: i64, + ) { + let mut connection = pool.acquire().await.expect("expiry test connection"); + sqlx::query("SET session_replication_role=replica") + .execute(&mut *connection) + .await + .expect("disable authority immutability trigger"); + sqlx::query( + "UPDATE protected_object_authority \ + SET expires_at=clock_timestamp() + ($4 * INTERVAL '1 microsecond') \ + WHERE community_id=$1 AND object_kind=$2 AND object_key=$3", + ) + .bind(community_id) + .bind(object_kind as i16) + .bind(object_key.as_slice()) + .bind(offset_microseconds) + .execute(&mut *connection) + .await + .expect("overwrite publication expiry for boundary test"); + sqlx::query("SET session_replication_role=origin") + .execute(&mut *connection) + .await + .expect("restore authority immutability trigger"); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn protected_publication_is_atomic_replayable_and_finally_rechecked() { + let admin_url = std::env::var("TEST_DATABASE_URL") + .unwrap_or_else(|_| crate::authorization_version::loopback_test_database_url()); + let admin = PgPool::connect(&admin_url).await.expect("connect admin"); + let name = format!("s5_protected_publication_{}", Uuid::new_v4().simple()); + sqlx::query(sqlx::AssertSqlSafe(format!("CREATE DATABASE {name}"))) + .execute(&admin) + .await + .expect("create scratch database"); + let split = admin_url.rfind('/').expect("database URL path"); + let scratch_url = format!("{}/{}", &admin_url[..split], name); + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect(&scratch_url) + .await + .expect("connect scratch database"); + crate::migration::run_migrations(&pool) + .await + .expect("migrate scratch database"); + + let community_uuid = Uuid::new_v4(); + let community = CommunityId::from_uuid(community_uuid); + sqlx::query("INSERT INTO communities (id,host) VALUES ($1,$2)") + .bind(community_uuid) + .bind(format!("publication-{}.example", community_uuid.simple())) + .execute(&pool) + .await + .expect("insert community"); + sqlx::query( + "INSERT INTO identity_enrollment_policies \ + (community_id,policy_revision,enrollment_mode,policy_digest,effective_at) \ + VALUES ($1,1,1,$2,clock_timestamp() - INTERVAL '1 second')", + ) + .bind(community_uuid) + .bind(vec![1_u8; 32]) + .execute(&pool) + .await + .expect("insert policy"); + let binding_id = Uuid::new_v4(); + let actor_pubkey = [2_u8; 32]; + let mut seed = pool.acquire().await.expect("seed connection"); + sqlx::query("SET session_replication_role=replica") + .execute(&mut *seed) + .await + .expect("disable seed triggers"); + let binding_version: i64 = sqlx::query_scalar( + "INSERT INTO identity_bindings \ + (community_id,binding_id,issuer,subject,principal_fingerprint,event_author_pubkey, \ + binding_state,lifecycle_revision,binding_provenance,policy_revision, \ + enrollment_evidence_digest,birth_history_id,creation_operation_id, \ + creation_request_fingerprint) \ + VALUES ($1,$2,'issuer','subject',$3,$4,1,1,1,1,$5,$6,$7,$8) \ + RETURNING binding_version", + ) + .bind(community_uuid) + .bind(binding_id) + .bind(vec![3_u8; 32]) + .bind(actor_pubkey.as_slice()) + .bind(vec![4_u8; 32]) + .bind(Uuid::new_v4()) + .bind(Uuid::new_v4()) + .bind(vec![5_u8; 32]) + .fetch_one(&mut *seed) + .await + .expect("insert binding"); + sqlx::query("SET session_replication_role=origin") + .execute(&mut *seed) + .await + .expect("restore seed triggers"); + drop(seed); + sqlx::query( + "INSERT INTO authorization_invalidation_domains (community_id,current_generation) \ + VALUES ($1,0)", + ) + .bind(community_uuid) + .execute(&pool) + .await + .expect("activate invalidation"); + let db = Db::from_pool(pool.clone()); + db.install_authorization_event_capacity( + community, + AuthorizationEventCapacityPolicy::new(100, 1 << 20, 16 << 10).expect("valid capacity"), + ) + .await + .expect("install event capacity"); + let mut live_notices = db + .subscribe_authorization_invalidations() + .await + .expect("install ready independent listener"); + + let target_fingerprint = [6_u8; 32]; + let repository = ProtectedPublicationCoordinate { + community_id: community, + object_kind: AuthorizationProtectedObjectKind::Repository, + object_key: protected_object_key( + community, + AuthorizationProtectedObjectKind::Repository, + target_fingerprint, + ), + target_fingerprint, + }; + let initial_fence = AuthorizationLeaseFence::from_bytes([7_u8; 32]).expect("valid fence"); + let issued_at = Utc::now() - Duration::seconds(1); + let expires_at = Utc::now() + Duration::minutes(10); + let initial_authority = ProtectedPublicationAuthority { + community_id: community, + correlation_id: Uuid::new_v4(), + actor: AuthorizationEventActor::test_direct(community), + capability: RouteCapability::GitWrite, + actor_pubkey, + owner_pubkey: None, + binding_id, + binding_version: u64::try_from(binding_version).expect("positive binding version"), + relationship_id: None, + relationship_revision: None, + delegation_conditions_fingerprint: None, + source_request_fingerprint: [8_u8; 32], + target_fingerprint, + transport: ProofTransport::Nip42, + transport_context_fingerprint: [9_u8; 32], + policy_revision: 1, + invalidation_generation: 0, + authority_epoch: 1, + fence: initial_fence, + issued_at, + expires_at, + }; + let first_operation = ProtectedPublicationOperationIdentity::new( + Uuid::new_v4(), + Uuid::new_v4(), + Uuid::new_v4(), + ) + .expect("valid first operation"); + let first_digest = [10_u8; 32]; + let first = db + .commit_staged_protected_publication(request( + repository.clone(), + first_operation, + initial_authority.clone(), + first_digest, + None, + )) + .await + .expect("commit first publication"); + assert_eq!( + first.disposition(), + ProtectedPublicationDisposition::Applied + ); + assert_eq!(first.result_digest(), first_digest); + assert_eq!(first.witness().authority_epoch(), 1); + assert_eq!(first.witness().fence(), initial_fence); + assert!(matches!( + live_notices + .recv() + .await + .expect("receive first publication advance"), + crate::authorization_invalidation::AuthorizationInvalidationNotice::ProtectedPublicationAdvanced { + authority_epoch: 1, + .. + } + )); + assert!(db + .recheck_protected_publication_witness(first.witness()) + .await + .expect("recheck first witness")); + let first_manifest = db + .authorization_operation_version_delta( + community, + first_operation.operation_id(), + first.witness().request_fingerprint, + ) + .await + .expect("load first manifest"); + assert_eq!(first_manifest.components().len(), 1); + assert_eq!(first_manifest.components()[0].before_version(), 0); + assert_eq!(first_manifest.components()[0].after_version(), 1); + + let replay = db + .commit_staged_protected_publication(request( + repository.clone(), + first_operation, + initial_authority.clone(), + first_digest, + None, + )) + .await + .expect("exactly replay first publication"); + assert_eq!( + replay.disposition(), + ProtectedPublicationDisposition::ExactReplay + ); + assert_eq!(replay.result_digest(), first_digest); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), live_notices.recv()) + .await + .is_err(), + "exact publication replay emits no live notice" + ); + let event_count: i64 = sqlx::query_scalar( + "SELECT count(*) FROM authorization_events WHERE community_id=$1 AND operation_id=$2", + ) + .bind(community_uuid) + .bind(first_operation.operation_id()) + .fetch_one(&pool) + .await + .expect("count replay events"); + assert_eq!(event_count, 1); + + let conflict_operation = ProtectedPublicationOperationIdentity::new( + Uuid::new_v4(), + Uuid::new_v4(), + Uuid::new_v4(), + ) + .expect("valid conflict operation"); + let mut next_authority = initial_authority.clone(); + next_authority.correlation_id = Uuid::new_v4(); + next_authority.authority_epoch = first.witness().authority_epoch(); + next_authority.fence = first.witness().fence(); + next_authority.source_request_fingerprint = [11_u8; 32]; + assert!(matches!( + db.commit_staged_protected_publication(request( + repository.clone(), + conflict_operation, + next_authority.clone(), + [12_u8; 32], + Some([99_u8; 32]), + )) + .await, + Err(ProtectedPublicationError::Conflict) + )); + let conflict_receipts: i64 = sqlx::query_scalar( + "SELECT count(*) FROM authorization_operation_receipts \ + WHERE community_id=$1 AND operation_id=$2", + ) + .bind(community_uuid) + .bind(conflict_operation.operation_id()) + .fetch_one(&pool) + .await + .expect("count conflict receipts"); + assert_eq!(conflict_receipts, 0, "parent conflict must mutate nothing"); + + let second_operation = ProtectedPublicationOperationIdentity::new( + Uuid::new_v4(), + Uuid::new_v4(), + Uuid::new_v4(), + ) + .expect("valid second operation"); + let second_digest = [13_u8; 32]; + let second = db + .commit_staged_protected_publication(request( + repository.clone(), + second_operation, + next_authority, + second_digest, + Some(first_digest), + )) + .await + .expect("commit replacement publication"); + assert_eq!(second.result_digest(), second_digest); + assert_eq!(second.witness().authority_epoch(), 2); + assert!(matches!( + live_notices + .recv() + .await + .expect("receive replacement publication advance"), + crate::authorization_invalidation::AuthorizationInvalidationNotice::ProtectedPublicationAdvanced { + authority_epoch: 2, + .. + } + )); + assert!(!db + .recheck_protected_publication_witness(first.witness()) + .await + .expect("old witness is stale")); + assert!(db + .recheck_protected_publication_witness(second.witness()) + .await + .expect("new witness is current")); + let resolved = db + .protected_publication_witness(&repository) + .await + .expect("resolve current witness") + .expect("publication exists"); + assert_eq!(resolved, *second.witness()); + + overwrite_publication_expiry_for_test( + &pool, + community_uuid, + AuthorizationProtectedObjectKind::Repository, + &repository.object_key, + 60_000_000, + ) + .await; + let future_expiry = db + .protected_publication_witness(&repository) + .await + .expect("load before expiry") + .expect("future authority remains current"); + assert!(future_expiry.authoritative_now() < future_expiry.expires_at()); + + overwrite_publication_expiry_for_test( + &pool, + community_uuid, + AuthorizationProtectedObjectKind::Repository, + &repository.object_key, + 0, + ) + .await; + assert!(db + .protected_publication_witness(&repository) + .await + .expect("load at exclusive expiry") + .is_none()); + assert!(!db + .recheck_protected_publication_witness(&future_expiry) + .await + .expect("expired final recheck is closed")); + + overwrite_publication_expiry_for_test( + &pool, + community_uuid, + AuthorizationProtectedObjectKind::Repository, + &repository.object_key, + -1, + ) + .await; + assert!(db + .protected_publication_witness(&repository) + .await + .expect("load after expiry") + .is_none()); + let media_target = [14_u8; 32]; + let media = ProtectedPublicationCoordinate { + community_id: community, + object_kind: AuthorizationProtectedObjectKind::Media, + object_key: protected_object_key( + community, + AuthorizationProtectedObjectKind::Media, + media_target, + ), + target_fingerprint: media_target, + }; + let media_operation = ProtectedPublicationOperationIdentity::new( + Uuid::new_v4(), + Uuid::new_v4(), + Uuid::new_v4(), + ) + .expect("valid media operation"); + let mut media_authority = initial_authority; + media_authority.capability = RouteCapability::MediaWrite; + media_authority.target_fingerprint = media_target; + media_authority.correlation_id = Uuid::new_v4(); + media_authority.source_request_fingerprint = [15_u8; 32]; + media_authority.fence = + AuthorizationLeaseFence::from_bytes([16_u8; 32]).expect("valid media fence"); + let media_digest = [17_u8; 32]; + let media_applied = db + .commit_staged_protected_publication(request( + media.clone(), + media_operation, + media_authority.clone(), + media_digest, + None, + )) + .await + .expect("commit media publication"); + let media_replay = db + .commit_staged_protected_publication(request( + media, + media_operation, + media_authority, + media_digest, + None, + )) + .await + .expect("replay media publication"); + assert_eq!( + media_replay.disposition(), + ProtectedPublicationDisposition::ExactReplay + ); + assert_eq!(media_replay.result_digest(), media_applied.result_digest()); + + db.latch_authorization_event_failure( + community, + AuthorizationAuditFailureCode::StorageUnavailable, + ) + .await + .expect("latch audit unavailable"); + let moderation_target = [18_u8; 32]; + let moderation = ProtectedPublicationCoordinate { + community_id: community, + object_kind: AuthorizationProtectedObjectKind::ModerationTarget, + object_key: protected_object_key( + community, + AuthorizationProtectedObjectKind::ModerationTarget, + moderation_target, + ), + target_fingerprint: moderation_target, + }; + let audit_failure_operation = ProtectedPublicationOperationIdentity::new( + Uuid::new_v4(), + Uuid::new_v4(), + Uuid::new_v4(), + ) + .expect("valid audit failure operation"); + let mut denied_authority = authority(RouteCapability::Moderation); + denied_authority.community_id = community; + denied_authority.correlation_id = Uuid::new_v4(); + denied_authority.actor = AuthorizationEventActor::test_direct(community); + denied_authority.actor_pubkey = actor_pubkey; + denied_authority.binding_id = binding_id; + denied_authority.binding_version = + u64::try_from(binding_version).expect("positive binding version"); + denied_authority.target_fingerprint = moderation_target; + denied_authority.policy_revision = 1; + denied_authority.invalidation_generation = 0; + denied_authority.authority_epoch = 1; + denied_authority.fence = + AuthorizationLeaseFence::from_bytes([19_u8; 32]).expect("valid moderation fence"); + denied_authority.issued_at = Utc::now() - Duration::seconds(1); + denied_authority.expires_at = Utc::now() + Duration::minutes(10); + assert!(matches!( + db.commit_staged_protected_publication(request( + moderation.clone(), + audit_failure_operation, + denied_authority, + [20_u8; 32], + None, + )) + .await, + Err(ProtectedPublicationError::AuditUnavailable) + )); + let failed_receipts: i64 = sqlx::query_scalar( + "SELECT count(*) FROM authorization_operation_receipts \ + WHERE community_id=$1 AND operation_id=$2", + ) + .bind(community_uuid) + .bind(audit_failure_operation.operation_id()) + .fetch_one(&pool) + .await + .expect("count failed audit receipts"); + assert_eq!(failed_receipts, 0); + assert!(db + .protected_publication_witness(&moderation) + .await + .expect("resolve failed publication") + .is_none()); + + sqlx::query( + "UPDATE authorization_invalidation_domains \ + SET current_generation=1,updated_at=clock_timestamp() WHERE community_id=$1", + ) + .bind(community_uuid) + .execute(&pool) + .await + .expect("advance invalidation after publication"); + assert!(!db + .recheck_protected_publication_witness(second.witness()) + .await + .expect("invalidation makes the publication witness stale")); + + pool.close().await; + let _ = sqlx::query(sqlx::AssertSqlSafe(format!( + "DROP DATABASE IF EXISTS {name} WITH (FORCE)" + ))) + .execute(&admin) + .await; + } +} diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 6d28b621c04..04add8aef50 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -19,8 +19,16 @@ pub mod archived_identities; pub mod authorization_admission; /// Immutable provider-free authorization events and capacity controls. pub mod authorization_events; +/// Durable authorization invalidation epochs and snapshots. +pub mod authorization_invalidation; /// Immutable provider-free enrollment-policy reconciliation. pub mod authorization_policy; +/// Provider-free authorization resolution against current bindings. +pub mod authorization_resolver; +/// Durable recovery of interrupted authorization operations. +pub mod authorization_restore; +/// Versioned authorization state and protected publication coordinates. +pub mod authorization_version; /// Channel and membership persistence. pub mod channel; /// Direct message channel persistence. diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index c7718ea0256..83775423759 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -562,7 +562,7 @@ mod tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 34); + assert_eq!(migrations.len(), 35); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -1084,6 +1084,40 @@ mod tests { "migration 0036 missing Invitation object-kind closure: {required}", ); } + + assert_eq!(migrations[34].version, 37); + let invalidation_restore_runtime = migrations[34].sql.as_str(); + for required in [ + "authorization_invalidation_domains", + "authorization_operation_version_delta_manifests", + "client_status_revisions", + "max_events_per_domain BETWEEN 1 AND 1000000", + "max_bytes_per_domain BETWEEN 1 AND 4294967296", + "max_envelope_bytes BETWEEN 1 AND 65536", + "ADD GENERATED ALWAYS AS IDENTITY", + "pg_get_serial_sequence", + "nip_fi_0037_ephemeral_status_required", + ] { + assert!( + invalidation_restore_runtime.contains(required), + "migration 0037 missing runtime catalog closure: {required}", + ); + } + assert!( + !invalidation_restore_runtime.contains("CREATE TABLE client_status_revisions"), + "migration 0037 must keep client status connection-local", + ); + for required in [ + "OR NEW.authority_operation_id IS NOT NULL", + "OR NEW.authority_request_fingerprint IS NOT NULL", + "OR NEW.authority_epoch IS NOT NULL", + "OR NEW.authority_fence IS NOT NULL", + ] { + assert!( + invalidation_restore_runtime.contains(required), + "final projection insert guard must reserve authority binding for the deferred guard: {required}", + ); + } } #[test] @@ -1326,7 +1360,7 @@ mod tests { run_migrations(&pool) .await .expect("retry succeeds after operator repair"); - assert_eq!(applied_versions(&pool).await.last().copied(), Some(36)); + assert_eq!(applied_versions(&pool).await.last().copied(), Some(37)); } #[tokio::test] diff --git a/crates/buzz-db/src/migration/tests/migration_nip_fi_tests.rs b/crates/buzz-db/src/migration/tests/migration_nip_fi_tests.rs index 316ef42d137..a9d68fba26a 100644 --- a/crates/buzz-db/src/migration/tests/migration_nip_fi_tests.rs +++ b/crates/buzz-db/src/migration/tests/migration_nip_fi_tests.rs @@ -1214,15 +1214,15 @@ async fn nip_fi_direct_final_catalog_and_behavior() { for (limits, expected_constraint) in [ ( - (10_001_i64, 16_777_216_i64, 16_384_i32), + (1_000_001_i64, 4_294_967_296_i64, 65_536_i32), "authorization_event_capacity_max_events", ), ( - (10_000_i64, 16_777_217_i64, 16_384_i32), + (1_000_000_i64, 4_294_967_297_i64, 65_536_i32), "authorization_event_capacity_max_bytes", ), ( - (10_000_i64, 16_777_216_i64, 16_385_i32), + (1_000_000_i64, 4_294_967_296_i64, 65_537_i32), "authorization_event_capacity_max_envelope", ), ] { @@ -1243,7 +1243,7 @@ async fn nip_fi_direct_final_catalog_and_behavior() { sqlx::query( "INSERT INTO authorization_event_capacity \ (community_id,max_events_per_domain,max_bytes_per_domain,max_envelope_bytes) \ - VALUES ($1,10000,16777216,16384)", + VALUES ($1,1000000,4294967296,65536)", ) .bind(community_id) .execute(&pool) diff --git a/migrations/0037_nip_fi_invalidation_restore_runtime.sql b/migrations/0037_nip_fi_invalidation_restore_runtime.sql new file mode 100644 index 00000000000..82f4b56331b --- /dev/null +++ b/migrations/0037_nip_fi_invalidation_restore_runtime.sql @@ -0,0 +1,196 @@ +-- Provider-free invalidation/restore runtime and desired-catalog closure. +-- +-- Migration 0030 directly installs the invalidation domains, floors, and +-- exact operation-version manifests consumed by the authorization runtime. +-- This migration fails closed if that authority is missing or if durable +-- kind-24244 status history reappears. It then converts the publication +-- allocator to an identity-backed sequence without changing the +-- target-serialized allocation contract. + +-- Keep database admission policy aligned with the public Rust contract. The +-- previous checks were intentionally stricter while the audit envelope was +-- still being bounded; replacing them is monotonic because every existing +-- policy already satisfies these expanded limits. +ALTER TABLE authorization_event_capacity + DROP CONSTRAINT authorization_event_capacity_max_events, + DROP CONSTRAINT authorization_event_capacity_max_bytes, + DROP CONSTRAINT authorization_event_capacity_max_envelope, + ADD CONSTRAINT authorization_event_capacity_max_events + CHECK (max_events_per_domain BETWEEN 1 AND 1000000) NOT VALID, + ADD CONSTRAINT authorization_event_capacity_max_bytes + CHECK (max_bytes_per_domain BETWEEN 1 AND 4294967296) NOT VALID, + ADD CONSTRAINT authorization_event_capacity_max_envelope + CHECK (max_envelope_bytes BETWEEN 1 AND 65536) NOT VALID; +ALTER TABLE authorization_event_capacity + VALIDATE CONSTRAINT authorization_event_capacity_max_events, + VALIDATE CONSTRAINT authorization_event_capacity_max_bytes, + VALIDATE CONSTRAINT authorization_event_capacity_max_envelope; + +DO $$ +DECLARE + old_sequence_value BIGINT; + old_sequence_called BOOLEAN; + allocated_floor BIGINT; + identity_sequence TEXT; + existing_identity "char"; + event_primary_key_definition TEXT; + event_primary_key_oid OID; + event_primary_key_count BIGINT; +BEGIN + IF to_regclass('public.authorization_invalidation_domains') IS NULL + OR to_regclass('public.authorization_invalidation_floors') IS NULL + OR to_regclass('public.authorization_operation_version_delta_manifests') IS NULL + OR to_regclass('public.authorization_operation_version_deltas') IS NULL + THEN + RAISE EXCEPTION 'authorization invalidation/restore authority is incomplete' + USING ERRCODE = 'check_violation', + CONSTRAINT = 'nip_fi_0037_runtime_prerequisites'; + END IF; + IF to_regclass('public.client_status_revisions') IS NOT NULL THEN + RAISE EXCEPTION 'kind 24244 status history must remain connection-local' + USING ERRCODE = 'check_violation', + CONSTRAINT = 'nip_fi_0037_ephemeral_status_required'; + END IF; + + SELECT pg_get_constraintdef(oid, true) + INTO STRICT event_primary_key_definition + FROM pg_constraint + WHERE conrelid = 'events'::regclass + AND conname = 'events_pkey' + AND contype = 'p'; + IF event_primary_key_definition = + 'PRIMARY KEY (community_id, created_at, id)' + THEN + -- The schema catalog normalizes this partitioned key to put the range + -- key first. The column set and uniqueness are unchanged; tenant/id + -- reads remain covered by idx_events_community_id. + EXECUTE 'ALTER TABLE events DROP CONSTRAINT events_pkey'; + EXECUTE 'ALTER TABLE events ADD CONSTRAINT events_pkey ' + 'PRIMARY KEY (created_at, community_id, id)'; + ELSIF event_primary_key_definition <> + 'PRIMARY KEY (created_at, community_id, id)' + THEN + RAISE EXCEPTION 'events primary key has unexpected definition: %', + event_primary_key_definition + USING ERRCODE = 'check_violation', + CONSTRAINT = 'nip_fi_0037_events_primary_key'; + END IF; + + SELECT oid INTO STRICT event_primary_key_oid + FROM pg_constraint + WHERE conrelid = 'events'::regclass + AND conname = 'events_pkey' + AND contype = 'p'; + SELECT count(*) INTO event_primary_key_count + FROM pg_constraint + WHERE (oid = event_primary_key_oid OR conparentid = event_primary_key_oid) + AND pg_get_constraintdef(oid, true) = + 'PRIMARY KEY (created_at, community_id, id)'; + IF event_primary_key_count <> 9 THEN + RAISE EXCEPTION 'events primary key did not propagate to all partitions' + USING ERRCODE = 'check_violation', + CONSTRAINT = 'nip_fi_0037_events_primary_key'; + END IF; + + LOCK TABLE protected_publication_projection_outbox IN ACCESS EXCLUSIVE MODE; + + SELECT attidentity INTO STRICT existing_identity + FROM pg_attribute + WHERE attrelid = 'protected_publication_projection_outbox'::regclass + AND attname = 'publication_sequence' + AND NOT attisdropped; + IF existing_identity <> '' THEN + RAISE EXCEPTION 'publication sequence is already identity-backed' + USING ERRCODE = 'check_violation', + CONSTRAINT = 'nip_fi_0037_allocator_precondition'; + END IF; + IF pg_get_serial_sequence( + 'protected_publication_projection_outbox', + 'publication_sequence' + ) IS DISTINCT FROM + 'public.protected_publication_projection_sequence_v1' + THEN + RAISE EXCEPTION 'publication sequence has unexpected allocator ownership' + USING ERRCODE = 'check_violation', + CONSTRAINT = 'nip_fi_0037_allocator_precondition'; + END IF; + + SELECT last_value, is_called + INTO STRICT old_sequence_value, old_sequence_called + FROM protected_publication_projection_sequence_v1; + SELECT GREATEST( + COALESCE(max(publication_sequence), 0), + CASE WHEN old_sequence_called THEN old_sequence_value ELSE 0 END + ) + INTO allocated_floor + FROM protected_publication_projection_outbox; + + EXECUTE 'ALTER SEQUENCE protected_publication_projection_sequence_v1 OWNED BY NONE'; + EXECUTE 'DROP SEQUENCE protected_publication_projection_sequence_v1'; + EXECUTE 'ALTER TABLE protected_publication_projection_outbox ' + 'ALTER COLUMN publication_sequence ADD GENERATED ALWAYS AS IDENTITY'; + + identity_sequence := pg_get_serial_sequence( + 'protected_publication_projection_outbox', + 'publication_sequence' + ); + IF identity_sequence IS NULL THEN + RAISE EXCEPTION 'identity publication allocator was not created' + USING ERRCODE = 'check_violation', + CONSTRAINT = 'nip_fi_0037_allocator_identity'; + END IF; + PERFORM setval( + identity_sequence::regclass, + GREATEST(allocated_floor, 1), + allocated_floor > 0 + ); +END; +$$; + +-- PostgreSQL fills an identity default before BEFORE INSERT triggers. The +-- preliminary value is intentionally discarded: the final value is consumed +-- only after the exact protected target lock, preserving commit/abort ordering +-- while allowing the desired schema to reproduce sequence ownership. These +-- values are ordering tokens, not contiguous public identifiers, so aborts +-- and the discarded preliminary allocation may leave harmless gaps. +CREATE OR REPLACE FUNCTION protected_publication_projection_insert_guard_v1() +RETURNS TRIGGER AS $$ +BEGIN + IF NEW.delivery_state <> 1 + OR NEW.attempt_count <> 0 + OR NEW.last_attempt_at IS NOT NULL + OR NEW.delivered_at IS NOT NULL + OR NEW.failure_code <> 0 + OR NEW.authority_operation_id IS NOT NULL + OR NEW.authority_request_fingerprint IS NOT NULL + OR NEW.authority_epoch IS NOT NULL + OR NEW.authority_fence IS NOT NULL + THEN + RAISE EXCEPTION 'protected publication projection must start pending' + USING ERRCODE = 'check_violation', + CONSTRAINT = 'protected_publication_projection_initial_state'; + END IF; + PERFORM pg_advisory_xact_lock( + hashtextextended( + jsonb_build_array( + 'buzz:nip-fi:protected-publication-projection:v1', + NEW.community_id::TEXT, + NEW.object_kind, + encode(NEW.object_key, 'hex'), + NEW.projection_kind, + NEW.projection_key + )::TEXT, + 0 + ) + ); + NEW.publication_sequence := nextval( + pg_get_serial_sequence( + 'protected_publication_projection_outbox', + 'publication_sequence' + )::regclass + ); + NEW.created_at := transaction_timestamp(); + NEW.next_attempt_at := transaction_timestamp(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; diff --git a/schema/schema.sql b/schema/schema.sql index 1b42db97cb8..d0ba303febd 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -213,9 +213,9 @@ CREATE TABLE IF NOT EXISTS authorization_event_capacity ( CONSTRAINT authorization_event_capacity_failure_code_check CHECK (failure_code IS NULL OR (failure_code IN (1, 2, 3))), CONSTRAINT authorization_event_capacity_generations CHECK (recovery_generation <= failure_generation), CONSTRAINT authorization_event_capacity_health_state_check CHECK (health_state IN (1, 2)), - CONSTRAINT authorization_event_capacity_max_bytes CHECK (max_bytes_per_domain >= 1 AND max_bytes_per_domain <= 16777216), - CONSTRAINT authorization_event_capacity_max_envelope CHECK (max_envelope_bytes >= 1 AND max_envelope_bytes <= 16384), - CONSTRAINT authorization_event_capacity_max_events CHECK (max_events_per_domain >= 1 AND max_events_per_domain <= 10000), + CONSTRAINT authorization_event_capacity_max_bytes CHECK (max_bytes_per_domain >= 1 AND max_bytes_per_domain <= 4294967296), + CONSTRAINT authorization_event_capacity_max_envelope CHECK (max_envelope_bytes >= 1 AND max_envelope_bytes <= 65536), + CONSTRAINT authorization_event_capacity_max_events CHECK (max_events_per_domain >= 1 AND max_events_per_domain <= 1000000), CONSTRAINT authorization_event_capacity_reserve_bytes CHECK (restrictive_reserve_bytes = CASE WHEN max_bytes_per_domain > max_envelope_bytes THEN GREATEST(max_envelope_bytes::bigint, LEAST(262144::bigint, max_bytes_per_domain / 8)) @@ -1892,7 +1892,7 @@ CREATE TABLE IF NOT EXISTS protected_publication_projection_outbox ( delivered_at timestamptz, failure_code smallint DEFAULT 0 NOT NULL, created_at timestamptz DEFAULT transaction_timestamp() NOT NULL, - publication_sequence bigint NOT NULL, + publication_sequence bigint GENERATED ALWAYS AS IDENTITY, CONSTRAINT protected_publication_projection_outbox_pkey PRIMARY KEY (community_id, operation_id, projection_kind), CONSTRAINT protected_publication_project_community_id_publication_sequ_key UNIQUE (community_id, publication_sequence), CONSTRAINT protected_publication_project_community_id_operation_id_re_fkey FOREIGN KEY (community_id, operation_id, request_fingerprint) REFERENCES authorization_operation_receipts (community_id, operation_id, request_fingerprint) DEFERRABLE INITIALLY DEFERRED, @@ -1907,6 +1907,7 @@ CREATE TABLE IF NOT EXISTS protected_publication_projection_outbox ( CONSTRAINT protected_publication_projection_outbox_check CHECK ((attempt_count = 0) = (last_attempt_at IS NULL)), CONSTRAINT protected_publication_projection_outbox_check1 CHECK (last_attempt_at IS NULL OR last_attempt_at >= created_at), CONSTRAINT protected_publication_projection_outbox_check2 CHECK (delivered_at IS NULL OR delivered_at >= created_at), + CONSTRAINT protected_publication_projection_outbox_check3 CHECK ((delivery_state = 1 AND delivered_at IS NULL AND failure_code IN (0, 1, 2)) OR (delivery_state = 2 AND delivered_at IS NOT NULL AND failure_code = 0) OR (delivery_state = 3 AND delivered_at IS NULL AND failure_code >= 3 AND failure_code <= 6)), CONSTRAINT protected_publication_projection_outbox_delivery_state_check CHECK (delivery_state IN (1, 2, 3)), CONSTRAINT protected_publication_projection_outbox_failure_code_check CHECK (failure_code >= 0 AND failure_code <= 6), CONSTRAINT protected_publication_projection_outbox_object_key_check CHECK (octet_length(object_key) = 32 AND object_key <> decode(repeat('00'::text, 32), 'hex'::text)), @@ -4419,13 +4420,20 @@ BEGIN jsonb_build_array( 'buzz:nip-fi:protected-publication-projection:v1', NEW.community_id::TEXT, + NEW.object_kind, + encode(NEW.object_key, 'hex'), NEW.projection_kind, NEW.projection_key )::TEXT, 0 ) ); - NEW.publication_sequence := nextval('protected_publication_projection_sequence_v1'); + NEW.publication_sequence := nextval( + pg_get_serial_sequence( + 'protected_publication_projection_outbox', + 'publication_sequence' + )::regclass + ); NEW.created_at := transaction_timestamp(); NEW.next_attempt_at := transaction_timestamp(); RETURN NEW; @@ -6047,8 +6055,5 @@ ALTER TABLE ONLY moderation_reports ALTER TABLE ONLY protected_object_authority ADD CONSTRAINT protected_object_authority_check1 CHECK ((((owner_pubkey IS NULL) AND (delegated_relationship_id IS NULL) AND (delegated_relationship_revision IS NULL) AND (delegation_conditions_fingerprint IS NULL)) OR ((owner_pubkey IS NOT NULL) AND (delegated_relationship_id IS NOT NULL) AND (delegated_relationship_revision IS NOT NULL) AND (delegation_conditions_fingerprint IS NOT NULL)))); -ALTER TABLE ONLY protected_publication_projection_outbox - ADD CONSTRAINT protected_publication_projection_outbox_check3 CHECK ((((delivery_state = 1) AND (delivered_at IS NULL) AND (failure_code = ANY (ARRAY[0, 1, 2]))) OR ((delivery_state = 2) AND (delivered_at IS NOT NULL) AND (failure_code = 0)) OR ((delivery_state = 3) AND (delivered_at IS NULL) AND (failure_code >= 3) AND (failure_code <= 6)))); - ALTER TABLE ONLY push_leases ADD CONSTRAINT push_leases_check CHECK (((active AND (app_profile IS NOT NULL) AND (endpoint_hash IS NOT NULL) AND (endpoint_grant IS NOT NULL) AND (max_class IS NOT NULL) AND (subscriptions IS NOT NULL)) OR ((NOT active) AND (app_profile IS NULL) AND (endpoint_hash IS NULL) AND (endpoint_grant IS NULL) AND (max_class IS NULL) AND (subscriptions IS NULL))));