Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions memex/aspire/Memex.Database.Migration/MigrationRegistry.cs
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ public static class MigrationRegistry
new V48_RetypeBuiltinFeedbackToPlugin(),
new V49_FixAccessAssignmentMainNode(),
new V50_RescopePlatformAdminGrants(),
new V51_DropInvalidPartitionSchemas(),
];

/// <summary>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
using MeshWeaver.Mesh;
using Microsoft.Extensions.Logging;

namespace Memex.Database.Migration.Migrations;

/// <summary>
/// Drop schemas whose names could never be valid partition segments (#714).
///
/// <para>Bug: before the partition router learned to reject malformed first segments and
/// before schema creation was gated to partition-owning creates, navigating to (or probing)
/// an arbitrary top-level URL lazily provisioned a real schema + tables. memex and
/// memexcloud accumulated querystring-shaped junk like
/// <c>search?q=query%20syntax&amp;hq=scope%3adescendants</c> (nine variants) and
/// <c>login?returnurl=https%3a%2f%2f…</c>. <see cref="V03_DropRogueSchemas"/> only dropped a
/// hardcoded bare-word list, which cannot match these.</para>
///
/// <para>This migration is pattern-based, not name-list based: it enumerates every
/// non-system schema and drops exactly those that FAIL the shared segment-validity rule
/// (<see cref="PartitionDefinition.IsValidPartitionSegment"/> — the SAME predicate the
/// router and the provisioning boundary enforce), i.e. names that do not start with a
/// letter/digit, contain anything other than a letter/digit/<c>.</c>/<c>-</c>/<c>_</c>, or
/// exceed 63 UTF-8 bytes. <b>The rule is Unicode-aware, NOT ASCII-only</b>
/// (<c>char.IsLetterOrDigit</c>): an accented schema such as <c>müller</c> PASSES and is
/// never dropped — do not read this as an <c>[A-Za-z0-9._-]</c> whitelist. Every legitimate schema —
/// partitions, their <c>*_versions</c> twins, <c>admin</c>/<c>auth</c>/<c>apitoken</c>/
/// <c>system_access</c>/<c>event_log</c> — passes the rule and is untouched; a schema that
/// fails it can never be routed to (the router's validity check precedes even its
/// registered-partition lookup), so it can hold nothing reachable — this also covers
/// legacy email-named user schemas (<c>…@…</c>), stranded-unreachable since the router
/// hardening. Drops are best-effort CASCADE with per-schema logging and a pre-drop
/// row-count audit so a surprise is visible in the log (mirrors
/// <see cref="V03_DropRogueSchemas"/> / <see cref="V38_DropLegacyProviderSchema"/>), plus a
/// <c>searchable_schemas</c> de-registration so queries stop fanning out to dropped schemas.</para>
/// </summary>
public sealed class V51_DropInvalidPartitionSchemas : IMigration
{
public int Version => 51;
public string Description => "Drop junk schemas whose names can never be valid partition segments (#714)";

public async Task RunAsync(MigrationContext ctx)
{
// Enumerate candidates: every schema except the engine's own (pg_catalog, pg_toast,
// pg_temp_*, …) and the SQL-standard information_schema / public. Validity is decided
// in C# by the ONE shared segment rule — never a duplicated regex or a name list.
var invalid = new List<string>();
await using (var listCmd = ctx.DataSource.CreateCommand("""
SELECT nspname FROM pg_namespace
WHERE nspname NOT LIKE 'pg%'
AND nspname NOT IN ('information_schema', 'public')
ORDER BY nspname
"""))
await using (var reader = await listCmd.ExecuteReaderAsync())
while (await reader.ReadAsync())
{
var name = reader.GetString(0);
if (!PartitionDefinition.IsValidPartitionSegment(name))
invalid.Add(name);
}

if (invalid.Count == 0)
{
ctx.Logger.LogInformation("Repair v51: no invalid-named schemas found — nothing to drop.");
return;
}

ctx.Logger.LogInformation(
"Repair v51: found {Count} schema(s) failing the partition-segment rule: {Schemas}",
invalid.Count, string.Join(", ", invalid.Select(s => $"\"{s}\"")));

foreach (var schema in invalid)
{
// Audit what we're removing before the CASCADE so a surprise is visible in the log
// (mirrors V38). Tolerant — a junk schema may not even have a mesh_nodes table.
try
{
await using var countCmd = ctx.DataSource.CreateCommand(
"SELECT count(*) FROM \"" + schema.Replace("\"", "\"\"") + "\".mesh_nodes");
var rows = await countCmd.ExecuteScalarAsync();
ctx.Logger.LogInformation(
"Repair v51: dropping invalid-named schema \"{Schema}\" ({Rows} mesh_nodes rows).",
schema, rows);
}
catch (Exception ex)
{
ctx.Logger.LogInformation(
"Repair v51: dropping invalid-named schema \"{Schema}\" (no readable mesh_nodes table: {Reason}).",
schema, ex.Message);
}

// Best-effort, log-and-continue (mirrors V03): a failed DROP (lock contention, an
// unexpected dependency) must NOT abort the whole migration run and block the deploy.
try
{
var quoted = schema.Replace("\"", "\"\"");
await using (var dropCmd = ctx.DataSource.CreateCommand(
$"DROP SCHEMA IF EXISTS \"{quoted}\" CASCADE"))
await dropCmd.ExecuteNonQueryAsync();
// The versions twin of a junk schema fails the rule itself and is dropped by its
// own iteration; this explicit twin drop (V03/V38 parity) is an idempotent no-op
// when it is already gone.
await using (var dropVCmd = ctx.DataSource.CreateCommand(
$"DROP SCHEMA IF EXISTS \"{quoted}_versions\" CASCADE"))
await dropVCmd.ExecuteNonQueryAsync();
ctx.Logger.LogInformation("Repair v51: dropped invalid-named schema \"{Schema}\"", schema);
}
catch (Exception ex)
{
ctx.Logger.LogWarning(ex, "Repair v51: failed to drop schema \"{Schema}\" (continuing)", schema);
}

// De-register from the searchable-schemas index (mirrors V38): a dangling row keeps
// cross-schema queries fanning out to the dropped schema. Tolerant — the table may
// not exist on every deployment shape, and a missing row is a no-op.
try
{
await using var deregCmd = ctx.DataSource.CreateCommand(
"DELETE FROM public.searchable_schemas WHERE lower(schema_name) = lower($1)");
deregCmd.Parameters.AddWithValue(schema);
var removed = await deregCmd.ExecuteNonQueryAsync();
if (removed > 0)
ctx.Logger.LogInformation(
"Repair v51: de-registered \"{Schema}\" from searchable_schemas.", schema);
}
catch (Exception ex)
{
ctx.Logger.LogWarning(ex,
"Repair v51: could not de-register \"{Schema}\" from searchable_schemas (continuing).", schema);
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,26 @@ public IObservable<NodeValidationResult> Validate(NodeValidationContext context)
if (string.IsNullOrEmpty(partitionName))
return Observable.Return(NodeValidationResult.Valid());

// 🚨 Defense-in-depth (#714): the partition id becomes the backing schema name.
// A malformed id (URL/query-string shaped — '?', '%', ':', '=', '&', whitespace;
// >63 chars; not starting with a letter/digit) must NEVER provision a schema. The
// storage routers refuse to ROUTE such segments (the same shared
// PartitionDefinition.IsValidPartitionSegment rule), so a partition created with
// one would be permanently unreachable anyway — reject the create with a speaking
// error naming the offending id instead of provisioning junk.
// 🚨 NO `_`-prefix exemption. A `_`-prefixed name is a SATELLITE CONTAINER segment
// ({P}/_Access/…, {P}/_Thread/…) or a global-satellite NAMESPACE, never a partition:
// the router resolves a `_`-prefixed first segment ONLY through a registered
// PartitionDefinition with an EXPLICIT schema (`_Access` → `system_access`) and never
// derives one from the name, so a schema named `_access` is unroutable by construction —
// exactly the ghost this guard exists to prevent. A `Space` create with id `_Access`
// is therefore a bug in the CALLER, and rejecting it loud is the point.
if (!PartitionDefinition.IsValidPartitionSegment(partitionName))
return Observable.Return(NodeValidationResult.Invalid(
$"Cannot create '{context.Node.NodeType}' with id '{partitionName}': "
+ $"{PartitionDefinition.PartitionSegmentRequirement}.",
NodeRejectionReason.InvalidPath));

var providers = _hub.ServiceProvider.GetServices<IPartitionStorageProvider>().ToList();
if (providers.Count == 0)
return Observable.Return(NodeValidationResult.Valid());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,22 @@ private async Task<PartitionDefinition> EnsureSchemaAsync(
var schema = !string.IsNullOrEmpty(def.Schema) ? def.Schema : def.Namespace;
if (string.IsNullOrEmpty(schema)) return def;

// 🚨 Defense-in-depth (#714): this is the LAST stop before
// public.ensure_partition_schema materialises the schema, so validate the resolved
// schema name here no matter which code path called (EnsurePartitionProvisioned,
// the boot-time static-partition seeding, tests). A URL/query-string-shaped name
// (`search?q=…`, `login?returnurl=…`) must never become a schema — fail LOUD,
// naming the offender, instead of provisioning junk the router can never route to.
// (Explicit schemas of the `_`-prefix globals — `system_access` etc. — are ordinary
// identifiers and pass; the rule constrains shape, not the registered names. That is
// precisely why a `_`-prefixed name must NOT be exempted here: `_Access` reaching this
// point means the caller resolved the schema from the NAMESPACE instead of the
// registered definition, and `_access` is a schema the router can never route to.)
if (!PartitionDefinition.IsValidPartitionSegment(schema))
throw new ArgumentException(
$"Cannot provision partition '{def.Namespace}' (schema '{schema}'): "
+ $"{PartitionDefinition.PartitionSegmentRequirement}.");

if (_schemasInitialized.ContainsKey(schema)) return def;

// Single source of truth for per-partition DDL: the public.ensure_partition_schema
Expand Down Expand Up @@ -330,6 +346,21 @@ public IObservable<Unit> EnsurePartitionProvisioned(string @namespace)
{
if (string.IsNullOrEmpty(@namespace))
return Observable.Return(Unit.Default);
// 🚨 Defense-in-depth (#714): the namespace becomes the schema name verbatim
// (lowercased). The router already refuses to ROUTE malformed segments, but a
// deliberate top-level create — or any future caller — must never reach
// public.ensure_partition_schema with a URL-shaped id (`search?q=…`,
// `login?returnurl=…`: the junk schemas that filled the memex/memexcloud DBs).
// Fail BEFORE the promise-cache so the rejection is loud (OnError naming the
// offending id), never a cached silent no-op the caller mistakes for success.
// A `_`-prefixed namespace is rejected like any other malformed name: global
// satellites (`_Access`) get their schema from a REGISTERED PartitionDefinition
// (`system_access`), never from this namespace-derived path — see EnsureSchemaAsync.
if (!PartitionDefinition.IsValidPartitionSegment(@namespace))
return Observable.Throw<Unit>(new ArgumentException(
$"Cannot provision partition '{@namespace}': "
+ $"{PartitionDefinition.PartitionSegmentRequirement}.",
nameof(@namespace)));
var def = new PartitionDefinition
{
Namespace = @namespace,
Expand Down
18 changes: 4 additions & 14 deletions src/MeshWeaver.Hosting.PostgreSql/PostgreSqlPathRoutingAdapter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,10 @@ public PostgreSqlPathRoutingAdapter(PostgreSqlPartitionStorageProvider provider)
// be lazily CREATE SCHEMA'd. Prod 2026-06-05: the atioz DB filled with exactly
// these garbage schemas (request URLs routed as mesh paths) and corrupted itself.
// → null so no schema is created; the write falls through to the next provider.
if (!IsValidPartitionSegment(seg))
// The charset rule is the shared PartitionDefinition.IsValidPartitionSegment —
// the SAME predicate the provisioning boundary enforces (#714), so a segment
// this router refuses to route can also never be provisioned.
if (!PartitionDefinition.IsValidPartitionSegment(seg))
return null;
// Valid partition segment → schema is the lowercased first segment. If a
// richer PartitionDefinition was registered for it (e.g. a non-default
Expand All @@ -140,19 +143,6 @@ public PostgreSqlPathRoutingAdapter(PostgreSqlPartitionStorageProvider provider)
};
}

/// <summary>
/// A partition's first path segment becomes a Postgres schema, so it must be a simple
/// identifier: start with a letter/digit, then only letters/digits/<c>. - _</c>, ≤63
/// chars (PG identifier limit). Rejects URL/query-string-shaped segments
/// (containing <c>? = &amp; % # :</c>, whitespace, …) that the partition router would
/// otherwise materialise as garbage schemas — the atioz DB-corruption root cause
/// (2026-06-05). (<c>_</c>-prefixed global-satellite segments are handled before this.)
/// </summary>
internal static bool IsValidPartitionSegment(string seg) =>
seg.Length is > 0 and <= 63
&& char.IsLetterOrDigit(seg[0])
&& seg.All(c => char.IsLetterOrDigit(c) || c is '.' or '-' or '_');

/// <summary>
/// Schema-bound adapter cache: once we've materialised the
/// PostgreSqlStorageAdapter for a (def.Schema), reuse it.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,15 @@ public IObservable<Unit> EnsurePartitionProvisioned(string @namespace)
{
if (string.IsNullOrEmpty(@namespace))
return Observable.Return(Unit.Default);
// 🚨 Defense-in-depth (#714, PG lineage): the namespace becomes the schema name
// verbatim (lowercased) — a malformed top-level id (URL/query-string shaped) must
// never provision a schema, no matter which code path calls. Fail BEFORE the
// promise-cache so the rejection is loud (OnError naming the offending id).
if (!PartitionDefinition.IsValidPartitionSegment(@namespace))
return Observable.Throw<Unit>(new ArgumentException(
$"Cannot provision partition '{@namespace}': "
+ $"{PartitionDefinition.PartitionSegmentRequirement}.",
nameof(@namespace)));
var def = new PartitionDefinition
{
Namespace = @namespace,
Expand Down
18 changes: 3 additions & 15 deletions src/MeshWeaver.Hosting.Snowflake/SnowflakePathRoutingAdapter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,9 @@ public SnowflakePathRoutingAdapter(SnowflakePartitionStorageProvider provider)
// be lazily CREATE SCHEMA'd. Prod 2026-06-05 (PG): the atioz DB filled with exactly
// these garbage schemas (request URLs routed as mesh paths) and corrupted itself.
// → null so no schema is created; the write falls through to the next provider.
if (!IsValidPartitionSegment(seg))
// The charset rule is the shared PartitionDefinition.IsValidPartitionSegment —
// one predicate for both backends AND the provisioning boundary (#714).
if (!PartitionDefinition.IsValidPartitionSegment(seg))
return null;
// Valid partition segment → schema is the lowercased first segment. If a
// richer PartitionDefinition was registered for it (e.g. a non-default
Expand All @@ -149,20 +151,6 @@ public SnowflakePathRoutingAdapter(SnowflakePartitionStorageProvider provider)
};
}

/// <summary>
/// A partition's first path segment becomes a Snowflake schema, so it must be a simple
/// identifier: start with a letter/digit, then only letters/digits/<c>. - _</c>, ≤63
/// chars (kept identical to the PG rule — same 63-char cap — so a path routable on one
/// backend is routable on the other). Rejects URL/query-string-shaped segments
/// (containing <c>? = &amp; % # :</c>, whitespace, …) that the partition router would
/// otherwise materialise as garbage schemas — the atioz DB-corruption root cause
/// (2026-06-05). (<c>_</c>-prefixed global-satellite segments are handled before this.)
/// </summary>
internal static bool IsValidPartitionSegment(string seg) =>
seg.Length is > 0 and <= 63
&& char.IsLetterOrDigit(seg[0])
&& seg.All(c => char.IsLetterOrDigit(c) || c is '.' or '-' or '_');

/// <summary>
/// Schema-bound adapter cache: once we've materialised the
/// <see cref="SnowflakeStorageAdapter"/> for a (def.Schema), reuse it. Every adapter
Expand Down
Loading
Loading