Repository navigation
feat(server): add result-side tripwires for query_sql and /sql #104
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
+153
−11
Merged
Changes from all commits
Commits
Show all changes
2 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| /** | ||
| * Result-side tripwires for query_sql. | ||
| * | ||
| * Cheap heuristics over the SQL text + results that warn about observed | ||
| * recurring correctness traps. | ||
| */ | ||
| import type { SqlResult } from "./ponder-client.js" | ||
|
|
||
| const AXLUSDC = "0xeb466342c4d449bc9f53a865d5cb90586f405215" | ||
|
|
||
| export function sqlNotices(sql: string, result: SqlResult): string[] { | ||
| const notices: string[] = [] | ||
| const q = sql.toLowerCase() | ||
|
|
||
| // -- Two payment channels: fp_rail_settled is streaming only -- | ||
| const settled = q.includes("fp_rail_settled") | ||
| const oneTime = q.includes("fp_one_time_payment") | ||
| const aggregating = /\bsum\s*\(/.test(q) || /\brevenue\b/.test(q) || /\bvolume\b/.test(q) | ||
| if (settled && !oneTime && aggregating) { | ||
| notices.push( | ||
| "fp_rail_settled is only the STREAMING channel. fp_one_time_payment is a parallel channel (often larger) with no gross column (gross = net_payee_amount + network_fee + operator_commission). Total volume = both; do not total from fp_rail_settled alone.", | ||
| ) | ||
| } | ||
|
|
||
| // -- Per-token decimals: a fixed /1e18 zeroes 6-decimal axlUSDC -- | ||
| // Only FilecoinPay amounts carry a token dimension. tx_meta gas is always FIL | ||
| // and /1e18 is its correct scaling, so gas-only queries must stay quiet. | ||
| const touchesPayments = /\bfp_[a-z_]+/.test(q) | ||
| const dividesBy1e18 = /\/\s*1e18\b/.test(q) || /\/\s*1_?0{18}\b/.test(q) | ||
| const sumsAmount = | ||
| /\bsum\s*\(/.test(q) && /(amount|net_payee|network_fee|operator_commission|value|price)/.test(q) | ||
| const mentionsToken = /\btoken\b/.test(q) | ||
| if (touchesPayments && dividesBy1e18) { | ||
| notices.push( | ||
| "Decimals are per-token: axlUSDC = 6 (1e6), USDFC/FIL = 18 (1e18). A fixed /1e18 collapses axlUSDC value to ~0. JOIN fp_rail_created.token and scale each token by its own decimals.", | ||
| ) | ||
| } else if (touchesPayments && sumsAmount && !mentionsToken) { | ||
| notices.push( | ||
| "Summing amounts without GROUP BY token mixes 6-decimal (axlUSDC) and 18-decimal (USDFC/FIL) values. Scale per token before summing.", | ||
| ) | ||
| } | ||
|
|
||
| // -- Result-shape: a result set spanning multiple tokens including axlUSDC -- | ||
| const tokenCol = result.columns.find((c) => c.toLowerCase() === "token") | ||
| if (tokenCol && result.rows.length > 1) { | ||
| const tokens = new Set(result.rows.map((r) => String(r[tokenCol]).toLowerCase())) | ||
| if (tokens.size > 1 && tokens.has(AXLUSDC)) { | ||
| notices.push( | ||
| "This result spans multiple tokens including axlUSDC (6-decimal). Scale each row by its own token's decimals; do not divide the whole set by one scalar.", | ||
| ) | ||
| } | ||
| } | ||
|
|
||
| return notices | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| import { describe, expect, test } from "vitest" | ||
| import { sqlNotices } from "../src/tripwires.js" | ||
|
|
||
| const AXLUSDC = "0xeb466342c4d449bc9f53a865d5cb90586f405215" | ||
| const USDFC = "0x80b98d3aa09ffff255c3ba4a241111ff1262f045" | ||
| const empty = { columns: [], rows: [], rowCount: 0 } | ||
|
|
||
| const has = (notices: string[], substr: string) => notices.some((n) => n.includes(substr)) | ||
|
|
||
| describe("sqlNotices tripwires", () => { | ||
| test("two-channel: fp_rail_settled SUM without fp_one_time_payment warns", () => { | ||
| const n = sqlNotices("SELECT SUM(total_net_payee_amount) FROM fp_rail_settled", empty) | ||
| expect(has(n, "fp_one_time_payment")).toBe(true) | ||
| }) | ||
|
|
||
| test("two-channel: silent when both channels present", () => { | ||
| const n = sqlNotices( | ||
| "SELECT SUM(s.total_net_payee_amount) FROM fp_rail_settled s UNION ALL SELECT SUM(o.net_payee_amount) FROM fp_one_time_payment o", | ||
| empty, | ||
| ) | ||
| expect(has(n, "fp_one_time_payment is a parallel")).toBe(false) | ||
| }) | ||
|
|
||
| test("per-token: a fixed /1e18 warns", () => { | ||
| const n = sqlNotices("SELECT SUM(total_settled_amount)/1e18 FROM fp_rail_settled", empty) | ||
| expect(has(n, "collapses axlUSDC")).toBe(true) | ||
| }) | ||
|
|
||
| test("per-token: SUM(amount) without token grouping warns", () => { | ||
| const n = sqlNotices("SELECT SUM(amount) FROM fp_deposit", empty) | ||
| expect(has(n, "without GROUP BY token")).toBe(true) | ||
| }) | ||
|
|
||
| test("per-token: SUM with token grouping does not warn about mixing", () => { | ||
| const n = sqlNotices("SELECT token, SUM(amount) FROM fp_deposit GROUP BY token", empty) | ||
| expect(has(n, "without GROUP BY token")).toBe(false) | ||
| }) | ||
|
|
||
| test("result-shape: mixed-token result including axlUSDC warns", () => { | ||
| const result = { | ||
| columns: ["token", "n"], | ||
| rows: [{ token: USDFC, n: "5" }, { token: AXLUSDC, n: "3" }], | ||
| rowCount: 2, | ||
| } | ||
| const n = sqlNotices("SELECT token, COUNT(*) n FROM fp_rail_created GROUP BY token", result) | ||
| expect(has(n, "spans multiple tokens")).toBe(true) | ||
| }) | ||
|
|
||
| test("result-shape: single-token result does not warn", () => { | ||
| const result = { columns: ["token", "n"], rows: [{ token: USDFC, n: "5" }], rowCount: 1 } | ||
| const n = sqlNotices("SELECT token, COUNT(*) n FROM fp_rail_created GROUP BY token", result) | ||
| expect(has(n, "spans multiple tokens")).toBe(false) | ||
| }) | ||
|
|
||
| test("per-token: the documented tx_meta gas recipe stays quiet", () => { | ||
| const n = sqlNotices( | ||
| "SELECT tx_hash, gas_used * effective_gas_price / 1e18 AS fil FROM tx_meta ORDER BY fil DESC LIMIT 10", | ||
| empty, | ||
| ) | ||
| expect(n).toHaveLength(0) | ||
| }) | ||
|
|
||
| test("per-token: SUM over gas without payments data stays quiet", () => { | ||
| const n = sqlNotices( | ||
| "SELECT SUM(gas_used * effective_gas_price) FROM tx_meta WHERE block_number > 6000000", | ||
| empty, | ||
| ) | ||
| expect(n).toHaveLength(0) | ||
| }) | ||
|
|
||
| test("per-token: gas scaling alongside payments data still warns", () => { | ||
| const n = sqlNotices( | ||
| "SELECT SUM(s.total_settled_amount)/1e18 FROM fp_rail_settled s JOIN tx_meta m USING (tx_hash)", | ||
| empty, | ||
| ) | ||
| expect(has(n, "collapses axlUSDC")).toBe(true) | ||
| }) | ||
|
|
||
| test("clean analytical query produces no notices", () => { | ||
| const n = sqlNotices("SELECT provider_id, COUNT(*) FROM fwss_data_set_created GROUP BY provider_id", empty) | ||
| expect(n).toHaveLength(0) | ||
| }) | ||
| }) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.