perp-fees harness: 4 data bugs (Paradex, gains, dYdX, GMX)

harnesses/perp-fees/cmd/script/dydx.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"sort"
 	"strconv"
 	"time"
 )
@@ -62,18 +63,45 @@ func fetchDYdX(v VenueConfig) PerpSample {
 		s.FetchLatencyMs = time.Since(start).Milliseconds()
 		return s
 	}
-	bestBid, _ := strconv.ParseFloat(book.Bids[0].Price, 64)
-	bestAsk, _ := strconv.ParseFloat(book.Asks[0].Price, 64)
-	mid := (bestBid + bestAsk) / 2
-	s.MidPrice = mid
 
-	// Walk asks
-	levels := make([]bookLevel, 0, len(book.Asks))
+	// Convert and sort. The dYdX v4 indexer's orderbook endpoint returns
+	// levels in insertion order, NOT sorted by price — walking as-is means
+	// the "top of book" starts at whichever level was most recently posted,
+	// which inflates the effective price by a factor that matches the
+	// observed 1.7-2x too-high headline (asks walked out of order push the
+	// weighted average well past the true best offer). Sort asks ascending
+	// and bids descending before consuming.
+	asks := make([]bookLevel, 0, len(book.Asks))
 	for _, a := range book.Asks {
 		px, _ := strconv.ParseFloat(a.Price, 64)
 		sz, _ := strconv.ParseFloat(a.Size, 64)
-		levels = append(levels, bookLevel{Px: px, Sz: sz})
+		if px > 0 && sz > 0 {
+			asks = append(asks, bookLevel{Px: px, Sz: sz})
+		}
 	}
+	bids := make([]bookLevel, 0, len(book.Bids))
+	for _, b := range book.Bids {
+		px, _ := strconv.ParseFloat(b.Price, 64)
+		sz, _ := strconv.ParseFloat(b.Size, 64)
+		if px > 0 && sz > 0 {
+			bids = append(bids, bookLevel{Px: px, Sz: sz})
+		}
+	}
+	if len(asks) == 0 || len(bids) == 0 {
+		s.Err = "empty_orderbook"
+		s.FetchLatencyMs = time.Since(start).Milliseconds()
+		return s
+	}
+	sort.Slice(asks, func(i, j int) bool { return asks[i].Px < asks[j].Px })
+	sort.Slice(bids, func(i, j int) bool { return bids[i].Px > bids[j].Px })
+
+	bestBid := bids[0].Px
+	bestAsk := asks[0].Px
+	mid := (bestBid + bestAsk) / 2
+	s.MidPrice = mid
+
+	// Walk asks (long open = buying against the ask side).
+	levels := asks
 	effective, err := walkBookForNotional(levels, v.NotionalUSD)
 	if err != nil {
 		s.Err = fmt.Sprintf("walk: %v", err)

harnesses/perp-fees/cmd/script/gains.go

@@ -225,8 +225,13 @@ func findGainsPair(client *http.Client, asset string) (*gainsPair, error) {
 	}
 	gainsCacheMu.Unlock()
 
+	// Gains v8 has grown well past 60 pairs; a narrow scan silently misses
+	// USD-quoted pairs (SOL/USD sits past the first crypto majors) and lets
+	// a same-symbol non-USD pair win the cache slot. 200 is enough headroom
+	// for the current listed universe with the 3-consecutive-errors early
+	// exit still handling the tail.
 	consecutiveErrors := 0
-	for i := 0; i < 60; i++ {
+	for i := 0; i < 200; i++ {
 		p, err := gainsReadPair(client, i)
 		if err != nil {
 			fmt.Printf("[PERP][gains] pair scan idx %d: %v\n", i, err)
@@ -240,12 +245,21 @@ func findGainsPair(client *http.Client, asset string) (*gainsPair, error) {
 		if p.From == "" {
 			continue
 		}
+		// Only cache USD-quoted pairs. Gains lists cross pairs like SOL/BTC
+		// alongside SOL/USD, and keying the cache by From alone would let a
+		// non-USD pair (with a different feeIndex/spreadP) win the slot for
+		// an asset that also has a USD pair. That was the source of the
+		// uniform-4.333-bps SOL bug: SOL's non-USD pair pinned the cache to
+		// a crypto-major feeIndex before SOL/USD was reached in the scan.
+		if !strings.EqualFold(p.To, "USD") {
+			continue
+		}
 		gainsCacheMu.Lock()
 		gainsPairCache[strings.ToUpper(p.From)] = p
 		gainsPairIdxCache[strings.ToUpper(p.From)] = i
 		gainsPairAt[strings.ToUpper(p.From)] = time.Now()
 		gainsCacheMu.Unlock()
-		if strings.EqualFold(p.From, asset) && strings.EqualFold(p.To, "USD") {
+		if strings.EqualFold(p.From, asset) {
 			return p, nil
 		}
 		time.Sleep(150 * time.Millisecond)

harnesses/perp-fees/cmd/script/gmx.go

@@ -38,11 +38,18 @@ type gmxGqlReq struct {
 type gmxMarketInfo struct {
 	Data struct {
 		MarketInfos []struct {
-			ID                                       string `json:"id"`
-			PositionFeeFactorForPositiveImpact       string `json:"positionFeeFactorForPositiveImpact"`
-			PositionFeeFactorForNegativeImpact       string `json:"positionFeeFactorForNegativeImpact"`
+			ID                                 string `json:"id"`
+			PositionFeeFactorForPositiveImpact string `json:"positionFeeFactorForPositiveImpact"`
+			PositionFeeFactorForNegativeImpact string `json:"positionFeeFactorForNegativeImpact"`
+			// Some Subsquid deployments expose an unsuffixed field name.
+			// We fall back to it when the impact-branch fields are absent
+			// so a schema rename does not silently zero the fee.
+			PositionFeeFactor string `json:"positionFeeFactor"`
 		} `json:"marketInfos"`
 	} `json:"data"`
+	Errors []struct {
+		Message string `json:"message"`
+	} `json:"errors"`
 }
 
 type gmxMarketsRESTItem struct {
@@ -73,40 +80,66 @@ func fetchGMX(v VenueConfig) PerpSample {
 	// 1) Position fee from Subsquid (live on-chain factor)
 	q := gmxGqlReq{
 		Query: `query { marketInfos(where: {id_eq: "` + market + `"}) {
-			id positionFeeFactorForPositiveImpact positionFeeFactorForNegativeImpact
+			id positionFeeFactorForPositiveImpact positionFeeFactorForNegativeImpact positionFeeFactor
 		}}`,
 	}
 	bodyBytes, _ := json.Marshal(q)
 	req, _ := http.NewRequest("POST", gmxSubsquid, bytes.NewBuffer(bodyBytes))
 	req.Header.Set("Content-Type", "application/json")
 	resp, err := client.Do(req)
 	if err != nil {
-		s.Err = fmt.Sprintf("gql: %v", err)
+		s.Err = fmt.Sprintf("subsquid_failed: %v", err)
 		s.FetchLatencyMs = time.Since(start).Milliseconds()
 		return s
 	}
 	defer resp.Body.Close()
 	respBody, _ := io.ReadAll(resp.Body)
 	if resp.StatusCode != 200 {
-		s.Err = fmt.Sprintf("gql_status_%d: %s", resp.StatusCode, truncate(string(respBody), 200))
+		s.Err = fmt.Sprintf("subsquid_failed: status_%d: %s", resp.StatusCode, truncate(string(respBody), 200))
 		s.FetchLatencyMs = time.Since(start).Milliseconds()
 		return s
 	}
 	var info gmxMarketInfo
 	if err := json.Unmarshal(respBody, &info); err != nil {
-		s.Err = fmt.Sprintf("gql_parse: %v", err)
+		s.Err = fmt.Sprintf("subsquid_failed: parse: %v", err)
+		s.FetchLatencyMs = time.Since(start).Milliseconds()
+		return s
+	}
+	if len(info.Errors) > 0 {
+		s.Err = fmt.Sprintf("subsquid_failed: gql_error: %s", info.Errors[0].Message)
 		s.FetchLatencyMs = time.Since(start).Milliseconds()
 		return s
 	}
 	if len(info.Data.MarketInfos) == 0 {
-		s.Err = "market_not_found"
+		s.Err = "subsquid_failed: market_not_found"
 		s.FetchLatencyMs = time.Since(start).Milliseconds()
 		return s
 	}
 	m := info.Data.MarketInfos[0]
 	// FLOAT_PRECISION = 1e30. positionFeeFactor = factor * 1e30.
 	// e.g. "600000000000000000000000000" = 0.0006 = 6 bps.
-	negativeFee := factor1e30ToBps(m.PositionFeeFactorForNegativeImpact)
+	//
+	// Branch choice: report the NEGATIVE-impact factor. Opening a long
+	// against a bid-heavy book (or a short against an ask-heavy book) pays
+	// the negative-impact branch, which is the worst-case opening cost and
+	// matches the "all-in fee at open" methodology used for the CEXes here.
+	// If the schema is on an older/unsuffixed version we fall back to
+	// positionFeeFactor so a rename does not silently zero the reported fee.
+	rawFactor := m.PositionFeeFactorForNegativeImpact
+	if rawFactor == "" {
+		rawFactor = m.PositionFeeFactor
+	}
+	if rawFactor == "" {
+		s.Err = "subsquid_failed: missing positionFeeFactor fields"
+		s.FetchLatencyMs = time.Since(start).Milliseconds()
+		return s
+	}
+	negativeFee := factor1e30ToBps(rawFactor)
+	if negativeFee == 0 {
+		s.Err = fmt.Sprintf("subsquid_failed: zero fee factor for market %s", market)
+		s.FetchLatencyMs = time.Since(start).Milliseconds()
+		return s
+	}
 	s.TakerFeeBps = negativeFee
 	// On GMX there's no orderbook spread — slippage = priceImpact, which
 	// is ~0 for $1k notional vs. multi-million pool TVL.

harnesses/perp-fees/cmd/script/paradex.go

@@ -83,15 +83,21 @@ func fetchParadex(v VenueConfig) PerpSample {
 		sz, _ := strconv.ParseFloat(a[1], 64)
 		levels = append(levels, bookLevel{Px: px, Sz: sz})
 	}
-	effective, err := walkBookForNotional(levels, v.NotionalUSD)
+	// Paradex REST caps depth at 100 levels. When the walker eats through
+	// more than ~90% of the visible book the "effective price" is dominated
+	// by the tail levels and stops being a real quote (methodology: skip
+	// when depth thins out). Cap the fill ratio to skip such tiers rather
+	// than publish an inflated bps.
+	const paradexMaxFillRatio = 0.9
+	effective, err := walkBookForNotionalCapped(levels, v.NotionalUSD, paradexMaxFillRatio)
 	if err != nil {
 		s.Err = fmt.Sprintf("walk: %v", err)
 		s.FetchLatencyMs = time.Since(start).Milliseconds()
 		return s
 	}
 	s.SpreadBps = (effective - mid) / mid * 10000
 	s.AllInBps = s.TakerFeeBps + s.SpreadBps
-	applyBookTiers(&s, levels, mid)
+	applyBookTiersCapped(&s, levels, mid, paradexMaxFillRatio)
 
 	// 3) Funding: per 8h period, normalize to per hour.
 	var fund paradexFunding

harnesses/perp-fees/cmd/script/walk.go

@@ -50,6 +50,33 @@ func walkBookForNotional(levels []bookLevel, notional float64) (float64, error)
 	return notional / totalQty, nil
 }
 
+// totalBookNotional sums price*size across every level, giving the total
+// visible USD depth on this side of the book.
+func totalBookNotional(levels []bookLevel) float64 {
+	total := 0.0
+	for _, l := range levels {
+		if l.Px <= 0 || l.Sz <= 0 {
+			continue
+		}
+		total += l.Px * l.Sz
+	}
+	return total
+}
+
+// walkBookForNotionalCapped walks like walkBookForNotional but returns an
+// error when the walk would consume more than maxFillRatio of the total
+// visible book depth. Depth-capped venues (Paradex depth=100) can otherwise
+// publish an eaten-through effective price when the tier notional is close
+// to or larger than the visible book. Use this for venues where the fetched
+// depth is a hard ceiling on visible liquidity.
+func walkBookForNotionalCapped(levels []bookLevel, notional, maxFillRatio float64) (float64, error) {
+	total := totalBookNotional(levels)
+	if total > 0 && notional > total*maxFillRatio {
+		return 0, fmt.Errorf("book_too_thin: %.2f of visible %.2f > %.0f%%", notional, total, maxFillRatio*100)
+	}
+	return walkBookForNotional(levels, notional)
+}
+
 // applyBookTiers walks the already-fetched ask levels once per tier notional
 // and fills s.Tiers with taker fee + tier spread. Call it after s.TakerFeeBps
 // is known. Costs no extra API calls: the book was fetched for the headline
@@ -76,6 +103,27 @@ func applyBookTiers(s *PerpSample, levels []bookLevel, mid float64) {
 	}
 }
 
+// applyBookTiersCapped is like applyBookTiers but skips any tier whose
+// notional would consume more than maxFillRatio of the visible book depth.
+// Use this on venues that expose a hard depth cap (e.g. Paradex depth=100)
+// where the top of a shallow book gives a misleading "effective price" when
+// the walk eats through the entire visible side.
+func applyBookTiersCapped(s *PerpSample, levels []bookLevel, mid, maxFillRatio float64) {
+	for _, n := range tierNotionals {
+		effective, err := walkBookForNotionalCapped(levels, n, maxFillRatio)
+		if err != nil {
+			s.SkippedTiers = append(s.SkippedTiers, notionalLabel(n))
+			continue
+		}
+		spread := (effective - mid) / mid * 10000
+		s.Tiers = append(s.Tiers, TierSample{
+			Notional:  notionalLabel(n),
+			SpreadBps: spread,
+			AllInBps:  s.TakerFeeBps + spread,
+		})
+	}
+}
+
 // applyFlatTiers publishes the same all-in figure at every tier. Used by the
 // oracle-priced venues (GMX v2, gains.trade) where the harness-read cost is
 // a fixed fraction of position size, so the bps figure does not change with