diff --git a/setup.py b/setup.py index 4f0b23599f..b031706c46 100644 --- a/setup.py +++ b/setup.py @@ -1480,9 +1480,14 @@ def main() -> int: for i, c in enumerate(CONTEXTS, 1): say(f" {i}) {c // 1024}K tokens" + (" (recommended for your GPU)" if c == rec_ctx else "")) ctx = CONTEXTS[int(ask("Context?", [str(i) for i in range(1, 6)], str(CONTEXTS.index(rec_ctx) + 1), a.yes)) - 1] - if model in ("IQ3_XXS", "IQ3_S") and ram < 90 and ctx > 131072: - warn(f"{model} with a 262K context needs more than 64 GB of RAM ({MODELS[model]['arena_gb']:.0f} GB of experts " - "+ the context): using 128K") + # The experts' arena plus the context's KV (in RAM from 64K up: ~13.7 KB/token at 8 bits) must fit, with room + # for everything else. Counted, not a fixed 90 GB: 24 GB of room keeps the rule for 64 GB PCs as it was (128K + # for both 3-bit models), while a box with ~84 GB keeps the 262K it asked for (measured: a Colab A100-40G runs + # IQ3_S at 262K with images in 56 of its 83.5 GiB). + need_gb = MODELS[model].get("arena_gb", 0) + ctx * 13 * 1056 / 1e9 + 24 + if model in ("IQ3_XXS", "IQ3_S") and ram < need_gb and ctx > 131072: + warn(f"{model} with a {ctx // 1024}K context needs about {need_gb:.0f} GB of RAM " + f"({MODELS[model]['arena_gb']:.0f} GB of experts + the context + room for the rest): using 128K") ctx = 131072 ok(f"context: {ctx} tokens") # the KV cache (the model's memory of the conversation): 8-bit, or 4-bit after a Hadamard rotation (PR #21)