{ "about": "Lessons from 17+ LoRA fine-tunes of gemma-3-4b-it as a command router on one 8 GB consumer GPU. Each claim is limited to this task, this base model and this recipe. Accuracy numbers are passed / runnable cases; see versions.json for definitions.", "findings": [ { "id": "F01", "title": "LoRA rank, not more data, moved the hardest capability", "claim": "Three rounds of more data (v3, v4, v5) left the clarify capability at 56-60%. Raising rank from 16 to 64 on the same 757 examples lifted overall accuracy 4 pp and clarify to ~75%; rank 112 lifted clarify to ~85%. Dropping back to rank 64 on a later corpus cost clarify 13 pp.", "evidence": { "overall_r16_vs_r64": [ 0.8902, 0.9306 ], "clarify_by_rank_865set": { "r16": 0.521, "r64": 0.755, "r112": 0.851 }, "v10_r64_vs_v9_r112_clarify": [ 0.73, 0.86 ], "v10_vs_v9_overall_1125set": [ 0.9164, 0.9307 ] }, "sources": [ "bake_v6wide_vs_v5_summary.json", "LOCAL_LLM_PHASE2_ROADMAP.md:996-1013,1061-1062", "bake_v10r64_vs_v12_summary.json" ], "measured_on": "2026-09-20" }, { "id": "F02", "title": "Rank 112 is the ceiling on an 8 GB card at ~7k-token examples", "claim": "With chunked MLP, save_on_cpu and a bf16 lm_head, rank 112 peaks at ~6.65 GB in real runs; rank 128 and DoRA (at 64 and 112) OOM on the longest example. Rank 112 over 64 bought +0.5 pp on the 865 set, inside run-to-run noise there, but held clarify gains later.", "evidence": { "real_peak_r112_mb": "6643-6658", "probe_r128": "OOM", "probe_dora_r64": "OOM", "probe_dora_r112": "OOM", "v7_vs_v6": [ 0.9434, 0.9387 ] }, "sources": [ "vram.json", "probe_rank128.log", "probe_dora_r64.log", "probe_dora_r112.log", "bake_v7r112_vs_v6wide_summary.json" ], "measured_on": "2026-09-18" }, { "id": "F03", "title": "Incremental patching on a trained adapter caused catastrophic forgetting", "claim": "Continuing v3's adapter for 8 epochs on only 59 targeted examples dropped accuracy 7.3 pp on v3's own test set. Two targeted capabilities improved; the main target (clarify) got worse; four untouched capabilities collapsed to 25-56%. Mixing the same 59 examples into the full corpus and retraining from base (v5) recovered every collapsed capability and matched v3 exactly.", "evidence": { "v3": "709/790", "v4": "651/790", "v5": "709/790", "v4_collapsed": { "run_agent": 0.25, "resume_agent": 0.417, "dispatch_code": 0.524, "calendar_availability": 0.562 }, "v5_same_capabilities": 1.0 }, "sources": [ "LOCAL_LLM_PHASE2_ROADMAP.md:867-918,953-973", "bake_v4_summary.json", "bake_v5_summary.json" ], "measured_on": "2026-09-15" }, { "id": "F04", "title": "Held-out eval loss does not pick the best checkpoint", "claim": "In every run with an eval set, eval loss bottomed early (step 75-125 of ~440 in most runs) and drifted up, while accuracy kept improving or held. The lowest-loss v9 checkpoint (step 175) scored 6.5 pp worse overall than the final. v12 (2 epochs) had the lowest final eval loss of the program and lower accuracy than v11, whose final eval loss was among the highest. v15 showed the 'better' loss curve and lost on accuracy. The eval targets carry partial arguments, which the program notes suggests rewards under-confident outputs.", "evidence": { "v9_step175": { "eval_loss": 0.1306, "accuracy": 0.8658 }, "v9_final": { "eval_loss": 0.1861, "accuracy": 0.9305 }, "final_eval_loss": { "v11": 0.3997, "v12": 0.2934, "v13": 0.3497, "v15": 0.3385, "v17": 0.4378 }, "best_eval_step": { "v9": 175, "v10": 125, "v11": 75, "v12": 125, "v13": 100, "v15": 300, "v16": 100, "v17": 100 }, "accuracy_1131set": { "v11": 0.9637, "v12": 0.9558 } }, "partial_arg_explanation_inferred": true, "sources": [ "train_v9-r112_relaunch_1789754626.log", "train_v1[0-7]*_relaunch_*.log (eval loss by checkpoint)", "bake_v9_vs_v11_summary.json", "bake_v12e2_vs_v11_summary.json", "LOCAL_LLM_PHASE2_ROADMAP.md:1045,1196-1197,1308-1311" ], "measured_on": "2026-09-23" }, { "id": "F05", "title": "Accuracy plateaus well before the end, and the final checkpoint is as good as any", "claim": "A held-out accuracy sweep of all 16 v9 checkpoints showed accuracy flat from step 200 of 405, with no mid-run peak worth picking.", "evidence": { "held_out_108_by_step": { "25": 86, "50": 94, "75": 96, "100": 86, "125": 95, "150": 94, "175": 96, "200": 100, "225": 99, "250": 100, "275": 97, "300": 99, "325": 99, "350": 100, "375": 100, "400": 99, "405": 99 } }, "conflict": "The program notes say the plateau is '91.7-92.6% from step 200'; the sweep summary has step 275 at 97/108 (89.8%). The plateau claim holds loosely; the stated range is slightly too tight. Trust the summary.", "sources": [ "bake_heldout_v9sweep_summary.json", "LOCAL_LLM_PHASE2_ROADMAP.md:1059-1060" ], "measured_on": "2026-09-19" }, { "id": "F06", "title": "A third epoch is worth about one point", "claim": "Same data and recipe at 2 epochs (v12) lost 0.8 pp overall and 4 held-out cases versus 3 epochs (v11), mostly on clarify. It saved about 4 hours of training, which buys nothing at inference.", "evidence": { "v11": { "passed": 1090, "held_out": "108/110" }, "v12": { "passed": 1081, "held_out": "104/110" }, "runtime_h": { "v11": 11.8, "v12": 7.9 } }, "sources": [ "bake_v12e2_vs_v11_summary.json", "train_v11-r112_relaunch_1789910846.log", "train_v12-r112-e2_relaunch_1789974074.log" ], "measured_on": "2026-09-21" }, { "id": "F07", "title": "rsLoRA raised the headline and eroded the safety guards", "claim": "rsLoRA at alpha = rank (scale 10.6) flatlined and was killed. At alpha 32 (scale 3.0) it posted the program's best overall accuracy, but every gained case was train-flagged; it lost 5 held-out cases and failed held-out guards the plain recipe passes, mostly cases with pasted or quoted content. Its checkpoint average did not recover them.", "evidence": { "v13_overall": 0.9752, "v11_overall": 0.9637, "v13_held_out": "103/110", "v11_held_out": "108/110", "v13_train_flagged": 0.9794, "guards_174": { "v11": "169, 1 held-out breach", "v13": "168, 3 held-out breaches (2 pasted/quoted content)" }, "a112_eval_loss_step50": 0.577, "v11_eval_loss_step50": 0.33 }, "sources": [ "bake_v13_vs_v11_summary.json", "bake_guards_baseline_summary.json", "bake_v13avg_vs_v11avg_summary.json", "LOCAL_LLM_PHASE2_ROADMAP.md:1212-1240" ], "measured_on": "2026-09-22" }, { "id": "F08", "title": "LoRA dropout did not fit at rank 112 and did not improve generalization where it fit", "claim": "Dropout 0.1 on top of v13 died three times at rank 112 (one Xid 31 fault, two OOMs). At equal steps (step 225) dropout raised overall accuracy 3.2 pp, all on train-flagged cases, and widened the train-to-held-out gap from 1.2 to 6.8 pp; it did improve guards (148 to 157 of 174). At rank 96 it trained clean but bought one held-out case over v13, cost 0.7 pp, and was vetoed on guards.", "evidence": { "ck225_nodrop": { "overall": 0.8382, "train": 0.8394, "held_out": "91/110", "guards": "148/174, 10 held-out breaches" }, "ck225_drop10": { "overall": 0.87, "train": 0.8766, "held_out": "89/110", "guards": "157/174, 7 held-out breaches" }, "v15": { "overall": 0.9682, "held_out": "104/110", "guards": "165/174, 4 held-out breaches, 3 pasted/quoted content" } }, "sources": [ "bake_dropout_ab_ck225_summary.json", "bake_guards_dropout_ab_summary.json", "bake_v15_vs_v13_summary.json", "bake_guards_v15_summary.json", "train_v14-r112-drop10_relaunch_*.log" ], "measured_on": "2026-09-23" }, { "id": "F09", "title": "Checkpoint averaging is noise on this recipe", "claim": "An exact mean of v11's last seven checkpoints scored +0.4 pp on one bake and +0.0 on a re-bake, with identical held-out both times. The rsLoRA average scored 0.4 pp below its final checkpoint.", "evidence": { "v11_avg": [ 0.9691, 0.9637 ], "v11": [ 0.9646, 0.9637 ], "v13_avg": 0.9717, "v13": 0.9761 }, "method_note": "averaging done exactly by concatenating LoRA A/B along rank (r = n x r0, alpha fixed), with an extra 1/sqrt(n) on B for rsLoRA", "sources": [ "bake_v11avg_vs_v11_summary.json", "bake_v13avg_vs_v11avg_summary.json", "average_checkpoints.py" ], "measured_on": "2026-09-22" }, { "id": "F10", "title": "Data quantity beat data purity: a 25% disjoint holdout cost 8.5 pp", "claim": "Holding a quarter of the gold set out of training (v16, 873 vs 1,167 examples, recipe unchanged) lost 8.5 pp on 82 cases neither model saw. The same experiment showed the plain recipe was not inflated by memorization: v11 scored 96.3% on those never-seen cases against a 96.4% headline. Recommendation recorded: hold back ~10%, not 25%.", "evidence": { "fair_82": { "v11": 0.963, "v16": 0.878 }, "v11_trained_v16_not_317": { "v11": 0.946, "v16": 0.826 }, "v16_overall_1148": 0.9016, "v11_overall_1148": 0.9591 }, "sources": [ "bake_v16.log", "LOCAL_LLM_PHASE2_ROADMAP.md:1324-1352" ], "measured_on": "2026-09-24" }, { "id": "F11", "title": "The test set was mostly training data, but the plain recipe still generalized", "claim": "Through v7, ~79% of the 865-case test set was also in the training corpus and the held-out slice was almost all must-not-fire guards with no training target; the untuned 12B beat every fine-tune on that slice until v9. Once guards got targets (v8/v9) and the harness was fixed, fine-tunes passed the 12B on held-out. The later disjoint test (F10) then showed the plain recipe was not a memorization score.", "evidence": { "v7_865": { "train_flagged": 0.963, "held_out": 0.871 }, "untuned_12b_865_held_out": 0.941, "v9_vs_12b_held_out_1125": [ 0.9167, 0.8889 ] }, "unverified_parts": "the 79% overlap and the v7/12B slice numbers are from the ledger and could not be recomputed (train flags changed after 09-18)", "sources": [ "router_ledger.html (findings)", "LOCAL_LLM_PHASE2_ROADMAP.md:1028-1032", "bake_12b_vs_v12_summary.json" ], "measured_on": "2026-09-19" }, { "id": "F12", "title": "Negative guards need training targets", "claim": "Must-not-fire cases had no gold label, so nothing taught them. Authoring targets for 126 of them lifted the guard families 19-50 pp in v8, and they held in v9. A later 86-case adversarial guard batch cut held-out breaches in v17 relative to v11 over repeated passes.", "evidence": { "guard_family_gain_pp_v8": "19-50", "not_run_pause_guard": [ 0.67, 0.92, 0.96 ], "guards_267_three_passes": { "v11": 250, "v17": 256 } }, "sources": [ "LOCAL_LLM_PHASE2_ROADMAP.md:1030-1036,1044", "router_ledger.html (findings)", "bake_guards_recheck_v11_*_summary.json", "bake_guards_recheck_v17_*_summary.json" ], "measured_on": "2026-09-25" }, { "id": "F13", "title": "Guard batches need counterweights or they teach over-refusal", "claim": "On 127 never-seen cases the weakest group for both models was counterweights: quote something, then genuinely issue a command. The program notes concludes the real residual weakness is over-refusal, not over-action, and that a guard batch shipped without counterweights would have pushed further that way.", "evidence": { "counterweights_24": { "v11": 0.833, "v16": 0.667 }, "adversarial_guards_62": { "v11": 0.855, "v16": 0.903 } }, "sources": [ "LOCAL_LLM_PHASE2_ROADMAP.md:1354-1374" ], "measured_on": "2026-09-24" }, { "id": "F14", "title": "Label conflicts cost more than missing data", "claim": "Four new clarify labels on bare confirmation words contradicted older gold labels and dropped gate_approve from 100% to 70% in v8. Removing them restored it to 94% in v9 with no other change to that capability.", "evidence": { "gate_approve": [ 1.0, 0.7, 0.94 ] }, "sources": [ "LOCAL_LLM_PHASE2_ROADMAP.md:1044-1046", "router_ledger.html (findings)" ], "measured_on": "2026-09-19" }, { "id": "F15", "title": "A single bake pass over a small set overstated a margin by about 2x", "claim": "The one-pass guard bake that justified switching production to v17 scored it 257/267 with 1 held-out breach against v11's 247 with 4. Three fresh passes each, same conditions, gave v17 256 with 4 breaches and v11 250 with 5, identical every pass. The real margin is 6 cases, not 10, and both models still miss some pasted-text cases, including one the single pass had counted as a v17 pass.", "evidence": { "single_pass": { "v17": "257, 1 breach", "v11": "247, 4 breaches" }, "three_passes": { "v17": "256, 4 breaches (x3)", "v11": "250, 5 breaches (x3)" } }, "open_question": "Why the single pass differed from four identical v11 passes (247 vs 250) at temperature 0 is not explained in the sources.", "sources": [ "bake_guards_v17_summary.json", "bake_guards_recheck_v11_[1-3]_summary.json", "bake_guards_recheck_v17_[1-3]_summary.json", "LOCAL_LLM_PHASE2_ROADMAP.md:1420-1444" ], "measured_on": "2026-09-25" }, { "id": "F16", "title": "Run-to-run noise differs by model; repeat bakes before comparing", "claim": "Three all-caps passes: v11 landed within one case of itself (spread 0.09 pp); v9 (marvin-v1.2) swung 14 cases (1.18 pp). Sub-point deltas against a noisy model are meaningless without repeats. Per-capability slices with n of about 8 need intervals.", "evidence": { "v11_passes": [ 1145, 1144, 1145 ], "v9_passes": [ 1098, 1112, 1112 ], "total": 1191, "example_interval": { "guard:open_folder": { "cases": 8, "accuracy": 0.875, "ci": [ 0.785, 0.931 ] } } }, "sources": [ "bake_allcaps_x3_summary.json", "capability_intervals_v11.json" ], "measured_on": "2026-09-22" }, { "id": "F17", "title": "The overall number can rank the worst router first; a guard veto is needed", "claim": "v13 (97.6%) and v15 (96.8%) both beat v11 (96.4%) overall and both obey instructions hidden in pasted text more often. The 174-case guard bake (about 4 minutes) was the only check that separated them. The release rule was changed to: veto any held-out guard breach the incumbent does not have, then rank by held-out accuracy.", "evidence": { "overall": { "v13": 0.9752, "v15": 0.9682, "v11": 0.9637 }, "held_out_guard_breaches_174": { "v11": 1, "v13": 3, "v15": 4 }, "guard_bake_wall_clock_s": [ 220.6, 230.5 ] }, "sources": [ "bake_v15_vs_v13_summary.json", "bake_guards_baseline_summary.json", "bake_guards_v15_summary.json", "pick_release_winner.py docstring" ], "measured_on": "2026-09-23" }, { "id": "F18", "title": "A harness channel bug scored every context case context-free for weeks", "claim": "Bake scripts and both live-model test harnesses routed on one channel while each case's setup seeded context on another, so ~50 context-dependent cases were scored without context for every model. Fixing it moved v9's held-out from 86.8% to 91.7% on the same weights.", "evidence": { "v9_held_out_before_fix": 0.8681, "v9_held_out_after_fix": 0.9167 }, "sources": [ "LOCAL_LLM_PHASE2_ROADMAP.md:1054-1057", "bake_v9_vs_v11_summary.json", "bake_12b_vs_v12_summary.json" ], "measured_on": "2026-09-19" }, { "id": "F19", "title": "Grammar-constrained decoding helps only when the enum matches what the router accepts", "claim": "Forcing the JSON schema and capability enum at decode time first cost 0.7 pp: the enum was built from the visible capability docs and omitted one hidden capability, so all 8 of its cases were forced to clarify. With every accepted capability in the enum it was +0.1 pp and cut unparseable outputs from 3 to 1.", "evidence": { "first": 0.9573, "plain": 0.9637, "fixed": 0.9655, "unparseable": [ 3, 1 ] }, "sources": [ "bake_v11_schema_on_summary.json", "bake_v11_schema_off_summary.json", "bake_v11_schema_on_fixed_summary.json", "LOCAL_LLM_PHASE2_ROADMAP.md:1181-1202" ], "measured_on": "2026-09-21" }, { "id": "F20", "title": "One-line capability docs earn their prompt tokens", "claim": "Stripping the one-line capability descriptions (24% of the routing prompt, ~1,669 tokens) cost 2.9 pp. Guards were untouched; the losses were disambiguation between capabilities whose names do not explain themselves, and unparseable outputs rose from 3 to 8.", "evidence": { "with_docs": 0.9637, "terse": 0.9355, "broke": 39, "fixed": 7, "guards_174_both": 169, "unparseable": [ 3, 8 ] }, "sources": [ "bake_v11_terse_summary.json", "bake_guards_terse_summary.json", "LOCAL_LLM_PHASE2_ROADMAP.md:1242-1249" ], "measured_on": "2026-09-22" }, { "id": "F21", "title": "Q8_0 beat Q4_0 by 3.2 pp on the fine-tuned router", "claim": "Quantizing the same merged weights to Q4_0 cost 3.2 pp for about 15% speed. Q8_0 stayed for every release.", "evidence": { "q8": 0.8936, "q4": 0.8613, "s_per_case": { "q8": 1.506, "q4": 1.278 } }, "sources": [ "bake_v5_q4vq8_summary.json" ], "measured_on": "2026-09-16" }, { "id": "F22", "title": "Inference speed is set by model size, not training choices", "claim": "Every rank-112 fine-tune ran 1.17-1.34 s per case on the same card; rank 64 was a few percent faster (time ratio 0.913 vs rank 112 in one head-to-head; the ledger says 4%); the untuned 12B was 2.1-2.7 s. A model that lands split across CPU and GPU (because other models are resident) slows to ~2.8 s per case or worse, so evict between bake halves.", "evidence": { "r112_s_per_case": "1.17-1.34", "r64_vs_r112_time_ratio": 0.913, "r64_vs_r16_time_ratio": 0.917, "untuned_12b": [ 2.124, 2.685 ], "split_cpu_gpu": 2.8 }, "sources": [ "bake summaries (mean_s_per_case)", "bake_v10r64_vs_v12_summary.json", "bake_v6wide_vs_v5_summary.json", "LOCAL_LLM_PHASE2_ROADMAP.md:1021-1024" ], "measured_on": "2026-09-25" }, { "id": "F23", "title": "A 4B fine-tune beat the untuned 12B it replaced", "claim": "On the 1,125-case set the untuned gemma4:12b-it-qat scored 82.1% (held-out 88.9%) at 2.7 s per case; the best 4B fine-tune scored 96.4% (held-out 96.3%) at 1.25 s. The first fine-tune (v1, 152 examples) had scored 4.9 pp below the 12B; the 490-example v2 was the first to beat it.", "evidence": { "untuned_12b": { "accuracy": 0.8213, "held_out": 0.8889, "s_per_case": 2.685 }, "v11": { "accuracy": 0.9636, "held_out": 0.963, "s_per_case": 1.245 }, "v1_vs_12b_550": [ 0.8036, 0.8527 ], "v2_vs_12b_550": [ 0.8964, 0.8527 ] }, "sources": [ "bake_12b_vs_v12_summary.json", "bake_v11_vs_v12_summary.json", "bake_v2_summary.json" ], "measured_on": "2026-09-21" }, { "id": "F24", "title": "Measure the memory ceiling with a calibrated probe instead of crashing into it", "claim": "A one-step probe on the longest example over-estimates a real run's peak by ~4.7%. Scaling the probe by that factor predicted rank 96 + dropout at 6,793 MB; the real run peaked at 6,629 MB (2.5% error). Discovering the rank-112 dropout ceiling by training had cost three crashes and about 11 GPU-hours.", "evidence": { "probe_mb": 7127.6, "predicted_mb": 6793, "actual_mb": 6629, "error_pct": 2.5 }, "sources": [ "vram.json (calibration)", "v15_r96_chain.sh:5-10", "train_v15-r96-drop10_relaunch_1790142410.log" ], "measured_on": "2026-09-23" }, { "id": "F25", "title": "Deterministic pre-router steps remove whole miss classes, invisibly to the bake", "claim": "v11's largest miss class was 14 short confirmations routed to clarify. A deterministic step before the router now resolves a bare yes or no against real pending state with no model call. Separately, regex shortcuts route some standard session phrasings before any model call. Bakes call the model directly, so production accuracy on those phrasings is better than the bake shows.", "evidence": { "confirmation_miss_class": 14, "session_positives_31": { "routed_by_regex_correctly": 10, "routed_by_regex_wrongly": 0 }, "backstop_covered_gold_cases_dropped_for_v16": 58 }, "sources": [ "LOCAL_LLM_PHASE2_ROADMAP.md:1165-1168,1376-1382", "holdout-manifest-expanded.json (backstop_covered)" ], "measured_on": "2026-09-24" }, { "id": "F26", "title": "Small models need the context window set explicitly", "claim": "Ollama gave each model a VRAM-derived default context of 4096 tokens while the routing prompt was ~5,650. For one model Ollama silently trimmed the prompt to 2,050 tokens, cutting off the capability list at its front, with its truncated flag still 0; routing still returned plausible answers, so misses looked like flakiness. Sending num_ctx explicitly on every call and a test that bounds prompt tokens against it fixed it.", "evidence": { "default_num_ctx": 4096, "routing_prompt_tokens": 5650, "trimmed_to_tokens": 2050, "num_ctx_now": 9216 }, "sources": [ "CLAUDE.md (Lessons learned: Ollama default context window)", "LOCAL_LLM_PHASE2_ROADMAP.md:1063" ], "measured_on": "2026-09-07" }, { "id": "F27", "title": "New capabilities often need little or no training", "claim": "v11 handled 25 session-capability paraphrases and discriminators at 88% having never seen those capabilities, and refused all 16 session-injection guard attempts.", "evidence": { "session_paraphrases_25": 0.88, "session_injection_guards_16": 1.0 }, "sources": [ "LOCAL_LLM_PHASE2_ROADMAP.md:1356-1370" ], "measured_on": "2026-09-24" }, { "id": "F28", "title": "Fine-tunes trade cautious misses for confident ones", "claim": "v2 beat the untuned 12B by 4.3 pp, but a larger share of its remaining misses were confident picks of a forbidden capability rather than an honest clarify. v7 likewise fixed clarify but lost ground on context and must-not-route cases.", "evidence": { "forbidden_pick_share_of_misses": { "untuned_12b": "6/81", "v1": "8/108", "v2": "12/57" }, "v7_context_cases_n31": [ 0.968, 0.871 ] }, "sources": [ "LOCAL_LLM_PHASE2_ROADMAP.md:753-762,1008-1013" ], "measured_on": "2026-09-18" }, { "id": "F29", "title": "AMD and NVIDIA need the same memory for the same QLoRA step", "claim": "The identical 4-bit QLoRA probe at 7,177 tokens peaked at 5,721.3 MB (rank 16) and 6,406.2 MB (rank 64) on an RX 7900 XTX under ROCm 7.2.1 on Windows, and at 5,788.9 MB and 6,465.0 MB on an RTX 3060 Ti under CUDA on Linux: within 1.2%, AMD slightly lower. Memory before the step matched to within 0.5 MB. A VRAM budget measured on one vendor carries over to the other for this recipe.", "evidence": { "peak_mb_r16": { "rx_7900_xtx": 5721.3, "rtx_3060_ti": 5788.9 }, "peak_mb_r64": { "rx_7900_xtx": 6406.2, "rtx_3060_ti": 6465.0 }, "pre_step_mb_r64": { "rx_7900_xtx": 3597.7, "rtx_3060_ti": 3597.3 } }, "sources": [ "vram.json cross_gpu" ], "measured_on": "2026-10-15" }, { "id": "F30", "title": "LoRA dropout costs about 170 MB at the probe, not 350", "claim": "Turning lora_dropout from 0 to 0.1 raised the one-step peak by 174.5, 173.7, 166.8 and 168.6 MB at ranks 96, 112, 128 and 256 on the RX 7900 XTX. The cost is close to flat across rank. The earlier 350 MB figure on the VRAM page was an estimate from crash margins, not a measurement.", "evidence": { "dropout_cost_mb_by_rank": { "96": 174.5, "112": 173.7, "128": 166.8, "256": 168.6 } }, "sources": [ "vram.json cross_gpu" ], "measured_on": "2026-10-15" }, { "id": "F31", "title": "Rank cost is linear; DoRA and bf16 add large fixed costs", "claim": "On the RX 7900 XTX at 7,177 tokens, the peak rose from 5,721.3 MB at rank 16 to 9,338.3 MB at rank 256, about 15.1 MB per rank, which matches the 15 MB per rank measured earlier on the 8 GB NVIDIA card. DoRA added about 1.83 GB at ranks 64 and 112 and made the step about 38% slower. Loading the base model in bf16 instead of 4-bit added about 5.1 GB at every rank. Without the memory fixes, the step needed about 36 GB.", "evidence": { "peak_mb_by_rank_4bit": { "16": 5721.3, "64": 6406.2, "96": 6884.8, "112": 7194.5, "128": 7395.9, "192": 8383.4, "256": 9338.3 }, "dora_extra_mb": { "64": 1834.5, "112": 1824.3 }, "bf16_extra_mb": { "16": 5105.0, "64": 5103.3, "112": 5123.0, "256": 5104.9 }, "no_fixes_peak_mb": { "16": 35829.2, "112": 36581.3 }, "peak_mb_by_seq_len_r112": { "2048": 5585.8, "4096": 6207.9, "7177": 7194.5, "8192": 7486.0, "12288": 8917.2, "16384": 10523.6 } }, "sources": [ "vram.json cross_gpu" ], "measured_on": "2026-10-15" }, { "id": "F32", "title": "A 20B mixture-of-experts model generates faster than dense 9B and 12B models", "claim": "On an RX 7900 XTX under Ollama 0.30.10, gpt-oss:20b generated about 116-123 tokens per second, against 84 for qwen3.5:9b and 66-68 for gemma4:12b-it-qat, and processed prompts 1.4 to 2.5 times faster than either. It needs the most memory, 12.3 GB resident, and the slowest cold load at 14.7 s. For a card with room to hold it, size in parameters is a poor guide to speed.", "evidence": { "gen_tok_per_s_short_long": { "gpt-oss:20b": [ 123.0, 116.2 ], "qwen3.5:9b": [ 84.9, 84.2 ], "gemma4:12b-it-qat": [ 68.4, 66.4 ] }, "prompt_tok_per_s_short_long": { "gpt-oss:20b": [ 3730.2, 4118.1 ], "qwen3.5:9b": [ 2588.9, 2704.7 ], "gemma4:12b-it-qat": [ 1526.1, 1661.9 ] }, "resident_mb": { "gpt-oss:20b": 12261, "qwen3.5:9b": 5642, "gemma4:12b-it-qat": 7601 } }, "inferred": true, "note": "The mixture-of-experts explanation (only a few billion parameters active per token) is the usual reading of gpt-oss's design; this bench measured speed only.", "sources": [ "inference.json" ], "measured_on": "2026-10-15" } ], "tried_and_closed": [ { "method": "incremental patch on a trained adapter", "verdict": "forgetting; retrain from base", "finding": "F03" }, { "method": "Q4_0 quantization", "verdict": "-3.2 pp; Q8_0 stays", "finding": "F21" }, { "method": "rank 128", "verdict": "OOM", "finding": "F02" }, { "method": "DoRA at rank 64 and 112", "verdict": "OOM inside the step", "finding": "F02" }, { "method": "checkpoint selection by eval loss", "verdict": "picked a 6.5 pp worse checkpoint", "finding": "F04" }, { "method": "rank 64 on the enriched corpus", "verdict": "-1.4 pp, clarify -13 pp", "finding": "F01" }, { "method": "two epochs", "verdict": "-0.8 pp", "finding": "F06" }, { "method": "grammar-constrained decoding", "verdict": "+0.1 pp with the right enum; kept on", "finding": "F19" }, { "method": "checkpoint averaging", "verdict": "noise", "finding": "F09" }, { "method": "rsLoRA", "verdict": "higher headline, worse guards", "finding": "F07" }, { "method": "LoRA dropout", "verdict": "does not fit at r112; no generalization gain at r96", "finding": "F08" }, { "method": "25% disjoint holdout corpus", "verdict": "-8.5 pp on the fair slice", "finding": "F10" }, { "method": "terse prompt without capability docs", "verdict": "-2.9 pp", "finding": "F20" }, { "method": "decoding temperature as a lever", "verdict": "already 0", "source": "router_ledger.html (tried and closed)" }, { "method": "MoE upcycling / widening", "verdict": "considered, not tried: a 9B upcycled to 4 experts needs ~13.5 GB quantized for weights alone", "inferred": true, "source": "LOCAL_LLM_PHASE2_ROADMAP.md:1109-1118" }, { "method": "two-model disagreement cascade", "verdict": "dropped: no untuned local model was left that beat the fine-tune on any slice", "source": "LOCAL_LLM_PHASE2_ROADMAP.md:1175; router_ledger.html" } ], "untried": [ "LR sweep (2e-4 linear never varied)", "LoRA+", "NEFTune, higher weight decay", "class-balanced loss", "LoRA on embeddings or the output head (needs more VRAM)", "a bigger base (gemma-3-12b, Qwen3.5-9B) on rented hardware", "training on real logged corrections (none collected before the program closed)" ] }