round 42: add elastic Metal vision (strata-vision + mmproj BF16 + 2D M-RoPE GENI)
11234537 unmodified lines43444546474849505120 unmodified lines72737475767778798037 unmodified lines`regolith` runs `swift-1.5-qwen3.8-flash-next` (`qwen4_exp`, 512 experts, 36 gated deltanet + 12 sparse attention layers + 51.2b n-gram ple table + neural mtp draft head) on apple silicon via a patched `strata` metal engine.`regolith` runs `swift-1.5-qwen3.8-flash-next` (`qwen4_exp`, 512 experts, 36 gated deltanet + 12 sparse attention layers + 51.2b n-gram ple table + neural mtp draft head + bf16 vision projector) on apple silicon via a patched `strata` metal engine.
`./scripts/setup_apple_silicon.sh` clones the pinned `strata` and `llama.cpp` trees into `vendor/`, applies `scripts/strata_metal_complete.diff`, downloads apple's `metal-cpp` headers, compiles `strata_window_cli`, and streams + repacks the unpruned `k=512` `iq2_xs` model (`3.90 gb` dense `.gguf` + `36.23 gb` `.experts` + `28.80 gb` `ple-iq4_nl.gguf` + `889 mb` `mtp-q2_0.gguf`) into `models/K512_iq2_xs/`.
`./scripts/serve_k512_local.sh` starts the local server on `http://127.0.0.1:8080` (`/v1/chat/completions` and `/v1/messages`, 128k context, `6.5 gb` wired expert arena, neural mtp + suffix speculative decoding).37 unmodified lines| neural mtp draft head (`mtp-q2_0.gguf`) | `0.85 gb` | `0.85 gb` | `0.71 gb` `mlock`ed `q2_0` experts + `0.09 gb` `q8_0` dense + `0.05 gb` `40,525`-tok draft vocab (`0.18 gb` bf16 released) || context state (`32,896` / `131,200` cells) | `0.58 gb` | `2.12 gb` | `36` gdn recurrent states (`112 mb`, `0 b/tok` growth) + `12` qsa `int8` kv & pooled indexers (`14.2 kib/cell`) || wired moe expert arena (`4,129` expert slots) | `6.50 gb` | `6.50 gb` | `mlock`ed (`33.6%` of all `24,576` experts; remaining `29.7 gb` of `36.23 gb` `.experts` streamed from ssd) || ple n-gram table page cache reserve | `0.50 gb` | `0.50 gb` | `28.80 gb` `ple-iq4_nl.gguf` (`51.2b` params) stays on ssd (`16` row reads/token) || elastic vision encoder (`strata-vision` + `mmproj` bf16) | `0.00 gb` | `0.00 gb` | `0.92 gb` active only during image turns (`vram_elastic`, `60s` idle unload; `.sve` cached by sha-256) || **steady-state decode total** | **`8.79 gb` (`9.5–9.8 gb` rss)** | **`10.30 gb` (`10.8 gb` rss)** | **`9.60` / `11.10 gb` planned against `18.80 gb` cap (`>13 gb` left free for macos)** || transient prefill scratch (`PromptGemm`, `MT=2048`) | `+1.12 gb` | `+1.32 gb` | separate `MAP_ANON` buffer during prefill only, `munmap`ped immediately after prefill |
## optimizations20 unmodified lines- single-command-buffer gpu neural mtp draft chain (`mtp_chain.metal`, `STRATA_METAL_MTP_CHAIN=1`) with `SuffixDrafter::propose_after` hybrid continuation- 64-cell attention sink (`STRATA_METAL_MTP_SINK=64`) + 256-cell local sliding window (`STRATA_METAL_MTP_WIN=256`, synced with `mtp_prefill` `tail_cap`) + `mtp_wrote` valid-run kv tracking + dead mtp indexer elimination (`QSA_ROW0=24` k/v-only `qsa_prep`)- 24 kib `tail_bak_` qsa raw-key snapshot/restore in `Window::run` / `Window::commit` for unrestricted `T <= 4` speculative rollback across 4-cell qsa block boundaries- opt-in online cost-model `SpecGate` (`STRATA_SPEC_GATE=suffix|1`), step-aware `STRATA_METAL_SPEC_MIN_P_SCHEDULE` / `STRATA_METAL_SPEC_MIN_CUM_P`, and exact/typical `temp > 0` speculative verification (`STRATA_SPEC_SAMPLE=exact|typical`)- elastic metal vision (`strata-vision` + `mmproj-Swift-Qwen3.8-Flash-Next-BF16.gguf` + 2d `(t, h, w)` m-rope `qsa_prep#mrope` + `GENI` `.sve` embedding injection + multi-turn image kv reuse)- 4-thread `F_NOCACHE` nvme session save/load (`--save-session` / `--load-session`, `Window::session_save` / `session_load`) serializing all 36 gdn + 12 qsa + ple + mtp sliding-window states (`mtp_cell0_`, `mtp_cell1_`, `mtp_sink_`) directly from/into `MTLBuffer`s at `5.2 gb/s`- bonsai-style automatic nvme prompt & conversation `.kvc` cache (`--cache-disk` / `--cache-disk-size 16384` in `--serve`): startup `.kvc` header indexing (`< 1 ms`), post-`DONE` zero-latency auto-save with obsolete-prefix pruning, 2-prompt shared-prefix learning & `.pin` checkpointing (`prompts-seen.bin`), zero-copy `prefix-share` delta `.kvc` storage against pinned prefixes (`.parent` + `SessionHeader` `base_cells`, storing only `[c0, n_cells)` positional cells while reading `[0, c0)` directly from the parent `.kvc`), and automatic cross-conversation / post-restart nvme restore + `scr` splice over disk- suffix cache reuse (`scr`, `Window::scr_relocate` + `strata::core::scr_plan` with `max_runs = 64` + 2-pass non-overlapping left/right-shift relocation before gap prefill + `scr_carry` `0..3` newline pad insertion for `Δ ≡ 0 (mod 4)` + `<path>.pads` + in-place `active_path` `.kvc` replacement across clm edits + 2-slot mru `ck_ring` in `--serve`) relocating qsa group-0 neox rope keys/pooled keys and mtp cell bounds in place while retaining post-edit 36-layer gdn state (`lcp >= 0` multi-span clm memory edits & stripped-`<think>` turns)
37 unmodified lines343 unmodified linesExpand all3443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803817 unmodified lines38939039139236339339439539639736 unmodified lines4344354364374384394404414424434444454464474484494504514528 unmodified lines343 unmodified linesExpand all jtxt = jinja_p.read_text(encoding="utf-8") if "reasoning_effort|default('xhigh')" in jtxt: jinja_p.write_text(jtxt.replace("reasoning_effort|default('xhigh')", "reasoning_effort|default('low')"), encoding="utf-8")
# Write 0-tensor vocab.gguf for strata-vision (mtmd vocab_only header) vocab_out = out_dir / "vocab.gguf" if not vocab_out.exists(): rd_v = MR.Reader(gguf_out) rd_v.f.close() vocab_hdr = struct.pack("<4sIQQ", b"GGUF", rd_v.version, 0, len(rd_v.kv)) + b"".join(raw for _, raw in rd_v.kv) vocab_out.write_bytes(vocab_hdr) del rd_v, vocab_hdr
# Optional vision projector (mmproj-Swift-Qwen3.8-Flash-Next-BF16.gguf, 907.54 MB) mmproj_out = out_dir / "mmproj-Swift-Qwen3.8-Flash-Next-BF16.gguf" mmproj_url = "https://huggingface.co/ukisai/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-GGUF/resolve/main/mmproj-Swift-Qwen3.8-Flash-Next-BF16.gguf" if "--no-vision" not in sys.argv and (not mmproj_out.exists() or mmproj_out.stat().st_size < 900_000_000): print("Downloading mmproj-Swift-Qwen3.8-Flash-Next-BF16.gguf (907.5 MB)...", flush=True) mm_sz = head_size(mmproj_url) tmp_mm = mmproj_out.with_name(mmproj_out.name + ".part") fd_m = os.open(str(tmp_mm), os.O_CREAT | os.O_RDWR | os.O_TRUNC, 0o644) try: os.ftruncate(fd_m, mm_sz) with ThreadPoolExecutor(max_workers=16) as ex_m: jb_m = start_blob(ex_m, mmproj_url, 0, mm_sz, chunk_mb=16) os.pwrite(fd_m, collect_blob(jb_m), 0) os.fsync(fd_m) finally: os.close(fd_m) os.replace(tmp_mm, mmproj_out)
vexe = ROOT / "vendor" / "strata" / "build-vision" / "bin" / "strata-vision" has_vision = vexe.exists() and mmproj_out.exists()
# Write strata-k512-iq2xs.json using checkout ROOT cfg_path = out_dir / "strata-k512-iq2xs.json" cfg = { "exe": str(ROOT / "vendor" / "strata" / "build-metal" / "strata_window_cli"),7 unmodified lines "--spec-min-p", "0.35", "--suffix-draft", "3", "--cache-disk", str(out_dir / "cache_disk"), "--cache-disk-size", "16384", ], ] + (["--vision"] if has_vision else []), "cwd": str(ROOT / "vendor" / "strata"), "tokenizer": str(tok_dir), "model_name": "swift-1.5-k512-iq2xs", "log": "/tmp/strata-k512-serve.log",36 unmodified lines "STRATA_METAL_DRAFT_MOCE": "1", }, "reasoning_effort": "low", } if has_vision: cfg["vram_elastic"] = True cfg["vision"] = { "exe": str(vexe), "mmproj": str(mmproj_out), "model": str(vocab_out), "gpu": True, "max_tokens": 1024, "elastic": True, "elastic_idle_s": 60, } cfg_path.write_text(json.dumps(cfg, indent=2) + "\n", encoding="utf-8")
print( f"COMPLETE in {time.monotonic()-t0:.1f}s! "8 unmodified lines207 unmodified linesExpand all2082092102112122132142152162172182192202212222232242252261 unmodified line22822923023122123223323423523628 unmodified lines2652662672682692702712722732742752762772782792802812822832 unmodified lines207 unmodified linesExpand all jinja_p.write_text(jtxt.replace("reasoning_effort|default('xhigh')", "reasoning_effort|default('low')"), encoding="utf-8")
healed_gguf = model_dir / "K256-q2_0-healed-metal.gguf" active_gguf = healed_gguf if healed_gguf.exists() else out_gguf vocab_out = model_dir / "vocab.gguf" if not vocab_out.exists(): rd_v = R.Reader(active_gguf) rd_v.f.close() vocab_out.write_bytes(struct.pack("<4sIQQ", b"GGUF", rd_v.version, 0, len(rd_v.kv)) + b"".join(raw for _, raw in rd_v.kv)) del rd_v vexe = root / "vendor" / "strata" / "build-vision" / "bin" / "strata-vision" mmproj_p = root / "models" / "K512_iq2_xs" / "mmproj-Swift-Qwen3.8-Flash-Next-BF16.gguf" if not mmproj_p.exists(): mmproj_p = model_dir / "mmproj-Swift-Qwen3.8-Flash-Next-BF16.gguf" has_vision = vexe.exists() and mmproj_p.exists() cfg = { "exe": str(root / "vendor" / "strata" / "build-metal" / "strata_window_cli"), "args": [ "--metal-gguf", str(active_gguf),1 unmodified line "--n-expert", "256", "--max-context", "131072", "--cache-disk", str(model_dir / "cache_disk"), "--cache-disk-size", "16384", ], ] + (["--vision"] if has_vision else []), "cwd": str(root / "vendor" / "strata"), "tokenizer": str(tok_dir), "model_name": "swift-1.5-k256-healed" if healed_gguf.exists() else "swift-1.5-k256", "log": "/tmp/strata-k256-serve.log",28 unmodified lines "repeat_penalty": 1.08, }, "reasoning_effort": "low", } if has_vision: cfg["vram_elastic"] = True cfg["vision"] = { "exe": str(vexe), "mmproj": str(mmproj_p), "model": str(vocab_out), "gpu": True, "max_tokens": 1024, "elastic": True, "elastic_idle_s": 60, } (model_dir / "strata-k256.json").write_text(json.dumps(cfg, indent=2) + "\n", encoding="utf-8") return 0
2 unmodified lines12334656789#!/usr/bin/env bashset -euo pipefailexport STRATA_OMIT_IMAGES_WHEN_NO_VISION=1ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"UV_BIN="${UV:-$(command -v uv || echo "$HOME/.local/bin/uv")}"exec "$UV_BIN" run --with regex --with jinja2 python3 \exec "$UV_BIN" run --with regex --with jinja2 --with pillow python3 \ "$ROOT/vendor/strata/serve/server.py" \ --engine strata \ --config "$ROOT/models/K256_reap_l2_sum/strata-k256.json" \ --port "${PORT:-8080}" "$@"12334656789#!/usr/bin/env bashset -euo pipefailexport STRATA_OMIT_IMAGES_WHEN_NO_VISION=1ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"UV_BIN="${UV:-$(command -v uv || echo "$HOME/.local/bin/uv")}"exec "$UV_BIN" run --with regex --with jinja2 python3 \exec "$UV_BIN" run --with regex --with jinja2 --with pillow python3 \ "$ROOT/vendor/strata/serve/server.py" \ --engine strata \ --config "$ROOT/models/K512_iq2_xs/strata-k512-iq2xs.json" \ --port "${PORT:-8080}" "$@"40 unmodified lines41424344454647484950515253545556571 unmodified line40 unmodified lines -DSTRATA_GGML_DIR="$ROOT/vendor/llama.cpp"
ninja -C "$ROOT/vendor/strata/build-metal" strata_window_cli convert_k256_q2_0 heal_k256_q2_0
cmake -S "$ROOT/vendor/strata/tools/vision" -B "$ROOT/vendor/strata/build-vision" -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLAMA_DIR="$ROOT/vendor/llama.cpp" \ -DSTRATA_VISION_CUDA=OFF \ -DSTRATA_PORTABLE=OFF \ -DGGML_METAL=ON
ninja -C "$ROOT/vendor/strata/build-vision" strata-vision
UV_BIN="${UV:-$(command -v uv || echo "$HOME/.local/bin/uv")}"if [ "${SKIP_MODEL_DOWNLOAD:-0}" != "1" ]; then "$UV_BIN" run --with regex python3 "$ROOT/scripts/build_k512_iq2_xs_metal.py" "$UV_BIN" run --with numpy python3 "$ROOT/scripts/build_mtp_q2_0.py"1 unmodified line32 unmodified lines3334353637383940414243892 unmodified linesExpand all9369379389399409419429439449459469479 unmodified lines95795895996096196296396496596696796896929 unmodified lines99910001001100210039911004100599310061007100810091010101110121013101410154 unmodified lines1020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810498 unmodified lines105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371032113811391140114111424 unmodified lines1147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117810501179118011811182118314 unmodified lines11981199120012011073107412021203120412051206120734 unmodified lines12421243124412451117124612471248124912502 unmodified lines125312541255125612571258125912601131126112621263126412651136126612671268126911401270127112721273127413 unmodified lines1288128912901291116212921293129412951296229 unmodified linesExpand all15261527152815291530140015311532153315341535153614061537153815391540154123 unmodified lines156515661567156815691570157115721573374 unmodified linesExpand all32 unmodified linesvoid* window_create();void window_destroy(void* w);bool window_init(void* w, const strata::core::ModelGeometry& g, strata::core::SessionState& ss, int max_t, int64_t max_cells, int64_t* n_vocab, std::string& err);void window_mrope_host_set(const int32_t* host_table);bool prompt_run(const int64_t* tokens, int64_t n, int64_t pos0, const float* const* embd_rows, const std::function<bool(const float*, int64_t, int64_t, std::string&)>& on_chunk, std::string& err);bool prompt_run_rows(const int64_t* tokens, int64_t n, int64_t pos0, const std::function<bool(const float*, int64_t, int64_t, std::string&)>& on_chunk, std::string& err);bool window_run(void* w, int T, const int32_t* tokens, int64_t pos0, bool head, int32_t* out, std::string& err);bool window_commit(void* w, int n_keep, std::string& err);892 unmodified linesExpand all } }; std::vector<int64_t> cached_ids; std::vector<int64_t> cached_pads; std::vector<std::pair<int64_t, uint64_t>> cached_imgs; std::vector<int32_t> mrope_host((size_t) (max_cells + 16) * 3, 0); std::vector<float> img_rows; std::vector<const float*> row_ptr; std::string active_path; bool ram_dirty = false; std::vector<float> lg_buf((size_t) n_vocab); std::mt19937 rng(42);9 unmodified lines if (ram_dirty && cached_ids.size() >= 32) disk_cache.save_current(cached_ids, cached_pads, false, err, &active_path); break; } if (line == "STOP") continue; if (line == "VRAM" || line.rfind("VRAM ", 0) == 0) { std::printf("VRAM expert_slots=4129 expert_cache_mib=6656 expert_cache_full_mib=6656 vram_free_mib=13312\n"); std::fflush(stdout); continue; } if (line.rfind("SAVE ", 0) == 0) { std::string path = line.substr(5); while (!path.empty() && (path.back() == '\r' || path.back() == ' ')) path.pop_back(); const double t0 = now_ms();29 unmodified lines } std::fflush(stdout); continue; } const bool geni = line.rfind("GENI ", 0) == 0; if (line.rfind("GEN ", 0) != 0) continue; if (!geni && line.rfind("GEN ", 0) != 0) continue;
std::istringstream iss(line.substr(4)); std::istringstream iss(line.substr(geni ? 5 : 4)); int max_new = 512; if (!(iss >> max_new)) continue; float temp = 0.0f, top_p = 1.0f, min_p = 0.05f, pen_rep = 1.05f, pen_pres = 0.2f; int top_k = 40; std::vector<uint64_t> req_img_keys; std::string tok_str; while (iss >> tok_str) { auto eq = tok_str.find('='); if (eq != std::string::npos) {4 unmodified lines else if (k == "min_p") min_p = std::stof(v); else if (k == "penalty_repeat") pen_rep = std::stof(v); else if (k == "penalty_present") pen_pres = std::stof(v); else if (k == "seed") rng.seed((uint32_t) std::stoul(v)); else if (k == "img_keys") { req_img_keys.clear(); const char* q = v.c_str(); while (*q != '\0') { char* qe = nullptr; const unsigned long long hk = std::strtoull(q, &qe, 16); if (qe == q) { req_img_keys.clear(); break; } req_img_keys.push_back((uint64_t) hk); q = *qe == ',' ? qe + 1 : qe; if (*qe != ',' && *qe != '\0') { req_img_keys.clear(); break; } } } } else { break; } } std::string emb_path; if (geni) { emb_path = tok_str; tok_str.clear(); iss >> tok_str; } std::vector<int64_t> raw_ids; std::stringstream tss(tok_str); std::string item; while (std::getline(tss, item, ',')) {8 unmodified lines max_new = (int) std::max<int64_t>(1, max_cells - (int64_t) raw_ids.size() - 1); }
const double t_pre0 = now_ms(); std::vector<std::pair<int64_t, uint64_t>> req_imgs; row_ptr.clear(); img_rows.clear(); if (geni) { constexpr int64_t kImagePad = 248056; const int64_t n = (int64_t) raw_ids.size(); row_ptr.assign((size_t) n, nullptr); const int64_t cells = (int64_t) mrope_host.size() / 3; auto put = [&](int64_t c, int64_t t, int64_t h, int64_t w) { mrope_host[(size_t) c * 3] = (int32_t) t; mrope_host[(size_t) c * 3 + 1] = (int32_t) h; mrope_host[(size_t) c * 3 + 2] = (int32_t) w; }; struct Img { int64_t n, nx, ny; size_t off; }; std::vector<Img> imgs; std::string ve; std::FILE* f = std::fopen(emb_path.c_str(), "rb"); if (!f) ve = "cannot open " + emb_path; while (f && ve.empty()) { int32_t hdr[5]; const size_t got = std::fread(hdr, sizeof(int32_t), 5, f); if (got == 0) break; if (got != 5 || hdr[0] != 0x31455653 || hdr[1] < 1 || hdr[2] < 1 || hdr[3] < 1 || (int64_t) hdr[2] * hdr[3] != hdr[1] || hdr[4] != 2560) { ve = "bad embeddings file (expected strata-vision records of width 2560)"; break; } const size_t off = img_rows.size(), cnt = (size_t) hdr[1] * (size_t) hdr[4]; img_rows.resize(off + cnt); if (std::fread(img_rows.data() + off, sizeof(float), cnt, f) != cnt) { ve = "short embeddings file"; break; } imgs.push_back({hdr[1], hdr[2], hdr[3], off}); } if (f) std::fclose(f); int64_t p = 0, i = 0; size_t k = 0; while (ve.empty() && i < n) { if (raw_ids[(size_t) i] != kImagePad) { put(i, p, p, p); ++p; ++i; continue; } if (k >= imgs.size()) { ve = "the prompt has more images than the embeddings file"; break; } const Img& im = imgs[k++]; uint64_t h = DiskCache::fnv1a(&im.n, 3); if (req_img_keys.size() == imgs.size()) { h ^= req_img_keys[k - 1] * 1099511628211ull; } else { h ^= DiskCache::fnv1a(reinterpret_cast<const int64_t*>(img_rows.data() + im.off), (size_t) im.n * (2560 * sizeof(float) / sizeof(int64_t))); } req_imgs.push_back({i, h}); for (int64_t j = 0; j < im.n && ve.empty(); ++j) { if (i + j >= n || raw_ids[(size_t) (i + j)] != kImagePad) { ve = "image has fewer <|image_pad|> tokens than rows"; } } for (int64_t j = 0; j < im.n && ve.empty(); ++j) { const int64_t y = j / im.nx, x = j % im.nx; put(i + j, p, p + y, p + x); row_ptr[(size_t) (i + j)] = img_rows.data() + im.off + (size_t) j * 2560; } i += im.n; p += std::max(im.nx, im.ny); } if (ve.empty() && k != imgs.size()) ve = "the embeddings file has more images than the prompt"; if (ve.empty() && n > 0 && raw_ids[(size_t) (n - 1)] == kImagePad) ve = "the prompt cannot end in an image"; for (int64_t c = n; ve.empty() && c < cells; ++c) put(c, p + (c - n), p + (c - n), p + (c - n)); if (!ve.empty()) { me::window_mrope_host_set(nullptr); std::printf("ERR %s\n", ve.c_str()); std::fflush(stdout); continue; } me::window_mrope_host_set(mrope_host.data()); } else { me::window_mrope_host_set(nullptr); }
std::vector<int64_t> ids = raw_ids; // Carry forward any 0..3 SCR alignment pads from prior turns that still lie in the shared prefix if (!cached_pads.empty() && !cached_ids.empty()) { if (!geni && cached_imgs.empty() && !cached_pads.empty() && !cached_ids.empty()) { std::vector<int64_t> carried_ids; cached_pads = strata::core::scr_carry(raw_ids, cached_ids, cached_pads, kScrPadToken, carried_ids); if ((int64_t) carried_ids.size() + max_new <= max_cells) { ids = std::move(carried_ids);4 unmodified lines int64_t n_pre = std::max<int64_t>(0, (int64_t) ids.size() - 1); size_t lcp = 0; const size_t lim0 = std::min(cached_ids.size(), (size_t) n_pre); while (lcp < lim0 && cached_ids[lcp] == ids[lcp]) ++lcp; if (geni != !cached_imgs.empty()) { // If switching between text and vision (or if image hashes differ inside 0..lcp), clamp lcp before the first differing image size_t img_cap = lim0; const size_t n_im = std::max(cached_imgs.size(), req_imgs.size()); for (size_t ii = 0; ii < n_im; ++ii) { if (ii >= cached_imgs.size() || ii >= req_imgs.size() || cached_imgs[ii] != req_imgs[ii]) { const int64_t p_cut = std::min(ii < cached_imgs.size() ? cached_imgs[ii].first : (int64_t) lim0, ii < req_imgs.size() ? req_imgs[ii].first : (int64_t) lim0); img_cap = (size_t) std::max<int64_t>(0, p_cut); break; } } lcp = std::min(lcp, img_cap); } else if (geni) { for (size_t ii = 0; ii < req_imgs.size(); ++ii) { if (ii >= cached_imgs.size() || cached_imgs[ii] != req_imgs[ii]) { const int64_t p_cut = std::min(ii < cached_imgs.size() ? cached_imgs[ii].first : (int64_t) lcp, req_imgs[ii].first); lcp = std::min(lcp, (size_t) std::max<int64_t>(0, p_cut)); break; } } }
// Estimate how many tokens the live RAM state + ck_ring can reuse before checking the NVMe tier int64_t ram_reused = 0; if (lcp == cached_ids.size() && lcp > 0) { ram_reused = (int64_t) lcp; } else if (cached_ids.size() >= 32 && n_pre >= 32) { } else if (!geni && cached_imgs.empty() && cached_ids.size() >= 32 && n_pre >= 32) { std::vector<int64_t> ck_lens = {(int64_t) cached_ids.size()}; for (const auto& s : ck_ring) { if (!s.ids.empty() && s.ids.size() <= cached_ids.size() && std::equal(s.ids.begin(), s.ids.end(), cached_ids.begin())) {14 unmodified lines } } }
// Check NVMe disk tier if it can beat the live in-RAM reuse by >= 32 tokens if (disk_cache.enabled()) { // Check NVMe disk tier if it can beat the live in-RAM reuse by >= 32 tokens (text conversations only) if (!geni && disk_cache.enabled()) { int64_t disk_reused = 0; const int disk_idx = disk_cache.find_best(raw_ids, max_new, max_cells, kScrPadToken, disk_reused); if (disk_idx >= 0 && disk_reused >= ram_reused + 32) { if (ram_dirty && cached_ids.size() >= 32) {34 unmodified lines ram_dirty = false; } }
const int64_t learn_at = (disk_cache.enabled() && cached_pads.empty()) const int64_t learn_at = (!geni && disk_cache.enabled() && cached_pads.empty()) ? disk_cache.observe_prefix(ids, n_pre) : -1; int64_t reused = 0;
2 unmodified lines std::vector<int32_t> nxt((size_t) k); for (int64_t t = 0; t < k; ++t) nxt[(size_t) t] = (int32_t) ids[(size_t) (p0 + t + 1)]; return me::mtp_prefill(nxt.data(), k, p0, e); }; const float* const* embd_ptr = geni ? row_ptr.data() : nullptr; auto run_prefill_span = [&](int64_t start_p, int64_t end_p) -> bool { if (end_p <= start_p) return true; if (learn_at > start_p && learn_at <= end_p && cached_pads.empty()) { if (!me::prompt_run_rows(ids.data() + start_p, learn_at - start_p, start_p, on_chunk, err)) return false; if (!me::prompt_run(ids.data() + start_p, learn_at - start_p, start_p, embd_ptr, on_chunk, err)) return false; std::vector<int64_t> pfx(ids.begin(), ids.begin() + learn_at); save_ck_at(pfx); disk_cache.save_current(pfx, {}, /*pinned=*/true, err); if (end_p > learn_at) { if (!me::prompt_run_rows(ids.data() + learn_at, end_p - learn_at, learn_at, on_chunk, err)) return false; if (!me::prompt_run(ids.data() + learn_at, end_p - learn_at, learn_at, embd_ptr, on_chunk, err)) return false; } return true; } return me::prompt_run_rows(ids.data() + start_p, end_p - start_p, start_p, on_chunk, err); return me::prompt_run(ids.data() + start_p, end_p - start_p, start_p, embd_ptr, on_chunk, err); };
if (lcp == cached_ids.size() && lcp > 0) { reused = (int64_t) lcp;13 unmodified lines cached_ids.assign(ids.begin(), ids.begin() + n_pre); ram_dirty = true; } else { // 2. Check SCR mid-context fold [A B C] -> [A B' C D] (including stripped <think> blocks) via scr_plan if (cached_ids.size() >= 32 && n_pre >= 32) { if (!geni && cached_imgs.empty() && cached_ids.size() >= 32 && n_pre >= 32) { std::vector<int64_t> ck_lens = {(int64_t) cached_ids.size()}; for (const auto& s : ck_ring) { if (!s.ids.empty() && s.ids.size() <= cached_ids.size() && std::equal(s.ids.begin(), s.ids.end(), cached_ids.begin())) {229 unmodified linesExpand all } cur_tok = next_tok; } } cached_imgs = req_imgs; save_ck_at(cached_ids); if (!geni) save_ck_at(cached_ids); const double decode_ms = now_ms() - t_dec0; if (std::strcmp(finish, "error") != 0) { std::printf("DONE %d %zu %.1f %.1f %s 0 0 %lld\n", gen, ids.size(), prefill_ms, decode_ms, finish, (long long) reused); std::fflush(stdout); if (disk_cache.enabled() && cached_ids.size() >= 32) { if (!geni && disk_cache.enabled() && cached_ids.size() >= 32) { disk_cache.save_current(cached_ids, cached_pads, false, err, &active_path); ram_dirty = false; } }23 unmodified lines else if (a == "--tokens-file" && i + 1 < argc) tokens_file = argv[++i]; else if (a == "--n-expert" && i + 1 < argc) n_expert = std::atoi(argv[++i]); else if ((a == "--ctx" || a == "--max-context") && i + 1 < argc) max_cells = std::atoll(argv[++i]); else if (a == "--serve") serve_mode = true; else if (a == "--vision") { /* enabled automatically via GENI */ } else if (a == "--n-prompt" && i + 1 < argc) n_prompt = std::atoi(argv[++i]); else if (a == "--n-gen" && i + 1 < argc) n_gen = std::atoi(argv[++i]); else if (a == "--suffix-draft" && i + 1 < argc) suffix_draft = std::max(0, std::min(7, std::atoi(argv[++i]))); else if (a == "--mtp-gguf" && i + 1 < argc) mtp_gguf = argv[++i];374 unmodified linesExpand all