[{"name":"models","kind":"directory","size":0,"lastCommitHash":"6d1917060d72cfffc6b54391ef8006b7f51361b9","lastCommitWhen":"2026-10-13T19:13:57-04:00","lastCommitMessage":"bonsai: fused CUDA kernel for DeltaNet conv path (gdn-conv)\n\nOne CUDA kernel for the DeltaNet conv path (concat -> ssm_conv -> silu -> conv-state writes): the concat is never materialized; same arithmetic. Bit-identical; MTP decode +1.5%. GGML_CUDA_SSM_CONV_CONCAT_FUSION=0 turns it off.\n"},{"name":"CMakeLists.txt","kind":"file","size":1924,"lastCommitHash":"5a32f7b66ef6cfb3e60deea26e3454cc6ad3438c","lastCommitWhen":"2026-08-21T19:52:34+02:00","lastCommitMessage":"model: add dots3-note (#27060)\n\n* text: conversion\n\n* init impl\n\n* address review comments\n\n* fix rope\n\n* move to a new llama_kv_cache_dsa_iswa"},{"name":"llama-adapter.cpp","kind":"file","size":18454,"lastCommitHash":"10bf611e533d81f739128304991c5e133c6aebd8","lastCommitWhen":"2026-08-16T09:38:01+03:00","lastCommitMessage":"llama : check LoRA tensor data is within file bounds (#27056)\n\n* llama : check LoRA tensor data is within file bounds\n\n* Update src/llama-adapter.cpp\n\nCo-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>\n\n---------\n\nCo-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>"},{"name":"llama-adapter.h","kind":"file","size":2240,"lastCommitHash":"312cf03328496df9c56fe843952cf02a8964c59a","lastCommitWhen":"2026-03-18T12:03:26+02:00","lastCommitMessage":"llama : re-enable manual LoRA adapter free (#19983)\n\n* Re-enable manual LoRA adapter free\n\n* Remove stale \"all adapters must be loaded before context creation\" stale comments"},{"name":"llama-arch.cpp","kind":"file","size":87417,"lastCommitHash":"279df66449f597f9b91d1976865486ba33e2b984","lastCommitWhen":"2026-09-25T11:22:35-07:00","lastCommitMessage":"spec : add DFlash2 support (local convolution + candidate selector) (#27342) (#27816) (#261)\n\n* spec : add DFlash2 support (local convolution + candidate selector) (#27342)\n\n* support DFlash2\n\n* Add p_min in DFlash2\n\nAssisted-by: Claude Opus 5\n\n* Revert unnecessary changes\n\nAssisted-by: Claude Opus 5\n\n* Revert draft sampling in rejection sampling\n\nAssisted-by: Claude Opus 5\n\n* Refactor code structure\n\nAssisted-by: Claude Opus 5\n\n* Delete embedding scaling\n\nAssisted-by: Claude Opus 5\n\n* Gate output transforms on DFlash2\n\nAssisted-by: Claude Opus 5\n\n* Optimize Dflash 2 cost\n\nAssisted-by: Claude Opus 5\n\n* Avoid using atoi\n\nAssisted-by: Claude Opus 5\n\n* Modify comments\n\nAssisted-by: Claude Opus 5\n\n* Move llama_model_dflash_selector_top_k to llama-ext.h\n\nAssisted-by: Claude Opus 5\n\n* Formatting\n\nAssisted-by: Claude Opus 5\n\n* Apply patch to fix the mrope bug\n\nAssisted-by: Claude Opus 5\n\n* fix ci\n\nAssisted-by: Claude Opus 5\n\n* Fix graph number calculation\n\nAssisted-by: Claude Opus 5\n\n* rename hid and unary\n\nAssisted-by: Claude Opus 5\n\n---------\n\n\n\n\n* revert top-k.cu changes\n\n---------\n\n\n\n(cherry picked from commit b10f9ca58c89ccfc3653ac01e979dd085d582b76)\n\nCo-authored-by: Xuan-Son Nguyen <son@huggingface.co>\nCo-authored-by: Zihan Zhang <tiancaizhangdaxian@sjtu.edu.cn>\nCo-authored-by: Jian Chen <jianchen0311@gmail.com>"},{"name":"llama-arch.h","kind":"file","size":24159,"lastCommitHash":"279df66449f597f9b91d1976865486ba33e2b984","lastCommitWhen":"2026-09-25T11:22:35-07:00","lastCommitMessage":"spec : add DFlash2 support (local convolution + candidate selector) (#27342) (#27816) (#261)\n\n* spec : add DFlash2 support (local convolution + candidate selector) (#27342)\n\n* support DFlash2\n\n* Add p_min in DFlash2\n\nAssisted-by: Claude Opus 5\n\n* Revert unnecessary changes\n\nAssisted-by: Claude Opus 5\n\n* Revert draft sampling in rejection sampling\n\nAssisted-by: Claude Opus 5\n\n* Refactor code structure\n\nAssisted-by: Claude Opus 5\n\n* Delete embedding scaling\n\nAssisted-by: Claude Opus 5\n\n* Gate output transforms on DFlash2\n\nAssisted-by: Claude Opus 5\n\n* Optimize Dflash 2 cost\n\nAssisted-by: Claude Opus 5\n\n* Avoid using atoi\n\nAssisted-by: Claude Opus 5\n\n* Modify comments\n\nAssisted-by: Claude Opus 5\n\n* Move llama_model_dflash_selector_top_k to llama-ext.h\n\nAssisted-by: Claude Opus 5\n\n* Formatting\n\nAssisted-by: Claude Opus 5\n\n* Apply patch to fix the mrope bug\n\nAssisted-by: Claude Opus 5\n\n* fix ci\n\nAssisted-by: Claude Opus 5\n\n* Fix graph number calculation\n\nAssisted-by: Claude Opus 5\n\n* rename hid and unary\n\nAssisted-by: Claude Opus 5\n\n---------\n\n\n\n\n* revert top-k.cu changes\n\n---------\n\n\n\n(cherry picked from commit b10f9ca58c89ccfc3653ac01e979dd085d582b76)\n\nCo-authored-by: Xuan-Son Nguyen <son@huggingface.co>\nCo-authored-by: Zihan Zhang <tiancaizhangdaxian@sjtu.edu.cn>\nCo-authored-by: Jian Chen <jianchen0311@gmail.com>"},{"name":"llama-batch.cpp","kind":"file","size":31148,"lastCommitHash":"904cb4ae461799612c62c826737bb23ddc3ec7fb","lastCommitWhen":"2026-10-13T19:13:57-04:00","lastCommitMessage":"bonsai: suffix cache reuse (SCR) for linear attention hybrids (scr)\n\nSuffix cache reuse: after an edit in the middle of a cached prompt (compaction, a trimmed tool output, a system prompt change), the attention KV of the text after the edit is moved to its new positions (q8_0 keys re-rotated) and the DeltaNet layers continue from the pre-edit state, so only the new text is prefilled. Approximate (KL ~0.017 nats/token after a compaction); per request \"scr\", server default --scr.\n"},{"name":"llama-batch.h","kind":"file","size":6822,"lastCommitHash":"904cb4ae461799612c62c826737bb23ddc3ec7fb","lastCommitWhen":"2026-10-13T19:13:57-04:00","lastCommitMessage":"bonsai: suffix cache reuse (SCR) for linear attention hybrids (scr)\n\nSuffix cache reuse: after an edit in the middle of a cached prompt (compaction, a trimmed tool output, a system prompt change), the attention KV of the text after the edit is moved to its new positions (q8_0 keys re-rotated) and the DeltaNet layers continue from the pre-edit state, so only the new text is prefilled. Approximate (KL ~0.017 nats/token after a compaction); per request \"scr\", server default --scr.\n"},{"name":"llama-chat.cpp","kind":"file","size":41872,"lastCommitHash":"bb771cbd2bfd92cfc99263e75053f5eb9b936d84","lastCommitWhen":"2026-05-28T13:13:33+02:00","lastCommitMessage":"chat : add Granite 4.1 chat template (#23518)"},{"name":"llama-chat.h","kind":"file","size":2275,"lastCommitHash":"bb771cbd2bfd92cfc99263e75053f5eb9b936d84","lastCommitWhen":"2026-05-28T13:13:33+02:00","lastCommitMessage":"chat : add Granite 4.1 chat template (#23518)"},{"name":"llama-context.cpp","kind":"file","size":166328,"lastCommitHash":"904cb4ae461799612c62c826737bb23ddc3ec7fb","lastCommitWhen":"2026-10-13T19:13:57-04:00","lastCommitMessage":"bonsai: suffix cache reuse (SCR) for linear attention hybrids (scr)\n\nSuffix cache reuse: after an edit in the middle of a cached prompt (compaction, a trimmed tool output, a system prompt change), the attention KV of the text after the edit is moved to its new positions (q8_0 keys re-rotated) and the DeltaNet layers continue from the pre-edit state, so only the new text is prefilled. Approximate (KL ~0.017 nats/token after a compaction); per request \"scr\", server default --scr.\n"},{"name":"llama-context.h","kind":"file","size":15239,"lastCommitHash":"904cb4ae461799612c62c826737bb23ddc3ec7fb","lastCommitWhen":"2026-10-13T19:13:57-04:00","lastCommitMessage":"bonsai: suffix cache reuse (SCR) for linear attention hybrids (scr)\n\nSuffix cache reuse: after an edit in the middle of a cached prompt (compaction, a trimmed tool output, a system prompt change), the attention KV of the text after the edit is moved to its new positions (q8_0 keys re-rotated) and the DeltaNet layers continue from the pre-edit state, so only the new text is prefilled. Approximate (KL ~0.017 nats/token after a compaction); per request \"scr\", server default --scr.\n"},{"name":"llama-cparams.cpp","kind":"file","size":100,"lastCommitHash":"c311ac664d68d10781a3e7b9f02d9d9520837d80","lastCommitWhen":"2025-06-15T10:08:58+03:00","lastCommitMessage":"cparams : rename LLAMA_MAX_PARALLEL_SEQUENCES to LLAMA_MAX_SEQ (#14188)\n\nggml-ci"},{"name":"llama-cparams.h","kind":"file","size":2772,"lastCommitHash":"6067dbea986ed1359db98c42ab06eb49426bbf16","lastCommitWhen":"2026-09-17T21:25:55-07:00","lastCommitMessage":"dspark: add corrected draft runtime and shared packed target head\n"},{"name":"llama-ext.h","kind":"file","size":11916,"lastCommitHash":"904cb4ae461799612c62c826737bb23ddc3ec7fb","lastCommitWhen":"2026-10-13T19:13:57-04:00","lastCommitMessage":"bonsai: suffix cache reuse (SCR) for linear attention hybrids (scr)\n\nSuffix cache reuse: after an edit in the middle of a cached prompt (compaction, a trimmed tool output, a system prompt change), the attention KV of the text after the edit is moved to its new positions (q8_0 keys re-rotated) and the DeltaNet layers continue from the pre-edit state, so only the new text is prefilled. Approximate (KL ~0.017 nats/token after a compaction); per request \"scr\", server default --scr.\n"},{"name":"llama-grammar.cpp","kind":"file","size":56224,"lastCommitHash":"cd0fa6051abd04fd7f82ccb42ad378ec43885455","lastCommitWhen":"2026-08-05T07:39:10-05:00","lastCommitMessage":"grammar : degrade max repetition >= 2000 to unbounded (#26613)"},{"name":"llama-grammar.h","kind":"file","size":6961,"lastCommitHash":"cef1d23c5a33156c44a206c1f4bc146f4db729f9","lastCommitWhen":"2026-01-03T16:02:43-06:00","lastCommitMessage":"common/grammar : replace problematic backtracking regex `[\\s\\S]*` (#18342)\n\n* grammar : add support for std::regex_search() with trigger patterns\n\n* common : update hermes2 pro trigger to search instead of match\n\n* common : use regex_search with anchoring for partial matching\n\n* common : adjust regex partial tests to use new pattern\n\n* grammar : check pattern directly instead of adding a type\n\n* common : adjust existing patterns to match new semantics"},{"name":"llama-graph.cpp","kind":"file","size":144772,"lastCommitHash":"c0209b63a39312975f3cbff7670afda41f653cd9","lastCommitWhen":"2026-10-13T19:13:57-04:00","lastCommitMessage":"bonsai: in-place GDN recurrent state reading from cache rows (gdn-rows)\n\nThe DeltaNet layers read each seq's recurrent state straight from its cache row instead of gathering it first (only while every seq's row is its own cell; otherwise the old gather). Bit-identical; MTP decode +1.6-2.3%. GGML_GDN_STATE_GATHER=1 restores the gather.\n"},{"name":"llama-graph.h","kind":"file","size":54846,"lastCommitHash":"c0209b63a39312975f3cbff7670afda41f653cd9","lastCommitWhen":"2026-10-13T19:13:57-04:00","lastCommitMessage":"bonsai: in-place GDN recurrent state reading from cache rows (gdn-rows)\n\nThe DeltaNet layers read each seq's recurrent state straight from its cache row instead of gathering it first (only while every seq's row is its own cell; otherwise the old gather). Bit-identical; MTP decode +1.6-2.3%. GGML_GDN_STATE_GATHER=1 restores the gather.\n"},{"name":"llama-hparams.cpp","kind":"file","size":8175,"lastCommitHash":"7221e24f579efbb5fd2a2ced1fffaef1aff2fe0d","lastCommitWhen":"2026-08-19T16:53:31+02:00","lastCommitMessage":"model : GraniteSWAForCausalLM / GraniteMoeSWAForCausalLM (#25505)\n\n* feat(convert): Add conversion for GraniteSWAForCausalLM\n\nBranch: GraniteSWAForCausalLM\nAI-usage: full (Bob, OpenCode + Qwen3.6-35b)\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* feat(llama): Add granite_swa support\n\nBranch: GraniteSWAForCausalLM\nAI-usage: full (Bob, OpenCode + Qwen3.6-35b)\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* feat(conversion): Add conversion infra for rope_pattern array\n\nNOTE: There is other work also targeting this, so this may be\nremoved depending on merge order.\n\nBranch: GraniteSWAForCausalLM\nAI-usage: full (Bob)\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* fix(conversion): Fix SWA pattern logic and support for non-rope layers\n\nBranch: GraniteSWAForCausalLM\nAI-usage: full (Bob)\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* feat(conversion): Add support for GraniteMoeSWA\n\nBranch: GraniteSWAForCausalLM\nAI-usage: full (Bob)\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* feat: Add llama_hparams::has_rope and arch constants\n\nNOTE: This shadows the work done for Granite Speech\nhttps://github.com/ggml-org/llama.cpp/pull/25107\n\nBranch: GraniteSWAForCausalLM\nAI-usage: full (Bob)\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* feat: Add support for per-layer rope determination\n\nBranch: GraniteSWAForCausalLM\nAI-usage: full (Bob)\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* style: Fix failing flake8 for extra newlines\n\nBranch: GraniteSWAForCausalLM\nAI-usage: none\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* test: Write out SLIDING_WINDOW_PATTERN in llama-model-saver\n\nBranch: GraniteSWAForCausalLM\nAI-usage: full (OpenCode + Qwen3.6-35b)\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* fix(convert): Fix missing registration for GraniteMoeSWAForCausalLM\n\nBranch: GraniteSWAForCausalLM\nAI-usage: none\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* fix: Load MoE params as optional\n\nBranch: GraniteSWAForCausalLM\nAI-usage: draft (OpenCode + Qwen3.6-35b)\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* feat: Handle MoE params in conversion\n\nbranch: GraniteSWAForCausalLM\nAI-usage: full (OpenCode + Qwen3.6-35b)\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* style: Remove unnecessary newline\n\nAI-usage: none\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* fix: Remove unnecessary tensor additions to GRANITE architecture\n\nBranch: GraniteSWAForCausalLM\nAI-usage: none\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* fix: Correctly handle naming for ffn gate inp\n\nBranch: GraniteSWAForCausalLM\nAI-usage: none\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* fix: Always default hparams.rope_pattern to 1s\n\nThis isn't strictly necessary, but it will allow other models to rely on\nhparams.has_rope(il) without needting to prepopulate.\n\nBranch: GraniteSWAForCausalLM\nAI-usage: none\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* feat: Move to has_rope for all granite model architectures\n\nNow that we have a proper hparam for this, it's better to use it and not\nrequire a hacky fallback in the hparam method itself.\n\nBranch: GraniteSWAForCausalLM\nAI-usage: none\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* feat: No hacky rope_finetuned fallback in has_rope\n\nBranch: GraniteSWAForCausalLM\nAI-usage: none\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* fix: Fully remove rope hparam filling in granitemoe\n\nThere are no granitemoe models that use NoPE (it's not actually used in the\nlayer building below), so this was just dead code.\n\nBranch: GraniteSWAForCausalLM\nAI-usage: none\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* fix: Save out rope_pattern in model-saver\n\nBranch: GraniteSWAForCausalLM\nAI-usage: none\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* fix: Set hparams.rope_finetuned for round trip\n\nSince the value is _read_ from rope_finetuned, we need to persist it when\nthe model is saved with the saver.\n\nBranch: GraniteSWAForCausalLM\nAI-usage: none\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* fix: Code review cleanup\n\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\nCo-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>\n\nCo-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>\n\n* refactor: Keep gate/up fused for MoE path\n\nBranch: GraniteSWAForCausalLM\nAI-usage: full (Claude + Sonnet 5)\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* fix: Skip GRANITE_SWA in model saver\n\nhttps://github.com/ggml-org/llama.cpp/pull/25505#discussion_r3773175651\n\nKeeping is_swa_impl in the saver can break other models.\n\nBranch: GraniteSWAForCausalLM\nAI-usage: none\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* add sliding window pattern for model in test\n\n* style: Fix indentation\n\nBranch: GraniteSWAForCausalLM\nAI-usage: none\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* fix: Fix \\r\\n\n\nThanks Claude!\n\nBranch: GraniteSWAForCausalLM\nAI-usage: none\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* feat: Keep shared expert fused\n\nBranch: GraniteSWAForCausalLM\nAI-usage: full (Claude + Sonnet 5)\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\n* style: More indentation fixes\n\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\n\nCo-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>\n\n---------\n\nSigned-off-by: Gabe Goodhart <ghart@us.ibm.com>\nCo-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@huggingface.co>"},{"name":"llama-hparams.h","kind":"file","size":17734,"lastCommitHash":"279df66449f597f9b91d1976865486ba33e2b984","lastCommitWhen":"2026-09-25T11:22:35-07:00","lastCommitMessage":"spec : add DFlash2 support (local convolution + candidate selector) (#27342) (#27816) (#261)\n\n* spec : add DFlash2 support (local convolution + candidate selector) (#27342)\n\n* support DFlash2\n\n* Add p_min in DFlash2\n\nAssisted-by: Claude Opus 5\n\n* Revert unnecessary changes\n\nAssisted-by: Claude Opus 5\n\n* Revert draft sampling in rejection sampling\n\nAssisted-by: Claude Opus 5\n\n* Refactor code structure\n\nAssisted-by: Claude Opus 5\n\n* Delete embedding scaling\n\nAssisted-by: Claude Opus 5\n\n* Gate output transforms on DFlash2\n\nAssisted-by: Claude Opus 5\n\n* Optimize Dflash 2 cost\n\nAssisted-by: Claude Opus 5\n\n* Avoid using atoi\n\nAssisted-by: Claude Opus 5\n\n* Modify comments\n\nAssisted-by: Claude Opus 5\n\n* Move llama_model_dflash_selector_top_k to llama-ext.h\n\nAssisted-by: Claude Opus 5\n\n* Formatting\n\nAssisted-by: Claude Opus 5\n\n* Apply patch to fix the mrope bug\n\nAssisted-by: Claude Opus 5\n\n* fix ci\n\nAssisted-by: Claude Opus 5\n\n* Fix graph number calculation\n\nAssisted-by: Claude Opus 5\n\n* rename hid and unary\n\nAssisted-by: Claude Opus 5\n\n---------\n\n\n\n\n* revert top-k.cu changes\n\n---------\n\n\n\n(cherry picked from commit b10f9ca58c89ccfc3653ac01e979dd085d582b76)\n\nCo-authored-by: Xuan-Son Nguyen <son@huggingface.co>\nCo-authored-by: Zihan Zhang <tiancaizhangdaxian@sjtu.edu.cn>\nCo-authored-by: Jian Chen <jianchen0311@gmail.com>"},{"name":"llama-impl.cpp","kind":"file","size":6131,"lastCommitHash":"58190cc84d846d8575ba26e8486bc29d9fd8ad55","lastCommitWhen":"2026-04-06T01:40:38+02:00","lastCommitMessage":"llama : correct platform-independent loading of BOOL metadata (#21428)\n\n* model-loader : fix GGUF bool array conversion\n\n* model-loader : fix remaining GGUF bool pointer uses"},{"name":"llama-impl.h","kind":"file","size":3076,"lastCommitHash":"90e0f5cfcb6cdb4b7b60a4f81b0a26e542149ad5","lastCommitWhen":"2026-07-08T18:18:09+08:00","lastCommitMessage":"llama: refactor fused ops (#24646)"},{"name":"llama-io.cpp","kind":"file","size":447,"lastCommitHash":"0754b7b6fe6109909786bdaa763111167b7410c8","lastCommitWhen":"2026-05-02T18:03:25+03:00","lastCommitMessage":"server : avoid checkpoint data host copies (#22558)\n\n* server : avoid checkpoint data host copies\n\n* llama : refactor llama_io_read_i"},{"name":"llama-io.h","kind":"file","size":812,"lastCommitHash":"d6e7b033a42b2880496d1e9db1df5527e48a8869","lastCommitWhen":"2026-05-05T06:35:07+03:00","lastCommitMessage":"llama : add option to save memory in device buffers (#22679)\n\n* llama : add option to save memory in device buffers\n\n* tests : extend llama-save-load-state"},{"name":"llama-kv-cache-dsa-iswa.cpp","kind":"file","size":10702,"lastCommitHash":"5a32f7b66ef6cfb3e60deea26e3454cc6ad3438c","lastCommitWhen":"2026-08-21T19:52:34+02:00","lastCommitMessage":"model: add dots3-note (#27060)\n\n* text: conversion\n\n* init impl\n\n* address review comments\n\n* fix rope\n\n* move to a new llama_kv_cache_dsa_iswa"},{"name":"llama-kv-cache-dsa-iswa.h","kind":"file","size":4188,"lastCommitHash":"5a32f7b66ef6cfb3e60deea26e3454cc6ad3438c","lastCommitWhen":"2026-08-21T19:52:34+02:00","lastCommitMessage":"model: add dots3-note (#27060)\n\n* text: conversion\n\n* init impl\n\n* address review comments\n\n* fix rope\n\n* move to a new llama_kv_cache_dsa_iswa"},{"name":"llama-kv-cache-dsa.cpp","kind":"file","size":8340,"lastCommitHash":"563dec81c1c538aac0fad465ea933eb2a621a183","lastCommitWhen":"2026-08-03T14:56:30+02:00","lastCommitMessage":"llama : allocate indexer cache only in \"full\" indexer layers (#26474)\n\nCo-authored-by: Stanisław Szymczyk <sszymczy@gmail.com>"},{"name":"llama-kv-cache-dsa.h","kind":"file","size":4271,"lastCommitHash":"563dec81c1c538aac0fad465ea933eb2a621a183","lastCommitWhen":"2026-08-03T14:56:30+02:00","lastCommitMessage":"llama : allocate indexer cache only in \"full\" indexer layers (#26474)\n\nCo-authored-by: Stanisław Szymczyk <sszymczy@gmail.com>"},{"name":"llama-kv-cache-dsv4.cpp","kind":"file","size":77915,"lastCommitHash":"bf0a29cc16066d313f6a8b52025620287a75cbc4","lastCommitWhen":"2026-08-24T09:20:25+03:00","lastCommitMessage":"Deepseek 4: `-sm tensor` (#26490)\n\n* DSV4: sm tensor\n\n* set coarser granularity for head splits\n\n* fix dspark\n\n* add model saving for dsv4 + allow dflash to return on specific device\n\n* add comment about dsv4 seq_rm\n\n* simplify\n\n* add shared expert delayed allreduce\n\n* remove special test for dsv4"},{"name":"llama-kv-cache-dsv4.h","kind":"file","size":14530,"lastCommitHash":"596a5795bdd6da317ea103fc06c0a71c296e3669","lastCommitWhen":"2026-08-02T20:55:34+08:00","lastCommitMessage":"DeepseekV4 MTP + DSpark (#25784)"},{"name":"llama-kv-cache-iswa.cpp","kind":"file","size":11491,"lastCommitHash":"230ea9d214320c5e79cc8166ed708ac60514c71e","lastCommitWhen":"2026-07-08T15:55:19+08:00","lastCommitMessage":"llama-batch: add n_keep_tail in split_equal for recurrent models (#25278)"},{"name":"llama-kv-cache-iswa.h","kind":"file","size":4842,"lastCommitHash":"8c146a8366304c871efc26057cc90370ccf58dad","lastCommitWhen":"2026-06-29T16:58:51+08:00","lastCommitMessage":"DeepSeek V4  (#24162)\n\n* convert: add dsv4 conversion\n\n* add basic setup\n\n* add llm_graph_input_dsv4\n\n* add save-load state\n\n* add sinkhorn eps - correction by @fairydreaming\n\n* add rope fix\n\n* cleanup dead code\n\n* fix bugs\n\n* support pro model: added by @fairydreaming\n\n* remove redundant V cache\n\n* Chat template\n\n* remove debugging leftovers\n\n* Add mechanism for inlining templates based on architecture\n\n* s/deepseek-v4-flash/deepseek4/g\n\n* s/deepseek-v4-flash/deepseek4/g continued\n\n* enable graph reuse\n\n* enable FA\n\n* fix test llama archs\n\n* rename\n\n* compatibility with antirez ds4 GGUFs\n\n* simplified set_gguf_parameters() by calling super class method, replaced moe.score_func with expert_gating_func.\n\n* reserve worst-case kv-cache\n\n* revert max split inputs\n\n* address review comments\n\n* add padding to enable FA\n\n* pad only the final value of plan.n_kv to 256\n\n* remove built-in cpp chat template\n\n* cont: remove cpp built-in template\n\n* rm outdated test\n\n* replace ggml_view_3d() with ggml_reshape_3d()\n\nCo-authored-by: Georgi Gerganov <ggerganov@gmail.com>\n\n* only support n_seq=1 for now\n\n* remove unused var\n\n* cont: remove unused var\n\n* use scale bias\n\n* use correct ptr for can_reuse\n\n* remove gen-chat-inline-templates.py\n\n* simplify graph reuse\n\n* cont: cleanup\n\n* remove unused inputs\n\n* enable partial checkpointing\n\n* add correct shape for kq_mask + set llama_model_n_swa to 0 for dsv4\n\n* precompute source_idx + add comment about dummy write\n\n* support multi-seq\n\n* remove restored_trim_pos\n\n* use split_equal when possible\n\n* fix indent\n\n* address review comments\n\n* use LLM_KV\n\n* fix ci\n\n---------\n\nCo-authored-by: Piotr Wilkin <piotr.wilkin@syndatis.com>\nCo-authored-by: Stanisław Szymczyk <sszymczy@gmail.com>\nCo-authored-by: Xuan Son Nguyen <son@huggingface.co>\nCo-authored-by: fairydreaming <166155368+fairydreaming@users.noreply.github.com>\nCo-authored-by: Georgi Gerganov <ggerganov@gmail.com>"},{"name":"llama-kv-cache-msa.cpp","kind":"file","size":12540,"lastCommitHash":"67d5978bb1ba490070ddbd2a7c59c7ec5a60b7cc","lastCommitWhen":"2026-08-03T16:30:08+03:00","lastCommitMessage":"model: M3: Move MSA into a new memory implementation (#26338)\n\n* Move MSA logic from llama-kv-cache into llama-kv-cache-msa\n\n* cont : minor\n\n* cont : ws fix\n\n---------\n\nCo-authored-by: Georgi Gerganov <ggerganov@gmail.com>"},{"name":"llama-kv-cache-msa.h","kind":"file","size":5714,"lastCommitHash":"67d5978bb1ba490070ddbd2a7c59c7ec5a60b7cc","lastCommitWhen":"2026-08-03T16:30:08+03:00","lastCommitMessage":"model: M3: Move MSA into a new memory implementation (#26338)\n\n* Move MSA logic from llama-kv-cache into llama-kv-cache-msa\n\n* cont : minor\n\n* cont : ws fix\n\n---------\n\nCo-authored-by: Georgi Gerganov <ggerganov@gmail.com>"},{"name":"llama-kv-cache.cpp","kind":"file","size":100539,"lastCommitHash":"904cb4ae461799612c62c826737bb23ddc3ec7fb","lastCommitWhen":"2026-10-13T19:13:57-04:00","lastCommitMessage":"bonsai: suffix cache reuse (SCR) for linear attention hybrids (scr)\n\nSuffix cache reuse: after an edit in the middle of a cached prompt (compaction, a trimmed tool output, a system prompt change), the attention KV of the text after the edit is moved to its new positions (q8_0 keys re-rotated) and the DeltaNet layers continue from the pre-edit state, so only the new text is prefilled. Approximate (KL ~0.017 nats/token after a compaction); per request \"scr\", server default --scr.\n"},{"name":"llama-kv-cache.h","kind":"file","size":16524,"lastCommitHash":"904cb4ae461799612c62c826737bb23ddc3ec7fb","lastCommitWhen":"2026-10-13T19:13:57-04:00","lastCommitMessage":"bonsai: suffix cache reuse (SCR) for linear attention hybrids (scr)\n\nSuffix cache reuse: after an edit in the middle of a cached prompt (compaction, a trimmed tool output, a system prompt change), the attention KV of the text after the edit is moved to its new positions (q8_0 keys re-rotated) and the DeltaNet layers continue from the pre-edit state, so only the new text is prefilled. Approximate (KL ~0.017 nats/token after a compaction); per request \"scr\", server default --scr.\n"},{"name":"llama-kv-cells.h","kind":"file","size":13800,"lastCommitHash":"904cb4ae461799612c62c826737bb23ddc3ec7fb","lastCommitWhen":"2026-10-13T19:13:57-04:00","lastCommitMessage":"bonsai: suffix cache reuse (SCR) for linear attention hybrids (scr)\n\nSuffix cache reuse: after an edit in the middle of a cached prompt (compaction, a trimmed tool output, a system prompt change), the attention KV of the text after the edit is moved to its new positions (q8_0 keys re-rotated) and the DeltaNet layers continue from the pre-edit state, so only the new text is prefilled. Approximate (KL ~0.017 nats/token after a compaction); per request \"scr\", server default --scr.\n"},{"name":"llama-memory-hybrid-iswa.cpp","kind":"file","size":10166,"lastCommitHash":"230ea9d214320c5e79cc8166ed708ac60514c71e","lastCommitWhen":"2026-07-08T15:55:19+08:00","lastCommitMessage":"llama-batch: add n_keep_tail in split_equal for recurrent models (#25278)"},{"name":"llama-memory-hybrid-iswa.h","kind":"file","size":4565,"lastCommitHash":"255582687b8dd211fdbc582e43ab842491554e94","lastCommitWhen":"2026-05-16T20:06:23+08:00","lastCommitMessage":"llama + spec: MTP Support  (#22673)\n\n* spec: support MTP\n\n* fix batch size\n\n* rename files\n\n* cont : simplify (#7)\n\n* MTP: clean-up (#9)\n\n* MTP: clean-up\n\n* review: use llama_context_type instead of llama_graph_type\n\n* review: remove llama_model_has_mtp\n\n* review: fix convert issues\n\n* convert: fix pycheck\n\n* review: formatting\n\n* use `mtp-` for identifying mtp models\n\n* convert: fix mtp conversion\n\n* mtp -> draft-mtp\n\n* remove unused llama_arch\n\n* add need_embd in speculative\n\n* llama: allow partial seq_rm for GDN models for speculative decoding\n\nCurrently speculative checkpoint needs to restart from a checkpoint\nafter some draft tokens are not accepted, this leads to some wastage in\nrunning the target again. This PR adds the ability to rollback upto\n`draft_max` by storing the GDN intermediates.\n\n* fix pending state\n\n* vulkan: add GDN partial rollback\n\n* meta: extend check to axis 1\n\n* metal: add GDN partial rollback\n\nExtend the gated delta net kernel to store intermediate states for\npartial rollback support on the Metal backend.\n\n- Add K (snapshot slot count) as a function constant\n- Read input state from slot 0 of the 3D state tensor\n- Write intermediate states to different slots during token loop\n- For K=1, maintain backward-compatible single-slot behavior\n\nRef: https://github.com/ggml-org/llama.cpp/commit/8c05923630110223669f069af2000e9cf10c02bc\n\nAssisted-by: llama.cpp:local pi\n\n* delta_net_base: use ggml_pad instead of new_tensor\n\n* review: add need_rs_seq\n\n* review: rename part_bounded to n_rs\n\n* review: deslop comments\n\n* review: rename, add asserts\n\n* server : adjust checkpoint logic (#11)\n\n* server : adjust checkpoint logic\n\n* cont : rm asserts\n\n* server-context: fix early exit\n\n* spec : fix compatibility with n-gram and add TODOs (#13)\n\n* metal : cleanup\n\n* llama : fix faulty bitwise check in recurrent memory\n\n* server : disable RS-based MTP in combination with other spec types\n\n* spec : add TODOs\n\n* cont : fix comment\n\n* cont : update comment\n\n* common : fix logic for ngram + mtp compat\n\n* llama-memory: enable checkpointing with partial rollback\n\n* cont: add test-case for loading into a dirty ctx\n\n* llama-memory-recurrent: clear rs_idx in clear\n\n* download: fix mtp path\n\n* llama-arch: fix enorm op\n\n* docs: update docs\n\n* conversion: fix type annotations\n\n---------\n\nCo-authored-by: Georgi Gerganov <ggerganov@gmail.com>"},{"name":"llama-memory-hybrid.cpp","kind":"file","size":9539,"lastCommitHash":"230ea9d214320c5e79cc8166ed708ac60514c71e","lastCommitWhen":"2026-07-08T15:55:19+08:00","lastCommitMessage":"llama-batch: add n_keep_tail in split_equal for recurrent models (#25278)"},{"name":"llama-memory-hybrid.h","kind":"file","size":4382,"lastCommitHash":"255582687b8dd211fdbc582e43ab842491554e94","lastCommitWhen":"2026-05-16T20:06:23+08:00","lastCommitMessage":"llama + spec: MTP Support  (#22673)\n\n* spec: support MTP\n\n* fix batch size\n\n* rename files\n\n* cont : simplify (#7)\n\n* MTP: clean-up (#9)\n\n* MTP: clean-up\n\n* review: use llama_context_type instead of llama_graph_type\n\n* review: remove llama_model_has_mtp\n\n* review: fix convert issues\n\n* convert: fix pycheck\n\n* review: formatting\n\n* use `mtp-` for identifying mtp models\n\n* convert: fix mtp conversion\n\n* mtp -> draft-mtp\n\n* remove unused llama_arch\n\n* add need_embd in speculative\n\n* llama: allow partial seq_rm for GDN models for speculative decoding\n\nCurrently speculative checkpoint needs to restart from a checkpoint\nafter some draft tokens are not accepted, this leads to some wastage in\nrunning the target again. This PR adds the ability to rollback upto\n`draft_max` by storing the GDN intermediates.\n\n* fix pending state\n\n* vulkan: add GDN partial rollback\n\n* meta: extend check to axis 1\n\n* metal: add GDN partial rollback\n\nExtend the gated delta net kernel to store intermediate states for\npartial rollback support on the Metal backend.\n\n- Add K (snapshot slot count) as a function constant\n- Read input state from slot 0 of the 3D state tensor\n- Write intermediate states to different slots during token loop\n- For K=1, maintain backward-compatible single-slot behavior\n\nRef: https://github.com/ggml-org/llama.cpp/commit/8c05923630110223669f069af2000e9cf10c02bc\n\nAssisted-by: llama.cpp:local pi\n\n* delta_net_base: use ggml_pad instead of new_tensor\n\n* review: add need_rs_seq\n\n* review: rename part_bounded to n_rs\n\n* review: deslop comments\n\n* review: rename, add asserts\n\n* server : adjust checkpoint logic (#11)\n\n* server : adjust checkpoint logic\n\n* cont : rm asserts\n\n* server-context: fix early exit\n\n* spec : fix compatibility with n-gram and add TODOs (#13)\n\n* metal : cleanup\n\n* llama : fix faulty bitwise check in recurrent memory\n\n* server : disable RS-based MTP in combination with other spec types\n\n* spec : add TODOs\n\n* cont : fix comment\n\n* cont : update comment\n\n* common : fix logic for ngram + mtp compat\n\n* llama-memory: enable checkpointing with partial rollback\n\n* cont: add test-case for loading into a dirty ctx\n\n* llama-memory-recurrent: clear rs_idx in clear\n\n* download: fix mtp path\n\n* llama-arch: fix enorm op\n\n* docs: update docs\n\n* conversion: fix type annotations\n\n---------\n\nCo-authored-by: Georgi Gerganov <ggerganov@gmail.com>"},{"name":"llama-memory-recurrent.cpp","kind":"file","size":43056,"lastCommitHash":"c0209b63a39312975f3cbff7670afda41f653cd9","lastCommitWhen":"2026-10-13T19:13:57-04:00","lastCommitMessage":"bonsai: in-place GDN recurrent state reading from cache rows (gdn-rows)\n\nThe DeltaNet layers read each seq's recurrent state straight from its cache row instead of gathering it first (only while every seq's row is its own cell; otherwise the old gather). Bit-identical; MTP decode +1.6-2.3%. GGML_GDN_STATE_GATHER=1 restores the gather.\n"},{"name":"llama-memory-recurrent.h","kind":"file","size":6391,"lastCommitHash":"c0209b63a39312975f3cbff7670afda41f653cd9","lastCommitWhen":"2026-10-13T19:13:57-04:00","lastCommitMessage":"bonsai: in-place GDN recurrent state reading from cache rows (gdn-rows)\n\nThe DeltaNet layers read each seq's recurrent state straight from its cache row instead of gathering it first (only while every seq's row is its own cell; otherwise the old gather). Bit-identical; MTP decode +1.6-2.3%. GGML_GDN_STATE_GATHER=1 restores the gather.\n"},{"name":"llama-memory.cpp","kind":"file","size":1534,"lastCommitHash":"745f11fed09ba2303f3f494efb12286d382608e5","lastCommitWhen":"2025-06-30T18:03:03+03:00","lastCommitMessage":"memory : correctly handle failure in apply() (#14438)\n\nggml-ci"},{"name":"llama-memory.h","kind":"file","size":4669,"lastCommitHash":"04eb4c446d22b63449d5dc41c038987d4d8cc3a6","lastCommitWhen":"2026-06-07T20:50:54+08:00","lastCommitMessage":"llama : add Gemma4 MTP (#23398)"},{"name":"llama-mmap.cpp","kind":"file","size":25152,"lastCommitHash":"5cbfb18075c95437e4ac7fb50e3baf88fe137a87","lastCommitWhen":"2026-04-30T14:17:52-07:00","lastCommitMessage":"Update llama-mmap to use ftello/fseeko (#22497)\n\n* Update llama-mmap to work with 32-bit wasm and >2GB models\n\n* Update to gguf.cpp style"},{"name":"llama-mmap.h","kind":"file","size":1663,"lastCommitHash":"36dafba5c476297261692bfb24c49ec657030c62","lastCommitWhen":"2026-03-25T12:53:16+02:00","lastCommitMessage":"llama: fix llama-model-saver (#20503)\n\n* llama : add fd-based model loading via llama_model_load_from_fd\n\n* llama : address review feedback for fd-based model loading\n\n* llama : use FILE pointer instead of fd in public API\n\n* llama : use FILE pointer consistently, address review feedback\n\n* fixup\n\n* fix tensor names\n\n* fix llama-model-saver\n\n* roundtrip tests\n\n* fixup\n\n* refactor tests\n\n* fix prints\n\n* fix model saving\n\n* fix CI, disable Chameleon\n\n* print seed\n\n---------\n\nCo-authored-by: Siddhesh2377 <siddheshsonar2377@gmail.com>"},{"name":"llama-model-loader.cpp","kind":"file","size":75281,"lastCommitHash":"01fd9521c92e7882a3fa1083bb933e3bd9305bef","lastCommitWhen":"2026-09-04T13:12:38-07:00","lastCommitMessage":"ggml: add PTQ1_0, ternary at group 128 (1.75 bpw, lossless vs PQ2_0) (#148)\n\n* ggml: add PTQ1_0, Prism ternary at group 128 (1.75 bpw)\n\nUpstream TQ1_0 packs ternary weights base-3 at 5 trits per byte, but carries a\nsingle fp16 scale per 256 weights. The ternary checkpoints this targets carry a\nscale per 128, so a 256-wide scale has to discard one of the two group scales it\nstraddles. Measured on a ternary checkpoint of this family: every trit survives and the zero pattern\nis preserved exactly, but the smaller-alpha group of each block is reconstructed\nat the larger alpha, giving 6.43% weight error that passes into activations at\nfull strength because it is multiplicative on the weight rather than noise.\n\nPTQ1_0 keeps TQ1_0's trit packing and our group of 128:\n\n  qs[24] + qh[2] + fp16 d = 28 bytes / 128 weights = 1.7500 bpw\n\nagainst PQ2_0's 34 bytes = 2.1250 bpw, so 17.6% smaller with no loss of\ninformation: PQ2_0 spends two bits on a value carrying log2(3) = 1.585 bits.\n\ntest-quantize-fns reports ptq1_0 identical to pq2_0 on both metrics, absolute\nquantization error 0.008678 and dot product error 0.141111, versus tq1_0's\n0.008681 and 0.141345. A round trip of real folded ternary weights is\nbit-identical to the PQ2_0 reconstruction, so the only error either format\ncarries is the shared fp16 scale rounding.\n\nA distinct type id (143) rather than changing TQ1_0 in place, for the same reason\nPQ2_0 is distinct from upstream's group-64 Q2_0: TQ1_0 is a wire format, and\nreinterpreting its block width would silently misread every existing TQ1_0 gguf.\n\nTQ1_0's qs staging is fixed at 32-then-16 bytes, which cannot cover 24, so the\nstages are generalised to 32/16/8. At TQ1_0's 48-byte qs that reduces to exactly\nits original behaviour.\n\nScope: codec, ggml traits, CPU vec_dot against Q8_0, and the ftype plumbing so\nllama-quantize can target it. The vec_dot decodes a block to element order before\nthe dot, sharing the traversal with dequantize so the two cannot drift; it is\ngeneric C on every arch, with no SIMD path yet. CUDA, Metal and Vulkan kernels,\nand the repack gemv/gemm paths, are not implemented.\n\n* ggml: wire PTQ1_0 into quantize dispatch, ftype map and row validation\n\nThree plumbing gaps the traits entry alone did not cover, each of which failed\nend-to-end quantization in a different way:\n\n  ggml_quantize_chunk   - without a dispatch case the returned byte count did not\n                          match the row size and GGML_ASSERT(result == nrows *\n                          row_size) aborted a worker thread mid-tensor, leaving a\n                          truncated output file behind.\n  ggml_ftype -> wtype   - the ftype could not resolve to a ggml type.\n  validate_row_data     - \"invalid type 143\", so quantization refused the result\n                          it had just produced.\n\nVerified end to end on a 1.7B ternary model with the embedding and output tensors\npinned to f16 in both runs, so only the quantized tensors differ:\n\n  quantized tensors  PQ2_0 357.47 MiB -> PTQ1_0 294.47 MiB, 17.62% smaller\n                     (1.75/2.125 bpw predicts 17.65%)\n  perplexity, 4 chunks at ctx 512, identical to every printed digit:\n                     per chunk 11.3796 / 14.7182 / 14.5014 / 12.6944 for both,\n                     final 12.6944 +/- 1.12596 for both\n\nThe perplexity run goes through the new CPU vec_dot, so that path is exercised\nrather than only the codec.\n\n* metal: PTQ1_0 kernels — correct, but 4.2x slower decode than PQ2_0\n\nAdds the Metal path for PTQ1_0: an element accessor for the base-3 layout, both\ndequantize entry points, a mat-vec kernel, mul_mm and mul_mm_id instantiations\nfor f32 and f16, get_rows, the simdgroup constants and both pipeline-selection\nswitches. Also enumerates PTQ1_0 in test-backend-ops, which otherwise silently\nskipped the new type and reported a clean pass.\n\nCorrect: 1251/1251 MUL_MAT tests pass on Apple M5 Pro. End to end on\na 1.7B ternary model the Metal perplexity is identical to PQ2_0 to every printed\ndigit, 12.6935 +/- 1.12567 for both.\n\nSlow, and the reason is structural. Measured on M5 Pro:\n\n                    PQ2_0            PTQ1_0\n  pp128        4505 tok/s       2590 tok/s   0.57x\n  tg32          182.5 tok/s      43.6 tok/s   0.24x\n\nBase-3 packing makes element order non-positional, so a thread owning 16\ncontiguous weights cannot index bytes directly, and threads owning different trit\nindices reload the same bytes. Hoisting the trit factor out of the inner loop for\nspans below element 80 took decode from 29.8 to 43.6 tok/s, but the remaining gap\nneeds the thread mapping changed so one thread consumes all five trits of a byte\nit loads once, instead of five threads each loading it for one trit.\n\nThis reproduces what the earlier TQ1_0 work already found on other backends:\nbase-3 is ALU-bound rather than bandwidth-bound, and the packing win does not pay\nfor itself at decode without that restructure. The 17.6% file-size reduction is\nreal and lossless; the speed is not there yet.\n\n* metal: PTQ1_0 decode 29.8 -> 165.3 tok/s, now 0.89x of PQ2_0\n\nFour changes, each measured on Apple M5 Pro against a 1.7B ternary model, with\nMUL_MAT staying 1251/1251 and Metal perplexity identical to PQ2_0 throughout:\n\n  1. byte-owning threads         29.8 -> 136.4   a thread now consumes all five\n     trits of a byte it loads once, instead of eight threads each reloading the\n     same qs bytes for one trit index; the block's 26 bytes are read once total\n  2. y staged in registers      136.4 -> 143.5   y was being re-read from device\n     memory once per row; it is now gathered once per block and reused, with the\n     ternary offset folded into a single sumy subtraction\n  3. N_R0 8 -> 4                143.5 -> 163.8   rows-per-thread was the real\n     limiter: 16 rows gives 94 tok/s, 8 gives 144, 4 gives 164, 2 gives 163\n  4. balanced qh                163.8 -> 165.3   qh's 8 elements went to threads\n     0 and 1 only, so those did 19 elements while the rest did 15 and the whole\n     simdgroup waited; one element per thread makes it 16 each and drops yl to 16\n\nTrit extraction is the base-3 remainder recurrence, t = (v*3)>>8 with\nv = (v*3)&0xFF, two integer ops per trit with no table. A 256-entry lookup was\ntried and is not better for decode: it adds a dependent load per byte, 26 extra\nper block against PQ2_0's 32 total loads. It does help the mul_mm path, which\nmust produce 16 contiguous elements and cannot use the byte-owning mapping, so\ndequantize keeps the table.\n\nFinal, PQ2_0 vs PTQ1_0 on M5 Pro:\n\n  pp128   4541 -> 3998 tok/s   0.88x\n  tg32     186 ->  165 tok/s   0.89x\n  size    357 ->  294 MiB      17.6% smaller, and lossless\n\nPrefill's remaining gap is structural: mul_mm's dequantize contract is 16\ncontiguous elements, so five threads still touch the same 16 bytes for different\ntrit indices. Closing it needs mul_mm itself to stage a decoded block in\nthreadgroup memory, which is a larger change than this.\n\n* cuda: PTQ1_0 dequantize and MMVQ paths — NOT COMPILED, no toolchain here\n\nAdds the CUDA surface for PTQ1_0, mirroring PQ2_0: type traits, a device trit\naccessor, the float2 dequantize entry point, the six dequant dispatch sites in\nconvert.cu, get_rows, the MMVQ dot with its VDR constant and three dispatch\nsites, and both supports_op type lists.\n\n**This code has never been compiled or executed.** The machine it was written on\nis an Apple M5 Pro with no CUDA toolchain, so nvcc was unavailable. It must be\nbuilt and run on a CUDA host before anyone relies on it.\n\nWhat *is* verified is the part where a bug would actually hide. The device\naccessor's element mapping is pure integer logic, so it was transcribed verbatim\nto the host and checked against the CPU codec's own traversal over 2,560,000\nelement positions across 20,000 random blocks: zero mismatches. That check is\nkept as tests/test-ptq1_0-element-map.cpp so it can be rerun.\n\nTrit extraction uses the same base-3 remainder recurrence as the Metal path,\nt = (v*3)>>8 with v = (v*3)&0xFF. The MMVQ dot gathers four trits at a time and\npacks them into an int as signed int8 lanes so accumulation still goes through\ndp4a, since base-3 packing leaves no byte-aligned run inside a 32-element chunk.\n\nMMQ is deliberately not implemented. Types without it fall back to dequantize\nplus cuBLAS for prefill, which is correct if not optimal, and writing an MMQ\ntile loader blind against tile layouts I cannot test would be worse than leaving\nthe fallback in place.\n\n* metal: tune PTQ1_0 to N_SG=1, and record what optimising it ruled out\n\nN_R0=4, N_SG=1. Decode lands at 162-166 tok/s across runs against PQ2_0's 186,\nso 0.87-0.89x; the spread between N_SG 1 and 2 is inside run-to-run noise\n(+/- 3.4 on a single measurement).\n\nFive further optimisations were tried and MEASURED, and four of them lost. Kept\nhere so nobody spends the same day twice:\n\n  256-entry trit lookup table       142.1  vs 143.5 recurrence   no decode gain\n                                    (helps mul_mm, which keeps it)\n  independent pow3 extraction       159.8  vs 166.0 recurrence   WORSE\n  contiguous byte ownership         162.0  vs 165.3 scattered    WORSE\n  N_R0 = 3 / 5 / 6                  129.8 / 156.7 / 149.6        all worse than 4\n  N_SG = 8                          161.1                        worse than 1-2\n\nWhat that rules out. The lookup table shows decode is not bound by trit\narithmetic. The pow3 variant shows it is not bound by the recurrence's five-deep\nserial dependency either -- four rows per thread already supply enough\nindependent work, so fewer ops beat more ILP. Contiguous byte runs made it worse\nbecause thread 5 straddles the qs 16/8 chunk boundary, and the resulting per-byte\nbranch cost more than consolidating the loads saved.\n\nWhere the remaining 11% actually lives, per 128 weights:\n\n  PQ2_0   128 x (shift, mask, cvt, fma)                = 512 ops, 34 B read\n  PTQ1_0  128 x (mul, shift, cvt, fma, mask-for-next)  = 640 ops, 28 B read\n          ALU 1.25x, bytes 0.82x\n\nA purely ALU-bound kernel would sit at 0.80x and a purely bandwidth-bound one at\n1.21x. Measured 0.89x sits between them, which is about where this instruction\nand byte mix should land. Extracting a base-3 digit is inherently a multiply\nwhere a 2-bit field is a shift, so this gap is the price of packing at the\n1.585-bit information limit rather than on a 2-bit boundary, not an\nimplementation defect.\n\nThe one lever left with a real mechanism behind it is prefill, not decode. In\ndecode each row has its own weights so there is nothing to share, but mul_mm\nreuses a weight tile across many output columns, and its dequantize contract\nforces 16 contiguous elements, so five threads still touch the same 16 bytes.\nStaging a cooperatively decoded block in threadgroup memory would remove that\nredundancy outright. Prefill is at 0.87x (3946 vs 4492).\n\nPerplexity remains identical to PQ2_0 at 12.6935 +/- 1.12567, MUL_MAT 1251/1251.\n\n* metal, cpu: close PTQ1_0 type-list gaps, add the missing mul_mv_ext r1_5\n\nSelf-review compared every switch that names PQ2_0 against those naming PTQ1_0\nand found the new type absent from ten support enumerations: seven in\nggml-cpu/ops.cpp, including ggml_compute_forward_get_rows_f32, one in\nggml-metal-ops.cpp and two in ggml-metal-device.m. PTQ1_0 is traits-compatible\nwith PQ2_0 there -- same block size, same to_float and from_float shape -- so it\nbelongs in each.\n\nClosing those gaps then exposed a real defect. Adding the type to the Metal\nsupport lists enabled the mul_mv_ext path for wider n, which immediately failed\nwith \"kernel not found in any metal library\" for\nkernel_mul_mv_ext_ptq1_0_f32_r1_5: PQ2_0 instantiates r1_2 through r1_5 and only\nr1_2 through r1_4 had been written, because the grep that enumerated them was\ntruncated by head. Instantiating r1_5 fixes it, and the instantiation set is now\nderived from PQ2_0's rather than transcribed.\n\nWorth noting the earlier 1251/1251 passes were not wrong, they were narrow: with\nthe type missing from the support lists the wider-n path was never selected, so\nthe absent kernel could not be reached. The gap and the defect concealed each\nother.\n\nAfter: MUL_MAT 1251/1251 on M5 Pro, Backend MTL0 OK, Metal perplexity identical\nto PQ2_0 at 12.6935 +/- 1.12567, decode 165 tok/s and prefill 3947.\n\n* tests: verify the CUDA MMVQ dot numerically on the host\n\nThe CUDA path still cannot be compiled here, but its arithmetic can be checked.\nThis stubs the three intrinsics the kernel uses -- __dp4a, __low2float and the\n4-byte int load -- transcribes vec_dot_ptq1_0_q8_1 and the device trit accessor\nverbatim, and compares them against a float reference built by dequantising the\nblock and the q8_1 activations independently.\n\nResult over 5000 random weight blocks with random q8_1 activations:\n\n  exact integer-accumulator checks   20000, mismatches 0\n  error / sum of term magnitudes     2.399e-08\n\nThe integer test is the one that matters: the kernel accumulates in int via dp4a\nbefore scaling, so if the trit extraction, the int8 lane packing or the chunk-to-\nq8_1 pairing were wrong, that accumulator would differ. It does not, on any of\nthe 20000 chunks.\n\nA first pass reported 7.062e-05 \"relative error\" and looked like a failure. That\nwas cancellation: the reference total is a signed sum that can land near zero, so\ndividing by it inflates the ratio. Measured against the sum of term magnitudes\ninstead, which cancellation cannot distort, the error is 2.4e-08, i.e. float32\nrounding of the four per-chunk products.\n\nTogether with tests/test-ptq1_0-element-map.cpp this leaves only compilation\nunproven for the CUDA path, not correctness of its math.\n\n* cuda: move ptq1_0_trit to common.cuh so vecdotq.cuh can see it — BUILDS NOW\n\nFirst compile of the CUDA path, on an RTX 5090 with CUDA 12.8 and\nCMAKE_CUDA_ARCHITECTURES=120, failed with\n\n  vecdotq.cuh(822): error: identifier \"ptq1_0_trit\" is undefined\n\nThe accessor was defined in dequantize.cuh, which vecdotq.cuh does not include.\nPQ2_0's dot never needed a shared helper -- it decodes 2-bit fields inline with\nbyte_perm -- so nothing in the file it was mirrored from exposed the dependency.\nBoth headers include common.cuh, so the helper belongs there.\n\nVerified on the 5090 after the fix:\n\n  test-backend-ops -o MUL_MAT   1283/1283 passed, Backend CUDA0: OK\n                                45 ptq1_0 cases OK, 0 failures\n  host element-map test         2,560,000 positions, 0 mismatches\n  host MMVQ dot test            20,000 integer checks, 0 mismatches\n\nThe two host tests were already green before this, which is the point worth\nrecording: they verified the arithmetic and could not have caught a missing\ndeclaration. Compilation was the only gate that would find this, and the earlier\ncommit saying the CUDA path had never been built was accurate about the risk.\n\n* vulkan: PTQ1_0 support, mirrored from Q1_0 not Q2_0 — UNTESTED as committed\n\nAdds the Vulkan path for PTQ1_0: the block struct and QUANT_K defines, the trit\naccessor plus dequantize/dequantize4/get_dm, the mul_mm load branch, a standalone\ndequant shader, the shader-generator registration, and 16 pipeline sites in\nggml-vulkan.cpp mirrored from Q1_0's.\n\nMirrored from Q1_0 deliberately. Q1_0 is type 41 at group 128 with correct Vulkan\nkernels; Q2_0 in this tree is group 128 in ggml-common.h but its Vulkan side still\ndeclares QUANT_K_Q2_0 64, because upstream's group-64 kernels were merged over a\ngroup-128 type and only some of the GLSL was re-derived. Pattern-matching off Q2_0\nwould have inherited that. Q1_0 also has no MMVQ entry, so this follows the same\ndequantize-based path rather than inventing one.\n\nOne divergence from Q1_0 worth naming: PTQ1_0 puts the scale LAST in the block\n(qs[24], qh[2], then d) where Q1_0 puts d first. The GLSL struct mirrors the C\nfield order exactly. Getting that wrong misindexes every block silently instead of\nfailing, which is the same class of error as the Q2_0 breakage above.\n\nNot wired, and it is a real gap rather than an oversight: on-device quantization\n(cpy_f32_quant, set_rows, cpy_quant_f32). Those need a base-3 ENCODER in GLSL,\nwhich is a separate piece of work and is not needed to run a model -- weights are\nquantized offline by llama-quantize.\n\n**This has not been compiled or run.** There was no Vulkan toolchain on the\nmachine it was written on. It is pushed so it can be built and tested on a GPU\nbox, and it should be treated as unverified until test-backend-ops says otherwise.\n\n* cuda: PTQ1_0 MMVQ takes the whole block — measured, and still 6-9x slower\n\nBenchmarked the CUDA path for the first time. It is correct and slow, and the\nnumbers matter more than the change: us/run on a 4090 at m=4096, k=14336, against\npq2_0 on the same run.\n\n        n=1     n=2     n=3     n=8    n=512\n  q1_0   8.88    9.82   13.05   30.14   225.4\n  pq2_0  9.75   11.05   14.92   31.27   232.4\n  ptq1_0 46.8    64.6    99.1   287.3   525.6\n\nThe per-chunk version asked \"which byte holds element e\" per element, a branch plus\na variable-length recurrence, and iqs differs per lane so the branch diverged. Taking\nthe whole 128-weight block per call (VDR 1 -> 4) makes every lane walk the same 26\nbytes in the same order. That bought about 9% at n=2, 70.6 -> 64.6, and no more.\n\nSo divergence was not the dominant cost, and the comment now says so. What remains is\nload shape: this issues 128 scalar int8 reads of the q8_1 activations, one per element,\nwhere pq2_0 issues eight dp4a calls over 4-byte words.\n\nThe fix is identified but deliberately not guessed at. Below element 80 the element\nstride is 16 and 4 divides 16, so four consecutive elements are four consecutive BYTES\nat the same trit index. Four recurrences packed into one int would feed dp4a against a\n4-byte q8_1 load, matching pq2_0's shape. That needs a build cycle to verify, and this\nsession has already shown twice -- on the CPU vec_dot and on the Metal mapping -- that\npredictions about where GPU time goes are wrong about as often as they are right.\n\nCorrectness held throughout: test-backend-ops -o MUL_MAT 1283/1283 on both an RTX 5090\nand an RTX 4090, Backend CUDA0 OK, 45 ptq1_0 cases OK and 0 bad, before and after.\n\nNote n=512 is the cuBLAS dequantize path rather than MMVQ, and is 2.3x rather than 6-9x,\nwhich is consistent with the problem being in the MMVQ load pattern specifically.\n\n* vulkan: fix three build breaks — shared ptq1_0.glsl, skip coopmat2, drop cm2 pipelines\n\nThe Vulkan commit did not build. Three errors, one root cause each, all found by a\nteammate on an Intel Arc B390 and independently by windows CI:\n\n1. mul_mm_cm2.comp: 'dequantFuncA' undeclared. Adding the type to type_names in\n   vulkan-shaders-gen.cpp auto-enrols it in coopmat2 generation, but\n   dequant_funcs_cm2.glsl has no PTQ1_0 decoder (Q1_0 has six entries). That failed\n   the whole Vulkan shader build, not just this type. Now skipped for ptq1_0, which\n   falls back to the scalar and coopmat1 paths that are actually implemented.\n\n2. mul_mm_funcs.glsl: 'ptq1_0_trit' no matching function. The helper lived in\n   dequant_funcs.glsl; mul_mm.comp includes types/dot_product/mul_mm_id/mul_mm_funcs\n   and never dequant_funcs.glsl. **This is the same bug as the CUDA one fixed an hour\n   earlier, same helper, different backend**: defined in the dequantize chain, called\n   from the matmul chain. Fixed the same way, with a shared header\n   vulkan-shaders/ptq1_0.glsl behind an include guard, included from both. Not\n   duplicated: two copies would have to stay in step with the CPU codec in\n   ggml-quants.c, and a divergence there appears as wrong matmul results, never as a\n   build error.\n\n3. ggml-vulkan.cpp: 'matmul_ptq1_0_f16_cm2_len' undeclared. Registrations for cm2\n   pipelines whose shaders no longer exist once (1) is fixed. Both removed, with a\n   comment recording that the absence is deliberate. (1) and (3) had to be fixed\n   together or one error just trades for the other.\n\nVerified locally this time, which is the other half of the lesson: glslc, molten-vk,\nvulkan-headers and vulkan-loader were already installed on the machine that wrote the\noriginal commit. It was pushed unverified on the false assumption that no toolchain\nwas present, and a local build finds all three in minutes.\n\n  -DGGML_VULKAN=ON            builds clean\n  SPIR-V generated            1991 shaders, 54 of them ptq1_0, incl. coopmat1, no cm2\n  test-backend-ops MUL_MAT    1009/1009, Backend Vulkan0: OK, no regression\n\nThat local run does NOT verify the kernel. MoltenVK reports every ptq1_0 case \"not\nsupported\", and it reports pq2_0 the same way (q1_0 gets 28 of 156), so this is an\nApple/MoltenVK coverage limit rather than a defect in the new type. Kernel correctness\ncomes from the Arc B390 run on the patched tree: 1009/1009 with 78 ptq1_0 cases\nactually executed and 0 failures, on KHR_coopmat.\n\nStill absent by design: MMVQ (no q8_1 pipelines, absent from should_use_mmvq),\ncoopmat2, and on-device quantization, which needs a base-3 encoder in GLSL.\n\n* vulkan: make PTQ1_0 reachable, and guard the coopmat2 path\n\nThe PTQ1_0 pipelines were created but the type was absent from every\nselection and support enumeration, so supports_op returned false, the\nscheduler never offered MUL_MAT/MUL_MAT_ID/GET_ROWS to Vulkan, and the\ngenerated shaders were unreachable. The build looked green while nothing\nexecuted on the backend. Add the type alongside GGML_TYPE_Q1_0 in the\nfive pipeline getters and the two supports_op gates.\n\nAdding it there alone is not safe, because PTQ1_0 is the only type in\nthose lists with no coopmat2 pipelines: dequant_funcs_cm2.glsl carries no\ndecoder for it, so vulkan-shaders-gen skips cm2 generation. The cm2 slots\nare still allocated by vk_matmul_pipeline2's constructor, so the mm getter\nwould return a non-null pipeline whose l/m/s are all null, and the\ncaller's \"mmp == nullptr\" fallback would not fire; the mul_mat_id getter\nwould instead trip GGML_ASSERT(support_fp32acc). Both getters now return\nnullptr when the cm2 slots are empty, which routes the caller to the\ndequant + f16 matmul fallback it already implements. That path needs\nggml_vk_get_to_fp16, which is one of the gates opened above.\n\nThe cpy, SET_ROWS and DUP gates are deliberately left alone:\ncopy_to_quant.comp has no PTQ1_0 block, so quantizing to this type on\ndevice is unimplemented and a refusal there is correct.\n\nVerified on Intel Arc B390 (KHR_coopmat): ptq1_0 goes from 0 OK / 78\ndeclined to 28 OK / 50 declined / 0 failures, matching q1_0 and q2_0 case\nfor case, byte-identical across two runs. The coopmat2 guard is reasoned\nfrom the pipeline-creation structure and is not covered by that run,\nwhich is a coopmat1 device; it has no coopmat2 hardware behind it yet.\n\n* cuda: accelerate PTQ1_0 MMVQ with packed DP4A\n\nAssisted-by: OpenAI Codex"},{"name":"llama-model-loader.h","kind":"file","size":7701,"lastCommitHash":"0596704284bffe8350eef4a9510316258ed6b87a","lastCommitWhen":"2026-08-18T16:22:32+02:00","lastCommitMessage":"quant : Optimise memory usage by evicting weights after processing each layer (#22877)\n\n* Evict weights from memory after processing each layer\n\n* Revert changes\n\n* Move unmap to libllama\n\n* Unmap weights offloaded to backend\n\n* Change member's constness\n\n* Remove unmap weights offloaded to backend"},{"name":"llama-model-saver.cpp","kind":"file","size":25037,"lastCommitHash":"bf0a29cc16066d313f6a8b52025620287a75cbc4","lastCommitWhen":"2026-08-24T09:20:25+03:00","lastCommitMessage":"Deepseek 4: `-sm tensor` (#26490)\n\n* DSV4: sm tensor\n\n* set coarser granularity for head splits\n\n* fix dspark\n\n* add model saving for dsv4 + allow dflash to return on specific device\n\n* add comment about dsv4 seq_rm\n\n* simplify\n\n* add shared expert delayed allreduce\n\n* remove special test for dsv4"},{"name":"llama-model-saver.h","kind":"file","size":1312,"lastCommitHash":"36dafba5c476297261692bfb24c49ec657030c62","lastCommitWhen":"2026-03-25T12:53:16+02:00","lastCommitMessage":"llama: fix llama-model-saver (#20503)\n\n* llama : add fd-based model loading via llama_model_load_from_fd\n\n* llama : address review feedback for fd-based model loading\n\n* llama : use FILE pointer instead of fd in public API\n\n* llama : use FILE pointer consistently, address review feedback\n\n* fixup\n\n* fix tensor names\n\n* fix llama-model-saver\n\n* roundtrip tests\n\n* fixup\n\n* refactor tests\n\n* fix prints\n\n* fix model saving\n\n* fix CI, disable Chameleon\n\n* print seed\n\n---------\n\nCo-authored-by: Siddhesh2377 <siddheshsonar2377@gmail.com>"},{"name":"llama-model.cpp","kind":"file","size":166362,"lastCommitHash":"279df66449f597f9b91d1976865486ba33e2b984","lastCommitWhen":"2026-09-25T11:22:35-07:00","lastCommitMessage":"spec : add DFlash2 support (local convolution + candidate selector) (#27342) (#27816) (#261)\n\n* spec : add DFlash2 support (local convolution + candidate selector) (#27342)\n\n* support DFlash2\n\n* Add p_min in DFlash2\n\nAssisted-by: Claude Opus 5\n\n* Revert unnecessary changes\n\nAssisted-by: Claude Opus 5\n\n* Revert draft sampling in rejection sampling\n\nAssisted-by: Claude Opus 5\n\n* Refactor code structure\n\nAssisted-by: Claude Opus 5\n\n* Delete embedding scaling\n\nAssisted-by: Claude Opus 5\n\n* Gate output transforms on DFlash2\n\nAssisted-by: Claude Opus 5\n\n* Optimize Dflash 2 cost\n\nAssisted-by: Claude Opus 5\n\n* Avoid using atoi\n\nAssisted-by: Claude Opus 5\n\n* Modify comments\n\nAssisted-by: Claude Opus 5\n\n* Move llama_model_dflash_selector_top_k to llama-ext.h\n\nAssisted-by: Claude Opus 5\n\n* Formatting\n\nAssisted-by: Claude Opus 5\n\n* Apply patch to fix the mrope bug\n\nAssisted-by: Claude Opus 5\n\n* fix ci\n\nAssisted-by: Claude Opus 5\n\n* Fix graph number calculation\n\nAssisted-by: Claude Opus 5\n\n* rename hid and unary\n\nAssisted-by: Claude Opus 5\n\n---------\n\n\n\n\n* revert top-k.cu changes\n\n---------\n\n\n\n(cherry picked from commit b10f9ca58c89ccfc3653ac01e979dd085d582b76)\n\nCo-authored-by: Xuan-Son Nguyen <son@huggingface.co>\nCo-authored-by: Zihan Zhang <tiancaizhangdaxian@sjtu.edu.cn>\nCo-authored-by: Jian Chen <jianchen0311@gmail.com>"},{"name":"llama-model.h","kind":"file","size":34017,"lastCommitHash":"279df66449f597f9b91d1976865486ba33e2b984","lastCommitWhen":"2026-09-25T11:22:35-07:00","lastCommitMessage":"spec : add DFlash2 support (local convolution + candidate selector) (#27342) (#27816) (#261)\n\n* spec : add DFlash2 support (local convolution + candidate selector) (#27342)\n\n* support DFlash2\n\n* Add p_min in DFlash2\n\nAssisted-by: Claude Opus 5\n\n* Revert unnecessary changes\n\nAssisted-by: Claude Opus 5\n\n* Revert draft sampling in rejection sampling\n\nAssisted-by: Claude Opus 5\n\n* Refactor code structure\n\nAssisted-by: Claude Opus 5\n\n* Delete embedding scaling\n\nAssisted-by: Claude Opus 5\n\n* Gate output transforms on DFlash2\n\nAssisted-by: Claude Opus 5\n\n* Optimize Dflash 2 cost\n\nAssisted-by: Claude Opus 5\n\n* Avoid using atoi\n\nAssisted-by: Claude Opus 5\n\n* Modify comments\n\nAssisted-by: Claude Opus 5\n\n* Move llama_model_dflash_selector_top_k to llama-ext.h\n\nAssisted-by: Claude Opus 5\n\n* Formatting\n\nAssisted-by: Claude Opus 5\n\n* Apply patch to fix the mrope bug\n\nAssisted-by: Claude Opus 5\n\n* fix ci\n\nAssisted-by: Claude Opus 5\n\n* Fix graph number calculation\n\nAssisted-by: Claude Opus 5\n\n* rename hid and unary\n\nAssisted-by: Claude Opus 5\n\n---------\n\n\n\n\n* revert top-k.cu changes\n\n---------\n\n\n\n(cherry picked from commit b10f9ca58c89ccfc3653ac01e979dd085d582b76)\n\nCo-authored-by: Xuan-Son Nguyen <son@huggingface.co>\nCo-authored-by: Zihan Zhang <tiancaizhangdaxian@sjtu.edu.cn>\nCo-authored-by: Jian Chen <jianchen0311@gmail.com>"},{"name":"llama-quant.cpp","kind":"file","size":61589,"lastCommitHash":"d2c9ddd053aa97ed19f9705cfccd487378d11950","lastCommitWhen":"2026-09-21T20:39:12+03:00","lastCommitMessage":"ggml-cpu: keep PQ2_0 rows that are not multiples of 256 on the Q8_0 activation path\n\nP2 from the review of 10df29881. The Q8_K activation switch was applied to every\nPQ2_0 tensor via the static trait table, but PQ2_0 stores 128 weights per block,\nso 128, 384, 640, ... are all legal row widths and appear in existing models.\nQ8_K only exists in 256-element blocks, so those rows cannot be expressed in it.\n\nGuarding the quantizer, as the previous commit did, does not help: tensors inside\nan already published model never pass through tensor_type_fallback. That guard is\nreverted here - it only cost PQ2_0 coverage on shapes that are now handled.\n\nSelection is now shape-aware. ggml_cpu_vec_dot_type / ggml_cpu_vec_dot return\nQ8_0 + ggml_vec_dot_pq2_0_q8_0 when ne[0] % QK_K != 0 and the trait-table entry\notherwise, and are used at all six sites that previously read the table directly,\nincluding both ggml_graph_plan work-size computations - a mismatch there would\nsize the activation buffer for a different type than the compute path consumes.\nllamafile_sgemm already dispatches on Btype, so those rows also pick up\ntinyBLAS_PQ2_AVX rather than falling back to the scalar dot.\n\nRows that are multiples of 256 - which is every PQ2_0 tensor in a typical model -\ntake an unchanged code path and keep the faster Q8_K kernels.\n\nNew test tests/test-pq2-row-shapes.cpp exercises pre-existing PQ2_0 tensors\nrather than the quantizer: it reference-encodes a PQ2_0 weight tensor the way a\n.gguf holds one, runs ggml_mul_mat through ggml_graph_compute_with_ctx for\nK in {128, 256, 384, 512, 640} x N in {1, 4, 16}, and checks the result against\ndequantized weights times exact float activations.\n\nBefore (Release, x86-64 AVX2 + AVX-VNNI) - the assert the report hit is not\nenabled in this configuration, so the failure is silently wrong output and reads\npast the activation buffer:\n\n  K= 128 N= 1 FAIL rel_rms=1.0000      K= 384 N= 1 FAIL rel_rms=0.4793\n  K= 128 N= 4 FAIL rel_rms=1.0000      K= 384 N= 4 FAIL rel_rms=0.9874\n  K= 128 N=16 FAIL rel_rms=1.0000      K= 384 N=16 FAIL rel_rms=4.3e36\n  K= 256 all  OK   rel_rms<=0.0053     K= 640 all  FAIL up to rel_rms=3.3e36\n  K= 512 all  OK   rel_rms<=0.0068\n\nAfter: 15/15 OK, rel_rms <= 0.0068. The K=256 and K=512 figures are unchanged to\nall digits, so the fast path is untouched.\n\nRegression checks:\n  test-backend-ops -o MUL_MAT    -b CPU  1294/1294\n  test-backend-ops -o MUL_MAT_ID -b CPU  1022/1022\n  test-quantize-fns                      pass\n  llama-bench Bonsai-2-27B-PQ2_0 tg32    3.97 +/- 0.09 t/s (PR range 3.9-4.2)\n\nThat model's 402 PQ2_0 tensors are all 256-multiples, so its graph provably does\nnot change; the branch is evaluated once per matmul node, not per row.\n\naarch64 cross-build (gcc-aarch64-linux-gnu), all defined, none undefined:\n  T ggml_vec_dot_pq2_0_q8_0\n  T ggml_vec_dot_pq2_0_q8_0_generic\n  T ggml_vec_dot_pq2_0_q8_K\n\nCo-Authored-By: Claude Opus 5 <noreply@anthropic.com>\nClaude-Session: https://claude.ai/code/session_01QrdPnP5KL449GsoWBZx52R\n"},{"name":"llama-quant.h","kind":"file","size":13,"lastCommitHash":"f66f5829276650cd83a087ab2cfed1a760183ea1","lastCommitWhen":"2025-01-03T10:18:53+02:00","lastCommitMessage":"llama : refactor `src/llama.cpp` (#10902)\n\n* llama : scatter llama.cpp into multiple modules (wip)\n\n* llama : control-vector -> adapter\n\n* llama : arch\n\n* llama : mmap\n\nggml-ci\n\n* ci : remove BUILD_SHARED_LIBS=OFF\n\nggml-ci\n\n* llama : arch (cont)\n\nggml-ci\n\n* llama : chat\n\nggml-ci\n\n* llama : model\n\nggml-ci\n\n* llama : hparams\n\nggml-ci\n\n* llama : adapter\n\nggml-ci\n\n* examples : fix\n\nggml-ci\n\n* rebase\n\nggml-ci\n\n* minor\n\n* llama : kv cache\n\nggml-ci\n\n* llama : impl\n\nggml-ci\n\n* llama : batch\n\nggml-ci\n\n* cont\n\nggml-ci\n\n* llama : context\n\nggml-ci\n\n* minor\n\n* llama : context (cont)\n\nggml-ci\n\n* llama : model loader\n\nggml-ci\n\n* common : update lora\n\nggml-ci\n\n* llama : quant\n\nggml-ci\n\n* llama : quant (cont)\n\nggml-ci\n\n* minor [no ci]"},{"name":"llama-sampler.cpp","kind":"file","size":151735,"lastCommitHash":"dd1ea524333b1e697489067d7a4c39c60d32beee","lastCommitWhen":"2026-08-10T16:58:56+03:00","lastCommitMessage":"llama : support multi-output backend sampling (#25532)\n\n* Enable backend sampling with token speculation\n\n* Clamp the mask sum before converting it into the sampled index\n\n* Add a numeric context parameter declaring the maximum outputs one sequence\n\n* More fixes\n\n* Don't reuse memory for output views.\n\n* Match dist between CPU and GPU\n\n* Fix CPU and backend sampling mismatches\n\n* Simpify some of the changes\n\n* Fix tests on Vulkan\n\n* More test fixes\n\n* Rebase changes\n\n* Rebase and address review comments\n\n* Address review comments\n\n* Address review comments\n\n* Update src/llama-sampler.cpp\n\nCo-authored-by: Georgi Gerganov <ggerganov@gmail.com>\n\n---------\n\nCo-authored-by: Georgi Gerganov <ggerganov@gmail.com>"},{"name":"llama-sampler.h","kind":"file","size":982,"lastCommitHash":"dd1ea524333b1e697489067d7a4c39c60d32beee","lastCommitWhen":"2026-08-10T16:58:56+03:00","lastCommitMessage":"llama : support multi-output backend sampling (#25532)\n\n* Enable backend sampling with token speculation\n\n* Clamp the mask sum before converting it into the sampled index\n\n* Add a numeric context parameter declaring the maximum outputs one sequence\n\n* More fixes\n\n* Don't reuse memory for output views.\n\n* Match dist between CPU and GPU\n\n* Fix CPU and backend sampling mismatches\n\n* Simpify some of the changes\n\n* Fix tests on Vulkan\n\n* More test fixes\n\n* Rebase changes\n\n* Rebase and address review comments\n\n* Address review comments\n\n* Address review comments\n\n* Update src/llama-sampler.cpp\n\nCo-authored-by: Georgi Gerganov <ggerganov@gmail.com>\n\n---------\n\nCo-authored-by: Georgi Gerganov <ggerganov@gmail.com>"},{"name":"llama-vocab.cpp","kind":"file","size":182463,"lastCommitHash":"d8df12ebc4592b55dcecf97a32953623d031efdc","lastCommitWhen":"2026-08-17T20:13:15+02:00","lastCommitMessage":"vocab : support integer tokenizer scores (#27260)"},{"name":"llama-vocab.h","kind":"file","size":7276,"lastCommitHash":"1f66c3ce1c26c95db3fadb734086c7d9fba23bb9","lastCommitWhen":"2026-07-22T09:54:08+08:00","lastCommitMessage":"Add support for Laguna XS.2 & M.1 (#25165)"},{"name":"llama.cpp","kind":"file","size":22043,"lastCommitHash":"4c1a0af40d88c7fbb3b15c85bf2e8016d1d5b64c","lastCommitWhen":"2026-08-14T15:14:19+03:00","lastCommitMessage":"llama : allow virtual igpu devices (#26953)\n\n* llama : allow virtual igpu devices\n\n* cont : better comment"},{"name":"unicode-data.cpp","kind":"file","size":168374,"lastCommitHash":"458367a90606448a9c0262b276947c9e536086e0","lastCommitWhen":"2024-10-08T13:27:04+02:00","lastCommitMessage":"server : better security control for public deployments (#9776)\n\n* server : more explicit endpoint access settings\r\n\r\n* protect /props endpoint\r\n\r\n* fix tests\r\n\r\n* update server docs\r\n\r\n* fix typo\r\n\r\n* fix tests"},{"name":"unicode-data.h","kind":"file","size":630,"lastCommitHash":"a39ab216aa624308fda7fa84439c6b61dc98b87a","lastCommitWhen":"2024-10-02T15:49:55+02:00","lastCommitMessage":"llama : reduce compile time and binary size (#9712)\n\n* llama : speed up compile time\r\n\r\n* fix build\r\n\r\n* fix build (2)"},{"name":"unicode.cpp","kind":"file","size":55284,"lastCommitHash":"afd439df1f081ce0f6bc76965aa0a0ed762ecc91","lastCommitWhen":"2026-08-18T15:15:22+02:00","lastCommitMessage":"unicode : include '~' in collapsed symbol class (#26972)\n\nThe collapsed \\p{S} class was missing '~', which split \" ~\" into\nseparate pre-tokens and prevented the Ġ~ BPE merge used by DeepSeek V4.\nThis caused re-tokenized prompts to diverge from sampled tokens and\nbroke KV cache reuse.\n\nAssisted-by: Codex"},{"name":"unicode.h","kind":"file","size":3967,"lastCommitHash":"b069b10ab48f25ba119e59d0b8bf35d4f06e093f","lastCommitWhen":"2026-04-03T10:33:03+02:00","lastCommitMessage":"vocab: fix Gemma4 tokenizer (#21343)\n\n* seems to work\n\n* fix case with new line\n\nCo-authored-by: sayap <sokann@gmail.com>\n\n* gemma 4: fix pre tok regex\n\n---------\n\nCo-authored-by: Xuan Son Nguyen <son@huggingface.co>\nCo-authored-by: sayap <sokann@gmail.com>"}]