at least 128 reused tokens and a tenth of what a plain request would prefill
static bool scr_pays_eval(const common_scr_plan & plan, int32_t n_tokens) {
- return !plan.empty() && plan.n_reused() >= std::max(512, (n_tokens - plan.restore) / 10);
+ return !plan.empty() && plan.n_reused() >= std::max(128, (n_tokens - plan.restore) / 10);
}
static arm_stats compare(const std::vector<float> & ref, const std::vector<float> & arm, const std::vector<llama_token> & teacher, int32_t n_vocab) {
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 6016f357..01e7eaa3 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -1840,8 +1840,8 @@ private:
SLT_TRC(slot, " - checking sim = %.3f (%zu/%zu) > %.3f\n", f_sim_cur, lcp_len, task.tokens.size(), slot_prompt_similarity);
- // select the current slot if the criteria match
- if (f_sim_cur > f_sim_best && f_sim_cur > slot_prompt_similarity) {
+ // select the current slot if the criteria match (or if it shares at least 32 tokens, e.g. system prompt)
+ if (f_sim_cur > f_sim_best && (f_sim_cur > slot_prompt_similarity || lcp_len >= 32)) {
f_sim_best = f_sim_cur;
ret = &slot;
@@ -1938,9 +1938,13 @@ private:
ret->prompt_save(*prompt_cache);
- // a slot chosen for SCR keeps its state: the splice reuses it
+ // a slot chosen for SCR keeps its state: the splice reuses it.
+ // also keep the slot state if it already shares tokens with the task, so we don't
+ // discard warm prefix cache in VRAM when the prompt cache has nothing better to load.
if (!scr_keep && !ret->prompt_load(*prompt_cache, task.tokens, skip)) {
- ret->prompt_clear();
+ if (ret->prompt.tokens.get_common_prefix(task.tokens) == 0) {
+ ret->prompt_clear();
+ }
}
prompt_cache->update();
@@ -2697,9 +2701,9 @@ private:
return scr_plan(slot.prompt, scr_n_state(slot), tokens);
}
- // the plan pays when it reuses at least 512 tokens and a tenth of what a plain request would prefill
+ // the plan pays when it reuses at least 128 tokens and a tenth of what a plain request would prefill
static bool scr_pays(const common_scr_plan & plan, int32_t n_tokens) {
- return !plan.empty() && plan.n_reused() >= std::max(512, (n_tokens - plan.restore) / 10);
+ return !plan.empty() && plan.n_reused() >= std::max(128, (n_tokens - plan.restore) / 10);
}
// loads the prompt-cache entry (RAM or disk) a splice can reuse more of than the slot's own state into the slot.
diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
index f9a4b636..fac61f1b 100644
--- a/tools/server/server-task.cpp
+++ b/tools/server/server-task.cpp
@@ -2326,8 +2326,8 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
SRV_TRC(" - prompt with length %7zu, lcp = %7d, f_keep = %.3f, f_sim = %.3f\n", it->prompt.tokens.size(), lcp_cur, f_keep_cur, f_sim_cur);
- // don't trash large prompts
- if (f_keep_cur < 0.25f) {
+ // don't trash large prompts unless the matching prefix is at least 32 tokens
+ if (f_keep_cur < 0.25f && lcp_cur < 32) {
continue;
}
@@ -2352,7 +2352,8 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
const float f_keep_cur = float(lcp_cur) / e.tokens.size();
const float f_sim_cur = float(lcp_cur) / tokens_new.size();
- if (f_keep_cur < 0.25f) {
+ // don't trash large prompts unless the matching prefix is at least 32 tokens
+ if (f_keep_cur < 0.25f && lcp_cur < 32) {
continue;
}