diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
index ad6565c7..e67050a9 100644
--- a/tools/server/server-task.cpp
+++ b/tools/server/server-task.cpp
@@ -1749,7 +1749,8 @@ constexpr size_t META_OFFSET = 16;
constexpr size_t META_SIZE = 512;
constexpr size_t META_FIXED = 74;
-constexpr size_t FAMILY_TOKENS = 256;
+constexpr size_t FAMILY_TOKENS = 256; // cap on the system block that is hashed
+constexpr size_t FAMILY_USER_TOKENS = 32; // without a system block: the start of the first user turn
constexpr size_t FAMILY_MIN_N = 8;
constexpr size_t FAMILIES_MAX = 500;
constexpr size_t CLASSIFY_HEAD = 1500;
@@ -2503,6 +2504,19 @@ void server_prompt_cache::policy_init(const cache_policy & policy, std::function
this->post_maint = std::move(post_maint);
}
+// a family is the service that sent the prompt: its system block, which stays the same across calls while the
+// user turn carries the per-call query or page; without one, the opening of the first user turn
+uint64_t server_prompt_cache::family_of(const llama_tokens & tokens) const {
+ auto it = user_marker.empty() ? tokens.end()
+ : std::search(tokens.begin(), tokens.end(), user_marker.begin(), user_marker.end());
+ size_t n = it == tokens.end() ? tokens.size() : (size_t) (it - tokens.begin());
+ if (n == 0) {
+ n = user_marker.size() + FAMILY_USER_TOKENS;
+ }
+ n = std::min({ n, tokens.size(), FAMILY_TOKENS });
+ return fnv1a(tokens.data(), n * sizeof(llama_token));
+}
+
void server_prompt_cache::policy_start() {
policy_thread = std::thread([this] { policy_loop(); });
}
@@ -2588,8 +2602,7 @@ void server_prompt_cache::note_saved(server_prompt_cache_state & state) {
}
if (m.family == 0) {
- const size_t n = std::min(tokens.size(), FAMILY_TOKENS);
- m.family = fnv1a(tokens.data(), n * sizeof(llama_token));
+ m.family = family_of(tokens);
}
auto & fam = families[m.family];
if (!m.family_saved) {
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index fcf7ae34..c4c37df9 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -815,6 +815,9 @@ private:
llama_tokens user_marker;
std::function<void()> post_maint;
+ // hash of the prompt's system block (or, without one, the start of the first user turn)
+ uint64_t family_of(const llama_tokens & tokens) const;
+
// server loop only
std::map<uint64_t, family_stat> families;
bool families_dirty = false;