diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
index ad6565c7..e67050a9 100644
--- a/tools/server/server-task.cpp
+++ b/tools/server/server-task.cpp
@@ -1749,7 +1749,8 @@ constexpr size_t META_OFFSET = 16;
 constexpr size_t META_SIZE   = 512;
 constexpr size_t META_FIXED  = 74;
 
-constexpr size_t FAMILY_TOKENS = 256;
+constexpr size_t FAMILY_TOKENS = 256; // cap on the system block that is hashed
+constexpr size_t FAMILY_USER_TOKENS = 32; // without a system block: the start of the first user turn
 constexpr size_t FAMILY_MIN_N  = 8;
 constexpr size_t FAMILIES_MAX  = 500;
 constexpr size_t CLASSIFY_HEAD = 1500;
@@ -2503,6 +2504,19 @@ void server_prompt_cache::policy_init(const cache_policy & policy, std::function
     this->post_maint  = std::move(post_maint);
 }
 
+// a family is the service that sent the prompt: its system block, which stays the same across calls while the
+// user turn carries the per-call query or page; without one, the opening of the first user turn
+uint64_t server_prompt_cache::family_of(const llama_tokens & tokens) const {
+    auto it = user_marker.empty() ? tokens.end()
+                                  : std::search(tokens.begin(), tokens.end(), user_marker.begin(), user_marker.end());
+    size_t n = it == tokens.end() ? tokens.size() : (size_t) (it - tokens.begin());
+    if (n == 0) {
+        n = user_marker.size() + FAMILY_USER_TOKENS;
+    }
+    n = std::min({ n, tokens.size(), FAMILY_TOKENS });
+    return fnv1a(tokens.data(), n * sizeof(llama_token));
+}
+
 void server_prompt_cache::policy_start() {
     policy_thread = std::thread([this] { policy_loop(); });
 }
@@ -2588,8 +2602,7 @@ void server_prompt_cache::note_saved(server_prompt_cache_state & state) {
     }
 
     if (m.family == 0) {
-        const size_t n = std::min(tokens.size(), FAMILY_TOKENS);
-        m.family = fnv1a(tokens.data(), n * sizeof(llama_token));
+        m.family = family_of(tokens);
     }
     auto & fam = families[m.family];
     if (!m.family_saved) {
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index fcf7ae34..c4c37df9 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -815,6 +815,9 @@ private:
     llama_tokens          user_marker;
     std::function<void()> post_maint;
 
+    // hash of the prompt's system block (or, without one, the start of the first user turn)
+    uint64_t family_of(const llama_tokens & tokens) const;
+
     // server loop only
     std::map<uint64_t, family_stat> families;
     bool families_dirty = false;