diff --git a/common/arg.cpp b/common/arg.cpp
index e096bfad..be415ee0 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -1724,6 +1724,21 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.cache_disk_mib = value;
}
).set_env("LLAMA_ARG_CACHE_DISK_SIZE").set_examples({LLAMA_EXAMPLE_SERVER}));
+ add_opt(common_arg(
+ {"--cache-embd-ram"}, "N",
+ string_format("RAM for image embeddings by image content in MiB, so an image that has to be prefilled again\n"
+ "skips the vision encoder (default: %d, 0 - off)", params.cache_embd_ram_mib),
+ [](common_params & params, int value) {
+ params.cache_embd_ram_mib = value;
+ }
+ ).set_env("LLAMA_ARG_CACHE_EMBD_RAM").set_examples({LLAMA_EXAMPLE_SERVER}));
+ add_opt(common_arg(
+ {"--cache-embd-disk"}, "N",
+ string_format("disk space for image embeddings under <--cache-disk>/embd in MiB (default: %d, 0 - off)", params.cache_embd_disk_mib),
+ [](common_params & params, int value) {
+ params.cache_embd_disk_mib = value;
+ }
+ ).set_env("LLAMA_ARG_CACHE_EMBD_DISK").set_examples({LLAMA_EXAMPLE_SERVER}));
add_opt(common_arg(
{"--cache-ttl-ephemeral"}, "N",
string_format("seconds a prompt cache entry of class ephemeral lives after its last use (default: %d)", params.cache_ttl_ephemeral),
diff --git a/common/common.h b/common/common.h
index 550bef61..cc89d4b1 100644
--- a/common/common.h
+++ b/common/common.h
@@ -621,6 +621,8 @@ struct common_params {
int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.
std::string cache_disk_path; // directory of the prompt cache's disk tier, empty = off
int32_t cache_disk_mib = 51200; // disk tier size limit in MiB, 0 = no limit
+ int32_t cache_embd_ram_mib = 256; // image embedding cache in RAM, MiB, 0 = off
+ int32_t cache_embd_disk_mib = 4096; // image embedding cache under <cache_disk_path>/embd, MiB, 0 = off
int32_t cache_ttl_ephemeral = 3600; // seconds an unused prompt cache entry of each class is kept
int32_t cache_ttl_default = 3*86400;
int32_t cache_ttl_keep = 30*86400;
diff --git a/tools/server/CMakeLists.txt b/tools/server/CMakeLists.txt
index 280bd9e1..1f6ad1f2 100644
--- a/tools/server/CMakeLists.txt
+++ b/tools/server/CMakeLists.txt
@@ -15,6 +15,8 @@ add_library(${TARGET} STATIC
server-common.h
server-context.cpp
server-context.h
+ server-embd-cache.cpp
+ server-embd-cache.h
server-stream.cpp
server-stream.h
server-tools.cpp
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index f657fdf0..0609fa71 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -1,6 +1,7 @@
#include "server-context.h"
#include "server-chat.h"
#include "server-common.h"
+#include "server-embd-cache.h"
#include "server-http.h"
#include "server-task.h"
#include "server-queue.h"
@@ -205,6 +206,7 @@ struct server_slot {
// multimodal
mtmd_context * mctx = nullptr;
mtmd::batch_ptr mbatch = nullptr;
+ server_embd_cache * embd_cache = nullptr; // may be null
// speculative decoding
common_speculative * spec;
@@ -734,38 +736,51 @@ static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch
const auto & chunk = input_tokens.find_chunk(idx);
int32_t res = 0;
+ // an image's rows are cached by its content id and token grid (the decoder position of its last token)
+ const mtmd_image_tokens * image = mtmd_input_chunk_get_tokens_image(chunk.get());
+ server_embd_cache * cache = image ? slot.embd_cache : nullptr;
+ const mtmd_decoder_pos grid = image ? mtmd_image_tokens_get_decoder_pos(image, 0, mtmd_image_tokens_get_n_tokens(image) - 1) : mtmd_decoder_pos{};
+ const size_t n_floats = (size_t) llama_model_n_embd_inp(llama_get_model(slot.ctx_tgt)) * mtmd_input_chunk_get_n_tokens(chunk.get());
+
+ auto decode = [&](float * embd) -> int32_t {
+ void * cb_data = slot.spec;
+ static auto cb = [](llama_batch batch, void * user_data) {
+ common_speculative * spec = static_cast<common_speculative *>(user_data);
+ if (!common_speculative_process(spec, batch)) {
+ return 1;
+ }
+ return 0;
+ };
+
+ llama_pos new_n_past; // unused for now
+ res = mtmd_helper_decode_image_chunk(
+ mctx,
+ slot.ctx_tgt,
+ chunk.get(),
+ embd,
+ slot.prompt.tokens.pos_next(),
+ slot.id,
+ llama_n_batch(slot.ctx_tgt),
+ &new_n_past,
+ cb,
+ cb_data
+ );
+ if (res != 0) {
+ SLT_ERR(slot, "failed to decode mtmd chunk, idx = %zu, res = %d\n", idx, res);
+ return -1;
+ }
+ n_tokens_out = mtmd_input_chunk_get_n_tokens(chunk.get());
+ return 0; // success
+ };
+
auto try_decode = [&]() -> int32_t {
if (mbatch) {
float * embd = mtmd_batch_get_output_embd(mbatch.get(), chunk.get());
if (embd) {
- void * cb_data = slot.spec;
- static auto cb = [](llama_batch batch, void * user_data) {
- common_speculative * spec = static_cast<common_speculative *>(user_data);
- if (!common_speculative_process(spec, batch)) {
- return 1;
- }
- return 0;
- };
-
- llama_pos new_n_past; // unused for now
- res = mtmd_helper_decode_image_chunk(
- mctx,
- slot.ctx_tgt,
- chunk.get(),
- embd,
- slot.prompt.tokens.pos_next(),
- slot.id,
- llama_n_batch(slot.ctx_tgt),
- &new_n_past,
- cb,
- cb_data
- );
- if (res != 0) {
- SLT_ERR(slot, "failed to decode mtmd chunk, idx = %zu, res = %d\n", idx, res);
- return -1;
+ if (cache) {
+ cache->put(mtmd_input_chunk_get_id(chunk.get()), grid.x + 1, grid.y + 1, embd, n_floats);
}
- n_tokens_out = mtmd_input_chunk_get_n_tokens(chunk.get());
- return 0; // success
+ return decode(embd);
}
}
return 1; // (non-error) need to create & encode batch
@@ -781,6 +796,16 @@ static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch
return res;
}
+ if (cache) {
+ const int64_t t0 = ggml_time_us();
+ std::vector<float> rows;
+ if (cache->get(mtmd_input_chunk_get_id(chunk.get()), grid.x + 1, grid.y + 1, n_floats, rows)) {
+ SLT_INF(slot, "image at idx = %zu (%zu tokens): cached embedding in %.1f ms, encoder skipped\n", idx,
+ mtmd_input_chunk_get_n_tokens(chunk.get()), (ggml_time_us() - t0) / 1e3);
+ return decode(rows.data());
+ }
+ }
+
// otherwise, the batch is either uninitialized or is used up
// we need to create & encode a new batch
mbatch.reset(mtmd_batch_init(mctx));
@@ -805,11 +830,13 @@ static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch
// TODO @ngxson : move this log line to debug when it become more stable
SLT_TRC(slot, "encoding mtmd batch from idx = %zu, n_chunks = %d\n", idx, n_added);
+ const int64_t t_encode = ggml_time_us();
res = mtmd_batch_encode(mbatch.get());
if (res != 0) {
SLT_ERR(slot, "failed to encode mtmd batch for chunk idx = %zu, res = %d\n", idx, res);
return -1;
}
+ SLT_INF(slot, "encoded %d media chunk(s) from idx = %zu in %.1f ms\n", n_added, idx, (ggml_time_us() - t_encode) / 1e3);
return try_decode();
}
@@ -900,6 +927,20 @@ private:
std::unique_ptr<server_prompt_cache> prompt_cache;
+ // created with the multimodal projector, kept across sleep
+ std::unique_ptr<server_embd_cache> embd_cache;
+
+ // what the image embeddings depend on besides the image: the projector file and the encoder settings
+ std::string media_identity() const {
+ std::error_code ec;
+ const auto & path = params_base.mmproj.path;
+ const auto mtime = std::filesystem::last_write_time(path, ec).time_since_epoch().count();
+ const auto size = std::filesystem::file_size(path, ec);
+ const char * bi = getenv("GGML_CUDA_BATCH_INVARIANT");
+ return string_format("%s|%llu|%lld|gpu=%d|min=%d|max=%d|bi=%s", path.c_str(), (unsigned long long) size, (long long) mtime,
+ params_base.mmproj_use_gpu ? 1 : 0, params_base.image_min_tokens, params_base.image_max_tokens, bi ? bi : "");
+ }
+
server_metrics metrics;
// queued prompt stats - llama_decode() is async, so the timing is only valid after a sync
@@ -1169,6 +1210,14 @@ private:
}
SRV_INF("loaded multimodal model, '%s'\n", mmproj_path.c_str());
+ if (!embd_cache && (params_base.cache_embd_ram_mib > 0 || params_base.cache_embd_disk_mib > 0)) {
+ const bool disk = !params_base.cache_disk_path.empty() && params_base.cache_embd_disk_mib > 0;
+ embd_cache = std::make_unique<server_embd_cache>(media_identity(),
+ 1024ull*1024ull*(size_t) std::max(params_base.cache_embd_ram_mib, 0),
+ disk ? params_base.cache_disk_path + "/embd" : "",
+ 1024ull*1024ull*(size_t) std::max(params_base.cache_embd_disk_mib, 0));
+ }
+
if (params_base.ctx_shift) {
params_base.ctx_shift = false;
SRV_WRN("%s\n", "ctx_shift is not supported by multimodal, it will be disabled");
@@ -1292,6 +1341,7 @@ private:
slot.stats.speculative = slot.can_speculate();
slot.mctx = mctx;
+ slot.embd_cache = embd_cache.get();
slot.prompt.tokens.has_mtmd = mctx != nullptr;
SLT_TRC(slot, "new slot, n_ctx = %d\n", slot.n_ctx);
@@ -1393,7 +1443,7 @@ private:
(unsigned long long) llama_model_size(model_tgt), (unsigned long long) llama_model_n_params(model_tgt), (long long) mtime,
ggml_type_name(params_base.cache_type_k), ggml_type_name(params_base.cache_type_v),
ggml_type_name(params_base.speculative.draft.cache_type_k), ggml_type_name(params_base.speculative.draft.cache_type_v));
- prompt_cache->disk_init(params_base.cache_disk_path, params_base.cache_disk_mib, key);
+ prompt_cache->disk_init(params_base.cache_disk_path, params_base.cache_disk_mib, key, mctx ? media_identity() : "");
}
prompt_cache->policy_start();
} else {
@@ -2468,10 +2518,19 @@ private:
return scr_pays(plan, task.tokens.size()) ? plan.n_prefix + plan.n_reused() : (int32_t) prompt.tokens.get_common_prefix(task.tokens);
};
+ // loading an entry consumes it, so one counts only for a splice that pays or, like in prompt_cache.load(),
+ // for a plain prefix of at least a quarter of it: a few tokens of chat template must not take an entry.
// planning against every cached prompt is O(cache size); only prompts that start like this one are planned
const auto value_of = [&](const server_prompt & prompt, int32_t n_state) {
const int32_t lcp = prompt.tokens.get_common_prefix(task.tokens);
- return lcp < 16 ? lcp : value(scr_plan(prompt, n_state, task.tokens), prompt);
+ if (lcp < 16) {
+ return -1;
+ }
+ const common_scr_plan plan = scr_plan(prompt, n_state, task.tokens);
+ if (scr_pays(plan, task.tokens.size())) {
+ return plan.n_prefix + plan.n_reused();
+ }
+ return 4*lcp >= prompt.n_tokens() ? lcp : -1;
};
const int32_t n_own = value(scr_plan(slot, task.tokens), slot.prompt);
@@ -2490,10 +2549,10 @@ private:
}
}
- // disk entries are text prompts without their checkpoints in the index
- const std::string path = prompt_cache->disk_best([&](const llama_tokens & tokens) {
+ // the disk index holds the prompts (images as placeholder chunks) without their checkpoints
+ const std::string path = prompt_cache->disk_best([&](const server_tokens & tokens) {
server_prompt prompt;
- prompt.tokens = server_tokens(tokens, false);
+ prompt.tokens = tokens.clone();
return value_of(prompt, prompt.n_tokens());
}, n_best);
@@ -2506,7 +2565,7 @@ private:
}
// saving can evict RAM entries, so a RAM entry is looked up again by its id
- auto best = !path.empty() ? prompt_cache->disk_promote(path, task.tokens.has_mtmd) :
+ auto best = !path.empty() ? prompt_cache->disk_promote(path) :
std::find_if(prompt_cache->states.begin(), prompt_cache->states.end(), [&](const server_prompt_cache_state & s) { return s.prompt.meta.id == id_best; });
if (best == prompt_cache->states.end()) {
return;
@@ -3077,8 +3136,8 @@ private:
if (task.type == SERVER_TASK_TYPE_CACHE_CLEAR) {
const std::string & scope = task.cache_scope;
const bool all = scope == "all";
- if (!all && scope != "slots" && scope != "ram" && scope != "disk") {
- send_error(task, "scope must be slots, ram, disk or all", ERROR_TYPE_INVALID_REQUEST);
+ if (!all && scope != "slots" && scope != "ram" && scope != "disk" && scope != "embd") {
+ send_error(task, "scope must be slots, ram, disk, embd or all", ERROR_TYPE_INVALID_REQUEST);
break;
}
if (all || scope == "slots") {
@@ -3103,6 +3162,9 @@ private:
if (prompt_cache && (all || scope == "disk")) {
cleared["disk"] = prompt_cache->clear_disk(task.cache_include_pinned);
}
+ if (embd_cache && (all || scope == "embd")) {
+ cleared["embd"] = embd_cache->clear();
+ }
SRV_INF("cache cleared (%s): %s\n", scope.c_str(), cleared.dump().c_str());
}
@@ -3122,6 +3184,9 @@ private:
res->id = task.id;
res->data = prompt_cache ? prompt_cache->info() : json::object();
res->data["slots"] = slots_info;
+ if (embd_cache) {
+ res->data["embd"] = embd_cache->info();
+ }
if (task.type == SERVER_TASK_TYPE_CACHE_CLEAR) {
res->data["cleared"] = cleared;
}
@@ -5243,7 +5308,7 @@ void server_routes::init_routes() {
return res;
};
- // GET /cache: slots and both prompt-cache tiers; POST /cache/clear?scope=slots|ram|disk|all
+ // GET /cache: slots, both prompt-cache tiers and the image embedding cache; POST /cache/clear?scope=slots|ram|disk|embd|all
this->get_cache = [this](const server_http_req & req) {
return handle_cache(req, SERVER_TASK_TYPE_CACHE_GET, "");
};
diff --git a/tools/server/server-embd-cache.cpp b/tools/server/server-embd-cache.cpp
new file mode 100644
index 00000000..5d4035ae
--- /dev/null
+++ b/tools/server/server-embd-cache.cpp
@@ -0,0 +1,275 @@
+#include "server-embd-cache.h"
+
+#include <algorithm>
+#include <cstdio>
+#include <cstring>
+#include <filesystem>
+
+namespace {
+
+constexpr uint32_t EMBD_MAGIC = 0x424d4542; // "BEMB"
+constexpr uint32_t EMBD_VERSION = 1;
+
+// magic, version, identity, nx, ny, n_floats, then the rows and a checksum of their bytes
+constexpr size_t EMBD_HEADER = 4 + 4 + 8 + 4 + 4 + 8;
+
+uint64_t fnv1a(const void * data, size_t n, uint64_t h = 1469598103934665603ull) {
+ const auto * p = static_cast<const uint8_t *>(data);
+ for (size_t i = 0; i < n; ++i) {
+ h = (h ^ p[i]) * 1099511628211ull;
+ }
+ return h;
+}
+
+// fnv-1a over 64-bit words: a corruption check, ~8x faster than bytewise on 20 MiB of rows
+uint64_t checksum(const float * data, size_t n_floats) {
+ const size_t n_bytes = n_floats * sizeof(float);
+ uint64_t h = 1469598103934665603ull;
+ size_t i = 0;
+ for (; i + 8 <= n_bytes; i += 8) {
+ uint64_t w;
+ memcpy(&w, (const uint8_t *) data + i, 8);
+ h = (h ^ w) * 1099511628211ull;
+ }
+ return fnv1a((const uint8_t *) data + i, n_bytes - i, h);
+}
+
+std::string ram_key(const std::string & id, uint32_t nx, uint32_t ny) {
+ return id + "-" + std::to_string(nx) + "x" + std::to_string(ny);
+}
+
+// ids are content hashes (hex); anything else stays out of file names
+bool valid_id(const std::string & id) {
+ return !id.empty() && id.size() <= 128 &&
+ std::all_of(id.begin(), id.end(), [](char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); });
+}
+
+struct embd_file {
+ FILE * f;
+ embd_file(const std::string & path, const char * mode) : f(fopen(path.c_str(), mode)) {}
+ ~embd_file() {
+ if (f) {
+ fclose(f);
+ }
+ }
+};
+
+template <typename T> bool write_val(FILE * f, const T & v) {
+ return fwrite(&v, sizeof(v), 1, f) == 1;
+}
+
+template <typename T> bool read_val(FILE * f, T & v) {
+ return fread(&v, sizeof(v), 1, f) == 1;
+}
+
+bool read_header(FILE * f, uint64_t ident, uint32_t & nx, uint32_t & ny, uint64_t & n_floats) {
+ uint32_t magic, version;
+ uint64_t id;
+ return read_val(f, magic) && read_val(f, version) && read_val(f, id) && read_val(f, nx) && read_val(f, ny) && read_val(f, n_floats) &&
+ magic == EMBD_MAGIC && version == EMBD_VERSION && id == ident && n_floats < (1ull << 32);
+}
+
+} // namespace
+
+server_embd_cache::server_embd_cache(const std::string & identity, size_t ram_limit, const std::string & dir, size_t disk_limit)
+ : ident(fnv1a(identity.data(), identity.size())), ram_limit(ram_limit), dir(dir), disk_limit(disk_limit) {
+ namespace fs = std::filesystem;
+
+ if (this->dir.empty()) {
+ return;
+ }
+ std::error_code ec;
+ fs::create_directories(this->dir, ec);
+ if (ec) {
+ SRV_ERR("embedding cache: cannot create %s: %s, disk tier off\n", this->dir.c_str(), ec.message().c_str());
+ this->dir.clear();
+ return;
+ }
+
+ // oldest first, so the clock replays the files' order of use
+ std::vector<std::pair<fs::file_time_type, std::string>> found;
+ for (const auto & de : fs::directory_iterator(this->dir, ec)) {
+ const auto ext = de.path().extension();
+ if (ext == ".tmp") {
+ fs::remove(de.path(), ec);
+ continue;
+ }
+ if (ext != ".emb") {
+ continue;
+ }
+ uint32_t nx, ny;
+ uint64_t n_floats;
+ bool ok;
+ {
+ embd_file f(de.path().string(), "rb");
+ ok = f.f && read_header(f.f, ident, nx, ny, n_floats) &&
+ de.file_size(ec) == EMBD_HEADER + n_floats * sizeof(float) + sizeof(uint64_t);
+ }
+ if (!ok) {
+ // another mmproj or encoder setting
+ fs::remove(de.path(), ec);
+ continue;
+ }
+ found.push_back({ de.last_write_time(ec), de.path().filename().string() });
+ }
+ std::sort(found.begin(), found.end());
+ for (const auto & [_, name] : found) {
+ const size_t size = fs::file_size(this->dir + "/" + name, ec);
+ disk[name] = { size, ++disk_clock };
+ disk_bytes += size;
+ }
+ disk_evict_locked();
+
+ SRV_INF("embedding cache: RAM %.0f MiB, disk %s with %zu images, %.1f MiB (limit %.0f MiB)\n", ram_limit / 1048576.0,
+ this->dir.c_str(), disk.size(), disk_bytes / 1048576.0, disk_limit / 1048576.0);
+}
+
+bool server_embd_cache::get(const std::string & id, uint32_t nx, uint32_t ny, size_t n_floats, std::vector<float> & out) {
+ std::lock_guard<std::mutex> lock(mutex);
+
+ const std::string key = ram_key(id, nx, ny);
+ const auto it = ram_index.find(key);
+ if (it != ram_index.end() && it->second->data.size() == n_floats) {
+ ram.splice(ram.end(), ram, it->second);
+ out = it->second->data;
+ n_hits_ram++;
+ return true;
+ }
+
+ if (!dir.empty() && valid_id(id) && disk_read_locked(key + ".emb", n_floats, out)) {
+ ram_put_locked(key, std::vector<float>(out));
+ n_hits_disk++;
+ return true;
+ }
+
+ n_misses++;
+ return false;
+}
+
+void server_embd_cache::put(const std::string & id, uint32_t nx, uint32_t ny, const float * data, size_t n_floats) {
+ std::lock_guard<std::mutex> lock(mutex);
+
+ const std::string key = ram_key(id, nx, ny);
+ if (ram_index.count(key) == 0) {
+ ram_put_locked(key, std::vector<float>(data, data + n_floats));
+ }
+ if (!dir.empty() && valid_id(id) && disk.count(key + ".emb") == 0) {
+ disk_write_locked(key + ".emb", nx, ny, data, n_floats);
+ }
+}
+
+void server_embd_cache::ram_put_locked(const std::string & key, std::vector<float> && data) {
+ const size_t size = data.size() * sizeof(float);
+ if (size > ram_limit) {
+ return;
+ }
+ while (!ram.empty() && ram_bytes + size > ram_limit) {
+ ram_bytes -= ram.front().data.size() * sizeof(float);
+ ram_index.erase(ram.front().key);
+ ram.pop_front();
+ }
+ ram.push_back({ key, std::move(data) });
+ ram_index[key] = std::prev(ram.end());
+ ram_bytes += size;
+}
+
+bool server_embd_cache::disk_read_locked(const std::string & name, size_t n_floats, std::vector<float> & out) {
+ const auto it = disk.find(name);
+ if (it == disk.end()) {
+ return false;
+ }
+ const std::string path = dir + "/" + name;
+ bool ok;
+ {
+ embd_file f(path, "rb");
+ uint32_t nx, ny;
+ uint64_t n, sum;
+ ok = f.f && read_header(f.f, ident, nx, ny, n) && n == n_floats;
+ if (ok) {
+ out.resize(n_floats);
+ ok = fread(out.data(), sizeof(float), n_floats, f.f) == n_floats && read_val(f.f, sum) && sum == checksum(out.data(), n_floats);
+ }
+ }
+ if (!ok) {
+ SRV_WRN("embedding cache: cannot read %s, removing it\n", path.c_str());
+ std::error_code ec;
+ std::filesystem::remove(path, ec);
+ disk_bytes -= it->second.size;
+ disk.erase(it);
+ return false;
+ }
+ it->second.used = ++disk_clock;
+ std::error_code ec;
+ std::filesystem::last_write_time(path, std::filesystem::file_time_type::clock::now(), ec);
+ return true;
+}
+
+void server_embd_cache::disk_write_locked(const std::string & name, uint32_t nx, uint32_t ny, const float * data, size_t n_floats) {
+ const size_t size = EMBD_HEADER + n_floats * sizeof(float) + sizeof(uint64_t);
+ if (size > disk_limit) {
+ return;
+ }
+ const std::string path = dir + "/" + name;
+ const std::string tmp = path + ".tmp";
+
+ bool ok;
+ {
+ embd_file f(tmp, "wb");
+ ok = f.f && write_val(f.f, EMBD_MAGIC) && write_val(f.f, EMBD_VERSION) && write_val(f.f, ident) && write_val(f.f, nx) && write_val(f.f, ny) &&
+ write_val(f.f, (uint64_t) n_floats) && fwrite(data, sizeof(float), n_floats, f.f) == n_floats &&
+ write_val(f.f, checksum(data, n_floats)) && fflush(f.f) == 0;
+ }
+ std::error_code ec;
+ if (ok) {
+ std::filesystem::rename(tmp, path, ec);
+ ok = !ec;
+ }
+ if (!ok) {
+ SRV_WRN("embedding cache: cannot write %s\n", path.c_str());
+ std::filesystem::remove(tmp, ec);
+ return;
+ }
+ disk[name] = { size, ++disk_clock };
+ disk_bytes += size;
+ disk_evict_locked();
+}
+
+void server_embd_cache::disk_evict_locked() {
+ std::error_code ec;
+ while (disk_bytes > disk_limit && !disk.empty()) {
+ auto victim = std::min_element(disk.begin(), disk.end(), [](const auto & a, const auto & b) { return a.second.used < b.second.used; });
+ std::filesystem::remove(dir + "/" + victim->first, ec);
+ disk_bytes -= victim->second.size;
+ disk.erase(victim);
+ }
+}
+
+json server_embd_cache::info() const {
+ std::lock_guard<std::mutex> lock(mutex);
+ json res = {
+ { "ram", { { "images", ram.size() }, { "bytes", ram_bytes }, { "limit_bytes", ram_limit } } },
+ { "hits_ram", n_hits_ram },
+ { "hits_disk", n_hits_disk },
+ { "misses", n_misses },
+ };
+ if (!dir.empty()) {
+ res["disk"] = { { "dir", dir }, { "images", disk.size() }, { "bytes", disk_bytes }, { "limit_bytes", disk_limit } };
+ }
+ return res;
+}
+
+size_t server_embd_cache::clear() {
+ std::lock_guard<std::mutex> lock(mutex);
+ size_t n = ram.size();
+ ram.clear();
+ ram_index.clear();
+ ram_bytes = 0;
+ std::error_code ec;
+ for (const auto & [name, _] : disk) {
+ std::filesystem::remove(dir + "/" + name, ec);
+ }
+ n += disk.size();
+ disk.clear();
+ disk_bytes = 0;
+ return n;
+}
diff --git a/tools/server/server-embd-cache.h b/tools/server/server-embd-cache.h
new file mode 100644
index 00000000..f5ce9488
--- /dev/null
+++ b/tools/server/server-embd-cache.h
@@ -0,0 +1,64 @@
+#pragma once
+
+#include "server-common.h"
+
+#include <cstdint>
+#include <list>
+#include <mutex>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+// image embeddings (the vision encoder's output rows) by image content, so an image the prompt cache cannot
+// reuse is decoded without running the encoder again. a RAM LRU tier and an optional disk tier, each under a
+// byte limit. the key folds in everything else the rows depend on (mmproj file, image token limits, encoder
+// device), so a cached row is the one a fresh encode of the same image computes. thread-safe
+struct server_embd_cache {
+ // identity: mmproj file and encoder settings; dir empty = no disk tier
+ server_embd_cache(const std::string & identity, size_t ram_limit, const std::string & dir, size_t disk_limit);
+
+ // the rows of the image with this content id and token grid; false if not cached
+ bool get(const std::string & id, uint32_t nx, uint32_t ny, size_t n_floats, std::vector<float> & out);
+
+ void put(const std::string & id, uint32_t nx, uint32_t ny, const float * data, size_t n_floats);
+
+ json info() const;
+
+ // drops both tiers; returns the number of entries dropped
+ size_t clear();
+
+private:
+ struct ram_entry {
+ std::string key;
+ std::vector<float> data;
+ };
+
+ struct disk_entry {
+ size_t size;
+ int64_t used; // order of use, larger = more recent
+ };
+
+ uint64_t ident;
+ size_t ram_limit;
+ std::string dir;
+ size_t disk_limit;
+
+ mutable std::mutex mutex;
+
+ std::list<ram_entry> ram; // most recently used last
+ std::unordered_map<std::string, std::list<ram_entry>::iterator> ram_index;
+ size_t ram_bytes = 0;
+
+ std::unordered_map<std::string, disk_entry> disk; // by file name
+ size_t disk_bytes = 0;
+ int64_t disk_clock = 0;
+
+ uint64_t n_hits_ram = 0;
+ uint64_t n_hits_disk = 0;
+ uint64_t n_misses = 0;
+
+ void ram_put_locked(const std::string & key, std::vector<float> && data);
+ bool disk_read_locked(const std::string & name, size_t n_floats, std::vector<float> & out);
+ void disk_write_locked(const std::string & name, uint32_t nx, uint32_t ny, const float * data, size_t n_floats);
+ void disk_evict_locked();
+};
diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
index 0a83f4d0..044e2480 100644
--- a/tools/server/server-task.cpp
+++ b/tools/server/server-task.cpp
@@ -1743,8 +1743,12 @@ namespace {
constexpr uint32_t DISK_MAGIC = 0x43564b42; // "BKVC"
constexpr uint32_t DISK_VERSION = 2;
+constexpr uint32_t DISK_VERSION_MEDIA = 3;
-// v2: magic, version, key, META_SIZE bytes of metadata (rewritten in place), then the v1 body
+// v2: magic, version, key, META_SIZE bytes of metadata (rewritten in place), then the v1 body.
+// v3 (prompts with images): as v2, but the token list is the media key followed by server_tokens::serialize()
+// (tokens plus the placeholder media chunks: id, grid, position layout). text prompts stay v2, so a binary that
+// predates v3 deletes only the image files (it reads them as another format), and keeps the rest
constexpr size_t META_OFFSET = 16;
constexpr size_t META_SIZE = 512;
constexpr size_t META_FIXED = 74;
@@ -1993,29 +1997,83 @@ bool get_bytes(FILE * f, std::vector<uint8_t> & v) {
return get(f, v.data(), n);
}
-// magic, version, key, metadata (v2) and the prompt tokens: enough to index a file without reading its state.
-// a v1 file has no metadata; has_meta is false for it
-bool read_header(FILE * f, uint64_t key, llama_tokens & tokens, uint32_t & version, cache_meta & meta) {
+// the number of media chunks in t, or -1 if its chunk map does not cover exactly its media positions
+int64_t media_count(const server_tokens & t) {
+ int64_t n = 0;
+ for (size_t i = 0; i < t.size();) {
+ if (t[i] != LLAMA_TOKEN_NULL) {
+ ++i;
+ continue;
+ }
+ const auto [s, e] = t.media_span(i);
+ if (s != i || e <= i || e > t.size()) {
+ return -1;
+ }
+ for (size_t j = s; j < e; ++j) {
+ if (t[j] != LLAMA_TOKEN_NULL) {
+ return -1;
+ }
+ }
+ n++;
+ i = e;
+ }
+ // chunks at text positions or past the end
+ int64_t n_map = 0;
+ try {
+ t.find_chunk(0);
+ n_map++;
+ } catch (const std::exception &) {
+ }
+ for (auto p = t.find_next_media_chunk(0); p.first; p = t.find_next_media_chunk(p.second)) {
+ n_map++;
+ }
+ return n_map == n ? n : -1;
+}
+
+// magic, version, key, metadata (v2, v3) and the prompt tokens with their media chunks (v3): enough to index a
+// file without reading its state. a v1 file has no metadata. a v3 file is only valid under the same media key
+bool read_header(FILE * f, uint64_t key, uint64_t media, server_tokens & tokens, uint32_t & version, cache_meta & meta) {
uint32_t magic;
uint64_t k;
uint64_t n;
if (!get(f, magic) || !get(f, version) || !get(f, k)) {
return false;
}
- if (magic != DISK_MAGIC || (version != 1 && version != DISK_VERSION) || k != key) {
+ if (magic != DISK_MAGIC || (version != 1 && version != DISK_VERSION && version != DISK_VERSION_MEDIA) || k != key) {
return false;
}
- if (version == DISK_VERSION) {
+ if (version >= DISK_VERSION) {
std::vector<uint8_t> b(META_SIZE);
if (!get(f, b.data(), META_SIZE) || !meta_decode(b, meta)) {
return false;
}
}
+ if (version == DISK_VERSION_MEDIA) {
+ uint64_t m;
+ if (!get(f, m) || m != media || media == 0) {
+ return false;
+ }
+ }
if (!get(f, n) || n > (1ull << 26)) {
return false;
}
- tokens.resize(n);
- return get(f, tokens.data(), n * sizeof(llama_token));
+ llama_tokens raw(n);
+ if (!get(f, raw.data(), n * sizeof(llama_token))) {
+ return false;
+ }
+ if (version != DISK_VERSION_MEDIA) {
+ tokens = server_tokens(raw, media != 0);
+ return true;
+ }
+ if (raw.empty() || raw[0] != LLAMA_TOKEN_NULL) {
+ return false;
+ }
+ try {
+ tokens = server_tokens::deserialize(raw, true);
+ } catch (const std::exception &) {
+ return false;
+ }
+ return media_count(tokens) > 0;
}
bool write_meta(const std::string & path, const cache_meta & meta) {
@@ -2027,15 +2085,6 @@ bool write_meta(const std::string & path, const cache_meta & meta) {
return put(f.f, b.data(), b.size()) && fflush(f.f) == 0;
}
-size_t common_prefix(const llama_tokens & a, const server_tokens & b) {
- const size_t n = std::min(a.size(), b.size());
- size_t i = 0;
- while (i < n && b[i] != LLAMA_TOKEN_NULL && a[i] == b[i]) {
- ++i;
- }
- return i;
-}
-
// the token list, media positions as LLAMA_TOKEN_NULL (get_tokens() refuses a multimodal server)
llama_tokens raw_tokens(const server_tokens & t) {
llama_tokens res(t.size());
@@ -2045,9 +2094,18 @@ llama_tokens raw_tokens(const server_tokens & t) {
return res;
}
-// a is a prefix of b
-bool is_prefix(const llama_tokens & a, const llama_tokens & b) {
- return a.size() <= b.size() && std::equal(a.begin(), a.end(), b.begin());
+bool has_media(const server_tokens & t) {
+ for (size_t i = 0; i < t.size(); ++i) {
+ if (t[i] == LLAMA_TOKEN_NULL) {
+ return true;
+ }
+ }
+ return false;
+}
+
+// a is a prefix of b; a media chunk matches by id and size
+bool is_prefix(const server_tokens & a, const server_tokens & b) {
+ return a.size() <= b.size() && a.get_common_prefix(b) == a.size();
}
size_t count_seq(const llama_tokens & tokens, const llama_tokens & seq) {
@@ -2063,13 +2121,14 @@ size_t count_seq(const llama_tokens & tokens, const llama_tokens & seq) {
return n;
}
-json entry_json(const cache_meta & m, size_t n_tokens, size_t bytes, int64_t now) {
+json entry_json(const cache_meta & m, size_t n_tokens, size_t n_images, size_t bytes, int64_t now) {
return {
{ "id", hex_id(m.id) },
{ "class", cache_class_name(m.cls) },
{ "source", cache_source_name(m.source) },
{ "classified", m.classified() },
{ "n_tokens", n_tokens },
+ { "images", n_images },
{ "bytes", bytes },
{ "hits", m.hits },
{ "age", now - m.created },
@@ -2089,7 +2148,7 @@ struct tier_report {
std::map<std::string, std::pair<size_t, size_t>> by_class;
std::vector<std::pair<int64_t, json>> entries;
- void add(const cache_meta & m, size_t n_tokens, size_t n_bytes, int64_t now) {
+ void add(const cache_meta & m, size_t n_tokens, size_t n_images, size_t n_bytes, int64_t now) {
prompts++;
tokens += n_tokens;
bytes += n_bytes;
@@ -2099,7 +2158,7 @@ struct tier_report {
auto & c = by_class[cache_class_name(m.cls)];
c.first++;
c.second += n_bytes;
- entries.push_back({ m.last_used, entry_json(m, n_tokens, n_bytes, now) });
+ entries.push_back({ m.last_used, entry_json(m, n_tokens, n_images, n_bytes, now) });
}
void to_json(json & out) {
@@ -2288,7 +2347,7 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
expire_disk_locked();
for (const auto & e : disk) {
- const int lcp_cur = common_prefix(e.tokens, tokens_new);
+ const int lcp_cur = e.tokens.get_common_prefix(tokens_new);
const float f_keep_cur = float(lcp_cur) / e.tokens.size();
const float f_sim_cur = float(lcp_cur) / tokens_new.size();
@@ -2309,7 +2368,7 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
bool from_disk = false;
if (!disk_best.empty()) {
- const auto it = disk_promote(disk_best, tokens_new.has_mtmd);
+ const auto it = disk_promote(disk_best);
if (it != states.end()) {
SRV_INF("prompt cache: f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best);
it_best = it;
@@ -2331,12 +2390,12 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok
return true;
}
-std::list<server_prompt_cache_state>::iterator server_prompt_cache::disk_promote(const std::string & path, bool has_mtmd) {
+std::list<server_prompt_cache_state>::iterator server_prompt_cache::disk_promote(const std::string & path) {
namespace fs = std::filesystem;
const int64_t t_start = ggml_time_us();
server_prompt_cache_state state;
- const bool ok = disk_read(path, has_mtmd, state);
+ const bool ok = disk_read(path, state);
std::lock_guard<std::mutex> lock(disk_mutex);
std::error_code ec;
@@ -2350,7 +2409,8 @@ std::list<server_prompt_cache_state>::iterator server_prompt_cache::disk_promote
return states.end();
}
- SRV_INF("prompt cache: read %zu tokens from disk in %.2f s\n", state.prompt.tokens.size(), (ggml_time_us() - t_start) / 1e6);
+ SRV_INF("prompt cache: read %zu tokens, %lld images from disk in %.2f s\n", state.prompt.tokens.size(),
+ (long long) std::max<int64_t>(0, media_count(state.prompt.tokens)), (ggml_time_us() - t_start) / 1e6);
if (it != disk.end()) {
// the file stays: it records the use, and is most recently used, here and after a restart
it->meta.last_used = unix_now();
@@ -2358,7 +2418,7 @@ std::list<server_prompt_cache_state>::iterator server_prompt_cache::disk_promote
set_expiry(it->meta);
state.prompt.meta = it->meta;
state.prompt.meta.hits--; // the slot counts this use when the request continues the prompt
- if (it->version == DISK_VERSION) {
+ if (it->version >= DISK_VERSION) {
write_meta(it->path, it->meta);
}
disk.splice(disk.end(), disk, it);
@@ -2369,7 +2429,7 @@ std::list<server_prompt_cache_state>::iterator server_prompt_cache::disk_promote
return std::prev(states.end());
}
-std::string server_prompt_cache::disk_best(const std::function<int32_t(const llama_tokens &)> & score, int32_t & n_best) {
+std::string server_prompt_cache::disk_best(const std::function<int32_t(const server_tokens &)> & score, int32_t & n_best) {
std::string best;
std::lock_guard<std::mutex> lock(disk_mutex);
if (!disk_thread.joinable()) {
@@ -3002,12 +3062,13 @@ server_prompt_cache::~server_prompt_cache() {
}
}
-void server_prompt_cache::disk_init(const std::string & dir, int32_t limit_mib, const std::string & key) {
+void server_prompt_cache::disk_init(const std::string & dir, int32_t limit_mib, const std::string & key, const std::string & media) {
namespace fs = std::filesystem;
disk_dir = dir;
disk_limit = 1024ull*1024ull*(size_t) std::max(limit_mib, 0);
disk_key = fnv1a(key.data(), key.size());
+ disk_media = media.empty() ? 0 : std::max<uint64_t>(1, fnv1a(media.data(), media.size()));
std::error_code ec;
fs::create_directories(dir, ec);
@@ -3029,16 +3090,16 @@ void server_prompt_cache::disk_init(const std::string & dir, int32_t limit_mib,
if (ext != ".kvc") {
continue;
}
- llama_tokens tokens;
+ server_tokens tokens;
uint32_t version = 0;
cache_meta meta;
bool ok;
{
disk_file f(de.path().string(), "rb");
- ok = f.f && read_header(f.f, disk_key, tokens, version, meta);
+ ok = f.f && read_header(f.f, disk_key, disk_media, tokens, version, meta);
}
if (!ok) {
- // another model, cache layout or format
+ // another model, cache layout, format or multimodal projector
fs::remove(de.path(), ec);
continue;
}
@@ -3051,14 +3112,15 @@ void server_prompt_cache::disk_init(const std::string & dir, int32_t limit_mib,
meta.id = new_entry_id();
meta.created = t;
meta.last_used = t;
- meta.family = fnv1a(tokens.data(), std::min(tokens.size(), FAMILY_TOKENS) * sizeof(llama_token));
+ const llama_tokens raw = raw_tokens(tokens);
+ meta.family = fnv1a(raw.data(), std::min(raw.size(), FAMILY_TOKENS) * sizeof(llama_token));
if (detokenize) {
- meta.excerpt = user_excerpt(detokenize(llama_tokens(tokens.begin(), tokens.begin() + std::min(tokens.size(), CLASSIFY_HEAD))));
+ meta.excerpt = user_excerpt(detokenize(llama_tokens(raw.begin(), raw.begin() + std::min(raw.size(), CLASSIFY_HEAD))));
}
set_expiry(meta);
n_v1++;
}
- found.push_back({ mtime, { de.path().string(), std::move(tokens), (size_t) de.file_size(ec), std::move(meta), version } });
+ found.emplace_back(mtime, disk_entry { de.path().string(), std::move(tokens), (size_t) de.file_size(ec), std::move(meta), version });
}
std::sort(found.begin(), found.end(), [](const auto & a, const auto & b) { return a.first < b.first; });
@@ -3073,17 +3135,21 @@ void server_prompt_cache::disk_init(const std::string & dir, int32_t limit_mib,
total += e.size;
}
- SRV_INF("prompt cache: disk tier %s, %zu prompts (%zu v1), %.1f GiB (limit %.1f GiB)\n",
- dir.c_str(), disk.size(), n_v1, total / (1024.0*1024.0*1024.0), disk_limit / (1024.0*1024.0*1024.0));
+ size_t n_media = 0;
+ for (const auto & e : disk) {
+ n_media += e.version == DISK_VERSION_MEDIA;
+ }
+
+ SRV_INF("prompt cache: disk tier %s, %zu prompts (%zu v1, %zu with images), %.1f GiB (limit %.1f GiB)\n",
+ dir.c_str(), disk.size(), n_v1, n_media, total / (1024.0*1024.0*1024.0), disk_limit / (1024.0*1024.0*1024.0));
disk_thread = std::thread([this] { disk_write_loop(); });
}
void server_prompt_cache::evict(std::list<server_prompt_cache_state>::iterator it) {
if (disk_thread.joinable() && !expired(it->prompt.meta, unix_now())) {
- // text prompts only: media chunks are not serialized
- const llama_tokens tokens = raw_tokens(it->prompt.tokens);
- if (std::find(tokens.begin(), tokens.end(), LLAMA_TOKEN_NULL) == tokens.end()) {
+ // a prompt with images needs the media key its file is read back under
+ if (disk_media != 0 || !has_media(it->prompt.tokens)) {
std::lock_guard<std::mutex> lock(disk_mutex);
pending.splice(pending.end(), states, it);
disk_cv.notify_all();
@@ -3131,8 +3197,20 @@ void server_prompt_cache::disk_write_loop() {
namespace {
-// the whole file at path (via path.tmp): header, metadata, then the v1 body
-bool write_file(const std::string & path, uint64_t key, const llama_tokens & tokens, const server_prompt_cache_state & state, const cache_meta & meta) {
+// what a file stores as its token list: the tokens of a text prompt, server_tokens::serialize() of one with media
+llama_tokens disk_tokens(const server_tokens & t) {
+ if (!has_media(t)) {
+ return raw_tokens(t);
+ }
+ const std::vector<char> blob = t.serialize();
+ llama_tokens res(blob.size() / sizeof(llama_token));
+ memcpy(res.data(), blob.data(), res.size() * sizeof(llama_token));
+ return res;
+}
+
+// the whole file at path (via path.tmp): header, metadata, then the v1 body. media != 0 writes v3: the media key
+// before the token list, which then holds the serialized media chunks too
+bool write_file(const std::string & path, uint64_t key, uint64_t media, const llama_tokens & tokens, const server_prompt_cache_state & state, const cache_meta & meta) {
const std::string tmp = path + ".tmp";
bool ok;
{
@@ -3141,7 +3219,8 @@ bool write_file(const std::string & path, uint64_t key, const llama_tokens & tok
if (ok) {
setvbuf(f.f, nullptr, _IOFBF, 8u << 20);
const auto mb = meta_encode(meta);
- ok = put(f.f, DISK_MAGIC) && put(f.f, DISK_VERSION) && put(f.f, key) && put(f.f, mb.data(), mb.size()) &&
+ ok = put(f.f, DISK_MAGIC) && put(f.f, media != 0 ? DISK_VERSION_MEDIA : DISK_VERSION) && put(f.f, key) &&
+ put(f.f, mb.data(), mb.size()) && (media == 0 || put(f.f, media)) &&
put(f.f, (uint64_t) tokens.size()) && put(f.f, tokens.data(), tokens.size() * sizeof(llama_token)) &&
put(f.f, (uint64_t) state.prompt.checkpoints.size());
for (const auto & ckpt : state.prompt.checkpoints) {
@@ -3166,7 +3245,7 @@ bool write_file(const std::string & path, uint64_t key, const llama_tokens & tok
} // namespace
void server_prompt_cache::disk_rewrite_meta_locked(disk_entry & e) {
- if (e.version == DISK_VERSION) {
+ if (e.version >= DISK_VERSION) {
if (!write_meta(e.path, e.meta)) {
SRV_WRN("prompt cache: cannot update metadata of %s\n", e.path.c_str());
}
@@ -3179,20 +3258,20 @@ void server_prompt_cache::disk_rewrite_meta_locked(disk_entry & e) {
// rewrites a v1 file as v2 so its metadata can be stored
void server_prompt_cache::disk_upgrade(const std::string & path) {
server_prompt_cache_state state;
- if (!disk_read(path, false, state)) {
+ if (!disk_read(path, state)) {
return;
}
cache_meta meta;
{
std::lock_guard<std::mutex> lock(disk_mutex);
auto it = std::find_if(disk.begin(), disk.end(), [&](const disk_entry & e) { return e.path == path; });
- if (it == disk.end() || it->version == DISK_VERSION) {
+ if (it == disk.end() || it->version != 1) {
return;
}
meta = it->meta;
}
- const llama_tokens tokens = raw_tokens(state.prompt.tokens);
- const bool ok = write_file(path, disk_key, tokens, state, meta);
+ // v1 files hold text prompts only
+ const bool ok = write_file(path, disk_key, 0, raw_tokens(state.prompt.tokens), state, meta);
std::lock_guard<std::mutex> lock(disk_mutex);
auto it = std::find_if(disk.begin(), disk.end(), [&](const disk_entry & e) { return e.path == path; });
@@ -3267,7 +3346,8 @@ void server_prompt_cache::disk_evict_locked(const std::string & newest) {
void server_prompt_cache::disk_write(server_prompt_cache_state & state) {
namespace fs = std::filesystem;
- const llama_tokens tokens = raw_tokens(state.prompt.tokens);
+ const server_tokens & tokens = state.prompt.tokens;
+ const bool media = has_media(tokens);
cache_meta meta;
{
std::lock_guard<std::mutex> lock(disk_mutex);
@@ -3285,10 +3365,12 @@ void server_prompt_cache::disk_write(server_prompt_cache_state & state) {
}
const int64_t t_start = ggml_time_us();
- const uint64_t h = fnv1a(tokens.data(), tokens.size() * sizeof(llama_token));
+ // the name hashes the media chunks too: prompts with equal text and other images get files of their own
+ const llama_tokens words = disk_tokens(tokens);
+ const uint64_t h = fnv1a(words.data(), words.size() * sizeof(llama_token));
const std::string path = string_format("%s/%016llx-%zu.kvc", disk_dir.c_str(), (unsigned long long) h, tokens.size());
- if (!write_file(path, disk_key, tokens, state, meta)) {
+ if (!write_file(path, disk_key, media ? disk_media : 0, words, state, meta)) {
SRV_ERR("prompt cache: failed to write %s\n", path.c_str());
return;
}
@@ -3311,7 +3393,7 @@ void server_prompt_cache::disk_write(server_prompt_cache_state & state) {
}
}
disk.remove_if([&](const disk_entry & e) { return e.path == path; });
- disk.push_back({ path, tokens, size, latest, DISK_VERSION });
+ disk.push_back({ path, tokens.clone(), size, latest, media ? DISK_VERSION_MEDIA : DISK_VERSION });
if (meta_encode(latest) != meta_encode(meta)) {
set_expiry(disk.back().meta);
write_meta(path, disk.back().meta);
@@ -3324,8 +3406,8 @@ void server_prompt_cache::disk_write(server_prompt_cache_state & state) {
total += e.size;
}
- SRV_INF("prompt cache: wrote %zu tokens (%s, %s) to disk, %.1f MiB in %.2f s (%zu prompts, %.1f GiB)\n",
- tokens.size(), hex_id(meta.id).c_str(), cache_class_name(meta.cls), size / (1024.0*1024.0), (ggml_time_us() - t_start) / 1e6,
+ SRV_INF("prompt cache: wrote %zu tokens, %lld images (%s, %s) to disk, %.1f MiB in %.2f s (%zu prompts, %.1f GiB)\n",
+ tokens.size(), (long long) std::max<int64_t>(0, media_count(tokens)), hex_id(meta.id).c_str(), cache_class_name(meta.cls), size / (1024.0*1024.0), (ggml_time_us() - t_start) / 1e6,
disk.size(), total / (1024.0*1024.0*1024.0));
}
@@ -3336,7 +3418,7 @@ json server_prompt_cache::info() {
{
tier_report rep;
for (const auto & s : states) {
- rep.add(s.prompt.meta, s.prompt.tokens.size(), s.size(), now);
+ rep.add(s.prompt.meta, s.prompt.tokens.size(), std::max<int64_t>(0, media_count(s.prompt.tokens)), s.size(), now);
}
rep.to_json(ram);
}
@@ -3359,15 +3441,18 @@ json server_prompt_cache::info() {
std::lock_guard<std::mutex> lock(disk_mutex);
tier_report rep;
size_t n_v1 = 0;
+ size_t n_media = 0;
for (const auto & e : disk) {
- rep.add(e.meta, e.tokens.size(), e.size, now);
+ rep.add(e.meta, e.tokens.size(), std::max<int64_t>(0, media_count(e.tokens)), e.size, now);
n_v1 += e.version == 1;
+ n_media += e.version == DISK_VERSION_MEDIA;
}
json d = {
{ "dir", disk_dir },
{ "limit_bytes", disk_limit },
{ "pending", pending.size() },
{ "v1_files", n_v1 },
+ { "image_files", n_media },
};
rep.to_json(d);
res["disk"] = d;
@@ -3396,18 +3481,18 @@ size_t server_prompt_cache::clear_disk(bool include_pinned) {
return n;
}
-bool server_prompt_cache::disk_read(const std::string & path, bool has_mtmd, server_prompt_cache_state & out) const {
+bool server_prompt_cache::disk_read(const std::string & path, server_prompt_cache_state & out) const {
disk_file f(path, "rb");
if (!f.f) {
return false;
}
setvbuf(f.f, nullptr, _IOFBF, 8u << 20);
- llama_tokens tokens;
+ server_tokens tokens;
uint32_t version;
cache_meta meta;
uint64_t n_ckpt;
- if (!read_header(f.f, disk_key, tokens, version, meta) || !get(f.f, n_ckpt) || n_ckpt > 4096) {
+ if (!read_header(f.f, disk_key, disk_media, tokens, version, meta) || !get(f.f, n_ckpt) || n_ckpt > 4096) {
return false;
}
std::list<common_prompt_checkpoint> checkpoints;
@@ -3425,7 +3510,7 @@ bool server_prompt_cache::disk_read(const std::string & path, bool has_mtmd, ser
return false;
}
- out.prompt.tokens = server_tokens(tokens, has_mtmd);
+ out.prompt.tokens = std::move(tokens);
out.prompt.checkpoints = std::move(checkpoints);
out.prompt.meta = std::move(meta);
out.data = std::move(data);
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index 21554dfe..2575c9cf 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -741,17 +741,19 @@ struct server_prompt_cache {
bool load_entry(std::list<server_prompt_cache_state>::iterator it, server_prompt & prompt, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot, bool keep_entry);
// the disk entry whose tokens score highest, above n_best (which it raises to that score); empty if none
- std::string disk_best(const std::function<int32_t(const llama_tokens &)> & score, int32_t & n_best);
+ std::string disk_best(const std::function<int32_t(const server_tokens &)> & score, int32_t & n_best);
// reads a disk entry into RAM and returns it, or states.end() if the file cannot be read (it is then removed)
- std::list<server_prompt_cache_state>::iterator disk_promote(const std::string & path, bool has_mtmd);
+ std::list<server_prompt_cache_state>::iterator disk_promote(const std::string & path);
void update();
// second tier on disk: an entry the RAM limit evicts is written to dir instead of dropped, and read back
// when it is the best prefix match for a new prompt; flush() writes what is still in RAM (on shutdown).
- // key names the model and cache layout; files written under another key are deleted. text prompts only.
- void disk_init(const std::string & dir, int32_t limit_mib, const std::string & key);
+ // key names the model and cache layout; files written under another key are deleted. media names the
+ // multimodal projector and its image settings (empty without one): a prompt with images is stored with its
+ // media chunk map (v3) and only read back under the same media key.
+ void disk_init(const std::string & dir, int32_t limit_mib, const std::string & key, const std::string & media);
void flush();
@@ -790,11 +792,11 @@ struct server_prompt_cache {
private:
struct disk_entry {
- std::string path;
- llama_tokens tokens;
- size_t size;
- cache_meta meta;
- uint32_t version;
+ std::string path;
+ server_tokens tokens; // media chunks are placeholders
+ size_t size;
+ cache_meta meta;
+ uint32_t version;
};
struct family_stat {
@@ -820,6 +822,7 @@ private:
std::string disk_dir;
size_t disk_limit = 0;
uint64_t disk_key = 0;
+ uint64_t disk_media = 0; // 0 = no multimodal projector
// oldest first; the lists and stop are guarded by disk_mutex
std::list<disk_entry> disk;
@@ -860,7 +863,7 @@ private:
void disk_write_loop();
void disk_write(server_prompt_cache_state & state);
void disk_upgrade(const std::string & path);
- bool disk_read(const std::string & path, bool has_mtmd, server_prompt_cache_state & out) const;
+ bool disk_read(const std::string & path, server_prompt_cache_state & out) const;
void disk_evict_locked(const std::string & newest);
void disk_rewrite_meta_locked(disk_entry & e);