diff --git a/common/arg.cpp b/common/arg.cpp index e096bfad..be415ee0 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -1724,6 +1724,21 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.cache_disk_mib = value; } ).set_env("LLAMA_ARG_CACHE_DISK_SIZE").set_examples({LLAMA_EXAMPLE_SERVER})); + add_opt(common_arg( + {"--cache-embd-ram"}, "N", + string_format("RAM for image embeddings by image content in MiB, so an image that has to be prefilled again\n" + "skips the vision encoder (default: %d, 0 - off)", params.cache_embd_ram_mib), + [](common_params & params, int value) { + params.cache_embd_ram_mib = value; + } + ).set_env("LLAMA_ARG_CACHE_EMBD_RAM").set_examples({LLAMA_EXAMPLE_SERVER})); + add_opt(common_arg( + {"--cache-embd-disk"}, "N", + string_format("disk space for image embeddings under <--cache-disk>/embd in MiB (default: %d, 0 - off)", params.cache_embd_disk_mib), + [](common_params & params, int value) { + params.cache_embd_disk_mib = value; + } + ).set_env("LLAMA_ARG_CACHE_EMBD_DISK").set_examples({LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( {"--cache-ttl-ephemeral"}, "N", string_format("seconds a prompt cache entry of class ephemeral lives after its last use (default: %d)", params.cache_ttl_ephemeral), diff --git a/common/common.h b/common/common.h index 550bef61..cc89d4b1 100644 --- a/common/common.h +++ b/common/common.h @@ -621,6 +621,8 @@ struct common_params { int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc. std::string cache_disk_path; // directory of the prompt cache's disk tier, empty = off int32_t cache_disk_mib = 51200; // disk tier size limit in MiB, 0 = no limit + int32_t cache_embd_ram_mib = 256; // image embedding cache in RAM, MiB, 0 = off + int32_t cache_embd_disk_mib = 4096; // image embedding cache under /embd, MiB, 0 = off int32_t cache_ttl_ephemeral = 3600; // seconds an unused prompt cache entry of each class is kept int32_t cache_ttl_default = 3*86400; int32_t cache_ttl_keep = 30*86400; diff --git a/tools/server/CMakeLists.txt b/tools/server/CMakeLists.txt index 280bd9e1..1f6ad1f2 100644 --- a/tools/server/CMakeLists.txt +++ b/tools/server/CMakeLists.txt @@ -15,6 +15,8 @@ add_library(${TARGET} STATIC server-common.h server-context.cpp server-context.h + server-embd-cache.cpp + server-embd-cache.h server-stream.cpp server-stream.h server-tools.cpp diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index f657fdf0..0609fa71 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -1,6 +1,7 @@ #include "server-context.h" #include "server-chat.h" #include "server-common.h" +#include "server-embd-cache.h" #include "server-http.h" #include "server-task.h" #include "server-queue.h" @@ -205,6 +206,7 @@ struct server_slot { // multimodal mtmd_context * mctx = nullptr; mtmd::batch_ptr mbatch = nullptr; + server_embd_cache * embd_cache = nullptr; // may be null // speculative decoding common_speculative * spec; @@ -734,38 +736,51 @@ static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch const auto & chunk = input_tokens.find_chunk(idx); int32_t res = 0; + // an image's rows are cached by its content id and token grid (the decoder position of its last token) + const mtmd_image_tokens * image = mtmd_input_chunk_get_tokens_image(chunk.get()); + server_embd_cache * cache = image ? slot.embd_cache : nullptr; + const mtmd_decoder_pos grid = image ? mtmd_image_tokens_get_decoder_pos(image, 0, mtmd_image_tokens_get_n_tokens(image) - 1) : mtmd_decoder_pos{}; + const size_t n_floats = (size_t) llama_model_n_embd_inp(llama_get_model(slot.ctx_tgt)) * mtmd_input_chunk_get_n_tokens(chunk.get()); + + auto decode = [&](float * embd) -> int32_t { + void * cb_data = slot.spec; + static auto cb = [](llama_batch batch, void * user_data) { + common_speculative * spec = static_cast(user_data); + if (!common_speculative_process(spec, batch)) { + return 1; + } + return 0; + }; + + llama_pos new_n_past; // unused for now + res = mtmd_helper_decode_image_chunk( + mctx, + slot.ctx_tgt, + chunk.get(), + embd, + slot.prompt.tokens.pos_next(), + slot.id, + llama_n_batch(slot.ctx_tgt), + &new_n_past, + cb, + cb_data + ); + if (res != 0) { + SLT_ERR(slot, "failed to decode mtmd chunk, idx = %zu, res = %d\n", idx, res); + return -1; + } + n_tokens_out = mtmd_input_chunk_get_n_tokens(chunk.get()); + return 0; // success + }; + auto try_decode = [&]() -> int32_t { if (mbatch) { float * embd = mtmd_batch_get_output_embd(mbatch.get(), chunk.get()); if (embd) { - void * cb_data = slot.spec; - static auto cb = [](llama_batch batch, void * user_data) { - common_speculative * spec = static_cast(user_data); - if (!common_speculative_process(spec, batch)) { - return 1; - } - return 0; - }; - - llama_pos new_n_past; // unused for now - res = mtmd_helper_decode_image_chunk( - mctx, - slot.ctx_tgt, - chunk.get(), - embd, - slot.prompt.tokens.pos_next(), - slot.id, - llama_n_batch(slot.ctx_tgt), - &new_n_past, - cb, - cb_data - ); - if (res != 0) { - SLT_ERR(slot, "failed to decode mtmd chunk, idx = %zu, res = %d\n", idx, res); - return -1; + if (cache) { + cache->put(mtmd_input_chunk_get_id(chunk.get()), grid.x + 1, grid.y + 1, embd, n_floats); } - n_tokens_out = mtmd_input_chunk_get_n_tokens(chunk.get()); - return 0; // success + return decode(embd); } } return 1; // (non-error) need to create & encode batch @@ -781,6 +796,16 @@ static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch return res; } + if (cache) { + const int64_t t0 = ggml_time_us(); + std::vector rows; + if (cache->get(mtmd_input_chunk_get_id(chunk.get()), grid.x + 1, grid.y + 1, n_floats, rows)) { + SLT_INF(slot, "image at idx = %zu (%zu tokens): cached embedding in %.1f ms, encoder skipped\n", idx, + mtmd_input_chunk_get_n_tokens(chunk.get()), (ggml_time_us() - t0) / 1e3); + return decode(rows.data()); + } + } + // otherwise, the batch is either uninitialized or is used up // we need to create & encode a new batch mbatch.reset(mtmd_batch_init(mctx)); @@ -805,11 +830,13 @@ static int process_mtmd_chunk(const server_slot & slot, mtmd::batch_ptr & mbatch // TODO @ngxson : move this log line to debug when it become more stable SLT_TRC(slot, "encoding mtmd batch from idx = %zu, n_chunks = %d\n", idx, n_added); + const int64_t t_encode = ggml_time_us(); res = mtmd_batch_encode(mbatch.get()); if (res != 0) { SLT_ERR(slot, "failed to encode mtmd batch for chunk idx = %zu, res = %d\n", idx, res); return -1; } + SLT_INF(slot, "encoded %d media chunk(s) from idx = %zu in %.1f ms\n", n_added, idx, (ggml_time_us() - t_encode) / 1e3); return try_decode(); } @@ -900,6 +927,20 @@ private: std::unique_ptr prompt_cache; + // created with the multimodal projector, kept across sleep + std::unique_ptr embd_cache; + + // what the image embeddings depend on besides the image: the projector file and the encoder settings + std::string media_identity() const { + std::error_code ec; + const auto & path = params_base.mmproj.path; + const auto mtime = std::filesystem::last_write_time(path, ec).time_since_epoch().count(); + const auto size = std::filesystem::file_size(path, ec); + const char * bi = getenv("GGML_CUDA_BATCH_INVARIANT"); + return string_format("%s|%llu|%lld|gpu=%d|min=%d|max=%d|bi=%s", path.c_str(), (unsigned long long) size, (long long) mtime, + params_base.mmproj_use_gpu ? 1 : 0, params_base.image_min_tokens, params_base.image_max_tokens, bi ? bi : ""); + } + server_metrics metrics; // queued prompt stats - llama_decode() is async, so the timing is only valid after a sync @@ -1169,6 +1210,14 @@ private: } SRV_INF("loaded multimodal model, '%s'\n", mmproj_path.c_str()); + if (!embd_cache && (params_base.cache_embd_ram_mib > 0 || params_base.cache_embd_disk_mib > 0)) { + const bool disk = !params_base.cache_disk_path.empty() && params_base.cache_embd_disk_mib > 0; + embd_cache = std::make_unique(media_identity(), + 1024ull*1024ull*(size_t) std::max(params_base.cache_embd_ram_mib, 0), + disk ? params_base.cache_disk_path + "/embd" : "", + 1024ull*1024ull*(size_t) std::max(params_base.cache_embd_disk_mib, 0)); + } + if (params_base.ctx_shift) { params_base.ctx_shift = false; SRV_WRN("%s\n", "ctx_shift is not supported by multimodal, it will be disabled"); @@ -1292,6 +1341,7 @@ private: slot.stats.speculative = slot.can_speculate(); slot.mctx = mctx; + slot.embd_cache = embd_cache.get(); slot.prompt.tokens.has_mtmd = mctx != nullptr; SLT_TRC(slot, "new slot, n_ctx = %d\n", slot.n_ctx); @@ -1393,7 +1443,7 @@ private: (unsigned long long) llama_model_size(model_tgt), (unsigned long long) llama_model_n_params(model_tgt), (long long) mtime, ggml_type_name(params_base.cache_type_k), ggml_type_name(params_base.cache_type_v), ggml_type_name(params_base.speculative.draft.cache_type_k), ggml_type_name(params_base.speculative.draft.cache_type_v)); - prompt_cache->disk_init(params_base.cache_disk_path, params_base.cache_disk_mib, key); + prompt_cache->disk_init(params_base.cache_disk_path, params_base.cache_disk_mib, key, mctx ? media_identity() : ""); } prompt_cache->policy_start(); } else { @@ -2468,10 +2518,19 @@ private: return scr_pays(plan, task.tokens.size()) ? plan.n_prefix + plan.n_reused() : (int32_t) prompt.tokens.get_common_prefix(task.tokens); }; + // loading an entry consumes it, so one counts only for a splice that pays or, like in prompt_cache.load(), + // for a plain prefix of at least a quarter of it: a few tokens of chat template must not take an entry. // planning against every cached prompt is O(cache size); only prompts that start like this one are planned const auto value_of = [&](const server_prompt & prompt, int32_t n_state) { const int32_t lcp = prompt.tokens.get_common_prefix(task.tokens); - return lcp < 16 ? lcp : value(scr_plan(prompt, n_state, task.tokens), prompt); + if (lcp < 16) { + return -1; + } + const common_scr_plan plan = scr_plan(prompt, n_state, task.tokens); + if (scr_pays(plan, task.tokens.size())) { + return plan.n_prefix + plan.n_reused(); + } + return 4*lcp >= prompt.n_tokens() ? lcp : -1; }; const int32_t n_own = value(scr_plan(slot, task.tokens), slot.prompt); @@ -2490,10 +2549,10 @@ private: } } - // disk entries are text prompts without their checkpoints in the index - const std::string path = prompt_cache->disk_best([&](const llama_tokens & tokens) { + // the disk index holds the prompts (images as placeholder chunks) without their checkpoints + const std::string path = prompt_cache->disk_best([&](const server_tokens & tokens) { server_prompt prompt; - prompt.tokens = server_tokens(tokens, false); + prompt.tokens = tokens.clone(); return value_of(prompt, prompt.n_tokens()); }, n_best); @@ -2506,7 +2565,7 @@ private: } // saving can evict RAM entries, so a RAM entry is looked up again by its id - auto best = !path.empty() ? prompt_cache->disk_promote(path, task.tokens.has_mtmd) : + auto best = !path.empty() ? prompt_cache->disk_promote(path) : std::find_if(prompt_cache->states.begin(), prompt_cache->states.end(), [&](const server_prompt_cache_state & s) { return s.prompt.meta.id == id_best; }); if (best == prompt_cache->states.end()) { return; @@ -3077,8 +3136,8 @@ private: if (task.type == SERVER_TASK_TYPE_CACHE_CLEAR) { const std::string & scope = task.cache_scope; const bool all = scope == "all"; - if (!all && scope != "slots" && scope != "ram" && scope != "disk") { - send_error(task, "scope must be slots, ram, disk or all", ERROR_TYPE_INVALID_REQUEST); + if (!all && scope != "slots" && scope != "ram" && scope != "disk" && scope != "embd") { + send_error(task, "scope must be slots, ram, disk, embd or all", ERROR_TYPE_INVALID_REQUEST); break; } if (all || scope == "slots") { @@ -3103,6 +3162,9 @@ private: if (prompt_cache && (all || scope == "disk")) { cleared["disk"] = prompt_cache->clear_disk(task.cache_include_pinned); } + if (embd_cache && (all || scope == "embd")) { + cleared["embd"] = embd_cache->clear(); + } SRV_INF("cache cleared (%s): %s\n", scope.c_str(), cleared.dump().c_str()); } @@ -3122,6 +3184,9 @@ private: res->id = task.id; res->data = prompt_cache ? prompt_cache->info() : json::object(); res->data["slots"] = slots_info; + if (embd_cache) { + res->data["embd"] = embd_cache->info(); + } if (task.type == SERVER_TASK_TYPE_CACHE_CLEAR) { res->data["cleared"] = cleared; } @@ -5243,7 +5308,7 @@ void server_routes::init_routes() { return res; }; - // GET /cache: slots and both prompt-cache tiers; POST /cache/clear?scope=slots|ram|disk|all + // GET /cache: slots, both prompt-cache tiers and the image embedding cache; POST /cache/clear?scope=slots|ram|disk|embd|all this->get_cache = [this](const server_http_req & req) { return handle_cache(req, SERVER_TASK_TYPE_CACHE_GET, ""); }; diff --git a/tools/server/server-embd-cache.cpp b/tools/server/server-embd-cache.cpp new file mode 100644 index 00000000..5d4035ae --- /dev/null +++ b/tools/server/server-embd-cache.cpp @@ -0,0 +1,275 @@ +#include "server-embd-cache.h" + +#include +#include +#include +#include + +namespace { + +constexpr uint32_t EMBD_MAGIC = 0x424d4542; // "BEMB" +constexpr uint32_t EMBD_VERSION = 1; + +// magic, version, identity, nx, ny, n_floats, then the rows and a checksum of their bytes +constexpr size_t EMBD_HEADER = 4 + 4 + 8 + 4 + 4 + 8; + +uint64_t fnv1a(const void * data, size_t n, uint64_t h = 1469598103934665603ull) { + const auto * p = static_cast(data); + for (size_t i = 0; i < n; ++i) { + h = (h ^ p[i]) * 1099511628211ull; + } + return h; +} + +// fnv-1a over 64-bit words: a corruption check, ~8x faster than bytewise on 20 MiB of rows +uint64_t checksum(const float * data, size_t n_floats) { + const size_t n_bytes = n_floats * sizeof(float); + uint64_t h = 1469598103934665603ull; + size_t i = 0; + for (; i + 8 <= n_bytes; i += 8) { + uint64_t w; + memcpy(&w, (const uint8_t *) data + i, 8); + h = (h ^ w) * 1099511628211ull; + } + return fnv1a((const uint8_t *) data + i, n_bytes - i, h); +} + +std::string ram_key(const std::string & id, uint32_t nx, uint32_t ny) { + return id + "-" + std::to_string(nx) + "x" + std::to_string(ny); +} + +// ids are content hashes (hex); anything else stays out of file names +bool valid_id(const std::string & id) { + return !id.empty() && id.size() <= 128 && + std::all_of(id.begin(), id.end(), [](char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); }); +} + +struct embd_file { + FILE * f; + embd_file(const std::string & path, const char * mode) : f(fopen(path.c_str(), mode)) {} + ~embd_file() { + if (f) { + fclose(f); + } + } +}; + +template bool write_val(FILE * f, const T & v) { + return fwrite(&v, sizeof(v), 1, f) == 1; +} + +template bool read_val(FILE * f, T & v) { + return fread(&v, sizeof(v), 1, f) == 1; +} + +bool read_header(FILE * f, uint64_t ident, uint32_t & nx, uint32_t & ny, uint64_t & n_floats) { + uint32_t magic, version; + uint64_t id; + return read_val(f, magic) && read_val(f, version) && read_val(f, id) && read_val(f, nx) && read_val(f, ny) && read_val(f, n_floats) && + magic == EMBD_MAGIC && version == EMBD_VERSION && id == ident && n_floats < (1ull << 32); +} + +} // namespace + +server_embd_cache::server_embd_cache(const std::string & identity, size_t ram_limit, const std::string & dir, size_t disk_limit) + : ident(fnv1a(identity.data(), identity.size())), ram_limit(ram_limit), dir(dir), disk_limit(disk_limit) { + namespace fs = std::filesystem; + + if (this->dir.empty()) { + return; + } + std::error_code ec; + fs::create_directories(this->dir, ec); + if (ec) { + SRV_ERR("embedding cache: cannot create %s: %s, disk tier off\n", this->dir.c_str(), ec.message().c_str()); + this->dir.clear(); + return; + } + + // oldest first, so the clock replays the files' order of use + std::vector> found; + for (const auto & de : fs::directory_iterator(this->dir, ec)) { + const auto ext = de.path().extension(); + if (ext == ".tmp") { + fs::remove(de.path(), ec); + continue; + } + if (ext != ".emb") { + continue; + } + uint32_t nx, ny; + uint64_t n_floats; + bool ok; + { + embd_file f(de.path().string(), "rb"); + ok = f.f && read_header(f.f, ident, nx, ny, n_floats) && + de.file_size(ec) == EMBD_HEADER + n_floats * sizeof(float) + sizeof(uint64_t); + } + if (!ok) { + // another mmproj or encoder setting + fs::remove(de.path(), ec); + continue; + } + found.push_back({ de.last_write_time(ec), de.path().filename().string() }); + } + std::sort(found.begin(), found.end()); + for (const auto & [_, name] : found) { + const size_t size = fs::file_size(this->dir + "/" + name, ec); + disk[name] = { size, ++disk_clock }; + disk_bytes += size; + } + disk_evict_locked(); + + SRV_INF("embedding cache: RAM %.0f MiB, disk %s with %zu images, %.1f MiB (limit %.0f MiB)\n", ram_limit / 1048576.0, + this->dir.c_str(), disk.size(), disk_bytes / 1048576.0, disk_limit / 1048576.0); +} + +bool server_embd_cache::get(const std::string & id, uint32_t nx, uint32_t ny, size_t n_floats, std::vector & out) { + std::lock_guard lock(mutex); + + const std::string key = ram_key(id, nx, ny); + const auto it = ram_index.find(key); + if (it != ram_index.end() && it->second->data.size() == n_floats) { + ram.splice(ram.end(), ram, it->second); + out = it->second->data; + n_hits_ram++; + return true; + } + + if (!dir.empty() && valid_id(id) && disk_read_locked(key + ".emb", n_floats, out)) { + ram_put_locked(key, std::vector(out)); + n_hits_disk++; + return true; + } + + n_misses++; + return false; +} + +void server_embd_cache::put(const std::string & id, uint32_t nx, uint32_t ny, const float * data, size_t n_floats) { + std::lock_guard lock(mutex); + + const std::string key = ram_key(id, nx, ny); + if (ram_index.count(key) == 0) { + ram_put_locked(key, std::vector(data, data + n_floats)); + } + if (!dir.empty() && valid_id(id) && disk.count(key + ".emb") == 0) { + disk_write_locked(key + ".emb", nx, ny, data, n_floats); + } +} + +void server_embd_cache::ram_put_locked(const std::string & key, std::vector && data) { + const size_t size = data.size() * sizeof(float); + if (size > ram_limit) { + return; + } + while (!ram.empty() && ram_bytes + size > ram_limit) { + ram_bytes -= ram.front().data.size() * sizeof(float); + ram_index.erase(ram.front().key); + ram.pop_front(); + } + ram.push_back({ key, std::move(data) }); + ram_index[key] = std::prev(ram.end()); + ram_bytes += size; +} + +bool server_embd_cache::disk_read_locked(const std::string & name, size_t n_floats, std::vector & out) { + const auto it = disk.find(name); + if (it == disk.end()) { + return false; + } + const std::string path = dir + "/" + name; + bool ok; + { + embd_file f(path, "rb"); + uint32_t nx, ny; + uint64_t n, sum; + ok = f.f && read_header(f.f, ident, nx, ny, n) && n == n_floats; + if (ok) { + out.resize(n_floats); + ok = fread(out.data(), sizeof(float), n_floats, f.f) == n_floats && read_val(f.f, sum) && sum == checksum(out.data(), n_floats); + } + } + if (!ok) { + SRV_WRN("embedding cache: cannot read %s, removing it\n", path.c_str()); + std::error_code ec; + std::filesystem::remove(path, ec); + disk_bytes -= it->second.size; + disk.erase(it); + return false; + } + it->second.used = ++disk_clock; + std::error_code ec; + std::filesystem::last_write_time(path, std::filesystem::file_time_type::clock::now(), ec); + return true; +} + +void server_embd_cache::disk_write_locked(const std::string & name, uint32_t nx, uint32_t ny, const float * data, size_t n_floats) { + const size_t size = EMBD_HEADER + n_floats * sizeof(float) + sizeof(uint64_t); + if (size > disk_limit) { + return; + } + const std::string path = dir + "/" + name; + const std::string tmp = path + ".tmp"; + + bool ok; + { + embd_file f(tmp, "wb"); + ok = f.f && write_val(f.f, EMBD_MAGIC) && write_val(f.f, EMBD_VERSION) && write_val(f.f, ident) && write_val(f.f, nx) && write_val(f.f, ny) && + write_val(f.f, (uint64_t) n_floats) && fwrite(data, sizeof(float), n_floats, f.f) == n_floats && + write_val(f.f, checksum(data, n_floats)) && fflush(f.f) == 0; + } + std::error_code ec; + if (ok) { + std::filesystem::rename(tmp, path, ec); + ok = !ec; + } + if (!ok) { + SRV_WRN("embedding cache: cannot write %s\n", path.c_str()); + std::filesystem::remove(tmp, ec); + return; + } + disk[name] = { size, ++disk_clock }; + disk_bytes += size; + disk_evict_locked(); +} + +void server_embd_cache::disk_evict_locked() { + std::error_code ec; + while (disk_bytes > disk_limit && !disk.empty()) { + auto victim = std::min_element(disk.begin(), disk.end(), [](const auto & a, const auto & b) { return a.second.used < b.second.used; }); + std::filesystem::remove(dir + "/" + victim->first, ec); + disk_bytes -= victim->second.size; + disk.erase(victim); + } +} + +json server_embd_cache::info() const { + std::lock_guard lock(mutex); + json res = { + { "ram", { { "images", ram.size() }, { "bytes", ram_bytes }, { "limit_bytes", ram_limit } } }, + { "hits_ram", n_hits_ram }, + { "hits_disk", n_hits_disk }, + { "misses", n_misses }, + }; + if (!dir.empty()) { + res["disk"] = { { "dir", dir }, { "images", disk.size() }, { "bytes", disk_bytes }, { "limit_bytes", disk_limit } }; + } + return res; +} + +size_t server_embd_cache::clear() { + std::lock_guard lock(mutex); + size_t n = ram.size(); + ram.clear(); + ram_index.clear(); + ram_bytes = 0; + std::error_code ec; + for (const auto & [name, _] : disk) { + std::filesystem::remove(dir + "/" + name, ec); + } + n += disk.size(); + disk.clear(); + disk_bytes = 0; + return n; +} diff --git a/tools/server/server-embd-cache.h b/tools/server/server-embd-cache.h new file mode 100644 index 00000000..f5ce9488 --- /dev/null +++ b/tools/server/server-embd-cache.h @@ -0,0 +1,64 @@ +#pragma once + +#include "server-common.h" + +#include +#include +#include +#include +#include +#include + +// image embeddings (the vision encoder's output rows) by image content, so an image the prompt cache cannot +// reuse is decoded without running the encoder again. a RAM LRU tier and an optional disk tier, each under a +// byte limit. the key folds in everything else the rows depend on (mmproj file, image token limits, encoder +// device), so a cached row is the one a fresh encode of the same image computes. thread-safe +struct server_embd_cache { + // identity: mmproj file and encoder settings; dir empty = no disk tier + server_embd_cache(const std::string & identity, size_t ram_limit, const std::string & dir, size_t disk_limit); + + // the rows of the image with this content id and token grid; false if not cached + bool get(const std::string & id, uint32_t nx, uint32_t ny, size_t n_floats, std::vector & out); + + void put(const std::string & id, uint32_t nx, uint32_t ny, const float * data, size_t n_floats); + + json info() const; + + // drops both tiers; returns the number of entries dropped + size_t clear(); + +private: + struct ram_entry { + std::string key; + std::vector data; + }; + + struct disk_entry { + size_t size; + int64_t used; // order of use, larger = more recent + }; + + uint64_t ident; + size_t ram_limit; + std::string dir; + size_t disk_limit; + + mutable std::mutex mutex; + + std::list ram; // most recently used last + std::unordered_map::iterator> ram_index; + size_t ram_bytes = 0; + + std::unordered_map disk; // by file name + size_t disk_bytes = 0; + int64_t disk_clock = 0; + + uint64_t n_hits_ram = 0; + uint64_t n_hits_disk = 0; + uint64_t n_misses = 0; + + void ram_put_locked(const std::string & key, std::vector && data); + bool disk_read_locked(const std::string & name, size_t n_floats, std::vector & out); + void disk_write_locked(const std::string & name, uint32_t nx, uint32_t ny, const float * data, size_t n_floats); + void disk_evict_locked(); +}; diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 0a83f4d0..044e2480 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -1743,8 +1743,12 @@ namespace { constexpr uint32_t DISK_MAGIC = 0x43564b42; // "BKVC" constexpr uint32_t DISK_VERSION = 2; +constexpr uint32_t DISK_VERSION_MEDIA = 3; -// v2: magic, version, key, META_SIZE bytes of metadata (rewritten in place), then the v1 body +// v2: magic, version, key, META_SIZE bytes of metadata (rewritten in place), then the v1 body. +// v3 (prompts with images): as v2, but the token list is the media key followed by server_tokens::serialize() +// (tokens plus the placeholder media chunks: id, grid, position layout). text prompts stay v2, so a binary that +// predates v3 deletes only the image files (it reads them as another format), and keeps the rest constexpr size_t META_OFFSET = 16; constexpr size_t META_SIZE = 512; constexpr size_t META_FIXED = 74; @@ -1993,29 +1997,83 @@ bool get_bytes(FILE * f, std::vector & v) { return get(f, v.data(), n); } -// magic, version, key, metadata (v2) and the prompt tokens: enough to index a file without reading its state. -// a v1 file has no metadata; has_meta is false for it -bool read_header(FILE * f, uint64_t key, llama_tokens & tokens, uint32_t & version, cache_meta & meta) { +// the number of media chunks in t, or -1 if its chunk map does not cover exactly its media positions +int64_t media_count(const server_tokens & t) { + int64_t n = 0; + for (size_t i = 0; i < t.size();) { + if (t[i] != LLAMA_TOKEN_NULL) { + ++i; + continue; + } + const auto [s, e] = t.media_span(i); + if (s != i || e <= i || e > t.size()) { + return -1; + } + for (size_t j = s; j < e; ++j) { + if (t[j] != LLAMA_TOKEN_NULL) { + return -1; + } + } + n++; + i = e; + } + // chunks at text positions or past the end + int64_t n_map = 0; + try { + t.find_chunk(0); + n_map++; + } catch (const std::exception &) { + } + for (auto p = t.find_next_media_chunk(0); p.first; p = t.find_next_media_chunk(p.second)) { + n_map++; + } + return n_map == n ? n : -1; +} + +// magic, version, key, metadata (v2, v3) and the prompt tokens with their media chunks (v3): enough to index a +// file without reading its state. a v1 file has no metadata. a v3 file is only valid under the same media key +bool read_header(FILE * f, uint64_t key, uint64_t media, server_tokens & tokens, uint32_t & version, cache_meta & meta) { uint32_t magic; uint64_t k; uint64_t n; if (!get(f, magic) || !get(f, version) || !get(f, k)) { return false; } - if (magic != DISK_MAGIC || (version != 1 && version != DISK_VERSION) || k != key) { + if (magic != DISK_MAGIC || (version != 1 && version != DISK_VERSION && version != DISK_VERSION_MEDIA) || k != key) { return false; } - if (version == DISK_VERSION) { + if (version >= DISK_VERSION) { std::vector b(META_SIZE); if (!get(f, b.data(), META_SIZE) || !meta_decode(b, meta)) { return false; } } + if (version == DISK_VERSION_MEDIA) { + uint64_t m; + if (!get(f, m) || m != media || media == 0) { + return false; + } + } if (!get(f, n) || n > (1ull << 26)) { return false; } - tokens.resize(n); - return get(f, tokens.data(), n * sizeof(llama_token)); + llama_tokens raw(n); + if (!get(f, raw.data(), n * sizeof(llama_token))) { + return false; + } + if (version != DISK_VERSION_MEDIA) { + tokens = server_tokens(raw, media != 0); + return true; + } + if (raw.empty() || raw[0] != LLAMA_TOKEN_NULL) { + return false; + } + try { + tokens = server_tokens::deserialize(raw, true); + } catch (const std::exception &) { + return false; + } + return media_count(tokens) > 0; } bool write_meta(const std::string & path, const cache_meta & meta) { @@ -2027,15 +2085,6 @@ bool write_meta(const std::string & path, const cache_meta & meta) { return put(f.f, b.data(), b.size()) && fflush(f.f) == 0; } -size_t common_prefix(const llama_tokens & a, const server_tokens & b) { - const size_t n = std::min(a.size(), b.size()); - size_t i = 0; - while (i < n && b[i] != LLAMA_TOKEN_NULL && a[i] == b[i]) { - ++i; - } - return i; -} - // the token list, media positions as LLAMA_TOKEN_NULL (get_tokens() refuses a multimodal server) llama_tokens raw_tokens(const server_tokens & t) { llama_tokens res(t.size()); @@ -2045,9 +2094,18 @@ llama_tokens raw_tokens(const server_tokens & t) { return res; } -// a is a prefix of b -bool is_prefix(const llama_tokens & a, const llama_tokens & b) { - return a.size() <= b.size() && std::equal(a.begin(), a.end(), b.begin()); +bool has_media(const server_tokens & t) { + for (size_t i = 0; i < t.size(); ++i) { + if (t[i] == LLAMA_TOKEN_NULL) { + return true; + } + } + return false; +} + +// a is a prefix of b; a media chunk matches by id and size +bool is_prefix(const server_tokens & a, const server_tokens & b) { + return a.size() <= b.size() && a.get_common_prefix(b) == a.size(); } size_t count_seq(const llama_tokens & tokens, const llama_tokens & seq) { @@ -2063,13 +2121,14 @@ size_t count_seq(const llama_tokens & tokens, const llama_tokens & seq) { return n; } -json entry_json(const cache_meta & m, size_t n_tokens, size_t bytes, int64_t now) { +json entry_json(const cache_meta & m, size_t n_tokens, size_t n_images, size_t bytes, int64_t now) { return { { "id", hex_id(m.id) }, { "class", cache_class_name(m.cls) }, { "source", cache_source_name(m.source) }, { "classified", m.classified() }, { "n_tokens", n_tokens }, + { "images", n_images }, { "bytes", bytes }, { "hits", m.hits }, { "age", now - m.created }, @@ -2089,7 +2148,7 @@ struct tier_report { std::map> by_class; std::vector> entries; - void add(const cache_meta & m, size_t n_tokens, size_t n_bytes, int64_t now) { + void add(const cache_meta & m, size_t n_tokens, size_t n_images, size_t n_bytes, int64_t now) { prompts++; tokens += n_tokens; bytes += n_bytes; @@ -2099,7 +2158,7 @@ struct tier_report { auto & c = by_class[cache_class_name(m.cls)]; c.first++; c.second += n_bytes; - entries.push_back({ m.last_used, entry_json(m, n_tokens, n_bytes, now) }); + entries.push_back({ m.last_used, entry_json(m, n_tokens, n_images, n_bytes, now) }); } void to_json(json & out) { @@ -2288,7 +2347,7 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok expire_disk_locked(); for (const auto & e : disk) { - const int lcp_cur = common_prefix(e.tokens, tokens_new); + const int lcp_cur = e.tokens.get_common_prefix(tokens_new); const float f_keep_cur = float(lcp_cur) / e.tokens.size(); const float f_sim_cur = float(lcp_cur) / tokens_new.size(); @@ -2309,7 +2368,7 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok bool from_disk = false; if (!disk_best.empty()) { - const auto it = disk_promote(disk_best, tokens_new.has_mtmd); + const auto it = disk_promote(disk_best); if (it != states.end()) { SRV_INF("prompt cache: f_keep = %.3f, f_sim = %.3f\n", f_keep_best, f_sim_best); it_best = it; @@ -2331,12 +2390,12 @@ bool server_prompt_cache::load(server_prompt & prompt, const server_tokens & tok return true; } -std::list::iterator server_prompt_cache::disk_promote(const std::string & path, bool has_mtmd) { +std::list::iterator server_prompt_cache::disk_promote(const std::string & path) { namespace fs = std::filesystem; const int64_t t_start = ggml_time_us(); server_prompt_cache_state state; - const bool ok = disk_read(path, has_mtmd, state); + const bool ok = disk_read(path, state); std::lock_guard lock(disk_mutex); std::error_code ec; @@ -2350,7 +2409,8 @@ std::list::iterator server_prompt_cache::disk_promote return states.end(); } - SRV_INF("prompt cache: read %zu tokens from disk in %.2f s\n", state.prompt.tokens.size(), (ggml_time_us() - t_start) / 1e6); + SRV_INF("prompt cache: read %zu tokens, %lld images from disk in %.2f s\n", state.prompt.tokens.size(), + (long long) std::max(0, media_count(state.prompt.tokens)), (ggml_time_us() - t_start) / 1e6); if (it != disk.end()) { // the file stays: it records the use, and is most recently used, here and after a restart it->meta.last_used = unix_now(); @@ -2358,7 +2418,7 @@ std::list::iterator server_prompt_cache::disk_promote set_expiry(it->meta); state.prompt.meta = it->meta; state.prompt.meta.hits--; // the slot counts this use when the request continues the prompt - if (it->version == DISK_VERSION) { + if (it->version >= DISK_VERSION) { write_meta(it->path, it->meta); } disk.splice(disk.end(), disk, it); @@ -2369,7 +2429,7 @@ std::list::iterator server_prompt_cache::disk_promote return std::prev(states.end()); } -std::string server_prompt_cache::disk_best(const std::function & score, int32_t & n_best) { +std::string server_prompt_cache::disk_best(const std::function & score, int32_t & n_best) { std::string best; std::lock_guard lock(disk_mutex); if (!disk_thread.joinable()) { @@ -3002,12 +3062,13 @@ server_prompt_cache::~server_prompt_cache() { } } -void server_prompt_cache::disk_init(const std::string & dir, int32_t limit_mib, const std::string & key) { +void server_prompt_cache::disk_init(const std::string & dir, int32_t limit_mib, const std::string & key, const std::string & media) { namespace fs = std::filesystem; disk_dir = dir; disk_limit = 1024ull*1024ull*(size_t) std::max(limit_mib, 0); disk_key = fnv1a(key.data(), key.size()); + disk_media = media.empty() ? 0 : std::max(1, fnv1a(media.data(), media.size())); std::error_code ec; fs::create_directories(dir, ec); @@ -3029,16 +3090,16 @@ void server_prompt_cache::disk_init(const std::string & dir, int32_t limit_mib, if (ext != ".kvc") { continue; } - llama_tokens tokens; + server_tokens tokens; uint32_t version = 0; cache_meta meta; bool ok; { disk_file f(de.path().string(), "rb"); - ok = f.f && read_header(f.f, disk_key, tokens, version, meta); + ok = f.f && read_header(f.f, disk_key, disk_media, tokens, version, meta); } if (!ok) { - // another model, cache layout or format + // another model, cache layout, format or multimodal projector fs::remove(de.path(), ec); continue; } @@ -3051,14 +3112,15 @@ void server_prompt_cache::disk_init(const std::string & dir, int32_t limit_mib, meta.id = new_entry_id(); meta.created = t; meta.last_used = t; - meta.family = fnv1a(tokens.data(), std::min(tokens.size(), FAMILY_TOKENS) * sizeof(llama_token)); + const llama_tokens raw = raw_tokens(tokens); + meta.family = fnv1a(raw.data(), std::min(raw.size(), FAMILY_TOKENS) * sizeof(llama_token)); if (detokenize) { - meta.excerpt = user_excerpt(detokenize(llama_tokens(tokens.begin(), tokens.begin() + std::min(tokens.size(), CLASSIFY_HEAD)))); + meta.excerpt = user_excerpt(detokenize(llama_tokens(raw.begin(), raw.begin() + std::min(raw.size(), CLASSIFY_HEAD)))); } set_expiry(meta); n_v1++; } - found.push_back({ mtime, { de.path().string(), std::move(tokens), (size_t) de.file_size(ec), std::move(meta), version } }); + found.emplace_back(mtime, disk_entry { de.path().string(), std::move(tokens), (size_t) de.file_size(ec), std::move(meta), version }); } std::sort(found.begin(), found.end(), [](const auto & a, const auto & b) { return a.first < b.first; }); @@ -3073,17 +3135,21 @@ void server_prompt_cache::disk_init(const std::string & dir, int32_t limit_mib, total += e.size; } - SRV_INF("prompt cache: disk tier %s, %zu prompts (%zu v1), %.1f GiB (limit %.1f GiB)\n", - dir.c_str(), disk.size(), n_v1, total / (1024.0*1024.0*1024.0), disk_limit / (1024.0*1024.0*1024.0)); + size_t n_media = 0; + for (const auto & e : disk) { + n_media += e.version == DISK_VERSION_MEDIA; + } + + SRV_INF("prompt cache: disk tier %s, %zu prompts (%zu v1, %zu with images), %.1f GiB (limit %.1f GiB)\n", + dir.c_str(), disk.size(), n_v1, n_media, total / (1024.0*1024.0*1024.0), disk_limit / (1024.0*1024.0*1024.0)); disk_thread = std::thread([this] { disk_write_loop(); }); } void server_prompt_cache::evict(std::list::iterator it) { if (disk_thread.joinable() && !expired(it->prompt.meta, unix_now())) { - // text prompts only: media chunks are not serialized - const llama_tokens tokens = raw_tokens(it->prompt.tokens); - if (std::find(tokens.begin(), tokens.end(), LLAMA_TOKEN_NULL) == tokens.end()) { + // a prompt with images needs the media key its file is read back under + if (disk_media != 0 || !has_media(it->prompt.tokens)) { std::lock_guard lock(disk_mutex); pending.splice(pending.end(), states, it); disk_cv.notify_all(); @@ -3131,8 +3197,20 @@ void server_prompt_cache::disk_write_loop() { namespace { -// the whole file at path (via path.tmp): header, metadata, then the v1 body -bool write_file(const std::string & path, uint64_t key, const llama_tokens & tokens, const server_prompt_cache_state & state, const cache_meta & meta) { +// what a file stores as its token list: the tokens of a text prompt, server_tokens::serialize() of one with media +llama_tokens disk_tokens(const server_tokens & t) { + if (!has_media(t)) { + return raw_tokens(t); + } + const std::vector blob = t.serialize(); + llama_tokens res(blob.size() / sizeof(llama_token)); + memcpy(res.data(), blob.data(), res.size() * sizeof(llama_token)); + return res; +} + +// the whole file at path (via path.tmp): header, metadata, then the v1 body. media != 0 writes v3: the media key +// before the token list, which then holds the serialized media chunks too +bool write_file(const std::string & path, uint64_t key, uint64_t media, const llama_tokens & tokens, const server_prompt_cache_state & state, const cache_meta & meta) { const std::string tmp = path + ".tmp"; bool ok; { @@ -3141,7 +3219,8 @@ bool write_file(const std::string & path, uint64_t key, const llama_tokens & tok if (ok) { setvbuf(f.f, nullptr, _IOFBF, 8u << 20); const auto mb = meta_encode(meta); - ok = put(f.f, DISK_MAGIC) && put(f.f, DISK_VERSION) && put(f.f, key) && put(f.f, mb.data(), mb.size()) && + ok = put(f.f, DISK_MAGIC) && put(f.f, media != 0 ? DISK_VERSION_MEDIA : DISK_VERSION) && put(f.f, key) && + put(f.f, mb.data(), mb.size()) && (media == 0 || put(f.f, media)) && put(f.f, (uint64_t) tokens.size()) && put(f.f, tokens.data(), tokens.size() * sizeof(llama_token)) && put(f.f, (uint64_t) state.prompt.checkpoints.size()); for (const auto & ckpt : state.prompt.checkpoints) { @@ -3166,7 +3245,7 @@ bool write_file(const std::string & path, uint64_t key, const llama_tokens & tok } // namespace void server_prompt_cache::disk_rewrite_meta_locked(disk_entry & e) { - if (e.version == DISK_VERSION) { + if (e.version >= DISK_VERSION) { if (!write_meta(e.path, e.meta)) { SRV_WRN("prompt cache: cannot update metadata of %s\n", e.path.c_str()); } @@ -3179,20 +3258,20 @@ void server_prompt_cache::disk_rewrite_meta_locked(disk_entry & e) { // rewrites a v1 file as v2 so its metadata can be stored void server_prompt_cache::disk_upgrade(const std::string & path) { server_prompt_cache_state state; - if (!disk_read(path, false, state)) { + if (!disk_read(path, state)) { return; } cache_meta meta; { std::lock_guard lock(disk_mutex); auto it = std::find_if(disk.begin(), disk.end(), [&](const disk_entry & e) { return e.path == path; }); - if (it == disk.end() || it->version == DISK_VERSION) { + if (it == disk.end() || it->version != 1) { return; } meta = it->meta; } - const llama_tokens tokens = raw_tokens(state.prompt.tokens); - const bool ok = write_file(path, disk_key, tokens, state, meta); + // v1 files hold text prompts only + const bool ok = write_file(path, disk_key, 0, raw_tokens(state.prompt.tokens), state, meta); std::lock_guard lock(disk_mutex); auto it = std::find_if(disk.begin(), disk.end(), [&](const disk_entry & e) { return e.path == path; }); @@ -3267,7 +3346,8 @@ void server_prompt_cache::disk_evict_locked(const std::string & newest) { void server_prompt_cache::disk_write(server_prompt_cache_state & state) { namespace fs = std::filesystem; - const llama_tokens tokens = raw_tokens(state.prompt.tokens); + const server_tokens & tokens = state.prompt.tokens; + const bool media = has_media(tokens); cache_meta meta; { std::lock_guard lock(disk_mutex); @@ -3285,10 +3365,12 @@ void server_prompt_cache::disk_write(server_prompt_cache_state & state) { } const int64_t t_start = ggml_time_us(); - const uint64_t h = fnv1a(tokens.data(), tokens.size() * sizeof(llama_token)); + // the name hashes the media chunks too: prompts with equal text and other images get files of their own + const llama_tokens words = disk_tokens(tokens); + const uint64_t h = fnv1a(words.data(), words.size() * sizeof(llama_token)); const std::string path = string_format("%s/%016llx-%zu.kvc", disk_dir.c_str(), (unsigned long long) h, tokens.size()); - if (!write_file(path, disk_key, tokens, state, meta)) { + if (!write_file(path, disk_key, media ? disk_media : 0, words, state, meta)) { SRV_ERR("prompt cache: failed to write %s\n", path.c_str()); return; } @@ -3311,7 +3393,7 @@ void server_prompt_cache::disk_write(server_prompt_cache_state & state) { } } disk.remove_if([&](const disk_entry & e) { return e.path == path; }); - disk.push_back({ path, tokens, size, latest, DISK_VERSION }); + disk.push_back({ path, tokens.clone(), size, latest, media ? DISK_VERSION_MEDIA : DISK_VERSION }); if (meta_encode(latest) != meta_encode(meta)) { set_expiry(disk.back().meta); write_meta(path, disk.back().meta); @@ -3324,8 +3406,8 @@ void server_prompt_cache::disk_write(server_prompt_cache_state & state) { total += e.size; } - SRV_INF("prompt cache: wrote %zu tokens (%s, %s) to disk, %.1f MiB in %.2f s (%zu prompts, %.1f GiB)\n", - tokens.size(), hex_id(meta.id).c_str(), cache_class_name(meta.cls), size / (1024.0*1024.0), (ggml_time_us() - t_start) / 1e6, + SRV_INF("prompt cache: wrote %zu tokens, %lld images (%s, %s) to disk, %.1f MiB in %.2f s (%zu prompts, %.1f GiB)\n", + tokens.size(), (long long) std::max(0, media_count(tokens)), hex_id(meta.id).c_str(), cache_class_name(meta.cls), size / (1024.0*1024.0), (ggml_time_us() - t_start) / 1e6, disk.size(), total / (1024.0*1024.0*1024.0)); } @@ -3336,7 +3418,7 @@ json server_prompt_cache::info() { { tier_report rep; for (const auto & s : states) { - rep.add(s.prompt.meta, s.prompt.tokens.size(), s.size(), now); + rep.add(s.prompt.meta, s.prompt.tokens.size(), std::max(0, media_count(s.prompt.tokens)), s.size(), now); } rep.to_json(ram); } @@ -3359,15 +3441,18 @@ json server_prompt_cache::info() { std::lock_guard lock(disk_mutex); tier_report rep; size_t n_v1 = 0; + size_t n_media = 0; for (const auto & e : disk) { - rep.add(e.meta, e.tokens.size(), e.size, now); + rep.add(e.meta, e.tokens.size(), std::max(0, media_count(e.tokens)), e.size, now); n_v1 += e.version == 1; + n_media += e.version == DISK_VERSION_MEDIA; } json d = { { "dir", disk_dir }, { "limit_bytes", disk_limit }, { "pending", pending.size() }, { "v1_files", n_v1 }, + { "image_files", n_media }, }; rep.to_json(d); res["disk"] = d; @@ -3396,18 +3481,18 @@ size_t server_prompt_cache::clear_disk(bool include_pinned) { return n; } -bool server_prompt_cache::disk_read(const std::string & path, bool has_mtmd, server_prompt_cache_state & out) const { +bool server_prompt_cache::disk_read(const std::string & path, server_prompt_cache_state & out) const { disk_file f(path, "rb"); if (!f.f) { return false; } setvbuf(f.f, nullptr, _IOFBF, 8u << 20); - llama_tokens tokens; + server_tokens tokens; uint32_t version; cache_meta meta; uint64_t n_ckpt; - if (!read_header(f.f, disk_key, tokens, version, meta) || !get(f.f, n_ckpt) || n_ckpt > 4096) { + if (!read_header(f.f, disk_key, disk_media, tokens, version, meta) || !get(f.f, n_ckpt) || n_ckpt > 4096) { return false; } std::list checkpoints; @@ -3425,7 +3510,7 @@ bool server_prompt_cache::disk_read(const std::string & path, bool has_mtmd, ser return false; } - out.prompt.tokens = server_tokens(tokens, has_mtmd); + out.prompt.tokens = std::move(tokens); out.prompt.checkpoints = std::move(checkpoints); out.prompt.meta = std::move(meta); out.data = std::move(data); diff --git a/tools/server/server-task.h b/tools/server/server-task.h index 21554dfe..2575c9cf 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -741,17 +741,19 @@ struct server_prompt_cache { bool load_entry(std::list::iterator it, server_prompt & prompt, llama_context * ctx_tgt, llama_context * ctx_dft, int32_t id_slot, bool keep_entry); // the disk entry whose tokens score highest, above n_best (which it raises to that score); empty if none - std::string disk_best(const std::function & score, int32_t & n_best); + std::string disk_best(const std::function & score, int32_t & n_best); // reads a disk entry into RAM and returns it, or states.end() if the file cannot be read (it is then removed) - std::list::iterator disk_promote(const std::string & path, bool has_mtmd); + std::list::iterator disk_promote(const std::string & path); void update(); // second tier on disk: an entry the RAM limit evicts is written to dir instead of dropped, and read back // when it is the best prefix match for a new prompt; flush() writes what is still in RAM (on shutdown). - // key names the model and cache layout; files written under another key are deleted. text prompts only. - void disk_init(const std::string & dir, int32_t limit_mib, const std::string & key); + // key names the model and cache layout; files written under another key are deleted. media names the + // multimodal projector and its image settings (empty without one): a prompt with images is stored with its + // media chunk map (v3) and only read back under the same media key. + void disk_init(const std::string & dir, int32_t limit_mib, const std::string & key, const std::string & media); void flush(); @@ -790,11 +792,11 @@ struct server_prompt_cache { private: struct disk_entry { - std::string path; - llama_tokens tokens; - size_t size; - cache_meta meta; - uint32_t version; + std::string path; + server_tokens tokens; // media chunks are placeholders + size_t size; + cache_meta meta; + uint32_t version; }; struct family_stat { @@ -820,6 +822,7 @@ private: std::string disk_dir; size_t disk_limit = 0; uint64_t disk_key = 0; + uint64_t disk_media = 0; // 0 = no multimodal projector // oldest first; the lists and stop are guarded by disk_mutex std::list disk; @@ -860,7 +863,7 @@ private: void disk_write_loop(); void disk_write(server_prompt_cache_state & state); void disk_upgrade(const std::string & path); - bool disk_read(const std::string & path, bool has_mtmd, server_prompt_cache_state & out) const; + bool disk_read(const std::string & path, server_prompt_cache_state & out) const; void disk_evict_locked(const std::string & newest); void disk_rewrite_meta_locked(disk_entry & e);