From 34be26e2074c803beec5bba82fee39208fb37856 Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 17:43:32 -0700 Subject: [PATCH 01/29] refactor media_object is_audio flag into media type enum --- gpttype_adapter.cpp | 12 ++++++------ otherarch/otherarch.h | 5 +++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index a93b1eb63ca7..a09b1bb5f8ea 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -2560,7 +2560,7 @@ static bool kcpp_parse_attached_media_placeholder( { candidate = image_count + number - 1; } - else if(number <= (int) media_objects.size() && media_objects[number - 1].is_audio) + else if(number <= (int) media_objects.size() && media_objects[number - 1].mediatype == MEDIA_TYPE_AUDIO) { candidate = number - 1; } @@ -2571,7 +2571,7 @@ static bool kcpp_parse_attached_media_placeholder( { candidate = number - 1; } - else if(number <= (int) media_objects.size() && !media_objects[number - 1].is_audio) + else if(number <= (int) media_objects.size() && media_objects[number - 1].mediatype == MEDIA_TYPE_IMAGE) { candidate = number - 1; } @@ -5325,7 +5325,7 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int { std::string media_obj = media_objects[i].b64data; const std::vector media_data_buffer = kcpp_base64_decode(media_obj); - mtmd::bitmap bitmap(media_objects[i].is_audio + mtmd::bitmap bitmap(media_objects[i].mediatype == MEDIA_TYPE_AUDIO ? mtmd_helper_bitmap_init_from_buf(mtmd_ctx, media_data_buffer.data(), media_data_buffer.size(),false).bitmap : kcpp_mtmd_bitmap_init_image_from_buf(media_data_buffer.data(), media_data_buffer.size(), vision_max_res)); if(!bitmap.ptr) @@ -5377,7 +5377,7 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int continue; } media_chunk chunk; - chunk.is_audio = media_objects[i].is_audio; + chunk.mediatype = media_objects[i].mediatype; chunk.mtmd_chunk = mtmd_input_chunk_copy(mtmdchunk); chunk.clp_image_tokens = mtmd_input_chunk_get_n_pos(mtmdchunk); mediatokensneeded += chunk.clp_image_tokens; @@ -5706,7 +5706,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) { media_object lv; lv.b64data = item; - lv.is_audio = false; + lv.mediatype = MEDIA_TYPE_IMAGE; TokenizeString("\n\n", lv.chunk_end_seq, file_format, false); media_objects.push_back(lv); new_media_composite += item; @@ -5719,7 +5719,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) { media_object lv; lv.b64data = item; - lv.is_audio = true; + lv.mediatype = MEDIA_TYPE_AUDIO; TokenizeString("\n\n", lv.chunk_end_seq, file_format, false); media_objects.push_back(lv); new_media_composite += item; diff --git a/otherarch/otherarch.h b/otherarch/otherarch.h index 41ff95ace4a1..a03d52531b79 100644 --- a/otherarch/otherarch.h +++ b/otherarch/otherarch.h @@ -509,9 +509,10 @@ struct mpt_model { std::map tensors; }; +enum MediaType { MEDIA_TYPE_IMAGE, MEDIA_TYPE_AUDIO, MEDIA_TYPE_VIDEO }; struct media_chunk { - bool is_audio = false; //if true its audio, otherwise its vision + MediaType mediatype = MEDIA_TYPE_IMAGE; void * mtmd_chunk = nullptr; // mtmd_input_chunk, owned by this chunk int32_t clp_image_tokens = 0; //holds number of tokens used in this chunk int32_t nx = 0; //only used for 2d roped images @@ -521,7 +522,7 @@ struct media_object { std::string b64data = ""; std::vector mediachunks; - bool is_audio = false; //if true its audio, otherwise its vision + MediaType mediatype = MEDIA_TYPE_IMAGE; std::vector chunk_start_seq; std::vector chunk_end_seq; }; From 04df7a266284a6ebd2d125f07c286d4567881b99 Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 17:52:10 -0700 Subject: [PATCH 02/29] add video fields to generation_inputs and load_model_inputs contract --- expose.h | 7 +++++++ koboldcpp.py | 13 +++++++++++++ 2 files changed, 20 insertions(+) diff --git a/expose.h b/expose.h index 56d7c0317918..44179cf27fd6 100644 --- a/expose.h +++ b/expose.h @@ -49,6 +49,11 @@ struct load_model_inputs const int visionmaxres = 2048; const int visionmintokens = -1; const int visionmaxtokens = -1; + const int videomaxframes = 32; + const float videofps = 2.0f; + const int videomintokens = -1; + const int videomaxtokens = -1; + const char * ffmpegpath = ""; const bool use_mmap = false; const bool use_mlock = false; const bool use_mtp = false; @@ -100,6 +105,8 @@ struct generation_inputs const char ** images = nullptr; const int audio_len = 0; const char ** audio = nullptr; + const int videos_len = 0; + const char ** videos = nullptr; const int max_context_length = 0; const int max_length = 0; const float temperature = 0.0f; diff --git a/koboldcpp.py b/koboldcpp.py index ae378e959065..084fd0c2c20e 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -48,6 +48,7 @@ tensor_split_max = 16 images_max = 16 audio_max = 16 +videos_max = 2 bias_min_value = -100.0 bias_max_value = 100.0 logprobs_max = 10 @@ -271,6 +272,11 @@ class load_model_inputs(ctypes.Structure): ("visionmaxres", ctypes.c_int), ("visionmintokens", ctypes.c_int), ("visionmaxtokens", ctypes.c_int), + ("videomaxframes", ctypes.c_int), + ("videofps", ctypes.c_float), + ("videomintokens", ctypes.c_int), + ("videomaxtokens", ctypes.c_int), + ("ffmpegpath", ctypes.c_char_p), ("use_mmap", ctypes.c_bool), ("use_mlock", ctypes.c_bool), ("use_mtp", ctypes.c_bool), @@ -321,6 +327,8 @@ class generation_inputs(ctypes.Structure): ("images", ctypes.POINTER(ctypes.c_char_p)), ("audio_len", ctypes.c_int), ("audio", ctypes.POINTER(ctypes.c_char_p)), + ("videos_len", ctypes.c_int), + ("videos", ctypes.POINTER(ctypes.c_char_p)), ("max_context_length", ctypes.c_int), ("max_length", ctypes.c_int), ("temperature", ctypes.c_float), @@ -1994,6 +2002,11 @@ def load_model(model_filename): vmaxtk = max(vmintk,vmaxtk) inputs.visionmintokens = vmintk inputs.visionmaxtokens = vmaxtk + inputs.videomaxframes = 32 + inputs.videofps = 2.0 + inputs.videomintokens = -1 + inputs.videomaxtokens = -1 + inputs.ffmpegpath = "".encode("UTF-8") inputs.use_smartcontext = args.smartcontext if args.parallelrequests > 1 and not args.noshift: print("\nWarning: Continuous batching is enabled, so context shifting has been disabled automatically.\n") From 60adf8b759584639988b59330d610abb8475927d Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 17:52:21 -0700 Subject: [PATCH 03/29] marshal videos genparam into generation_inputs struct --- koboldcpp.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/koboldcpp.py b/koboldcpp.py index 084fd0c2c20e..f8e335bcb4e6 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -1130,7 +1130,7 @@ def is_incomplete_utf8_sequence(byte_seq): #note, this will only flag INCOMPLETE def strip_base64_prefix(encoded_data): if not encoded_data: return "" - if encoded_data.startswith("data:image") or encoded_data.startswith("data:audio"): + if encoded_data.startswith("data:image") or encoded_data.startswith("data:audio") or encoded_data.startswith("data:video"): encoded_data = encoded_data.split(',', 1)[-1] return encoded_data @@ -2119,6 +2119,7 @@ def generate(genparams, stream_flag=False): guidance_scale = tryparsefloat(genparams.get('guidance_scale', 1.0),1.0) images = genparams.get('images', []) audio = genparams.get('audio', []) + videos = genparams.get('videos', []) max_context_length = tryparseint(genparams.get('max_context_length', maxctx),maxctx) max_length = tryparseint(genparams.get('max_length', args.defaultgenamt),args.defaultgenamt) temperature = tryparsefloat(genparams.get('temperature', adapter_obj.get("temperature", 0.7)),0.7) @@ -2199,6 +2200,11 @@ def generate(genparams, stream_flag=False): inputs.audio = (ctypes.c_char_p * inputs.audio_len)() for n, item in enumerate(audio): inputs.audio[n] = item.encode("UTF-8") + videos = videos[-videos_max:] + inputs.videos_len = len(videos) + inputs.videos = (ctypes.c_char_p * inputs.videos_len)() + for n, item in enumerate(videos): + inputs.videos[n] = strip_base64_prefix(item).encode("UTF-8") global showmaxctxwarning if max_context_length > maxctx: From 300165330a4a0c092f3e1a8f68cb3539bcbe9f53 Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 17:52:39 -0700 Subject: [PATCH 04/29] populate video media_objects and skip video eval safely in gpttype_adapter --- gpttype_adapter.cpp | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index a09b1bb5f8ea..aae004577785 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -4379,7 +4379,7 @@ static bool batch_inputs_eligible(const generation_inputs & inputs) { return false; } - if(draft_ctx || guidance_ctx || inputs.images_len>0 || inputs.audio_len>0) + if(draft_ctx || guidance_ctx || inputs.images_len>0 || inputs.audio_len>0 || inputs.videos_len>0) { return false; } @@ -4391,7 +4391,7 @@ static bool batch_inputs_eligible(const generation_inputs & inputs) { return false; } - if(inputs.images_len > 0 || inputs.audio_len > 0 || inputs.guidance_scale != 1.0f) + if(inputs.images_len > 0 || inputs.audio_len > 0 || inputs.videos_len > 0 || inputs.guidance_scale != 1.0f) { return false; } @@ -5323,6 +5323,12 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int for(int i=0;i media_data_buffer = kcpp_base64_decode(media_obj); mtmd::bitmap bitmap(media_objects[i].mediatype == MEDIA_TYPE_AUDIO @@ -5725,6 +5731,19 @@ generation_outputs gpttype_generate(const generation_inputs inputs) new_media_composite += item; } } + for(int x=0;x Date: Wed, 22 Jul 2026 17:57:43 -0700 Subject: [PATCH 05/29] enable MTMD_VIDEO and probe video capability at load --- CMakeLists.txt | 1 + Makefile | 2 +- gpttype_adapter.cpp | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b3f782aafd64..e5c461565204 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,6 +62,7 @@ add_compile_definitions(GGML_USE_CPU_REPACK) add_compile_definitions(GGML_USE_RPC) add_compile_definitions(NOMINMAX) add_compile_definitions(_REGEX_MAX_STACK_COUNT=5000) +add_compile_definitions(MTMD_VIDEO) if (GGML_HIP_FORCE_ROCWMMA_FATTN_GFX12) add_compile_definitions(GGML_HIP_ROCWMMA_FATTN_GFX12) diff --git a/Makefile b/Makefile index 3402953c2b4b..6399d2af77c9 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ CFLAGS += -fsanitize=undefined -fsanitize-undefined-trap-on-error CXXFLAGS += -fsanitize=undefined -fsanitize-undefined-trap-on-error endif CFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c11 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC -CXXFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c++17 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC +CXXFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c++17 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC -DMTMD_VIDEO ifndef KCPP_DEBUG CFLAGS += -DNDEBUG -s diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index aae004577785..8540ac5b32eb 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -87,6 +87,7 @@ int speculative_chunk_amt = 4; //do it in chunks of this many tokens bool generation_finished; bool audio_multimodal_supported = false; bool vision_multimodal_supported = false; +bool video_multimodal_supported = false; float last_process_time = 0; float last_eval_time = 0; int last_token_count = 0; @@ -3016,6 +3017,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in } audio_multimodal_supported = false; vision_multimodal_supported = false; + video_multimodal_supported = false; use_mrope = false; overridden_jinja_template = inputs.jinja_template; @@ -3658,6 +3660,11 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in } vision_multimodal_supported = mtmd_support_vision(mtmd_ctx); audio_multimodal_supported = mtmd_support_audio(mtmd_ctx); + video_multimodal_supported = mtmd_helper_support_video(mtmd_ctx); + if(vision_multimodal_supported && video_multimodal_supported) + { + printf("Video input enabled (requires ffmpeg/ffprobe on PATH at runtime).\n"); + } } const llama_vocab * tmpvocab = llama_model_get_vocab(llamamodel); From c7deb6240cd4bfcf9785a2745234f9e194b88fdc Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 17:59:24 -0700 Subject: [PATCH 06/29] accept video content parts in chat completions paths --- embd_res/kcpp_docs.embd | 7 +++++++ koboldcpp.py | 41 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/embd_res/kcpp_docs.embd b/embd_res/kcpp_docs.embd index ed44c09f9a15..e4afe4783bf7 100644 --- a/embd_res/kcpp_docs.embd +++ b/embd_res/kcpp_docs.embd @@ -197,6 +197,13 @@ "type": "string" } }, + "videos": { + "description": "KoboldCpp ONLY. If set, takes an array of up to 2 base64 encoded strings, each one representing a video to be processed.", + "type": "array", + "items": { + "type": "string" + } + }, "trim_stop": { "default": true, "description": "KoboldCpp ONLY. If true, also removes detected stop_sequences from the output and truncates all text after them. If false, output will also include stop sequence and potentially a few additional characters.", diff --git a/koboldcpp.py b/koboldcpp.py index f8e335bcb4e6..de7adfd93794 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -2396,7 +2396,7 @@ def continuous_batching_python_eligible(genparams, api_format): if not getattr(args, "noshift", False) or getattr(args, "smartcontext", False) or getattr(args, "draftmodel", "") or getattr(args, "usemtp", False) or getattr(args, "enableguidance", False): utfprint("Batching disabled due to loaded settings",2) return False - if genparams.get("negative_prompt") or genparams.get("images") or genparams.get("audio"): + if genparams.get("negative_prompt") or genparams.get("images") or genparams.get("audio") or genparams.get("videos"): utfprint("Batching disabled due to media",2) return False if genparams.get("grammar") or genparams.get("grammar_retain_state") or genparams.get("banned_tokens") or genparams.get("banned_strings"): @@ -3848,6 +3848,9 @@ def parse(self, parser): elif item.get("type")=="input_audio": turn_text += f"\n(Attached Audio {mediacount})\n" mediacount += 1 + elif item.get("type")=="video_url" or item.get("type")=="input_video": + turn_text += f"\n(Attached Video {mediacount})\n" + mediacount += 1 else: normalized.append(item) if turn_text: @@ -4121,9 +4124,10 @@ def determine_tool_json_to_use(genparams, curr_ctx, assistant_message_start, is_ images_added = [] #sometimes images are needed to make a decision too audio_added = [] + videos_added = [] if messages: - images_added, audio_added = sweep_media_from_messages(messages) + images_added, audio_added, videos_added = sweep_media_from_messages(messages) reversed_messages = list(reversed(messages)) for message in reversed_messages: if message["role"] == "user": @@ -4181,6 +4185,8 @@ def determine_tool_json_to_use(genparams, curr_ctx, assistant_message_start, is_ temp_poll["images"] = images_added if len(audio_added)>0: temp_poll["audio"] = audio_added + if len(videos_added)>0: + temp_poll["videos"] = videos_added temp_poll_result = generate(genparams=temp_poll) temp_poll_text = temp_poll_result['text'].strip().rstrip('.') temp_poll_data_arr = extract_json_from_string(temp_poll_text) @@ -4240,6 +4246,7 @@ def compress_tools_array(tools_array): def sweep_media_from_messages(messages_array): images = [] audio = [] + videos = [] for message in messages_array: curr_content = message.get("content", None) if isinstance(curr_content, list): @@ -4252,6 +4259,14 @@ def sweep_media_from_messages(messages_array): data = item.get("input_audio", {}).get("data") if data: audio.append(data) + elif item.get("type") == "video_url": + url = item.get("video_url", {}).get("url", "") + if url.startswith("data:video"): + videos.append(url.split(",", 1)[1]) + elif item.get("type") == "input_video": + data = item.get("input_video", {}).get("data") + if data: + videos.append(data) elif message.get("role", "")=="tool" and isinstance(curr_content, str): #handle mcp returned images try: mcp_pl = json.loads(curr_content) @@ -4265,7 +4280,7 @@ def sweep_media_from_messages(messages_array): if imgs_ollama: for img in imgs_ollama: images.append(img) - return images, audio + return images, audio, videos def transform_genparams(genparams, api_format, use_jinja): @@ -4357,6 +4372,7 @@ def transform_genparams(genparams, api_format, use_jinja): tools_message_end = adapter_obj.get("tools_end", "") images_added = [] audio_added = [] + videos_added = [] continue_assistant_turn = genparams.get('continue_assistant_turn', True) latest_turn_was_assistant = False latest_turn_was_tool = False @@ -4390,6 +4406,7 @@ def transform_genparams(genparams, api_format, use_jinja): message_index = 0 attachedimgid = 0 attachedaudid = 0 + attachedvidid = 0 jinja_output = None jinjatools = genparams.get('tools', []) if use_jinja and cached_chat_template: @@ -4412,7 +4429,7 @@ def transform_genparams(genparams, api_format, use_jinja): if jinjatools and len(jinjatools)>0: genparams["using_openai_tools"] = True # handle media - images_added, audio_added = sweep_media_from_messages(messages_array) + images_added, audio_added, videos_added = sweep_media_from_messages(messages_array) else: if jinjatools: # inject the tools list at the top of the context window, even if context has shifted @@ -4481,6 +4498,16 @@ def transform_genparams(genparams, api_format, use_jinja): audio_added.append(item['input_audio']['data']) attachedaudid += 1 messages_string += f"\n(Attached Audio {attachedaudid})\n" + elif item['type']=="video_url": + if 'video_url' in item and item['video_url'] and item['video_url']['url'] and item['video_url']['url'].startswith("data:video"): + videos_added.append(item['video_url']['url'].split(",", 1)[1]) + attachedvidid += 1 + messages_string += f"\n(Attached Video {attachedvidid})\n" + elif item['type']=="input_video": + if 'input_video' in item and item['input_video'] and item['input_video']['data']: + videos_added.append(item['input_video']['data']) + attachedvidid += 1 + messages_string += f"\n(Attached Video {attachedvidid})\n" elif isinstance(item, str): messages_string += item # If item is just a string, append it directly @@ -4534,6 +4561,8 @@ def transform_genparams(genparams, api_format, use_jinja): genparams["images"] = images_added if len(audio_added)>0: genparams["audio"] = audio_added + if len(videos_added)>0: + genparams["videos"] = videos_added if len(genparams.get('stop_sequence', []))==0: #only set stop seq if it wont overwrite existing genparams["stop_sequence"] = [user_message_start.strip(),assistant_message_start.strip()] else: @@ -4610,6 +4639,9 @@ def transform_genparams(genparams, api_format, use_jinja): elif part.get("type") == "input_image": img = part.get("image_url", part.get("source", {})) parts.append({"type": "image_url", "image_url": {"url": img}}) + elif part.get("type") == "input_video": + vid = part.get("video_url", part.get("source", {})) + parts.append({"type": "video_url", "video_url": {"url": vid}}) content = parts converted.append({"role": role, "content": content}) elif isinstance(item, str): @@ -4672,6 +4704,7 @@ def normalize_anthropic_content_block(item): return [{"type": "text", "text": doc_text}] else: return [{"type": "text", "text": "(Attached Unknown Document)"}] + # Anthropic Messages API has no video content block type, so video is intentionally unsupported here. else: return [item] # pass through unknown types (including tool_use, tool_result - handled below) From 4a5963d6f2541fd27d1f67387a31522b0ea60071 Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 18:06:10 -0700 Subject: [PATCH 07/29] add video input CLI flags and budget warning Adds --videomaxframes, --videofps, --videomaxtokens, --videomintokens, and --ffmpegpath, wires them into load_model_inputs (replacing the previously hardcoded defaults), and prints an effective video config plus a context-budget warning when worst-case video tokens could exceed max context. GUI launcher deferred: the vision fields use a hand-tuned pixel-positioned grid (row/padx offsets, per-field tooltips, save/load/export plumbing) that isn't a trivial copy, so video stays CLI-only for now. --- koboldcpp.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/koboldcpp.py b/koboldcpp.py index de7adfd93794..211a20c5a2f0 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -2002,11 +2002,18 @@ def load_model(model_filename): vmaxtk = max(vmintk,vmaxtk) inputs.visionmintokens = vmintk inputs.visionmaxtokens = vmaxtk - inputs.videomaxframes = 32 - inputs.videofps = 2.0 - inputs.videomintokens = -1 - inputs.videomaxtokens = -1 - inputs.ffmpegpath = "".encode("UTF-8") + inputs.videomaxframes = args.videomaxframes if args.videomaxframes > 0 else 32 + inputs.videofps = args.videofps + vidmintk = args.videomintokens + vidmaxtk = args.videomaxtokens + vidmintk = -1 if vidmintk<-1 else vidmintk + vidmaxtk = -1 if vidmaxtk<-1 else vidmaxtk + if(vidmintk!=-1 or vidmaxtk!=-1) and (vidmintk==-1 or vidmaxtk==-1): #if exactly one of the args is -1 + vidmintk = max(vidmintk,vidmaxtk) + vidmaxtk = max(vidmintk,vidmaxtk) + inputs.videomintokens = vidmintk + inputs.videomaxtokens = vidmaxtk + inputs.ffmpegpath = args.ffmpegpath.encode("UTF-8") if args.ffmpegpath else "".encode("UTF-8") inputs.use_smartcontext = args.smartcontext if args.parallelrequests > 1 and not args.noshift: print("\nWarning: Continuous batching is enabled, so context shifting has been disabled automatically.\n") @@ -11929,6 +11936,13 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): has_audio_support = False has_vision_support = False + if args.mmproj and args.mmproj!="": + effective_video_max_tokens = args.videomaxtokens if args.videomaxtokens>0 else (args.visionmaxtokens if args.visionmaxtokens>0 else 256) + print(f"Video Config: maxframes={args.videomaxframes}, fps={args.videofps}, mintokens={args.videomintokens}, maxtokens={args.videomaxtokens} (effective maxtokens/frame: {effective_video_max_tokens})") + worst_case_video_tokens = args.videomaxframes * effective_video_max_tokens + if worst_case_video_tokens > maxctx: + print(f"Warning: Worst-case video token usage ({worst_case_video_tokens} = {args.videomaxframes} frames x {effective_video_max_tokens} tokens/frame) may exceed the max context size ({maxctx}). Consider lowering --videomaxframes, --videofps, or --videomaxtokens.") + if not loadok: exitcounter = 999 exit_with_error(3,"Could not load text model: " + modelname) @@ -12507,6 +12521,7 @@ def range_checker(arg: str): advparser.add_argument("--exportconfig", help="Exports the current selected arguments as a .kcpps settings file", metavar=('[filename]'), type=str, default="") advparser.add_argument("--exporttemplate", help="Exports the current selected arguments as a .kcppt template file", metavar=('[filename]'), type=str, default="") advparser.add_argument("--failsafe", help="Use failsafe mode, extremely old CPU compatibility mode that should work on all devices.", action='store_true') + advparser.add_argument("--ffmpegpath", metavar=('[directory]'), help="Specify a directory containing the ffmpeg/ffprobe binaries used for video frame extraction. If unset, searches PATH.", type=str, default="") advparser.add_argument("--foreground", help="Windows only. Sends the terminal to the foreground every time a new prompt is generated. This helps avoid some idle slowdown issues.", action='store_true') advparser.add_argument("--gendefaults", metavar=('{"parameter":"value",...}'), help="Sets extra default parameters for some fields in API requests, as a JSON string.", default="") advparser.add_argument("--gendefaultsoverwrite", help="Allow the gendefaults parameters to overwrite the original value in API payloads.", action='store_true') @@ -12567,6 +12582,10 @@ def range_checker(arg: str): advparser.add_argument("--usemlock","--mlock", help="Enables mlock, preventing the RAM used to load the model from being paged out. Not usually recommended.", action='store_true') compatgroup3 = advparser.add_mutually_exclusive_group() compatgroup3.add_argument("--usemmap", help="If set, uses mmap to load model.", action='store_true') + advparser.add_argument("--videofps", metavar=('[fps]'), help="Sets the video frame sampling rate in frames per second. Values <=0 use the video's native fps (default 2.0).", type=float, default=2.0) + advparser.add_argument("--videomaxframes", metavar=('[frames]'), help="Sets the maximum number of decoded frames per video (default 32).", type=int, default=32) + advparser.add_argument("--videomaxtokens", metavar=('[tokens]'), help="Override the maximum tokens per video frame for the MMProj embedding. If -1, inherits --visionmaxtokens (default -1).", type=int, default=-1) + advparser.add_argument("--videomintokens", metavar=('[tokens]'), help="Override the minimum tokens per video frame for the MMProj embedding. If -1, inherits --visionmintokens (default -1).", type=int, default=-1) advparser.add_argument("--visionmaxres", metavar=('[max px]'), help="Clamp MMProj vision maximum allowed resolution. Allowed values are between 512 to 2048 px (default 1024).", type=int, default=default_visionmaxres) advparser.add_argument("--visionmaxtokens","--image-max-tokens", metavar=('[tokens]'), help="Override the maximum tokens for the MMProj embedding (default -1).", type=int, default=-1) advparser.add_argument("--visionmintokens","--image-min-tokens", metavar=('[tokens]'), help="Override the minimum tokens for the MMProj embedding (default -1).", type=int, default=-1) From da1673f0a68731537a35d0edc300742856819e63 Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 18:07:39 -0700 Subject: [PATCH 08/29] reword --videomaxframes help as a sampling budget, not a truncation cap C++ side downsamples fps to spread the frame budget evenly across a video's full duration, falling back to truncation only if needed. --- koboldcpp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koboldcpp.py b/koboldcpp.py index 211a20c5a2f0..9ed01445ac11 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -12583,7 +12583,7 @@ def range_checker(arg: str): compatgroup3 = advparser.add_mutually_exclusive_group() compatgroup3.add_argument("--usemmap", help="If set, uses mmap to load model.", action='store_true') advparser.add_argument("--videofps", metavar=('[fps]'), help="Sets the video frame sampling rate in frames per second. Values <=0 use the video's native fps (default 2.0).", type=float, default=2.0) - advparser.add_argument("--videomaxframes", metavar=('[frames]'), help="Sets the maximum number of decoded frames per video (default 32).", type=int, default=32) + advparser.add_argument("--videomaxframes", metavar=('[frames]'), help="Sets the max frames sampled per video; longer videos are downsampled evenly across their full duration to fit (default 32).", type=int, default=32) advparser.add_argument("--videomaxtokens", metavar=('[tokens]'), help="Override the maximum tokens per video frame for the MMProj embedding. If -1, inherits --visionmaxtokens (default -1).", type=int, default=-1) advparser.add_argument("--videomintokens", metavar=('[tokens]'), help="Override the minimum tokens per video frame for the MMProj embedding. If -1, inherits --visionmintokens (default -1).", type=int, default=-1) advparser.add_argument("--visionmaxres", metavar=('[max px]'), help="Clamp MMProj vision maximum allowed resolution. Allowed values are between 512 to 2048 px (default 1024).", type=int, default=default_visionmaxres) From e98778645eb25a39211db7aaca6ee8f6d25aa04a Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 18:08:19 -0700 Subject: [PATCH 09/29] implement MTMD_VIDEO frame-streaming eval path in PrepareMediaEmbds Stream video frames via mtmd_helper_video, tokenize frames and timestamp texts into ordered media chunks, enforce videomaxframes cap and optional videomaxtokens per-frame pixel downscale. Extend the (Attached Video N) placeholder parser and wire videofps/videomaxframes/videomaxtokens/ffmpegpath globals. Guarded by MTMD_VIDEO with graceful skips on all failure paths. --- gpttype_adapter.cpp | 266 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 259 insertions(+), 7 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 8540ac5b32eb..349f1b3b8f41 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -144,6 +144,10 @@ static std::vector media_object_token_counts; //per media object dummy toke static std::string media_composite_image_signature = ""; //for identifying when the media changes, we need to invalidate the cache static int current_media_identifier = MEDIA_TOKEN_IDENTIFIER_A; static int vision_max_res = 2048; +static float video_fps = 2.0f; +static int video_max_frames = 32; +static int video_max_tokens = -1; //-1 = inherit vision budget via shared clip context; >0 = explicit per-frame token cap +static std::string ffmpeg_path = ""; //empty = search PATH static bool use_mrope = false; static kcpp_params * kcpp_data = nullptr; @@ -2515,23 +2519,31 @@ static bool kcpp_parse_attached_media_placeholder( size_t pos, int image_count, int audio_count, + int video_count, int & media_index, size_t & placeholder_len) { const std::string image_prefix = "(Attached Image "; const std::string audio_prefix = "(Attached Audio "; - bool is_audio = false; + const std::string video_prefix = "(Attached Video "; + MediaType mtype = MEDIA_TYPE_IMAGE; size_t prefix_len = 0; if(prompt.compare(pos, image_prefix.size(), image_prefix) == 0) { + mtype = MEDIA_TYPE_IMAGE; prefix_len = image_prefix.size(); } else if(prompt.compare(pos, audio_prefix.size(), audio_prefix) == 0) { - is_audio = true; + mtype = MEDIA_TYPE_AUDIO; prefix_len = audio_prefix.size(); } + else if(prompt.compare(pos, video_prefix.size(), video_prefix) == 0) + { + mtype = MEDIA_TYPE_VIDEO; + prefix_len = video_prefix.size(); + } else { return false; @@ -2555,7 +2567,7 @@ static bool kcpp_parse_attached_media_placeholder( } int candidate = -1; - if(is_audio) + if(mtype == MEDIA_TYPE_AUDIO) { if(number <= audio_count) { @@ -2566,6 +2578,17 @@ static bool kcpp_parse_attached_media_placeholder( candidate = number - 1; } } + else if(mtype == MEDIA_TYPE_VIDEO) + { + if(number <= video_count) + { + candidate = image_count + audio_count + number - 1; + } + else if(number <= (int) media_objects.size() && media_objects[number - 1].mediatype == MEDIA_TYPE_VIDEO) + { + candidate = number - 1; + } + } else { if(number <= image_count) @@ -2607,7 +2630,8 @@ static bool kcpp_tokenize_prompt_with_inline_media( FileFormat file_format, bool add_bos, int image_count, - int audio_count) + int audio_count, + int video_count) { output_tokens.clear(); bool inserted_media = false; @@ -2630,7 +2654,7 @@ static bool kcpp_tokenize_prompt_with_inline_media( { int media_index = -1; size_t placeholder_len = 0; - if(kcpp_parse_attached_media_placeholder(prompt, pos, image_count, audio_count, media_index, placeholder_len)) + if(kcpp_parse_attached_media_placeholder(prompt, pos, image_count, audio_count, video_count, media_index, placeholder_len)) { append_text(text_start, pos); if(add_bos && !emitted_anything) @@ -2985,6 +3009,10 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in kcpp_data->vision_min_tokens = inputs.visionmintokens; kcpp_data->vision_max_tokens = inputs.visionmaxtokens; vision_max_res = inputs.visionmaxres; + video_fps = inputs.videofps; + video_max_frames = inputs.videomaxframes; + video_max_tokens = inputs.videomaxtokens; //videomintokens has no separate action; per-frame min is inherited via the shared clip context + ffmpeg_path = (inputs.ffmpegpath ? inputs.ffmpegpath : ""); if(isGguf && kcpp_pipeline_parallelism) { //double the logical batch, while keeping the physical batch the same, pipeline parallel set GGML_SCHED_MAX_COPIES to 2 @@ -5318,6 +5346,48 @@ static mtmd_bitmap * kcpp_mtmd_bitmap_init_image_from_buf(const unsigned char * return bitmap; } +#ifdef MTMD_VIDEO +//downscale an already-decoded RGB video frame so its larger dimension does not exceed maxdims. +//takes ownership of the input bitmap; returns a new bitmap (or the original if no resize was needed). +static mtmd_bitmap * kcpp_downscale_bitmap(mtmd_bitmap * bitmap, int maxdims) +{ + if(bitmap == nullptr || maxdims <= 0) + { + return bitmap; + } + int nx = (int)mtmd_bitmap_get_nx(bitmap); + int ny = (int)mtmd_bitmap_get_ny(bitmap); + if(nx <= maxdims && ny <= maxdims) + { + return bitmap; + } + const float aspect_ratio = static_cast(nx) / ny; + int new_width = nx; + int new_height = ny; + if(aspect_ratio > 1.0f) + { + new_width = maxdims; + new_height = std::max(1, static_cast(maxdims / aspect_ratio)); + } + else + { + new_height = maxdims; + new_width = std::max(1, static_cast(maxdims * aspect_ratio)); + } + uint8_t * resized_image = (uint8_t *)malloc((size_t)new_width * new_height * 3); + if(resized_image != nullptr && stbir_resize_uint8(mtmd_bitmap_get_data(bitmap), nx, ny, 0, resized_image, new_width, new_height, 0, 3)) + { + mtmd_bitmap * newbmp = mtmd_bitmap_init(new_width, new_height, resized_image); + free(resized_image); + mtmd_bitmap_free(bitmap); + return newbmp; + } + printf("\nWarning: MTMD video frame resize failed, using original frame."); + free(resized_image); + return bitmap; +} +#endif + //this function prepares the mtmd chunks for media. it's only needed when media changes static void PrepareMediaEmbds(const int nctx, const std::vector & media_intro, const std::vector & media_outro) { @@ -5332,9 +5402,190 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int { if(media_objects[i].mediatype == MEDIA_TYPE_VIDEO) { +#ifdef MTMD_VIDEO + const std::vector video_data_buffer = kcpp_base64_decode(media_objects[i].b64data); + mtmd_helper_video_init_params vparams = mtmd_helper_video_init_params_default(); + vparams.fps_target = video_fps; + vparams.ffmpeg_bin_dir = ffmpeg_path.empty() ? nullptr : ffmpeg_path.c_str(); + mtmd_helper::video_ptr vctx(mtmd_helper_video_init_from_buf(mtmd_ctx, video_data_buffer.data(), video_data_buffer.size(), vparams)); + if(!vctx) + { + media_object_token_counts.push_back(0); + printf("\nError: MTMD video %d failed - ffmpeg/ffprobe not found or unreadable video, skipping.", i); + continue; + } + + int mediatokensneeded = 0; + bool seen_media_embedding = false; + bool used_fallback_boundary_tokens = false; + std::vector fallback_start_seq; + std::vector fallback_end_seq; + int frames_processed = 0; + bool frame_cap_hit = false; + + //frames inherit the image resolution clamp; when an explicit video token budget is set we tighten it. + //no public patch-size accessor is reachable from this TU, so approximate the token budget with a + //conservative 14px patch to derive a pixel cap (otherwise frames just use vision_max_res). + int frame_max_res = vision_max_res; + if(video_max_tokens > 0) + { + const int patch_px = 14; + int tok_res = (int)(std::sqrt((double)video_max_tokens) * patch_px); + if(tok_res > 0 && (frame_max_res <= 0 || tok_res < frame_max_res)) + { + frame_max_res = tok_res; + } + } + + auto append_video_chunks = [&](mtmd::input_chunks & chunks) + { + for(size_t j=0;j fallback_tokens; + TokenizeString(seen_media_embedding ? "" : "", fallback_tokens, file_format, false); + if(fallback_tokens.size() > 0) + { + if(seen_media_embedding) + { + fallback_end_seq.insert(fallback_end_seq.end(), fallback_tokens.begin(), fallback_tokens.end()); + } + else + { + fallback_start_seq.insert(fallback_start_seq.end(), fallback_tokens.begin(), fallback_tokens.end()); + } + } + used_fallback_boundary_tokens = true; + continue; + } + media_chunk chunk; + chunk.mediatype = media_objects[i].mediatype; + chunk.mtmd_chunk = mtmd_input_chunk_copy(mtmdchunk); + chunk.clp_image_tokens = mtmd_input_chunk_get_n_pos(mtmdchunk); + mediatokensneeded += chunk.clp_image_tokens; + media_objects[i].mediachunks.push_back(chunk); + if(mtmd_input_chunk_get_type(mtmdchunk) != MTMD_INPUT_CHUNK_TYPE_TEXT) + { + seen_media_embedding = true; + } + } + }; + + while(true) + { + mtmd_bitmap * raw_bitmap = nullptr; + char * raw_text = nullptr; + int32_t rd = mtmd_helper_video_read_next(vctx.get(), &raw_bitmap, &raw_text); + if(rd == -1) //EOF + { + break; + } + if(rd == -2) //error - keep already-collected chunks + { + printf("\nWarning: MTMD video %d read error, stopping with %d frames collected.", i, frames_processed); + break; + } + if(raw_bitmap != nullptr) + { + if(frames_processed >= video_max_frames) + { + mtmd_bitmap_free(raw_bitmap); + frame_cap_hit = true; + break; + } + mtmd::bitmap framebmp(kcpp_downscale_bitmap(raw_bitmap, frame_max_res)); + if(!framebmp.ptr) + { + printf("\nWarning: MTMD video %d frame %d failed to prepare, skipping frame.", i, frames_processed); + continue; + } + mtmd_input_text inp_txt = { + mtmd_default_marker(), + /* add_special */ false, + /* parse_special */ true, + }; + mtmd::input_chunks chunks(mtmd_input_chunks_init()); + std::vector bitmaps = { framebmp.ptr.get() }; + int32_t tokenized = mtmd_tokenize(mtmd_ctx, chunks.ptr.get(), &inp_txt, bitmaps.data(), bitmaps.size()); + if(tokenized != 0) + { + printf("\nWarning: MTMD video %d frame %d failed to tokenize (status %d), skipping frame.", i, frames_processed, tokenized); + continue; + } + append_video_chunks(chunks); + ++frames_processed; + } + else if(raw_text != nullptr) + { + std::string tstext = raw_text; + free(raw_text); + mtmd_input_text inp_txt = { + tstext.c_str(), + /* add_special */ false, + /* parse_special */ false, + }; + mtmd::input_chunks chunks(mtmd_input_chunks_init()); + std::vector nobitmaps; + int32_t tokenized = mtmd_tokenize(mtmd_ctx, chunks.ptr.get(), &inp_txt, nobitmaps.data(), nobitmaps.size()); + if(tokenized != 0) + { + printf("\nWarning: MTMD video %d timestamp failed to tokenize (status %d), skipping.", i, tokenized); + continue; + } + append_video_chunks(chunks); + } + } + + if(frame_cap_hit) + { + printf("\nvideo truncated to %d frames (videomaxframes)", frames_processed); + } + if(fallback_start_seq.size() > 0) + { + media_objects[i].chunk_start_seq.insert(media_objects[i].chunk_start_seq.end(), fallback_start_seq.begin(), fallback_start_seq.end()); + } + if(fallback_end_seq.size() > 0) + { + media_objects[i].chunk_end_seq.insert(media_objects[i].chunk_end_seq.begin(), fallback_end_seq.begin(), fallback_end_seq.end()); + } + if(used_fallback_boundary_tokens) + { + printf("\nWarning: MTMD video %d produced invalid model-specific boundary tokens. Falling back to generic and marker text.", i); + } + const int boundarytokensneeded = media_objects[i].chunk_start_seq.size() + media_objects[i].chunk_end_seq.size(); + mediatokensneeded += boundarytokensneeded; + if(debugmode==1 && !is_quiet) + { + printf("\nMTMD Video %i used %d frames, Tokens: %d",i,frames_processed,mediatokensneeded); + } + if(mediatokensneeded>0 && mediatokensneeded < nctx) + { + media_object_token_counts.push_back(mediatokensneeded); + int tokcnt = mediatokensneeded; + if(i==0) + { + tokcnt += introsize + outrosize; + } + const int media_token = kcpp_media_token_for_index(i); + for(int n=0;n media_data_buffer = kcpp_base64_decode(media_obj); @@ -5974,7 +6225,8 @@ generation_outputs gpttype_generate(const generation_inputs inputs) file_format, add_bos_token, inputs.images_len, - inputs.audio_len); + inputs.audio_len, + inputs.videos_len); } if(!media_inserted_inline) { From 6b15f41326b13375867ca54277633ded8db58020 Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 18:10:32 -0700 Subject: [PATCH 10/29] downsample video fps to fit frame budget instead of truncating Query mtmd_helper_video_get_info; when estimated frame count exceeds videomaxframes, re-init the stream at a reduced fps so sampled frames cover the full duration. The hard videomaxframes stop remains as a safety net for estimation error; if n_frames is unknown (-1) behavior is unchanged. --- gpttype_adapter.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 349f1b3b8f41..e7d32cf65e82 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -5415,6 +5415,28 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int continue; } + //spread the frame budget across the whole video by lowering fps instead of truncating the tail. + //n_frames is the estimated total at the effective fps (-1 if unknown); when it exceeds the budget, + //re-init the stream at a reduced fps so sampled frames cover the full duration. + mtmd_helper_video_info vinfo = mtmd_helper_video_get_info(vctx.get()); + if(vinfo.n_frames > video_max_frames && vinfo.fps > 0.0f) + { + float duration_sec = (float)vinfo.n_frames / vinfo.fps; + if(duration_sec > 0.0f) + { + float adjusted_fps = ((float)video_max_frames / duration_sec) * 0.995f; //safety factor so rounding stays within budget + printf("\nvideo: %d frames at %.2f fps exceeds budget of %d, downsampling to %.3f fps to sample the full duration", vinfo.n_frames, vinfo.fps, video_max_frames, adjusted_fps); + vparams.fps_target = adjusted_fps; + vctx.reset(mtmd_helper_video_init_from_buf(mtmd_ctx, video_data_buffer.data(), video_data_buffer.size(), vparams)); + if(!vctx) + { + media_object_token_counts.push_back(0); + printf("\nError: MTMD video %d failed to re-init at adjusted fps, skipping.", i); + continue; + } + } + } + int mediatokensneeded = 0; bool seen_media_embedding = false; bool used_fallback_boundary_tokens = false; From fb9d839305cfafd24e3d8707eeb91d0ee95d68ed Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 18:13:56 -0700 Subject: [PATCH 11/29] hash media items for cache signature instead of concatenating base64 --- gpttype_adapter.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index e7d32cf65e82..aa80cca94c2f 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -141,6 +141,18 @@ static mtmd_context * mtmd_ctx = nullptr; //for multimodal media static std::vector media_objects; static std::vector last_media_mem; //for storing dummy tokens that will be consumed by mtmd static std::vector media_object_token_counts; //per media object dummy token counts for inline placeholders +static std::string fnv1a64_hex(const std::string &data) //cheap fixed-size digest, used to avoid concatenating full base64 payloads for cache signatures +{ + uint64_t hash = 14695981039346656037ULL; + for(unsigned char c : data) + { + hash ^= c; + hash *= 1099511628211ULL; + } + char buf[17]; + snprintf(buf, sizeof(buf), "%016llx", (unsigned long long)hash); + return std::string(buf); +} static std::string media_composite_image_signature = ""; //for identifying when the media changes, we need to invalidate the cache static int current_media_identifier = MEDIA_TOKEN_IDENTIFIER_A; static int vision_max_res = 2048; @@ -5995,7 +6007,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) lv.mediatype = MEDIA_TYPE_IMAGE; TokenizeString("\n\n", lv.chunk_end_seq, file_format, false); media_objects.push_back(lv); - new_media_composite += item; + new_media_composite += "img:" + fnv1a64_hex(item) + ";"; } } for(int x=0;x Date: Wed, 22 Jul 2026 18:33:18 -0700 Subject: [PATCH 12/29] cap attached videos before placeholder numbering and warn on drop --- koboldcpp.py | 50 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/koboldcpp.py b/koboldcpp.py index 9ed01445ac11..fb8a2a2a841a 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -2207,7 +2207,9 @@ def generate(genparams, stream_flag=False): inputs.audio = (ctypes.c_char_p * inputs.audio_len)() for n, item in enumerate(audio): inputs.audio[n] = item.encode("UTF-8") - videos = videos[-videos_max:] + if len(videos) > videos_max: + print(f"Warning: too many videos attached, only the first {videos_max} are used.") + videos = videos[:videos_max] inputs.videos_len = len(videos) inputs.videos = (ctypes.c_char_p * inputs.videos_len)() for n, item in enumerate(videos): @@ -3842,6 +3844,8 @@ def parse(self, parser): m["content"] = "" # fix image placeholders, erase them and slap a reference onto the turn text message mediacount = 1 + video_count = 0 + videos_dropped_jinja = False for m in messages: if isinstance(m.get("content"), list): normalized = [] @@ -3856,13 +3860,19 @@ def parse(self, parser): turn_text += f"\n(Attached Audio {mediacount})\n" mediacount += 1 elif item.get("type")=="video_url" or item.get("type")=="input_video": - turn_text += f"\n(Attached Video {mediacount})\n" - mediacount += 1 + if video_count < videos_max: + turn_text += f"\n(Attached Video {mediacount})\n" + mediacount += 1 + video_count += 1 + else: + videos_dropped_jinja = True else: normalized.append(item) if turn_text: normalized.append({"type": "text","text": turn_text}) m["content"] = normalized + if videos_dropped_jinja: + print(f"Warning: too many videos attached, only the first {videos_max} are used.") for m in messages: # Fix tool_calls arguments and content if parsable if m.get("tool_calls"): for tc in m["tool_calls"]: @@ -4254,6 +4264,7 @@ def sweep_media_from_messages(messages_array): images = [] audio = [] videos = [] + videos_dropped = False for message in messages_array: curr_content = message.get("content", None) if isinstance(curr_content, list): @@ -4269,11 +4280,17 @@ def sweep_media_from_messages(messages_array): elif item.get("type") == "video_url": url = item.get("video_url", {}).get("url", "") if url.startswith("data:video"): - videos.append(url.split(",", 1)[1]) + if len(videos) < videos_max: + videos.append(url.split(",", 1)[1]) + else: + videos_dropped = True elif item.get("type") == "input_video": data = item.get("input_video", {}).get("data") if data: - videos.append(data) + if len(videos) < videos_max: + videos.append(data) + else: + videos_dropped = True elif message.get("role", "")=="tool" and isinstance(curr_content, str): #handle mcp returned images try: mcp_pl = json.loads(curr_content) @@ -4287,6 +4304,8 @@ def sweep_media_from_messages(messages_array): if imgs_ollama: for img in imgs_ollama: images.append(img) + if videos_dropped: + print(f"Warning: too many videos attached, only the first {videos_max} are used.") return images, audio, videos @@ -4414,6 +4433,7 @@ def transform_genparams(genparams, api_format, use_jinja): attachedimgid = 0 attachedaudid = 0 attachedvidid = 0 + videos_dropped_legacy = False jinja_output = None jinjatools = genparams.get('tools', []) if use_jinja and cached_chat_template: @@ -4507,14 +4527,20 @@ def transform_genparams(genparams, api_format, use_jinja): messages_string += f"\n(Attached Audio {attachedaudid})\n" elif item['type']=="video_url": if 'video_url' in item and item['video_url'] and item['video_url']['url'] and item['video_url']['url'].startswith("data:video"): - videos_added.append(item['video_url']['url'].split(",", 1)[1]) - attachedvidid += 1 - messages_string += f"\n(Attached Video {attachedvidid})\n" + if attachedvidid < videos_max: + videos_added.append(item['video_url']['url'].split(",", 1)[1]) + attachedvidid += 1 + messages_string += f"\n(Attached Video {attachedvidid})\n" + else: + videos_dropped_legacy = True elif item['type']=="input_video": if 'input_video' in item and item['input_video'] and item['input_video']['data']: - videos_added.append(item['input_video']['data']) - attachedvidid += 1 - messages_string += f"\n(Attached Video {attachedvidid})\n" + if attachedvidid < videos_max: + videos_added.append(item['input_video']['data']) + attachedvidid += 1 + messages_string += f"\n(Attached Video {attachedvidid})\n" + else: + videos_dropped_legacy = True elif isinstance(item, str): messages_string += item # If item is just a string, append it directly @@ -4555,6 +4581,8 @@ def transform_genparams(genparams, api_format, use_jinja): messages_string += assistant_message_end elif message['role'] == "tool": messages_string += tools_message_end + if videos_dropped_legacy: + print(f"Warning: too many videos attached, only the first {videos_max} are used.") messages_string += assistant_message_gen if (latest_turn_was_assistant and continue_assistant_turn): #allow continue a prefill, chop off end messages_string = messages_string[:-(len(assistant_message_gen)+len(assistant_message_end))] From cb00e58a7a135418d300e5a05af67275dd9b2dc9 Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 18:34:26 -0700 Subject: [PATCH 13/29] enforce videomintokens by upscaling undersized video frames --- gpttype_adapter.cpp | 47 +++++++++++++++++++++++++++++++++++---------- koboldcpp.py | 2 +- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index aa80cca94c2f..0b8931935220 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -159,6 +159,7 @@ static int vision_max_res = 2048; static float video_fps = 2.0f; static int video_max_frames = 32; static int video_max_tokens = -1; //-1 = inherit vision budget via shared clip context; >0 = explicit per-frame token cap +static int video_min_tokens = -1; //-1 = no floor; >0 = explicit per-frame token floor, undersized frames get upscaled static std::string ffmpeg_path = ""; //empty = search PATH static bool use_mrope = false; @@ -3023,7 +3024,8 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in vision_max_res = inputs.visionmaxres; video_fps = inputs.videofps; video_max_frames = inputs.videomaxframes; - video_max_tokens = inputs.videomaxtokens; //videomintokens has no separate action; per-frame min is inherited via the shared clip context + video_max_tokens = inputs.videomaxtokens; + video_min_tokens = inputs.videomintokens; //undersized frames are upscaled at eval time; the model's own baked clip minimum still applies downstream regardless ffmpeg_path = (inputs.ffmpegpath ? inputs.ffmpegpath : ""); if(isGguf && kcpp_pipeline_parallelism) { @@ -5359,32 +5361,44 @@ static mtmd_bitmap * kcpp_mtmd_bitmap_init_image_from_buf(const unsigned char * } #ifdef MTMD_VIDEO -//downscale an already-decoded RGB video frame so its larger dimension does not exceed maxdims. +//resize an already-decoded RGB video frame so its larger dimension stays within [mindims, maxdims]: +//downscales if the frame exceeds maxdims, upscales if it's smaller than mindims (either bound <=0 disables that side). //takes ownership of the input bitmap; returns a new bitmap (or the original if no resize was needed). -static mtmd_bitmap * kcpp_downscale_bitmap(mtmd_bitmap * bitmap, int maxdims) +static mtmd_bitmap * kcpp_resize_bitmap_bounds(mtmd_bitmap * bitmap, int mindims, int maxdims) { - if(bitmap == nullptr || maxdims <= 0) + if(bitmap == nullptr) { return bitmap; } int nx = (int)mtmd_bitmap_get_nx(bitmap); int ny = (int)mtmd_bitmap_get_ny(bitmap); - if(nx <= maxdims && ny <= maxdims) + + int target_dim = 0; //0 = no resize needed + if(maxdims > 0 && (nx > maxdims || ny > maxdims)) + { + target_dim = maxdims; + } + else if(mindims > 0 && nx < mindims && ny < mindims) + { + target_dim = mindims; + } + if(target_dim <= 0) { return bitmap; } + const float aspect_ratio = static_cast(nx) / ny; int new_width = nx; int new_height = ny; if(aspect_ratio > 1.0f) { - new_width = maxdims; - new_height = std::max(1, static_cast(maxdims / aspect_ratio)); + new_width = target_dim; + new_height = std::max(1, static_cast(target_dim / aspect_ratio)); } else { - new_height = maxdims; - new_width = std::max(1, static_cast(maxdims * aspect_ratio)); + new_height = target_dim; + new_width = std::max(1, static_cast(target_dim * aspect_ratio)); } uint8_t * resized_image = (uint8_t *)malloc((size_t)new_width * new_height * 3); if(resized_image != nullptr && stbir_resize_uint8(mtmd_bitmap_get_data(bitmap), nx, ny, 0, resized_image, new_width, new_height, 0, 3)) @@ -5471,6 +5485,19 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int } } + //mirror of the max heuristic above: derive a pixel floor from videomintokens so undersized frames + //get upscaled before tokenization. the model's own baked clip minimum still applies downstream regardless. + int frame_min_res = 0; + if(video_min_tokens > 0) + { + const int patch_px = 14; + frame_min_res = (int)(std::sqrt((double)video_min_tokens) * patch_px); + if(frame_max_res > 0 && frame_min_res > frame_max_res) + { + frame_min_res = frame_max_res; //never let the floor exceed the cap + } + } + auto append_video_chunks = [&](mtmd::input_chunks & chunks) { for(size_t j=0;j & media_int frame_cap_hit = true; break; } - mtmd::bitmap framebmp(kcpp_downscale_bitmap(raw_bitmap, frame_max_res)); + mtmd::bitmap framebmp(kcpp_resize_bitmap_bounds(raw_bitmap, frame_min_res, frame_max_res)); if(!framebmp.ptr) { printf("\nWarning: MTMD video %d frame %d failed to prepare, skipping frame.", i, frames_processed); diff --git a/koboldcpp.py b/koboldcpp.py index fb8a2a2a841a..030e4942debd 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -12613,7 +12613,7 @@ def range_checker(arg: str): advparser.add_argument("--videofps", metavar=('[fps]'), help="Sets the video frame sampling rate in frames per second. Values <=0 use the video's native fps (default 2.0).", type=float, default=2.0) advparser.add_argument("--videomaxframes", metavar=('[frames]'), help="Sets the max frames sampled per video; longer videos are downsampled evenly across their full duration to fit (default 32).", type=int, default=32) advparser.add_argument("--videomaxtokens", metavar=('[tokens]'), help="Override the maximum tokens per video frame for the MMProj embedding. If -1, inherits --visionmaxtokens (default -1).", type=int, default=-1) - advparser.add_argument("--videomintokens", metavar=('[tokens]'), help="Override the minimum tokens per video frame for the MMProj embedding. If -1, inherits --visionmintokens (default -1).", type=int, default=-1) + advparser.add_argument("--videomintokens", metavar=('[tokens]'), help="Sets a minimum tokens per video frame for the MMProj embedding; undersized frames are upscaled to meet this floor. If -1, no floor is applied (default -1).", type=int, default=-1) advparser.add_argument("--visionmaxres", metavar=('[max px]'), help="Clamp MMProj vision maximum allowed resolution. Allowed values are between 512 to 2048 px (default 1024).", type=int, default=default_visionmaxres) advparser.add_argument("--visionmaxtokens","--image-max-tokens", metavar=('[tokens]'), help="Override the maximum tokens for the MMProj embedding (default -1).", type=int, default=-1) advparser.add_argument("--visionmintokens","--image-min-tokens", metavar=('[tokens]'), help="Override the minimum tokens for the MMProj embedding (default -1).", type=int, default=-1) From 6d9520eb803bec4bdbc21dab0951ae665f9770ae Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 18:36:57 -0700 Subject: [PATCH 14/29] document why fnv1a64_hex is implemented locally --- gpttype_adapter.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 0b8931935220..831a2d07c3ae 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -141,7 +141,11 @@ static mtmd_context * mtmd_ctx = nullptr; //for multimodal media static std::vector media_objects; static std::vector last_media_mem; //for storing dummy tokens that will be consumed by mtmd static std::vector media_object_token_counts; //per media object dummy token counts for inline placeholders -static std::string fnv1a64_hex(const std::string &data) //cheap fixed-size digest, used to avoid concatenating full base64 payloads for cache signatures +//standard FNV-1a 64-bit digest, implemented locally: the only in-repo FNV helpers are file-static inside +//vendored code (tools/mtmd, ggml-rpc) which we keep pristine, and std::hash is implementation-defined +//(not stable across compilers/platforms), while signatures must compare deterministically. Cheap fixed-size +//digest avoids concatenating full base64 payloads for cache signatures; non-cryptographic is fine here. +static std::string fnv1a64_hex(const std::string &data) { uint64_t hash = 14695981039346656037ULL; for(unsigned char c : data) From afc2b34aeb1e61be6f10460e4982b1ec251407b6 Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 18:39:54 -0700 Subject: [PATCH 15/29] keep most recent videos when capping, drop oldest --- koboldcpp.py | 74 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 28 deletions(-) diff --git a/koboldcpp.py b/koboldcpp.py index 030e4942debd..fb1c4d788e8b 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -2208,8 +2208,8 @@ def generate(genparams, stream_flag=False): for n, item in enumerate(audio): inputs.audio[n] = item.encode("UTF-8") if len(videos) > videos_max: - print(f"Warning: too many videos attached, only the first {videos_max} are used.") - videos = videos[:videos_max] + print(f"Note: {len(videos)} videos attached, keeping only the most recent {videos_max}.") + videos = videos[-videos_max:] inputs.videos_len = len(videos) inputs.videos = (ctypes.c_char_p * inputs.videos_len)() for n, item in enumerate(videos): @@ -3844,8 +3844,10 @@ def parse(self, parser): m["content"] = "" # fix image placeholders, erase them and slap a reference onto the turn text message mediacount = 1 - video_count = 0 - videos_dropped_jinja = False + #pre-count videos so the cap below keeps only the most recent videos_max (sliding window over the chat) + total_videos = count_video_parts_in_messages(messages) + videos_to_skip = max(0, total_videos - videos_max) + video_seen = 0 for m in messages: if isinstance(m.get("content"), list): normalized = [] @@ -3860,19 +3862,17 @@ def parse(self, parser): turn_text += f"\n(Attached Audio {mediacount})\n" mediacount += 1 elif item.get("type")=="video_url" or item.get("type")=="input_video": - if video_count < videos_max: + video_seen += 1 + if video_seen > videos_to_skip: turn_text += f"\n(Attached Video {mediacount})\n" mediacount += 1 - video_count += 1 - else: - videos_dropped_jinja = True else: normalized.append(item) if turn_text: normalized.append({"type": "text","text": turn_text}) m["content"] = normalized - if videos_dropped_jinja: - print(f"Warning: too many videos attached, only the first {videos_max} are used.") + if videos_to_skip > 0: + print(f"Note: {total_videos} videos attached, keeping only the most recent {videos_max}.") for m in messages: # Fix tool_calls arguments and content if parsable if m.get("tool_calls"): for tc in m["tool_calls"]: @@ -4260,11 +4260,29 @@ def compress_tools_array(tools_array): return tools_array_filtered +def count_video_parts_in_messages(messages_array): + #counts video content parts across a chat messages array, used to determine how many of the + #oldest videos to skip so a videos_max cap keeps only the most recent videos (sliding window). + total = 0 + for message in messages_array: + curr_content = message.get("content", None) + if isinstance(curr_content, list): + for item in curr_content: + if item.get("type") == "video_url": + if item.get("video_url", {}).get("url", "").startswith("data:video"): + total += 1 + elif item.get("type") == "input_video": + if item.get("input_video", {}).get("data"): + total += 1 + return total + def sweep_media_from_messages(messages_array): images = [] audio = [] videos = [] - videos_dropped = False + total_videos = count_video_parts_in_messages(messages_array) + videos_to_skip = max(0, total_videos - videos_max) + videos_seen = 0 for message in messages_array: curr_content = message.get("content", None) if isinstance(curr_content, list): @@ -4280,17 +4298,15 @@ def sweep_media_from_messages(messages_array): elif item.get("type") == "video_url": url = item.get("video_url", {}).get("url", "") if url.startswith("data:video"): - if len(videos) < videos_max: + videos_seen += 1 + if videos_seen > videos_to_skip: videos.append(url.split(",", 1)[1]) - else: - videos_dropped = True elif item.get("type") == "input_video": data = item.get("input_video", {}).get("data") if data: - if len(videos) < videos_max: + videos_seen += 1 + if videos_seen > videos_to_skip: videos.append(data) - else: - videos_dropped = True elif message.get("role", "")=="tool" and isinstance(curr_content, str): #handle mcp returned images try: mcp_pl = json.loads(curr_content) @@ -4304,8 +4320,8 @@ def sweep_media_from_messages(messages_array): if imgs_ollama: for img in imgs_ollama: images.append(img) - if videos_dropped: - print(f"Warning: too many videos attached, only the first {videos_max} are used.") + if videos_to_skip > 0: + print(f"Note: {total_videos} videos attached, keeping only the most recent {videos_max}.") return images, audio, videos @@ -4433,7 +4449,6 @@ def transform_genparams(genparams, api_format, use_jinja): attachedimgid = 0 attachedaudid = 0 attachedvidid = 0 - videos_dropped_legacy = False jinja_output = None jinjatools = genparams.get('tools', []) if use_jinja and cached_chat_template: @@ -4465,6 +4480,11 @@ def transform_genparams(genparams, api_format, use_jinja): exist_mem = genparams.get('memory', "") genparams["memory"] = tools_string + exist_mem + #pre-count videos so the cap below keeps only the most recent videos_max (sliding window over the chat) + total_videos_legacy = count_video_parts_in_messages(messages_array) + videos_to_skip_legacy = max(0, total_videos_legacy - videos_max) + videos_seen_legacy = 0 + for message in messages_array: message_index += 1 latest_turn_was_assistant = False @@ -4527,20 +4547,18 @@ def transform_genparams(genparams, api_format, use_jinja): messages_string += f"\n(Attached Audio {attachedaudid})\n" elif item['type']=="video_url": if 'video_url' in item and item['video_url'] and item['video_url']['url'] and item['video_url']['url'].startswith("data:video"): - if attachedvidid < videos_max: + videos_seen_legacy += 1 + if videos_seen_legacy > videos_to_skip_legacy: videos_added.append(item['video_url']['url'].split(",", 1)[1]) attachedvidid += 1 messages_string += f"\n(Attached Video {attachedvidid})\n" - else: - videos_dropped_legacy = True elif item['type']=="input_video": if 'input_video' in item and item['input_video'] and item['input_video']['data']: - if attachedvidid < videos_max: + videos_seen_legacy += 1 + if videos_seen_legacy > videos_to_skip_legacy: videos_added.append(item['input_video']['data']) attachedvidid += 1 messages_string += f"\n(Attached Video {attachedvidid})\n" - else: - videos_dropped_legacy = True elif isinstance(item, str): messages_string += item # If item is just a string, append it directly @@ -4581,8 +4599,8 @@ def transform_genparams(genparams, api_format, use_jinja): messages_string += assistant_message_end elif message['role'] == "tool": messages_string += tools_message_end - if videos_dropped_legacy: - print(f"Warning: too many videos attached, only the first {videos_max} are used.") + if videos_to_skip_legacy > 0: + print(f"Note: {total_videos_legacy} videos attached, keeping only the most recent {videos_max}.") messages_string += assistant_message_gen if (latest_turn_was_assistant and continue_assistant_turn): #allow continue a prefill, chop off end messages_string = messages_string[:-(len(assistant_message_gen)+len(assistant_message_end))] From 12c84fb8b08615f231582362300402aec6dc5372 Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 20:13:14 -0700 Subject: [PATCH 16/29] set mtmd_input_text text_len on all media tokenize calls to fix lost marker --- gpttype_adapter.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 831a2d07c3ae..53a15efd93c2 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -5566,8 +5566,10 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int printf("\nWarning: MTMD video %d frame %d failed to prepare, skipping frame.", i, frames_processed); continue; } + const std::string media_marker = mtmd_default_marker(); mtmd_input_text inp_txt = { - mtmd_default_marker(), + media_marker.c_str(), + media_marker.size(), /* add_special */ false, /* parse_special */ true, }; @@ -5588,6 +5590,7 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int free(raw_text); mtmd_input_text inp_txt = { tstext.c_str(), + tstext.size(), /* add_special */ false, /* parse_special */ false, }; @@ -5663,8 +5666,10 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int printf("\nError: MTMD media %d failed to load!",i); continue; } + const std::string media_marker = mtmd_default_marker(); mtmd_input_text inp_txt = { - mtmd_default_marker(), + media_marker.c_str(), + media_marker.size(), /* add_special */ false, /* parse_special */ true, }; From a947ccf2bfc733d06450a1efc9b34b0e193963e7 Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 20:16:33 -0700 Subject: [PATCH 17/29] note maxrequestsize limit on video attachments at model load --- koboldcpp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/koboldcpp.py b/koboldcpp.py index fb1c4d788e8b..7c2a16c5fdfb 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -11988,6 +11988,9 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): worst_case_video_tokens = args.videomaxframes * effective_video_max_tokens if worst_case_video_tokens > maxctx: print(f"Warning: Worst-case video token usage ({worst_case_video_tokens} = {args.videomaxframes} frames x {effective_video_max_tokens} tokens/frame) may exceed the max context size ({maxctx}). Consider lowering --videomaxframes, --videofps, or --videomaxtokens.") + effective_maxrequestsize = int(args.maxrequestsize) if args.maxrequestsize else 32 + if effective_maxrequestsize <= 32: + print(f"Note: video attachments are limited by --maxrequestsize (currently {effective_maxrequestsize}MB); base64 encoding adds ~33% overhead, so large videos may need a higher value.") if not loadok: exitcounter = 999 From 296b16db0e2844db81f14f31622643b7b00df95a Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 20:30:27 -0700 Subject: [PATCH 18/29] decode video via temp file to avoid ffmpeg feeder-thread teardown deadlock --- gpttype_adapter.cpp | 55 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 53a15efd93c2..44ab90062d51 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -5416,6 +5416,47 @@ static mtmd_bitmap * kcpp_resize_bitmap_bounds(mtmd_bitmap * bitmap, int mindims free(resized_image); return bitmap; } + +//stage decoded video bytes to a temp file so we decode via the file-based helper API instead of the +//buffer API. buffer input (pipe:0) spawns a feeder thread that blocks writing the whole video into +//ffmpeg's stdin; tearing that context down before ffmpeg's stdout is drained (e.g. the throwaway +//probe used for fps downsampling) leaves the feeder blocked on a pipe the OS keeps open, deadlocking +//the server on the join in the context destructor. file input has no feeder and is seekable, so +//teardown is always clean. returns "" on failure. +static std::string kcpp_write_temp_video_file(const std::vector & data, const std::string & uniq) +{ +#if defined(_WIN32) + const char * tmpdir = getenv("TEMP"); + if(tmpdir == nullptr || tmpdir[0] == '\0') { tmpdir = getenv("TMP"); } + if(tmpdir == nullptr || tmpdir[0] == '\0') { tmpdir = "."; } + const char sep = '\\'; +#else + const char * tmpdir = getenv("TMPDIR"); + if(tmpdir == nullptr || tmpdir[0] == '\0') { tmpdir = "/tmp"; } + const char sep = '/'; +#endif + std::string path = std::string(tmpdir) + sep + "kcpp_vid_" + uniq + ".tmp"; + FILE * tf = fopen(path.c_str(), "wb"); + if(tf == nullptr) + { + return ""; + } + size_t written = data.empty() ? 0 : fwrite(data.data(), 1, data.size(), tf); + fclose(tf); + if(written != data.size()) + { + std::remove(path.c_str()); + return ""; + } + return path; +} + +//removes the staged temp file when the video branch exits by any path (including early continues) +struct kcpp_temp_file_guard +{ + std::string path; + ~kcpp_temp_file_guard() { if(!path.empty()) { std::remove(path.c_str()); } } +}; #endif //this function prepares the mtmd chunks for media. it's only needed when media changes @@ -5434,10 +5475,18 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int { #ifdef MTMD_VIDEO const std::vector video_data_buffer = kcpp_base64_decode(media_objects[i].b64data); + std::string video_temp_path = kcpp_write_temp_video_file(video_data_buffer, fnv1a64_hex(media_objects[i].b64data)); + kcpp_temp_file_guard video_temp_guard{video_temp_path}; + if(video_temp_path.empty()) + { + media_object_token_counts.push_back(0); + printf("\nError: MTMD video %d failed - could not stage temp file for decoding, skipping.", i); + continue; + } mtmd_helper_video_init_params vparams = mtmd_helper_video_init_params_default(); vparams.fps_target = video_fps; vparams.ffmpeg_bin_dir = ffmpeg_path.empty() ? nullptr : ffmpeg_path.c_str(); - mtmd_helper::video_ptr vctx(mtmd_helper_video_init_from_buf(mtmd_ctx, video_data_buffer.data(), video_data_buffer.size(), vparams)); + mtmd_helper::video_ptr vctx(mtmd_helper_video_init(mtmd_ctx, video_temp_path.c_str(), vparams)); if(!vctx) { media_object_token_counts.push_back(0); @@ -5455,9 +5504,9 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int if(duration_sec > 0.0f) { float adjusted_fps = ((float)video_max_frames / duration_sec) * 0.995f; //safety factor so rounding stays within budget - printf("\nvideo: %d frames at %.2f fps exceeds budget of %d, downsampling to %.3f fps to sample the full duration", vinfo.n_frames, vinfo.fps, video_max_frames, adjusted_fps); + printf("\nvideo: %d frames at %.2f fps exceeds budget of %d, downsampling to %.3f fps to sample the full duration\n", vinfo.n_frames, vinfo.fps, video_max_frames, adjusted_fps); vparams.fps_target = adjusted_fps; - vctx.reset(mtmd_helper_video_init_from_buf(mtmd_ctx, video_data_buffer.data(), video_data_buffer.size(), vparams)); + vctx.reset(mtmd_helper_video_init(mtmd_ctx, video_temp_path.c_str(), vparams)); if(!vctx) { media_object_token_counts.push_back(0); From d2e19a421cd3e0f41295161a3b13df66d15f9421 Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 21:20:00 -0700 Subject: [PATCH 19/29] aim downsampled fps so the last frame lands inside the frame budget --- gpttype_adapter.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 44ab90062d51..580c9dc470fb 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -5498,12 +5498,16 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int //n_frames is the estimated total at the effective fps (-1 if unknown); when it exceeds the budget, //re-init the stream at a reduced fps so sampled frames cover the full duration. mtmd_helper_video_info vinfo = mtmd_helper_video_get_info(vctx.get()); + bool downsampled = false; if(vinfo.n_frames > video_max_frames && vinfo.fps > 0.0f) { float duration_sec = (float)vinfo.n_frames / vinfo.fps; if(duration_sec > 0.0f) { - float adjusted_fps = ((float)video_max_frames / duration_sec) * 0.995f; //safety factor so rounding stays within budget + //place frames at t=0..duration inclusive: spreading (max-1) intervals across the whole + //video makes the first AND last sampled frame land inside the budget, instead of the + //(max/duration) rate that emits max+1 frames and drops the frame nearest the end. + float adjusted_fps = (video_max_frames > 1) ? ((float)(video_max_frames - 1) / duration_sec) : (1.0f / duration_sec); printf("\nvideo: %d frames at %.2f fps exceeds budget of %d, downsampling to %.3f fps to sample the full duration\n", vinfo.n_frames, vinfo.fps, video_max_frames, adjusted_fps); vparams.fps_target = adjusted_fps; vctx.reset(mtmd_helper_video_init(mtmd_ctx, video_temp_path.c_str(), vparams)); @@ -5513,6 +5517,7 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int printf("\nError: MTMD video %d failed to re-init at adjusted fps, skipping.", i); continue; } + downsampled = true; } } @@ -5657,7 +5662,14 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int if(frame_cap_hit) { - printf("\nvideo truncated to %d frames (videomaxframes)", frames_processed); + if(downsampled) + { + printf("\nNote: video %d produced trailing frame(s) beyond the %d-frame budget from fps rounding; dropped.", i, video_max_frames); + } + else + { + printf("\nvideo truncated to %d frames (videomaxframes); rest of the video was not sampled", frames_processed); + } } if(fallback_start_seq.size() > 0) { From e2cac91b354ba41cabdb6c2089400dd061bf8f8d Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 21:41:22 -0700 Subject: [PATCH 20/29] probe container duration to downsample videos lacking stream metadata --- gpttype_adapter.cpp | 106 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 100 insertions(+), 6 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 580c9dc470fb..a220c84fbd5d 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -5457,6 +5457,96 @@ struct kcpp_temp_file_guard std::string path; ~kcpp_temp_file_guard() { if(!path.empty()) { std::remove(path.c_str()); } } }; + +//run ffprobe with the given argument string and return the LAST non-empty line of its stdout. +//keeping only the last line bounds memory for the per-packet query (one line per packet) while still +//capturing the single value emitted by the format-duration query. +static std::string kcpp_run_ffprobe_lastline(const std::string & ffprobe_bin, const std::string & args) +{ + std::string cmd = "\"" + ffprobe_bin + "\" " + args; +#if defined(_WIN32) + cmd = "\"" + cmd + "\""; //cmd.exe /c strips one outer quote pair; wrap so quoted bin+path both survive + FILE * pp = _popen(cmd.c_str(), "r"); +#else + FILE * pp = popen(cmd.c_str(), "r"); +#endif + if(pp == nullptr) + { + return ""; + } + std::string last; + char buf[128]; + while(fgets(buf, sizeof(buf), pp) != nullptr) + { + std::string line = buf; + //trim trailing CR/LF/space so atof sees a clean number and empties are recognised + while(!line.empty() && (line.back()=='\n' || line.back()=='\r' || line.back()==' ' || line.back()=='\t')) + { + line.pop_back(); + } + if(!line.empty()) + { + last = line; + } + } +#if defined(_WIN32) + _pclose(pp); +#else + pclose(pp); +#endif + return last; +} + +//probe the container duration (seconds) of a staged video file via our own ffprobe. needed because the +//vendored get_info reports n_frames=-1 whenever the stream lacks both a stream-level duration and +//nb_frames (common in webm/mkv, where duration lives at the container/format level), which would +//otherwise skip fps downsampling and hard-truncate long videos at videomaxframes. tiered so that +//animated webp (which the webp demuxer usually reports with NO duration at either stream OR format +//level) is still rescued by summing container packet timestamps. returns <=0 on failure. +static float kcpp_probe_video_duration(const std::string & path, const std::string & bin_dir) +{ + //resolve the ffprobe binary the same way the vendored helper's video_resolve_bin does + std::string ffprobe_bin; + if(bin_dir.empty()) + { + ffprobe_bin = "ffprobe"; + } + else + { + ffprobe_bin = bin_dir; + char lastc = ffprobe_bin.empty() ? '\0' : ffprobe_bin.back(); + if(lastc != '/' && lastc != '\\') + { +#if defined(_WIN32) + ffprobe_bin += '\\'; +#else + ffprobe_bin += '/'; +#endif + } + ffprobe_bin += "ffprobe"; + } +#if defined(_WIN32) + ffprobe_bin += ".exe"; +#endif + + std::string qpath = "\"" + path + "\""; + //tier 1: cheap format-level duration (one line). works for mp4/mov and most containers. + std::string out = kcpp_run_ffprobe_lastline(ffprobe_bin, + "-v quiet -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 " + qpath); + float duration = out.empty() ? -1.0f : (float)atof(out.c_str()); //atof returns 0 on "N/A"/garbage + if(duration > 0.0f) + { + return duration; + } + //tier 2: last video packet presentation timestamp. container demux only (no decode), so it stays + //fast even on large files, and covers animated webp/webm whose format duration reads N/A. this is + //the last frame's start time (one interval short of the true end) which slightly under-estimates + //duration - fine, it only makes the downsample fps marginally lower. + out = kcpp_run_ffprobe_lastline(ffprobe_bin, + "-v quiet -select_streams v:0 -show_entries packet=pts_time -of csv=p=0 " + qpath); + duration = out.empty() ? -1.0f : (float)atof(out.c_str()); + return (duration > 0.0f) ? duration : -1.0f; +} #endif //this function prepares the mtmd chunks for media. it's only needed when media changes @@ -5495,20 +5585,24 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int } //spread the frame budget across the whole video by lowering fps instead of truncating the tail. - //n_frames is the estimated total at the effective fps (-1 if unknown); when it exceeds the budget, - //re-init the stream at a reduced fps so sampled frames cover the full duration. + //determine the duration to decide: get_info's n_frames is the estimated total at the effective + //fps but is -1 when the container exposes neither stream duration nor nb_frames (common in + //webm/mkv), so fall back to our own container-level ffprobe rather than hard-truncating. mtmd_helper_video_info vinfo = mtmd_helper_video_get_info(vctx.get()); bool downsampled = false; - if(vinfo.n_frames > video_max_frames && vinfo.fps > 0.0f) + float duration_sec = (vinfo.n_frames > 0 && vinfo.fps > 0.0f) + ? ((float)vinfo.n_frames / vinfo.fps) + : kcpp_probe_video_duration(video_temp_path, ffmpeg_path); + if(duration_sec > 0.0f) { - float duration_sec = (float)vinfo.n_frames / vinfo.fps; - if(duration_sec > 0.0f) + float est_frames = duration_sec * video_fps; + if(est_frames > (float)video_max_frames) { //place frames at t=0..duration inclusive: spreading (max-1) intervals across the whole //video makes the first AND last sampled frame land inside the budget, instead of the //(max/duration) rate that emits max+1 frames and drops the frame nearest the end. float adjusted_fps = (video_max_frames > 1) ? ((float)(video_max_frames - 1) / duration_sec) : (1.0f / duration_sec); - printf("\nvideo: %d frames at %.2f fps exceeds budget of %d, downsampling to %.3f fps to sample the full duration\n", vinfo.n_frames, vinfo.fps, video_max_frames, adjusted_fps); + printf("\nvideo: ~%d frames (%.1fs) at %.2f fps exceeds budget of %d, downsampling to %.3f fps to sample the full duration\n", (int)(est_frames + 0.5f), duration_sec, video_fps, video_max_frames, adjusted_fps); vparams.fps_target = adjusted_fps; vctx.reset(mtmd_helper_video_init(mtmd_ctx, video_temp_path.c_str(), vparams)); if(!vctx) From 7c9e5f7af6fce0500a1819efd705efe9e895ad81 Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 22:43:32 -0700 Subject: [PATCH 21/29] respect quiet mode in video pipeline logging --- gpttype_adapter.cpp | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index a220c84fbd5d..c9df5de23155 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -259,6 +259,19 @@ static inline void log_callback_off(ggml_log_level level, const char* text, void return; } +//vendored mtmd/mtmd-helper emit their ffmpeg probe/decode progress ("probe: launching", "start_ffmpeg", +//"read_next: proc_alive=", per-batch decode timings, etc.) at DEBUG/INFO via a global logger whose default +//callback prints every level unconditionally - ignoring kcpp's --quiet. route it through our verbosity gate: +//only surface that spam in full --debugmode, but always let warnings and errors through. +static inline void mtmd_log_filter(ggml_log_level level, const char* text, void*) { + bool showall = (debugmode==1 && !is_quiet); + if(!showall && level < GGML_LOG_LEVEL_WARN) { + return; + } + fputs(text, stderr); + fflush(stderr); +} + static inline void string_trim_whitespace(std::string & s) { auto nul = std::find(s.begin(), s.end(), '\0'); //remove everything after the first NUL if (nul != s.end()) { @@ -3704,6 +3717,8 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in fprintf(stderr, "%s: error: failed to load mmproj model!\n", __func__); return ModelLoadResult::FAIL; } + //gate vendored mtmd/mtmd-helper logging through our verbosity filter (also sets mtmd_log_set internally) + mtmd_helper_log_set(mtmd_log_filter, nullptr); vision_multimodal_supported = mtmd_support_vision(mtmd_ctx); audio_multimodal_supported = mtmd_support_audio(mtmd_ctx); video_multimodal_supported = mtmd_helper_support_video(mtmd_ctx); @@ -5602,7 +5617,10 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int //video makes the first AND last sampled frame land inside the budget, instead of the //(max/duration) rate that emits max+1 frames and drops the frame nearest the end. float adjusted_fps = (video_max_frames > 1) ? ((float)(video_max_frames - 1) / duration_sec) : (1.0f / duration_sec); - printf("\nvideo: ~%d frames (%.1fs) at %.2f fps exceeds budget of %d, downsampling to %.3f fps to sample the full duration\n", (int)(est_frames + 0.5f), duration_sec, video_fps, video_max_frames, adjusted_fps); + if(!is_quiet && debugmode!=-1) + { + printf("\nvideo: ~%d frames (%.1fs) at %.2f fps exceeds budget of %d, downsampling to %.3f fps to sample the full duration\n", (int)(est_frames + 0.5f), duration_sec, video_fps, video_max_frames, adjusted_fps); + } vparams.fps_target = adjusted_fps; vctx.reset(mtmd_helper_video_init(mtmd_ctx, video_temp_path.c_str(), vparams)); if(!vctx) @@ -5758,7 +5776,10 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int { if(downsampled) { - printf("\nNote: video %d produced trailing frame(s) beyond the %d-frame budget from fps rounding; dropped.", i, video_max_frames); + if(!is_quiet && debugmode!=-1) + { + printf("\nNote: video %d produced trailing frame(s) beyond the %d-frame budget from fps rounding; dropped.", i, video_max_frames); + } } else { From d31c7ff0d6351c9824156a77c861559b1befac5c Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 23:05:04 -0700 Subject: [PATCH 22/29] add TODO outlining future video audio-track embedding --- gpttype_adapter.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index c9df5de23155..1b45997c4df2 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -5786,6 +5786,14 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int printf("\nvideo truncated to %d frames (videomaxframes); rest of the video was not sampled", frames_processed); } } + //TODO(future): embed the video's audio track alongside the sampled frames for audio-capable mmprojs. + //Shape: (1) run ffmpeg on video_temp_path (-vn -> PCM/WAV); skip silently when the container has no + //audio stream; (2) feed the extracted bytes through the existing MEDIA_TYPE_AUDIO ingestion and append + //the resulting chunks after the frame/timestamp chunks of this same media object (the chunk/token + //accounting below is already generic); (3) gate on mtmd_support_audio(mtmd_ctx) plus a --videoaudio + //toggle; (4) count audio tokens in the launch-time context budget warning (~750 tokens per 30s for + //whisper-style encoders). Interleaving audio segments between timestamped frames is possible but + //untested against model training formats - ship append-after-frames first. if(fallback_start_seq.size() > 0) { media_objects[i].chunk_start_seq.insert(media_objects[i].chunk_start_seq.end(), fallback_start_seq.begin(), fallback_start_seq.end()); From 0435a5decc5d600a0ad4702b6064d8c41ee26a73 Mon Sep 17 00:00:00 2001 From: Reithan Date: Wed, 22 Jul 2026 23:30:40 -0700 Subject: [PATCH 23/29] reformat video timestamps to MM:SS and append duration note for model readability --- gpttype_adapter.cpp | 61 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 1b45997c4df2..35a7f5f6a63e 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -5562,6 +5562,26 @@ static float kcpp_probe_video_duration(const std::string & path, const std::stri duration = out.empty() ? -1.0f : (float)atof(out.c_str()); return (duration > 0.0f) ? duration : -1.0f; } + +//format a whole-second video position as MM:SS (or H:MM:SS past an hour) - a notation the model parses +//reliably, unlike the vendored compact [XmY.YYs] form which Gemma misreads (e.g. [0m10.45s] as ~0.5s). +static std::string kcpp_format_video_timestamp(int total_seconds) +{ + if(total_seconds < 0) { total_seconds = 0; } + int hours = total_seconds / 3600; + int mins = (total_seconds % 3600) / 60; + int secs = total_seconds % 60; + char buf[32]; + if(hours > 0) + { + snprintf(buf, sizeof(buf), "%d:%02d:%02d", hours, mins, secs); + } + else + { + snprintf(buf, sizeof(buf), "%02d:%02d", mins, secs); + } + return std::string(buf); +} #endif //this function prepares the mtmd chunks for media. it's only needed when media changes @@ -5754,6 +5774,20 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int { std::string tstext = raw_text; free(raw_text); + //the vendored timestamp arrives as the compact "[XmY.YYs]" which the model misparses (it + //read [0m10.45s] as ~0.5s); rewrite to a familiar "[MM:SS]" before tokenizing. anything + //that doesn't match the pattern (prompt header, future vendored format) passes through + //unchanged so no text is ever dropped. + int ts_min = 0; + float ts_sec = 0.0f; + if(sscanf(tstext.c_str(), "[%dm%fs]", &ts_min, &ts_sec) == 2) + { + tstext = "[" + kcpp_format_video_timestamp((int)(ts_min * 60 + ts_sec + 0.5f)) + "]"; + if(debugmode==1 && !is_quiet) + { + printf("\nMTMD Video %d timestamp reformatted to %s", i, tstext.c_str()); + } + } mtmd_input_text inp_txt = { tstext.c_str(), tstext.size(), @@ -5786,6 +5820,33 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int printf("\nvideo truncated to %d frames (videomaxframes); rest of the video was not sampled", frames_processed); } } + //append an explicit end-of-clip duration note in the same familiar notation so the model can state + //how long the video is; skip when the duration is unknown or no frames were actually embedded. + if(duration_sec > 0.0f && frames_processed > 0) + { + std::string durtext = "(video duration: " + kcpp_format_video_timestamp((int)(duration_sec + 0.5f)) + ")"; + mtmd_input_text inp_txt = { + durtext.c_str(), + durtext.size(), + /* add_special */ false, + /* parse_special */ false, + }; + mtmd::input_chunks chunks(mtmd_input_chunks_init()); + std::vector nobitmaps; + int32_t tokenized = mtmd_tokenize(mtmd_ctx, chunks.ptr.get(), &inp_txt, nobitmaps.data(), nobitmaps.size()); + if(tokenized == 0) + { + append_video_chunks(chunks); + if(debugmode==1 && !is_quiet) + { + printf("\nMTMD Video %d duration note: %s", i, durtext.c_str()); + } + } + else + { + printf("\nWarning: MTMD video %d duration note failed to tokenize (status %d), skipping.", i, tokenized); + } + } //TODO(future): embed the video's audio track alongside the sampled frames for audio-capable mmprojs. //Shape: (1) run ffmpeg on video_temp_path (-vn -> PCM/WAV); skip silently when the container has no //audio stream; (2) feed the extracted bytes through the existing MEDIA_TYPE_AUDIO ingestion and append From 141e72f80bcd4b964b656cc7a542d352032327d2 Mon Sep 17 00:00:00 2001 From: Reithan Date: Thu, 23 Jul 2026 00:26:07 -0700 Subject: [PATCH 24/29] filter llama core log spam during media generations per quiet/debug tiers --- gpttype_adapter.cpp | 48 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 35a7f5f6a63e..5c1277efa555 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -272,6 +272,47 @@ static inline void mtmd_log_filter(ggml_log_level level, const char* text, void* fflush(stderr); } +//koboldcpp never installs a global llama logger, so llama's default callback prints every level - including +//the DEBUG/INFO "set_causal_attn"/"sched_reserve" chatter that mtmd's causal-attn toggling triggers on every +//media-bearing generation - ignoring --quiet. this filter is only installed when mtmd is active (keeping +//non-multimodal builds bit-identical). it stays disarmed until load fully completes so load-time logs print +//as before; once armed, llama core logs are gated by tier. +static bool kcpp_llama_logfilter_armed = false; +static void kcpp_llama_log_filter(ggml_log_level level, const char* text, void* ud) { + (void)ud; + static bool last_shown = true; //CONT fragments continue the previous line, so follow its decision + bool show; + if(!kcpp_llama_logfilter_armed) + { + show = true; //pre-load: behave like the default callback + } + else if(level == GGML_LOG_LEVEL_CONT) + { + show = last_shown; + } + else if(level == GGML_LOG_LEVEL_WARN || level == GGML_LOG_LEVEL_ERROR) + { + show = true; + } + else if(level == GGML_LOG_LEVEL_DEBUG) + { + show = (debugmode==1 && !is_quiet); + } + else //INFO (and NONE): normal-verbosity info, dropped only under --quiet + { + show = !is_quiet; + } + if(level != GGML_LOG_LEVEL_CONT) + { + last_shown = show; + } + if(show) + { + fputs(text, stderr); + fflush(stderr); + } +} + static inline void string_trim_whitespace(std::string & s) { auto nul = std::find(s.begin(), s.end(), '\0'); //remove everything after the first NUL if (nul != s.end()) { @@ -3719,6 +3760,10 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in } //gate vendored mtmd/mtmd-helper logging through our verbosity filter (also sets mtmd_log_set internally) mtmd_helper_log_set(mtmd_log_filter, nullptr); + //also gate llama core logs (set_causal_attn/sched_reserve spam that media generations trigger). disarmed + //until load completes so load-time logs still print; runs after the fitting code's log save/restore blocks. + kcpp_llama_logfilter_armed = false; + llama_log_set(kcpp_llama_log_filter, nullptr); vision_multimodal_supported = mtmd_support_vision(mtmd_ctx); audio_multimodal_supported = mtmd_support_audio(mtmd_ctx); video_multimodal_supported = mtmd_helper_support_video(mtmd_ctx); @@ -3797,6 +3842,9 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in { printf("\nModel Warmup Failed! (code:%d)\n",er); } + //load (incl. warmup decodes above) is done - arm the llama log filter so generation-time core log + //spam now respects the quiet/debug tiers. no-op unless the filter was installed (mtmd active). + kcpp_llama_logfilter_armed = true; return ModelLoadResult::SUCCESS; } else if (file_format == FileFormat::RWKV_1 || file_format==FileFormat::RWKV_2) From 88effd2be99fc38d3c67156b716e4035c7e23e07 Mon Sep 17 00:00:00 2001 From: Reithan Date: Thu, 23 Jul 2026 23:08:35 -0700 Subject: [PATCH 25/29] budget media context by real KV cell counts instead of position advance --- gpttype_adapter.cpp | 23 +++++++++++++++-------- otherarch/otherarch.h | 3 ++- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 5c1277efa555..ca82e08db7c2 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -5702,6 +5702,7 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int } int mediatokensneeded = 0; + int mediacellsneeded = 0; //real KV cells: for M-RoPE each frame consumes nx*ny cells while only advancing the position by max(nx,ny) bool seen_media_embedding = false; bool used_fallback_boundary_tokens = false; std::vector fallback_start_seq; @@ -5722,7 +5723,6 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int frame_max_res = tok_res; } } - //mirror of the max heuristic above: derive a pixel floor from videomintokens so undersized frames //get upscaled before tokenization. the model's own baked clip minimum still applies downstream regardless. int frame_min_res = 0; @@ -5763,7 +5763,9 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int chunk.mediatype = media_objects[i].mediatype; chunk.mtmd_chunk = mtmd_input_chunk_copy(mtmdchunk); chunk.clp_image_tokens = mtmd_input_chunk_get_n_pos(mtmdchunk); + chunk.clp_kv_cells = (int32_t)mtmd_input_chunk_get_n_tokens(mtmdchunk); mediatokensneeded += chunk.clp_image_tokens; + mediacellsneeded += chunk.clp_kv_cells; media_objects[i].mediachunks.push_back(chunk); if(mtmd_input_chunk_get_type(mtmdchunk) != MTMD_INPUT_CHUNK_TYPE_TEXT) { @@ -5917,11 +5919,12 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int } const int boundarytokensneeded = media_objects[i].chunk_start_seq.size() + media_objects[i].chunk_end_seq.size(); mediatokensneeded += boundarytokensneeded; + mediacellsneeded += boundarytokensneeded; if(debugmode==1 && !is_quiet) { - printf("\nMTMD Video %i used %d frames, Tokens: %d",i,frames_processed,mediatokensneeded); + printf("\nMTMD Video %i used %d frames, Tokens: %d, KV Cells: %d",i,frames_processed,mediatokensneeded,mediacellsneeded); } - if(mediatokensneeded>0 && mediatokensneeded < nctx) + if(mediatokensneeded>0 && mediatokensneeded < nctx && mediacellsneeded < nctx) { media_object_token_counts.push_back(mediatokensneeded); int tokcnt = mediatokensneeded; @@ -5939,7 +5942,7 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int { media_object_token_counts.push_back(0); media_composite_image_signature = ""; //force invalidate - printf("\nWarning: Video excluded - Context size too low or no frames decoded! (needed %d)\nVideo will be IGNORED! You probably want to relaunch with a larger context size!\n",mediatokensneeded); + printf("\nWarning: Video excluded - Context size too low or no frames decoded! (needed %d KV cells)\nVideo will be IGNORED! You probably want to relaunch with a larger context size!\n",mediacellsneeded); } continue; #else @@ -5978,6 +5981,7 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int } int mediatokensneeded = 0; + int mediacellsneeded = 0; //real KV cells: M-RoPE image grids consume nx*ny cells but advance the position by only max(nx,ny) bool seen_media_embedding = false; bool used_fallback_boundary_tokens = false; std::vector fallback_start_seq; @@ -6007,7 +6011,9 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int chunk.mediatype = media_objects[i].mediatype; chunk.mtmd_chunk = mtmd_input_chunk_copy(mtmdchunk); chunk.clp_image_tokens = mtmd_input_chunk_get_n_pos(mtmdchunk); + chunk.clp_kv_cells = (int32_t)mtmd_input_chunk_get_n_tokens(mtmdchunk); mediatokensneeded += chunk.clp_image_tokens; + mediacellsneeded += chunk.clp_kv_cells; media_objects[i].mediachunks.push_back(chunk); if(mtmd_input_chunk_get_type(mtmdchunk) != MTMD_INPUT_CHUNK_TYPE_TEXT) { @@ -6028,11 +6034,12 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int } const int boundarytokensneeded = media_objects[i].chunk_start_seq.size() + media_objects[i].chunk_end_seq.size(); mediatokensneeded += boundarytokensneeded; + mediacellsneeded += boundarytokensneeded; if(debugmode==1 && !is_quiet) { - printf("\nMTMD Media %i used Tokens: %d",i,mediatokensneeded); + printf("\nMTMD Media %i used Tokens: %d, KV Cells: %d",i,mediatokensneeded,mediacellsneeded); } - if(mediatokensneeded>0 && mediatokensneeded < nctx) + if(mediatokensneeded>0 && mediatokensneeded < nctx && mediacellsneeded < nctx) { media_object_token_counts.push_back(mediatokensneeded); int tokcnt = mediatokensneeded; @@ -6050,7 +6057,7 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int { media_object_token_counts.push_back(0); media_composite_image_signature = ""; //force invalidate - printf("\nWarning: Media excluded - Context size too low or not enough mtmd tokens! (needed %d)\nMedia will be IGNORED! You probably want to relaunch with a larger context size!\n",mediatokensneeded); + printf("\nWarning: Media excluded - Context size too low or not enough mtmd tokens! (needed %d KV cells)\nMedia will be IGNORED! You probably want to relaunch with a larger context size!\n",mediacellsneeded); } } } @@ -7855,7 +7862,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) media_chunk chunk = media_objects[curr_media_index].mediachunks[j]; if(allow_regular_prints) { - printf("\rProcessing Media Embedding %d (%d tokens)",(curr_media_index+1), chunk.clp_image_tokens); + printf("\rProcessing Media Embedding %d (%d tokens, %d KV cells)",(curr_media_index+1), chunk.clp_image_tokens, chunk.clp_kv_cells); } bool err = kcpp_eval_media(llama_ctx_v4,chunk,kcpp_data->n_batch,&n_past); mediatokensevaled += chunk.clp_image_tokens; diff --git a/otherarch/otherarch.h b/otherarch/otherarch.h index a03d52531b79..1d3a73598ad0 100644 --- a/otherarch/otherarch.h +++ b/otherarch/otherarch.h @@ -514,7 +514,8 @@ struct media_chunk { MediaType mediatype = MEDIA_TYPE_IMAGE; void * mtmd_chunk = nullptr; // mtmd_input_chunk, owned by this chunk - int32_t clp_image_tokens = 0; //holds number of tokens used in this chunk + int32_t clp_image_tokens = 0; //position advance of this chunk (max(nx,ny) for M-RoPE, real token count otherwise) + int32_t clp_kv_cells = 0; //real embedding token count = KV cells this chunk occupies (nx*ny for M-RoPE grids) int32_t nx = 0; //only used for 2d roped images int32_t ny = 0; }; From ad39eb5fcb4062ca792fc510b4846a862641f111 Mon Sep 17 00:00:00 2001 From: Reithan Date: Thu, 23 Jul 2026 23:09:09 -0700 Subject: [PATCH 26/29] cap video frame resolution and frame count by context KV cell budget --- gpttype_adapter.cpp | 49 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index ca82e08db7c2..5c81ec310b25 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -5709,6 +5709,12 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int std::vector fallback_end_seq; int frames_processed = 0; bool frame_cap_hit = false; + bool cell_budget_hit = false; + //hard ceiling on the KV cells one video may occupy: the position-based token counts above cannot + //see the M-RoPE inflation, so without this a high-res video passes every check and then exhausts + //the KV cache mid-eval (decode error "failed to find a memory slot"). keep a quarter of the + //context free for the prompt, response and other media. + const int video_cell_budget = (nctx / 4) * 3; //frames inherit the image resolution clamp; when an explicit video token budget is set we tighten it. //no public patch-size accessor is reachable from this TU, so approximate the token budget with a @@ -5723,6 +5729,30 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int frame_max_res = tok_res; } } + //also cap frame resolution by the per-video KV cell budget, spread over the frames we expect to + //sample, so dynamic-resolution encoders (Qwen-VL: cells scale with pixel area) downsize to fit the + //context instead of tripping the cell guard below and losing the tail of the video. the 14px patch + //heuristic under-estimates for merged-patch models, erring toward smaller frames + full coverage. + { + int expected_frames = video_max_frames; + if(duration_sec > 0.0f && !downsampled) + { + int est = (int)(duration_sec * video_fps) + 1; + expected_frames = std::min(expected_frames, std::max(1, est)); + } + const int patch_px = 14; + int per_frame_cells = video_cell_budget / std::max(1, expected_frames); + int budget_res = (int)(std::sqrt((double)per_frame_cells) * patch_px); + if(budget_res > 0 && (frame_max_res <= 0 || budget_res < frame_max_res)) + { + frame_max_res = budget_res; + if(!is_quiet && debugmode!=-1) + { + printf("\nvideo: frame resolution capped at %dpx so ~%d frames fit the context budget (%d cells)\n", frame_max_res, expected_frames, video_cell_budget); + } + } + } + //mirror of the max heuristic above: derive a pixel floor from videomintokens so undersized frames //get upscaled before tokenization. the model's own baked clip minimum still applies downstream regardless. int frame_min_res = 0; @@ -5773,6 +5803,15 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int } } }; + auto chunks_kv_cells = [](mtmd::input_chunks & chunks) -> int + { + int cells = 0; + for(size_t j=0;j & media_int printf("\nWarning: MTMD video %d frame %d failed to tokenize (status %d), skipping frame.", i, frames_processed, tokenized); continue; } + //exact per-frame guard: measured cells, so it holds even where the pixel-cap heuristic misestimates + if(mediacellsneeded + chunks_kv_cells(chunks) > video_cell_budget) + { + cell_budget_hit = true; + break; + } append_video_chunks(chunks); ++frames_processed; } @@ -5870,6 +5915,10 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int printf("\nvideo truncated to %d frames (videomaxframes); rest of the video was not sampled", frames_processed); } } + if(cell_budget_hit) + { + printf("\nvideo truncated to %d frames: sampled frames occupy %d KV cells of a %d-cell budget (context %d). Lower the video resolution, raise context size, or reduce videomaxframes for full coverage.", frames_processed, mediacellsneeded, video_cell_budget, nctx); + } //append an explicit end-of-clip duration note in the same familiar notation so the model can state //how long the video is; skip when the duration is unknown or no frames were actually embedded. if(duration_sec > 0.0f && frames_processed > 0) From 88b0011ef0d4b1993f4dae3abba4cf96d46e3172 Mon Sep 17 00:00:00 2001 From: Reithan Date: Fri, 24 Jul 2026 00:12:25 -0700 Subject: [PATCH 27/29] calibrate video frame resolution from measured first-frame cell cost --- gpttype_adapter.cpp | 90 +++++++++++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 36 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 5c81ec310b25..35204bd2c837 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -5713,45 +5713,26 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int //hard ceiling on the KV cells one video may occupy: the position-based token counts above cannot //see the M-RoPE inflation, so without this a high-res video passes every check and then exhausts //the KV cache mid-eval (decode error "failed to find a memory slot"). keep a quarter of the - //context free for the prompt, response and other media. - const int video_cell_budget = (nctx / 4) * 3; - - //frames inherit the image resolution clamp; when an explicit video token budget is set we tighten it. - //no public patch-size accessor is reachable from this TU, so approximate the token budget with a - //conservative 14px patch to derive a pixel cap (otherwise frames just use vision_max_res). - int frame_max_res = vision_max_res; - if(video_max_tokens > 0) + //context free for the prompt, response and other media. an explicit videomaxtokens tightens it. + int video_cell_budget = (nctx / 4) * 3; + if(video_max_tokens > 0 && video_max_tokens < video_cell_budget) { - const int patch_px = 14; - int tok_res = (int)(std::sqrt((double)video_max_tokens) * patch_px); - if(tok_res > 0 && (frame_max_res <= 0 || tok_res < frame_max_res)) - { - frame_max_res = tok_res; - } + video_cell_budget = video_max_tokens; } - //also cap frame resolution by the per-video KV cell budget, spread over the frames we expect to - //sample, so dynamic-resolution encoders (Qwen-VL: cells scale with pixel area) downsize to fit the - //context instead of tripping the cell guard below and losing the tail of the video. the 14px patch - //heuristic under-estimates for merged-patch models, erring toward smaller frames + full coverage. + //spread the budget over the frames we expect to sample. the first tokenized frame is measured + //against this per-frame share to calibrate the resolution cap (inside the loop below): actual + //pixels-per-token varies too much across encoders (Qwen3-VL ~32px/token side, Gemma4 clamps + //internally to <=280 tokens) for any fixed heuristic to convert cells to pixels accurately. + int expected_frames = video_max_frames; + if(duration_sec > 0.0f && !downsampled) { - int expected_frames = video_max_frames; - if(duration_sec > 0.0f && !downsampled) - { - int est = (int)(duration_sec * video_fps) + 1; - expected_frames = std::min(expected_frames, std::max(1, est)); - } - const int patch_px = 14; - int per_frame_cells = video_cell_budget / std::max(1, expected_frames); - int budget_res = (int)(std::sqrt((double)per_frame_cells) * patch_px); - if(budget_res > 0 && (frame_max_res <= 0 || budget_res < frame_max_res)) - { - frame_max_res = budget_res; - if(!is_quiet && debugmode!=-1) - { - printf("\nvideo: frame resolution capped at %dpx so ~%d frames fit the context budget (%d cells)\n", frame_max_res, expected_frames, video_cell_budget); - } - } + int est = (int)(duration_sec * video_fps) + 1; + expected_frames = std::min(expected_frames, std::max(1, est)); } + const int per_frame_cell_budget = std::max(1, video_cell_budget / std::max(1, expected_frames)); + + //frames inherit the image resolution clamp; the measured calibration below can only tighten it + int frame_max_res = vision_max_res; //mirror of the max heuristic above: derive a pixel floor from videomintokens so undersized frames //get upscaled before tokenization. the model's own baked clip minimum still applies downstream regardless. @@ -5856,7 +5837,44 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int printf("\nWarning: MTMD video %d frame %d failed to tokenize (status %d), skipping frame.", i, frames_processed, tokenized); continue; } - //exact per-frame guard: measured cells, so it holds even where the pixel-cap heuristic misestimates + //calibrate the resolution cap off the first frame's measured cell cost - tokenization is + //cheap (no encoder run) and exact, and every frame of a video shares one resolution, so + //one measurement holds for the rest. only shrinks when the frame overshoots its share. + if(frames_processed == 0) + { + const int framecells = chunks_kv_cells(chunks); + const int longside = (int)std::max(framebmp.nx(), framebmp.ny()); + if(framecells > per_frame_cell_budget && longside > 1) + { + //cells scale with pixel area, so shrink the long side by sqrt of the overshoot + int calib_res = (int)((double)longside * std::sqrt((double)per_frame_cell_budget / (double)framecells)); + calib_res = std::max(64, calib_res); + if(calib_res < longside) + { + frame_max_res = calib_res; + if(frame_min_res > frame_max_res) + { + frame_min_res = frame_max_res; //never let the floor exceed the cap + } + if(!is_quiet && debugmode!=-1) + { + printf("\nvideo: frame resolution capped at %dpx (measured %d KV cells at %dpx) so ~%d frames fit the %d-cell context budget\n", frame_max_res, framecells, longside, expected_frames, video_cell_budget); + } + mtmd::bitmap calibbmp(kcpp_resize_bitmap_bounds(framebmp.ptr.release(), frame_min_res, frame_max_res)); + if(calibbmp.ptr) + { + framebmp.ptr = std::move(calibbmp.ptr); + mtmd::input_chunks rechunks(mtmd_input_chunks_init()); + std::vector rebitmaps = { framebmp.ptr.get() }; + if(mtmd_tokenize(mtmd_ctx, rechunks.ptr.get(), &inp_txt, rebitmaps.data(), rebitmaps.size()) == 0) + { + chunks.ptr = std::move(rechunks.ptr); + } + } + } + } + } + //exact per-frame guard: measured cells, so it holds even where the calibration misestimates if(mediacellsneeded + chunks_kv_cells(chunks) > video_cell_budget) { cell_budget_hit = true; From ec16b8d62185c51813473590ba6c3381f2673e67 Mon Sep 17 00:00:00 2001 From: Reithan Date: Fri, 24 Jul 2026 00:30:50 -0700 Subject: [PATCH 28/29] iterate frame resolution calibration via interpolation search on measured cells --- gpttype_adapter.cpp | 64 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 35204bd2c837..c1c8595ba9ac 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -5839,26 +5839,68 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int } //calibrate the resolution cap off the first frame's measured cell cost - tokenization is //cheap (no encoder run) and exact, and every frame of a video shares one resolution, so - //one measurement holds for the rest. only shrinks when the frame overshoots its share. + //one calibration holds for the rest. cell cost scales with pixel area (a token covers a + //fixed pixel patch), so this is an interpolation search in area space: one probe in the + //common case, extra probes only inside an encoder's internal token clamp where measured + //cost stops tracking area. only ever shrinks - a frame within its share stays untouched. if(frames_processed == 0) { - const int framecells = chunks_kv_cells(chunks); + const int orig_cells = chunks_kv_cells(chunks); const int longside = (int)std::max(framebmp.nx(), framebmp.ny()); - if(framecells > per_frame_cell_budget && longside > 1) + if(orig_cells > per_frame_cell_budget && longside > 1) { - //cells scale with pixel area, so shrink the long side by sqrt of the overshoot - int calib_res = (int)((double)longside * std::sqrt((double)per_frame_cell_budget / (double)framecells)); - calib_res = std::max(64, calib_res); - if(calib_res < longside) + int probe_res = longside; + int probe_cells = orig_cells; + int best_res = 0; //resolution of the probe that fit the share, 0 = none fit + int low_res = 0; //cheapest failing probe, fallback when a min-token floor blocks fitting + int low_cells = orig_cells; + for(int probes=0; probes<5 && probe_cells > per_frame_cell_budget; ++probes) { - frame_max_res = calib_res; + int next_res = (int)((double)probe_res * std::sqrt((double)per_frame_cell_budget / (double)probe_cells)); + next_res = std::max(64, next_res); + if(next_res >= probe_res) + { + break; //can't shrink further + } + probe_res = next_res; + //probe a throwaway copy so the final resize is made straight from the original frame + mtmd::bitmap probebmp(mtmd_bitmap_init(framebmp.nx(), framebmp.ny(), framebmp.data())); + probebmp.ptr.reset(kcpp_resize_bitmap_bounds(probebmp.ptr.release(), 0, probe_res)); + if(!probebmp.ptr) + { + break; + } + mtmd::input_chunks probechunks(mtmd_input_chunks_init()); + std::vector probebitmaps = { probebmp.ptr.get() }; + if(mtmd_tokenize(mtmd_ctx, probechunks.ptr.get(), &inp_txt, probebitmaps.data(), probebitmaps.size()) != 0) + { + break; + } + probe_cells = chunks_kv_cells(probechunks); + if(probe_cells <= per_frame_cell_budget) + { + best_res = probe_res; + } + else if(probe_cells < low_cells) + { + low_cells = probe_cells; + low_res = probe_res; + } + } + //when no probe fit the share, an encoder min-token floor may still have bottomed the + //cost out (take the cheapest probed size), or a fixed-output encoder ignores resolution + //entirely (keep full quality); either way the exact budget guard below bounds the total. + const int chosen_res = (best_res > 0) ? best_res : ((low_cells < orig_cells) ? low_res : 0); + if(chosen_res > 0) + { + frame_max_res = chosen_res; if(frame_min_res > frame_max_res) { frame_min_res = frame_max_res; //never let the floor exceed the cap } if(!is_quiet && debugmode!=-1) { - printf("\nvideo: frame resolution capped at %dpx (measured %d KV cells at %dpx) so ~%d frames fit the %d-cell context budget\n", frame_max_res, framecells, longside, expected_frames, video_cell_budget); + printf("\nvideo: frame resolution capped at %dpx (%d KV cells, was %d at %dpx) so ~%d frames fit the %d-cell context budget\n", frame_max_res, (best_res > 0) ? probe_cells : low_cells, orig_cells, longside, expected_frames, video_cell_budget); } mtmd::bitmap calibbmp(kcpp_resize_bitmap_bounds(framebmp.ptr.release(), frame_min_res, frame_max_res)); if(calibbmp.ptr) @@ -5872,6 +5914,10 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int } } } + else if(!is_quiet && debugmode!=-1) + { + printf("\nvideo: frame cell cost (%d) does not shrink with resolution for this model; keeping %dpx frames, the cell budget may truncate frame count\n", orig_cells, longside); + } } } //exact per-frame guard: measured cells, so it holds even where the calibration misestimates From 6c91637d4405fa748f270ab4623131133db565cc Mon Sep 17 00:00:00 2001 From: Reithan Date: Fri, 24 Jul 2026 00:39:50 -0700 Subject: [PATCH 29/29] make frame resolution calibration bidirectional with bracketed probes --- gpttype_adapter.cpp | 71 +++++++++++++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index c1c8595ba9ac..40c6c181706c 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -5842,30 +5842,57 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int //one calibration holds for the rest. cell cost scales with pixel area (a token covers a //fixed pixel patch), so this is an interpolation search in area space: one probe in the //common case, extra probes only inside an encoder's internal token clamp where measured - //cost stops tracking area. only ever shrinks - a frame within its share stays untouched. + //cost stops tracking area. a frame already within its share never enters this block, and + //probes are bracketed strictly below the original size, so frames are never upscaled. if(frames_processed == 0) { const int orig_cells = chunks_kv_cells(chunks); const int longside = (int)std::max(framebmp.nx(), framebmp.ny()); if(orig_cells > per_frame_cell_budget && longside > 1) { + //bracket: hi = smallest resolution known to exceed the share (starts at the original + //size, probes stay strictly below it), best = largest known to fit. interpolate toward + //the share from either side and accept once a fit lands within 90% of it, so an + //undershooting probe gets refined back upward instead of locked in. + int hi_res = longside; + int hi_cells = orig_cells; + int best_res = 0; + int best_cells = 0; int probe_res = longside; int probe_cells = orig_cells; - int best_res = 0; //resolution of the probe that fit the share, 0 = none fit - int low_res = 0; //cheapest failing probe, fallback when a min-token floor blocks fitting - int low_cells = orig_cells; - for(int probes=0; probes<5 && probe_cells > per_frame_cell_budget; ++probes) + int prev_res = -1; + int prev_cells = -1; + for(int probes=0; probes<6; ++probes) { - int next_res = (int)((double)probe_res * std::sqrt((double)per_frame_cell_budget / (double)probe_cells)); - next_res = std::max(64, next_res); - if(next_res >= probe_res) + if(best_res > 0 && best_cells * 10 >= per_frame_cell_budget * 9) { - break; //can't shrink further + break; //close enough to the share, stop refining + } + int next_res; + if(prev_res > 0 && prev_res != probe_res && prev_cells == probe_cells) + { + //two resolutions measured an identical cost (inside an encoder's internal + //clamp), interpolation has no signal here - step geometrically to escape + next_res = probe_res / 2; + } + else + { + next_res = (int)((double)probe_res * std::sqrt((double)per_frame_cell_budget / (double)probe_cells)); + } + const int res_floor = std::max(64, best_res + 1); + const int res_ceil = hi_res - 1; + if(res_floor > res_ceil) + { + break; //bracket exhausted + } + next_res = std::min(std::max(next_res, res_floor), res_ceil); + if(next_res == probe_res) + { + break; //nothing new to learn } - probe_res = next_res; //probe a throwaway copy so the final resize is made straight from the original frame mtmd::bitmap probebmp(mtmd_bitmap_init(framebmp.nx(), framebmp.ny(), framebmp.data())); - probebmp.ptr.reset(kcpp_resize_bitmap_bounds(probebmp.ptr.release(), 0, probe_res)); + probebmp.ptr.reset(kcpp_resize_bitmap_bounds(probebmp.ptr.release(), 0, next_res)); if(!probebmp.ptr) { break; @@ -5876,21 +5903,29 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int { break; } + prev_res = probe_res; + prev_cells = probe_cells; + probe_res = next_res; probe_cells = chunks_kv_cells(probechunks); if(probe_cells <= per_frame_cell_budget) { - best_res = probe_res; + if(probe_res > best_res) + { + best_res = probe_res; + best_cells = probe_cells; + } } - else if(probe_cells < low_cells) + else if(probe_res < hi_res) { - low_cells = probe_cells; - low_res = probe_res; + hi_res = probe_res; + hi_cells = probe_cells; } } //when no probe fit the share, an encoder min-token floor may still have bottomed the - //cost out (take the cheapest probed size), or a fixed-output encoder ignores resolution + //cost out (take the cheapest failing size), or a fixed-output encoder ignores resolution //entirely (keep full quality); either way the exact budget guard below bounds the total. - const int chosen_res = (best_res > 0) ? best_res : ((low_cells < orig_cells) ? low_res : 0); + const int chosen_res = (best_res > 0) ? best_res : ((hi_cells < orig_cells) ? hi_res : 0); + const int chosen_cells = (best_res > 0) ? best_cells : hi_cells; if(chosen_res > 0) { frame_max_res = chosen_res; @@ -5900,7 +5935,7 @@ static void PrepareMediaEmbds(const int nctx, const std::vector & media_int } if(!is_quiet && debugmode!=-1) { - printf("\nvideo: frame resolution capped at %dpx (%d KV cells, was %d at %dpx) so ~%d frames fit the %d-cell context budget\n", frame_max_res, (best_res > 0) ? probe_cells : low_cells, orig_cells, longside, expected_frames, video_cell_budget); + printf("\nvideo: frame resolution capped at %dpx (%d KV cells, was %d at %dpx) so ~%d frames fit the %d-cell context budget\n", frame_max_res, chosen_cells, orig_cells, longside, expected_frames, video_cell_budget); } mtmd::bitmap calibbmp(kcpp_resize_bitmap_bounds(framebmp.ptr.release(), frame_min_res, frame_max_res)); if(calibbmp.ptr)