Add video input support for VLM inference

CMakeLists.txt

@@ -62,6 +62,7 @@ add_compile_definitions(GGML_USE_CPU_REPACK)
 add_compile_definitions(GGML_USE_RPC)
 add_compile_definitions(NOMINMAX)
 add_compile_definitions(_REGEX_MAX_STACK_COUNT=5000)
+add_compile_definitions(MTMD_VIDEO)
 
 if (GGML_HIP_FORCE_ROCWMMA_FATTN_GFX12)
     add_compile_definitions(GGML_HIP_ROCWMMA_FATTN_GFX12)

Makefile

@@ -56,7 +56,7 @@ CFLAGS += -fsanitize=undefined -fsanitize-undefined-trap-on-error
 CXXFLAGS += -fsanitize=undefined -fsanitize-undefined-trap-on-error
 endif
 CFLAGS   += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c11 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC
-CXXFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c++17 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC
+CXXFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c++17 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC -DMTMD_VIDEO
 
 ifndef KCPP_DEBUG
 CFLAGS += -DNDEBUG -s

embd_res/kcpp_docs.embd

@@ -197,6 +197,13 @@
                                "type": "string"
                             }
                          },
+                         "videos": {
+                            "description": "KoboldCpp ONLY. If set, takes an array of up to 2 base64 encoded strings, each one representing a video to be processed.",
+                            "type": "array",
+                            "items": {
+                               "type": "string"
+                            }
+                         },
                          "trim_stop": {
                             "default": true,
                             "description": "KoboldCpp ONLY. If true, also removes detected stop_sequences from the output and truncates all text after them. If false, output will also include stop sequence and potentially a few additional characters.",

expose.h

@@ -49,6 +49,11 @@ struct load_model_inputs
     const int visionmaxres = 2048;
     const int visionmintokens = -1;
     const int visionmaxtokens = -1;
+    const int videomaxframes = 32;
+    const float videofps = 2.0f;
+    const int videomintokens = -1;
+    const int videomaxtokens = -1;
+    const char * ffmpegpath = "";
     const bool use_mmap = false;
     const bool use_mlock = false;
     const bool use_mtp = false;
@@ -100,6 +105,8 @@ struct generation_inputs
     const char ** images = nullptr;
     const int audio_len = 0;
     const char ** audio = nullptr;
+    const int videos_len = 0;
+    const char ** videos = nullptr;
     const int max_context_length = 0;
     const int max_length = 0;
     const float temperature = 0.0f;