Skip to content

server: support live prefix rewind for GLM-5.3 and preserve common prefix on forks #1005

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
aaa2015 wants to merge 2 commits into
base: main
Choose a base branch
from
+52 −16
Open

server: support live prefix rewind for GLM-5.3 and preserve common prefix on forks #1005

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
acb0976
server: support live prefix rewind for GLM-5.3 and preserve common pr…
aaa2015 Sep 9, 2026
20cda55
server: let live prefix rewind accept Qwen3.8 too
aaa2015 Sep 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion ds4.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71838,6 +71838,23 @@ const char *ds4_qwen4_reasoning_effort_text(ds4_think_mode mode) {
return NULL;
}

bool ds4_engine_can_rewind(ds4_engine *e) {
if (!e) return false;
/* The question here is narrow: does ds4_session_rewind() roll the engine
* back while keeping the checkpoint, or does it clear checkpoint_valid and
* force the very rebuild this helper exists to avoid? Keep this in step
* with the branches in ds4_session_rewind().
*
* GLM rolls back through its own frontier. Qwen3.8 restores a verify
* snapshot when one matches the position and otherwise resets the graph
* and replays the kept transcript -- either way it reports the checkpoint
* as preserved. DeepSeek's DSpark compressors cannot be rolled back by
* truncating their row counts and keep no frontier, so they stay out until
* that path can restore its state. */
return ds4_engine_is_glm_dsa(e) || ds4_engine_is_glm53(e) ||
ds4_engine_is_qwen4(e);
}

/* Decode gate firing schedule for the TP transport (see ds4_tp_identity).
* Resident GLM splits attention and FFN on sparse layers. Streaming keeps
* attention replicated and exchanges only the routed FFN partial. */
Expand Down Expand Up @@ -85175,11 +85192,12 @@ void ds4_session_rewind(ds4_session *s, int pos) {
} else {
qwen4_graph_reset(g);
}
/* Qwen eval replays the kept transcript if reset left the graph behind. */
state_ok = true;
s->qwen4_rewound = logit_row < 0;
}
#endif
/* This block only knows how to roll back GLM state; the glm53/MTP calls
* below are meaningless for other engines. */
if (s->checkpoint_valid && ds4_session_is_glm(s)) {
state_ok = !s->glm_graph.glm53 || ds4_session_glm_mtp_rewind(s, pos);
}
Expand Down
1 change: 1 addition & 0 deletions ds4.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,7 @@ bool ds4_engine_is_glm53(ds4_engine *e);
bool ds4_engine_is_qwen4(ds4_engine *e);
/* Qwen3.8 reasoning-effort system instruction for a think mode (NULL when none) */
const char *ds4_qwen4_reasoning_effort_text(ds4_think_mode mode);
bool ds4_engine_can_rewind(ds4_engine *e);
const char *ds4_backend_name(ds4_backend backend);
bool ds4_think_mode_enabled(ds4_think_mode mode);
int ds4_think_mode_level(ds4_think_mode mode);
Expand Down
47 changes: 32 additions & 15 deletions ds4_server.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11854,11 +11854,19 @@ static void trace_write_cache_diag(
}
}

/* Return the rewind target, or -1 when no live rewind applies. *full_prefix
* reports that the whole prompt is already cached, so the caller is handing the
* final prompt token back to the sampler. */
static int live_prefix_rewind_target(bool backend_can_rewind,
int old_pos, int prompt_len, int common) {
if (!backend_can_rewind || prompt_len <= 1 || prompt_len >= old_pos) return -1;
if (common != prompt_len) return -1;
return prompt_len - 1;
int old_pos, int prompt_len, int common,
bool *full_prefix) {
if (full_prefix) *full_prefix = false;
if (!backend_can_rewind || common <= 1 || common >= old_pos) return -1;
if (common == prompt_len) {
if (full_prefix) *full_prefix = true;
return prompt_len - 1;
}
return common;
}

static void trace_time(FILE *fp) {
Expand Down Expand Up @@ -13376,9 +13384,10 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) {
"Anthropic continuation state is not available; retry by replaying the full messages history");
return;
} else if (cached == 0 && live_vision_match) {
bool full_prefix = false;
const int rewind_to = live_prefix_rewind_target(
ds4_engine_is_glm_dsa(s->engine), old_pos,
j->req.prompt.len, common);
ds4_engine_can_rewind(s->engine), old_pos,
j->req.prompt.len, common, &full_prefix);
if (rewind_to >= 0) {
pthread_mutex_lock(&s->inference_mu);
ds4_session_rewind(slot->session, rewind_to);
Expand All @@ -13395,11 +13404,14 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) {
cache_source = "memory-rewind";
cache_diag.rewind_to = rewind_to;
server_log(DS4_LOG_KVCACHE,
"ds4-server: rewound GLM live prefix from %d to %d; final prompt token will be reevaluated",
old_pos, rewind_to);
"ds4-server: rewound live prefix from %d to %d; %s",
old_pos, rewind_to,
full_prefix ?
"final prompt token will be reevaluated" :
"suffix tokens will be evaluated");
} else {
server_log(DS4_LOG_KVCACHE,
"ds4-server: GLM live prefix rewind from %d to %d requires rebuild",
"ds4-server: live prefix rewind from %d to %d requires rebuild",
old_pos, rewind_to);
}
} else {
Expand Down Expand Up @@ -20214,12 +20226,17 @@ static void test_model_metadata_clamps_completion_to_context(void) {
}

static void test_live_prefix_rewind_target(void) {
TEST_ASSERT(live_prefix_rewind_target(true, 17, 8, 8) == 7);
TEST_ASSERT(live_prefix_rewind_target(true, 49826, 48379, 48379) == 48378);
TEST_ASSERT(live_prefix_rewind_target(false, 17, 8, 8) == -1);
TEST_ASSERT(live_prefix_rewind_target(true, 17, 8, 7) == -1);
TEST_ASSERT(live_prefix_rewind_target(true, 8, 8, 8) == -1);
TEST_ASSERT(live_prefix_rewind_target(true, 17, 1, 1) == -1);
bool full = false;
TEST_ASSERT(live_prefix_rewind_target(true, 17, 8, 8, &full) == 7 && full);
TEST_ASSERT(live_prefix_rewind_target(true, 49826, 48379, 48379, &full) == 48378 && full);
TEST_ASSERT(live_prefix_rewind_target(false, 17, 8, 8, &full) == -1 && !full);
TEST_ASSERT(live_prefix_rewind_target(true, 17, 8, 7, &full) == 7 && !full);
TEST_ASSERT(live_prefix_rewind_target(true, 165755, 4977, 4973, &full) == 4973 && !full);
TEST_ASSERT(live_prefix_rewind_target(true, 8, 8, 8, &full) == -1 && !full);
TEST_ASSERT(live_prefix_rewind_target(true, 17, 1, 1, &full) == -1 && !full);
/* The out-parameter is optional: callers that only need the target pass NULL. */
TEST_ASSERT(live_prefix_rewind_target(true, 17, 8, 8, NULL) == 7);
TEST_ASSERT(live_prefix_rewind_target(false, 17, 8, 8, NULL) == -1);
}

static void test_client_socket_nonblocking_flag(void) {
Expand Down