From c6f756eccb9a3af7e9d301f124db2db3595ffb06 Mon Sep 17 00:00:00 2001 From: Mohd Quamar Tyagi <104281681+Tyagiquamar@users.noreply.github.com> Date: Sun, 1 Nov 2026 15:49:37 +0530 Subject: [PATCH] fix(test-runner): split $$ heredocs and escaped backticks in splitQueries() The statement splitter in the upstream SQL test harness only knew single quotes, double quotes, backticks, backslash escapes outside backticks, and the two SQL comment styles. Two constructs in the server lexer were missing: - Dollar quoting ($$...$$ and $tag$...$tag$). A quote inside the heredoc toggled the splitter's quote state, so every following statement was merged into one, and a `;` inside the heredoc could split it apart. - A backslash-escaped backtick inside a quoted identifier, which closed the identifier early and left it open for the rest of the file. Both are implemented the way the server lexer does it (see src/Parsers/Lexer.cpp): the opening delimiter is `$`, an optional tag of word characters, and `$` again, and the heredoc runs to the next occurrence of that same delimiter; an unterminated delimiter is not a heredoc. Quoted identifiers now honor backslash escapes like string literals. Statement counts on the upstream files from #1096 now match the official lexer: 03011_definitive_guide_to_cast 82 -> 91, 04038_ascii_cjk_tokenizer 28 -> 43, 04097_tuple_inside_nullable_subcolumn_descendants 15 -> 18, and the 02683/02685/02686/02687/03032 heredoc files split at the right boundary. A 150-file sample of tests/queries/0_stateless produces unchanged output. --- .../__tests__/split-queries.test.ts | 64 +++++++++++++++++++ .../src/split-queries.ts | 44 ++++++++++++- 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/tests/clickhouse-test-runner/__tests__/split-queries.test.ts b/tests/clickhouse-test-runner/__tests__/split-queries.test.ts index c00537863..a0d5e8b6e 100644 --- a/tests/clickhouse-test-runner/__tests__/split-queries.test.ts +++ b/tests/clickhouse-test-runner/__tests__/split-queries.test.ts @@ -77,4 +77,68 @@ describe("splitQueries", () => { "SELECT 2", ]); }); + + it("does not split on quotes or semicolons inside a $$ heredoc", () => { + // Condensed 03032_save_bad_json_escape_sequences: the heredoc body holds + // double quotes and backslashes that must not toggle the quote state. + const sql = [ + "select * from format(JSONEachRow, $$", + '{"key" : "\\u"}', + '{"key" : "\\ud800\\"}', + "$$);", + "SELECT 2;", + ].join("\n"); + expect(splitQueries(sql)).toEqual([ + [ + "select * from format(JSONEachRow, $$", + '{"key" : "\\u"}', + '{"key" : "\\ud800\\"}', + "$$)", + ].join("\n"), + "SELECT 2", + ]); + }); + + it("does not split on quotes or semicolons inside a $tag$ heredoc", () => { + expect( + splitQueries("SELECT $doc$it's; a trap$doc$ AS s; SELECT 2"), + ).toEqual(["SELECT $doc$it's; a trap$doc$ AS s", "SELECT 2"]); + }); + + it("keeps the boundaries of statements that follow a heredoc", () => { + // 02685_bson2 and friends: the heredoc body itself contains an apostrophe. + expect(splitQueries("SELECT $$it's$$; SELECT 2; SELECT 3")).toEqual([ + "SELECT $$it's$$", + "SELECT 2", + "SELECT 3", + ]); + }); + + it("treats an unterminated $$ as ordinary characters", () => { + expect(splitQueries("SELECT $$; SELECT 2")).toEqual([ + "SELECT $$", + "SELECT 2", + ]); + }); + + it("does not treat other bare words starting with $ as heredocs", () => { + expect(splitQueries("SELECT $foo, '$bar'; SELECT 2")).toEqual([ + "SELECT $foo, '$bar'", + "SELECT 2", + ]); + }); + + it("handles a backslash-escaped backtick in a quoted identifier", () => { + // 04097_tuple_inside_nullable_subcolumn_descendants: the escaped backtick + // used to close the identifier early, leaving the identifier open and + // merging the rest of the file into one statement. + const sql = + "CREATE TABLE t (tup Nullable(Tuple(`a\\`b` Nullable(String)))) ENGINE = Memory;" + + " SELECT 1; SELECT 2"; + expect(splitQueries(sql)).toEqual([ + "CREATE TABLE t (tup Nullable(Tuple(`a\\`b` Nullable(String)))) ENGINE = Memory", + "SELECT 1", + "SELECT 2", + ]); + }); }); diff --git a/tests/clickhouse-test-runner/src/split-queries.ts b/tests/clickhouse-test-runner/src/split-queries.ts index 9c9d9e24b..4833f25ac 100644 --- a/tests/clickhouse-test-runner/src/split-queries.ts +++ b/tests/clickhouse-test-runner/src/split-queries.ts @@ -1,3 +1,34 @@ +/** `[A-Za-z0-9_]`, matching the server lexer's `isWordCharASCII`. */ +function isWordChar(code: number): boolean { + return ( + (code >= 0x61 && code <= 0x7a) || // a-z + (code >= 0x41 && code <= 0x5a) || // A-Z + (code >= 0x30 && code <= 0x39) || // 0-9 + code === 0x5f // _ + ); +} + +/** + * Returns the exclusive end offset of the dollar-quoted heredoc that starts at + * `start` (which points at a `$`), or `-1` when that `$` does not open one. + * + * Mirrors the server lexer: the opening delimiter is `$`, an optional tag of + * word characters, and `$` again (so `$$` and `$tag$`); the heredoc runs to the + * next occurrence of the same delimiter. An unterminated delimiter is not a + * heredoc, and its `$` is then an ordinary character. + */ +function findHeredocEnd(sql: string, start: number): number { + const tagEnd = sql.indexOf("$", start + 1); + if (tagEnd === -1) return -1; + for (let i = start + 1; i < tagEnd; i++) { + if (!isWordChar(sql.charCodeAt(i))) return -1; + } + const delimiter = sql.slice(start, tagEnd + 1); + const end = sql.indexOf(delimiter, tagEnd + 1); + if (end === -1) return -1; + return end + delimiter.length; +} + export function splitQueries(sql: string): string[] { const queries: string[] = []; let current = ""; @@ -16,7 +47,7 @@ export function splitQueries(sql: string): string[] { continue; } - if ((inSingleQuote || inDoubleQuote) && ch === "\\") { + if ((inSingleQuote || inDoubleQuote || inBacktick) && ch === "\\") { current += ch; escaping = true; continue; @@ -41,6 +72,17 @@ export function splitQueries(sql: string): string[] { i = end - 1; continue; } + // Dollar-quoted heredoc: $$ ... $$ or $tag$ ... $tag$. The server lexer + // reads the whole region as a single token, so quotes, comment markers + // and semicolons inside it must not affect the split. + if (ch === "$") { + const heredocEnd = findHeredocEnd(sql, i); + if (heredocEnd !== -1) { + current += sql.slice(i, heredocEnd); + i = heredocEnd - 1; + continue; + } + } } if (!inDoubleQuote && !inBacktick && ch === "'") {