From 85e906e58202b2ae0ab782f6f411485acfd55f9e Mon Sep 17 00:00:00 2001 From: Bryce Kwon Date: Sat, 24 Oct 2026 09:17:41 -1000 Subject: Escape the same characters in the highlighter A lexer may hand back the bytes of one separator, or the CR and LF of a line ending, in different tokens, so line endings become line feeds before lexing and the separators are escaped once the tokens are joined. --- MANUAL.txt | 8 ++++++ custom/extensions/syntax-highlight.lua | 45 ++++++++++++++++++++++++++---- tests/extensions/test-syntax-highlight.lua | 29 +++++++++++++++++++ tests/t0502-syntax-highlight.sh | 11 ++++++++ 4 files changed, 87 insertions(+), 6 deletions(-) diff --git a/MANUAL.txt b/MANUAL.txt index 8060528..59e4b04 100644 --- a/MANUAL.txt +++ b/MANUAL.txt @@ -861,6 +861,14 @@ source filter:: be filtered is available on standard input and the filtered contents is expected on standard output. + The line numbers beside the output are counted from the line feeds of + the input, so the output has to break lines exactly where those fall. + A browser also starts a line at a carriage return with no line feed + after it, and may at a vertical tab, a form feed, or the Unicode next + line, line separator and paragraph separator, which is why cgit and + syntax-highlight.lua show each of those as an escape sequence in + span.escape. + trailer filter:: This filter is given two parameters, the key of the trailer being formatted, such as "Fixes", and a string indicating the originating diff --git a/custom/extensions/syntax-highlight.lua b/custom/extensions/syntax-highlight.lua index fdc4fa0..c178f25 100644 --- a/custom/extensions/syntax-highlight.lua +++ b/custom/extensions/syntax-highlight.lua @@ -95,10 +95,32 @@ local scintillua = nil local filename = "" local chunks = {} -local escape_map = { ["&"] = "&", ["<"] = "<", [">"] = ">" } +-- Besides the markup characters, everything a browser may start a new line +-- at is shown as an escape sequence, since the line numbers beside the code +-- count only line feeds. cgit escapes a blob it serves unhighlighted the same +-- way. +local escape_map = { + ["&"] = "&", + ["<"] = "<", + [">"] = ">", + ["\r"] = "\\r", + ["\v"] = "\\v", + ["\f"] = "\\f", + ["\194\133"] = "\\u0085", + ["\226\128\168"] = "\\u2028", + ["\226\128\169"] = "\\u2029", +} local function escape(s) - return (string.gsub(s, "[&<>]", escape_map)) + return (string.gsub(s, "[&<>\r\v\f]", escape_map)) +end + +-- The next line and the two Unicode separators take several bytes, which a +-- lexer may hand back in different tokens, so they are escaped once the +-- tokens are joined again. +local function escape_separators(s) + s = string.gsub(s, "\194\133", escape_map) + return (string.gsub(s, "\226\128[\168\169]", escape_map)) end local function scintillua_path() @@ -208,7 +230,7 @@ local function highlight(text) if pos <= #text then out[#out + 1] = escape(string.sub(text, pos)) end - return table.concat(out) + return escape_separators(table.concat(out)) end function filter_open(name) @@ -226,8 +248,13 @@ end -- source. function filter_close() local text = table.concat(chunks) + local fits = #text <= max_bytes chunks = {} - if #text <= max_bytes then + -- A browser reads CR LF as one line feed only while the two stay + -- together, and a lexer could hand them back in different tokens. + -- Every carriage return left after this is shown by escape(). + text = string.gsub(text, "\r\n", "\n") + if fits then if scintillua == nil then scintillua = load_scintillua() or false end @@ -241,8 +268,14 @@ function filter_close() end local pos = 1 while pos <= #text do - html(escape(string.sub(text, pos, pos + slice_bytes - 1))) - pos = pos + slice_bytes + local stop = pos + slice_bytes - 1 + -- A slice runs on to the end of the character it would cut, so + -- no separator is split where escape_separators cannot see it. + while string.find(text, "^[\128-\191]", stop + 1) do + stop = stop + 1 + end + html(escape_separators(escape(string.sub(text, pos, stop)))) + pos = stop + 1 end return 0 end diff --git a/tests/extensions/test-syntax-highlight.lua b/tests/extensions/test-syntax-highlight.lua index a87cd80..f831b01 100644 --- a/tests/extensions/test-syntax-highlight.lua +++ b/tests/extensions/test-syntax-highlight.lua @@ -33,6 +33,17 @@ if mode == "nolexers" then out = render("big.fake", string.rep("x", 30000)) h.equals("slicing loses nothing at the boundaries", out, string.rep("x<y>", 30000)) + out = render("hello.fake", "a\rb\vc\fd\r\ne") + h.equals("the fallback escapes what a browser breaks at", out, + "a\\rb\\vc" .. + "\\fd\ne") + + -- The line separator starts on the last byte of the first slice, and + -- its other two bytes would land in the next one. + out = render("big.fake", string.rep("x", 64 * 1024 - 1) .. "\226\128\168y") + h.equals("a separator across a slice boundary is still escaped", out, + string.rep("x", 64 * 1024 - 1) .. "\\u2028y") + h.finish() end @@ -70,6 +81,24 @@ h.equals("an extension with no lexer serves escaped text", out, "import <x> out = render("noextension", "words ") h.equals("a file with no extension serves escaped text", out, "words <here>") +-- The line numbers beside the code count line feeds alone, so each character +-- a browser would start a line at is an escape, and CR LF reaches the page as +-- the line feed a browser would have made of it. +out = render("hello.fake", "if\r42\vzz\f'a'\r\nif") +h.equals("what a browser breaks at is escaped between tokens", out, + "if\\r" .. + "42\\v" .. + "zz\\f" .. + "'a'\nif") + +-- The fake lexer hands back every byte of a separator as a token of its own. +out = render("hello.fake", "a\194\133b\226\128\168c\226\128\169d") +h.equals("a separator split between tokens is escaped whole", out, + "a\\u0085" .. + "b\\u2028" .. + "c\\u2029" .. + "d") + out = render("big.fake", string.rep("a", 512 * 1024 + 1)) h.excludes("an oversized blob skips the lexing pass", out, "42 & " tmp ' +test_expect_success CGIT_LUA 'a lone carriage return reaches the page as an escape' ' + ( + cd repos/fake && + printf "if\\rzz\\n" >cr.fake && + git add cr.fake && + git commit -m "add a lone carriage return" + ) && + cgit_url "fake/tree/cr.fake" >tmp && + grep "if\\\\r" tmp +' + test_done -- cgit 2.9.1