Escape the same characters in the highlighter

A lexer may hand back the bytes of one separator, or the CR and LF of a
line ending, in different tokens, so line endings become line feeds
before lexing and the separators are escaped once the tokens are joined.

4 files changed, 87 insertions, 6 deletions

diff --git a/MANUAL.txt b/MANUAL.txtindex 8060528..59e4b04 100644--- a/MANUAL.txt+++ b/MANUAL.txt@@ -861,6 +861,14 @@ source filter:: 	be filtered is available on standard input and the filtered contents is 	expected on standard output. +	The line numbers beside the output are counted from the line feeds of+	the input, so the output has to break lines exactly where those fall.+	A browser also starts a line at a carriage return with no line feed+	after it, and may at a vertical tab, a form feed, or the Unicode next+	line, line separator and paragraph separator, which is why cgit and+	syntax-highlight.lua show each of those as an escape sequence in+	span.escape.+ trailer filter:: 	This filter is given two parameters, the key of the trailer being 	formatted, such as "Fixes", and a string indicating the originating
diff --git a/MANUAL.txt b/MANUAL.txt
index 8060528..59e4b04 100644
--- a/MANUAL.txt
+++ b/MANUAL.txt
@@ -861,6 +861,14 @@ source filter::
 	be filtered is available on standard input and the filtered contents is
 	expected on standard output.
 
+	The line numbers beside the output are counted from the line feeds of
+	the input, so the output has to break lines exactly where those fall.
+	A browser also starts a line at a carriage return with no line feed
+	after it, and may at a vertical tab, a form feed, or the Unicode next
+	line, line separator and paragraph separator, which is why cgit and
+	syntax-highlight.lua show each of those as an escape sequence in
+	span.escape.
+
 trailer filter::
 	This filter is given two parameters, the key of the trailer being
 	formatted, such as "Fixes", and a string indicating the originating
diff --git a/custom/extensions/syntax-highlight.lua b/custom/extensions/syntax-highlight.luaindex fdc4fa0..c178f25 100644--- a/custom/extensions/syntax-highlight.lua+++ b/custom/extensions/syntax-highlight.lua@@ -95,10 +95,32 @@ local scintillua = nil local filename = "" local chunks = {} -local escape_map = { ["&"] = "&amp;", ["<"] = "&lt;", [">"] = "&gt;" }+-- Besides the markup characters, everything a browser may start a new line+-- at is shown as an escape sequence, since the line numbers beside the code+-- count only line feeds. cgit escapes a blob it serves unhighlighted the same+-- way.+local escape_map = {+	["&"] = "&amp;",+	["<"] = "&lt;",+	[">"] = "&gt;",+	["\r"] = "<span class='escape'>\\r</span>",+	["\v"] = "<span class='escape'>\\v</span>",+	["\f"] = "<span class='escape'>\\f</span>",+	["\194\133"] = "<span class='escape'>\\u0085</span>",+	["\226\128\168"] = "<span class='escape'>\\u2028</span>",+	["\226\128\169"] = "<span class='escape'>\\u2029</span>",+}  local function escape(s)-	return (string.gsub(s, "[&<>]", escape_map))+	return (string.gsub(s, "[&<>\r\v\f]", escape_map))+end++-- The next line and the two Unicode separators take several bytes, which a+-- lexer may hand back in different tokens, so they are escaped once the+-- tokens are joined again.+local function escape_separators(s)+	s = string.gsub(s, "\194\133", escape_map)+	return (string.gsub(s, "\226\128[\168\169]", escape_map)) end  local function scintillua_path()@@ -208,7 +230,7 @@ local function highlight(text) 	if pos <= #text then 		out[#out + 1] = escape(string.sub(text, pos)) 	end-	return table.concat(out)+	return escape_separators(table.concat(out)) end  function filter_open(name)@@ -226,8 +248,13 @@ end -- source. function filter_close() 	local text = table.concat(chunks)+	local fits = #text <= max_bytes 	chunks = {}-	if #text <= max_bytes then+	-- A browser reads CR LF as one line feed only while the two stay+	-- together, and a lexer could hand them back in different tokens.+	-- Every carriage return left after this is shown by escape().+	text = string.gsub(text, "\r\n", "\n")+	if fits then 		if scintillua == nil then 			scintillua = load_scintillua() or false 		end@@ -241,8 +268,14 @@ function filter_close() 	end 	local pos = 1 	while pos <= #text do-		html(escape(string.sub(text, pos, pos + slice_bytes - 1)))-		pos = pos + slice_bytes+		local stop = pos + slice_bytes - 1+		-- A slice runs on to the end of the character it would cut, so+		-- no separator is split where escape_separators cannot see it.+		while string.find(text, "^[\128-\191]", stop + 1) do+			stop = stop + 1+		end+		html(escape_separators(escape(string.sub(text, pos, stop))))+		pos = stop + 1 	end 	return 0 end
diff --git a/custom/extensions/syntax-highlight.lua b/custom/extensions/syntax-highlight.lua
index fdc4fa0..c178f25 100644
--- a/custom/extensions/syntax-highlight.lua
+++ b/custom/extensions/syntax-highlight.lua
@@ -95,10 +95,32 @@ local scintillua = nil
 local filename = ""
 local chunks = {}
 
-local escape_map = { ["&"] = "&amp;", ["<"] = "&lt;", [">"] = "&gt;" }
+-- Besides the markup characters, everything a browser may start a new line
+-- at is shown as an escape sequence, since the line numbers beside the code
+-- count only line feeds. cgit escapes a blob it serves unhighlighted the same
+-- way.
+local escape_map = {
+	["&"] = "&amp;",
+	["<"] = "&lt;",
+	[">"] = "&gt;",
+	["\r"] = "<span class='escape'>\\r</span>",
+	["\v"] = "<span class='escape'>\\v</span>",
+	["\f"] = "<span class='escape'>\\f</span>",
+	["\194\133"] = "<span class='escape'>\\u0085</span>",
+	["\226\128\168"] = "<span class='escape'>\\u2028</span>",
+	["\226\128\169"] = "<span class='escape'>\\u2029</span>",
+}
 
 local function escape(s)
-	return (string.gsub(s, "[&<>]", escape_map))
+	return (string.gsub(s, "[&<>\r\v\f]", escape_map))
+end
+
+-- The next line and the two Unicode separators take several bytes, which a
+-- lexer may hand back in different tokens, so they are escaped once the
+-- tokens are joined again.
+local function escape_separators(s)
+	s = string.gsub(s, "\194\133", escape_map)
+	return (string.gsub(s, "\226\128[\168\169]", escape_map))
 end
 
 local function scintillua_path()
@@ -208,7 +230,7 @@ local function highlight(text)
 	if pos <= #text then
 		out[#out + 1] = escape(string.sub(text, pos))
 	end
-	return table.concat(out)
+	return escape_separators(table.concat(out))
 end
 
 function filter_open(name)
@@ -226,8 +248,13 @@ end
 -- source.
 function filter_close()
 	local text = table.concat(chunks)
+	local fits = #text <= max_bytes
 	chunks = {}
-	if #text <= max_bytes then
+	-- A browser reads CR LF as one line feed only while the two stay
+	-- together, and a lexer could hand them back in different tokens.
+	-- Every carriage return left after this is shown by escape().
+	text = string.gsub(text, "\r\n", "\n")
+	if fits then
 		if scintillua == nil then
 			scintillua = load_scintillua() or false
 		end
@@ -241,8 +268,14 @@ function filter_close()
 	end
 	local pos = 1
 	while pos <= #text do
-		html(escape(string.sub(text, pos, pos + slice_bytes - 1)))
-		pos = pos + slice_bytes
+		local stop = pos + slice_bytes - 1
+		-- A slice runs on to the end of the character it would cut, so
+		-- no separator is split where escape_separators cannot see it.
+		while string.find(text, "^[\128-\191]", stop + 1) do
+			stop = stop + 1
+		end
+		html(escape_separators(escape(string.sub(text, pos, stop))))
+		pos = stop + 1
 	end
 	return 0
 end
diff --git a/tests/extensions/test-syntax-highlight.lua b/tests/extensions/test-syntax-highlight.luaindex a87cd80..f831b01 100644--- a/tests/extensions/test-syntax-highlight.lua+++ b/tests/extensions/test-syntax-highlight.lua@@ -33,6 +33,17 @@ if mode == "nolexers" then 	out = render("big.fake", string.rep("x<y>", 30000)) 	h.equals("slicing loses nothing at the boundaries", out, string.rep("x&lt;y&gt;", 30000)) +	out = render("hello.fake", "a\rb\vc\fd\r\ne")+	h.equals("the fallback escapes what a browser breaks at", out,+		"a<span class='escape'>\\r</span>b<span class='escape'>\\v</span>c" ..+		"<span class='escape'>\\f</span>d\ne")++	-- The line separator starts on the last byte of the first slice, and+	-- its other two bytes would land in the next one.+	out = render("big.fake", string.rep("x", 64 * 1024 - 1) .. "\226\128\168y")+	h.equals("a separator across a slice boundary is still escaped", out,+		string.rep("x", 64 * 1024 - 1) .. "<span class='escape'>\\u2028</span>y")+ 	h.finish() end @@ -70,6 +81,24 @@ h.equals("an extension with no lexer serves escaped text", out, "import &lt;x&gt out = render("noextension", "words <here>") h.equals("a file with no extension serves escaped text", out, "words &lt;here&gt;") +-- The line numbers beside the code count line feeds alone, so each character+-- a browser would start a line at is an escape, and CR LF reaches the page as+-- the line feed a browser would have made of it.+out = render("hello.fake", "if\r42\vzz\f'a'\r\nif")+h.equals("what a browser breaks at is escaped between tokens", out,+	"<span class='hl-keyword'>if</span><span class='escape'>\\r</span>" ..+	"<span class='hl-number'>42</span><span class='escape'>\\v</span>" ..+	"<span class='hl-keyword'>zz</span><span class='escape'>\\f</span>" ..+	"<span class='hl-string'>'a'</span>\n<span class='hl-keyword'>if</span>")++-- The fake lexer hands back every byte of a separator as a token of its own.+out = render("hello.fake", "a\194\133b\226\128\168c\226\128\169d")+h.equals("a separator split between tokens is escaped whole", out,+	"<span class='hl-keyword'>a</span><span class='escape'>\\u0085</span>" ..+	"<span class='hl-keyword'>b</span><span class='escape'>\\u2028</span>" ..+	"<span class='hl-keyword'>c</span><span class='escape'>\\u2029</span>" ..+	"<span class='hl-keyword'>d</span>")+ out = render("big.fake", string.rep("a", 512 * 1024 + 1)) h.excludes("an oversized blob skips the lexing pass", out, "<span") h.equals("and still arrives whole", #out, 512 * 1024 + 1)
diff --git a/tests/extensions/test-syntax-highlight.lua b/tests/extensions/test-syntax-highlight.lua
index a87cd80..f831b01 100644
--- a/tests/extensions/test-syntax-highlight.lua
+++ b/tests/extensions/test-syntax-highlight.lua
@@ -33,6 +33,17 @@ if mode == "nolexers" then
 	out = render("big.fake", string.rep("x<y>", 30000))
 	h.equals("slicing loses nothing at the boundaries", out, string.rep("x&lt;y&gt;", 30000))
 
+	out = render("hello.fake", "a\rb\vc\fd\r\ne")
+	h.equals("the fallback escapes what a browser breaks at", out,
+		"a<span class='escape'>\\r</span>b<span class='escape'>\\v</span>c" ..
+		"<span class='escape'>\\f</span>d\ne")
+
+	-- The line separator starts on the last byte of the first slice, and
+	-- its other two bytes would land in the next one.
+	out = render("big.fake", string.rep("x", 64 * 1024 - 1) .. "\226\128\168y")
+	h.equals("a separator across a slice boundary is still escaped", out,
+		string.rep("x", 64 * 1024 - 1) .. "<span class='escape'>\\u2028</span>y")
+
 	h.finish()
 end
 
@@ -70,6 +81,24 @@ h.equals("an extension with no lexer serves escaped text", out, "import &lt;x&gt
 out = render("noextension", "words <here>")
 h.equals("a file with no extension serves escaped text", out, "words &lt;here&gt;")
 
+-- The line numbers beside the code count line feeds alone, so each character
+-- a browser would start a line at is an escape, and CR LF reaches the page as
+-- the line feed a browser would have made of it.
+out = render("hello.fake", "if\r42\vzz\f'a'\r\nif")
+h.equals("what a browser breaks at is escaped between tokens", out,
+	"<span class='hl-keyword'>if</span><span class='escape'>\\r</span>" ..
+	"<span class='hl-number'>42</span><span class='escape'>\\v</span>" ..
+	"<span class='hl-keyword'>zz</span><span class='escape'>\\f</span>" ..
+	"<span class='hl-string'>'a'</span>\n<span class='hl-keyword'>if</span>")
+
+-- The fake lexer hands back every byte of a separator as a token of its own.
+out = render("hello.fake", "a\194\133b\226\128\168c\226\128\169d")
+h.equals("a separator split between tokens is escaped whole", out,
+	"<span class='hl-keyword'>a</span><span class='escape'>\\u0085</span>" ..
+	"<span class='hl-keyword'>b</span><span class='escape'>\\u2028</span>" ..
+	"<span class='hl-keyword'>c</span><span class='escape'>\\u2029</span>" ..
+	"<span class='hl-keyword'>d</span>")
+
 out = render("big.fake", string.rep("a", 512 * 1024 + 1))
 h.excludes("an oversized blob skips the lexing pass", out, "<span")
 h.equals("and still arrives whole", #out, 512 * 1024 + 1)
diff --git a/tests/t0502-syntax-highlight.sh b/tests/t0502-syntax-highlight.shindex 31ab7e7..ced9edd 100755--- a/tests/t0502-syntax-highlight.sh+++ b/tests/t0502-syntax-highlight.sh@@ -70,4 +70,15 @@ test_expect_success CGIT_LUA 'bytes between tokens reach the page escaped' ' 	grep "<span class=.hl-number.>42</span> &amp; " tmp ' +test_expect_success CGIT_LUA 'a lone carriage return reaches the page as an escape' '+	(+		cd repos/fake &&+		printf "if\\rzz\\n" >cr.fake &&+		git add cr.fake &&+		git commit -m "add a lone carriage return"+	) &&+	cgit_url "fake/tree/cr.fake" >tmp &&+	grep "<span class=.hl-keyword.>if</span><span class=.escape.>\\\\r</span>" tmp+'+ test_done
diff --git a/tests/t0502-syntax-highlight.sh b/tests/t0502-syntax-highlight.sh
index 31ab7e7..ced9edd 100755
--- a/tests/t0502-syntax-highlight.sh
+++ b/tests/t0502-syntax-highlight.sh
@@ -70,4 +70,15 @@ test_expect_success CGIT_LUA 'bytes between tokens reach the page escaped' '
 	grep "<span class=.hl-number.>42</span> &amp; " tmp
 '
 
+test_expect_success CGIT_LUA 'a lone carriage return reaches the page as an escape' '
+	(
+		cd repos/fake &&
+		printf "if\\rzz\\n" >cr.fake &&
+		git add cr.fake &&
+		git commit -m "add a lone carriage return"
+	) &&
+	cgit_url "fake/tree/cr.fake" >tmp &&
+	grep "<span class=.hl-keyword.>if</span><span class=.escape.>\\\\r</span>" tmp
+'
+
 test_done