Escape the same characters in the highlighter

A lexer may hand back the bytes of one separator, or the CR and LF of a
line ending, in different tokens, so line endings become line feeds
before lexing and the separators are escaped once the tokens are joined.

1 file changed, 39 insertions, 6 deletions

diff --git a/custom/extensions/syntax-highlight.lua b/custom/extensions/syntax-highlight.luaindex fdc4fa0..c178f25 100644--- a/custom/extensions/syntax-highlight.lua+++ b/custom/extensions/syntax-highlight.lua@@ -95,10 +95,32 @@ local scintillua = nil local filename = "" local chunks = {} -local escape_map = { ["&"] = "&amp;", ["<"] = "&lt;", [">"] = "&gt;" }+-- Besides the markup characters, everything a browser may start a new line+-- at is shown as an escape sequence, since the line numbers beside the code+-- count only line feeds. cgit escapes a blob it serves unhighlighted the same+-- way.+local escape_map = {+	["&"] = "&amp;",+	["<"] = "&lt;",+	[">"] = "&gt;",+	["\r"] = "<span class='escape'>\\r</span>",+	["\v"] = "<span class='escape'>\\v</span>",+	["\f"] = "<span class='escape'>\\f</span>",+	["\194\133"] = "<span class='escape'>\\u0085</span>",+	["\226\128\168"] = "<span class='escape'>\\u2028</span>",+	["\226\128\169"] = "<span class='escape'>\\u2029</span>",+}  local function escape(s)-	return (string.gsub(s, "[&<>]", escape_map))+	return (string.gsub(s, "[&<>\r\v\f]", escape_map))+end++-- The next line and the two Unicode separators take several bytes, which a+-- lexer may hand back in different tokens, so they are escaped once the+-- tokens are joined again.+local function escape_separators(s)+	s = string.gsub(s, "\194\133", escape_map)+	return (string.gsub(s, "\226\128[\168\169]", escape_map)) end  local function scintillua_path()@@ -208,7 +230,7 @@ local function highlight(text) 	if pos <= #text then 		out[#out + 1] = escape(string.sub(text, pos)) 	end-	return table.concat(out)+	return escape_separators(table.concat(out)) end  function filter_open(name)@@ -226,8 +248,13 @@ end -- source. function filter_close() 	local text = table.concat(chunks)+	local fits = #text <= max_bytes 	chunks = {}-	if #text <= max_bytes then+	-- A browser reads CR LF as one line feed only while the two stay+	-- together, and a lexer could hand them back in different tokens.+	-- Every carriage return left after this is shown by escape().+	text = string.gsub(text, "\r\n", "\n")+	if fits then 		if scintillua == nil then 			scintillua = load_scintillua() or false 		end@@ -241,8 +268,14 @@ function filter_close() 	end 	local pos = 1 	while pos <= #text do-		html(escape(string.sub(text, pos, pos + slice_bytes - 1)))-		pos = pos + slice_bytes+		local stop = pos + slice_bytes - 1+		-- A slice runs on to the end of the character it would cut, so+		-- no separator is split where escape_separators cannot see it.+		while string.find(text, "^[\128-\191]", stop + 1) do+			stop = stop + 1+		end+		html(escape_separators(escape(string.sub(text, pos, stop))))+		pos = stop + 1 	end 	return 0 end
diff --git a/custom/extensions/syntax-highlight.lua b/custom/extensions/syntax-highlight.lua
index fdc4fa0..c178f25 100644
--- a/custom/extensions/syntax-highlight.lua
+++ b/custom/extensions/syntax-highlight.lua
@@ -95,10 +95,32 @@ local scintillua = nil
 local filename = ""
 local chunks = {}
 
-local escape_map = { ["&"] = "&amp;", ["<"] = "&lt;", [">"] = "&gt;" }
+-- Besides the markup characters, everything a browser may start a new line
+-- at is shown as an escape sequence, since the line numbers beside the code
+-- count only line feeds. cgit escapes a blob it serves unhighlighted the same
+-- way.
+local escape_map = {
+	["&"] = "&amp;",
+	["<"] = "&lt;",
+	[">"] = "&gt;",
+	["\r"] = "<span class='escape'>\\r</span>",
+	["\v"] = "<span class='escape'>\\v</span>",
+	["\f"] = "<span class='escape'>\\f</span>",
+	["\194\133"] = "<span class='escape'>\\u0085</span>",
+	["\226\128\168"] = "<span class='escape'>\\u2028</span>",
+	["\226\128\169"] = "<span class='escape'>\\u2029</span>",
+}
 
 local function escape(s)
-	return (string.gsub(s, "[&<>]", escape_map))
+	return (string.gsub(s, "[&<>\r\v\f]", escape_map))
+end
+
+-- The next line and the two Unicode separators take several bytes, which a
+-- lexer may hand back in different tokens, so they are escaped once the
+-- tokens are joined again.
+local function escape_separators(s)
+	s = string.gsub(s, "\194\133", escape_map)
+	return (string.gsub(s, "\226\128[\168\169]", escape_map))
 end
 
 local function scintillua_path()
@@ -208,7 +230,7 @@ local function highlight(text)
 	if pos <= #text then
 		out[#out + 1] = escape(string.sub(text, pos))
 	end
-	return table.concat(out)
+	return escape_separators(table.concat(out))
 end
 
 function filter_open(name)
@@ -226,8 +248,13 @@ end
 -- source.
 function filter_close()
 	local text = table.concat(chunks)
+	local fits = #text <= max_bytes
 	chunks = {}
-	if #text <= max_bytes then
+	-- A browser reads CR LF as one line feed only while the two stay
+	-- together, and a lexer could hand them back in different tokens.
+	-- Every carriage return left after this is shown by escape().
+	text = string.gsub(text, "\r\n", "\n")
+	if fits then
 		if scintillua == nil then
 			scintillua = load_scintillua() or false
 		end
@@ -241,8 +268,14 @@ function filter_close()
 	end
 	local pos = 1
 	while pos <= #text do
-		html(escape(string.sub(text, pos, pos + slice_bytes - 1)))
-		pos = pos + slice_bytes
+		local stop = pos + slice_bytes - 1
+		-- A slice runs on to the end of the character it would cut, so
+		-- no separator is split where escape_separators cannot see it.
+		while string.find(text, "^[\128-\191]", stop + 1) do
+			stop = stop + 1
+		end
+		html(escape_separators(escape(string.sub(text, pos, stop))))
+		pos = stop + 1
 	end
 	return 0
 end