Escape the same characters in the highlighter

A lexer may hand back the bytes of one separator, or the CR and LF of a
line ending, in different tokens, so line endings become line feeds
before lexing and the separators are escaped once the tokens are joined.

1 file changed, 29 insertions, 0 deletions

diff --git a/tests/extensions/test-syntax-highlight.lua b/tests/extensions/test-syntax-highlight.luaindex a87cd80..f831b01 100644--- a/tests/extensions/test-syntax-highlight.lua+++ b/tests/extensions/test-syntax-highlight.lua@@ -33,6 +33,17 @@ if mode == "nolexers" then 	out = render("big.fake", string.rep("x<y>", 30000)) 	h.equals("slicing loses nothing at the boundaries", out, string.rep("x&lt;y&gt;", 30000)) +	out = render("hello.fake", "a\rb\vc\fd\r\ne")+	h.equals("the fallback escapes what a browser breaks at", out,+		"a<span class='escape'>\\r</span>b<span class='escape'>\\v</span>c" ..+		"<span class='escape'>\\f</span>d\ne")++	-- The line separator starts on the last byte of the first slice, and+	-- its other two bytes would land in the next one.+	out = render("big.fake", string.rep("x", 64 * 1024 - 1) .. "\226\128\168y")+	h.equals("a separator across a slice boundary is still escaped", out,+		string.rep("x", 64 * 1024 - 1) .. "<span class='escape'>\\u2028</span>y")+ 	h.finish() end @@ -70,6 +81,24 @@ h.equals("an extension with no lexer serves escaped text", out, "import &lt;x&gt out = render("noextension", "words <here>") h.equals("a file with no extension serves escaped text", out, "words &lt;here&gt;") +-- The line numbers beside the code count line feeds alone, so each character+-- a browser would start a line at is an escape, and CR LF reaches the page as+-- the line feed a browser would have made of it.+out = render("hello.fake", "if\r42\vzz\f'a'\r\nif")+h.equals("what a browser breaks at is escaped between tokens", out,+	"<span class='hl-keyword'>if</span><span class='escape'>\\r</span>" ..+	"<span class='hl-number'>42</span><span class='escape'>\\v</span>" ..+	"<span class='hl-keyword'>zz</span><span class='escape'>\\f</span>" ..+	"<span class='hl-string'>'a'</span>\n<span class='hl-keyword'>if</span>")++-- The fake lexer hands back every byte of a separator as a token of its own.+out = render("hello.fake", "a\194\133b\226\128\168c\226\128\169d")+h.equals("a separator split between tokens is escaped whole", out,+	"<span class='hl-keyword'>a</span><span class='escape'>\\u0085</span>" ..+	"<span class='hl-keyword'>b</span><span class='escape'>\\u2028</span>" ..+	"<span class='hl-keyword'>c</span><span class='escape'>\\u2029</span>" ..+	"<span class='hl-keyword'>d</span>")+ out = render("big.fake", string.rep("a", 512 * 1024 + 1)) h.excludes("an oversized blob skips the lexing pass", out, "<span") h.equals("and still arrives whole", #out, 512 * 1024 + 1)
diff --git a/tests/extensions/test-syntax-highlight.lua b/tests/extensions/test-syntax-highlight.lua
index a87cd80..f831b01 100644
--- a/tests/extensions/test-syntax-highlight.lua
+++ b/tests/extensions/test-syntax-highlight.lua
@@ -33,6 +33,17 @@ if mode == "nolexers" then
 	out = render("big.fake", string.rep("x<y>", 30000))
 	h.equals("slicing loses nothing at the boundaries", out, string.rep("x&lt;y&gt;", 30000))
 
+	out = render("hello.fake", "a\rb\vc\fd\r\ne")
+	h.equals("the fallback escapes what a browser breaks at", out,
+		"a<span class='escape'>\\r</span>b<span class='escape'>\\v</span>c" ..
+		"<span class='escape'>\\f</span>d\ne")
+
+	-- The line separator starts on the last byte of the first slice, and
+	-- its other two bytes would land in the next one.
+	out = render("big.fake", string.rep("x", 64 * 1024 - 1) .. "\226\128\168y")
+	h.equals("a separator across a slice boundary is still escaped", out,
+		string.rep("x", 64 * 1024 - 1) .. "<span class='escape'>\\u2028</span>y")
+
 	h.finish()
 end
 
@@ -70,6 +81,24 @@ h.equals("an extension with no lexer serves escaped text", out, "import &lt;x&gt
 out = render("noextension", "words <here>")
 h.equals("a file with no extension serves escaped text", out, "words &lt;here&gt;")
 
+-- The line numbers beside the code count line feeds alone, so each character
+-- a browser would start a line at is an escape, and CR LF reaches the page as
+-- the line feed a browser would have made of it.
+out = render("hello.fake", "if\r42\vzz\f'a'\r\nif")
+h.equals("what a browser breaks at is escaped between tokens", out,
+	"<span class='hl-keyword'>if</span><span class='escape'>\\r</span>" ..
+	"<span class='hl-number'>42</span><span class='escape'>\\v</span>" ..
+	"<span class='hl-keyword'>zz</span><span class='escape'>\\f</span>" ..
+	"<span class='hl-string'>'a'</span>\n<span class='hl-keyword'>if</span>")
+
+-- The fake lexer hands back every byte of a separator as a token of its own.
+out = render("hello.fake", "a\194\133b\226\128\168c\226\128\169d")
+h.equals("a separator split between tokens is escaped whole", out,
+	"<span class='hl-keyword'>a</span><span class='escape'>\\u0085</span>" ..
+	"<span class='hl-keyword'>b</span><span class='escape'>\\u2028</span>" ..
+	"<span class='hl-keyword'>c</span><span class='escape'>\\u2029</span>" ..
+	"<span class='hl-keyword'>d</span>")
+
 out = render("big.fake", string.rep("a", 512 * 1024 + 1))
 h.excludes("an oversized blob skips the lexing pass", out, "<span")
 h.equals("and still arrives whole", #out, 512 * 1024 + 1)