From 85e906e58202b2ae0ab782f6f411485acfd55f9e Mon Sep 17 00:00:00 2001 From: Bryce Kwon Date: Sat, 24 Oct 2026 09:17:41 -1000 Subject: Escape the same characters in the highlighter A lexer may hand back the bytes of one separator, or the CR and LF of a line ending, in different tokens, so line endings become line feeds before lexing and the separators are escaped once the tokens are joined. --- tests/extensions/test-syntax-highlight.lua | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'tests/extensions') diff --git a/tests/extensions/test-syntax-highlight.lua b/tests/extensions/test-syntax-highlight.lua index a87cd80..f831b01 100644 --- a/tests/extensions/test-syntax-highlight.lua +++ b/tests/extensions/test-syntax-highlight.lua @@ -33,6 +33,17 @@ if mode == "nolexers" then out = render("big.fake", string.rep("x", 30000)) h.equals("slicing loses nothing at the boundaries", out, string.rep("x<y>", 30000)) + out = render("hello.fake", "a\rb\vc\fd\r\ne") + h.equals("the fallback escapes what a browser breaks at", out, + "a\\rb\\vc" .. + "\\fd\ne") + + -- The line separator starts on the last byte of the first slice, and + -- its other two bytes would land in the next one. + out = render("big.fake", string.rep("x", 64 * 1024 - 1) .. "\226\128\168y") + h.equals("a separator across a slice boundary is still escaped", out, + string.rep("x", 64 * 1024 - 1) .. "\\u2028y") + h.finish() end @@ -70,6 +81,24 @@ h.equals("an extension with no lexer serves escaped text", out, "import <x> out = render("noextension", "words ") h.equals("a file with no extension serves escaped text", out, "words <here>") +-- The line numbers beside the code count line feeds alone, so each character +-- a browser would start a line at is an escape, and CR LF reaches the page as +-- the line feed a browser would have made of it. +out = render("hello.fake", "if\r42\vzz\f'a'\r\nif") +h.equals("what a browser breaks at is escaped between tokens", out, + "if\\r" .. + "42\\v" .. + "zz\\f" .. + "'a'\nif") + +-- The fake lexer hands back every byte of a separator as a token of its own. +out = render("hello.fake", "a\194\133b\226\128\168c\226\128\169d") +h.equals("a separator split between tokens is escaped whole", out, + "a\\u0085" .. + "b\\u2028" .. + "c\\u2029" .. + "d") + out = render("big.fake", string.rep("a", 512 * 1024 + 1)) h.excludes("an oversized blob skips the lexing pass", out, "