index : cgit

-- Unit checks for custom/extensions/syntax-highlight.lua, run under a
-- standalone Lua by t0502-syntax-highlight.sh with the script path as the
-- first argument. The filter remembers whether the lexers loaded for the
-- life of the interpreter, so the two sides of that split are two runs of
-- this file. In highlight mode the shell points CGIT_SCINTILLUA_PATH at the
-- fake-lexers directory beside this file and every path through a working
-- lexer is checked. In nolexers mode it points at an empty directory, so
-- the escaped fallback is what everything renders through, and the run ends
-- with every directory the filter probes hidden, for a server with no
-- Scintillua at all.

local test_directory = arg[0]:match("^(.*)/") or "."
local h = dofile(test_directory .. "/harness.lua")

local script = arg[1]
local mode = h.mode("highlight", "nolexers")

h.load(script)

local function render(filename, text)
	return h.run({ filename }, { text })
end

if mode == "nolexers" then
	local out, ret = render("hello.fake", "if 42")
	h.equals("without lexers the source passes through uncoloured", out, "if 42")
	h.equals("close answers zero", ret, 0)
	h.equals("and the error log says the lexers would not load", h.error_log(),
		"[cgit] Unable to load the Scintillua lexers: module 'lexer' not found:\n")

	out = render("hello.fake", "a <b> & c")
	h.equals("the fallback escapes markup", out, "a &lt;b&gt; &amp; c")
	h.equals("and the error log hears of it only once", h.error_log(), "")

	out = render("hello.fake", "a\0b")
	h.equals("a nul byte cuts the escaped blob short", out, "a")

	-- Large output leaves in slices, and a byte lost at a boundary would
	-- put the line numbers beside it out of step, so the whole escaped
	-- blob has to come back intact across several slices.
	out = render("big.fake", string.rep("x<y>", 30000))
	h.equals("slicing loses nothing at the boundaries", out, string.rep("x&lt;y&gt;", 30000))

	out = render("hello.fake", "a\rb\vc\fd\r\ne")
	h.equals("the fallback escapes what a browser breaks at", out,
		"a<span class='escape'>\\r</span>b<span class='escape'>\\v</span>c" ..
		"<span class='escape'>\\f</span>d\ne")

	-- The line separator starts on the last byte of the first slice, and
	-- its other two bytes would land in the next one.
	out = render("big.fake", string.rep("x", 64 * 1024 - 1) .. "\226\128\168y")
	h.equals("a separator across a slice boundary is still escaped", out,
		string.rep("x", 64 * 1024 - 1) .. "<span class='escape'>\\u2028</span>y")

	-- With the variable unset and no lexers beside cgitrc or in any directory
	-- the filter probes, it finds nothing to load, the usual case on a server
	-- without Scintillua.
	local file = assert(io.open(script, "rb"))
	local source = file:read("*a")
	file:close()
	for dir in source:match("local scintillua_dirs = (%b{})"):gmatch('"([^"]+)"') do
		h.redirect_file(dir .. "/lexer.lua", "/nonexistent/lexer.lua")
	end
	h.setenv("CGIT_SCINTILLUA_PATH", nil)
	h.setenv("CGIT_CONFIG", nil)
	h.load(script)
	out = render("hello.fake", "a <b>")
	h.equals("with no lexers found the source is served escaped", out, "a &lt;b&gt;")
	h.equals("and the error log says none were found", h.error_log(),
		"[cgit] Unable to find the Scintillua lexers, so source is shown uncoloured\n")

	-- The filter takes a module called lexer only when it is Scintillua, so
	-- one that is something else is never used. A Scintillua older than 6.0
	-- has no detect() and colours nothing, and lexers that need a missing
	-- lpeg say why in the log. Each loads the filter afresh with the module
	-- it finds forced through package.preload.
	local function load_with_lexer(loader)
		package.loaded.lexer = nil
		package.preload.lexer = loader
		h.setenv("CGIT_SCINTILLUA_PATH", "/nonexistent")
		h.load(script)
	end

	local used = false
	load_with_lexer(function()
		return { detect = function() used = true end }
	end)
	out = render("hello.fake", "a <b>")
	h.equals("a lexer module without load is passed over", out, "a &lt;b&gt;")
	h.check("and never used", not used)
	h.equals("and the error log stays empty", h.error_log(), "")

	load_with_lexer(function()
		return { load = function() return { lex = function() return { "keyword", 2 } end } end }
	end)
	out = render("hello.fake", "a <b>")
	h.equals("a Scintillua without detect colours nothing", out, "a &lt;b&gt;")
	h.equals("and logs nothing", h.error_log(), "")

	h.preload_failure("lpeg")
	load_with_lexer(function()
		return require("lpeg")
	end)
	out = render("hello.fake", "a <b>")
	h.equals("lexers that cannot load lpeg leave the source uncoloured", out, "a &lt;b&gt;")
	local log = h.error_log()
	local prefix = "[cgit] Unable to load the Scintillua lexers: "
	h.equals("and the error log says why", log:sub(1, #prefix), prefix)
	h.contains("naming lpeg", log, "lpeg deliberately unavailable")
	h.equals("on one line", log:find("\n", 1, true), #log)

	h.finish()
end

local out, ret = render("hello.fake", "if 42 'hi' & zz")
h.equals("loaded lexers leave the error log empty", h.error_log(), "")
h.contains("keywords are wrapped in their class", out, "<span class='hl-keyword'>if</span>")
h.contains("numbers are wrapped in their class", out, "<span class='hl-number'>42</span>")
h.contains("a dotted tag maps by its first component", out, "<span class='hl-string'>'hi'</span>")
h.contains("bytes between tokens are escaped", out, " &amp; ")
h.contains("later tokens still highlight", out, "<span class='hl-keyword'>zz</span>")
h.equals("close answers zero", ret, 0)

out = render("hello.fake", "'<b>&'")
h.equals("markup inside a token is escaped", out, "<span class='hl-string'>'&lt;b&gt;&amp;'</span>")

out = render("hello.fake", "-- note\nString x")
h.equals("comments and types take their classes", out,
	"<span class='hl-comment'>-- note</span>\n<span class='hl-type'>String</span> <span class='hl-keyword'>x</span>")

out = render("hello.fake", "$color :hover")
h.contains("a css property takes the number class", out, "<span class='hl-number'>$color</span>")
h.contains("a css pseudo-class takes the function class", out, "<span class='hl-func'>:hover</span>")

out = h.run({ "split.fake" }, { "i", "f" })
h.equals("writes are joined before lexing", out, "<span class='hl-keyword'>if</span>")

-- The short lexer tags four bytes and stops, and every byte after it still
-- has to reach the page.
out = render("cut.short", "abcd<ef>")
h.equals("a lexer stopping early leaves an escaped tail", out, "<span class='hl-keyword'>abcd</span>&lt;ef&gt;")

out = render("boom.badlex", "some <code>")
h.equals("a lexer that raises falls back to escaped text", out, "some &lt;code&gt;")

out = render("empty.nothing", "some <code>")
h.equals("a lexer that returns no tokens falls back to escaped text", out, "some &lt;code&gt;")

out = render("HELLO.FAKE", "if 42")
h.contains("an extension in capitals finds its lexer", out,
	"<span class='hl-keyword'>if</span>")

out = render("post-receive", "#!/usr/bin/fake\nif")
h.contains("a shebang picks the lexer of a file with no extension", out,
	"<span class='hl-keyword'>if</span>")

out = render("tool.py", "import <x>")
h.equals("an extension with no lexer serves escaped text", out, "import &lt;x&gt;")

out = render("noextension", "words <here>")
h.equals("a file with no extension serves escaped text", out, "words &lt;here&gt;")

-- The line numbers beside the code count line feeds alone, so each character
-- a browser would start a line at is an escape, and CR LF reaches the page as
-- the line feed a browser would have made of it.
out = render("hello.fake", "if\r42\vzz\f'a'\r\nif")
h.equals("what a browser breaks at is escaped between tokens", out,
	"<span class='hl-keyword'>if</span><span class='escape'>\\r</span>" ..
	"<span class='hl-number'>42</span><span class='escape'>\\v</span>" ..
	"<span class='hl-keyword'>zz</span><span class='escape'>\\f</span>" ..
	"<span class='hl-string'>'a'</span>\n<span class='hl-keyword'>if</span>")

-- The fake lexer hands back every byte of a separator as a token of its own.
out = render("hello.fake", "a\194\133b\226\128\168c\226\128\169d")
h.equals("a separator split between tokens is escaped whole", out,
	"<span class='hl-keyword'>a</span><span class='escape'>\\u0085</span>" ..
	"<span class='hl-keyword'>b</span><span class='escape'>\\u2028</span>" ..
	"<span class='hl-keyword'>c</span><span class='escape'>\\u2029</span>" ..
	"<span class='hl-keyword'>d</span>")

out = render("big.fake", string.rep("a", 512 * 1024 + 1))
h.excludes("an oversized blob skips the lexing pass", out, "<span")
h.equals("and still arrives whole", #out, 512 * 1024 + 1)

out = render("big.fake", string.rep("a", 512 * 1024))
h.equals("a blob of exactly the cap is still lexed", out:sub(1, 26), "<span class='hl-keyword'>a")

-- cgit's sink stops at the first NUL byte, as the harness does, so a blob
-- holding one loses everything from there on.
out = render("hello.fake", "a\0b")
h.equals("a nul byte cuts the highlighted blob short", out, "<span class='hl-keyword'>a</span>")

out, ret = render("empty.fake", "")
h.equals("an empty blob renders empty", out, "")
h.equals("an empty blob still answers zero", ret, 0)

h.finish()