Bound syntax-highlight memory and widen coverage

1 file changed, 150 insertions, 74 deletions

diff --git a/extensions/syntax-highlight.lua b/extensions/syntax-highlight.lua
index c88be87..8310504 100644
--- a/extensions/syntax-highlight.lua
+++ b/extensions/syntax-highlight.lua
@@ -1,30 +1,41 @@
--- Server-side syntax highlighting for the tree and blob views, used with
--- the source-filter setting in cgitrc and the lua: prefix so it runs in
--- cgit's embedded interpreter with no per-request process.
+-- Server-side syntax highlighting for the tree and blob views, used with the
+-- source-filter setting in cgitrc and the lua: prefix so it runs in cgit's
+-- embedded interpreter with no per-request process.
 --
 --     source-filter=lua:/usr/lib/cgit/extensions/syntax-highlight.lua
 --
--- Highlighting is deliberately not built into cgit itself. Without this
--- filter cgit serves plain escaped text, and any other program can take
--- this filter's place.
+-- Highlighting is deliberately not built into cgit itself. Without this filter
+-- cgit serves plain escaped text, and any other program can take its place.
+--
+-- SUPPORTED LUA
+--
+-- Lua 5.1 through 5.4 and LuaJIT. The Scintillua version matters too. Recent
+-- Scintillua (6.x) needs Lua 5.3 or newer to load its lexers, older Scintillua
+-- releases still load under 5.1 and 5.2. Pick a Scintillua release that matches
+-- the Lua cgit is built against.
 --
 -- REQUIREMENTS
 --
--- Two pieces, and BOTH must be installed. When either is missing the
--- filter serves plain escaped text by design, so uncolored code means
--- a missing dependency, not an error. Works with Lua 5.1 onward,
--- including LuaJIT.
+-- Two pieces, and BOTH must be installed. When either is missing the filter
+-- serves plain escaped text by design, so uncolored code means a missing
+-- dependency, not an error.
 --
--- 1. lpeg, the parsing module for the Lua that cgit is linked
---    against. Scintillua does NOT bundle it, it must come from the
---    system, and forgetting it is the usual reason nothing happens.
+-- 1. lpeg, the parsing module, for the Lua cgit is linked against. Scintillua
+--    does NOT bundle it, it must come from the system, and forgetting it is the
+--    usual reason nothing happens.
 --
---        Debian and Ubuntu   sudo apt install lua-lpeg
---        macOS               luarocks --lua-version 5.1 install lpeg
+--        # Debian and Ubuntu
+--        sudo apt install lua-lpeg
+--        # Fedora
+--        sudo dnf install lua-lpeg
+--        # Alpine
+--        sudo apk add lua5.1-lpeg
+--        # or with LuaRocks, matched to your Lua version
+--        sudo luarocks --lua-version 5.1 install lpeg
 --
--- 2. Scintillua, the lexer collection from the Textadept editor.
---    Roughly 120 languages as plain .lua files, nothing to compile.
---    Download a release and unpack it anywhere.
+-- 2. Scintillua, the lexer collection from the Textadept editor. Roughly 120
+--    languages as plain .lua files, nothing to compile. Download a release and
+--    unpack it anywhere.
 --
 --        https://orbitalquark.github.io/scintillua/
 --
@@ -32,75 +43,122 @@
 --
 --     $CGIT_SCINTILLUA_PATH        (used alone when set, no fallback)
 --     <dir of $CGIT_CONFIG>/scintillua/lexers
---     /usr/local/share/scintillua/lexers
---     /usr/share/scintillua/lexers
---     /opt/homebrew/share/scintillua/lexers
+--     the scintillua_dirs list in the CONFIGURATION block below
+--
+-- so either set the variable in the web server environment, or place (or
+-- symlink) the scintillua directory next to your cgitrc.
+--
+-- SECURITY
+--
+-- Every probed directory is placed on package.path and its Lua is executed in
+-- cgit's process. Make sure none of them is writable by other users, or someone
+-- who can write there gains code execution as the web server. On macOS in
+-- particular, /opt/homebrew/share is group-writable by default.
 --
--- so either set the variable in the web server environment, or place
--- (or symlink) the scintillua directory next to your cgitrc. When
--- Scintillua or lpeg is missing the filter passes text through escaped
--- and unhighlighted, so it is safe to enable before the dependencies
--- are installed.
+-- LIMITATIONS
+--
+-- cgit sends the filter output through a C string sink that stops at the first
+-- NUL byte, so a blob containing a NUL is truncated there. This affects binary
+-- files that slip past cgit's text detection, not ordinary source.
 --
 -- OUTPUT
 --
 -- Tokens are wrapped in <span> elements carrying the hl- classes that
--- assets/cgit.css styles. Every input byte is preserved, so the line
--- number gutter stays aligned.
+-- assets/cgit.css styles. Every input byte up to the first NUL is preserved, so
+-- the line number gutter stays aligned.
 
--- Files larger than this are passed through without lexing.
+--
+-- ===== CONFIGURATION =====
+--
+
+-- Files larger than this many bytes are served escaped but unhighlighted, so a
+-- huge blob does not cost a lexing pass. Kept well below cgit's max-blob-size.
 local max_bytes = 512 * 1024
 
-local function scintillua_path()
-	local env = os.getenv("CGIT_SCINTILLUA_PATH")
-	if env then
-		return env
-	end
-	local candidates = {
-		"/usr/local/share/scintillua/lexers",
-		"/usr/share/scintillua/lexers",
-		"/opt/homebrew/share/scintillua/lexers",
-	}
-	local config = os.getenv("CGIT_CONFIG")
-	if config then
-		local dir = string.match(config, "^(.*)/[^/]+$")
-		if dir then
-			table.insert(candidates, 1, dir .. "/scintillua/lexers")
-		end
-	end
-	for _, dir in ipairs(candidates) do
-		local f = io.open(dir .. "/lexer.lua", "r")
-		if f then
-			f:close()
-			return dir
-		end
-	end
-	return nil
-end
+-- Environment variable that, when set, points straight at the Scintillua
+-- lexers directory and is used alone.
+local scintillua_env = "CGIT_SCINTILLUA_PATH"
 
--- Scintillua tag name (first dotted component) to cgit css class.
+-- Directories probed for the lexers when that variable is not set. The
+-- directory of $CGIT_CONFIG, when set, is tried ahead of these. Keep every one
+-- of these unwritable by others, see the SECURITY note above.
+local scintillua_dirs = {
+	"/usr/local/share/scintillua/lexers",
+	"/usr/share/scintillua/lexers",
+	"/opt/homebrew/share/scintillua/lexers",
+}
+
+-- Scintillua tag name (its first dotted component) to a cgit css class. Only
+-- the six classes below exist in assets/cgit.css. Add a class there and a row
+-- here to style more token kinds. Tokens with no row render as plain text,
+-- which is what most themes want for operators and identifiers.
 local css = {
 	comment      = "hl-comment",
 	string       = "hl-string",
 	regex        = "hl-string",
 	number       = "hl-number",
+	constant     = "hl-number",
 	keyword      = "hl-keyword",
 	preprocessor = "hl-keyword",
+	tag          = "hl-keyword",
+	label        = "hl-keyword",
+	annotation   = "hl-keyword",
 	type         = "hl-type",
 	class        = "hl-type",
-	constant     = "hl-number",
+	attribute    = "hl-type",
 	["function"] = "hl-func",
 }
 
+-- Extension to lexer-name fixes for the fallback path, used only when this
+-- Scintillua has no detect(). Most extensions already equal their lexer name,
+-- these are the frequent exceptions. A wrong guess just falls back to plain
+-- text, so there is no harm in listing best-effort entries.
+local ext_lexer = {
+	py = "python", js = "javascript", ts = "typescript",
+	rb = "ruby", pl = "perl", pm = "perl", sh = "bash",
+	md = "markdown", htm = "html", yml = "yaml",
+	rs = "rust", c = "ansi_c", h = "ansi_c",
+}
+
+--
+-- =========================
+--
+
 local lexer_mod = nil
 local filename = ""
 local chunks = {}
 
+local escape_map = { ["&"] = "&amp;", ["<"] = "&lt;", [">"] = "&gt;" }
+
+-- Escape the three HTML metacharacters in a single pass.
 local function escape(s)
-	s = string.gsub(s, "&", "&amp;")
-	s = string.gsub(s, "<", "&lt;")
-	s = string.gsub(s, ">", "&gt;")
-	return s
+	return (string.gsub(s, "[&<>]", escape_map))
+end
+
+local function scintillua_path()
+	local env = os.getenv(scintillua_env)
+	if env then
+		return env
+	end
+	local candidates = {}
+	local config = os.getenv("CGIT_CONFIG")
+	if config then
+		local dir = string.match(config, "^(.*)/[^/]+$")
+		if dir then
+			candidates[#candidates + 1] = dir .. "/scintillua/lexers"
+		end
+	end
+	for _, d in ipairs(scintillua_dirs) do
+		candidates[#candidates + 1] = d
+	end
+	for _, dir in ipairs(candidates) do
+		local f = io.open(dir .. "/lexer.lua", "r")
+		if f then
+			f:close()
+			return dir
+		end
+	end
+	return nil
 end
 
 local function load_scintillua()
@@ -112,36 +170,43 @@ local function load_scintillua()
 		package.path = dir .. "/?.lua;" .. package.path
 	end
 	local ok, mod = pcall(require, "lexer")
-	-- A real Scintillua exposes load(); anything else on the path
-	-- that happens to be called lexer is not usable.
+	-- A real Scintillua exposes load(). Anything else on the path that happens
+	-- to be called lexer is not usable.
 	if ok and type(mod) == "table" and type(mod.load) == "function" then
 		return mod
 	end
 	return nil
 end
 
--- Resolve a lexer for the file, preferring Scintillua's own filename
--- detection when this version provides it.
+local function load_lexer_name(name)
+	if name == nil then
+		return nil
+	end
+	local ok, lex = pcall(lexer_mod.load, name)
+	if ok and lex then
+		return lex
+	end
+	return nil
+end
+
+-- Resolve a lexer for the file, preferring Scintillua's own filename detection
+-- when this version provides it, then an extension map, then the raw extension.
 local function lexer_for(name)
 	if type(lexer_mod.detect) == "function" then
 		local ok, lang = pcall(lexer_mod.detect, name)
 		if ok and lang then
-			local okl, lex = pcall(lexer_mod.load, lang)
-			if okl then
+			local lex = load_lexer_name(lang)
+			if lex then
 				return lex
 			end
 		end
-		return nil
 	end
 	local ext = string.match(name, "%.([^.]+)$")
 	if not ext then
 		return nil
 	end
-	local ok, lex = pcall(lexer_mod.load, string.lower(ext))
-	if ok then
-		return lex
-	end
-	return nil
+	ext = string.lower(ext)
+	return load_lexer_name(ext_lexer[ext]) or load_lexer_name(ext)
 end
 
 local function highlight(text)
@@ -197,6 +262,17 @@ function filter_close()
 			end
 		end
 	end
-	html(escape(text))
+	-- Fallback, escaped plain text emitted in slices so a large blob does not
+	-- cost a full-size second copy all at once.
+	local n = #text
+	if n == 0 then
+		html("")
+		return 0
+	end
+	local pos = 1
+	while pos <= n do
+		html(escape(string.sub(text, pos, pos + 65535)))
+		pos = pos + 65536
+	end
 	return 0
 end