Bound syntax-highlight memory and widen coverage

1 file changed, 150 insertions, 74 deletions

diff --git a/extensions/syntax-highlight.lua b/extensions/syntax-highlight.luaindex c88be87..8310504 100644--- a/extensions/syntax-highlight.lua+++ b/extensions/syntax-highlight.lua@@ -1,30 +1,41 @@--- Server-side syntax highlighting for the tree and blob views, used with--- the source-filter setting in cgitrc and the lua: prefix so it runs in--- cgit's embedded interpreter with no per-request process.+-- Server-side syntax highlighting for the tree and blob views, used with the+-- source-filter setting in cgitrc and the lua: prefix so it runs in cgit's+-- embedded interpreter with no per-request process. -- --     source-filter=lua:/usr/lib/cgit/extensions/syntax-highlight.lua ----- Highlighting is deliberately not built into cgit itself. Without this--- filter cgit serves plain escaped text, and any other program can take--- this filter's place.+-- Highlighting is deliberately not built into cgit itself. Without this filter+-- cgit serves plain escaped text, and any other program can take its place.+--+-- SUPPORTED LUA+--+-- Lua 5.1 through 5.4 and LuaJIT. The Scintillua version matters too. Recent+-- Scintillua (6.x) needs Lua 5.3 or newer to load its lexers, older Scintillua+-- releases still load under 5.1 and 5.2. Pick a Scintillua release that matches+-- the Lua cgit is built against. -- -- REQUIREMENTS ----- Two pieces, and BOTH must be installed. When either is missing the--- filter serves plain escaped text by design, so uncolored code means--- a missing dependency, not an error. Works with Lua 5.1 onward,--- including LuaJIT.+-- Two pieces, and BOTH must be installed. When either is missing the filter+-- serves plain escaped text by design, so uncolored code means a missing+-- dependency, not an error. ----- 1. lpeg, the parsing module for the Lua that cgit is linked---    against. Scintillua does NOT bundle it, it must come from the---    system, and forgetting it is the usual reason nothing happens.+-- 1. lpeg, the parsing module, for the Lua cgit is linked against. Scintillua+--    does NOT bundle it, it must come from the system, and forgetting it is the+--    usual reason nothing happens. -----        Debian and Ubuntu   sudo apt install lua-lpeg---        macOS               luarocks --lua-version 5.1 install lpeg+--        # Debian and Ubuntu+--        sudo apt install lua-lpeg+--        # Fedora+--        sudo dnf install lua-lpeg+--        # Alpine+--        sudo apk add lua5.1-lpeg+--        # or with LuaRocks, matched to your Lua version+--        sudo luarocks --lua-version 5.1 install lpeg ----- 2. Scintillua, the lexer collection from the Textadept editor.---    Roughly 120 languages as plain .lua files, nothing to compile.---    Download a release and unpack it anywhere.+-- 2. Scintillua, the lexer collection from the Textadept editor. Roughly 120+--    languages as plain .lua files, nothing to compile. Download a release and+--    unpack it anywhere. -- --        https://orbitalquark.github.io/scintillua/ --@@ -32,75 +43,122 @@ -- --     $CGIT_SCINTILLUA_PATH        (used alone when set, no fallback) --     <dir of $CGIT_CONFIG>/scintillua/lexers---     /usr/local/share/scintillua/lexers---     /usr/share/scintillua/lexers---     /opt/homebrew/share/scintillua/lexers+--     the scintillua_dirs list in the CONFIGURATION block below+--+-- so either set the variable in the web server environment, or place (or+-- symlink) the scintillua directory next to your cgitrc.+--+-- SECURITY+--+-- Every probed directory is placed on package.path and its Lua is executed in+-- cgit's process. Make sure none of them is writable by other users, or someone+-- who can write there gains code execution as the web server. On macOS in+-- particular, /opt/homebrew/share is group-writable by default. ----- so either set the variable in the web server environment, or place--- (or symlink) the scintillua directory next to your cgitrc. When--- Scintillua or lpeg is missing the filter passes text through escaped--- and unhighlighted, so it is safe to enable before the dependencies--- are installed.+-- LIMITATIONS+--+-- cgit sends the filter output through a C string sink that stops at the first+-- NUL byte, so a blob containing a NUL is truncated there. This affects binary+-- files that slip past cgit's text detection, not ordinary source. -- -- OUTPUT -- -- Tokens are wrapped in <span> elements carrying the hl- classes that--- assets/cgit.css styles. Every input byte is preserved, so the line--- number gutter stays aligned.+-- assets/cgit.css styles. Every input byte up to the first NUL is preserved, so+-- the line number gutter stays aligned. --- Files larger than this are passed through without lexing.+--+-- ===== CONFIGURATION =====+--++-- Files larger than this many bytes are served escaped but unhighlighted, so a+-- huge blob does not cost a lexing pass. Kept well below cgit's max-blob-size. local max_bytes = 512 * 1024 -local function scintillua_path()-	local env = os.getenv("CGIT_SCINTILLUA_PATH")-	if env then-		return env-	end-	local candidates = {-		"/usr/local/share/scintillua/lexers",-		"/usr/share/scintillua/lexers",-		"/opt/homebrew/share/scintillua/lexers",-	}-	local config = os.getenv("CGIT_CONFIG")-	if config then-		local dir = string.match(config, "^(.*)/[^/]+$")-		if dir then-			table.insert(candidates, 1, dir .. "/scintillua/lexers")-		end-	end-	for _, dir in ipairs(candidates) do-		local f = io.open(dir .. "/lexer.lua", "r")-		if f then-			f:close()-			return dir-		end-	end-	return nil-end+-- Environment variable that, when set, points straight at the Scintillua+-- lexers directory and is used alone.+local scintillua_env = "CGIT_SCINTILLUA_PATH" --- Scintillua tag name (first dotted component) to cgit css class.+-- Directories probed for the lexers when that variable is not set. The+-- directory of $CGIT_CONFIG, when set, is tried ahead of these. Keep every one+-- of these unwritable by others, see the SECURITY note above.+local scintillua_dirs = {+	"/usr/local/share/scintillua/lexers",+	"/usr/share/scintillua/lexers",+	"/opt/homebrew/share/scintillua/lexers",+}++-- Scintillua tag name (its first dotted component) to a cgit css class. Only+-- the six classes below exist in assets/cgit.css. Add a class there and a row+-- here to style more token kinds. Tokens with no row render as plain text,+-- which is what most themes want for operators and identifiers. local css = { 	comment      = "hl-comment", 	string       = "hl-string", 	regex        = "hl-string", 	number       = "hl-number",+	constant     = "hl-number", 	keyword      = "hl-keyword", 	preprocessor = "hl-keyword",+	tag          = "hl-keyword",+	label        = "hl-keyword",+	annotation   = "hl-keyword", 	type         = "hl-type", 	class        = "hl-type",-	constant     = "hl-number",+	attribute    = "hl-type", 	["function"] = "hl-func", } +-- Extension to lexer-name fixes for the fallback path, used only when this+-- Scintillua has no detect(). Most extensions already equal their lexer name,+-- these are the frequent exceptions. A wrong guess just falls back to plain+-- text, so there is no harm in listing best-effort entries.+local ext_lexer = {+	py = "python", js = "javascript", ts = "typescript",+	rb = "ruby", pl = "perl", pm = "perl", sh = "bash",+	md = "markdown", htm = "html", yml = "yaml",+	rs = "rust", c = "ansi_c", h = "ansi_c",+}++--+-- =========================+--+ local lexer_mod = nil local filename = "" local chunks = {} +local escape_map = { ["&"] = "&amp;", ["<"] = "&lt;", [">"] = "&gt;" }++-- Escape the three HTML metacharacters in a single pass. local function escape(s)-	s = string.gsub(s, "&", "&amp;")-	s = string.gsub(s, "<", "&lt;")-	s = string.gsub(s, ">", "&gt;")-	return s+	return (string.gsub(s, "[&<>]", escape_map))+end++local function scintillua_path()+	local env = os.getenv(scintillua_env)+	if env then+		return env+	end+	local candidates = {}+	local config = os.getenv("CGIT_CONFIG")+	if config then+		local dir = string.match(config, "^(.*)/[^/]+$")+		if dir then+			candidates[#candidates + 1] = dir .. "/scintillua/lexers"+		end+	end+	for _, d in ipairs(scintillua_dirs) do+		candidates[#candidates + 1] = d+	end+	for _, dir in ipairs(candidates) do+		local f = io.open(dir .. "/lexer.lua", "r")+		if f then+			f:close()+			return dir+		end+	end+	return nil end  local function load_scintillua()@@ -112,36 +170,43 @@ local function load_scintillua() 		package.path = dir .. "/?.lua;" .. package.path 	end 	local ok, mod = pcall(require, "lexer")-	-- A real Scintillua exposes load(); anything else on the path-	-- that happens to be called lexer is not usable.+	-- A real Scintillua exposes load(). Anything else on the path that happens+	-- to be called lexer is not usable. 	if ok and type(mod) == "table" and type(mod.load) == "function" then 		return mod 	end 	return nil end --- Resolve a lexer for the file, preferring Scintillua's own filename--- detection when this version provides it.+local function load_lexer_name(name)+	if name == nil then+		return nil+	end+	local ok, lex = pcall(lexer_mod.load, name)+	if ok and lex then+		return lex+	end+	return nil+end++-- Resolve a lexer for the file, preferring Scintillua's own filename detection+-- when this version provides it, then an extension map, then the raw extension. local function lexer_for(name) 	if type(lexer_mod.detect) == "function" then 		local ok, lang = pcall(lexer_mod.detect, name) 		if ok and lang then-			local okl, lex = pcall(lexer_mod.load, lang)-			if okl then+			local lex = load_lexer_name(lang)+			if lex then 				return lex 			end 		end-		return nil 	end 	local ext = string.match(name, "%.([^.]+)$") 	if not ext then 		return nil 	end-	local ok, lex = pcall(lexer_mod.load, string.lower(ext))-	if ok then-		return lex-	end-	return nil+	ext = string.lower(ext)+	return load_lexer_name(ext_lexer[ext]) or load_lexer_name(ext) end  local function highlight(text)@@ -197,6 +262,17 @@ function filter_close() 			end 		end 	end-	html(escape(text))+	-- Fallback, escaped plain text emitted in slices so a large blob does not+	-- cost a full-size second copy all at once.+	local n = #text+	if n == 0 then+		html("")+		return 0+	end+	local pos = 1+	while pos <= n do+		html(escape(string.sub(text, pos, pos + 65535)))+		pos = pos + 65536+	end 	return 0 end
diff --git a/extensions/syntax-highlight.lua b/extensions/syntax-highlight.lua
index c88be87..8310504 100644
--- a/extensions/syntax-highlight.lua
+++ b/extensions/syntax-highlight.lua
@@ -1,30 +1,41 @@
--- Server-side syntax highlighting for the tree and blob views, used with
--- the source-filter setting in cgitrc and the lua: prefix so it runs in
--- cgit's embedded interpreter with no per-request process.
+-- Server-side syntax highlighting for the tree and blob views, used with the
+-- source-filter setting in cgitrc and the lua: prefix so it runs in cgit's
+-- embedded interpreter with no per-request process.
 --
 --     source-filter=lua:/usr/lib/cgit/extensions/syntax-highlight.lua
 --
--- Highlighting is deliberately not built into cgit itself. Without this
--- filter cgit serves plain escaped text, and any other program can take
--- this filter's place.
+-- Highlighting is deliberately not built into cgit itself. Without this filter
+-- cgit serves plain escaped text, and any other program can take its place.
+--
+-- SUPPORTED LUA
+--
+-- Lua 5.1 through 5.4 and LuaJIT. The Scintillua version matters too. Recent
+-- Scintillua (6.x) needs Lua 5.3 or newer to load its lexers, older Scintillua
+-- releases still load under 5.1 and 5.2. Pick a Scintillua release that matches
+-- the Lua cgit is built against.
 --
 -- REQUIREMENTS
 --
--- Two pieces, and BOTH must be installed. When either is missing the
--- filter serves plain escaped text by design, so uncolored code means
--- a missing dependency, not an error. Works with Lua 5.1 onward,
--- including LuaJIT.
+-- Two pieces, and BOTH must be installed. When either is missing the filter
+-- serves plain escaped text by design, so uncolored code means a missing
+-- dependency, not an error.
 --
--- 1. lpeg, the parsing module for the Lua that cgit is linked
---    against. Scintillua does NOT bundle it, it must come from the
---    system, and forgetting it is the usual reason nothing happens.
+-- 1. lpeg, the parsing module, for the Lua cgit is linked against. Scintillua
+--    does NOT bundle it, it must come from the system, and forgetting it is the
+--    usual reason nothing happens.
 --
---        Debian and Ubuntu   sudo apt install lua-lpeg
---        macOS               luarocks --lua-version 5.1 install lpeg
+--        # Debian and Ubuntu
+--        sudo apt install lua-lpeg
+--        # Fedora
+--        sudo dnf install lua-lpeg
+--        # Alpine
+--        sudo apk add lua5.1-lpeg
+--        # or with LuaRocks, matched to your Lua version
+--        sudo luarocks --lua-version 5.1 install lpeg
 --
--- 2. Scintillua, the lexer collection from the Textadept editor.
---    Roughly 120 languages as plain .lua files, nothing to compile.
---    Download a release and unpack it anywhere.
+-- 2. Scintillua, the lexer collection from the Textadept editor. Roughly 120
+--    languages as plain .lua files, nothing to compile. Download a release and
+--    unpack it anywhere.
 --
 --        https://orbitalquark.github.io/scintillua/
 --
@@ -32,75 +43,122 @@
 --
 --     $CGIT_SCINTILLUA_PATH        (used alone when set, no fallback)
 --     <dir of $CGIT_CONFIG>/scintillua/lexers
---     /usr/local/share/scintillua/lexers
---     /usr/share/scintillua/lexers
---     /opt/homebrew/share/scintillua/lexers
+--     the scintillua_dirs list in the CONFIGURATION block below
+--
+-- so either set the variable in the web server environment, or place (or
+-- symlink) the scintillua directory next to your cgitrc.
+--
+-- SECURITY
+--
+-- Every probed directory is placed on package.path and its Lua is executed in
+-- cgit's process. Make sure none of them is writable by other users, or someone
+-- who can write there gains code execution as the web server. On macOS in
+-- particular, /opt/homebrew/share is group-writable by default.
 --
--- so either set the variable in the web server environment, or place
--- (or symlink) the scintillua directory next to your cgitrc. When
--- Scintillua or lpeg is missing the filter passes text through escaped
--- and unhighlighted, so it is safe to enable before the dependencies
--- are installed.
+-- LIMITATIONS
+--
+-- cgit sends the filter output through a C string sink that stops at the first
+-- NUL byte, so a blob containing a NUL is truncated there. This affects binary
+-- files that slip past cgit's text detection, not ordinary source.
 --
 -- OUTPUT
 --
 -- Tokens are wrapped in <span> elements carrying the hl- classes that
--- assets/cgit.css styles. Every input byte is preserved, so the line
--- number gutter stays aligned.
+-- assets/cgit.css styles. Every input byte up to the first NUL is preserved, so
+-- the line number gutter stays aligned.
 
--- Files larger than this are passed through without lexing.
+--
+-- ===== CONFIGURATION =====
+--
+
+-- Files larger than this many bytes are served escaped but unhighlighted, so a
+-- huge blob does not cost a lexing pass. Kept well below cgit's max-blob-size.
 local max_bytes = 512 * 1024
 
-local function scintillua_path()
-	local env = os.getenv("CGIT_SCINTILLUA_PATH")
-	if env then
-		return env
-	end
-	local candidates = {
-		"/usr/local/share/scintillua/lexers",
-		"/usr/share/scintillua/lexers",
-		"/opt/homebrew/share/scintillua/lexers",
-	}
-	local config = os.getenv("CGIT_CONFIG")
-	if config then
-		local dir = string.match(config, "^(.*)/[^/]+$")
-		if dir then
-			table.insert(candidates, 1, dir .. "/scintillua/lexers")
-		end
-	end
-	for _, dir in ipairs(candidates) do
-		local f = io.open(dir .. "/lexer.lua", "r")
-		if f then
-			f:close()
-			return dir
-		end
-	end
-	return nil
-end
+-- Environment variable that, when set, points straight at the Scintillua
+-- lexers directory and is used alone.
+local scintillua_env = "CGIT_SCINTILLUA_PATH"
 
--- Scintillua tag name (first dotted component) to cgit css class.
+-- Directories probed for the lexers when that variable is not set. The
+-- directory of $CGIT_CONFIG, when set, is tried ahead of these. Keep every one
+-- of these unwritable by others, see the SECURITY note above.
+local scintillua_dirs = {
+	"/usr/local/share/scintillua/lexers",
+	"/usr/share/scintillua/lexers",
+	"/opt/homebrew/share/scintillua/lexers",
+}
+
+-- Scintillua tag name (its first dotted component) to a cgit css class. Only
+-- the six classes below exist in assets/cgit.css. Add a class there and a row
+-- here to style more token kinds. Tokens with no row render as plain text,
+-- which is what most themes want for operators and identifiers.
 local css = {
 	comment      = "hl-comment",
 	string       = "hl-string",
 	regex        = "hl-string",
 	number       = "hl-number",
+	constant     = "hl-number",
 	keyword      = "hl-keyword",
 	preprocessor = "hl-keyword",
+	tag          = "hl-keyword",
+	label        = "hl-keyword",
+	annotation   = "hl-keyword",
 	type         = "hl-type",
 	class        = "hl-type",
-	constant     = "hl-number",
+	attribute    = "hl-type",
 	["function"] = "hl-func",
 }
 
+-- Extension to lexer-name fixes for the fallback path, used only when this
+-- Scintillua has no detect(). Most extensions already equal their lexer name,
+-- these are the frequent exceptions. A wrong guess just falls back to plain
+-- text, so there is no harm in listing best-effort entries.
+local ext_lexer = {
+	py = "python", js = "javascript", ts = "typescript",
+	rb = "ruby", pl = "perl", pm = "perl", sh = "bash",
+	md = "markdown", htm = "html", yml = "yaml",
+	rs = "rust", c = "ansi_c", h = "ansi_c",
+}
+
+--
+-- =========================
+--
+
 local lexer_mod = nil
 local filename = ""
 local chunks = {}
 
+local escape_map = { ["&"] = "&amp;", ["<"] = "&lt;", [">"] = "&gt;" }
+
+-- Escape the three HTML metacharacters in a single pass.
 local function escape(s)
-	s = string.gsub(s, "&", "&amp;")
-	s = string.gsub(s, "<", "&lt;")
-	s = string.gsub(s, ">", "&gt;")
-	return s
+	return (string.gsub(s, "[&<>]", escape_map))
+end
+
+local function scintillua_path()
+	local env = os.getenv(scintillua_env)
+	if env then
+		return env
+	end
+	local candidates = {}
+	local config = os.getenv("CGIT_CONFIG")
+	if config then
+		local dir = string.match(config, "^(.*)/[^/]+$")
+		if dir then
+			candidates[#candidates + 1] = dir .. "/scintillua/lexers"
+		end
+	end
+	for _, d in ipairs(scintillua_dirs) do
+		candidates[#candidates + 1] = d
+	end
+	for _, dir in ipairs(candidates) do
+		local f = io.open(dir .. "/lexer.lua", "r")
+		if f then
+			f:close()
+			return dir
+		end
+	end
+	return nil
 end
 
 local function load_scintillua()
@@ -112,36 +170,43 @@ local function load_scintillua()
 		package.path = dir .. "/?.lua;" .. package.path
 	end
 	local ok, mod = pcall(require, "lexer")
-	-- A real Scintillua exposes load(); anything else on the path
-	-- that happens to be called lexer is not usable.
+	-- A real Scintillua exposes load(). Anything else on the path that happens
+	-- to be called lexer is not usable.
 	if ok and type(mod) == "table" and type(mod.load) == "function" then
 		return mod
 	end
 	return nil
 end
 
--- Resolve a lexer for the file, preferring Scintillua's own filename
--- detection when this version provides it.
+local function load_lexer_name(name)
+	if name == nil then
+		return nil
+	end
+	local ok, lex = pcall(lexer_mod.load, name)
+	if ok and lex then
+		return lex
+	end
+	return nil
+end
+
+-- Resolve a lexer for the file, preferring Scintillua's own filename detection
+-- when this version provides it, then an extension map, then the raw extension.
 local function lexer_for(name)
 	if type(lexer_mod.detect) == "function" then
 		local ok, lang = pcall(lexer_mod.detect, name)
 		if ok and lang then
-			local okl, lex = pcall(lexer_mod.load, lang)
-			if okl then
+			local lex = load_lexer_name(lang)
+			if lex then
 				return lex
 			end
 		end
-		return nil
 	end
 	local ext = string.match(name, "%.([^.]+)$")
 	if not ext then
 		return nil
 	end
-	local ok, lex = pcall(lexer_mod.load, string.lower(ext))
-	if ok then
-		return lex
-	end
-	return nil
+	ext = string.lower(ext)
+	return load_lexer_name(ext_lexer[ext]) or load_lexer_name(ext)
 end
 
 local function highlight(text)
@@ -197,6 +262,17 @@ function filter_close()
 			end
 		end
 	end
-	html(escape(text))
+	-- Fallback, escaped plain text emitted in slices so a large blob does not
+	-- cost a full-size second copy all at once.
+	local n = #text
+	if n == 0 then
+		html("")
+		return 0
+	end
+	local pos = 1
+	while pos <= n do
+		html(escape(string.sub(text, pos, pos + 65535)))
+		pos = pos + 65536
+	end
 	return 0
 end