index : cgit

-- Server-side syntax highlighting for cgit's tree and blob views, named by
-- the source-filter setting and run inside cgit's embedded Lua interpreter
-- so a coloured blob costs no extra process per request. Tokens come from the
-- Scintillua lexers and reach the page wrapped in span elements carrying the
-- hl- classes that assets/cgit.css styles. Whenever a piece is missing or
-- will not load, from lpeg down to a single lexer, the filter falls back to
-- plain escaped text instead of failing. It runs on Lua 5.1 through 5.5 and
-- LuaJIT.
--
--     source-filter=lua:/usr/local/lib/cgit/filters/syntax-highlight.lua


-- Files larger than this many bytes are served escaped but unhighlighted, so
-- a huge blob does not cost a lexing pass. Kept well below cgit's
-- max-blob-size.
local max_bytes = 512 * 1024

-- Size of the pieces the unhighlighted fallback is written in, so a large
-- blob does not cost a full-size second copy all at once.
local slice_bytes = 64 * 1024

-- Set this in the web server environment to point straight at the Scintillua
-- lexers directory, in which case nothing else is probed.
local scintillua_env = "CGIT_SCINTILLUA_PATH"

-- Directories probed for the lexers when that variable is not set, tried
-- after the directory of $CGIT_CONFIG, so placing or symlinking a scintillua
-- directory next to cgitrc is enough for it to be found. The lexers come
-- from https://orbitalquark.github.io/scintillua/ and need lpeg, built for
-- the Lua cgit is linked against. Forgetting that is the usual reason
-- nothing is coloured.
--
--     # Debian and Ubuntu
--     sudo apt install lua-lpeg
--     # Fedora
--     sudo dnf install lua-lpeg
--     # Alpine
--     sudo apk add lua5.1-lpeg
--     # or with LuaRocks, matched to your Lua version
--     sudo luarocks --lua-version 5.1 install lpeg
--
-- Every directory probed goes on package.path and its Lua is executed in
-- cgit's process, so one that other users can write to hands them code
-- execution as the web server. On macOS /opt/homebrew/share is group-writable
-- by default, so check it before leaving it in this list.
local scintillua_dirs = {
	"/usr/local/share/scintillua/lexers",
	"/usr/share/scintillua/lexers",
	"/opt/homebrew/share/scintillua/lexers",
}

-- A Scintillua tag name, meaning its first dotted component, mapped to a cgit
-- css class. Only the six classes named here exist in assets/cgit.css, so
-- styling another kind of token means adding a class there and a row here. A
-- tag with no row renders as plain text, which is what most themes want for
-- operators and identifiers.
local css = {
	comment      = "hl-comment",
	string       = "hl-string",
	regex        = "hl-string",
	number       = "hl-number",
	constant     = "hl-number",
	keyword      = "hl-keyword",
	preprocessor = "hl-keyword",
	tag          = "hl-keyword",
	label        = "hl-keyword",
	annotation   = "hl-keyword",
	type         = "hl-type",
	class        = "hl-type",
	attribute    = "hl-type",
	["function"] = "hl-func",
}

-- Extension to lexer name fixes for the fallback path, reached only when
-- this Scintillua has no detect(). Most extensions already equal their lexer
-- name and these are the frequent exceptions.
local ext_lexer = {
	py = "python",
	js = "javascript",
	ts = "typescript",
	rb = "ruby",
	pl = "perl",
	pm = "perl",
	sh = "bash",
	md = "markdown",
	htm = "html",
	yml = "yaml",
	rs = "rust",
	c = "ansi_c",
	h = "ansi_c",
}


local scintillua = nil
local filename = ""
local chunks = {}

-- Besides the markup characters, everything a browser may start a new line
-- at is shown as an escape sequence, since the line numbers beside the code
-- count only line feeds. cgit escapes a blob it serves unhighlighted the same
-- way.
local escape_map = {
	["&"] = "&",
	["<"] = "&lt;",
	[">"] = "&gt;",
	["\r"] = "<span class='escape'>\\r</span>",
	["\v"] = "<span class='escape'>\\v</span>",
	["\f"] = "<span class='escape'>\\f</span>",
	["\194\133"] = "<span class='escape'>\\u0085</span>",
	["\226\128\168"] = "<span class='escape'>\\u2028</span>",
	["\226\128\169"] = "<span class='escape'>\\u2029</span>",
}

local function escape(s)
	return (string.gsub(s, "[&<>\r\v\f]", escape_map))
end

-- The next line and the two Unicode separators take several bytes, which a
-- lexer may hand back in different tokens, so they are escaped once the
-- tokens are joined again.
local function escape_separators(s)
	s = string.gsub(s, "\194\133", escape_map)
	return (string.gsub(s, "\226\128[\168\169]", escape_map))
end

local function scintillua_path()
	local env = os.getenv(scintillua_env)
	if env then
		return env
	end
	local candidates = {}
	local config = os.getenv("CGIT_CONFIG")
	if config then
		local dir = string.match(config, "^(.*)/[^/]+$")
		if dir then
			candidates[#candidates + 1] = dir .. "/scintillua/lexers"
		end
	end
	for _, dir in ipairs(scintillua_dirs) do
		candidates[#candidates + 1] = dir
	end
	-- A candidate counts only when lexer.lua is actually in it, so a
	-- directory that exists but holds no lexers does not shadow a
	-- later one.
	for _, dir in ipairs(candidates) do
		local f = io.open(dir .. "/lexer.lua", "r")
		if f then
			f:close()
			return dir
		end
	end
	return nil
end

local function load_scintillua()
	local dir = scintillua_path()
	if not dir then
		return nil
	end
	package.path = dir .. "/?.lua;" .. package.path
	local ok, mod = pcall(require, "lexer")
	-- A real Scintillua exposes load(), so anything else on the path that
	-- happens to be called lexer is rejected rather than used.
	if ok and type(mod) == "table" and type(mod.load) == "function" then
		return mod
	end
	return nil
end

-- Loading a lexer runs its Lua, and one written for another Scintillua can
-- raise, so a failure here just leaves this file uncoloured.
local function load_lexer_name(name)
	if name == nil then
		return nil
	end
	local ok, lexer = pcall(scintillua.load, name)
	if ok and lexer then
		return lexer
	end
	return nil
end

local function lexer_for(name, text)
	if type(scintillua.detect) == "function" then
		-- The first line goes along, so a script with no extension is still
		-- known by its shebang.
		local ok, lang = pcall(scintillua.detect, name, text:match("^[^\n]*"))
		if ok and lang then
			local lexer = load_lexer_name(lang)
			if lexer then
				return lexer
			end
		end
	end
	local ext = string.match(name, "%.([^.]+)$")
	if not ext then
		return nil
	end
	ext = string.lower(ext)
	return load_lexer_name(ext_lexer[ext]) or load_lexer_name(ext)
end

local function highlight(text)
	local lexer = lexer_for(filename, text)
	if not lexer then
		return nil
	end
	local ok, tokens = pcall(lexer.lex, lexer, text)
	if not ok or type(tokens) ~= "table" then
		return nil
	end
	local out = {}
	local pos = 1
	-- Scintillua returns one flat list of a tag name and the position
	-- just past the token it names, so a token is the text from where
	-- the one before it ended.
	for i = 1, #tokens, 2 do
		local tag = tokens[i]
		local stop = tokens[i + 1]
		local part = escape(string.sub(text, pos, stop - 1))
		local class = css[string.match(tag, "^[%w_]+")]
		if class and part ~= "" then
			part = "<span class='" .. class .. "'>" .. part .. "</span>"
		end
		out[#out + 1] = part
		pos = stop
	end
	-- A lexer can stop short of the end, and every byte still has to reach
	-- the page or the line number gutter beside it drifts out of step.
	if pos <= #text then
		out[#out + 1] = escape(string.sub(text, pos))
	end
	return escape_separators(table.concat(out))
end

function filter_open(name)
	filename = name or ""
	chunks = {}
end

function filter_write(str)
	chunks[#chunks + 1] = str
end

-- cgit takes filter output through a C string sink that stops at the first
-- NUL byte, so a write holding one loses everything from the NUL on. That
-- reaches binary files which slip past cgit's text detection, not ordinary
-- source.
function filter_close()
	local text = table.concat(chunks)
	local fits = #text <= max_bytes
	chunks = {}
	-- A browser reads CR LF as one line feed only while the two stay
	-- together, and a lexer could hand them back in different tokens.
	-- Every carriage return left after this is shown by escape().
	text = string.gsub(text, "\r\n", "\n")
	if fits then
		if scintillua == nil then
			scintillua = load_scintillua() or false
		end
		if scintillua then
			local ok, marked = pcall(highlight, text)
			if ok and marked then
				html(marked)
				return 0
			end
		end
	end
	local pos = 1
	while pos <= #text do
		local stop = pos + slice_bytes - 1
		-- A slice runs on to the end of the character it would cut, so
		-- no separator is split where escape_separators cannot see it.
		while string.find(text, "^[\128-\191]", stop + 1) do
			stop = stop + 1
		end
		html(escape_separators(escape(string.sub(text, pos, stop))))
		pos = stop + 1
	end
	return 0
end