Render the man pages that generators write

DocBook, pod2man and hand written pages leaked roff onto the page: font
escapes inside no fill blocks, comments, layout escapes such as \% and
\c, the colours and half line moves around DocBook's links, and the
requests and macro definitions their preambles open with. Those now
draw nothing, the common named characters and strings render, .EX and
pod2man's .Vb read as code, a .TP tag takes a line of its own and the
alternating font macros switch fonts between their arguments. Anything
rarer still falls back to its text.

1 file changed, 188 insertions, 52 deletions

diff --git a/custom/extensions/about-render.lua b/custom/extensions/about-render.luaindex 040768c..808dd08 100644--- a/custom/extensions/about-render.lua+++ b/custom/extensions/about-render.lua@@ -343,9 +343,10 @@ end  -- Markdown is a deliberate subset rather than CommonMark, leaving out -- reference links, raw HTML passthrough and nested lists, and inline markup--- does not span the end of a line. Man rendering covers the common macros and--- drops the rest. Both parse into a node tree before anything is emitted, so a--- failed parse falls back to plain text without leaving half a page behind.+-- does not span the end of a line. Man rendering covers the man(7) macros and+-- drops the roff requests and definitions that generators put around them.+-- Both parse into a node tree before anything is emitted, so a failed parse+-- falls back to plain text without leaving half a page behind.  local render_markdown, render_man @@ -712,41 +713,58 @@ if has_lpeg then 	-- and folds it into the nodes markdown emits, instead of nesting. 	local macro_line = S(".'") * space ^ 0 * C((1 - space) ^ 1) * space ^ 0 * C(P(1) ^ 0) -	local one_font = { B = "B", I = "I" }-	local two_font = { CB = "B", BI = "B", CI = "I" }+	-- Fonts by name and by position. Anything else, \fP for the previous+	-- font among them, falls back to roman.+	local fonts = {+		B = "B", I = "I", BI = "B", CB = "B", CI = "I",+		["2"] = "I", ["3"] = "B",+	}++	-- The named characters and strings generated pages use most. A name+	-- with no entry renders as nothing rather than as its spelling. 	local man_chars = {-		aq = "'",-		cq = "'",-		oq = "'",-		dq = '"',-		lq = '"',-		rq = '"',-		hy = "-",-		en = "-",-		em = "-",+		aq = "'", dq = '"', cq = "’", oq = "‘", lq = "“", rq = "”",+		hy = "-", en = "–", em = "—", bu = "•", co = "©", rg = "®",+		tm = "™", ti = "~", ha = "^", rs = "\\",+	}+	local man_strings = {+		lq = "“", rq = "”", R = "®", Tm = "™", Aq = "'",+		['L"'] = "“", ['R"'] = "”", 	}+ 	local backslash = P("\\")-	local function font_token(name) return { kind = "font", font = name } end 	local function text_token(text) return { kind = "text", text = text } end 	local skip_token = { kind = "skip" }+	local function skip() return skip_token end+	local function font_token(name)+		return { kind = "font", font = fonts[name] or "R" }+	end+	local function char_token(name) return text_token(man_chars[name] or "") end+	local function string_token(name)+		return text_token(man_strings[name] or "")+	end++	-- \( takes a two character name and \[ a longer one, bounded so that a+	-- line of unclosed openers stays linear.+	local named = P("(") * C(P(1) * P(1))+		+ P("[") * C((1 - P("]")) ^ -64) * P("]") +	-- Comments, size changes, colours, the half line moves DocBook puts+	-- around a footnote mark and conditional braces draw nothing, and nor do+	-- the escapes that only steer hyphenation or breaks. 	local man_inline_grammar = Ct((-		  (backslash * P("f") * P("(") * C(P(1) * P(1)))-			/ function(name) return font_token(two_font[name] or "R") end-		+ (backslash * P("f") * P("[") * C((1 - P("]")) ^ 0) * P("]"))-			/ function(name) return font_token(one_font[name] or "R") end-		+ (backslash * P("f") * C(P(1)))-			/ function(name) return font_token(one_font[name] or "R") end+		  (backslash * P("f") * (named + C(P(1)))) / font_token+		+ (backslash * S('"#') * P(1) ^ 0) / skip 		+ (backslash * P("-")) / function() return text_token("-") end 		+ (backslash * P("e")) / function() return text_token("\\") end-		+ (backslash * P(" ")) / function() return text_token(" ") end-		+ (backslash * S("&|^")) / function() return skip_token end-		+ (backslash * P("(") * C(P(1) * P(1)))-			/ function(name) return text_token(man_chars[name] or "") end-		+ (backslash * P("*") * (P("(") * P(1) * P(1) + P(1)))-			/ function() return skip_token end+		+ (backslash * S(" 0~")) / function() return text_token(" ") end+		+ (backslash * S("&|^%:cud{}")) / skip+		+ (backslash * named) / char_token+		+ (backslash * P("*") * (named + C(P(1)))) / string_token+		+ (backslash * P("s") * S("+-") ^ -1 * (named + R("09") * R("09") ^ -1)) / skip+		+ (backslash * S("mM") * (named + P(1))) / skip 		+ (backslash * C(P(1))) / text_token-		+ backslash / function() return skip_token end+		+ backslash / skip 		+ C((1 - backslash) ^ 1) / text_token 	) ^ 0) @@ -788,10 +806,75 @@ if has_lpeg then 		return nodes 	end +	-- A no fill block keeps its lines as they are, so only the text of+	-- each one is wanted there, without its fonts.+	local function man_plain(text)+		local parts = {}+		for _, token in ipairs(man_inline_grammar:match(text) or {}) do+			if token.kind == "text" then+				parts[#parts + 1] = token.text+			end+		end+		return table.concat(parts)+	end++	-- A macro's arguments divide at blanks, except inside double quotes.+	local function man_args(rest)+		local args, i = {}, 1+		while true do+			i = rest:find("%S", i)+			if not i then+				return args+			end+			local stop+			if rest:sub(i, i) == '"' then+				stop = rest:find('"', i + 1, true) or #rest + 1+				args[#args + 1] = rest:sub(i + 1, stop - 1)+			else+				stop = rest:find("%s", i) or #rest + 1+				args[#args + 1] = rest:sub(i, stop - 1)+			end+			i = stop + 1+		end+	end++	local function in_font(font, nodes)+		if font == "B" then+			return { { kind = "strong", kids = nodes } }+		elseif font == "I" then+			return { { kind = "em", kids = nodes } }+		end+		return nodes+	end++	-- Each alternating macro sets its arguments in its two fonts by turns+	-- with nothing between them, as in .BR ls (1).+	local single_font = { B = "B", I = "I" }+	local alternating = {+		BR = { "B", "R" }, RB = { "R", "B" }, IR = { "I", "R" },+		RI = { "R", "I" }, BI = { "B", "I" }, IB = { "I", "B" },+	}++	-- No fill blocks under their man(7) and groff names, and under the Vb+	-- and Ve that pod2man defines for its verbatim paragraphs.+	local nofill_start = { nf = true, EX = true, Vb = true }+	local nofill_end = { fi = true, EE = true, Ve = true }++	-- Macros that only begin a paragraph, Sp being pod2man's, ones that only+	-- set spacing or an index entry, and the requests that open a definition,+	-- which runs to a line of two dots.+	local breaks = {+		TH = true, PP = true, LP = true, P = true, HP = true, RS = true,+		RE = true, sp = true, Sp = true,+	}+	local silent = { PD = true, DT = true, IX = true }+	local definitions = { de = true, de1 = true, am = true, ig = true }+ 	local function parse_man(text) 		local lines = split_lines(text)-		local blocks = {}+		local blocks, defined = {}, {} 		local paragraph, nofill_lines, nofill = nil, nil, false+		local skipping, tag_pending = false, false 		local function flush_paragraph() 			if paragraph and #paragraph > 0 then 				blocks[#blocks + 1] = { kind = "para", lines = paragraph }@@ -808,54 +891,107 @@ if has_lpeg then 				nofill_lines = nil 			end 		end+		-- A tagged paragraph puts its tag on a line of its own above the+		-- text it describes. 		local function add_line(nodes)+			if tag_pending then+				nodes[#nodes + 1] = { kind = "break" }+				tag_pending = false+			end 			paragraph = paragraph or {} 			paragraph[#paragraph + 1] = nodes 		end 		for _, line in ipairs(lines) do 			local macro, rest = macro_line:match(line)-			if rest then-				rest = (rest:gsub('^"(.*)"$', "%1"))-			end-			if nofill then-				if macro == "fi" then+			if skipping then+				skipping = not line:find("^%.%.%s*$")+			elseif macro and macro:sub(1, 1) == "\\"+				or line:find("^[.']%s*$") then+				-- A comment, the close of a conditional block or an empty+				-- request.+			elseif nofill then+				if nofill_end[macro] then 					nofill = false 					flush_nofill()-				else-					nofill_lines[#nofill_lines + 1] = line+				elseif not macro then+					nofill_lines[#nofill_lines + 1] = man_plain(line) 				end+			elseif definitions[macro] then+				-- What a page defines for itself cannot be expanded here,+				-- so the definition is skipped and so are its uses.+				local name = man_args(rest)[1]+				if name and macro ~= "ig" then+					defined[name] = true+				end+				skipping = true 			elseif line:match("^%s*$") then 				flush_paragraph()-			elseif macro and macro:sub(1, 1) == "\\" then-				-- A roff comment, dropped. 			elseif macro == "SH" or macro == "SS" then 				flush_paragraph()+				tag_pending = false 				blocks[#blocks + 1] = { 					kind = "heading", 					level = (macro == "SH") and 2 or 3,-					kids = man_inline(rest),+					kids = man_inline(table.concat(man_args(rest), " ")), 				}-			elseif macro == "nf" then+			elseif nofill_start[macro] then 				flush_paragraph() 				nofill = true 				nofill_lines = {}-			elseif macro == "fi" then+			elseif nofill_end[macro] then 				flush_nofill()-			elseif macro == "B" or macro == "BR" or macro == "RB" or macro == "BI" or macro == "IB" then-				add_line({ { kind = "strong", kids = man_inline(rest) } })-			elseif macro == "I" or macro == "IR" or macro == "RI" then-				add_line({ { kind = "em", kids = man_inline(rest) } })-			elseif macro == "TH" or macro == "PP" or macro == "LP"-				or macro == "P" or macro == "TP" or macro == "IP"-				or macro == "HP" or macro == "RS" or macro == "RE"-				or macro == "sp" or macro == "br" then+			elseif single_font[macro] then+				local nodes = man_inline(table.concat(man_args(rest), " "))+				add_line(in_font(single_font[macro], nodes))+			elseif alternating[macro] then+				local pair, nodes = alternating[macro], {}+				for k, arg in ipairs(man_args(rest)) do+					local font = pair[(k - 1) % 2 + 1]+					for _, node in ipairs(in_font(font, man_inline(arg))) do+						nodes[#nodes + 1] = node+					end+				end+				add_line(nodes)+			elseif macro == "TP" then 				flush_paragraph()+				tag_pending = true+			elseif macro == "IP" then+				-- A bullet or a number leads the text on its line, and any+				-- other tag takes a line of its own.+				flush_paragraph()+				local tag = man_args(rest)[1]+				if tag and tag ~= "" then+					local nodes, plain = man_inline(tag), man_plain(tag)+					if plain:find("%w") and not plain:find("^%d+[.)]?$") then+						nodes[#nodes + 1] = { kind = "break" }+					else+						nodes[#nodes + 1] = { kind = "text", text = " " }+					end+					add_line(nodes)+				end+			elseif macro == "br" then+				if paragraph then+					local last = paragraph[#paragraph]+					last[#last + 1] = { kind = "break" }+				end+			elseif breaks[macro] then+				flush_paragraph()+			elseif macro+				and (silent[macro] or defined[macro] or macro:find("^%l")) then+				-- A macro the page defined, or a roff request, which in+				-- lower case only steers the layout. 			elseif macro then-				if rest ~= "" then-					add_line(man_inline(rest))+				local args = man_args(rest)+				if #args > 0 then+					add_line(man_inline(table.concat(args, " "))) 				end 			else-				add_line(man_inline(line))+				-- A line of nothing but escapes that draw nothing, such as+				-- a conditional's brace, adds no line.+				local nodes = man_inline(line)+				if #nodes > 0 then+					add_line(nodes)+				end 			end 		end 		flush_paragraph()
diff --git a/custom/extensions/about-render.lua b/custom/extensions/about-render.lua
index 040768c..808dd08 100644
--- a/custom/extensions/about-render.lua
+++ b/custom/extensions/about-render.lua
@@ -343,9 +343,10 @@ end
 
 -- Markdown is a deliberate subset rather than CommonMark, leaving out
 -- reference links, raw HTML passthrough and nested lists, and inline markup
--- does not span the end of a line. Man rendering covers the common macros and
--- drops the rest. Both parse into a node tree before anything is emitted, so a
--- failed parse falls back to plain text without leaving half a page behind.
+-- does not span the end of a line. Man rendering covers the man(7) macros and
+-- drops the roff requests and definitions that generators put around them.
+-- Both parse into a node tree before anything is emitted, so a failed parse
+-- falls back to plain text without leaving half a page behind.
 
 local render_markdown, render_man
 
@@ -712,41 +713,58 @@ if has_lpeg then
 	-- and folds it into the nodes markdown emits, instead of nesting.
 	local macro_line = S(".'") * space ^ 0 * C((1 - space) ^ 1) * space ^ 0 * C(P(1) ^ 0)
 
-	local one_font = { B = "B", I = "I" }
-	local two_font = { CB = "B", BI = "B", CI = "I" }
+	-- Fonts by name and by position. Anything else, \fP for the previous
+	-- font among them, falls back to roman.
+	local fonts = {
+		B = "B", I = "I", BI = "B", CB = "B", CI = "I",
+		["2"] = "I", ["3"] = "B",
+	}
+
+	-- The named characters and strings generated pages use most. A name
+	-- with no entry renders as nothing rather than as its spelling.
 	local man_chars = {
-		aq = "'",
-		cq = "'",
-		oq = "'",
-		dq = '"',
-		lq = '"',
-		rq = '"',
-		hy = "-",
-		en = "-",
-		em = "-",
+		aq = "'", dq = '"', cq = "’", oq = "‘", lq = "“", rq = "”",
+		hy = "-", en = "–", em = "—", bu = "•", co = "©", rg = "®",
+		tm = "™", ti = "~", ha = "^", rs = "\\",
+	}
+	local man_strings = {
+		lq = "“", rq = "”", R = "®", Tm = "™", Aq = "'",
+		['L"'] = "“", ['R"'] = "”",
 	}
+
 	local backslash = P("\\")
-	local function font_token(name) return { kind = "font", font = name } end
 	local function text_token(text) return { kind = "text", text = text } end
 	local skip_token = { kind = "skip" }
+	local function skip() return skip_token end
+	local function font_token(name)
+		return { kind = "font", font = fonts[name] or "R" }
+	end
+	local function char_token(name) return text_token(man_chars[name] or "") end
+	local function string_token(name)
+		return text_token(man_strings[name] or "")
+	end
+
+	-- \( takes a two character name and \[ a longer one, bounded so that a
+	-- line of unclosed openers stays linear.
+	local named = P("(") * C(P(1) * P(1))
+		+ P("[") * C((1 - P("]")) ^ -64) * P("]")
 
+	-- Comments, size changes, colours, the half line moves DocBook puts
+	-- around a footnote mark and conditional braces draw nothing, and nor do
+	-- the escapes that only steer hyphenation or breaks.
 	local man_inline_grammar = Ct((
-		  (backslash * P("f") * P("(") * C(P(1) * P(1)))
-			/ function(name) return font_token(two_font[name] or "R") end
-		+ (backslash * P("f") * P("[") * C((1 - P("]")) ^ 0) * P("]"))
-			/ function(name) return font_token(one_font[name] or "R") end
-		+ (backslash * P("f") * C(P(1)))
-			/ function(name) return font_token(one_font[name] or "R") end
+		  (backslash * P("f") * (named + C(P(1)))) / font_token
+		+ (backslash * S('"#') * P(1) ^ 0) / skip
 		+ (backslash * P("-")) / function() return text_token("-") end
 		+ (backslash * P("e")) / function() return text_token("\\") end
-		+ (backslash * P(" ")) / function() return text_token(" ") end
-		+ (backslash * S("&|^")) / function() return skip_token end
-		+ (backslash * P("(") * C(P(1) * P(1)))
-			/ function(name) return text_token(man_chars[name] or "") end
-		+ (backslash * P("*") * (P("(") * P(1) * P(1) + P(1)))
-			/ function() return skip_token end
+		+ (backslash * S(" 0~")) / function() return text_token(" ") end
+		+ (backslash * S("&|^%:cud{}")) / skip
+		+ (backslash * named) / char_token
+		+ (backslash * P("*") * (named + C(P(1)))) / string_token
+		+ (backslash * P("s") * S("+-") ^ -1 * (named + R("09") * R("09") ^ -1)) / skip
+		+ (backslash * S("mM") * (named + P(1))) / skip
 		+ (backslash * C(P(1))) / text_token
-		+ backslash / function() return skip_token end
+		+ backslash / skip
 		+ C((1 - backslash) ^ 1) / text_token
 	) ^ 0)
 
@@ -788,10 +806,75 @@ if has_lpeg then
 		return nodes
 	end
 
+	-- A no fill block keeps its lines as they are, so only the text of
+	-- each one is wanted there, without its fonts.
+	local function man_plain(text)
+		local parts = {}
+		for _, token in ipairs(man_inline_grammar:match(text) or {}) do
+			if token.kind == "text" then
+				parts[#parts + 1] = token.text
+			end
+		end
+		return table.concat(parts)
+	end
+
+	-- A macro's arguments divide at blanks, except inside double quotes.
+	local function man_args(rest)
+		local args, i = {}, 1
+		while true do
+			i = rest:find("%S", i)
+			if not i then
+				return args
+			end
+			local stop
+			if rest:sub(i, i) == '"' then
+				stop = rest:find('"', i + 1, true) or #rest + 1
+				args[#args + 1] = rest:sub(i + 1, stop - 1)
+			else
+				stop = rest:find("%s", i) or #rest + 1
+				args[#args + 1] = rest:sub(i, stop - 1)
+			end
+			i = stop + 1
+		end
+	end
+
+	local function in_font(font, nodes)
+		if font == "B" then
+			return { { kind = "strong", kids = nodes } }
+		elseif font == "I" then
+			return { { kind = "em", kids = nodes } }
+		end
+		return nodes
+	end
+
+	-- Each alternating macro sets its arguments in its two fonts by turns
+	-- with nothing between them, as in .BR ls (1).
+	local single_font = { B = "B", I = "I" }
+	local alternating = {
+		BR = { "B", "R" }, RB = { "R", "B" }, IR = { "I", "R" },
+		RI = { "R", "I" }, BI = { "B", "I" }, IB = { "I", "B" },
+	}
+
+	-- No fill blocks under their man(7) and groff names, and under the Vb
+	-- and Ve that pod2man defines for its verbatim paragraphs.
+	local nofill_start = { nf = true, EX = true, Vb = true }
+	local nofill_end = { fi = true, EE = true, Ve = true }
+
+	-- Macros that only begin a paragraph, Sp being pod2man's, ones that only
+	-- set spacing or an index entry, and the requests that open a definition,
+	-- which runs to a line of two dots.
+	local breaks = {
+		TH = true, PP = true, LP = true, P = true, HP = true, RS = true,
+		RE = true, sp = true, Sp = true,
+	}
+	local silent = { PD = true, DT = true, IX = true }
+	local definitions = { de = true, de1 = true, am = true, ig = true }
+
 	local function parse_man(text)
 		local lines = split_lines(text)
-		local blocks = {}
+		local blocks, defined = {}, {}
 		local paragraph, nofill_lines, nofill = nil, nil, false
+		local skipping, tag_pending = false, false
 		local function flush_paragraph()
 			if paragraph and #paragraph > 0 then
 				blocks[#blocks + 1] = { kind = "para", lines = paragraph }
@@ -808,54 +891,107 @@ if has_lpeg then
 				nofill_lines = nil
 			end
 		end
+		-- A tagged paragraph puts its tag on a line of its own above the
+		-- text it describes.
 		local function add_line(nodes)
+			if tag_pending then
+				nodes[#nodes + 1] = { kind = "break" }
+				tag_pending = false
+			end
 			paragraph = paragraph or {}
 			paragraph[#paragraph + 1] = nodes
 		end
 		for _, line in ipairs(lines) do
 			local macro, rest = macro_line:match(line)
-			if rest then
-				rest = (rest:gsub('^"(.*)"$', "%1"))
-			end
-			if nofill then
-				if macro == "fi" then
+			if skipping then
+				skipping = not line:find("^%.%.%s*$")
+			elseif macro and macro:sub(1, 1) == "\\"
+				or line:find("^[.']%s*$") then
+				-- A comment, the close of a conditional block or an empty
+				-- request.
+			elseif nofill then
+				if nofill_end[macro] then
 					nofill = false
 					flush_nofill()
-				else
-					nofill_lines[#nofill_lines + 1] = line
+				elseif not macro then
+					nofill_lines[#nofill_lines + 1] = man_plain(line)
 				end
+			elseif definitions[macro] then
+				-- What a page defines for itself cannot be expanded here,
+				-- so the definition is skipped and so are its uses.
+				local name = man_args(rest)[1]
+				if name and macro ~= "ig" then
+					defined[name] = true
+				end
+				skipping = true
 			elseif line:match("^%s*$") then
 				flush_paragraph()
-			elseif macro and macro:sub(1, 1) == "\\" then
-				-- A roff comment, dropped.
 			elseif macro == "SH" or macro == "SS" then
 				flush_paragraph()
+				tag_pending = false
 				blocks[#blocks + 1] = {
 					kind = "heading",
 					level = (macro == "SH") and 2 or 3,
-					kids = man_inline(rest),
+					kids = man_inline(table.concat(man_args(rest), " ")),
 				}
-			elseif macro == "nf" then
+			elseif nofill_start[macro] then
 				flush_paragraph()
 				nofill = true
 				nofill_lines = {}
-			elseif macro == "fi" then
+			elseif nofill_end[macro] then
 				flush_nofill()
-			elseif macro == "B" or macro == "BR" or macro == "RB" or macro == "BI" or macro == "IB" then
-				add_line({ { kind = "strong", kids = man_inline(rest) } })
-			elseif macro == "I" or macro == "IR" or macro == "RI" then
-				add_line({ { kind = "em", kids = man_inline(rest) } })
-			elseif macro == "TH" or macro == "PP" or macro == "LP"
-				or macro == "P" or macro == "TP" or macro == "IP"
-				or macro == "HP" or macro == "RS" or macro == "RE"
-				or macro == "sp" or macro == "br" then
+			elseif single_font[macro] then
+				local nodes = man_inline(table.concat(man_args(rest), " "))
+				add_line(in_font(single_font[macro], nodes))
+			elseif alternating[macro] then
+				local pair, nodes = alternating[macro], {}
+				for k, arg in ipairs(man_args(rest)) do
+					local font = pair[(k - 1) % 2 + 1]
+					for _, node in ipairs(in_font(font, man_inline(arg))) do
+						nodes[#nodes + 1] = node
+					end
+				end
+				add_line(nodes)
+			elseif macro == "TP" then
 				flush_paragraph()
+				tag_pending = true
+			elseif macro == "IP" then
+				-- A bullet or a number leads the text on its line, and any
+				-- other tag takes a line of its own.
+				flush_paragraph()
+				local tag = man_args(rest)[1]
+				if tag and tag ~= "" then
+					local nodes, plain = man_inline(tag), man_plain(tag)
+					if plain:find("%w") and not plain:find("^%d+[.)]?$") then
+						nodes[#nodes + 1] = { kind = "break" }
+					else
+						nodes[#nodes + 1] = { kind = "text", text = " " }
+					end
+					add_line(nodes)
+				end
+			elseif macro == "br" then
+				if paragraph then
+					local last = paragraph[#paragraph]
+					last[#last + 1] = { kind = "break" }
+				end
+			elseif breaks[macro] then
+				flush_paragraph()
+			elseif macro
+				and (silent[macro] or defined[macro] or macro:find("^%l")) then
+				-- A macro the page defined, or a roff request, which in
+				-- lower case only steers the layout.
 			elseif macro then
-				if rest ~= "" then
-					add_line(man_inline(rest))
+				local args = man_args(rest)
+				if #args > 0 then
+					add_line(man_inline(table.concat(args, " ")))
 				end
 			else
-				add_line(man_inline(line))
+				-- A line of nothing but escapes that draw nothing, such as
+				-- a conditional's brace, adds no line.
+				local nodes = man_inline(line)
+				if #nodes > 0 then
+					add_line(nodes)
+				end
 			end
 		end
 		flush_paragraph()