Render more of the markdown seen in readmes

A line of equals signs or dashes under a paragraph now makes a heading
of it, as on GitHub, instead of running into its text, and any other
rule ends the paragraph. Four spaces or a tab start a code block, and an
escaped pipe stays in its table cell. A link opening with two slashes
or backslashes in any mix leads to another site, so it renders as text
like the plain scheme relative one already did.

2 files changed, 131 insertions, 19 deletions

diff --git a/custom/extensions/about-render.lua b/custom/extensions/about-render.luaindex d7753a3..040768c 100644--- a/custom/extensions/about-render.lua+++ b/custom/extensions/about-render.lua@@ -81,11 +81,36 @@ local function split_lines(text) 	end end +-- A pipe escaped with a backslash belongs to its cell, even inside a code+-- span, so as on GitHub the backslash goes and the pipe stays. An even run of+-- backslashes escapes only itself, so the pipe after it still divides. local function split_cells(row)-	row = trim(row):gsub("^|", ""):gsub("|$", "")-	local cells = {}-	for cell in (row .. "|"):gmatch("(.-)|") do-		cells[#cells + 1] = trim(cell)+	row = trim(row)+	local cells, parts, start = {}, {}, 1+	if row:sub(1, 1) == "|" then+		start = 2+	end+	while true do+		local bar = row:find("|", start, true)+		if not bar then+			break+		end+		local slashes = 0+		while row:sub(bar - slashes - 1, bar - slashes - 1) == "\\" do+			slashes = slashes + 1+		end+		if slashes % 2 == 1 then+			parts[#parts + 1] = row:sub(start, bar - 2) .. "|"+		else+			parts[#parts + 1] = row:sub(start, bar - 1)+			cells[#cells + 1] = trim(table.concat(parts))+			parts = {}+		end+		start = bar + 1+	end+	if start <= #row or #parts > 0 then+		parts[#parts + 1] = row:sub(start)+		cells[#cells + 1] = trim(table.concat(parts)) 	end 	return cells end@@ -96,10 +121,11 @@ end -- so it is safe on Lua 5.1, where an embedded zero byte ends a pattern. local function safe_url(url) 	url = url:gsub("[%c%s]", "")-	-- A leading "//" or "/\" is scheme-relative, which a browser-	-- resolves to another origin, so it is not the local target the-	-- parser meant to allow.-	if url:sub(1, 2) == "//" or url:sub(1, 2) == "/\\" then+	-- A leading "//" is scheme-relative, which a browser resolves to+	-- another origin, and it reads a backslash there as a slash, so no+	-- pair of the two is the local target the parser meant to allow.+	local head = url:sub(1, 2):gsub("\\", "/")+	if head == "//" then 		return nil 	end 	local scheme = url:match("^(%a[%w%+%.%-]*):")@@ -316,11 +342,10 @@ end   -- Markdown is a deliberate subset rather than CommonMark, leaving out--- reference links, raw HTML passthrough, nested lists and setext headings,--- and inline markup does not span the end of a line. Man rendering covers the--- common macros and drops the rest. Both parse into a node tree before--- anything is emitted, so a failed parse falls back to plain text without--- leaving half a page behind.+-- reference links, raw HTML passthrough and nested lists, and inline markup+-- does not span the end of a line. Man rendering covers the common macros and+-- drops the rest. Both parse into a node tree before anything is emitted, so a+-- failed parse falls back to plain text without leaving half a page behind.  local render_markdown, render_man @@ -428,6 +453,11 @@ if has_lpeg then  	local lang_char = R("az", "AZ", "09") + S("_.+#-") 	local heading_line = C(P("#") * P("#") ^ -5) * space ^ 1 * C(P(1) ^ 0)+	-- A run of equals signs or dashes under a paragraph turns it into a+	-- heading, as on GitHub, rather than reading as more of its text.+	local setext_line = space ^ 0 * (C(P("=") ^ 1) + C(P("-") ^ 1))+		* space ^ 0 * eol+	local code_indent = P("    ") + P("\t") 	local function thematic(mark) 		return space ^ 0 * P(mark) * (space ^ 0 * P(mark)) ^ 2 * space ^ 0 * eol 	end@@ -454,6 +484,28 @@ if has_lpeg then  	local function ordered(mark) return mark:match("%d") ~= nil end +	local function follows_list(blocks)+		local last = blocks[#blocks]+		return last ~= nil and last.kind == "list"+	end++	-- An indented block runs on through blank lines, and up to four spaces+	-- or one tab come off each of its lines.+	local function continues_code(line)+		return code_indent:match(line) or not line:find("%S")+	end++	local function unindent(line)+		return line:match("^\t(.*)$") or line:match("^ ? ? ? ?(.*)$")+	end++	local function setext_level(line)+		local run = setext_line:match(line)+		if run then+			return run:sub(1, 1) == "=" and 1 or 2+		end+	end+ 	local function table_starts(lines, i) 		return i < #lines and lines[i]:find("|", 1, true) ~= nil 			and table_delimiter:match(lines[i + 1]) ~= nil@@ -589,17 +641,51 @@ if has_lpeg then 					ordered = is_ordered, 					items = items, 				}+			-- Four spaces or a tab start a code block, except straight+			-- after a list, where an indented paragraph more likely+			-- carries an item on.+			elseif code_indent:match(line) and not follows_list(blocks) then+				local code = {}+				while i <= n and continues_code(lines[i]) do+					code[#code + 1] = unindent(lines[i])+					i = i + 1+				end+				while code[#code]:match("^%s*$") do+					code[#code] = nil+				end+				blocks[#blocks + 1] = {+					kind = "code",+					text = table.concat(code, "\n") .. "\n",+				} 			else-				local paragraph = { line }+				local paragraph, level = { line }, nil 				i = i + 1-				while i <= n and not ends_paragraph(lines, i) do+				while i <= n do+					level = setext_level(lines[i])+					if level or ends_paragraph(lines, i)+						or break_line:match(lines[i]) then+						break+					end 					paragraph[#paragraph + 1] = lines[i] 					i = i + 1 				end-				blocks[#blocks + 1] = {-					kind = "para",-					lines = paragraph_lines(paragraph),-				}+				if level then+					local words = {}+					for k, text in ipairs(paragraph) do+						words[k] = trim(text)+					end+					blocks[#blocks + 1] = {+						kind = "heading",+						level = level,+						kids = parse_inline(table.concat(words, " ")),+					}+					i = i + 1+				else+					blocks[#blocks + 1] = {+						kind = "para",+						lines = paragraph_lines(paragraph),+					}+				end 			end 		end 		return blocks
diff --git a/custom/extensions/about-render.lua b/custom/extensions/about-render.lua
index d7753a3..040768c 100644
--- a/custom/extensions/about-render.lua
+++ b/custom/extensions/about-render.lua
@@ -81,11 +81,36 @@ local function split_lines(text)
 	end
 end
 
+-- A pipe escaped with a backslash belongs to its cell, even inside a code
+-- span, so as on GitHub the backslash goes and the pipe stays. An even run of
+-- backslashes escapes only itself, so the pipe after it still divides.
 local function split_cells(row)
-	row = trim(row):gsub("^|", ""):gsub("|$", "")
-	local cells = {}
-	for cell in (row .. "|"):gmatch("(.-)|") do
-		cells[#cells + 1] = trim(cell)
+	row = trim(row)
+	local cells, parts, start = {}, {}, 1
+	if row:sub(1, 1) == "|" then
+		start = 2
+	end
+	while true do
+		local bar = row:find("|", start, true)
+		if not bar then
+			break
+		end
+		local slashes = 0
+		while row:sub(bar - slashes - 1, bar - slashes - 1) == "\\" do
+			slashes = slashes + 1
+		end
+		if slashes % 2 == 1 then
+			parts[#parts + 1] = row:sub(start, bar - 2) .. "|"
+		else
+			parts[#parts + 1] = row:sub(start, bar - 1)
+			cells[#cells + 1] = trim(table.concat(parts))
+			parts = {}
+		end
+		start = bar + 1
+	end
+	if start <= #row or #parts > 0 then
+		parts[#parts + 1] = row:sub(start)
+		cells[#cells + 1] = trim(table.concat(parts))
 	end
 	return cells
 end
@@ -96,10 +121,11 @@ end
 -- so it is safe on Lua 5.1, where an embedded zero byte ends a pattern.
 local function safe_url(url)
 	url = url:gsub("[%c%s]", "")
-	-- A leading "//" or "/\" is scheme-relative, which a browser
-	-- resolves to another origin, so it is not the local target the
-	-- parser meant to allow.
-	if url:sub(1, 2) == "//" or url:sub(1, 2) == "/\\" then
+	-- A leading "//" is scheme-relative, which a browser resolves to
+	-- another origin, and it reads a backslash there as a slash, so no
+	-- pair of the two is the local target the parser meant to allow.
+	local head = url:sub(1, 2):gsub("\\", "/")
+	if head == "//" then
 		return nil
 	end
 	local scheme = url:match("^(%a[%w%+%.%-]*):")
@@ -316,11 +342,10 @@ end
 
 
 -- Markdown is a deliberate subset rather than CommonMark, leaving out
--- reference links, raw HTML passthrough, nested lists and setext headings,
--- and inline markup does not span the end of a line. Man rendering covers the
--- common macros and drops the rest. Both parse into a node tree before
--- anything is emitted, so a failed parse falls back to plain text without
--- leaving half a page behind.
+-- reference links, raw HTML passthrough and nested lists, and inline markup
+-- does not span the end of a line. Man rendering covers the common macros and
+-- drops the rest. Both parse into a node tree before anything is emitted, so a
+-- failed parse falls back to plain text without leaving half a page behind.
 
 local render_markdown, render_man
 
@@ -428,6 +453,11 @@ if has_lpeg then
 
 	local lang_char = R("az", "AZ", "09") + S("_.+#-")
 	local heading_line = C(P("#") * P("#") ^ -5) * space ^ 1 * C(P(1) ^ 0)
+	-- A run of equals signs or dashes under a paragraph turns it into a
+	-- heading, as on GitHub, rather than reading as more of its text.
+	local setext_line = space ^ 0 * (C(P("=") ^ 1) + C(P("-") ^ 1))
+		* space ^ 0 * eol
+	local code_indent = P("    ") + P("\t")
 	local function thematic(mark)
 		return space ^ 0 * P(mark) * (space ^ 0 * P(mark)) ^ 2 * space ^ 0 * eol
 	end
@@ -454,6 +484,28 @@ if has_lpeg then
 
 	local function ordered(mark) return mark:match("%d") ~= nil end
 
+	local function follows_list(blocks)
+		local last = blocks[#blocks]
+		return last ~= nil and last.kind == "list"
+	end
+
+	-- An indented block runs on through blank lines, and up to four spaces
+	-- or one tab come off each of its lines.
+	local function continues_code(line)
+		return code_indent:match(line) or not line:find("%S")
+	end
+
+	local function unindent(line)
+		return line:match("^\t(.*)$") or line:match("^ ? ? ? ?(.*)$")
+	end
+
+	local function setext_level(line)
+		local run = setext_line:match(line)
+		if run then
+			return run:sub(1, 1) == "=" and 1 or 2
+		end
+	end
+
 	local function table_starts(lines, i)
 		return i < #lines and lines[i]:find("|", 1, true) ~= nil
 			and table_delimiter:match(lines[i + 1]) ~= nil
@@ -589,17 +641,51 @@ if has_lpeg then
 					ordered = is_ordered,
 					items = items,
 				}
+			-- Four spaces or a tab start a code block, except straight
+			-- after a list, where an indented paragraph more likely
+			-- carries an item on.
+			elseif code_indent:match(line) and not follows_list(blocks) then
+				local code = {}
+				while i <= n and continues_code(lines[i]) do
+					code[#code + 1] = unindent(lines[i])
+					i = i + 1
+				end
+				while code[#code]:match("^%s*$") do
+					code[#code] = nil
+				end
+				blocks[#blocks + 1] = {
+					kind = "code",
+					text = table.concat(code, "\n") .. "\n",
+				}
 			else
-				local paragraph = { line }
+				local paragraph, level = { line }, nil
 				i = i + 1
-				while i <= n and not ends_paragraph(lines, i) do
+				while i <= n do
+					level = setext_level(lines[i])
+					if level or ends_paragraph(lines, i)
+						or break_line:match(lines[i]) then
+						break
+					end
 					paragraph[#paragraph + 1] = lines[i]
 					i = i + 1
 				end
-				blocks[#blocks + 1] = {
-					kind = "para",
-					lines = paragraph_lines(paragraph),
-				}
+				if level then
+					local words = {}
+					for k, text in ipairs(paragraph) do
+						words[k] = trim(text)
+					end
+					blocks[#blocks + 1] = {
+						kind = "heading",
+						level = level,
+						kids = parse_inline(table.concat(words, " ")),
+					}
+					i = i + 1
+				else
+					blocks[#blocks + 1] = {
+						kind = "para",
+						lines = paragraph_lines(paragraph),
+					}
+				end
 			end
 		end
 		return blocks
diff --git a/tests/extensions/test-about-render.lua b/tests/extensions/test-about-render.luaindex c4b16a8..7341385 100644--- a/tests/extensions/test-about-render.lua+++ b/tests/extensions/test-about-render.lua@@ -257,6 +257,26 @@ render("README.md", string.rep("# a\n", 50000)) render("README.md", "| a | b |\n|---|---|\n| x" .. string.rep(" ", 40000) .. "y | z |") h.check("a long run of spaces is trimmed quickly", os.clock() - started < 10) +out = render("README.md", "Title\n=====\n\nBody")+h.contains("equals signs under a line make a heading", out, "<h1 id='title'>Title</h1><p>Body</p>")++out = render("README.md", "Two\nlines\n---\nafter")+h.contains("dashes under a paragraph make a heading of it all", out,+	"<h2 id='two-lines'>Two lines</h2><p>after</p>")++out = render("README.md", "text\n***\nmore")+h.contains("a rule ends the paragraph above it", out, "<p>text</p><hr><p>more</p>")++out = render("README.md", "Para\n\n    code line\n\tnext <line>\n\nafter")+h.contains("an indented block renders as code", out,+	"<p>Para</p><pre><code>code line\nnext &lt;line&gt;\n</code></pre><p>after</p>")++out = render("README.md", "- item\n\n    more about it")+h.excludes("an indented paragraph after a list stays prose", out, "<pre>")++out = render("README.md", "[a](\\\\evil.example/x) [b](\\/evil.example/x)")+h.excludes("a scheme relative link spelt with backslashes renders no anchor", out, "<a ")+ out = render("README.md", "| a | b |\n| --- | --- |\n| 1 | 2 |") h.contains("a pipe table renders its head", out, "<thead><tr><th>a</th><th>b</th></tr></thead>") h.contains("a pipe table renders its body", out, "<tbody><tr><td>1</td><td>2</td></tr></tbody>")@@ -267,6 +287,12 @@ h.contains("a table cuts a paragraph short", out, "<p>Pick one:</p><table><thead out = render("README.md", "a | b in prose") h.excludes("a pipe without a delimiter row stays prose", out, "<table") +out = render("README.md", "| a | b |\n|---|---|\n| x \\| y | `p \\| q` |")+h.contains("an escaped pipe stays in its cell", out, "<td>x | y</td><td><code>p | q</code></td>")++out = render("README.md", "| a | b |\n|---|---|\n| x \\\\| y |")+h.contains("an escaped backslash still lets the pipe divide", out, "<td>x \\</td><td>y</td>")+ out = render("README.md", "- first\n- second") h.contains("dashes render an unordered list", out, "<ul><li>first</li><li>second</li></ul>") 
diff --git a/tests/extensions/test-about-render.lua b/tests/extensions/test-about-render.lua
index c4b16a8..7341385 100644
--- a/tests/extensions/test-about-render.lua
+++ b/tests/extensions/test-about-render.lua
@@ -257,6 +257,26 @@ render("README.md", string.rep("# a\n", 50000))
 render("README.md", "| a | b |\n|---|---|\n| x" .. string.rep(" ", 40000) .. "y | z |")
 h.check("a long run of spaces is trimmed quickly", os.clock() - started < 10)
 
+out = render("README.md", "Title\n=====\n\nBody")
+h.contains("equals signs under a line make a heading", out, "<h1 id='title'>Title</h1><p>Body</p>")
+
+out = render("README.md", "Two\nlines\n---\nafter")
+h.contains("dashes under a paragraph make a heading of it all", out,
+	"<h2 id='two-lines'>Two lines</h2><p>after</p>")
+
+out = render("README.md", "text\n***\nmore")
+h.contains("a rule ends the paragraph above it", out, "<p>text</p><hr><p>more</p>")
+
+out = render("README.md", "Para\n\n    code line\n\tnext <line>\n\nafter")
+h.contains("an indented block renders as code", out,
+	"<p>Para</p><pre><code>code line\nnext &lt;line&gt;\n</code></pre><p>after</p>")
+
+out = render("README.md", "- item\n\n    more about it")
+h.excludes("an indented paragraph after a list stays prose", out, "<pre>")
+
+out = render("README.md", "[a](\\\\evil.example/x) [b](\\/evil.example/x)")
+h.excludes("a scheme relative link spelt with backslashes renders no anchor", out, "<a ")
+
 out = render("README.md", "| a | b |\n| --- | --- |\n| 1 | 2 |")
 h.contains("a pipe table renders its head", out, "<thead><tr><th>a</th><th>b</th></tr></thead>")
 h.contains("a pipe table renders its body", out, "<tbody><tr><td>1</td><td>2</td></tr></tbody>")
@@ -267,6 +287,12 @@ h.contains("a table cuts a paragraph short", out, "<p>Pick one:</p><table><thead
 out = render("README.md", "a | b in prose")
 h.excludes("a pipe without a delimiter row stays prose", out, "<table")
 
+out = render("README.md", "| a | b |\n|---|---|\n| x \\| y | `p \\| q` |")
+h.contains("an escaped pipe stays in its cell", out, "<td>x | y</td><td><code>p | q</code></td>")
+
+out = render("README.md", "| a | b |\n|---|---|\n| x \\\\| y |")
+h.contains("an escaped backslash still lets the pipe divide", out, "<td>x \\</td><td>y</td>")
+
 out = render("README.md", "- first\n- second")
 h.contains("dashes render an unordered list", out, "<ul><li>first</li><li>second</li></ul>")