Show characters a browser breaks at as escapes

The HTML parser turns a carriage return with no line feed after it into
a line feed, and WebKit starts a new line at U+2028 and U+2029, while
git and the line numbers count line feeds alone, so each one set every
later line of the tree and blame pages a line below its number. CSS
Text makes vertical tab, form feed and next line forced breaks as well,
though no engine honours that yet. The escapes follow gitweb, which
shows control characters the same way.

8 files changed, 179 insertions, 10 deletions

diff --git a/README.txt b/README.txtindex 998c072..ceba98c 100644--- a/README.txt+++ b/README.txt@@ -98,6 +98,9 @@ script added through the css and js options has stable hooks to work with. * Relative ages are time elements with an age-* class and the timestamp in   data-ut, while a date shown in full is a plain time element. Line numbers are   anchors named n followed by the line, while enable-tree-linenumbers is on.+  A character in a file that a browser would start a new line at, though git+  does not, such as a carriage return with no line feed after it, is shown as+  an escape sequence in span.escape. * Colours, fonts and metrics are custom properties on div#cgit, so a theme can   redefine those alone. Every rule in cgit.css starts with div#cgit, apart from   the background and margins it sets on html.cgit-standalone, the class of the
diff --git a/README.txt b/README.txt
index 998c072..ceba98c 100644
--- a/README.txt
+++ b/README.txt
@@ -98,6 +98,9 @@ script added through the css and js options has stable hooks to work with.
 * Relative ages are time elements with an age-* class and the timestamp in
   data-ut, while a date shown in full is a plain time element. Line numbers are
   anchors named n followed by the line, while enable-tree-linenumbers is on.
+  A character in a file that a browser would start a new line at, though git
+  does not, such as a carriage return with no line feed after it, is shown as
+  an escape sequence in span.escape.
 * Colours, fonts and metrics are custom properties on div#cgit, so a theme can
   redefine those alone. Every rule in cgit.css starts with div#cgit, apart from
   the background and margins it sets on html.cgit-standalone, the class of the
diff --git a/assets/cgit.css b/assets/cgit.cssindex 0d2da53..2f53559 100644--- a/assets/cgit.css+++ b/assets/cgit.css@@ -653,6 +653,12 @@ div#cgit table.blob code {   font: inherit; } +div#cgit span.escape {+  color: var(--muted);+  outline: 1px dashed var(--border-mid);+  outline-offset: -1px;+}+ div#cgit table.blob td.linenumbers a, div#cgit table.ssdiff td.lineno a {   color: var(--muted);
diff --git a/assets/cgit.css b/assets/cgit.css
index 0d2da53..2f53559 100644
--- a/assets/cgit.css
+++ b/assets/cgit.css
@@ -653,6 +653,12 @@ div#cgit table.blob code {
   font: inherit;
 }
 
+div#cgit span.escape {
+  color: var(--muted);
+  outline: 1px dashed var(--border-mid);
+  outline-offset: -1px;
+}
+
 div#cgit table.blob td.linenumbers a,
 div#cgit table.ssdiff td.lineno a {
   color: var(--muted);
diff --git a/source/html.c b/source/html.cindex d1e28f8..d13ac5b 100644--- a/source/html.c+++ b/source/html.c@@ -258,6 +258,85 @@ ssize_t html_ntxt(const char *txt, size_t len) 	return left; } +/*+ * git ends a line only at a line feed. A browser also starts one at a carriage+ * return with no line feed after it, since the HTML parser turns that into a+ * line feed, and may at the other characters Unicode makes mandatory breaks.+ * Escaping those keeps one line on the page for every line git counts, so the+ * numbers beside the text stay in step with it.+ */+const char *html_line_break_escape(const char *p, const char *end, size_t *width)+{+	const char *esc;++	switch ((unsigned char)*p) {+	case '\r':+		// The parser reads CR LF as the single line feed git ends the+		// line with.+		if (p + 1 < end && p[1] == '\n')+			return NULL;+		*width = 1;+		return "\\r";+	case '\v':+		*width = 1;+		return "\\v";+	case '\f':+		*width = 1;+		return "\\f";+	case 0xc2:+		if (end - p < 2 || memcmp(p, "\xc2\x85", 2))+			return NULL;+		*width = 2;+		return "\\u0085";+	case 0xe2:+		if (end - p >= 3 && !memcmp(p, "\xe2\x80\xa8", 3))+			esc = "\\u2028";+		else if (end - p >= 3 && !memcmp(p, "\xe2\x80\xa9", 3))+			esc = "\\u2029";+		else+			return NULL;+		*width = 3;+		return esc;+	default:+		return NULL;+	}+}++void html_blob_txt(const char *txt, size_t len)+{+	const char *end = txt + len;+	const char *p;++	for (p = txt; p < end; p++) {+		const char *entity = NULL, *esc = NULL;+		size_t width = 1;++		if (*p == '<')+			entity = "&lt;";+		else if (*p == '>')+			entity = "&gt;";+		else if (*p == '&')+			entity = "&amp;";+		else+			esc = html_line_break_escape(p, end, &width);+		if (!entity && !esc)+			continue;+		html_raw(txt, p - txt);+		if (entity) {+			html(entity);+		} else {+			// The span lets a stylesheet tell the escape from the+			// same characters typed into the file.+			html("<span class='escape'>");+			html(esc);+			html("</span>");+		}+		p += width - 1;+		txt = p + 1;+	}+	html_raw(txt, end - txt);+}+ void html_attrf(const char *format, ...) { 	va_list args;
diff --git a/source/html.c b/source/html.c
index d1e28f8..d13ac5b 100644
--- a/source/html.c
+++ b/source/html.c
@@ -258,6 +258,85 @@ ssize_t html_ntxt(const char *txt, size_t len)
 	return left;
 }
 
+/*
+ * git ends a line only at a line feed. A browser also starts one at a carriage
+ * return with no line feed after it, since the HTML parser turns that into a
+ * line feed, and may at the other characters Unicode makes mandatory breaks.
+ * Escaping those keeps one line on the page for every line git counts, so the
+ * numbers beside the text stay in step with it.
+ */
+const char *html_line_break_escape(const char *p, const char *end, size_t *width)
+{
+	const char *esc;
+
+	switch ((unsigned char)*p) {
+	case '\r':
+		// The parser reads CR LF as the single line feed git ends the
+		// line with.
+		if (p + 1 < end && p[1] == '\n')
+			return NULL;
+		*width = 1;
+		return "\\r";
+	case '\v':
+		*width = 1;
+		return "\\v";
+	case '\f':
+		*width = 1;
+		return "\\f";
+	case 0xc2:
+		if (end - p < 2 || memcmp(p, "\xc2\x85", 2))
+			return NULL;
+		*width = 2;
+		return "\\u0085";
+	case 0xe2:
+		if (end - p >= 3 && !memcmp(p, "\xe2\x80\xa8", 3))
+			esc = "\\u2028";
+		else if (end - p >= 3 && !memcmp(p, "\xe2\x80\xa9", 3))
+			esc = "\\u2029";
+		else
+			return NULL;
+		*width = 3;
+		return esc;
+	default:
+		return NULL;
+	}
+}
+
+void html_blob_txt(const char *txt, size_t len)
+{
+	const char *end = txt + len;
+	const char *p;
+
+	for (p = txt; p < end; p++) {
+		const char *entity = NULL, *esc = NULL;
+		size_t width = 1;
+
+		if (*p == '<')
+			entity = "&lt;";
+		else if (*p == '>')
+			entity = "&gt;";
+		else if (*p == '&')
+			entity = "&amp;";
+		else
+			esc = html_line_break_escape(p, end, &width);
+		if (!entity && !esc)
+			continue;
+		html_raw(txt, p - txt);
+		if (entity) {
+			html(entity);
+		} else {
+			// The span lets a stylesheet tell the escape from the
+			// same characters typed into the file.
+			html("<span class='escape'>");
+			html(esc);
+			html("</span>");
+		}
+		p += width - 1;
+		txt = p + 1;
+	}
+	html_raw(txt, end - txt);
+}
+
 void html_attrf(const char *format, ...)
 {
 	va_list args;
diff --git a/source/html.h b/source/html.hindex c6c0e19..9cad3a6 100644--- a/source/html.h+++ b/source/html.h@@ -61,6 +61,19 @@ extern void html_txt(const char *txt);  */ extern ssize_t html_ntxt(const char *txt, size_t len); +/*+ * The escape sequence a blob's text shows for the character at p when a+ * browser would start a new line at it though git does not, or NULL for any+ * other character. The character's length in bytes is stored in width.+ */+extern const char *html_line_break_escape(const char *p, const char *end, size_t *width);++/*+ * Escape the text of a blob for the page beside its line numbers, writing+ * each character html_line_break_escape names as its escape sequence.+ */+extern void html_blob_txt(const char *txt, size_t len);+ extern void html_attr(const char *txt); extern void html_url_path(const char *txt); 
diff --git a/source/html.h b/source/html.h
index c6c0e19..9cad3a6 100644
--- a/source/html.h
+++ b/source/html.h
@@ -61,6 +61,19 @@ extern void html_txt(const char *txt);
  */
 extern ssize_t html_ntxt(const char *txt, size_t len);
 
+/*
+ * The escape sequence a blob's text shows for the character at p when a
+ * browser would start a new line at it though git does not, or NULL for any
+ * other character. The character's length in bytes is stored in width.
+ */
+extern const char *html_line_break_escape(const char *p, const char *end, size_t *width);
+
+/*
+ * Escape the text of a blob for the page beside its line numbers, writing
+ * each character html_line_break_escape names as its escape sequence.
+ */
+extern void html_blob_txt(const char *txt, size_t len);
+
 extern void html_attr(const char *txt);
 extern void html_url_path(const char *txt);
 
diff --git a/source/ui-blame.c b/source/ui-blame.cindex 8fd6c83..f60020c 100644--- a/source/ui-blame.c+++ b/source/ui-blame.c@@ -211,6 +211,15 @@ static size_t line_width(struct blame_scoreboard *sb, int line) 	size_t width = 0;  	while (pos < end) {+		size_t len;+		const char *esc = html_line_break_escape(pos, end, &len);++		// An escaped character takes the room of its escape sequence.+		if (esc) {+			width += strlen(esc);+			pos += len;+			continue;+		} 		width++; 		if (*pos++ == '\t') 			width = (width + TAB_WIDTH - 1) & ~(TAB_WIDTH - 1);@@ -297,9 +306,10 @@ static void print_blame_page(const struct object_id *oid, const char *path, 	cgit_tree_link("tree", NULL, NULL, ctx.qry.head, rev, path); 	html(")</div>\n"); -	// A NUL past the window buffer_is_binary sniffs would end html_txt-	// early while the hash and line number columns still cover the whole-	// file, so a blob holding one anywhere is treated as binary too.+	// A NUL past the window buffer_is_binary sniffs would end a Lua+	// filter's output early while the hash and line number columns still+	// cover the whole file, so a blob holding one anywhere is treated as+	// binary too. 	if (buffer_is_binary(buf, size) || memchr(buf, 0, size)) { 		struct blame_entry *ent = sb.ent, *next; @@ -334,7 +344,7 @@ static void print_blame_page(const struct object_id *oid, const char *path, 		cgit_close_filter(ctx.repo->source_filter); 		free(filter_arg); 	} else {-		html_txt(buf);+		html_blob_txt(buf, size); 	} 	html("</code></pre>"); 
diff --git a/source/ui-blame.c b/source/ui-blame.c
index 8fd6c83..f60020c 100644
--- a/source/ui-blame.c
+++ b/source/ui-blame.c
@@ -211,6 +211,15 @@ static size_t line_width(struct blame_scoreboard *sb, int line)
 	size_t width = 0;
 
 	while (pos < end) {
+		size_t len;
+		const char *esc = html_line_break_escape(pos, end, &len);
+
+		// An escaped character takes the room of its escape sequence.
+		if (esc) {
+			width += strlen(esc);
+			pos += len;
+			continue;
+		}
 		width++;
 		if (*pos++ == '\t')
 			width = (width + TAB_WIDTH - 1) & ~(TAB_WIDTH - 1);
@@ -297,9 +306,10 @@ static void print_blame_page(const struct object_id *oid, const char *path,
 	cgit_tree_link("tree", NULL, NULL, ctx.qry.head, rev, path);
 	html(")</div>\n");
 
-	// A NUL past the window buffer_is_binary sniffs would end html_txt
-	// early while the hash and line number columns still cover the whole
-	// file, so a blob holding one anywhere is treated as binary too.
+	// A NUL past the window buffer_is_binary sniffs would end a Lua
+	// filter's output early while the hash and line number columns still
+	// cover the whole file, so a blob holding one anywhere is treated as
+	// binary too.
 	if (buffer_is_binary(buf, size) || memchr(buf, 0, size)) {
 		struct blame_entry *ent = sb.ent, *next;
 
@@ -334,7 +344,7 @@ static void print_blame_page(const struct object_id *oid, const char *path,
 		cgit_close_filter(ctx.repo->source_filter);
 		free(filter_arg);
 	} else {
-		html_txt(buf);
+		html_blob_txt(buf, size);
 	}
 	html("</code></pre>");
 
diff --git a/source/ui-tree.c b/source/ui-tree.cindex 13a9787..1d8bb3d 100644--- a/source/ui-tree.c+++ b/source/ui-tree.c@@ -113,7 +113,7 @@ static void print_text_buffer(const char *filename, char *buf, size_t size) 	}  	html("<td class='lines'><pre><code>");-	html_txt(buf);+	html_blob_txt(buf, size); 	html("</code></pre></td></tr></table>\n"); } @@ -188,7 +188,7 @@ static bool print_object(const struct object_id *oid, const char *path, 		return false; 	} 	// buffer_is_binary only sniffs the front of the blob, and a NUL past-	// that window would end the escaped text render early while the line+	// that window would end a Lua filter's output early while the line 	// number column still counts the whole file, so check the rest too. 	is_binary = buffer_is_binary(buf, size) || memchr(buf, 0, size); 
diff --git a/source/ui-tree.c b/source/ui-tree.c
index 13a9787..1d8bb3d 100644
--- a/source/ui-tree.c
+++ b/source/ui-tree.c
@@ -113,7 +113,7 @@ static void print_text_buffer(const char *filename, char *buf, size_t size)
 	}
 
 	html("<td class='lines'><pre><code>");
-	html_txt(buf);
+	html_blob_txt(buf, size);
 	html("</code></pre></td></tr></table>\n");
 }
 
@@ -188,7 +188,7 @@ static bool print_object(const struct object_id *oid, const char *path,
 		return false;
 	}
 	// buffer_is_binary only sniffs the front of the blob, and a NUL past
-	// that window would end the escaped text render early while the line
+	// that window would end a Lua filter's output early while the line
 	// number column still counts the whole file, so check the rest too.
 	is_binary = buffer_is_binary(buf, size) || memchr(buf, 0, size);
 
diff --git a/tests/t0005-markup.sh b/tests/t0005-markup.shindex e3cb08d..829c839 100755--- a/tests/t0005-markup.sh+++ b/tests/t0005-markup.sh@@ -96,15 +96,17 @@ test_expect_success 'the stylesheet and the script name only the current hooks' 	grep "window.cgit.highlightLines" "$TEST_OUTPUT_DIRECTORY/../assets/cgit.js" ' -# The rest of the hooks need a tag, a symlink, a submodule, a trailer, clone-# urls, an empty repository and a second page of repositories, so they get a-# commit and a config of their own after the checks above have read the first-# ones.+# The rest of the hooks need a tag, a symlink, a submodule, a trailer, a lone+# carriage return, clone urls, an empty repository and a second page of+# repositories, so they get a commit and a config of their own after the checks+# above have read the first ones. test_expect_success 'set up the remaining kinds of entry' ' 	( 		cd repos/hooks && 		git tag -a -m "first release" v1 && 		test_ln_s_add file-1 link &&+		printf "a\\rb\\n" >cr.txt &&+		git add cr.txt && 		git update-index --add --cacheinfo "160000,$(git rev-parse HEAD),sub" && 		git commit -m "more kinds of entry" -m "Fixes: $(git rev-parse --short HEAD)" 	) &&@@ -139,6 +141,11 @@ test_expect_success 'the refs, summary and tree name every section and row' ' 	grep "<tr class=.mod.>" tmp ' +test_expect_success 'a character the browser would break a line at is an escape' '+	more "hooks/tree/cr.txt" >tmp &&+	grep "a<span class=.escape.>" tmp+'+ test_expect_success 'the commit page has its trailer table and diffstat' ' 	more "hooks/commit/" >tmp && 	grep "<table class=.commit-trailers.>" tmp &&
diff --git a/tests/t0005-markup.sh b/tests/t0005-markup.sh
index e3cb08d..829c839 100755
--- a/tests/t0005-markup.sh
+++ b/tests/t0005-markup.sh
@@ -96,15 +96,17 @@ test_expect_success 'the stylesheet and the script name only the current hooks'
 	grep "window.cgit.highlightLines" "$TEST_OUTPUT_DIRECTORY/../assets/cgit.js"
 '
 
-# The rest of the hooks need a tag, a symlink, a submodule, a trailer, clone
-# urls, an empty repository and a second page of repositories, so they get a
-# commit and a config of their own after the checks above have read the first
-# ones.
+# The rest of the hooks need a tag, a symlink, a submodule, a trailer, a lone
+# carriage return, clone urls, an empty repository and a second page of
+# repositories, so they get a commit and a config of their own after the checks
+# above have read the first ones.
 test_expect_success 'set up the remaining kinds of entry' '
 	(
 		cd repos/hooks &&
 		git tag -a -m "first release" v1 &&
 		test_ln_s_add file-1 link &&
+		printf "a\\rb\\n" >cr.txt &&
+		git add cr.txt &&
 		git update-index --add --cacheinfo "160000,$(git rev-parse HEAD),sub" &&
 		git commit -m "more kinds of entry" -m "Fixes: $(git rev-parse --short HEAD)"
 	) &&
@@ -139,6 +141,11 @@ test_expect_success 'the refs, summary and tree name every section and row' '
 	grep "<tr class=.mod.>" tmp
 '
 
+test_expect_success 'a character the browser would break a line at is an escape' '
+	more "hooks/tree/cr.txt" >tmp &&
+	grep "a<span class=.escape.>" tmp
+'
+
 test_expect_success 'the commit page has its trailer table and diffstat' '
 	more "hooks/commit/" >tmp &&
 	grep "<table class=.commit-trailers.>" tmp &&
diff --git a/tests/t0104-tree.sh b/tests/t0104-tree.shindex fefff4a..d464f8c 100755--- a/tests/t0104-tree.sh+++ b/tests/t0104-tree.sh@@ -31,6 +31,57 @@ test_expect_success 'the code is set in the font of its line numbers' ' 	grep "^  font: inherit;$" rule ' +# A browser starts a new line at more characters than git ends one at, and+# each would push the rest of the code a line below its numbers.+test_expect_success 'set up a file holding the characters a browser breaks at' '+	test_create_repo repos/breaks &&+	(+		cd repos/breaks &&+		printf "a\\rb\\vc\\fd\\302\\205e\\342\\200\\250f\\342\\200\\251g\\nh\\r\\ni\\r" >breaks.txt &&+		git add breaks.txt &&+		git commit -m breaks+	) &&+	{+		echo "virtual-root=/" &&+		echo "cache-size=0" &&+		echo "enable-blame=1" &&+		echo "repo.url=breaks" &&+		echo "repo.path=$PWD/repos/breaks/.git"+	} >breaksrc+'++test_expect_success 'each of them is shown as an escape in the tree' '+	CGIT_CONFIG="$PWD/breaksrc" QUERY_STRING="url=breaks/tree/breaks.txt" cgit >tmp &&+	grep "<code>a<span class=.escape.>\\\\r</span>b<span class=.escape.>\\\\v</span>c" tmp &&+	grep "c<span class=.escape.>\\\\f</span>d<span class=.escape.>\\\\u0085</span>e" tmp &&+	grep "e<span class=.escape.>\\\\u2028</span>f<span class=.escape.>\\\\u2029</span>g$" tmp+'++test_expect_success 'only a carriage return before a line feed is left as it is' '+	printf "h\\r\\n" >expect &&+	grep "^h" tmp >actual &&+	test_cmp expect actual &&+	grep "^i<span class=.escape.>\\\\r</span></code>" tmp+'++test_expect_success 'the numbers count only the lines git counts' '+	grep "<a id=.n3. href=.#n3.>3</a>" tmp &&+	! grep "<a id=.n4. href=.#n4.>4</a>" tmp+'++test_expect_success 'the blame page escapes them the same way' '+	CGIT_CONFIG="$PWD/breaksrc" QUERY_STRING="url=breaks/blame/breaks.txt" cgit >tmp &&+	grep "<code>a<span class=.escape.>\\\\r</span>b<span class=.escape.>\\\\v</span>c" tmp &&+	grep "e<span class=.escape.>\\\\u2028</span>f<span class=.escape.>\\\\u2029</span>g$" tmp &&+	grep "^i<span class=.escape.>\\\\r</span></code>" tmp+'++# The striped background is padded with spaces to the widest line, which is+# 31 columns once its six escapes are written out, though it is 18 bytes.+test_expect_success 'the blame stripes are as wide as the escaped line' '+	grep "^ \\{31\\}</pre></div>" tmp+'+ test_expect_success 'generate foo+bar/tree' 'cgit_url "foo%2bbar/tree" >tmp'  test_expect_success 'verify a+b link' '
diff --git a/tests/t0104-tree.sh b/tests/t0104-tree.sh
index fefff4a..d464f8c 100755
--- a/tests/t0104-tree.sh
+++ b/tests/t0104-tree.sh
@@ -31,6 +31,57 @@ test_expect_success 'the code is set in the font of its line numbers' '
 	grep "^  font: inherit;$" rule
 '
 
+# A browser starts a new line at more characters than git ends one at, and
+# each would push the rest of the code a line below its numbers.
+test_expect_success 'set up a file holding the characters a browser breaks at' '
+	test_create_repo repos/breaks &&
+	(
+		cd repos/breaks &&
+		printf "a\\rb\\vc\\fd\\302\\205e\\342\\200\\250f\\342\\200\\251g\\nh\\r\\ni\\r" >breaks.txt &&
+		git add breaks.txt &&
+		git commit -m breaks
+	) &&
+	{
+		echo "virtual-root=/" &&
+		echo "cache-size=0" &&
+		echo "enable-blame=1" &&
+		echo "repo.url=breaks" &&
+		echo "repo.path=$PWD/repos/breaks/.git"
+	} >breaksrc
+'
+
+test_expect_success 'each of them is shown as an escape in the tree' '
+	CGIT_CONFIG="$PWD/breaksrc" QUERY_STRING="url=breaks/tree/breaks.txt" cgit >tmp &&
+	grep "<code>a<span class=.escape.>\\\\r</span>b<span class=.escape.>\\\\v</span>c" tmp &&
+	grep "c<span class=.escape.>\\\\f</span>d<span class=.escape.>\\\\u0085</span>e" tmp &&
+	grep "e<span class=.escape.>\\\\u2028</span>f<span class=.escape.>\\\\u2029</span>g$" tmp
+'
+
+test_expect_success 'only a carriage return before a line feed is left as it is' '
+	printf "h\\r\\n" >expect &&
+	grep "^h" tmp >actual &&
+	test_cmp expect actual &&
+	grep "^i<span class=.escape.>\\\\r</span></code>" tmp
+'
+
+test_expect_success 'the numbers count only the lines git counts' '
+	grep "<a id=.n3. href=.#n3.>3</a>" tmp &&
+	! grep "<a id=.n4. href=.#n4.>4</a>" tmp
+'
+
+test_expect_success 'the blame page escapes them the same way' '
+	CGIT_CONFIG="$PWD/breaksrc" QUERY_STRING="url=breaks/blame/breaks.txt" cgit >tmp &&
+	grep "<code>a<span class=.escape.>\\\\r</span>b<span class=.escape.>\\\\v</span>c" tmp &&
+	grep "e<span class=.escape.>\\\\u2028</span>f<span class=.escape.>\\\\u2029</span>g$" tmp &&
+	grep "^i<span class=.escape.>\\\\r</span></code>" tmp
+'
+
+# The striped background is padded with spaces to the widest line, which is
+# 31 columns once its six escapes are written out, though it is 18 bytes.
+test_expect_success 'the blame stripes are as wide as the escaped line' '
+	grep "^ \\{31\\}</pre></div>" tmp
+'
+
 test_expect_success 'generate foo+bar/tree' 'cgit_url "foo%2bbar/tree" >tmp'
 
 test_expect_success 'verify a+b link' '