index : cgit

#!/bin/sh

# Exercises the cache. The first three tests set cache-size to nothing, one
# slot and the full table in turn, then count what the requests left behind.

test_description='Validate cache'
. ./setup.sh

# The counting tests below visit the same pages, the index plus six pages in
# each of two repositories, so the counts they expect stay tied to this one
# list of thirteen requests.
request_all_pages() {
	cgit_ok "$1" "url=" >/dev/null &&
	for repo in foo bar
	do
		for page in "" /refs /tree /log /diff /patch
		do
			cgit_ok "$1" "url=$repo$page" >/dev/null || return 1
		done
	done
}

test_expect_success 'a cache-size of 0 leaves no slot behind' '
	rm -f cache/* &&
	sed "s/^cache-size=.*/cache-size=0/" cgitrc >size0rc &&
	request_all_pages size0rc &&
	test_dir_is_empty cache
'

# With one slot every request lands on the same file, so the key it holds is
# all that keeps one page from being served for another.
test_expect_success 'one slot serves each request its own page' '
	rm -f cache/* &&
	sed "s/^cache-size=.*/cache-size=1/" cgitrc >size1rc &&
	cgit_ok size1rc "url=foo/log/" >/dev/null &&
	cgit_rc size1rc "url=bar/refs/" >refs.out 2>refs.err &&
	test_grep "^Status: 200" refs.out &&
	test_grep "data-page=.refs. data-repo=.bar." refs.out &&
	test_grep "<table class=.list refs." refs.out &&
	test_grep ! "data-repo=.foo." refs.out &&
	test_grep "holds a different key, consider a larger cache-size" refs.err &&
	cgit_rc size1rc "url=foo/log/" >log.out 2>log.err &&
	test_grep "data-page=.log. data-repo=.foo." log.out &&
	test_grep "<table class=.list log." log.out &&
	test_grep ! "data-repo=.bar." log.out &&
	test_grep "holds a different key, consider a larger cache-size" log.err &&
	ls cache >output &&
	test_line_count = 1 output
'

# The listing is a page like any other and takes a slot of its own, so a
# fresh render of it would list fourteen slots. Thirteen on the second
# request shows the first was replayed, which a dynamic ttl that never
# expires keeps true however slow the run.
test_expect_success 'a full table gives each request a slot of its own' '
	rm -f cache/* &&
	{
		echo "cache-dynamic-ttl=-1" &&
		sed "s/^cache-size=.*/cache-size=1021/" cgitrc
	} >size1021rc &&
	request_all_pages size1021rc &&
	ls cache >output &&
	test_line_count = 13 output &&
	cgit_ok size1021rc "url=foo/ls_cache" >output.full &&
	strip_headers <output.full >output &&
	test_line_count = 13 output &&
	ls cache >output &&
	test_line_count = 14 output &&
	cgit_ok size1021rc "url=foo/ls_cache" >output.second &&
	test_cmp output.full output.second
'

# Page output is buffered, so a slot only holds the whole page if the buffer is
# emptied before the size of what was generated is taken. The blob set up here
# is larger than one buffer, so a missing flush would leave the cached copy cut
# short.
test_expect_success 'set up a repo with a page larger than the output buffer' '
	mkrepo repos/bigpage 1 &&
	(
		cd repos/bigpage &&
		awk "BEGIN{for(i=0;i<20000;i++) print \"line \" i \" of the big file\"}" >big.txt &&
		git add big.txt &&
		git commit -m "add big.txt"
	) &&
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache2" &&
		echo "cache-size=64" &&
		echo "enable-tree-linenumbers=1" &&
		echo "repo.url=bigpage" &&
		echo "repo.path=$PWD/repos/bigpage/.git"
	} >bigrc &&
	rm -rf cache2 && mkdir cache2
'

test_expect_success 'the first request fills the slot and the second replays it' '
	cgit_rc bigrc "url=bigpage/tree/big.txt" >big.first &&
	test $(wc -c <big.first) -gt 65536 &&
	cgit_rc bigrc "url=bigpage/tree/big.txt" >big.second &&
	strip_headers <big.first >big.first.body &&
	strip_headers <big.second >big.second.body &&
	test_cmp big.first.body big.second.body
'

test_expect_success 'the cached body matches one generated with the cache off' '
	sed -e "s/^cache-size=64$/cache-size=0/" bigrc >bignocache &&
	cgit_rc bignocache "url=bigpage/tree/big.txt" >big.nocache &&
	strip_headers <big.nocache >big.nocache.body &&
	test_cmp big.nocache.body big.second.body
'

test_expect_success 'the page ends where it should, so nothing was dropped' '
	tail -c 200 big.second.body | grep "</html>"
'

# A slot stores its key ahead of the content and only the first few kilobytes of
# it are read back, so a key longer than that could never match. A request
# carrying one used to leave a slot behind on every visit that nothing could
# ever go on to use.
test_expect_success 'an over-long cache key leaves no slot behind' '
	rm -rf cache3 && mkdir cache3 &&
	sed -e "s|^cache-root=.*|cache-root=$PWD/cache3|" bigrc >bigkeyrc &&
	long=$(awk "BEGIN{s=\"\";for(i=0;i<6000;i++)s=s \"k\"; print s}") &&
	cgit_rc bigkeyrc "url=bigpage/&q=$long" >key.out 2>/dev/null &&
	grep "</html>" key.out &&
	test_dir_is_empty cache3
'

test_expect_success 'an ordinary key still fills a slot' '
	rm -rf cache3 && mkdir cache3 &&
	cgit_rc bigkeyrc "url=bigpage/" >/dev/null &&
	ls cache3 >key.slots &&
	test_line_count = 1 key.slots
'

# An error page fired after rendering began abandons the fill with part of
# the page already written into the lock file. That fragment must be
# discarded along with the file, not replayed to the visitor behind the
# error page. A commit whose parent object is missing renders its whole
# info table before the diff machinery fails, which makes it the trigger.
test_expect_success 'set up a repo missing a parent object' '
	mkrepo repos/broken 2 &&
	parent=$(git -C repos/broken rev-parse HEAD^) &&
	tip=$(git -C repos/broken rev-parse HEAD) &&
	rm "repos/broken/.git/objects/$(echo "$parent" | cut -c1-2)/$(echo "$parent" | cut -c3-)" &&
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache4" &&
		echo "cache-size=64" &&
		echo "repo.url=broken" &&
		echo "repo.path=$PWD/repos/broken/.git"
	} >brokenrc &&
	rm -rf cache4 && mkdir cache4
'

test_expect_success 'an error after output began ends the page with it and caches nothing' '
	cgit_rc brokenrc "url=broken/commit/&id=$tip" >broken.out &&
	grep "Not a commit" broken.out &&
	test $(grep -c "^Status:" broken.out) = 1 &&
	test_dir_is_empty cache4
'

# An error page reports a condition the repository may grow out of, and a
# request pinned to an object id would cache it under the never-expiring
# static ttl. Asking for a commit that does not exist yet must therefore
# leave nothing behind, or the real page could never appear once it does.
test_expect_success 'an error page leaves no slot behind' '
	rm -rf cache3 && mkdir cache3 &&
	CGIT_CONFIG="$PWD/bigkeyrc" \
	QUERY_STRING="url=bigpage/commit/&id=0123456789abcdef0123456789abcdef01234567" cgit >error.out &&
	grep "Not a commit" error.out &&
	test_dir_is_empty cache3
'

# The dumb transport serves files that already sit on the disk, so a copy in
# a slot would only double the disk they take.
test_expect_success 'a clone file leaves no slot behind' '
	rm -rf cache3 && mkdir cache3 &&
	cgit_rc bigkeyrc "url=bigpage/info/refs" >clone.out &&
	grep "refs/heads/master" clone.out &&
	cgit_rc bigkeyrc "url=bigpage/objects/info/packs" >packs.out &&
	grep "^Status: 200" packs.out &&
	test_dir_is_empty cache3
'

# A response past cache-max-slot-size is served whole but kept nowhere, both
# on a first fill and when it would have replaced an expired slot, which is
# dropped along with it. The summary page stays under the limit set here
# while the big blob page is well past it.
test_expect_success 'a response over cache-max-slot-size is served but not kept' '
	rm -rf cache3 && mkdir cache3 &&
	{
		echo "cache-max-slot-size=32" &&
		cat bigkeyrc
	} >slotsizerc &&
	cgit_rc slotsizerc "url=bigpage/tree/big.txt" >big.unkept &&
	tail -c 200 big.unkept | grep "</html>" &&
	test_dir_is_empty cache3 &&
	cgit_rc slotsizerc "url=bigpage/" >/dev/null &&
	ls cache3 >kept.slots &&
	test_line_count = 1 kept.slots
'

test_expect_success 'an expired slot is dropped when its refill is too large' '
	rm -rf cache3 && mkdir cache3 &&
	cgit_rc bigkeyrc "url=bigpage/tree/big.txt" >/dev/null &&
	ls cache3 >filled.slots &&
	test_line_count = 1 filled.slots &&
	touch -t 200001010000 cache3/* &&
	cgit_rc slotsizerc "url=bigpage/tree/big.txt" >big.refill &&
	tail -c 200 big.refill | grep "</html>" &&
	test_dir_is_empty cache3
'

# The summary page keeps a ttl of its own, while every other page that follows
# a branch rather than a pinned id takes cache-dynamic-ttl, whether or not its
# url names the branch.
test_expect_success 'pages following a branch take the dynamic ttl' '
	rm -rf cache3 && mkdir cache3 &&
	{
		echo "cache-dynamic-ttl=0" &&
		cat bigkeyrc
	} >dynamicrc &&
	cgit_rc dynamicrc "url=bigpage/log/" >/dev/null &&
	cgit_rc dynamicrc "url=bigpage/log/&h=master" >/dev/null &&
	test_dir_is_empty cache3 &&
	cgit_rc dynamicrc "url=bigpage/" >/dev/null &&
	cgit_rc dynamicrc "url=bigpage/summary/" >/dev/null &&
	ls cache3 >summary.slots &&
	test_line_count = 2 summary.slots
'

# A write into the lock file that fails on the page's last flush has to end
# as one that fails mid page does, with the fill abandoned and the page
# delivered from the lock file, rather than in a blank response. The limit
# is on files alone, so the page goes to the visitor through a pipe.
test_expect_success 'a failed final write still delivers the page' '
	rm -rf cache3 && mkdir cache3 &&
	(
		trap "" XFSZ &&
		ulimit -f 2 &&
		cgit_rc bigkeyrc "url=bigpage/" 2>flush.err
	) | cat >flush.out &&
	grep "^Status: 200" flush.out &&
	grep "</html>" flush.out &&
	! grep "called many times" flush.err
'

# The second request lands while the first is inside its filter, which says
# so through a marker file, since a fresh binary can take a moment to start.
start_first_request() {
	n=0
	rm -f filter-running
	cgit_rc lockrc "url=bigpage/about/" >first.out 2>first.err &
	while ! test -f filter-running && test $n -lt 10
	do
		sleep 1
		n=$((n + 1))
	done
}

overlap_requests() {
	start_first_request
	cgit_rc lockrc "url=bigpage/log/" >/dev/null 2>second.err
	wait
}

# The fill writes the page into the lock file through stdout, and an exec
# filter puts its own pipe on stdout while it runs. The slot lock has to
# outlast that, or a second request on the slot rebuilds it under the first
# and the first visitor gets the two pages spliced together. The second one
# finding the lock held is ordinary contention, which logs nothing.
test_expect_success 'a slot stays locked while an exec filter runs' '
	rm -rf cache3 && mkdir cache3 &&
	write_script slow-filter.sh <<-EOF &&
	touch "$PWD/filter-running"
	sleep 3
	cat
	EOF
	{
		echo "cache-size=1" &&
		sed "/^cache-size=/d" bigkeyrc &&
		echo "repo.readme=:file-1" &&
		echo "repo.about-filter=exec:$PWD/slow-filter.sh"
	} >lockrc &&
	overlap_requests &&
	test $(grep -c "^Status:" first.out) = 1 &&
	grep "id=.about." first.out &&
	! grep "Unable to publish" first.err &&
	! grep "Unable to lock" second.err
'

# A request that opens the lock file just before its holder renames it into
# place as the live slot goes on to lock the slot itself. strace holds the
# second request's flock back until the first has finished, and the check
# that the path still names the locked file turns that lock down, so the
# slot keeps the first request's key. strace needs ptrace and a version that
# can delay a call, which some systems do not give.
test_lazy_prereq STRACE_DELAY '
	strace -o /dev/null -e inject=flock:delay_enter=1 true
'

test_expect_success STRACE_DELAY 'a lock taken after the rename leaves the slot alone' '
	rm -rf cache3 && mkdir cache3 &&
	start_first_request &&
	strace -o /dev/null -e inject=flock:delay_enter=4000000 \
		-E CGIT_CONFIG="$PWD/lockrc" -E QUERY_STRING="url=bigpage/log/" \
		cgit >second.out 2>second.err &&
	wait &&
	grep "id=.about." first.out &&
	grep "</html>" second.out &&
	tr "\000" "\n" <"cache3/$(ls cache3)" | head -n 1 >slot.key &&
	grep "about" slot.key &&
	! grep "log" slot.key
'

# An flock lasts until the last copy of its descriptor closes, so a filter
# may hold the slot's lock file only as the stdout it writes the page to. A
# stray copy would keep the slot locked for as long as the filter, or
# anything it leaves running, lives on.
test_expect_success PERL 'a filter holds the lock file only as its stdout' '
	rm -rf cache3 && mkdir cache3 &&
	write_script fd-filter "$PERL_PATH" <<-\EOF &&
	use POSIX qw(fstat);
	1 while <STDIN>;
	my @lock = stat $ENV{LOCK_FILE} or die "$!\n";
	my @fds = grep {
		my @fd = fstat($_);
		@fd && $fd[0] == $lock[0] && $fd[1] == $lock[1]
	} 0 .. 255;
	print "lock file on fd @fds.\n";
	EOF
	{
		echo "cache-size=1" &&
		sed "/^cache-size=/d" bigkeyrc &&
		echo "repo.readme=:file-1" &&
		echo "repo.about-filter=exec:$PWD/fd-filter"
	} >fdrc &&
	LOCK_FILE="$PWD/cache3/00000000.lock" CGIT_CONFIG="$PWD/fdrc" \
	QUERY_STRING="url=bigpage/about/" cgit >fd.out &&
	grep -F "lock file on fd 1." fd.out
'

# A refresh that cannot take the lock serves the expired copy, and says why
# unless the reason is another request doing the refresh. A line appended
# to the copy shows that it is what was served.
test_expect_success SANITY 'a refresh that cannot lock serves the expired slot and says why' '
	rm -rf cache3 && mkdir cache3 &&
	cgit_rc bigkeyrc "url=bigpage/" >/dev/null &&
	slot=$(ls cache3) &&
	echo "the expired copy" >>"cache3/$slot" &&
	touch -t 200001010000 "cache3/$slot" &&
	test_when_finished "chmod u+w cache3" &&
	chmod a-w cache3 &&
	cgit_rc bigkeyrc "url=bigpage/" >stale.out 2>stale.err &&
	grep "the expired copy" stale.out &&
	grep "Unable to open .*\.lock: Permission denied" stale.err
'

# A lock file that cannot take the key goes with its lock, rather than
# staying behind locked while the page renders uncached. The limit is on
# files, so the log goes through the pipe as well.
test_expect_success 'a slot lock that cannot take its key is let go' '
	rm -rf cache3 && mkdir cache3 &&
	(
		trap "" XFSZ &&
		ulimit -f 0 &&
		cgit_rc bigkeyrc "url=bigpage/" 2>&1
	) | cat >nokey.out &&
	grep "</html>" nokey.out &&
	grep "Unable to write .*\.lock" nokey.out &&
	test_dir_is_empty cache3
'

# A scan is cached in a file named after a hash of its path and settings, so
# a first scan finds out what that name is. The index page is left uncached,
# which makes each request read the list afresh.
test_expect_success 'set up a cached scan of two repositories' '
	mkrepo scanned/one 1 &&
	mkrepo scanned/two 1 &&
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache5" &&
		echo "cache-size=64" &&
		echo "cache-index-ttl=0" &&
		echo "scan-path=$PWD/scanned"
	} >scanrc &&
	rm -rf cache5 && mkdir cache5 &&
	cgit_rc scanrc "url=" >/dev/null &&
	scanned_rc=$(ls cache5) &&
	test_path_is_file "cache5/$scanned_rc"
'

# A request that finds the scan's lock held and has no list to go on scans
# in its own process, leaving the file to the holder. That is only another
# request at work and no cause for a line in the log.
test_expect_success PERL 'a scan finding its lock held runs uncached' '
	write_script hold-lock "$PERL_PATH" <<-\EOF &&
	use Fcntl qw(:flock);
	open(my $lock, ">>", shift) or die "$!\n";
	flock($lock, LOCK_EX | LOCK_NB) or die "$!\n";
	exit(system(@ARGV) ? 1 : 0);
	EOF
	rm "cache5/$scanned_rc" &&
	CGIT_CONFIG="$PWD/scanrc" QUERY_STRING="url=" \
	./hold-lock "cache5/$scanned_rc.lock" cgit >held.out 2>held.err &&
	grep ">one</a>" held.out &&
	grep ">two</a>" held.out &&
	test_path_is_missing "cache5/$scanned_rc" &&
	test_must_be_empty held.err
'

# A run that died before its rename leaves a lock file behind, here longer
# than anything the next run writes and ending in a repository and a marker.
# Emptying the file before reuse keeps that tail out of both caches.
test_expect_success 'a stale lock file reaches neither the scan nor a slot' '
	{
		printf "%01024d\n" 0 0 0 0 &&
		echo "repo.url=ghost" &&
		echo "repo.path=$PWD/scanned/one/.git"
	} >stale.lock &&
	rm -f "cache5/$scanned_rc" &&
	cp stale.lock "cache5/$scanned_rc.lock" &&
	CGIT_CONFIG="$PWD/scanrc" QUERY_STRING="url=" cgit >scan.out &&
	grep ">one</a>" scan.out &&
	! grep ">ghost</a>" scan.out &&
	rm -rf cache3 && mkdir cache3 &&
	CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/" cgit >page.out &&
	slot=$(ls cache3) &&
	rm "cache3/$slot" &&
	{ cat page.out page.out && echo "STALE-TAIL"; } >"cache3/$slot.lock" &&
	CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/" cgit >/dev/null &&
	CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/" cgit >replay.out &&
	grep "</html>" replay.out &&
	! grep "STALE-TAIL" replay.out
'

# The cached list is read back as trusted config, so a config that would
# scan differently keeps a list of its own in the same cache-root, here one
# that takes only the repositories carrying an export marker.
test_expect_success 'a config that scans differently keeps its own list' '
	touch scanned/one/.git/export-ok &&
	{
		echo "strict-export=export-ok" &&
		cat scanrc
	} >strictrc &&
	cgit_rc scanrc "url=" >all.out &&
	cgit_rc strictrc "url=" >strict.out &&
	grep ">two</a>" all.out &&
	grep ">one</a>" strict.out &&
	! grep ">two</a>" strict.out
'

# A negative ttl never expires, so a list however old is served and kept. A
# rebuild runs in a child holding the request's copy of fd 9, so cat reaching
# the end of that pipe means any rebuild is done, as the positive ttl after
# it shows.
test_expect_success 'a negative cache-scan-ttl never expires the list' '
	{
		echo "cache-scan-ttl=-1" &&
		cat scanrc
	} >foreverrc &&
	{
		echo "repo.url=kept" &&
		echo "repo.path=$PWD/scanned/one/.git"
	} >>"cache5/$scanned_rc" &&
	touch -t 200001010000 "cache5/$scanned_rc" &&
	cgit_rc foreverrc "url=" 9>&1 >forever.out | cat &&
	grep ">kept</a>" forever.out &&
	grep "^repo.url=kept$" "cache5/$scanned_rc" &&
	cgit_rc scanrc "url=" 9>&1 >/dev/null | cat &&
	! grep "^repo.url=kept$" "cache5/$scanned_rc"
'

# A repository the scan found without a description goes into the list
# without one, so its pages do not describe themselves with the placeholder.
test_expect_success 'a cached scan leaves the description placeholder out' '
	rm -rf cache5 && mkdir cache5 &&
	cgit_rc scanrc "url=" >/dev/null &&
	! grep "^repo.desc=" cache5/rc-* &&
	cgit_rc scanrc "url=" >tmp &&
	grep "\[no description\]" tmp &&
	cgit_rc scanrc "url=one/" >tmp &&
	! grep "<meta name=.description." tmp
'

# The list is read back under the global settings, so what a repository set
# against them has to come back with it. This one sorts its branches by name
# where the site sorts by age, turns the site's about filter off and links a
# submodule of its own. A list scanned without the global filter is no good
# to a config that sets one, as it says nothing of the filter turned off.
test_expect_success 'a cached scan keeps what a repository sets' '
	mkrepo kept/one 1 &&
	sub=$(git -C kept/one rev-parse HEAD) &&
	(
		cd kept/one &&
		git update-index --add --cacheinfo 160000,$sub,submod &&
		git commit -m gitlink &&
		git branch aaa &&
		newest=$(echo newest | GIT_COMMITTER_DATE="@2000000000 +0000" git commit-tree HEAD^{tree} -p HEAD) &&
		git update-ref refs/heads/zzz $newest
	) &&
	{
		echo "branch-sort=name" &&
		echo "about-filter=" &&
		echo "readme=:file-1" &&
		echo "module-link.submod=https://example.org/s/?id=%s"
	} >kept/one/.git/cgitrc &&
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache6" &&
		echo "cache-size=64" &&
		echo "trust-scan-config=1" &&
		echo "branch-sort=age" &&
		echo "scan-path=$PWD/kept"
	} >unfilteredrc &&
	{
		echo "about-filter=exec:$FILTER_DIRECTORY/dump.sh" &&
		cat unfilteredrc
	} >keptrc &&
	rm -rf cache6 && mkdir cache6 &&
	cgit_rc unfilteredrc "url=" >/dev/null &&
	cgit_rc keptrc "url=" >/dev/null &&
	cgit_rc keptrc "url=one/refs/" >tmp &&
	sed -n "s/^<tr class=.branch.><td><a [^>]*>\([a-z]*\)<.*/\1/p" tmp >branches &&
	test_write_lines aaa master zzz >expect &&
	test_cmp expect branches &&
	cgit_rc keptrc "url=one/about/" >tmp &&
	grep "<pre class=.plaintext.>1$" tmp &&
	cgit_rc keptrc "url=one/tree/" >tmp &&
	grep "href=.https://example.org/s/?id=$sub." tmp
'

# A cache-size below zero turns the page cache off, and the scan with it, so
# the cache directory stays empty.
test_expect_success 'a negative cache-size leaves the scan uncached' '
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache-negative" &&
		echo "cache-size=-1" &&
		echo "scan-path=$PWD/scanned"
	} >negativerc &&
	rm -rf cache-negative && mkdir cache-negative &&
	cgit_rc negativerc "url=" >tmp &&
	grep ">one</a>" tmp &&
	test_dir_is_empty cache-negative
'

test_done