index : cgit

#!/bin/sh

# Exercises the cache. The first three tests set cache-size to nothing, one
# slot and the full table in turn, then count what the requests left behind.

test_description='Validate cache'
. ./setup.sh

# The three counting tests below visit the same pages, the index plus six
# pages in each of two repositories, so the counts they expect stay tied to
# this one list of thirteen requests.
request_all_pages() {
	cgit_url "" &&
	for repo in foo bar
	do
		for page in "" /refs /tree /log /diff /patch
		do
			cgit_url "$repo$page" || return 1
		done
	done
}

set_cache_size() {
	sed -e "s/^cache-size=.*$/cache-size=$1/" cgitrc >cgitrc.tmp &&
	mv -f cgitrc.tmp cgitrc
}

test_expect_success 'verify cache-size=0' '
	rm -f cache/* &&
	set_cache_size 0 &&
	request_all_pages &&
	test_dir_is_empty cache
'

test_expect_success 'verify cache-size=1' '
	rm -f cache/* &&
	set_cache_size 1 &&
	request_all_pages &&
	ls cache >output &&
	test_line_count = 1 output
'

test_expect_success 'verify cache-size=1021' '
	rm -f cache/* &&
	set_cache_size 1021 &&
	request_all_pages &&
	ls cache >output &&
	test_line_count = 13 output &&
	cgit_url "foo/ls_cache" >output.full &&
	strip_headers <output.full >output &&
	test_line_count = 13 output &&
	cgit_url "foo/ls_cache" >output.second &&
	test_cmp output.full output.second
'

# Page output is buffered, so a slot only holds the whole page if the buffer is
# emptied before the size of what was generated is taken. The blob set up here
# is larger than one buffer, so a missing flush would leave the cached copy cut
# short.
test_expect_success 'set up a repo with a page larger than the output buffer' '
	mkrepo repos/bigpage 1 &&
	(
		cd repos/bigpage &&
		awk "BEGIN{for(i=0;i<20000;i++) print \"line \" i \" of the big file\"}" >big.txt &&
		git add big.txt &&
		git commit -m "add big.txt"
	) &&
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache2" &&
		echo "cache-size=64" &&
		echo "enable-tree-linenumbers=1" &&
		echo "repo.url=bigpage" &&
		echo "repo.path=$PWD/repos/bigpage/.git"
	} >bigrc &&
	rm -rf cache2 && mkdir cache2
'

test_expect_success 'the first request fills the slot and the second replays it' '
	cgit_rc bigrc "url=bigpage/tree/big.txt" >big.first &&
	test $(wc -c <big.first) -gt 65536 &&
	cgit_rc bigrc "url=bigpage/tree/big.txt" >big.second &&
	strip_headers <big.first >big.first.body &&
	strip_headers <big.second >big.second.body &&
	test_cmp big.first.body big.second.body
'

test_expect_success 'the cached body matches one generated with the cache off' '
	sed -e "s/^cache-size=64$/cache-size=0/" bigrc >bignocache &&
	cgit_rc bignocache "url=bigpage/tree/big.txt" >big.nocache &&
	strip_headers <big.nocache >big.nocache.body &&
	test_cmp big.nocache.body big.second.body
'

test_expect_success 'the page ends where it should, so nothing was dropped' '
	tail -c 200 big.second.body | grep "</html>"
'

# A slot stores its key ahead of the content and only the first few kilobytes of
# it are read back, so a key longer than that could never match. A request
# carrying one used to leave a slot behind on every visit that nothing could
# ever go on to use.
test_expect_success 'an over-long cache key leaves no slot behind' '
	rm -rf cache3 && mkdir cache3 &&
	sed -e "s|^cache-root=.*|cache-root=$PWD/cache3|" bigrc >bigkeyrc &&
	long=$(awk "BEGIN{s=\"\";for(i=0;i<6000;i++)s=s \"k\"; print s}") &&
	cgit_rc bigkeyrc "url=bigpage/&q=$long" >key.out 2>/dev/null &&
	grep "</html>" key.out &&
	test_dir_is_empty cache3
'

test_expect_success 'an ordinary key still fills a slot' '
	rm -rf cache3 && mkdir cache3 &&
	cgit_rc bigkeyrc "url=bigpage/" >/dev/null &&
	ls cache3 >key.slots &&
	test_line_count = 1 key.slots
'

# An error page fired after rendering began abandons the fill with part of
# the page already written into the lock file. That fragment must be
# discarded along with the file, not replayed to the visitor behind the
# error page. A commit whose parent object is missing renders its whole
# info table before the diff machinery fails, which makes it the trigger.
test_expect_success 'set up a repo missing a parent object' '
	mkrepo repos/broken 2 &&
	parent=$(git -C repos/broken rev-parse HEAD^) &&
	tip=$(git -C repos/broken rev-parse HEAD) &&
	rm "repos/broken/.git/objects/$(echo "$parent" | cut -c1-2)/$(echo "$parent" | cut -c3-)" &&
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache4" &&
		echo "cache-size=64" &&
		echo "repo.url=broken" &&
		echo "repo.path=$PWD/repos/broken/.git"
	} >brokenrc &&
	rm -rf cache4 && mkdir cache4
'

test_expect_success 'an error after output began ends the page with it and caches nothing' '
	cgit_rc brokenrc "url=broken/commit/&id=$tip" >broken.out &&
	grep "Not a commit" broken.out &&
	test $(grep -c "^Status:" broken.out) = 1 &&
	test_dir_is_empty cache4
'

# An error page reports a condition the repository may grow out of, and a
# request pinned to an object id would cache it under the never-expiring
# static ttl. Asking for a commit that does not exist yet must therefore
# leave nothing behind, or the real page could never appear once it does.
test_expect_success 'an error page leaves no slot behind' '
	rm -rf cache3 && mkdir cache3 &&
	CGIT_CONFIG="$PWD/bigkeyrc" \
	QUERY_STRING="url=bigpage/commit/&id=0123456789abcdef0123456789abcdef01234567" cgit >error.out &&
	grep "Not a commit" error.out &&
	test_dir_is_empty cache3
'

# The dumb transport serves files that already sit on the disk, so a copy in
# a slot would only double the disk they take.
test_expect_success 'a clone file leaves no slot behind' '
	rm -rf cache3 && mkdir cache3 &&
	cgit_rc bigkeyrc "url=bigpage/info/refs" >clone.out &&
	grep "refs/heads/master" clone.out &&
	cgit_rc bigkeyrc "url=bigpage/objects/info/packs" >packs.out &&
	grep "^Status: 200" packs.out &&
	test_dir_is_empty cache3
'

# A response past cache-max-slot-size is served whole but kept nowhere, both
# on a first fill and when it would have replaced an expired slot, which is
# dropped along with it. The summary page stays under the limit set here
# while the big blob page is well past it.
test_expect_success 'a response over cache-max-slot-size is served but not kept' '
	rm -rf cache3 && mkdir cache3 &&
	{
		echo "cache-max-slot-size=32" &&
		cat bigkeyrc
	} >slotsizerc &&
	cgit_rc slotsizerc "url=bigpage/tree/big.txt" >big.unkept &&
	tail -c 200 big.unkept | grep "</html>" &&
	test_dir_is_empty cache3 &&
	cgit_rc slotsizerc "url=bigpage/" >/dev/null &&
	ls cache3 >kept.slots &&
	test_line_count = 1 kept.slots
'

test_expect_success 'an expired slot is dropped when its refill is too large' '
	rm -rf cache3 && mkdir cache3 &&
	cgit_rc bigkeyrc "url=bigpage/tree/big.txt" >/dev/null &&
	ls cache3 >filled.slots &&
	test_line_count = 1 filled.slots &&
	touch -t 200001010000 cache3/* &&
	cgit_rc slotsizerc "url=bigpage/tree/big.txt" >big.refill &&
	tail -c 200 big.refill | grep "</html>" &&
	test_dir_is_empty cache3
'

# The summary page keeps a ttl of its own, while every other page that follows
# a branch rather than a pinned id takes cache-dynamic-ttl, whether or not its
# url names the branch.
test_expect_success 'pages following a branch take the dynamic ttl' '
	rm -rf cache3 && mkdir cache3 &&
	{
		echo "cache-dynamic-ttl=0" &&
		cat bigkeyrc
	} >dynamicrc &&
	cgit_rc dynamicrc "url=bigpage/log/" >/dev/null &&
	cgit_rc dynamicrc "url=bigpage/log/&h=master" >/dev/null &&
	test_dir_is_empty cache3 &&
	cgit_rc dynamicrc "url=bigpage/" >/dev/null &&
	cgit_rc dynamicrc "url=bigpage/summary/" >/dev/null &&
	ls cache3 >summary.slots &&
	test_line_count = 2 summary.slots
'

# A write into the lock file that fails on the page's last flush has to end
# as one that fails mid page does, with the fill abandoned and the page
# delivered from the lock file, rather than in a blank response. The limit
# is on files alone, so the page goes to the visitor through a pipe.
test_expect_success 'a failed final write still delivers the page' '
	rm -rf cache3 && mkdir cache3 &&
	(
		trap "" XFSZ &&
		ulimit -f 2 &&
		cgit_rc bigkeyrc "url=bigpage/" 2>flush.err
	) | cat >flush.out &&
	grep "^Status: 200" flush.out &&
	grep "</html>" flush.out &&
	! grep "called many times" flush.err
'

# The second request lands while the first is inside its filter, which says
# so through a marker file, since a fresh binary can take a moment to start.
start_first_request() {
	n=0
	rm -f filter-running
	cgit_rc lockrc "url=bigpage/about/" >first.out 2>first.err &
	while ! test -f filter-running && test $n -lt 10
	do
		sleep 1
		n=$((n + 1))
	done
}

overlap_requests() {
	start_first_request
	cgit_rc lockrc "url=bigpage/log/" >/dev/null 2>second.err
	wait
}

# The fill writes the page into the lock file through stdout, and an exec
# filter puts its own pipe on stdout while it runs. The slot lock has to
# outlast that, or a second request on the slot rebuilds it under the first
# and the first visitor gets the two pages spliced together. The second one
# finding the lock held is ordinary contention, which logs nothing.
test_expect_success 'a slot stays locked while an exec filter runs' '
	rm -rf cache3 && mkdir cache3 &&
	write_script slow-filter.sh <<-EOF &&
	touch "$PWD/filter-running"
	sleep 3
	cat
	EOF
	{
		echo "cache-size=1" &&
		sed "/^cache-size=/d" bigkeyrc &&
		echo "repo.readme=:file-1" &&
		echo "repo.about-filter=exec:$PWD/slow-filter.sh"
	} >lockrc &&
	overlap_requests &&
	test $(grep -c "^Status:" first.out) = 1 &&
	grep "id=.about." first.out &&
	! grep "Unable to publish" first.err &&
	! grep "Unable to lock" second.err
'

# A request that opens the lock file just before its holder renames it into
# place as the live slot goes on to lock the slot itself. strace holds the
# second request's flock back until the first has finished, and the check
# that the path still names the locked file turns that lock down, so the
# slot keeps the first request's key. strace needs ptrace and a version that
# can delay a call, which some systems do not give.
test_lazy_prereq STRACE_DELAY '
	strace -o /dev/null -e inject=flock:delay_enter=1 true
'

test_expect_success STRACE_DELAY 'a lock taken after the rename leaves the slot alone' '
	rm -rf cache3 && mkdir cache3 &&
	start_first_request &&
	strace -o /dev/null -e inject=flock:delay_enter=4000000 \
		-E CGIT_CONFIG="$PWD/lockrc" -E QUERY_STRING="url=bigpage/log/" \
		cgit >second.out 2>second.err &&
	wait &&
	grep "id=.about." first.out &&
	grep "</html>" second.out &&
	tr "\000" "\n" <"cache3/$(ls cache3)" | head -n 1 >slot.key &&
	grep "about" slot.key &&
	! grep "log" slot.key
'

# An flock lasts until the last copy of its descriptor closes, so a filter
# may hold the slot's lock file only as the stdout it writes the page to. A
# stray copy would keep the slot locked for as long as the filter, or
# anything it leaves running, lives on.
test_expect_success PERL 'a filter holds the lock file only as its stdout' '
	rm -rf cache3 && mkdir cache3 &&
	write_script fd-filter "$PERL_PATH" <<-\EOF &&
	use POSIX qw(fstat);
	1 while <STDIN>;
	my @lock = stat $ENV{LOCK_FILE} or die "$!\n";
	my @fds = grep {
		my @fd = fstat($_);
		@fd && $fd[0] == $lock[0] && $fd[1] == $lock[1]
	} 0 .. 255;
	print "lock file on fd @fds.\n";
	EOF
	{
		echo "cache-size=1" &&
		sed "/^cache-size=/d" bigkeyrc &&
		echo "repo.readme=:file-1" &&
		echo "repo.about-filter=exec:$PWD/fd-filter"
	} >fdrc &&
	LOCK_FILE="$PWD/cache3/00000000.lock" CGIT_CONFIG="$PWD/fdrc" \
	QUERY_STRING="url=bigpage/about/" cgit >fd.out &&
	grep -F "lock file on fd 1." fd.out
'

# A refresh that cannot take the lock serves the expired copy, and says why
# unless the reason is another request doing the refresh. A line appended
# to the copy shows that it is what was served.
test_expect_success SANITY 'a refresh that cannot lock serves the expired slot and says why' '
	rm -rf cache3 && mkdir cache3 &&
	cgit_rc bigkeyrc "url=bigpage/" >/dev/null &&
	slot=$(ls cache3) &&
	echo "the expired copy" >>"cache3/$slot" &&
	touch -t 200001010000 "cache3/$slot" &&
	test_when_finished "chmod u+w cache3" &&
	chmod a-w cache3 &&
	cgit_rc bigkeyrc "url=bigpage/" >stale.out 2>stale.err &&
	grep "the expired copy" stale.out &&
	grep "Unable to open .*\.lock: Permission denied" stale.err
'

# A lock file that cannot take the key goes with its lock, rather than
# staying behind locked while the page renders uncached. The limit is on
# files, so the log goes through the pipe as well.
test_expect_success 'a slot lock that cannot take its key is let go' '
	rm -rf cache3 && mkdir cache3 &&
	(
		trap "" XFSZ &&
		ulimit -f 0 &&
		cgit_rc bigkeyrc "url=bigpage/" 2>&1
	) | cat >nokey.out &&
	grep "</html>" nokey.out &&
	grep "Unable to write .*\.lock" nokey.out &&
	test_dir_is_empty cache3
'

# A scan is cached in a file named after the hash of its path, so a first
# scan finds out what that name is. The index page is left uncached, which
# makes each request read the list afresh.
test_expect_success 'set up a cached scan of two repositories' '
	mkrepo scanned/one 1 &&
	mkrepo scanned/two 1 &&
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache5" &&
		echo "cache-size=64" &&
		echo "cache-index-ttl=0" &&
		echo "scan-path=$PWD/scanned"
	} >scanrc &&
	rm -rf cache5 && mkdir cache5 &&
	cgit_rc scanrc "url=" >/dev/null &&
	scanned_rc=$(ls cache5) &&
	test_path_is_file "cache5/$scanned_rc"
'

# A request that finds the scan's lock held and has no list to go on scans
# in its own process, leaving the file to the holder. That is only another
# request at work and no cause for a line in the log.
test_expect_success PERL 'a scan finding its lock held runs uncached' '
	write_script hold-lock "$PERL_PATH" <<-\EOF &&
	use Fcntl qw(:flock);
	open(my $lock, ">>", shift) or die "$!\n";
	flock($lock, LOCK_EX | LOCK_NB) or die "$!\n";
	exit(system(@ARGV) ? 1 : 0);
	EOF
	rm "cache5/$scanned_rc" &&
	CGIT_CONFIG="$PWD/scanrc" QUERY_STRING="url=" \
	./hold-lock "cache5/$scanned_rc.lock" cgit >held.out 2>held.err &&
	grep ">one</a>" held.out &&
	grep ">two</a>" held.out &&
	test_path_is_missing "cache5/$scanned_rc" &&
	test_must_be_empty held.err
'

# A run that died before its rename leaves a lock file behind, here longer
# than anything the next run writes and ending in a repository and a marker.
# Emptying the file before reuse keeps that tail out of both caches.
test_expect_success 'a stale lock file reaches neither the scan nor a slot' '
	{
		printf "%01024d\n" 0 0 0 0 &&
		echo "repo.url=ghost" &&
		echo "repo.path=$PWD/scanned/one/.git"
	} >stale.lock &&
	rm -f "cache5/$scanned_rc" &&
	cp stale.lock "cache5/$scanned_rc.lock" &&
	CGIT_CONFIG="$PWD/scanrc" QUERY_STRING="url=" cgit >scan.out &&
	grep ">one</a>" scan.out &&
	! grep ">ghost</a>" scan.out &&
	rm -rf cache3 && mkdir cache3 &&
	CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/" cgit >page.out &&
	slot=$(ls cache3) &&
	rm "cache3/$slot" &&
	{ cat page.out page.out && echo "STALE-TAIL"; } >"cache3/$slot.lock" &&
	CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/" cgit >/dev/null &&
	CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/" cgit >replay.out &&
	grep "</html>" replay.out &&
	! grep "STALE-TAIL" replay.out
'

test_done