index : cgit

#!/bin/sh

# Checks the page cache and the scan cache. A page is kept in a slot named
# after its request, filled under a lock that a second request must respect,
# and served stale while another request refreshes it, within the ttls and
# size bounds cgitrc sets. A scan-path list is kept in a file named after the
# settings that shaped it and rebuilt in the background once it expires.

test_description='Validate cache'
. ./setup.sh

# The counting tests below visit the same pages, the index plus six pages in
# each of two repositories, so the counts they expect stay tied to this one
# list of thirteen requests.
request_all_pages() {
	cgit_ok "$1" "url=" >/dev/null &&
	for repo in foo bar
	do
		for page in "" /refs /tree /log /diff /patch
		do
			cgit_ok "$1" "url=$repo$page" >/dev/null || return 1
		done
	done
}

# The PERL prerequisite says only that the build allows perl, so the tests
# with perl helpers ask the interpreter itself, and skip where it is missing.
test_lazy_prereq PERL_RUNS '
	"$PERL_PATH" -e 1
'

# Runs a command while holding the flock on the file named first, as another
# request at work on the same lock would. The command has to start inside the
# hold, which cgit_rc cannot do, so the alarm cgit_rc would set is set here,
# and a cgit that waits on the lock fails its test rather than stalling it.
write_hold_lock() {
	write_script hold-lock "$PERL_PATH" <<-\EOF
	use Fcntl qw(:flock);
	open(my $lock, ">>", shift) or die "$!\n";
	flock($lock, LOCK_EX | LOCK_NB) or die "$!\n";
	defined(my $pid = fork) or die "$!\n";
	if (!$pid) {
		alarm 60;
		exec @ARGV or die "$ARGV[0]: $!\n";
	}
	waitpid($pid, 0);
	exit($? ? 1 : 0);
	EOF
}

test_expect_success 'a cache-size of 0 leaves no slot behind' '
	rm -f cache/* &&
	sed "s/^cache-size=.*/cache-size=0/" cgitrc >size0rc &&
	request_all_pages size0rc &&
	test_dir_is_empty cache
'

# With one slot every request lands on the same file, so the key it holds is
# all that keeps one page from being served for another.
test_expect_success 'one slot serves each request its own page' '
	rm -f cache/* &&
	sed "s/^cache-size=.*/cache-size=1/" cgitrc >size1rc &&
	cgit_ok size1rc "url=foo/log/" >/dev/null &&
	cgit_rc size1rc "url=bar/refs/" >refs.out 2>refs.err &&
	test_grep "^Status: 200" refs.out &&
	test_grep "data-page=.refs. data-repo=.bar." refs.out &&
	test_grep "<table class=.list refs." refs.out &&
	test_grep ! "data-repo=.foo." refs.out &&
	test_grep "holds a different key, consider a larger cache-size" refs.err &&
	cgit_rc size1rc "url=foo/log/" >log.out 2>log.err &&
	test_grep "data-page=.log. data-repo=.foo." log.out &&
	test_grep "<table class=.list log." log.out &&
	test_grep ! "data-repo=.bar." log.out &&
	test_grep "holds a different key, consider a larger cache-size" log.err &&
	ls cache >output &&
	test_line_count = 1 output
'

# The listing is a page like any other and takes a slot of its own, so a
# fresh render of it would list fourteen slots. Thirteen on the second
# request shows the first was replayed, which a dynamic ttl that never
# expires keeps true however slow the run.
test_expect_success 'a full table gives each request a slot of its own' '
	rm -f cache/* &&
	{
		echo "cache-dynamic-ttl=-1" &&
		sed "s/^cache-size=.*/cache-size=1021/" cgitrc
	} >size1021rc &&
	request_all_pages size1021rc &&
	ls cache >output &&
	test_line_count = 13 output &&
	cgit_ok size1021rc "url=foo/ls_cache" >output.full &&
	strip_headers <output.full >output &&
	test_line_count = 13 output &&
	ls cache >output &&
	test_line_count = 14 output &&
	cgit_ok size1021rc "url=foo/ls_cache" >output.second &&
	test_cmp output.full output.second
'

# The listing names the cache path and the urls other visitors asked for, so
# it answers only where enable-cache-list is set, and as plain text.
test_expect_success 'the cache listing answers only where it is enabled' '
	sed -e "/^enable-cache-list=/d" -e "s/^cache-size=.*/cache-size=0/" cgitrc >nolistrc &&
	cgit_rc nolistrc "url=foo/ls_cache" >nolist.out &&
	test_grep "^Status: 404" nolist.out &&
	test_grep ! -F "$PWD/cache" nolist.out &&
	sed "s/^cache-size=.*/cache-size=0/" cgitrc >listrc &&
	cgit_ok listrc "url=foo/ls_cache" >list.out &&
	test_grep "^Content-Type: text/plain" list.out &&
	test_grep -F "$PWD/cache/" list.out
'

test_expect_success 'a cache listing without its directory says so' '
	sed -e "s|^cache-root=.*|cache-root=$PWD/no-such-cache|" -e "s/^cache-size=.*/cache-size=0/" cgitrc >nocacherc &&
	cgit_rc nocacherc "url=foo/ls_cache" >nocache.out 2>nocache.err &&
	test_grep "^Status: 200" nocache.out &&
	test_grep -F "Unable to open $PWD/no-such-cache: No such file or directory" nocache.err
'

# Page output is buffered, so a slot only holds the whole page if the buffer is
# emptied before the size of what was generated is taken. The blob set up here
# is larger than one buffer, so a missing flush would leave the cached copy cut
# short.
test_expect_success 'set up a repo with a page larger than the output buffer' '
	mkrepo repos/bigpage 1 &&
	(
		cd repos/bigpage &&
		awk "BEGIN{for(i=0;i<20000;i++) print \"line \" i \" of the big file\"}" >big.txt &&
		git add big.txt &&
		git commit -m "add big.txt"
	) &&
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache2" &&
		echo "cache-size=64" &&
		echo "enable-tree-linenumbers=1" &&
		echo "repo.url=bigpage" &&
		echo "repo.path=$PWD/repos/bigpage/.git"
	} >bigrc &&
	sed -e "s|^cache-root=.*|cache-root=$PWD/cache3|" bigrc >bigkeyrc &&
	{
		echo "cache-max-slot-size=32" &&
		cat bigkeyrc
	} >slotsizerc &&
	rm -rf cache2 && mkdir cache2
'

# The slot holds the key, a NUL and then the page, so the page it keeps is
# what follows the first line once the NUL is a newline. The replay has to
# match that and a render with the cache off.
test_expect_success 'a page larger than the buffer is kept and replayed whole' '
	cgit_ok bigrc "url=bigpage/tree/big.txt" >big.first &&
	test $(wc -c <big.first) -gt 65536 &&
	ls cache2 >big.slots &&
	test_line_count = 1 big.slots &&
	tr "\000" "\n" <"cache2/$(cat big.slots)" | sed 1d >slot.page &&
	test_cmp big.first slot.page &&
	cgit_ok bigrc "url=bigpage/tree/big.txt" >big.second &&
	test_cmp big.first big.second &&
	sed -e "s/^cache-size=64$/cache-size=0/" bigrc >bignocache &&
	cgit_ok bignocache "url=bigpage/tree/big.txt" >big.nocache &&
	strip_headers <big.nocache >big.nocache.body &&
	strip_headers <big.second >big.second.body &&
	test_cmp big.nocache.body big.second.body
'

# A slot is read back in one read of 4096 bytes, which has to take in the key
# and its NUL, so a longer key could never match. A request carrying one used
# to leave a slot behind on every visit that nothing could ever go on to use.
# The key is the scheme, host, path and query, each behind its length, so with
# no host or path a query of 4077 bytes makes a key of 4095.
test_expect_success 'a cache key past 4095 bytes leaves no slot behind' '
	rm -rf cache3 && mkdir cache3 &&
	cgit_rc bigkeyrc "url=bigpage/&q=$(printf "%04063d" 0)" >key.out 2>key.err &&
	test_grep "</html>" key.out &&
	test_grep "Cache key too long for a slot" key.err &&
	test_dir_is_empty cache3
'

test_expect_success 'a cache key of 4095 bytes fills a slot' '
	rm -rf cache3 && mkdir cache3 &&
	cgit_ok bigkeyrc "url=bigpage/&q=$(printf "%04062d" 0)" >/dev/null &&
	ls cache3 >key.slots &&
	test_line_count = 1 key.slots &&
	tr "\000" "\n" <"cache3/$(cat key.slots)" | awk "{ print length; exit }" >key.length &&
	echo 4095 >expect &&
	test_cmp expect key.length
'

# An error page fired after rendering began abandons the fill with part of
# the page already written into the lock file. That fragment must be
# discarded along with the file, not replayed to the visitor behind the
# error page. A commit whose parent object is missing renders its whole
# info table before the diff machinery fails, which makes it the trigger.
test_expect_success 'set up a repo missing a parent object' '
	mkrepo repos/broken 2 &&
	parent=$(git -C repos/broken rev-parse HEAD^) &&
	tip=$(git -C repos/broken rev-parse HEAD) &&
	rm "repos/broken/.git/objects/$(echo "$parent" | cut -c1-2)/$(echo "$parent" | cut -c3-)" &&
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache4" &&
		echo "cache-size=64" &&
		echo "repo.url=broken" &&
		echo "repo.path=$PWD/repos/broken/.git"
	} >brokenrc &&
	rm -rf cache4 && mkdir cache4
'

# The visitor gets the info table the commit page had rendered, then the
# error, then the end of the page. The tree page needs no parent, and the slot
# it fills shows the cache is in use for brokenrc at all.
test_expect_success 'an error after output began ends the page with it and caches nothing' '
	cgit_rc brokenrc "url=broken/commit/&id=$tip" >broken.out &&
	test $(grep -c "^Status:" broken.out) = 1 &&
	sed -n -e "/<table class=.commit-info.>/s/.*/info/p" -e "/Not a commit/s/.*/error/p" \
		-e "/<\/html>/s/.*/end/p" broken.out >broken.order &&
	test_write_lines info error end >expect &&
	test_cmp expect broken.order &&
	test_dir_is_empty cache4 &&
	cgit_ok brokenrc "url=broken/tree/" >/dev/null &&
	ls cache4 >broken.slots &&
	test_line_count = 1 broken.slots
'

# An error page reports a condition the repository may grow out of, and a
# request pinned to an object id would cache it under the never-expiring
# static ttl. Asking for a commit that does not exist yet must therefore
# leave nothing behind, or the real page could never appear once it does.
test_expect_success 'an error page leaves no slot behind' '
	rm -rf cache3 && mkdir cache3 &&
	cgit_rc bigkeyrc "url=bigpage/commit/&id=$(test_oid deadbeef)" >error.out &&
	test_grep "Not a commit" error.out &&
	test_dir_is_empty cache3
'

# The dumb transport serves files that already sit on the disk, so a copy in
# a slot would only double the disk they take.
test_expect_success 'a clone file leaves no slot behind' '
	rm -rf cache3 && mkdir cache3 &&
	cgit_rc bigkeyrc "url=bigpage/info/refs" >clone.out &&
	grep "refs/heads/master" clone.out &&
	cgit_rc bigkeyrc "url=bigpage/objects/info/packs" >packs.out &&
	grep "^Status: 200" packs.out &&
	test_dir_is_empty cache3
'

# A response past cache-max-slot-size is served whole but kept nowhere, both
# on a first fill and when it would have replaced an expired slot, which is
# dropped along with it. The summary page stays under the 32 kilobytes
# slotsizerc allows while the big blob page is well past it.
test_expect_success 'a response over cache-max-slot-size is served but not kept' '
	rm -rf cache3 && mkdir cache3 &&
	cgit_rc slotsizerc "url=bigpage/tree/big.txt" >big.unkept &&
	tail -c 200 big.unkept | grep "</html>" &&
	test_dir_is_empty cache3 &&
	cgit_rc slotsizerc "url=bigpage/" >/dev/null &&
	ls cache3 >kept.slots &&
	test_line_count = 1 kept.slots
'

test_expect_success 'an expired slot is dropped when its refill is too large' '
	rm -rf cache3 && mkdir cache3 &&
	cgit_ok bigkeyrc "url=bigpage/tree/big.txt" >/dev/null &&
	ls cache3 >filled.slots &&
	test_line_count = 1 filled.slots &&
	echo "the expired copy" >>"cache3/$(cat filled.slots)" &&
	touch -t 200001010000 cache3/* &&
	cgit_ok slotsizerc "url=bigpage/tree/big.txt" >big.refill &&
	test_grep ! "the expired copy" big.refill &&
	test_dir_is_empty cache3
'

# MANUAL says a cache-max-slot-size of 0 removes the limit, and cgit reads one
# below 0 the same way. Each value comes after a limit the big blob page is
# well past, so the page keeps its slot only if the value read last lifts it.
test_expect_success 'a cache-max-slot-size of 0 or below lifts the limit' '
	for size in 0 -1
	do
		rm -rf cache3 && mkdir cache3 &&
		{
			echo "cache-max-slot-size=32" &&
			echo "cache-max-slot-size=$size" &&
			cat bigkeyrc
		} >unlimitedrc &&
		cgit_ok unlimitedrc "url=bigpage/tree/big.txt" >/dev/null &&
		ls cache3 >unlimited.slots &&
		test_line_count = 1 unlimited.slots ||
		{ echo "cache-max-slot-size=$size kept no slot" && return 1; }
	done
'

# A slot past its ttl is rendered again, and the new page is both what the
# request gets and what the slot keeps. The ttl is long enough that the copy
# is still served after the commit, which shows the slot is in use.
test_expect_success 'an expired slot is refreshed with the page as it is now' '
	rm -rf cache3 && mkdir cache3 &&
	mkrepo repos/fresh 1 &&
	cgitrc_repo freshrc fresh "cache-root=$PWD/cache3" cache-size=64 cache-dynamic-ttl=60 &&
	cgit_ok freshrc "url=fresh/log/" >/dev/null &&
	slot=$(ls cache3) &&
	git -C repos/fresh commit --allow-empty -q -m "a newer commit" &&
	cgit_ok freshrc "url=fresh/log/" >stale.out &&
	test_grep ! "a newer commit" stale.out &&
	touch -t 200001010000 "cache3/$slot" &&
	cgit_ok freshrc "url=fresh/log/" >fresh.out &&
	test_grep "a newer commit" fresh.out &&
	tr "\000" "\n" <"cache3/$slot" >slot.txt &&
	test_grep "a newer commit" slot.txt
'

# A negative ttl never expires, however old the slot, which the line
# appended to it shows by being served.
test_expect_success 'a negative page ttl never expires the slot' '
	rm -rf cache3 && mkdir cache3 &&
	{
		echo "cache-dynamic-ttl=-1" &&
		cat bigkeyrc
	} >neverrc &&
	cgit_ok neverrc "url=bigpage/log/" >/dev/null &&
	slot=$(ls cache3) &&
	echo "the kept copy" >>"cache3/$slot" &&
	touch -t 200001010000 "cache3/$slot" &&
	cgit_ok neverrc "url=bigpage/log/" >never.out &&
	test_grep "the kept copy" never.out
'

# The summary page keeps a ttl of its own, while every other page that follows
# a branch rather than a pinned id takes cache-dynamic-ttl, whether or not its
# url names the branch.
test_expect_success 'pages following a branch take the dynamic ttl' '
	rm -rf cache3 && mkdir cache3 &&
	{
		echo "cache-dynamic-ttl=0" &&
		cat bigkeyrc
	} >dynamicrc &&
	cgit_rc dynamicrc "url=bigpage/log/" >/dev/null &&
	cgit_rc dynamicrc "url=bigpage/log/&h=master" >/dev/null &&
	test_dir_is_empty cache3 &&
	cgit_rc dynamicrc "url=bigpage/" >/dev/null &&
	cgit_rc dynamicrc "url=bigpage/summary/" >/dev/null &&
	ls cache3 >summary.slots &&
	test_line_count = 2 summary.slots
'

# Only a full object id names a page that can never change, so a page pinned
# to one alone takes the static ttl, which by default never expires. An
# abbreviation or a branch name, in either id, can come to mean another
# commit.
test_expect_success 'only a page pinned to full ids takes the static ttl' '
	rm -rf cache3 && mkdir cache3 &&
	{
		echo "cache-dynamic-ttl=0" &&
		cat bigkeyrc
	} >staticrc &&
	first=$(git -C repos/bigpage rev-parse HEAD~1) &&
	short=$(git -C repos/bigpage rev-parse --short HEAD~1) &&
	for query in "commit/&id=$short" "commit/&id=master" "diff/&id=$first&id2=$short" "diff/&id=master&id2=$first"
	do
		cgit_ok staticrc "url=bigpage/$query" >/dev/null || return 1
	done &&
	test_dir_is_empty cache3 &&
	cgit_ok staticrc "url=bigpage/commit/&id=$first" >/dev/null &&
	ls cache3 >static.slots &&
	test_line_count = 1 static.slots &&
	cgit_ok staticrc "url=bigpage/diff/&id=$first&id2=$first" >/dev/null &&
	ls cache3 >static.slots &&
	test_line_count = 2 static.slots
'

# Each kind of page takes a ttl of its own, and a ttl of 0 keeps that page out
# of the cache. The same page with its ttl at 1 fills a slot, so the empty
# cache-root is down to the 0 rather than to a page that is never cached.
test_expect_success 'a page ttl of 0 leaves that page uncached' '
	first=$(git -C repos/bigpage rev-parse HEAD~1) &&
	while read setting query
	do
		rm -rf cache3 && mkdir cache3 &&
		{
			echo "snapshots=tar.gz" &&
			echo "$setting=0" &&
			cat bigkeyrc &&
			echo "repo.readme=:file-1"
		} >ttl0rc &&
		sed "/^$setting=/s/=0/=1/" ttl0rc >ttl1rc &&
		cgit_ok ttl0rc "url=$query" >/dev/null &&
		test_dir_is_empty cache3 &&
		cgit_ok ttl1rc "url=$query" >/dev/null &&
		ls cache3 >ttl.slots &&
		test_line_count = 1 ttl.slots ||
		{ echo "$setting does not govern url=$query" && return 1; }
	done <<-EOF
	cache-index-ttl
	cache-summary-ttl bigpage/
	cache-about-ttl bigpage/about/
	cache-snapshot-ttl bigpage/snapshot/master.tar.gz
	cache-static-ttl bigpage/commit/&id=$first
	cache-dynamic-ttl bigpage/log/
	EOF
'

# A write into the lock file that fails on the page's last flush has to end
# as one that fails mid page does, with the fill abandoned and the page
# delivered from the lock file, rather than in a blank response. The limit
# is on files alone, so the page goes to the visitor through a pipe.
test_expect_success 'a failed final write still delivers the page' '
	rm -rf cache3 && mkdir cache3 &&
	(
		trap "" XFSZ &&
		ulimit -f 2 &&
		{ cgit_rc bigkeyrc "url=bigpage/" 2>flush.err; echo $? >flush.status; }
	) | cat >flush.out &&
	test "$(cat flush.status)" = 0 &&
	test_dir_is_empty cache3 &&
	test $(grep -c "^Status:" flush.out) = 1 &&
	test_grep "^Status: 200" flush.out &&
	test_grep ! "called many times" flush.err &&
	sed "s/^cache-size=.*/cache-size=0/" bigkeyrc >flushnocacherc &&
	cgit_ok flushnocacherc "url=bigpage/" >flush.nocache &&
	strip_headers <flush.nocache >expect &&
	strip_headers <flush.out >actual &&
	test_cmp expect actual
'

# The first request's filter says through a marker that it is running, and
# then holds the first request there until the test writes a marker of its
# own, so the second request is sure to land while the first holds the slot.
# A fresh binary can take a moment to start, so both waits are bounded loops.
start_first_request() {
	rm -f filter-running release
	cgit_rc lockrc "url=bigpage/about/" >first.out 2>first.err &
	first_pid=$!
	n=0
	while ! test -f filter-running && test $n -lt 30
	do
		sleep 1
		n=$((n + 1))
	done
	test -f filter-running
}

# Lets the first request go and returns its status.
finish_first_request() {
	touch release
	wait "$first_pid"
}

overlap_requests() {
	start_first_request &&
	cgit_rc lockrc "url=bigpage/log/" >second.out 2>second.err
	overlap_status=$?
	finish_first_request &&
	return $overlap_status
}

# The fill writes the page into the lock file through stdout, and an exec
# filter puts its own pipe on stdout while it runs. The slot lock has to
# outlast that, or a second request on the slot rebuilds it under the first
# and the first visitor gets the two pages spliced together. The second one
# finding the lock held is ordinary contention, which logs nothing and gets
# the page uncached. A network filesystem that fakes flock with record locks
# breaks this the same way, so the trash directory has to be local.
# A config of one slot puts every page in 00000000, as the slot is named after
# the key's hash modulo cache-size.
test_expect_success 'a slot stays locked while an exec filter runs' '
	rm -rf cache3 && mkdir cache3 &&
	write_script slow-filter.sh <<-EOF &&
	touch "$PWD/filter-running"
	n=0
	while ! test -f "$PWD/release" && test \$n -lt 60
	do
		sleep 1
		n=\$((n + 1))
	done
	cat
	EOF
	{
		echo "cache-size=1" &&
		sed "/^cache-size=/d" bigkeyrc &&
		echo "repo.readme=:file-1" &&
		echo "repo.about-filter=exec:$PWD/slow-filter.sh"
	} >lockrc &&
	overlap_requests &&
	test $(grep -c "^Status:" first.out) = 1 &&
	test_grep "<div id=.about.>1$" first.out &&
	test_grep "</html>" first.out &&
	test_must_be_empty first.err &&
	test_grep "data-page=.log." second.out &&
	test_grep "</html>" second.out &&
	test_must_be_empty second.err &&
	tr "\000" "\n" <cache3/00000000 | head -n 1 >slot.key &&
	test_grep "about" slot.key &&
	test_grep ! "log" slot.key
'

# A request that opens the lock file just before its holder renames it into
# place as the live slot goes on to lock the slot itself. strace holds the
# second request's flock back until the first has finished, and the check
# that the path still names the locked file turns that lock down, so the
# slot keeps the first request's key. This is the only test of that check.
# The test needs an strace that can trace the process and delay a call, and
# skips where there is none.
test_lazy_prereq STRACE_DELAY '
	strace -o /dev/null -e trace=%file,flock -e inject=flock:delay_enter=1 true
'

# Returns once the trace shows the second request has opened the lock file,
# which the first request still holds. strace has to start cgit itself, so
# the request is set up here rather than by cgit_rc.
start_traced_request() {
	rm -f second.trace
	strace -o second.trace -e trace=%file,flock \
		-e inject=flock:delay_enter=4000000 \
		-E CGIT_CONFIG="$PWD/lockrc" -E QUERY_STRING="url=bigpage/log/" \
		cgit >second.out 2>second.err &
	second_pid=$!
	n=0
	while ! grep "00000000\.lock" second.trace >/dev/null 2>&1 && test $n -lt 30
	do
		sleep 1
		n=$((n + 1))
	done
	grep "00000000\.lock" second.trace >/dev/null 2>&1
}

overlap_traced_requests() {
	second_pid=
	start_first_request &&
	start_traced_request
	overlap_status=$?
	finish_first_request || overlap_status=1
	if test -n "$second_pid"
	then
		wait "$second_pid" || overlap_status=1
	fi
	return $overlap_status
}

# The first request is let go as soon as the second has the lock file open,
# well inside the delay, so the flock that follows succeeds on a file that
# has become the slot. A trace showing it refused would mean the race was
# never reached. strace pads a short call out to a column before its result.
test_expect_success STRACE_DELAY 'a lock taken after the rename leaves the slot alone' '
	rm -rf cache3 && mkdir cache3 &&
	overlap_traced_requests &&
	test_grep -E "^flock\(.*\) += 0" second.trace &&
	test_grep "<div id=.about.>1$" first.out &&
	test_grep "</html>" second.out &&
	tr "\000" "\n" <cache3/00000000 | head -n 1 >slot.key &&
	test_grep "about" slot.key &&
	test_grep ! "log" slot.key
'

# An flock lasts until the last copy of its descriptor closes, so a filter
# may hold the slot's lock file only as the stdout it writes the page to. A
# stray copy would keep the slot locked for as long as the filter, or
# anything it leaves running, lives on. The visitor's stdout, put aside while
# the fill has stdout, must not reach the filter at all, or one that outlives
# the request would hold the visitor's connection open. The filter finds both
# files from its own path, since a variable set for cgit_rc alone would not
# portably reach cgit.
test_expect_success PERL_RUNS 'a filter gets the lock file only as stdout and never the client' '
	rm -rf cache3 && mkdir cache3 &&
	write_script fd-filter "$PERL_PATH" <<-\EOF &&
	use File::Basename qw(dirname);
	use POSIX qw(fstat);
	1 while <STDIN>;
	for my $file ("cache3/00000000.lock", "fd.out") {
		my $path = dirname($0) . "/$file";
		my @file = stat $path or die "$path: $!\n";
		my @fds = grep {
			my @fd = fstat($_);
			@fd && $fd[0] == $file[0] && $fd[1] == $file[1]
		} 0 .. 255;
		print "$file on fd ", @fds ? "@fds" : "none", ".\n";
	}
	EOF
	{
		echo "cache-size=1" &&
		sed "/^cache-size=/d" bigkeyrc &&
		echo "repo.readme=:file-1" &&
		echo "repo.about-filter=exec:$PWD/fd-filter"
	} >fdrc &&
	cgit_rc fdrc "url=bigpage/about/" >fd.out &&
	grep -F "cache3/00000000.lock on fd 1." fd.out &&
	grep -F "fd.out on fd none." fd.out
'

# A refresh that cannot take the lock serves the expired copy, and says why
# unless the reason is another request doing the refresh. A line appended
# to the copy shows that it is what was served.
test_expect_success SANITY 'a refresh that cannot lock serves the expired slot and says why' '
	rm -rf cache3 && mkdir cache3 &&
	cgit_rc bigkeyrc "url=bigpage/" >/dev/null &&
	slot=$(ls cache3) &&
	echo "the expired copy" >>"cache3/$slot" &&
	touch -t 200001010000 "cache3/$slot" &&
	test_when_finished "chmod u+w cache3" &&
	chmod a-w cache3 &&
	cgit_rc bigkeyrc "url=bigpage/" >stale.out 2>stale.err &&
	grep "the expired copy" stale.out &&
	grep "Unable to open .*\.lock: Permission denied" stale.err
'

# A refresh that finds the lock held is another request already at work on
# the slot, so the expired copy is served and nothing is logged.
test_expect_success PERL_RUNS 'a refresh finding the lock held serves the expired slot quietly' '
	write_hold_lock &&
	rm -rf cache3 && mkdir cache3 &&
	cgit_ok bigkeyrc "url=bigpage/" >/dev/null &&
	slot=$(ls cache3) &&
	echo "the expired copy" >>"cache3/$slot" &&
	touch -t 200001010000 "cache3/$slot" &&
	CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/" \
	./hold-lock "cache3/$slot.lock" cgit >held.out 2>held.err &&
	test_grep "^Status: 200" held.out &&
	test_grep "the expired copy" held.out &&
	test_must_be_empty held.err
'

# Valgrind cannot start under ulimit -f 0, as it writes a file for its own
# gdbserver before cgit runs, so the tests that set that limit run without it.
test -n "$CGIT_VALGRIND" || test_set_prereq NO_VALGRIND

# A lock file that cannot take the key goes with its lock, rather than
# staying behind locked while the page renders uncached. The limit is on
# files, so the log and cgit's exit status go through the pipe as well.
test_expect_success NO_VALGRIND 'a slot lock that cannot take its key is let go' '
	rm -rf cache3 && mkdir cache3 &&
	(
		trap "" XFSZ &&
		ulimit -f 0 &&
		{ cgit_rc bigkeyrc "url=bigpage/" 2>&1; echo "exit status $?"; }
	) | cat >nokey.out &&
	test_grep "^exit status 0$" nokey.out &&
	test_grep "</html>" nokey.out &&
	test_grep "Unable to write .*\.lock" nokey.out &&
	test_dir_is_empty cache3
'

# A scan is cached in a file named after a hash of its path and settings, so
# a first scan finds out what that name is. The index page is left uncached,
# which makes each request read the list afresh.
test_expect_success 'set up a cached scan of two repositories' '
	mkrepo scanned/one 1 &&
	mkrepo scanned/two 1 &&
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache5" &&
		echo "cache-size=64" &&
		echo "cache-index-ttl=0" &&
		echo "scan-path=$PWD/scanned"
	} >scanrc &&
	rm -rf cache5 && mkdir cache5 &&
	cgit_rc scanrc "url=" >/dev/null &&
	scanned_rc=$(ls cache5) &&
	test_path_is_file "cache5/$scanned_rc"
'

# A list that cannot be written leaves the request with the scan it made
# before the write failed, so each repository is listed once and nothing is
# left in the cache-root. The limit is on files, so the log and the exit
# status go through the pipe with the page.
test_expect_success NO_VALGRIND 'a scan whose list cannot be written lists each repository once' '
	rm -rf cache9 && mkdir cache9 &&
	sed "s|^cache-root=.*|cache-root=$PWD/cache9|" scanrc >unwritablerc &&
	(
		trap "" XFSZ &&
		ulimit -f 0 &&
		{ cgit_rc unwritablerc "url=" 2>&1; echo "exit status $?"; }
	) | cat >unwritable.out &&
	test_grep "^exit status 0$" unwritable.out &&
	test_grep "Unable to write .*/cache9/rc-[0-9a-f]*\.lock" unwritable.out &&
	test $(grep -c ">one</a>" unwritable.out) = 1 &&
	test $(grep -c ">two</a>" unwritable.out) = 1 &&
	test_dir_is_empty cache9
'

# A request that finds the scan's lock held and has no list to go on scans
# in its own process, leaving the file to the holder. That is only another
# request at work and no cause for a line in the log.
test_expect_success PERL_RUNS 'a scan finding its lock held runs uncached' '
	write_hold_lock &&
	rm "cache5/$scanned_rc" &&
	CGIT_CONFIG="$PWD/scanrc" QUERY_STRING="url=" \
	./hold-lock "cache5/$scanned_rc.lock" cgit >held.out 2>held.err &&
	grep ">one</a>" held.out &&
	grep ">two</a>" held.out &&
	test_path_is_missing "cache5/$scanned_rc" &&
	test_must_be_empty held.err
'

# A run that died before its rename leaves a lock file behind, here longer
# than anything the next run writes and ending in a repository and a marker.
# Emptying the file before reuse keeps that tail out of both caches. The page
# that rebuilt the list is drawn from the scan in memory, so it is the file,
# and a second request reading it back, that would show the ghost.
test_expect_success 'a stale lock file reaches neither the scan nor a slot' '
	{
		printf "%01024d\n" 0 0 0 0 &&
		echo "repo.url=ghost" &&
		echo "repo.path=$PWD/scanned/one/.git"
	} >stale.lock &&
	rm -f "cache5/$scanned_rc" &&
	cp stale.lock "cache5/$scanned_rc.lock" &&
	cgit_ok scanrc "url=" >scan.out &&
	test_grep ">one</a>" scan.out &&
	test_grep "^repo.url=one$" "cache5/$scanned_rc" &&
	test_grep ! "^repo.url=ghost$" "cache5/$scanned_rc" &&
	cgit_ok scanrc "url=" >scan.out &&
	test_grep ">one</a>" scan.out &&
	test_grep ! ">ghost</a>" scan.out &&
	rm -rf cache3 && mkdir cache3 &&
	cgit_ok bigkeyrc "url=bigpage/" >page.out &&
	slot=$(ls cache3) &&
	rm "cache3/$slot" &&
	{ cat page.out page.out && echo "STALE-TAIL"; } >"cache3/$slot.lock" &&
	cgit_ok bigkeyrc "url=bigpage/" >/dev/null &&
	cgit_ok bigkeyrc "url=bigpage/" >replay.out &&
	test_cmp page.out replay.out
'

# The cached list is read back as trusted config, so a config that would
# scan differently keeps a list of its own in the same cache-root, here one
# that takes only the repositories carrying an export marker.
test_expect_success 'a config that scans differently keeps its own list' '
	touch scanned/one/.git/export-ok &&
	{
		echo "strict-export=export-ok" &&
		cat scanrc
	} >strictrc &&
	cgit_rc scanrc "url=" >all.out &&
	cgit_rc strictrc "url=" >strict.out &&
	grep ">two</a>" all.out &&
	grep ">one</a>" strict.out &&
	! grep ">two</a>" strict.out
'

# A list scanned under trust-scan-config carries the filters a repository
# set, and the list is read back with no trust check, so a config that does
# not trust its scans has to keep a list of its own. Pages are left uncached,
# since the page cache is keyed on the request alone.
test_expect_success 'a config that trusts its scans keeps its own list' '
	mkrepo trusted/one 1 &&
	echo "commit-filter=exec:$FILTER_DIRECTORY/dump.sh" >trusted/one/.git/cgitrc &&
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache7" &&
		echo "cache-size=64" &&
		echo "cache-dynamic-ttl=0" &&
		echo "scan-path=$PWD/trusted"
	} >distrustrc &&
	{
		echo "trust-scan-config=1" &&
		cat distrustrc
	} >trustrc &&
	rm -rf cache7 && mkdir cache7 &&
	cgit_ok trustrc "url=one/commit/" >trust.out &&
	test_grep "<div class=.commit-subject.>COMMIT 1" trust.out &&
	cgit_rc distrustrc "url=one/commit/" >distrust.out 2>distrust.err &&
	test_grep "^Status: 200" distrust.out &&
	test_grep "<div class=.commit-subject.>commit 1" distrust.out &&
	test_grep "Ignoring commit-filter in $PWD/trusted/one/.git/: trust-scan-config is not set" distrust.err &&
	ls cache7/rc-* >lists &&
	test_line_count = 2 lists
'

# Every setting the scan reads, or hands each repository as a default, goes
# into the name of the list, so changing any one of them, or the path, must
# start a list of its own.
test_expect_success 'every setting the scan depends on keeps its own list' '
	mkrepo keyed/one 1 &&
	echo one >projects &&
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache8" &&
		echo "cache-size=64" &&
		echo "scan-path=$PWD/keyed"
	} >keyedrc &&
	rm -rf cache8 && mkdir cache8 &&
	cgit_ok keyedrc "url=" >/dev/null &&
	sed "s|^scan-path=.*|scan-path=$PWD/keyed/one|" keyedrc >keyed-setting-rc &&
	cgit_ok keyed-setting-rc "url=" >/dev/null &&
	ls cache8/rc-* >lists &&
	test_line_count = 2 lists &&
	lists=2 &&
	while read setting
	do
		{
			echo "$setting" &&
			cat keyedrc
		} >keyed-setting-rc &&
		cgit_ok keyed-setting-rc "url=" >/dev/null &&
		lists=$((lists + 1)) &&
		ls cache8/rc-* >lists &&
		test_line_count = $lists lists ||
		{ echo "$setting shares a list" && return 1; }
	done <<-EOF
	project-list=$PWD/projects
	strict-export=git-daemon-export-ok
	section=Sec
	module-link=https://example.org/%s
	clone-url=https://example.org/\$CGIT_REPO_URL
	scan-hidden-path=1
	case-sensitive-sort=0
	remove-suffix=1
	section-from-path=1
	enable-git-config=1
	trust-scan-config=1
	snapshots=zip
	max-stats=year
	branch-sort=age
	commit-sort=date
	enable-blame=1
	enable-commit-graph=1
	enable-commit-trailers=1
	enable-follow-links=1
	enable-gitmodules-links=1
	enable-html-serving=1
	enable-log-filecount=1
	enable-log-linecount=1
	enable-mailmap=0
	enable-remote-branches=1
	enable-subject-links=1
	about-filter=exec:$FILTER_DIRECTORY/dump.sh
	commit-filter=exec:$FILTER_DIRECTORY/dump.sh
	email-filter=exec:$FILTER_DIRECTORY/dump.sh
	source-filter=exec:$FILTER_DIRECTORY/dump.sh
	trailer-filter=exec:$FILTER_DIRECTORY/dump.sh
	EOF
'

# A negative ttl never expires, so a list however old is served and kept. A
# rebuild runs in a child holding the request's copy of fd 9, so cat reaching
# the end of that pipe means any rebuild is done, as the positive ttl after
# it shows. The ttl is not part of the list's name, so both configs share
# the list, which is scanned afresh here.
test_expect_success 'a negative cache-scan-ttl never expires the list' '
	{
		echo "cache-scan-ttl=-1" &&
		cat scanrc
	} >foreverrc &&
	rm -f "cache5/$scanned_rc" &&
	cgit_ok scanrc "url=" >/dev/null &&
	test_path_is_file "cache5/$scanned_rc" &&
	{
		echo "repo.url=kept" &&
		echo "repo.path=$PWD/scanned/one/.git"
	} >>"cache5/$scanned_rc" &&
	touch -t 200001010000 "cache5/$scanned_rc" &&
	{ cgit_rc foreverrc "url=" 9>&1 >forever.out 2>forever.err; echo $? >forever.status; } | cat &&
	test "$(cat forever.status)" = 0 &&
	test_must_be_empty forever.err &&
	test_grep ">kept</a>" forever.out &&
	test_grep "^repo.url=kept$" "cache5/$scanned_rc" &&
	{ cgit_rc scanrc "url=" 9>&1 >rescan.out 2>rescan.err; echo $? >rescan.status; } | cat &&
	test "$(cat rescan.status)" = 0 &&
	test_must_be_empty rescan.err &&
	test_grep ">kept</a>" rescan.out &&
	test_grep "^repo.url=one$" "cache5/$scanned_rc" &&
	test_grep ! "^repo.url=kept$" "cache5/$scanned_rc"
'

# Starts a request in the background whose stdin, stdout and stderr are pipes,
# and returns once the request is over, which is only when every process
# holding those pipes has let go, or fails after 30 seconds. The rescan child
# keeps fd 9, so the reader of that pipe, and with it the background job, ends
# only when the child does.
start_rescan_request() {
	rm -f rescan.done
	{
		{
			cat /dev/zero 2>/dev/null |
			cgit_rc pipelistrc "url=" 2>&1 |
			cat >rescan.out &&
			touch rescan.done
		} 9>&1 >/dev/null | cat
	} &
	rescan_pid=$!
	n=0
	while ! test -f rescan.done && test $n -lt 30
	do
		sleep 1
		n=$((n + 1))
	done
	test -f rescan.done
}

# Writes the project list into the pipe the child is held on, which fails
# when no child is there to read it, and waits for the child to end.
finish_rescan() {
	"$PERL_PATH" -e 'alarm 30; open(my $list, ">", shift) or die "$!\n"; print $list "one\n"' pipelist
	finish_status=$?
	wait "$rescan_pid"
	return $finish_status
}

rescan_behind_request() {
	start_rescan_request
	rescan_status=$?
	finish_rescan &&
	return $rescan_status
}

# An expired list is rebuilt by a child forked off the request, which answers
# from the stale list meanwhile. A web server reads the request's stdout until
# every holder is gone, so the child drops it, along with stdin and stderr. A
# project list that is a pipe holds the child until the test writes the list
# into it, after the request has ended.
test_expect_success PIPE,PERL_RUNS 'a background rescan lets the request finish' '
	mkrepo piped/one 1 &&
	echo one >pipelist &&
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache10" &&
		echo "cache-size=64" &&
		echo "cache-index-ttl=0" &&
		echo "project-list=$PWD/pipelist" &&
		echo "scan-path=$PWD/piped"
	} >pipelistrc &&
	rm -rf cache10 && mkdir cache10 &&
	cgit_ok pipelistrc "url=" >/dev/null &&
	list=$(ls cache10/rc-*) &&
	{
		echo "repo.url=kept" &&
		echo "repo.path=$PWD/piped/one/.git"
	} >>"$list" &&
	touch -t 200001010000 "$list" &&
	rm pipelist &&
	mkfifo pipelist &&
	rescan_behind_request &&
	test_grep ">kept</a>" rescan.out &&
	test_grep "</html>" rescan.out &&
	test_grep ! "^\[cgit\]" rescan.out &&
	test_grep "^repo.url=one$" "$list" &&
	test_grep ! "^repo.url=kept$" "$list"
'

# A repository the scan found without a description goes into the list
# without one, so its pages do not describe themselves with the placeholder.
test_expect_success 'a cached scan leaves the description placeholder out' '
	rm -rf cache5 && mkdir cache5 &&
	cgit_ok scanrc "url=" >/dev/null &&
	rc=$(ls cache5/rc-*) &&
	test_grep "^repo.url=one$" "$rc" &&
	test_grep ! "^repo.desc=" "$rc" &&
	cgit_ok scanrc "url=" >tmp &&
	test_grep "\[no description\]" tmp &&
	cgit_ok scanrc "url=one/" >tmp &&
	test_grep "data-page=.summary. data-repo=.one." tmp &&
	test_grep ! "<meta name=.description." tmp
'

# The list is read back under the global settings, so what a repository set
# against them has to come back with it. This one sorts its branches by name
# where the site sorts by age, turns the site's about filter off and links a
# submodule of its own. A list scanned without the global filter is no good
# to a config that sets one, as it says nothing of the filter turned off.
test_expect_success 'a cached scan keeps what a repository sets' '
	mkrepo kept/one 1 &&
	sub=$(git -C kept/one rev-parse HEAD) &&
	(
		cd kept/one &&
		git update-index --add --cacheinfo 160000,$sub,submod &&
		git commit -m gitlink &&
		git branch aaa &&
		newest=$(echo newest | GIT_COMMITTER_DATE="@2000000000 +0000" git commit-tree HEAD^{tree} -p HEAD) &&
		git update-ref refs/heads/zzz $newest
	) &&
	{
		echo "branch-sort=name" &&
		echo "about-filter=" &&
		echo "readme=:file-1" &&
		echo "module-link.submod=https://example.org/s/?id=%s"
	} >kept/one/.git/cgitrc &&
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache6" &&
		echo "cache-size=64" &&
		echo "trust-scan-config=1" &&
		echo "branch-sort=age" &&
		echo "scan-path=$PWD/kept"
	} >unfilteredrc &&
	{
		echo "about-filter=exec:$FILTER_DIRECTORY/dump.sh" &&
		cat unfilteredrc
	} >keptrc &&
	rm -rf cache6 && mkdir cache6 &&
	cgit_rc unfilteredrc "url=" >/dev/null &&
	cgit_rc keptrc "url=" >/dev/null &&
	cgit_rc keptrc "url=one/refs/" >tmp &&
	sed -n "s/^<tr class=.branch.><td><a [^>]*>\([a-z]*\)<.*/\1/p" tmp >branches &&
	test_write_lines aaa master zzz >expect &&
	test_cmp expect branches &&
	cgit_rc keptrc "url=one/about/" >tmp &&
	grep "<pre class=.plaintext.>1$" tmp &&
	cgit_rc keptrc "url=one/tree/" >tmp &&
	grep "href=.https://example.org/s/?id=$sub." tmp
'

# A cache-size below zero turns the page cache off, and the scan with it, so
# the cache directory stays empty.
test_expect_success 'a negative cache-size leaves the scan uncached' '
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache-negative" &&
		echo "cache-size=-1" &&
		echo "scan-path=$PWD/scanned"
	} >negativerc &&
	rm -rf cache-negative && mkdir cache-negative &&
	cgit_rc negativerc "url=" >tmp &&
	grep ">one</a>" tmp &&
	test_dir_is_empty cache-negative
'

test_done