Diff options
Diffstat (limited to '')
| -rwxr-xr-x | tests/t0003-cache.sh | 249 |
1 file changed, 193 insertions, 56 deletions
diff --git a/tests/t0003-cache.sh b/tests/t0003-cache.sh index 06cdd99..e07fcd0 100755 --- a/tests/t0003-cache.sh +++ b/tests/t0003-cache.sh @@ -2,8 +2,6 @@ # Exercises the cache. The first three tests set cache-size to nothing, one # slot and the full table in turn, then count what the requests left behind. -# The rest cover a page longer than the output buffer and a key too long to -# be read back. test_description='Validate cache' . ./setup.sh @@ -31,8 +29,7 @@ test_expect_success 'verify cache-size=0' ' rm -f cache/* && set_cache_size 0 && request_all_pages && - ls cache >output && - test_line_count = 0 output + test_dir_is_empty cache ' test_expect_success 'verify cache-size=1' ' @@ -79,29 +76,10 @@ test_expect_success 'set up a repo with a page larger than the output buffer' ' rm -rf cache2 && mkdir cache2 ' -# The second request lands while the first is inside its filter, which says -# so through a marker file, since a fresh binary can take a moment to start. -overlap_requests() { - n=0 - rm -f filter-running - CGIT_CONFIG="$PWD/lockrc" QUERY_STRING="url=bigpage/about/" cgit >first.out 2>first.err & - while ! test -f filter-running && test $n -lt 10 - do - sleep 1 - n=$((n + 1)) - done - CGIT_CONFIG="$PWD/lockrc" QUERY_STRING="url=bigpage/log/" cgit >second.out 2>second.err - wait -} - -bigpage_query() { - CGIT_CONFIG="$PWD/bigrc" QUERY_STRING="$1" cgit -} - test_expect_success 'the first request fills the slot and the second replays it' ' - bigpage_query "url=bigpage/tree/big.txt" >big.first && + cgit_rc bigrc "url=bigpage/tree/big.txt" >big.first && test $(wc -c <big.first) -gt 65536 && - bigpage_query "url=bigpage/tree/big.txt" >big.second && + cgit_rc bigrc "url=bigpage/tree/big.txt" >big.second && strip_headers <big.first >big.first.body && strip_headers <big.second >big.second.body && test_cmp big.first.body big.second.body @@ -109,7 +87,7 @@ test_expect_success 'the first request fills the slot and the second replays it' test_expect_success 'the cached body matches one generated with the cache off' ' sed -e "s/^cache-size=64$/cache-size=0/" bigrc >bignocache && - CGIT_CONFIG="$PWD/bignocache" QUERY_STRING="url=bigpage/tree/big.txt" cgit >big.nocache && + cgit_rc bignocache "url=bigpage/tree/big.txt" >big.nocache && strip_headers <big.nocache >big.nocache.body && test_cmp big.nocache.body big.second.body ' @@ -126,15 +104,14 @@ test_expect_success 'an over-long cache key leaves no slot behind' ' rm -rf cache3 && mkdir cache3 && sed -e "s|^cache-root=.*|cache-root=$PWD/cache3|" bigrc >bigkeyrc && long=$(awk "BEGIN{s=\"\";for(i=0;i<6000;i++)s=s \"k\"; print s}") && - CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/&q=$long" cgit >key.out 2>key.err && + cgit_rc bigkeyrc "url=bigpage/&q=$long" >key.out 2>/dev/null && grep "</html>" key.out && - ls cache3 >key.slots && - test_line_count = 0 key.slots + test_dir_is_empty cache3 ' test_expect_success 'an ordinary key still fills a slot' ' rm -rf cache3 && mkdir cache3 && - CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/" cgit >/dev/null && + cgit_rc bigkeyrc "url=bigpage/" >/dev/null && ls cache3 >key.slots && test_line_count = 1 key.slots ' @@ -160,11 +137,10 @@ test_expect_success 'set up a repo missing a parent object' ' ' test_expect_success 'an error after output began ends the page with it and caches nothing' ' - CGIT_CONFIG="$PWD/brokenrc" QUERY_STRING="url=broken/commit/&id=$tip" cgit >broken.out && + cgit_rc brokenrc "url=broken/commit/&id=$tip" >broken.out && grep "Not a commit" broken.out && test $(grep -c "^Status:" broken.out) = 1 && - ls cache4 >broken.slots && - test_line_count = 0 broken.slots + test_dir_is_empty cache4 ' # An error page reports a condition the repository may grow out of, and a @@ -176,20 +152,18 @@ test_expect_success 'an error page leaves no slot behind' ' CGIT_CONFIG="$PWD/bigkeyrc" \ QUERY_STRING="url=bigpage/commit/&id=0123456789abcdef0123456789abcdef01234567" cgit >error.out && grep "Not a commit" error.out && - ls cache3 >error.slots && - test_line_count = 0 error.slots + test_dir_is_empty cache3 ' # The dumb transport serves files that already sit on the disk, so a copy in # a slot would only double the disk they take. test_expect_success 'a clone file leaves no slot behind' ' rm -rf cache3 && mkdir cache3 && - CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/info/refs" cgit >clone.out && + cgit_rc bigkeyrc "url=bigpage/info/refs" >clone.out && grep "refs/heads/master" clone.out && - CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/objects/info/packs" cgit >packs.out && + cgit_rc bigkeyrc "url=bigpage/objects/info/packs" >packs.out && grep "^Status: 200" packs.out && - ls cache3 >clone.slots && - test_line_count = 0 clone.slots + test_dir_is_empty cache3 ' # A response past cache-max-slot-size is served whole but kept nowhere, both @@ -202,25 +176,23 @@ test_expect_success 'a response over cache-max-slot-size is served but not kept' echo "cache-max-slot-size=32" && cat bigkeyrc } >slotsizerc && - CGIT_CONFIG="$PWD/slotsizerc" QUERY_STRING="url=bigpage/tree/big.txt" cgit >big.unkept && + cgit_rc slotsizerc "url=bigpage/tree/big.txt" >big.unkept && tail -c 200 big.unkept | grep "</html>" && - ls cache3 >unkept.slots && - test_line_count = 0 unkept.slots && - CGIT_CONFIG="$PWD/slotsizerc" QUERY_STRING="url=bigpage/" cgit >/dev/null && + test_dir_is_empty cache3 && + cgit_rc slotsizerc "url=bigpage/" >/dev/null && ls cache3 >kept.slots && test_line_count = 1 kept.slots ' test_expect_success 'an expired slot is dropped when its refill is too large' ' rm -rf cache3 && mkdir cache3 && - CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/tree/big.txt" cgit >/dev/null && + cgit_rc bigkeyrc "url=bigpage/tree/big.txt" >/dev/null && ls cache3 >filled.slots && test_line_count = 1 filled.slots && touch -t 200001010000 cache3/* && - CGIT_CONFIG="$PWD/slotsizerc" QUERY_STRING="url=bigpage/tree/big.txt" cgit >big.refill && + cgit_rc slotsizerc "url=bigpage/tree/big.txt" >big.refill && tail -c 200 big.refill | grep "</html>" && - ls cache3 >refill.slots && - test_line_count = 0 refill.slots + test_dir_is_empty cache3 ' # The summary page keeps a ttl of its own, while every other page that follows @@ -232,12 +204,11 @@ test_expect_success 'pages following a branch take the dynamic ttl' ' echo "cache-dynamic-ttl=0" && cat bigkeyrc } >dynamicrc && - CGIT_CONFIG="$PWD/dynamicrc" QUERY_STRING="url=bigpage/log/" cgit >/dev/null && - CGIT_CONFIG="$PWD/dynamicrc" QUERY_STRING="url=bigpage/log/&h=master" cgit >/dev/null && - ls cache3 >dynamic.slots && - test_line_count = 0 dynamic.slots && - CGIT_CONFIG="$PWD/dynamicrc" QUERY_STRING="url=bigpage/" cgit >/dev/null && - CGIT_CONFIG="$PWD/dynamicrc" QUERY_STRING="url=bigpage/summary/" cgit >/dev/null && + cgit_rc dynamicrc "url=bigpage/log/" >/dev/null && + cgit_rc dynamicrc "url=bigpage/log/&h=master" >/dev/null && + test_dir_is_empty cache3 && + cgit_rc dynamicrc "url=bigpage/" >/dev/null && + cgit_rc dynamicrc "url=bigpage/summary/" >/dev/null && ls cache3 >summary.slots && test_line_count = 2 summary.slots ' @@ -251,17 +222,37 @@ test_expect_success 'a failed final write still delivers the page' ' ( trap "" XFSZ && ulimit -f 2 && - CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/" cgit 2>flush.err + cgit_rc bigkeyrc "url=bigpage/" 2>flush.err ) | cat >flush.out && grep "^Status: 200" flush.out && grep "</html>" flush.out && ! grep "called many times" flush.err ' +# The second request lands while the first is inside its filter, which says +# so through a marker file, since a fresh binary can take a moment to start. +start_first_request() { + n=0 + rm -f filter-running + cgit_rc lockrc "url=bigpage/about/" >first.out 2>first.err & + while ! test -f filter-running && test $n -lt 10 + do + sleep 1 + n=$((n + 1)) + done +} + +overlap_requests() { + start_first_request + cgit_rc lockrc "url=bigpage/log/" >/dev/null 2>second.err + wait +} + # The fill writes the page into the lock file through stdout, and an exec # filter puts its own pipe on stdout while it runs. The slot lock has to # outlast that, or a second request on the slot rebuilds it under the first -# and the first visitor gets the two pages spliced together. +# and the first visitor gets the two pages spliced together. The second one +# finding the lock held is ordinary contention, which logs nothing. test_expect_success 'a slot stays locked while an exec filter runs' ' rm -rf cache3 && mkdir cache3 && write_script slow-filter.sh <<-EOF && @@ -278,7 +269,153 @@ test_expect_success 'a slot stays locked while an exec filter runs' ' overlap_requests && test $(grep -c "^Status:" first.out) = 1 && grep "id=.about." first.out && - ! grep "Unable to publish" first.err + ! grep "Unable to publish" first.err && + ! grep "Unable to lock" second.err +' + +# A request that opens the lock file just before its holder renames it into +# place as the live slot goes on to lock the slot itself. strace holds the +# second request's flock back until the first has finished, and the check +# that the path still names the locked file turns that lock down, so the +# slot keeps the first request's key. strace needs ptrace and a version that +# can delay a call, which containers and older systems may not give. +test_lazy_prereq STRACE_DELAY ' + strace -o /dev/null -e inject=flock:delay_enter=1 true +' + +test_expect_success STRACE_DELAY 'a lock taken after the rename leaves the slot alone' ' + rm -rf cache3 && mkdir cache3 && + start_first_request && + strace -o /dev/null -e inject=flock:delay_enter=4000000 \ + -E CGIT_CONFIG="$PWD/lockrc" -E QUERY_STRING="url=bigpage/log/" \ + cgit >second.out 2>second.err && + wait && + grep "id=.about." first.out && + grep "</html>" second.out && + tr "\000" "\n" <"cache3/$(ls cache3)" | head -n 1 >slot.key && + grep "about" slot.key && + ! grep "log" slot.key +' + +# An flock lasts until the last copy of its descriptor closes, so a filter +# may hold the slot's lock file only as the stdout it writes the page to. A +# stray copy would keep the slot locked for as long as the filter, or +# anything it leaves running, lives on. +test_expect_success PERL 'a filter holds the lock file only as its stdout' ' + rm -rf cache3 && mkdir cache3 && + write_script fd-filter "$PERL_PATH" <<-\EOF && + use POSIX qw(fstat); + 1 while <STDIN>; + my @lock = stat $ENV{LOCK_FILE} or die "$!\n"; + my @fds = grep { + my @fd = fstat($_); + @fd && $fd[0] == $lock[0] && $fd[1] == $lock[1] + } 0 .. 255; + print "lock file on fd @fds.\n"; + EOF + { + echo "cache-size=1" && + sed "/^cache-size=/d" bigkeyrc && + echo "repo.readme=:file-1" && + echo "repo.about-filter=exec:$PWD/fd-filter" + } >fdrc && + LOCK_FILE="$PWD/cache3/00000000.lock" CGIT_CONFIG="$PWD/fdrc" \ + QUERY_STRING="url=bigpage/about/" cgit >fd.out && + grep -F "lock file on fd 1." fd.out +' + +# A refresh that cannot take the lock serves the expired copy, and says why +# unless the reason is another request doing the refresh. A line appended +# to the copy shows that it is what was served. +test_expect_success SANITY 'a refresh that cannot lock serves the expired slot and says why' ' + rm -rf cache3 && mkdir cache3 && + cgit_rc bigkeyrc "url=bigpage/" >/dev/null && + slot=$(ls cache3) && + echo "the expired copy" >>"cache3/$slot" && + touch -t 200001010000 "cache3/$slot" && + test_when_finished "chmod u+w cache3" && + chmod a-w cache3 && + cgit_rc bigkeyrc "url=bigpage/" >stale.out 2>stale.err && + grep "the expired copy" stale.out && + grep "Unable to open .*\.lock: Permission denied" stale.err +' + +# A lock file that cannot take the key goes with its lock, rather than +# staying behind locked while the page renders uncached. The limit is on +# files, so the log goes through the pipe as well. +test_expect_success 'a slot lock that cannot take its key is let go' ' + rm -rf cache3 && mkdir cache3 && + ( + trap "" XFSZ && + ulimit -f 0 && + cgit_rc bigkeyrc "url=bigpage/" 2>&1 + ) | cat >nokey.out && + grep "</html>" nokey.out && + grep "Unable to write .*\.lock" nokey.out && + test_dir_is_empty cache3 +' + +# A scan is cached in a file named after the hash of its path, so a first +# scan finds out what that name is. The index page is left uncached, which +# makes each request read the list afresh. +test_expect_success 'set up a cached scan of two repositories' ' + mkrepo scanned/one 1 && + mkrepo scanned/two 1 && + { + echo "virtual-root=/" && + echo "cache-root=$PWD/cache5" && + echo "cache-size=64" && + echo "cache-index-ttl=0" && + echo "scan-path=$PWD/scanned" + } >scanrc && + rm -rf cache5 && mkdir cache5 && + cgit_rc scanrc "url=" >/dev/null && + scanned_rc=$(ls cache5) && + test_path_is_file "cache5/$scanned_rc" +' + +# A request that finds the scan's lock held and has no list to go on scans +# in its own process, leaving the file to the holder. That is only another +# request at work and no cause for a line in the log. +test_expect_success PERL 'a scan finding its lock held runs uncached' ' + write_script hold-lock "$PERL_PATH" <<-\EOF && + use Fcntl qw(:flock); + open(my $lock, ">>", shift) or die "$!\n"; + flock($lock, LOCK_EX | LOCK_NB) or die "$!\n"; + exit(system(@ARGV) ? 1 : 0); + EOF + rm "cache5/$scanned_rc" && + CGIT_CONFIG="$PWD/scanrc" QUERY_STRING="url=" \ + ./hold-lock "cache5/$scanned_rc.lock" cgit >held.out 2>held.err && + grep ">one</a>" held.out && + grep ">two</a>" held.out && + test_path_is_missing "cache5/$scanned_rc" && + test_must_be_empty held.err +' + +# A run that died before its rename leaves a lock file behind, here longer +# than anything the next run writes and ending in a repository and a marker. +# Emptying the file before reuse keeps that tail out of both caches. +test_expect_success 'a stale lock file reaches neither the scan nor a slot' ' + { + printf "%01024d\n" 0 0 0 0 && + echo "repo.url=ghost" && + echo "repo.path=$PWD/scanned/one/.git" + } >stale.lock && + rm -f "cache5/$scanned_rc" && + cp stale.lock "cache5/$scanned_rc.lock" && + CGIT_CONFIG="$PWD/scanrc" QUERY_STRING="url=" cgit >scan.out && + grep ">one</a>" scan.out && + ! grep ">ghost</a>" scan.out && + rm -rf cache3 && mkdir cache3 && + CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/" cgit >page.out && + slot=$(ls cache3) && + rm "cache3/$slot" && + { cat page.out page.out && echo "STALE-TAIL"; } >"cache3/$slot.lock" && + CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/" cgit >/dev/null && + CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/" cgit >replay.out && + grep "</html>" replay.out && + ! grep "STALE-TAIL" replay.out ' test_done |