#!/bin/sh # Exercises the cache. The first three tests set cache-size to nothing, one # slot and the full table in turn, then count what the requests left behind. test_description='Validate cache' . ./setup.sh # The counting tests below visit the same pages, the index plus six pages in # each of two repositories, so the counts they expect stay tied to this one # list of thirteen requests. request_all_pages() { cgit_ok "$1" "url=" >/dev/null && for repo in foo bar do for page in "" /refs /tree /log /diff /patch do cgit_ok "$1" "url=$repo$page" >/dev/null || return 1 done done } test_expect_success 'a cache-size of 0 leaves no slot behind' ' rm -f cache/* && sed "s/^cache-size=.*/cache-size=0/" cgitrc >size0rc && request_all_pages size0rc && test_dir_is_empty cache ' # With one slot every request lands on the same file, so the key it holds is # all that keeps one page from being served for another. test_expect_success 'one slot serves each request its own page' ' rm -f cache/* && sed "s/^cache-size=.*/cache-size=1/" cgitrc >size1rc && cgit_ok size1rc "url=foo/log/" >/dev/null && cgit_rc size1rc "url=bar/refs/" >refs.out 2>refs.err && test_grep "^Status: 200" refs.out && test_grep "data-page=.refs. data-repo=.bar." refs.out && test_grep "log.out 2>log.err && test_grep "data-page=.log. data-repo=.foo." log.out && test_grep "
output && test_line_count = 1 output ' # The listing is a page like any other and takes a slot of its own, so a # fresh render of it would list fourteen slots. Thirteen on the second # request shows the first was replayed, which a dynamic ttl that never # expires keeps true however slow the run. test_expect_success 'a full table gives each request a slot of its own' ' rm -f cache/* && { echo "cache-dynamic-ttl=-1" && sed "s/^cache-size=.*/cache-size=1021/" cgitrc } >size1021rc && request_all_pages size1021rc && ls cache >output && test_line_count = 13 output && cgit_ok size1021rc "url=foo/ls_cache" >output.full && strip_headers output && test_line_count = 13 output && ls cache >output && test_line_count = 14 output && cgit_ok size1021rc "url=foo/ls_cache" >output.second && test_cmp output.full output.second ' # Page output is buffered, so a slot only holds the whole page if the buffer is # emptied before the size of what was generated is taken. The blob set up here # is larger than one buffer, so a missing flush would leave the cached copy cut # short. test_expect_success 'set up a repo with a page larger than the output buffer' ' mkrepo repos/bigpage 1 && ( cd repos/bigpage && awk "BEGIN{for(i=0;i<20000;i++) print \"line \" i \" of the big file\"}" >big.txt && git add big.txt && git commit -m "add big.txt" ) && { echo "virtual-root=/" && echo "cache-root=$PWD/cache2" && echo "cache-size=64" && echo "enable-tree-linenumbers=1" && echo "repo.url=bigpage" && echo "repo.path=$PWD/repos/bigpage/.git" } >bigrc && rm -rf cache2 && mkdir cache2 ' test_expect_success 'the first request fills the slot and the second replays it' ' cgit_rc bigrc "url=bigpage/tree/big.txt" >big.first && test $(wc -c big.second && strip_headers big.first.body && strip_headers big.second.body && test_cmp big.first.body big.second.body ' test_expect_success 'the cached body matches one generated with the cache off' ' sed -e "s/^cache-size=64$/cache-size=0/" bigrc >bignocache && cgit_rc bignocache "url=bigpage/tree/big.txt" >big.nocache && strip_headers big.nocache.body && test_cmp big.nocache.body big.second.body ' test_expect_success 'the page ends where it should, so nothing was dropped' ' tail -c 200 big.second.body | grep "" ' # A slot stores its key ahead of the content and only the first few kilobytes of # it are read back, so a key longer than that could never match. A request # carrying one used to leave a slot behind on every visit that nothing could # ever go on to use. test_expect_success 'an over-long cache key leaves no slot behind' ' rm -rf cache3 && mkdir cache3 && sed -e "s|^cache-root=.*|cache-root=$PWD/cache3|" bigrc >bigkeyrc && long=$(awk "BEGIN{s=\"\";for(i=0;i<6000;i++)s=s \"k\"; print s}") && cgit_rc bigkeyrc "url=bigpage/&q=$long" >key.out 2>/dev/null && grep "" key.out && test_dir_is_empty cache3 ' test_expect_success 'an ordinary key still fills a slot' ' rm -rf cache3 && mkdir cache3 && cgit_rc bigkeyrc "url=bigpage/" >/dev/null && ls cache3 >key.slots && test_line_count = 1 key.slots ' # An error page fired after rendering began abandons the fill with part of # the page already written into the lock file. That fragment must be # discarded along with the file, not replayed to the visitor behind the # error page. A commit whose parent object is missing renders its whole # info table before the diff machinery fails, which makes it the trigger. test_expect_success 'set up a repo missing a parent object' ' mkrepo repos/broken 2 && parent=$(git -C repos/broken rev-parse HEAD^) && tip=$(git -C repos/broken rev-parse HEAD) && rm "repos/broken/.git/objects/$(echo "$parent" | cut -c1-2)/$(echo "$parent" | cut -c3-)" && { echo "virtual-root=/" && echo "cache-root=$PWD/cache4" && echo "cache-size=64" && echo "repo.url=broken" && echo "repo.path=$PWD/repos/broken/.git" } >brokenrc && rm -rf cache4 && mkdir cache4 ' test_expect_success 'an error after output began ends the page with it and caches nothing' ' cgit_rc brokenrc "url=broken/commit/&id=$tip" >broken.out && grep "Not a commit" broken.out && test $(grep -c "^Status:" broken.out) = 1 && test_dir_is_empty cache4 ' # An error page reports a condition the repository may grow out of, and a # request pinned to an object id would cache it under the never-expiring # static ttl. Asking for a commit that does not exist yet must therefore # leave nothing behind, or the real page could never appear once it does. test_expect_success 'an error page leaves no slot behind' ' rm -rf cache3 && mkdir cache3 && CGIT_CONFIG="$PWD/bigkeyrc" \ QUERY_STRING="url=bigpage/commit/&id=0123456789abcdef0123456789abcdef01234567" cgit >error.out && grep "Not a commit" error.out && test_dir_is_empty cache3 ' # The dumb transport serves files that already sit on the disk, so a copy in # a slot would only double the disk they take. test_expect_success 'a clone file leaves no slot behind' ' rm -rf cache3 && mkdir cache3 && cgit_rc bigkeyrc "url=bigpage/info/refs" >clone.out && grep "refs/heads/master" clone.out && cgit_rc bigkeyrc "url=bigpage/objects/info/packs" >packs.out && grep "^Status: 200" packs.out && test_dir_is_empty cache3 ' # A response past cache-max-slot-size is served whole but kept nowhere, both # on a first fill and when it would have replaced an expired slot, which is # dropped along with it. The summary page stays under the limit set here # while the big blob page is well past it. test_expect_success 'a response over cache-max-slot-size is served but not kept' ' rm -rf cache3 && mkdir cache3 && { echo "cache-max-slot-size=32" && cat bigkeyrc } >slotsizerc && cgit_rc slotsizerc "url=bigpage/tree/big.txt" >big.unkept && tail -c 200 big.unkept | grep "" && test_dir_is_empty cache3 && cgit_rc slotsizerc "url=bigpage/" >/dev/null && ls cache3 >kept.slots && test_line_count = 1 kept.slots ' test_expect_success 'an expired slot is dropped when its refill is too large' ' rm -rf cache3 && mkdir cache3 && cgit_rc bigkeyrc "url=bigpage/tree/big.txt" >/dev/null && ls cache3 >filled.slots && test_line_count = 1 filled.slots && touch -t 200001010000 cache3/* && cgit_rc slotsizerc "url=bigpage/tree/big.txt" >big.refill && tail -c 200 big.refill | grep "" && test_dir_is_empty cache3 ' # The summary page keeps a ttl of its own, while every other page that follows # a branch rather than a pinned id takes cache-dynamic-ttl, whether or not its # url names the branch. test_expect_success 'pages following a branch take the dynamic ttl' ' rm -rf cache3 && mkdir cache3 && { echo "cache-dynamic-ttl=0" && cat bigkeyrc } >dynamicrc && cgit_rc dynamicrc "url=bigpage/log/" >/dev/null && cgit_rc dynamicrc "url=bigpage/log/&h=master" >/dev/null && test_dir_is_empty cache3 && cgit_rc dynamicrc "url=bigpage/" >/dev/null && cgit_rc dynamicrc "url=bigpage/summary/" >/dev/null && ls cache3 >summary.slots && test_line_count = 2 summary.slots ' # Only a full object id names a page that can never change, so a page pinned # to one alone takes the static ttl, which by default never expires. An # abbreviation or a branch name, in either id, can come to mean another # commit. test_expect_success 'only a page pinned to full ids takes the static ttl' ' rm -rf cache3 && mkdir cache3 && { echo "cache-dynamic-ttl=0" && cat bigkeyrc } >staticrc && first=$(git -C repos/bigpage rev-parse HEAD~1) && short=$(git -C repos/bigpage rev-parse --short HEAD~1) && for query in "commit/&id=$short" "commit/&id=master" "diff/&id=$first&id2=$short" "diff/&id=master&id2=$first" do cgit_ok staticrc "url=bigpage/$query" >/dev/null || return 1 done && test_dir_is_empty cache3 && cgit_ok staticrc "url=bigpage/commit/&id=$first" >/dev/null && ls cache3 >static.slots && test_line_count = 1 static.slots && cgit_ok staticrc "url=bigpage/diff/&id=$first&id2=$first" >/dev/null && ls cache3 >static.slots && test_line_count = 2 static.slots ' # A write into the lock file that fails on the page's last flush has to end # as one that fails mid page does, with the fill abandoned and the page # delivered from the lock file, rather than in a blank response. The limit # is on files alone, so the page goes to the visitor through a pipe. test_expect_success 'a failed final write still delivers the page' ' rm -rf cache3 && mkdir cache3 && ( trap "" XFSZ && ulimit -f 2 && cgit_rc bigkeyrc "url=bigpage/" 2>flush.err ) | cat >flush.out && grep "^Status: 200" flush.out && grep "" flush.out && ! grep "called many times" flush.err ' # The second request lands while the first is inside its filter, which says # so through a marker file, since a fresh binary can take a moment to start. start_first_request() { n=0 rm -f filter-running cgit_rc lockrc "url=bigpage/about/" >first.out 2>first.err & while ! test -f filter-running && test $n -lt 10 do sleep 1 n=$((n + 1)) done } overlap_requests() { start_first_request cgit_rc lockrc "url=bigpage/log/" >/dev/null 2>second.err wait } # The fill writes the page into the lock file through stdout, and an exec # filter puts its own pipe on stdout while it runs. The slot lock has to # outlast that, or a second request on the slot rebuilds it under the first # and the first visitor gets the two pages spliced together. The second one # finding the lock held is ordinary contention, which logs nothing. test_expect_success 'a slot stays locked while an exec filter runs' ' rm -rf cache3 && mkdir cache3 && write_script slow-filter.sh <<-EOF && touch "$PWD/filter-running" sleep 3 cat EOF { echo "cache-size=1" && sed "/^cache-size=/d" bigkeyrc && echo "repo.readme=:file-1" && echo "repo.about-filter=exec:$PWD/slow-filter.sh" } >lockrc && overlap_requests && test $(grep -c "^Status:" first.out) = 1 && grep "id=.about." first.out && ! grep "Unable to publish" first.err && ! grep "Unable to lock" second.err ' # A request that opens the lock file just before its holder renames it into # place as the live slot goes on to lock the slot itself. strace holds the # second request's flock back until the first has finished, and the check # that the path still names the locked file turns that lock down, so the # slot keeps the first request's key. strace needs ptrace and a version that # can delay a call, which some systems do not give. test_lazy_prereq STRACE_DELAY ' strace -o /dev/null -e inject=flock:delay_enter=1 true ' test_expect_success STRACE_DELAY 'a lock taken after the rename leaves the slot alone' ' rm -rf cache3 && mkdir cache3 && start_first_request && strace -o /dev/null -e inject=flock:delay_enter=4000000 \ -E CGIT_CONFIG="$PWD/lockrc" -E QUERY_STRING="url=bigpage/log/" \ cgit >second.out 2>second.err && wait && grep "id=.about." first.out && grep "" second.out && tr "\000" "\n" <"cache3/$(ls cache3)" | head -n 1 >slot.key && grep "about" slot.key && ! grep "log" slot.key ' # An flock lasts until the last copy of its descriptor closes, so a filter # may hold the slot's lock file only as the stdout it writes the page to. A # stray copy would keep the slot locked for as long as the filter, or # anything it leaves running, lives on. test_expect_success PERL 'a filter holds the lock file only as its stdout' ' rm -rf cache3 && mkdir cache3 && write_script fd-filter "$PERL_PATH" <<-\EOF && use POSIX qw(fstat); 1 while ; my @lock = stat $ENV{LOCK_FILE} or die "$!\n"; my @fds = grep { my @fd = fstat($_); @fd && $fd[0] == $lock[0] && $fd[1] == $lock[1] } 0 .. 255; print "lock file on fd @fds.\n"; EOF { echo "cache-size=1" && sed "/^cache-size=/d" bigkeyrc && echo "repo.readme=:file-1" && echo "repo.about-filter=exec:$PWD/fd-filter" } >fdrc && LOCK_FILE="$PWD/cache3/00000000.lock" CGIT_CONFIG="$PWD/fdrc" \ QUERY_STRING="url=bigpage/about/" cgit >fd.out && grep -F "lock file on fd 1." fd.out ' # A refresh that cannot take the lock serves the expired copy, and says why # unless the reason is another request doing the refresh. A line appended # to the copy shows that it is what was served. test_expect_success SANITY 'a refresh that cannot lock serves the expired slot and says why' ' rm -rf cache3 && mkdir cache3 && cgit_rc bigkeyrc "url=bigpage/" >/dev/null && slot=$(ls cache3) && echo "the expired copy" >>"cache3/$slot" && touch -t 200001010000 "cache3/$slot" && test_when_finished "chmod u+w cache3" && chmod a-w cache3 && cgit_rc bigkeyrc "url=bigpage/" >stale.out 2>stale.err && grep "the expired copy" stale.out && grep "Unable to open .*\.lock: Permission denied" stale.err ' # A lock file that cannot take the key goes with its lock, rather than # staying behind locked while the page renders uncached. The limit is on # files, so the log goes through the pipe as well. test_expect_success 'a slot lock that cannot take its key is let go' ' rm -rf cache3 && mkdir cache3 && ( trap "" XFSZ && ulimit -f 0 && cgit_rc bigkeyrc "url=bigpage/" 2>&1 ) | cat >nokey.out && grep "" nokey.out && grep "Unable to write .*\.lock" nokey.out && test_dir_is_empty cache3 ' # A scan is cached in a file named after a hash of its path and settings, so # a first scan finds out what that name is. The index page is left uncached, # which makes each request read the list afresh. test_expect_success 'set up a cached scan of two repositories' ' mkrepo scanned/one 1 && mkrepo scanned/two 1 && { echo "virtual-root=/" && echo "cache-root=$PWD/cache5" && echo "cache-size=64" && echo "cache-index-ttl=0" && echo "scan-path=$PWD/scanned" } >scanrc && rm -rf cache5 && mkdir cache5 && cgit_rc scanrc "url=" >/dev/null && scanned_rc=$(ls cache5) && test_path_is_file "cache5/$scanned_rc" ' # A request that finds the scan's lock held and has no list to go on scans # in its own process, leaving the file to the holder. That is only another # request at work and no cause for a line in the log. test_expect_success PERL 'a scan finding its lock held runs uncached' ' write_script hold-lock "$PERL_PATH" <<-\EOF && use Fcntl qw(:flock); open(my $lock, ">>", shift) or die "$!\n"; flock($lock, LOCK_EX | LOCK_NB) or die "$!\n"; exit(system(@ARGV) ? 1 : 0); EOF rm "cache5/$scanned_rc" && CGIT_CONFIG="$PWD/scanrc" QUERY_STRING="url=" \ ./hold-lock "cache5/$scanned_rc.lock" cgit >held.out 2>held.err && grep ">one" held.out && grep ">two" held.out && test_path_is_missing "cache5/$scanned_rc" && test_must_be_empty held.err ' # A run that died before its rename leaves a lock file behind, here longer # than anything the next run writes and ending in a repository and a marker. # Emptying the file before reuse keeps that tail out of both caches. The page # that rebuilt the list is drawn from the scan in memory, so it is the file, # and a second request reading it back, that would show the ghost. test_expect_success 'a stale lock file reaches neither the scan nor a slot' ' { printf "%01024d\n" 0 0 0 0 && echo "repo.url=ghost" && echo "repo.path=$PWD/scanned/one/.git" } >stale.lock && rm -f "cache5/$scanned_rc" && cp stale.lock "cache5/$scanned_rc.lock" && cgit_ok scanrc "url=" >scan.out && test_grep ">one" scan.out && test_grep "^repo.url=one$" "cache5/$scanned_rc" && test_grep ! "^repo.url=ghost$" "cache5/$scanned_rc" && cgit_ok scanrc "url=" >scan.out && test_grep ">one" scan.out && test_grep ! ">ghost" scan.out && rm -rf cache3 && mkdir cache3 && cgit_ok bigkeyrc "url=bigpage/" >page.out && slot=$(ls cache3) && rm "cache3/$slot" && { cat page.out page.out && echo "STALE-TAIL"; } >"cache3/$slot.lock" && cgit_ok bigkeyrc "url=bigpage/" >/dev/null && cgit_ok bigkeyrc "url=bigpage/" >replay.out && test_cmp page.out replay.out ' # The cached list is read back as trusted config, so a config that would # scan differently keeps a list of its own in the same cache-root, here one # that takes only the repositories carrying an export marker. test_expect_success 'a config that scans differently keeps its own list' ' touch scanned/one/.git/export-ok && { echo "strict-export=export-ok" && cat scanrc } >strictrc && cgit_rc scanrc "url=" >all.out && cgit_rc strictrc "url=" >strict.out && grep ">two" all.out && grep ">one" strict.out && ! grep ">two" strict.out ' # A list scanned under trust-scan-config carries the filters a repository # set, and the list is read back with no trust check, so a config that does # not trust its scans has to keep a list of its own. Pages are left uncached, # since the page cache is keyed on the request alone. test_expect_success 'a config that trusts its scans keeps its own list' ' mkrepo trusted/one 1 && echo "commit-filter=exec:$FILTER_DIRECTORY/dump.sh" >trusted/one/.git/cgitrc && { echo "virtual-root=/" && echo "cache-root=$PWD/cache7" && echo "cache-size=64" && echo "cache-dynamic-ttl=0" && echo "scan-path=$PWD/trusted" } >distrustrc && { echo "trust-scan-config=1" && cat distrustrc } >trustrc && rm -rf cache7 && mkdir cache7 && cgit_ok trustrc "url=one/commit/" >trust.out && test_grep "
COMMIT 1" trust.out && cgit_rc distrustrc "url=one/commit/" >distrust.out 2>distrust.err && test_grep "^Status: 200" distrust.out && test_grep "
commit 1" distrust.out && test_grep "Ignoring commit-filter in $PWD/trusted/one/.git/: trust-scan-config is not set" distrust.err && ls cache7/rc-* >lists && test_line_count = 2 lists ' # Every setting the scan reads, or hands each repository as a default, goes # into the name of the list, so changing any one of them, or the path, must # start a list of its own. test_expect_success 'every setting the scan depends on keeps its own list' ' mkrepo keyed/one 1 && echo one >projects && { echo "virtual-root=/" && echo "cache-root=$PWD/cache8" && echo "cache-size=64" && echo "scan-path=$PWD/keyed" } >keyedrc && rm -rf cache8 && mkdir cache8 && cgit_ok keyedrc "url=" >/dev/null && sed "s|^scan-path=.*|scan-path=$PWD/keyed/one|" keyedrc >keyed-setting-rc && cgit_ok keyed-setting-rc "url=" >/dev/null && ls cache8/rc-* >lists && test_line_count = 2 lists && lists=2 && while read setting do { echo "$setting" && cat keyedrc } >keyed-setting-rc && cgit_ok keyed-setting-rc "url=" >/dev/null && lists=$((lists + 1)) && ls cache8/rc-* >lists && test_line_count = $lists lists || { echo "$setting shares a list" && return 1; } done <<-EOF project-list=$PWD/projects strict-export=git-daemon-export-ok section=Sec module-link=https://example.org/%s clone-url=https://example.org/\$CGIT_REPO_URL scan-hidden-path=1 case-sensitive-sort=0 remove-suffix=1 section-from-path=1 enable-git-config=1 trust-scan-config=1 snapshots=zip max-stats=year branch-sort=age commit-sort=date enable-blame=1 enable-commit-graph=1 enable-commit-trailers=1 enable-follow-links=1 enable-gitmodules-links=1 enable-html-serving=1 enable-log-filecount=1 enable-log-linecount=1 enable-mailmap=0 enable-remote-branches=1 enable-subject-links=1 about-filter=exec:$FILTER_DIRECTORY/dump.sh commit-filter=exec:$FILTER_DIRECTORY/dump.sh email-filter=exec:$FILTER_DIRECTORY/dump.sh source-filter=exec:$FILTER_DIRECTORY/dump.sh trailer-filter=exec:$FILTER_DIRECTORY/dump.sh EOF ' # A negative ttl never expires, so a list however old is served and kept. A # rebuild runs in a child holding the request's copy of fd 9, so cat reaching # the end of that pipe means any rebuild is done, as the positive ttl after # it shows. test_expect_success 'a negative cache-scan-ttl never expires the list' ' { echo "cache-scan-ttl=-1" && cat scanrc } >foreverrc && { echo "repo.url=kept" && echo "repo.path=$PWD/scanned/one/.git" } >>"cache5/$scanned_rc" && touch -t 200001010000 "cache5/$scanned_rc" && cgit_rc foreverrc "url=" 9>&1 >forever.out | cat && grep ">kept" forever.out && grep "^repo.url=kept$" "cache5/$scanned_rc" && cgit_rc scanrc "url=" 9>&1 >/dev/null | cat && ! grep "^repo.url=kept$" "cache5/$scanned_rc" ' # A repository the scan found without a description goes into the list # without one, so its pages do not describe themselves with the placeholder. test_expect_success 'a cached scan leaves the description placeholder out' ' rm -rf cache5 && mkdir cache5 && cgit_rc scanrc "url=" >/dev/null && ! grep "^repo.desc=" cache5/rc-* && cgit_rc scanrc "url=" >tmp && grep "\[no description\]" tmp && cgit_rc scanrc "url=one/" >tmp && ! grep "kept/one/.git/cgitrc && { echo "virtual-root=/" && echo "cache-root=$PWD/cache6" && echo "cache-size=64" && echo "trust-scan-config=1" && echo "branch-sort=age" && echo "scan-path=$PWD/kept" } >unfilteredrc && { echo "about-filter=exec:$FILTER_DIRECTORY/dump.sh" && cat unfilteredrc } >keptrc && rm -rf cache6 && mkdir cache6 && cgit_rc unfilteredrc "url=" >/dev/null && cgit_rc keptrc "url=" >/dev/null && cgit_rc keptrc "url=one/refs/" >tmp && sed -n "s/^
]*>\([a-z]*\)<.*/\1/p" tmp >branches && test_write_lines aaa master zzz >expect && test_cmp expect branches && cgit_rc keptrc "url=one/about/" >tmp && grep "
1$" tmp &&
	cgit_rc keptrc "url=one/tree/" >tmp &&
	grep "href=.https://example.org/s/?id=$sub." tmp
'

# A cache-size below zero turns the page cache off, and the scan with it, so
# the cache directory stays empty.
test_expect_success 'a negative cache-size leaves the scan uncached' '
	{
		echo "virtual-root=/" &&
		echo "cache-root=$PWD/cache-negative" &&
		echo "cache-size=-1" &&
		echo "scan-path=$PWD/scanned"
	} >negativerc &&
	rm -rf cache-negative && mkdir cache-negative &&
	cgit_rc negativerc "url=" >tmp &&
	grep ">one" tmp &&
	test_dir_is_empty cache-negative
'

test_done