1$" first.out && test_grep "" first.out && test_must_be_empty first.err && test_grep "data-page=.log." second.out && test_grep "" second.out && test_must_be_empty second.err && tr "\000" "\n" slot.key && test_grep "about" slot.key && test_grep ! "log" slot.key ' # A request that opens the lock file just before its holder renames it into # place as the live slot goes on to lock the slot itself. strace holds the # second request's flock back until the first has finished, and the check # that the path still names the locked file turns that lock down, so the # slot keeps the first request's key. This is the only test of that check. # The test needs an strace that can trace the process and delay a call, and # skips where there is none. test_lazy_prereq STRACE_DELAY ' strace -o /dev/null -e trace=%file,flock -e inject=flock:delay_enter=1 true ' # Returns once the trace shows the second request has opened the lock file, # which the first request still holds. strace has to start cgit itself, so # the request is set up here rather than by cgit_rc. start_traced_request() { rm -f second.trace strace -o second.trace -e trace=%file,flock \ -e inject=flock:delay_enter=4000000 \ -E CGIT_CONFIG="$PWD/lockrc" -E QUERY_STRING="url=bigpage/log/" \ cgit >second.out 2>second.err & second_pid=$! n=0 while ! grep "00000000\.lock" second.trace >/dev/null 2>&1 && test $n -lt 30 do sleep 1 n=$((n + 1)) done grep "00000000\.lock" second.trace >/dev/null 2>&1 } overlap_traced_requests() { second_pid= start_first_request && start_traced_request overlap_status=$? finish_first_request || overlap_status=1 if test -n "$second_pid" then wait "$second_pid" || overlap_status=1 fi return $overlap_status } # The first request is let go as soon as the second has the lock file open, # well inside the delay, so the flock that follows succeeds on a file that # has become the slot. A trace showing it refused would mean the race was # never reached. strace pads a short call out to a column before its result. test_expect_success STRACE_DELAY 'a lock taken after the rename leaves the slot alone' ' rm -rf cache3 && mkdir cache3 && overlap_traced_requests && test_grep -E "^flock$.*$ += 0" second.trace && test_grep "

1$" first.out && test_grep "" second.out && tr "\000" "\n" slot.key && test_grep "about" slot.key && test_grep ! "log" slot.key ' # An flock lasts until the last copy of its descriptor closes, so a filter # may hold the slot's lock file only as the stdout it writes the page to. A # stray copy would keep the slot locked for as long as the filter, or # anything it leaves running, lives on. The visitor's stdout, put aside while # the fill has stdout, must not reach the filter at all, or one that outlives # the request would hold the visitor's connection open. The filter finds both # files from its own path, since a variable set for cgit_rc alone would not # portably reach cgit. test_expect_success PERL_RUNS 'a filter gets the lock file only as stdout and never the client' ' rm -rf cache3 && mkdir cache3 && write_script fd-filter "$PERL_PATH" <<-\EOF && use File::Basename qw(dirname); use POSIX qw(fstat); 1 while ; for my $file ("cache3/00000000.lock", "fd.out") { my $path = dirname($0) . "/$file"; my @file = stat $path or die "$path: $!\n"; my @fds = grep { my @fd = fstat($_); @fd && $fd[0] == $file[0] && $fd[1] == $file[1] } 0 .. 255; print "$file on fd ", @fds ? "@fds" : "none", ".\n"; } EOF { echo "cache-size=1" && sed "/^cache-size=/d" bigkeyrc && echo "repo.readme=:file-1" && echo "repo.about-filter=exec:$PWD/fd-filter" } >fdrc && cgit_rc fdrc "url=bigpage/about/" >fd.out && grep -F "cache3/00000000.lock on fd 1." fd.out && grep -F "fd.out on fd none." fd.out ' # A refresh that cannot take the lock serves the expired copy, and says why # unless the reason is another request doing the refresh. A line appended # to the copy shows that it is what was served. test_expect_success SANITY 'a refresh that cannot lock serves the expired slot and says why' ' rm -rf cache3 && mkdir cache3 && cgit_rc bigkeyrc "url=bigpage/" >/dev/null && slot=$(ls cache3) && echo "the expired copy" >>"cache3/$slot" && touch -t 200001010000 "cache3/$slot" && test_when_finished "chmod u+w cache3" && chmod a-w cache3 && cgit_rc bigkeyrc "url=bigpage/" >stale.out 2>stale.err && grep "the expired copy" stale.out && grep "Unable to open .*\.lock: Permission denied" stale.err ' # A refresh that finds the lock held is another request already at work on # the slot, so the expired copy is served and nothing is logged. test_expect_success PERL_RUNS 'a refresh finding the lock held serves the expired slot quietly' ' write_hold_lock && rm -rf cache3 && mkdir cache3 && cgit_ok bigkeyrc "url=bigpage/" >/dev/null && slot=$(ls cache3) && echo "the expired copy" >>"cache3/$slot" && touch -t 200001010000 "cache3/$slot" && CGIT_CONFIG="$PWD/bigkeyrc" QUERY_STRING="url=bigpage/" \ ./hold-lock "cache3/$slot.lock" cgit >held.out 2>held.err && test_grep "^Status: 200" held.out && test_grep "the expired copy" held.out && test_must_be_empty held.err ' # Valgrind cannot start under ulimit -f 0, as it writes a file for its own # gdbserver before cgit runs, so the tests that set that limit run without it. test -n "$CGIT_VALGRIND" || test_set_prereq NO_VALGRIND # A lock file that cannot take the key goes with its lock, rather than # staying behind locked while the page renders uncached. The limit is on # files, so the log and cgit's exit status go through the pipe as well. test_expect_success NO_VALGRIND 'a slot lock that cannot take its key is let go' ' rm -rf cache3 && mkdir cache3 && ( trap "" XFSZ && ulimit -f 0 && { cgit_rc bigkeyrc "url=bigpage/" 2>&1; echo "exit status $?"; } ) | cat >nokey.out && test_grep "^exit status 0$" nokey.out && test_grep "" nokey.out && test_grep "Unable to write .*\.lock" nokey.out && test_dir_is_empty cache3 ' # A scan is cached in a file named after a hash of its path and settings, so # a first scan finds out what that name is. The index page is left uncached, # which makes each request read the list afresh. test_expect_success 'set up a cached scan of two repositories' ' mkrepo scanned/one 1 && mkrepo scanned/two 1 && { echo "virtual-root=/" && echo "cache-root=$PWD/cache5" && echo "cache-size=64" && echo "cache-index-ttl=0" && echo "scan-path=$PWD/scanned" } >scanrc && rm -rf cache5 && mkdir cache5 && cgit_rc scanrc "url=" >/dev/null && scanned_rc=$(ls cache5) && test_path_is_file "cache5/$scanned_rc" ' # A list that cannot be written leaves the request with the scan it made # before the write failed, so each repository is listed once and nothing is # left in the cache-root. The limit is on files, so the log and the exit # status go through the pipe with the page. test_expect_success NO_VALGRIND 'a scan whose list cannot be written lists each repository once' ' rm -rf cache9 && mkdir cache9 && sed "s|^cache-root=.*|cache-root=$PWD/cache9|" scanrc >unwritablerc && ( trap "" XFSZ && ulimit -f 0 && { cgit_rc unwritablerc "url=" 2>&1; echo "exit status $?"; } ) | cat >unwritable.out && test_grep "^exit status 0$" unwritable.out && test_grep "Unable to write .*/cache9/rc-[0-9a-f]*\.lock" unwritable.out && test $(grep -c ">one" unwritable.out) = 1 && test $(grep -c ">two" unwritable.out) = 1 && test_dir_is_empty cache9 ' # A request that finds the scan's lock held and has no list to go on scans # in its own process, leaving the file to the holder. That is only another # request at work and no cause for a line in the log. test_expect_success PERL_RUNS 'a scan finding its lock held runs uncached' ' write_hold_lock && rm "cache5/$scanned_rc" && CGIT_CONFIG="$PWD/scanrc" QUERY_STRING="url=" \ ./hold-lock "cache5/$scanned_rc.lock" cgit >held.out 2>held.err && grep ">one" held.out && grep ">two" held.out && test_path_is_missing "cache5/$scanned_rc" && test_must_be_empty held.err ' # A run that died before its rename leaves a lock file behind, here longer # than anything the next run writes and ending in a repository and a marker. # Emptying the file before reuse keeps that tail out of both caches. The page # that rebuilt the list is drawn from the scan in memory, so it is the file, # and a second request reading it back, that would show the ghost. test_expect_success 'a stale lock file reaches neither the scan nor a slot' ' { printf "%01024d\n" 0 0 0 0 && echo "repo.url=ghost" && echo "repo.path=$PWD/scanned/one/.git" } >stale.lock && rm -f "cache5/$scanned_rc" && cp stale.lock "cache5/$scanned_rc.lock" && cgit_ok scanrc "url=" >scan.out && test_grep ">one" scan.out && test_grep "^repo.url=one$" "cache5/$scanned_rc" && test_grep ! "^repo.url=ghost$" "cache5/$scanned_rc" && cgit_ok scanrc "url=" >scan.out && test_grep ">one" scan.out && test_grep ! ">ghost" scan.out && rm -rf cache3 && mkdir cache3 && cgit_ok bigkeyrc "url=bigpage/" >page.out && slot=$(ls cache3) && rm "cache3/$slot" && { cat page.out page.out && echo "STALE-TAIL"; } >"cache3/$slot.lock" && cgit_ok bigkeyrc "url=bigpage/" >/dev/null && cgit_ok bigkeyrc "url=bigpage/" >replay.out && test_cmp page.out replay.out ' # The cached list is read back as trusted config, so a config that would # scan differently keeps a list of its own in the same cache-root, here one # that takes only the repositories carrying an export marker. test_expect_success 'a config that scans differently keeps its own list' ' touch scanned/one/.git/export-ok && { echo "strict-export=export-ok" && cat scanrc } >strictrc && cgit_rc scanrc "url=" >all.out && cgit_rc strictrc "url=" >strict.out && grep ">two" all.out && grep ">one" strict.out && ! grep ">two" strict.out ' # A list scanned under trust-scan-config carries the filters a repository # set, and the list is read back with no trust check, so a config that does # not trust its scans has to keep a list of its own. Pages are left uncached, # since the page cache is keyed on the request alone. test_expect_success 'a config that trusts its scans keeps its own list' ' mkrepo trusted/one 1 && echo "commit-filter=exec:$FILTER_DIRECTORY/dump.sh" >trusted/one/.git/cgitrc && { echo "virtual-root=/" && echo "cache-root=$PWD/cache7" && echo "cache-size=64" && echo "cache-dynamic-ttl=0" && echo "scan-path=$PWD/trusted" } >distrustrc && { echo "trust-scan-config=1" && cat distrustrc } >trustrc && rm -rf cache7 && mkdir cache7 && cgit_ok trustrc "url=one/commit/" >trust.out && test_grep "

COMMIT 1" trust.out && cgit_rc distrustrc "url=one/commit/" >distrust.out 2>distrust.err && test_grep "^Status: 200" distrust.out && test_grep "

commit 1" distrust.out && test_grep "Ignoring commit-filter in $PWD/trusted/one/.git/: trust-scan-config is not set" distrust.err && ls cache7/rc-* >lists && test_line_count = 2 lists ' # Every setting the scan reads, or hands each repository as a default, goes # into the name of the list, so changing any one of them, or the path, must # start a list of its own. test_expect_success 'every setting the scan depends on keeps its own list' ' mkrepo keyed/one 1 && echo one >projects && { echo "virtual-root=/" && echo "cache-root=$PWD/cache8" && echo "cache-size=64" && echo "scan-path=$PWD/keyed" } >keyedrc && rm -rf cache8 && mkdir cache8 && cgit_ok keyedrc "url=" >/dev/null && sed "s|^scan-path=.*|scan-path=$PWD/keyed/one|" keyedrc >keyed-setting-rc && cgit_ok keyed-setting-rc "url=" >/dev/null && ls cache8/rc-* >lists && test_line_count = 2 lists && lists=2 && while read setting do { echo "$setting" && cat keyedrc } >keyed-setting-rc && cgit_ok keyed-setting-rc "url=" >/dev/null && lists=$((lists + 1)) && ls cache8/rc-* >lists && test_line_count = $lists lists || { echo "$setting shares a list" && return 1; } done <<-EOF project-list=$PWD/projects strict-export=git-daemon-export-ok section=Sec module-link=https://example.org/%s clone-url=https://example.org/\$CGIT_REPO_URL scan-hidden-path=1 case-sensitive-sort=0 remove-suffix=1 section-from-path=1 enable-git-config=1 trust-scan-config=1 snapshots=zip max-stats=year branch-sort=age commit-sort=date enable-blame=1 enable-commit-graph=1 enable-commit-trailers=1 enable-follow-links=1 enable-gitmodules-links=1 enable-html-serving=1 enable-log-filecount=1 enable-log-linecount=1 enable-mailmap=0 enable-remote-branches=1 enable-subject-links=1 about-filter=exec:$FILTER_DIRECTORY/dump.sh commit-filter=exec:$FILTER_DIRECTORY/dump.sh email-filter=exec:$FILTER_DIRECTORY/dump.sh source-filter=exec:$FILTER_DIRECTORY/dump.sh trailer-filter=exec:$FILTER_DIRECTORY/dump.sh EOF ' # A negative ttl never expires, so a list however old is served and kept. A # rebuild runs in a child holding the request's copy of fd 9, so cat reaching # the end of that pipe means any rebuild is done, as the positive ttl after # it shows. The ttl is not part of the list's name, so both configs share # the list, which is scanned afresh here. test_expect_success 'a negative cache-scan-ttl never expires the list' ' { echo "cache-scan-ttl=-1" && cat scanrc } >foreverrc && rm -f "cache5/$scanned_rc" && cgit_ok scanrc "url=" >/dev/null && test_path_is_file "cache5/$scanned_rc" && { echo "repo.url=kept" && echo "repo.path=$PWD/scanned/one/.git" } >>"cache5/$scanned_rc" && touch -t 200001010000 "cache5/$scanned_rc" && { cgit_rc foreverrc "url=" 9>&1 >forever.out 2>forever.err; echo $? >forever.status; } | cat && test "$(cat forever.status)" = 0 && test_must_be_empty forever.err && test_grep ">kept" forever.out && test_grep "^repo.url=kept$" "cache5/$scanned_rc" && { cgit_rc scanrc "url=" 9>&1 >rescan.out 2>rescan.err; echo $? >rescan.status; } | cat && test "$(cat rescan.status)" = 0 && test_must_be_empty rescan.err && test_grep ">kept" rescan.out && test_grep "^repo.url=one$" "cache5/$scanned_rc" && test_grep ! "^repo.url=kept$" "cache5/$scanned_rc" ' # Starts a request in the background whose stdin, stdout and stderr are pipes, # and returns once the request is over, which is only when every process # holding those pipes has let go, or fails after 30 seconds. The rescan child # keeps fd 9, so the reader of that pipe, and with it the background job, ends # only when the child does. start_rescan_request() { rm -f rescan.done { { cat /dev/zero 2>/dev/null | cgit_rc pipelistrc "url=" 2>&1 | cat >rescan.out && touch rescan.done } 9>&1 >/dev/null | cat } & rescan_pid=$! n=0 while ! test -f rescan.done && test $n -lt 30 do sleep 1 n=$((n + 1)) done test -f rescan.done } # Writes the project list into the pipe the child is held on, which fails # when no child is there to read it, and waits for the child to end. finish_rescan() { "$PERL_PATH" -e 'alarm 30; open(my $list, ">", shift) or die "$!\n"; print $list "one\n"' pipelist finish_status=$? wait "$rescan_pid" return $finish_status } rescan_behind_request() { start_rescan_request rescan_status=$? finish_rescan && return $rescan_status } # An expired list is rebuilt by a child forked off the request, which answers # from the stale list meanwhile. A web server reads the request's stdout until # every holder is gone, so the child drops it, along with stdin and stderr. A # project list that is a pipe holds the child until the test writes the list # into it, after the request has ended. test_expect_success PIPE,PERL_RUNS 'a background rescan lets the request finish' ' mkrepo piped/one 1 && echo one >pipelist && { echo "virtual-root=/" && echo "cache-root=$PWD/cache10" && echo "cache-size=64" && echo "cache-index-ttl=0" && echo "project-list=$PWD/pipelist" && echo "scan-path=$PWD/piped" } >pipelistrc && rm -rf cache10 && mkdir cache10 && cgit_ok pipelistrc "url=" >/dev/null && list=$(ls cache10/rc-*) && { echo "repo.url=kept" && echo "repo.path=$PWD/piped/one/.git" } >>"$list" && touch -t 200001010000 "$list" && rm pipelist && mkfifo pipelist && rescan_behind_request && test_grep ">kept" rescan.out && test_grep "" rescan.out && test_grep ! "^\[cgit\]" rescan.out && test_grep "^repo.url=one$" "$list" && test_grep ! "^repo.url=kept$" "$list" ' # A repository the scan found without a description goes into the list # without one, so its pages do not describe themselves with the placeholder. test_expect_success 'a cached scan leaves the description placeholder out' ' rm -rf cache5 && mkdir cache5 && cgit_ok scanrc "url=" >/dev/null && rc=$(ls cache5/rc-*) && test_grep "^repo.url=one$" "$rc" && test_grep ! "^repo.desc=" "$rc" && cgit_ok scanrc "url=" >tmp && test_grep "\[no description\]" tmp && cgit_ok scanrc "url=one/" >tmp && test_grep "data-page=.summary. data-repo=.one." tmp && test_grep ! "kept/one/.git/cgitrc && { echo "virtual-root=/" && echo "cache-root=$PWD/cache6" && echo "cache-size=64" && echo "trust-scan-config=1" && echo "branch-sort=age" && echo "scan-path=$PWD/kept" } >unfilteredrc && { echo "about-filter=exec:$FILTER_DIRECTORY/dump.sh" && cat unfilteredrc } >keptrc && rm -rf cache6 && mkdir cache6 && cgit_rc unfilteredrc "url=" >/dev/null && cgit_rc keptrc "url=" >/dev/null && cgit_rc keptrc "url=one/refs/" >tmp && sed -n "s/^

1$" tmp && cgit_rc keptrc "url=one/tree/" >tmp && grep "href=.https://example.org/s/?id=$sub." tmp ' # A cache-size below zero turns the page cache off, and the scan with it, so # the cache directory stays empty. test_expect_success 'a negative cache-size leaves the scan uncached' ' { echo "virtual-root=/" && echo "cache-root=$PWD/cache-negative" && echo "cache-size=-1" && echo "scan-path=$PWD/scanned" } >negativerc && rm -rf cache-negative && mkdir cache-negative && cgit_rc negativerc "url=" >tmp && grep ">one" tmp && test_dir_is_empty cache-negative ' test_done