From 4f714264a62f0bfc040666d1a6b676d1b3fb117c Mon Sep 17 00:00:00 2001 From: Bryce Kwon Date: Sat, 7 Nov 2026 16:24:32 -1000 Subject: Pin the robots.txt rules Nothing in tests/ read robots.txt, so dropping the trailing slashes again went unnoticed. --- tests/t0301-security.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/t0301-security.sh b/tests/t0301-security.sh index a00ded2..f7befe8 100755 --- a/tests/t0301-security.sh +++ b/tests/t0301-security.sh @@ -671,4 +671,17 @@ test_expect_success 'a redirect keeps every query byte inside its Location' ' test_grep ! "^X-Injected" headers ' +# Every link cgit makes to the snapshot and blame pages ends the page name +# with a slash, and the rules do too, so a file whose name only starts with +# either word stays open to crawlers. +test_expect_success 'robots.txt turns crawlers away from snapshots and blame' ' + grep -v "^#" "$TEST_OUTPUT_DIRECTORY/../assets/robots.txt" >rules && + cat >expect <<-\EOF && + User-agent: * + Disallow: /*/snapshot/ + Disallow: /*/blame/ + EOF + test_cmp expect rules +' + test_done -- cgit 2.12.1