Differential D21607 Diff 51438 src/applications/system/controller/robots/PhabricatorRobotsPlatformController.php
Changeset View
Changeset View
Standalone View
Standalone View
src/applications/system/controller/robots/PhabricatorRobotsPlatformController.php
- This file was moved from src/applications/system/controller/PhabricatorRobotsPlatformController.php.
Show All 13 Lines | protected function newRobotsRules() { | ||||
// Note that this still allows commits (at '/rPxxxxx') to be indexed. | // Note that this still allows commits (at '/rPxxxxx') to be indexed. | ||||
// They're probably not hugely useful, but suffer fewer of the problems | // They're probably not hugely useful, but suffer fewer of the problems | ||||
// Diffusion suffers and are hard to omit with 'robots.txt'. | // Diffusion suffers and are hard to omit with 'robots.txt'. | ||||
$out[] = 'User-Agent: *'; | $out[] = 'User-Agent: *'; | ||||
$out[] = 'Disallow: /diffusion/'; | $out[] = 'Disallow: /diffusion/'; | ||||
$out[] = 'Disallow: /source/'; | $out[] = 'Disallow: /source/'; | ||||
// Add a small crawl delay (number of seconds between requests) for spiders | |||||
// which respect it. The intent here is to prevent spiders from affecting | |||||
// performance for users. The possible cost is slower indexing, but that | |||||
// seems like a reasonable tradeoff, since most Phabricator installs are | |||||
// probably not hugely concerned about cutting-edge SEO. | |||||
$out[] = 'Crawl-delay: 1'; | |||||
return $out; | return $out; | ||||
} | } | ||||
} | } |