Changeset View
Changeset View
Standalone View
Standalone View
support/PhabricatorStartup.php
| Show First 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | final class PhabricatorStartup { | ||||
| private static $rawInput; | private static $rawInput; | ||||
| private static $oldMemoryLimit; | private static $oldMemoryLimit; | ||||
| private static $phases; | private static $phases; | ||||
| // TODO: For now, disable rate limiting entirely by default. We need to | // TODO: For now, disable rate limiting entirely by default. We need to | ||||
| // iterate on it a bit for Conduit, some of the specific score levels, and | // iterate on it a bit for Conduit, some of the specific score levels, and | ||||
| // to deal with NAT'd offices. | // to deal with NAT'd offices. | ||||
| private static $maximumRate = 0; | private static $maximumRate = 0; | ||||
| private static $rateLimitToken; | |||||
| /* -( Accessing Request Information )-------------------------------------- */ | /* -( Accessing Request Information )-------------------------------------- */ | ||||
| /** | /** | ||||
| * @task info | * @task info | ||||
| */ | */ | ||||
| ▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Lines | public static function didStartup($start_time) { | ||||
| self::setupPHP(); | self::setupPHP(); | ||||
| self::verifyPHP(); | self::verifyPHP(); | ||||
| // If we've made it this far, the environment isn't completely broken so | // If we've made it this far, the environment isn't completely broken so | ||||
| // we can switch over to relying on our own exception recovery mechanisms. | // we can switch over to relying on our own exception recovery mechanisms. | ||||
| ini_set('display_errors', 0); | ini_set('display_errors', 0); | ||||
| if (isset($_SERVER['REMOTE_ADDR'])) { | $rate_token = self::getRateLimitToken(); | ||||
| self::rateLimitRequest($_SERVER['REMOTE_ADDR']); | if ($rate_token !== null) { | ||||
| self::rateLimitRequest($rate_token); | |||||
| } | } | ||||
| self::normalizeInput(); | self::normalizeInput(); | ||||
| self::verifyRewriteRules(); | self::verifyRewriteRules(); | ||||
| self::detectPostMaxSizeTriggered(); | self::detectPostMaxSizeTriggered(); | ||||
| ▲ Show 20 Lines • Show All 526 Lines • ▼ Show 20 Lines | /* -( Rate Limiting )------------------------------------------------------ */ | ||||
| * @task ratelimit | * @task ratelimit | ||||
| */ | */ | ||||
| public static function setMaximumRate($rate) { | public static function setMaximumRate($rate) { | ||||
| self::$maximumRate = $rate; | self::$maximumRate = $rate; | ||||
| } | } | ||||
| /** | /** | ||||
| * Set a token to identify the client for purposes of rate limiting. | |||||
| * | |||||
| * By default, the `REMOTE_ADDR` is used. If your install is behind a load | |||||
| * balancer, you may want to parse `X-Forwarded-For` and use that address | |||||
| * instead. | |||||
| * | |||||
| * @param string Client identity for rate limiting. | |||||
| */ | |||||
| public static function setRateLimitToken($token) { | |||||
| self::$rateLimitToken = $token; | |||||
| } | |||||
| /** | |||||
| * Get the current client identity for rate limiting. | |||||
| */ | |||||
| public static function getRateLimitToken() { | |||||
| if (self::$rateLimitToken !== null) { | |||||
| return self::$rateLimitToken; | |||||
| } | |||||
| if (isset($_SERVER['REMOTE_ADDR'])) { | |||||
| return $_SERVER['REMOTE_ADDR']; | |||||
| } | |||||
| return null; | |||||
| } | |||||
| /** | |||||
| * Check if the user (identified by `$user_identity`) has issued too many | * Check if the user (identified by `$user_identity`) has issued too many | ||||
| * requests recently. If they have, end the request with a 429 error code. | * requests recently. If they have, end the request with a 429 error code. | ||||
| * | * | ||||
| * The key just needs to identify the user. Phabricator uses both user PHIDs | * The key just needs to identify the user. Phabricator uses both user PHIDs | ||||
| * and user IPs as keys, tracking logged-in and logged-out users separately | * and user IPs as keys, tracking logged-in and logged-out users separately | ||||
| * and enforcing different limits. | * and enforcing different limits. | ||||
| * | * | ||||
| * @param string Some key which identifies the user making the request. | * @param string Some key which identifies the user making the request. | ||||
| * @return void If the user has exceeded the rate limit, this method | * @return void If the user has exceeded the rate limit, this method | ||||
| * does not return. | * does not return. | ||||
| * @task ratelimit | * @task ratelimit | ||||
| */ | */ | ||||
| public static function rateLimitRequest($user_identity) { | public static function rateLimitRequest($user_identity) { | ||||
| if (!self::canRateLimit()) { | if (!self::canRateLimit()) { | ||||
| return; | return; | ||||
| } | } | ||||
| $score = self::getRateLimitScore($user_identity); | $score = self::getRateLimitScore($user_identity); | ||||
| if ($score > (self::$maximumRate * self::getRateLimitBucketCount())) { | $limit = self::$maximumRate * self::getRateLimitBucketCount(); | ||||
| if ($score > $limit) { | |||||
| // Give the user some bonus points for getting rate limited. This keeps | // Give the user some bonus points for getting rate limited. This keeps | ||||
| // bad actors who keep slamming the 429 page locked out completely, | // bad actors who keep slamming the 429 page locked out completely, | ||||
| // instead of letting them get a burst of requests through every minute | // instead of letting them get a burst of requests through every minute | ||||
| // after a bucket expires. | // after a bucket expires. | ||||
| self::addRateLimitScore($user_identity, 50); | $penalty = 50; | ||||
| self::didRateLimit($user_identity); | |||||
| self::addRateLimitScore($user_identity, $penalty); | |||||
| $score += $penalty; | |||||
| self::didRateLimit($user_identity, $score, $limit); | |||||
| } | } | ||||
| } | } | ||||
| /** | /** | ||||
| * Add points to the rate limit score for some user. | * Add points to the rate limit score for some user. | ||||
| * | * | ||||
| * If users have earned more than 1000 points per minute across all the | * If users have earned more than 1000 points per minute across all the | ||||
| * buckets they'll be locked out of the application, so awarding 1 point per | * buckets they'll be locked out of the application, so awarding 1 point per | ||||
| * request roughly corresponds to allowing 1000 requests per second, while | * request roughly corresponds to allowing 1000 requests per second, while | ||||
| * awarding 50 points roughly corresponds to allowing 20 requests per second. | * awarding 50 points roughly corresponds to allowing 20 requests per second. | ||||
| * | * | ||||
| * @param string Some key which identifies the user making the request. | * @param string Some key which identifies the user making the request. | ||||
| * @param float The cost for this request; more points pushes them toward | * @param float The cost for this request; more points pushes them toward | ||||
| * the limit faster. | * the limit faster. | ||||
| * @return void | * @return void | ||||
| * @task ratelimit | * @task ratelimit | ||||
| */ | */ | ||||
| public static function addRateLimitScore($user_identity, $score) { | public static function addRateLimitScore($user_identity, $score) { | ||||
| if (!self::canRateLimit()) { | if (!self::canRateLimit()) { | ||||
| return; | return; | ||||
| } | } | ||||
| $is_apcu = (bool)function_exists('apcu_fetch'); | |||||
| $current = self::getRateLimitBucket(); | $current = self::getRateLimitBucket(); | ||||
| // There's a bit of a race here, if a second process reads the bucket before | // There's a bit of a race here, if a second process reads the bucket | ||||
| // this one writes it, but it's fine if we occasionally fail to record a | // before this one writes it, but it's fine if we occasionally fail to | ||||
| // user's score. If they're making requests fast enough to hit rate | // record a user's score. If they're making requests fast enough to hit | ||||
| // limiting, we'll get them soon. | // rate limiting, we'll get them soon enough. | ||||
| $bucket_key = self::getRateLimitBucketKey($current); | $bucket_key = self::getRateLimitBucketKey($current); | ||||
| if ($is_apcu) { | |||||
| $bucket = apcu_fetch($bucket_key); | |||||
| } else { | |||||
| $bucket = apc_fetch($bucket_key); | $bucket = apc_fetch($bucket_key); | ||||
| } | |||||
| if (!is_array($bucket)) { | if (!is_array($bucket)) { | ||||
| $bucket = array(); | $bucket = array(); | ||||
| } | } | ||||
| if (empty($bucket[$user_identity])) { | if (empty($bucket[$user_identity])) { | ||||
| $bucket[$user_identity] = 0; | $bucket[$user_identity] = 0; | ||||
| } | } | ||||
| $bucket[$user_identity] += $score; | $bucket[$user_identity] += $score; | ||||
| if ($is_apcu) { | |||||
| apcu_store($bucket_key, $bucket); | |||||
| } else { | |||||
| apc_store($bucket_key, $bucket); | apc_store($bucket_key, $bucket); | ||||
| } | } | ||||
| } | |||||
| /** | /** | ||||
| * Determine if rate limiting is available. | * Determine if rate limiting is available. | ||||
| * | * | ||||
| * Rate limiting depends on APC, and isn't available unless the APC user | * Rate limiting depends on APC, and isn't available unless the APC user | ||||
| * cache is available. | * cache is available. | ||||
| * | * | ||||
| * @return bool True if rate limiting is available. | * @return bool True if rate limiting is available. | ||||
| * @task ratelimit | * @task ratelimit | ||||
| */ | */ | ||||
| private static function canRateLimit() { | private static function canRateLimit() { | ||||
| if (!self::$maximumRate) { | if (!self::$maximumRate) { | ||||
| return false; | return false; | ||||
| } | } | ||||
| if (!function_exists('apc_fetch')) { | if (!function_exists('apc_fetch') && !function_exists('apcu_fetch')) { | ||||
| return false; | return false; | ||||
| } | } | ||||
| return true; | return true; | ||||
| } | } | ||||
| /** | /** | ||||
| ▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | /* -( Rate Limiting )------------------------------------------------------ */ | ||||
| /** | /** | ||||
| * Get the current rate limit score for a given user. | * Get the current rate limit score for a given user. | ||||
| * | * | ||||
| * @param string Unique key identifying the user. | * @param string Unique key identifying the user. | ||||
| * @return float The user's current score. | * @return float The user's current score. | ||||
| * @task ratelimit | * @task ratelimit | ||||
| */ | */ | ||||
| private static function getRateLimitScore($user_identity) { | private static function getRateLimitScore($user_identity) { | ||||
| $is_apcu = (bool)function_exists('apcu_fetch'); | |||||
| $min_key = self::getRateLimitMinKey(); | $min_key = self::getRateLimitMinKey(); | ||||
| // Identify the oldest bucket stored in APC. | // Identify the oldest bucket stored in APC. | ||||
| $cur = self::getRateLimitBucket(); | $cur = self::getRateLimitBucket(); | ||||
| if ($is_apcu) { | |||||
| $min = apcu_fetch($min_key); | |||||
| } else { | |||||
| $min = apc_fetch($min_key); | $min = apc_fetch($min_key); | ||||
| } | |||||
| // If we don't have any buckets stored yet, store the current bucket as | // If we don't have any buckets stored yet, store the current bucket as | ||||
| // the oldest bucket. | // the oldest bucket. | ||||
| if (!$min) { | if (!$min) { | ||||
| if ($is_apcu) { | |||||
| apcu_store($min_key, $cur); | |||||
| } else { | |||||
| apc_store($min_key, $cur); | apc_store($min_key, $cur); | ||||
| } | |||||
| $min = $cur; | $min = $cur; | ||||
| } | } | ||||
| // Destroy any buckets that are older than the minimum bucket we're keeping | // Destroy any buckets that are older than the minimum bucket we're keeping | ||||
| // track of. Under load this normally shouldn't do anything, but will clean | // track of. Under load this normally shouldn't do anything, but will clean | ||||
| // up an old bucket once per minute. | // up an old bucket once per minute. | ||||
| $count = self::getRateLimitBucketCount(); | $count = self::getRateLimitBucketCount(); | ||||
| for ($cursor = $min; $cursor < ($cur - $count); $cursor++) { | for ($cursor = $min; $cursor < ($cur - $count); $cursor++) { | ||||
| apc_delete(self::getRateLimitBucketKey($cursor)); | $bucket_key = self::getRateLimitBucketKey($cursor); | ||||
| if ($is_apcu) { | |||||
| apcu_delete($bucket_key); | |||||
| apcu_store($min_key, $cursor + 1); | |||||
| } else { | |||||
| apc_delete($bucket_key); | |||||
| apc_store($min_key, $cursor + 1); | apc_store($min_key, $cursor + 1); | ||||
| } | } | ||||
| } | |||||
| // Now, sum up the user's scores in all of the active buckets. | // Now, sum up the user's scores in all of the active buckets. | ||||
| $score = 0; | $score = 0; | ||||
| for (; $cursor <= $cur; $cursor++) { | for (; $cursor <= $cur; $cursor++) { | ||||
| $bucket = apc_fetch(self::getRateLimitBucketKey($cursor)); | $bucket_key = self::getRateLimitBucketKey($cursor); | ||||
| if ($is_apcu) { | |||||
| $bucket = apcu_fetch($bucket_key); | |||||
| } else { | |||||
| $bucket = apc_fetch($bucket_key); | |||||
| } | |||||
| if (isset($bucket[$user_identity])) { | if (isset($bucket[$user_identity])) { | ||||
| $score += $bucket[$user_identity]; | $score += $bucket[$user_identity]; | ||||
| } | } | ||||
| } | } | ||||
| return $score; | return $score; | ||||
| } | } | ||||
| /** | /** | ||||
| * Emit an HTTP 429 "Too Many Requests" response (indicating that the user | * Emit an HTTP 429 "Too Many Requests" response (indicating that the user | ||||
| * has exceeded application rate limits) and exit. | * has exceeded application rate limits) and exit. | ||||
| * | * | ||||
| * @return exit This method **does not return**. | * @return exit This method **does not return**. | ||||
| * @task ratelimit | * @task ratelimit | ||||
| */ | */ | ||||
| private static function didRateLimit() { | private static function didRateLimit($user_identity, $score, $limit) { | ||||
| $message = | $message = | ||||
| "TOO MANY REQUESTS\n". | "TOO MANY REQUESTS\n". | ||||
| "You are issuing too many requests too quickly.\n". | "You (\"{$user_identity}\") are issuing too many requests ". | ||||
| "To adjust limits, see \"Configuring a Preamble Script\" in the ". | "too quickly.\n"; | ||||
amckinley: Probably better not to send this information (and maybe log it instead). Legit users with over… | |||||
| "documentation."; | |||||
| header( | header( | ||||
| 'Content-Type: text/plain; charset=utf-8', | 'Content-Type: text/plain; charset=utf-8', | ||||
| $replace = true, | $replace = true, | ||||
| $http_error = 429); | $http_error = 429); | ||||
| echo $message; | echo $message; | ||||
| ▲ Show 20 Lines • Show All 58 Lines • Show Last 20 Lines | |||||
Probably better not to send this information (and maybe log it instead). Legit users with over-aggressive bots that get 429'd will probably go from oblivious to "oops, I'll cut my rate by 10x" without needing to know the details, but abuse traffic just gets a clue about how our rate limiting works and what the limits are.