diff --git a/src/applications/cache/PhabricatorCaches.php b/src/applications/cache/PhabricatorCaches.php index 860abcb130..f4eb7a918c 100644 --- a/src/applications/cache/PhabricatorCaches.php +++ b/src/applications/cache/PhabricatorCaches.php @@ -1,286 +1,347 @@ setCaches($caches); } /* -( Local Cache )-------------------------------------------------------- */ /** * Gets an immutable cache stack. * * This stack trades mutability away for improved performance. Normally, it is * APC + DB. * * In the general case with multiple web frontends, this stack can not be * cleared, so it is only appropriate for use if the value of a given key is * permanent and immutable. * * @return PhutilKeyValueCacheStack Best immutable stack available. * @task immutable */ public static function getImmutableCache() { static $cache; if (!$cache) { $caches = self::buildImmutableCaches(); $cache = self::newStackFromCaches($caches); } return $cache; } /** * Build the immutable cache stack. * * @return list List of caches. * @task immutable */ private static function buildImmutableCaches() { $caches = array(); $apc = new PhutilKeyValueCacheAPC(); if ($apc->isAvailable()) { $caches[] = $apc; } $caches[] = new PhabricatorKeyValueDatabaseCache(); return $caches; } /* -( Repository Graph Cache )--------------------------------------------- */ public static function getRepositoryGraphL1Cache() { static $cache; if (!$cache) { $caches = self::buildRepositoryGraphL1Caches(); $cache = self::newStackFromCaches($caches); } return $cache; } private static function buildRepositoryGraphL1Caches() { $caches = array(); $request = new PhutilKeyValueCacheInRequest(); $request->setLimit(32); $caches[] = $request; $apc = new PhutilKeyValueCacheAPC(); if ($apc->isAvailable()) { $caches[] = $apc; } return $caches; } public static function getRepositoryGraphL2Cache() { static $cache; if (!$cache) { $caches = self::buildRepositoryGraphL2Caches(); $cache = self::newStackFromCaches($caches); } return $cache; } private static function buildRepositoryGraphL2Caches() { $caches = array(); $caches[] = new PhabricatorKeyValueDatabaseCache(); return $caches; } /* -( Setup Cache )-------------------------------------------------------- */ /** * Highly specialized cache for performing setup checks. We use this cache * to determine if we need to run expensive setup checks when the page * loads. Without it, we would need to run these checks every time. * * Normally, this cache is just APC. In the absence of APC, this cache * degrades into a slow, quirky on-disk cache. * * NOTE: Do not use this cache for anything else! It is not a general-purpose * cache! * * @return PhutilKeyValueCacheStack Most qualified available cache stack. * @task setup */ public static function getSetupCache() { static $cache; if (!$cache) { $caches = self::buildSetupCaches(); $cache = self::newStackFromCaches($caches); } return $cache; } /** * @task setup */ private static function buildSetupCaches() { // In most cases, we should have APC. This is an ideal cache for our // purposes -- it's fast and empties on server restart. $apc = new PhutilKeyValueCacheAPC(); if ($apc->isAvailable()) { return array($apc); } // If we don't have APC, build a poor approximation on disk. This is still // much better than nothing; some setup steps are quite slow. $disk_path = self::getSetupCacheDiskCachePath(); if ($disk_path) { $disk = new PhutilKeyValueCacheOnDisk(); $disk->setCacheFile($disk_path); $disk->setWait(0.1); if ($disk->isAvailable()) { return array($disk); } } return array(); } /** * @task setup */ private static function getSetupCacheDiskCachePath() { // The difficulty here is in choosing a path which will change on server // restart (we MUST have this property), but as rarely as possible // otherwise (we desire this property to give the cache the best hit rate // we can). // In some setups, the parent PID is more stable and longer-lived that the // PID (e.g., under apache, our PID will be a worker while the ppid will // be the main httpd process). If we're confident we're running under such // a setup, we can try to use the PPID as the basis for our cache instead // of our own PID. $use_ppid = false; switch (php_sapi_name()) { case 'cli-server': // This is the PHP5.4+ built-in webserver. We should use the pid // (the server), not the ppid (probably a shell or something). $use_ppid = false; break; case 'fpm-fcgi': // We should be safe to use PPID here. $use_ppid = true; break; case 'apache2handler': // We're definitely safe to use the PPID. $use_ppid = true; break; } $pid_basis = getmypid(); if ($use_ppid) { if (function_exists('posix_getppid')) { $parent_pid = posix_getppid(); // On most systems, normal processes can never have PIDs lower than 100, // so something likely went wrong if we we get one of these. if ($parent_pid > 100) { $pid_basis = $parent_pid; } } } // If possible, we also want to know when the process launched, so we can // drop the cache if a process restarts but gets the same PID an earlier // process had. "/proc" is not available everywhere (e.g., not on OSX), but // check if we have it. $epoch_basis = null; $stat = @stat("/proc/{$pid_basis}"); if ($stat !== false) { $epoch_basis = $stat['ctime']; } $tmp_dir = sys_get_temp_dir(); $tmp_path = $tmp_dir.DIRECTORY_SEPARATOR.'phabricator-setup'; if (!file_exists($tmp_path)) { @mkdir($tmp_path); } $is_ok = self::testTemporaryDirectory($tmp_path); if (!$is_ok) { $tmp_path = $tmp_dir; $is_ok = self::testTemporaryDirectory($tmp_path); if (!$is_ok) { // We can't find anywhere to write the cache, so just bail. return null; } } $tmp_name = 'setup-'.$pid_basis; if ($epoch_basis) { $tmp_name .= '.'.$epoch_basis; } $tmp_name .= '.cache'; return $tmp_path.DIRECTORY_SEPARATOR.$tmp_name; } /** * @task setup */ private static function testTemporaryDirectory($dir) { if (!@file_exists($dir)) { return false; } if (!@is_dir($dir)) { return false; } if (!@is_writable($dir)) { return false; } return true; } private static function addProfilerToCaches(array $caches) { foreach ($caches as $key => $cache) { $pcache = new PhutilKeyValueCacheProfiler($cache); $pcache->setProfiler(PhutilServiceProfiler::getInstance()); $caches[$key] = $pcache; } return $caches; } private static function addNamespaceToCaches(array $caches) { $namespace = PhabricatorCaches::getNamespace(); if (!$namespace) { return $caches; } foreach ($caches as $key => $cache) { $ncache = new PhutilKeyValueCacheNamespace($cache); $ncache->setNamespace($namespace); $caches[$key] = $ncache; } return $caches; } + + /** + * Deflate a value, if deflation is available and has an impact. + * + * If the value is larger than 1KB, we have `gzdeflate()`, we successfully + * can deflate it, and it benefits from deflation, we deflate it. Otherwise + * we leave it as-is. + * + * Data can later be inflated with @{method:inflateData}. + * + * @param string String to attempt to deflate. + * @return string|null Deflated string, or null if it was not deflated. + * @task compress + */ + public static function maybeDeflateData($value) { + $len = strlen($value); + if ($len <= 1024) { + return null; + } + + if (!function_exists('gzdeflate')) { + return null; + } + + $deflated = gzdeflate($value); + if ($deflated === false) { + return null; + } + + $deflated_len = strlen($deflated); + if ($deflated_len >= ($len / 2)) { + return null; + } + + return $deflated; + } + + + /** + * Inflate data previously deflated by @{method:maybeDeflateData}. + * + * @param string Deflated data, from @{method:maybeDeflateData}. + * @return string Original, uncompressed data. + * @task compress + */ + public static function inflateData($value) { + if (!function_exists('gzinflate')) { + throw new Exception( + pht('gzinflate() is not available; unable to read deflated data!')); + } + + $value = gzinflate($value); + if ($value === false) { + throw new Exception(pht('Failed to inflate data!')); + } + + return $value; + } + + } diff --git a/src/applications/cache/PhabricatorKeyValueDatabaseCache.php b/src/applications/cache/PhabricatorKeyValueDatabaseCache.php index 822b90ffcf..aa641ab703 100644 --- a/src/applications/cache/PhabricatorKeyValueDatabaseCache.php +++ b/src/applications/cache/PhabricatorKeyValueDatabaseCache.php @@ -1,185 +1,170 @@ digestKeys(array_keys($keys)); $conn_w = $this->establishConnection('w'); $sql = array(); foreach ($map as $key => $hash) { $value = $keys[$key]; list($format, $storage_value) = $this->willWriteValue($key, $value); $sql[] = qsprintf( $conn_w, '(%s, %s, %s, %B, %d, %nd)', $hash, $key, $format, $storage_value, time(), $ttl ? (time() + $ttl) : null); } $guard = AphrontWriteGuard::beginScopedUnguardedWrites(); foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) { queryfx( $conn_w, 'INSERT INTO %T (cacheKeyHash, cacheKey, cacheFormat, cacheData, cacheCreated, cacheExpires) VALUES %Q ON DUPLICATE KEY UPDATE cacheKey = VALUES(cacheKey), cacheFormat = VALUES(cacheFormat), cacheData = VALUES(cacheData), cacheCreated = VALUES(cacheCreated), cacheExpires = VALUES(cacheExpires)', $this->getTableName(), $chunk); } unset($guard); } return $this; } public function getKeys(array $keys) { $results = array(); if ($keys) { $map = $this->digestKeys($keys); $rows = queryfx_all( $this->establishConnection('r'), 'SELECT * FROM %T WHERE cacheKeyHash IN (%Ls)', $this->getTableName(), $map); $rows = ipull($rows, null, 'cacheKey'); foreach ($keys as $key) { if (empty($rows[$key])) { continue; } $row = $rows[$key]; if ($row['cacheExpires'] && ($row['cacheExpires'] < time())) { continue; } try { $results[$key] = $this->didReadValue( $row['cacheFormat'], $row['cacheData']); } catch (Exception $ex) { // Treat this as a cache miss. phlog($ex); } } } return $results; } public function deleteKeys(array $keys) { if ($keys) { $map = $this->digestKeys($keys); queryfx( $this->establishConnection('w'), 'DELETE FROM %T WHERE cacheKeyHash IN (%Ls)', $this->getTableName(), $keys); } return $this; } public function destroyCache() { queryfx( $this->establishConnection('w'), 'DELETE FROM %T', $this->getTableName()); return $this; } /* -( Raw Cache Access )--------------------------------------------------- */ public function establishConnection($mode) { // TODO: This is the only concrete table we have on the database right // now. return id(new PhabricatorMarkupCache())->establishConnection($mode); } public function getTableName() { return 'cache_general'; } /* -( Implementation )----------------------------------------------------- */ private function digestKeys(array $keys) { $map = array(); foreach ($keys as $key) { $map[$key] = PhabricatorHash::digestForIndex($key); } return $map; } private function willWriteValue($key, $value) { if (!is_string($value)) { throw new Exception("Only strings may be written to the DB cache!"); } static $can_deflate; if ($can_deflate === null) { $can_deflate = function_exists('gzdeflate') && PhabricatorEnv::getEnvConfig('cache.enable-deflate'); } - // If the value is larger than 1KB, we have gzdeflate(), we successfully - // can deflate it, and it benefits from deflation, store it deflated. if ($can_deflate) { - $len = strlen($value); - if ($len > 1024) { - $deflated = gzdeflate($value); - if ($deflated !== false) { - $deflated_len = strlen($deflated); - if ($deflated_len < ($len / 2)) { - return array(self::CACHE_FORMAT_DEFLATE, $deflated); - } - } + $deflated = PhabricatorCaches::maybeDeflateData($value); + if ($deflated !== null) { + return array(self::CACHE_FORMAT_DEFLATE, $deflated); } } return array(self::CACHE_FORMAT_RAW, $value); } private function didReadValue($format, $value) { switch ($format) { case self::CACHE_FORMAT_RAW: return $value; case self::CACHE_FORMAT_DEFLATE: - if (!function_exists('gzinflate')) { - throw new Exception("No gzinflate() to read deflated cache."); - } - $value = gzinflate($value); - if ($value === false) { - throw new Exception("Failed to deflate cache."); - } - return $value; + return PhabricatorCaches::inflateData($value); default: throw new Exception("Unknown cache format."); } } } diff --git a/src/applications/differential/management/PhabricatorHunksManagementMigrateWorkflow.php b/src/applications/differential/management/PhabricatorHunksManagementMigrateWorkflow.php index 3065658743..2b59f1871d 100644 --- a/src/applications/differential/management/PhabricatorHunksManagementMigrateWorkflow.php +++ b/src/applications/differential/management/PhabricatorHunksManagementMigrateWorkflow.php @@ -1,48 +1,62 @@ setName('migrate') ->setExamples('**migrate**') ->setSynopsis(pht('Migrate hunks to modern storage.')) ->setArguments(array()); } public function execute(PhutilArgumentParser $args) { $saw_any_rows = false; $console = PhutilConsole::getConsole(); $table = new DifferentialHunkLegacy(); foreach (new LiskMigrationIterator($table) as $hunk) { $saw_any_rows = true; $id = $hunk->getID(); $console->writeOut("%s\n", pht('Migrating hunk %d...', $id)); $new_hunk = id(new DifferentialHunkModern()) ->setChangesetID($hunk->getChangesetID()) ->setOldOffset($hunk->getOldOffset()) ->setOldLen($hunk->getOldLen()) ->setNewOffset($hunk->getNewOffset()) ->setNewLen($hunk->getNewLen()) ->setChanges($hunk->getChanges()) ->setDateCreated($hunk->getDateCreated()) ->setDateModified($hunk->getDateModified()); $hunk->openTransaction(); $new_hunk->save(); $hunk->delete(); $hunk->saveTransaction(); + + $old_len = strlen($hunk->getChanges()); + $new_len = strlen($new_hunk->getData()); + if ($old_len) { + $diff_len = ($old_len - $new_len); + $console->writeOut( + "%s\n", + pht( + 'Saved %s bytes (%s).', + new PhutilNumber($diff_len), + sprintf('%.1f%%', 100 * ($diff_len / $old_len)))); + } + + break; } if ($saw_any_rows) { $console->writeOut("%s\n", pht('Done.')); } else { $console->writeOut("%s\n", pht('No rows to migrate.')); } } } diff --git a/src/applications/differential/storage/DifferentialHunkModern.php b/src/applications/differential/storage/DifferentialHunkModern.php index 50aa48e8c6..39168b3ebe 100644 --- a/src/applications/differential/storage/DifferentialHunkModern.php +++ b/src/applications/differential/storage/DifferentialHunkModern.php @@ -1,73 +1,103 @@ array( 'data' => true, ), ) + parent::getConfiguration(); } public function setChanges($text) { + $this->rawData = $text; + $this->dataEncoding = $this->detectEncodingForStorage($text); $this->dataType = self::DATATYPE_TEXT; $this->dataFormat = self::DATAFORMAT_RAW; $this->data = $text; return $this; } public function getChanges() { return $this->getUTF8StringFromStorage( $this->getRawData(), $this->getDataEncoding()); } - private function getRawData() { + public function save() { + $type = $this->getDataType(); - $data = $this->getData(); - - switch ($type) { - case self::DATATYPE_TEXT: - // In this storage type, the changes are stored on the object. - $data = $data; - break; - case self::DATATYPE_FILE: - default: - throw new Exception( - pht('Hunk has unsupported data type "%s"!', $type)); + $format = $this->getDataFormat(); + + // Before saving the data, attempt to compress it. + if ($type == self::DATATYPE_TEXT) { + if ($format == self::DATAFORMAT_RAW) { + $data = $this->getData(); + $deflated = PhabricatorCaches::maybeDeflateData($data); + if ($deflated !== null) { + $this->data = $deflated; + $this->dataFormat = self::DATAFORMAT_DEFLATED; + } + } } - $format = $this->getDataFormat(); - switch ($format) { - case self::DATAFORMAT_RAW: - // In this format, the changes are stored as-is. - $data = $data; - break; - case self::DATAFORMAT_DEFLATE: - default: - throw new Exception( - pht('Hunk has unsupported data encoding "%s"!', $type)); + return parent::save(); + } + + private function getRawData() { + if ($this->rawData === null) { + $type = $this->getDataType(); + $data = $this->getData(); + + switch ($type) { + case self::DATATYPE_TEXT: + // In this storage type, the changes are stored on the object. + $data = $data; + break; + case self::DATATYPE_FILE: + default: + throw new Exception( + pht('Hunk has unsupported data type "%s"!', $type)); + } + + $format = $this->getDataFormat(); + switch ($format) { + case self::DATAFORMAT_RAW: + // In this format, the changes are stored as-is. + $data = $data; + break; + case self::DATAFORMAT_DEFLATED: + $data = PhabricatorCaches::inflateData($data); + break; + default: + throw new Exception( + pht('Hunk has unsupported data encoding "%s"!', $type)); + } + + $this->rawData = $data; } - return $data; + return $this->rawData; } }