PhabricatorGlobalLock.php
51cb7a3db9e8master

PhabricatorGlobalLock.php
View Options

			<?php

			/**
			* Global, MySQL-backed lock. This is a high-reliability, low-performance
			* global lock.
			*
			* The lock is maintained by using GET_LOCK() in MySQL, and automatically
			* released when the connection terminates. Thus, this lock can safely be used
			* to control access to shared resources without implementing any sort of
			* timeout or override logic: the lock can't normally be stuck in a locked state
			* with no process actually holding the lock.
			*
			* However, acquiring the lock is moderately expensive (several network
			* roundtrips). This makes it unsuitable for tasks where lock performance is
			* important.
			*
			* $lock = PhabricatorGlobalLock::newLock('example');
			* $lock->lock();
			* do_contentious_things();
			* $lock->unlock();
			*
			* NOTE: This lock is not completely global; it is namespaced to the active
			* storage namespace so that unit tests running in separate table namespaces
			* are isolated from one another.
			*
			* @task construct Constructing Locks
			* @task impl Implementation
			*/
			final class PhabricatorGlobalLock extends PhutilLock {

			private $parameters;
			private $conn;
			private $externalConnection;
			private $log;
			private $disableLogging;

			private static $pool = array();


			/* -( Constructing Locks )------------------------------------------------- */


			public static function newLock($name, $parameters = array()) {
			$namespace = PhabricatorLiskDAO::getStorageNamespace();
			$namespace = PhabricatorHash::digestToLength($namespace, 20);

			$parts = array();
			ksort($parameters);
			foreach ($parameters as $key => $parameter) {
			if (!preg_match('/^[a-zA-Z0-9]+\z/', $key)) {
			throw new Exception(
			pht(
			'Lock parameter key "%s" must be alphanumeric.',
			$key));
			}

			if (!is_scalar($parameter) && !is_null($parameter)) {
			throw new Exception(
			pht(
			'Lock parameter for key "%s" must be a scalar.',
			$key));
			}

			$value = phutil_json_encode($parameter);
			$parts[] = "{$key}={$value}";
			}
			$parts = implode(', ', $parts);

			$local = "{$name}({$parts})";
			$local = PhabricatorHash::digestToLength($local, 20);

			$full_name = "ph:{$namespace}:{$local}";
			$lock = self::getLock($full_name);
			if (!$lock) {
			$lock = new PhabricatorGlobalLock($full_name);
			self::registerLock($lock);

			$lock->parameters = $parameters;
			}

			return $lock;
			}

			/**
			* Use a specific database connection for locking.
			*
			* By default, `PhabricatorGlobalLock` will lock on the "repository" database
			* (somewhat arbitrarily). In most cases this is fine, but this method can
			* be used to lock on a specific connection.
			*
			* @param AphrontDatabaseConnection
			* @return this
			*/
			public function setExternalConnection(AphrontDatabaseConnection $conn) {
			if ($this->conn) {
			throw new Exception(
			pht(
			'Lock is already held, and must be released before the '.
			'connection may be changed.'));
			}
			$this->externalConnection = $conn;
			return $this;
			}

			public function setDisableLogging($disable) {
			$this->disableLogging = $disable;
			return $this;
			}


			/* -( Connection Pool )---------------------------------------------------- */

			public static function getConnectionPoolSize() {
			return count(self::$pool);
			}

			public static function clearConnectionPool() {
			self::$pool = array();
			}

			public static function newConnection() {
			// NOTE: Use of the "repository" database is somewhat arbitrary, mostly
			// because the first client of locks was the repository daemons.

			// We must always use the same database for all locks, because different
			// databases may be on different hosts if the database is partitioned.

			// However, we don't access any tables so we could use any valid database.
			// We could build a database-free connection instead, but that's kind of
			// messy and unusual.

			$dao = new PhabricatorRepository();

			// NOTE: Using "force_new" to make sure each lock is on its own connection.

			// See T13627. This is critically important in versions of MySQL older
			// than MySQL 5.7, because they can not hold more than one lock per
			// connection simultaneously.

			return $dao->establishConnection('w', $force_new = true);
			}

			/* -( Implementation )----------------------------------------------------- */

			protected function doLock($wait) {
			$conn = $this->conn;

			if (!$conn) {
			if ($this->externalConnection) {
			$conn = $this->externalConnection;
			}
			}

			if (!$conn) {
			// Try to reuse a connection from the connection pool.
			$conn = array_pop(self::$pool);
			}

			if (!$conn) {
			$conn = self::newConnection();
			}

			// See T13627. We must never hold more than one lock per connection, so
			// make sure this connection has no existing locks. (Normally, we should
			// only be able to get here if callers explicitly provide the same external
			// connection to multiple locks.)

			if ($conn->isHoldingAnyLock()) {
			throw new Exception(
			pht(
			'Unable to establish lock on connection: this connection is '.
			'already holding a lock. Acquiring a second lock on the same '.
			'connection would release the first lock in MySQL versions '.
			'older than 5.7.'));
			}

			// NOTE: Since MySQL will disconnect us if we're idle for too long, we set
			// the wait_timeout to an enormous value, to allow us to hold the
			// connection open indefinitely (or, at least, for 24 days).
			$max_allowed_timeout = 2147483;
			queryfx($conn, 'SET wait_timeout = %d', $max_allowed_timeout);

			$lock_name = $this->getName();

			$result = queryfx_one(
			$conn,
			'SELECT GET_LOCK(%s, %f)',
			$lock_name,
			$wait);

			$ok = head($result);
			if (!$ok) {

			// See PHI1794. We failed to acquire the lock, but the connection itself
			// is still good. We're done with it, so add it to the pool, just as we
			// would if we were releasing the lock.

			// If we don't do this, we may establish a huge number of connections
			// very rapidly if many workers try to acquire a lock at once. For
			// example, this can happen if there are a large number of webhook tasks
			// in the queue.

			// See T13627. If this is an external connection, don't put it into
			// the shared connection pool.

			if (!$this->externalConnection) {
			self::$pool[] = $conn;
			}

			throw id(new PhutilLockException($lock_name))
			->setHint($this->newHint($lock_name, $wait));
			}

			$conn->rememberLock($lock_name);

			$this->conn = $conn;

			if ($this->shouldLogLock()) {
			$lock_context = $this->newLockContext();

			$log = id(new PhabricatorDaemonLockLog())
			->setLockName($lock_name)
			->setLockParameters($this->parameters)
			->setLockContext($lock_context)
			->save();

			$this->log = $log;
			}
			}

			protected function doUnlock() {
			$lock_name = $this->getName();

			$conn = $this->conn;

			try {
			$result = queryfx_one(
			$conn,
			'SELECT RELEASE_LOCK(%s)',
			$lock_name);
			$conn->forgetLock($lock_name);
			} catch (Exception $ex) {
			$result = array(null);
			}

			$ok = head($result);
			if (!$ok) {
			// TODO: We could throw here, but then this lock doesn't get marked
			// unlocked and we throw again later when exiting. It also doesn't
			// particularly matter for any current applications. For now, just
			// swallow the error.
			}

			$this->conn = null;

			if (!$this->externalConnection) {
			$conn->close();
			self::$pool[] = $conn;
			}

			if ($this->log) {
			$log = $this->log;
			$this->log = null;

			$conn = $log->establishConnection('w');
			queryfx(
			$conn,
			'UPDATE %T SET lockReleased = UNIX_TIMESTAMP() WHERE id = %d',
			$log->getTableName(),
			$log->getID());
			}
			}

			private function shouldLogLock() {
			if ($this->disableLogging) {
			return false;
			}

			$policy = id(new PhabricatorDaemonLockLogGarbageCollector())
			->getRetentionPolicy();
			if (!$policy) {
			return false;
			}

			return true;
			}

			private function newLockContext() {
			$context = array(
			'pid' => getmypid(),
			'host' => php_uname('n'),
			'sapi' => php_sapi_name(),
			);

			global $argv;
			if ($argv) {
			$context['argv'] = $argv;
			}

			$access_log = null;

			// TODO: There's currently no cohesive way to get the parameterized access
			// log for the current request across different request types. Web requests
			// have an "AccessLog", SSH requests have an "SSHLog", and other processes
			// (like scripts) have no log. But there's no method to say "give me any
			// log you've got". For now, just test if we have a web request and use the
			// "AccessLog" if we do, since that's the only one we actually read any
			// parameters from.

			// NOTE: "PhabricatorStartup" is only available from web requests, not
			// from CLI scripts.
			if (class_exists('PhabricatorStartup', false)) {
			$access_log = PhabricatorAccessLog::getLog();
			}

			if ($access_log) {
			$controller = $access_log->getData('C');
			if ($controller) {
			$context['controller'] = $controller;
			}

			$method = $access_log->getData('m');
			if ($method) {
			$context['method'] = $method;
			}
			}

			return $context;
			}

			private function newHint($lock_name, $wait) {
			if (!$this->shouldLogLock()) {
			return pht(
			'Enable the lock log for more detailed information about '.
			'which process is holding this lock.');
			}

			$now = PhabricatorTime::getNow();

			// First, look for recent logs. If other processes have been acquiring and
			// releasing this lock while we've been waiting, this is more likely to be
			// a contention/throughput issue than an issue with something hung while
			// holding the lock.
			$limit = 100;
			$logs = id(new PhabricatorDaemonLockLog())->loadAllWhere(
			'lockName = %s AND dateCreated >= %d ORDER BY id ASC LIMIT %d',
			$lock_name,
			($now - $wait),
			$limit);

			if ($logs) {
			if (count($logs) === $limit) {
			return pht(
			'During the last %s second(s) spent waiting for the lock, more '.
			'than %s other process(es) acquired it, so this is likely a '.
			'bottleneck. Use "bin/lock log --name %s" to review log activity.',
			new PhutilNumber($wait),
			new PhutilNumber($limit),
			$lock_name);
			} else {
			return pht(
			'During the last %s second(s) spent waiting for the lock, %s '.
			'other process(es) acquired it, so this is likely a '.
			'bottleneck. Use "bin/lock log --name %s" to review log activity.',
			new PhutilNumber($wait),
			phutil_count($logs),
			$lock_name);
			}
			}

			$last_log = id(new PhabricatorDaemonLockLog())->loadOneWhere(
			'lockName = %s ORDER BY id DESC LIMIT 1',
			$lock_name);

			if ($last_log) {
			$info = array();

			$acquired = $last_log->getDateCreated();
			$context = $last_log->getLockContext();

			$process_info = array();

			$pid = idx($context, 'pid');
			if ($pid) {
			$process_info[] = 'pid='.$pid;
			}

			$host = idx($context, 'host');
			if ($host) {
			$process_info[] = 'host='.$host;
			}

			$sapi = idx($context, 'sapi');
			if ($sapi) {
			$process_info[] = 'sapi='.$sapi;
			}

			$argv = idx($context, 'argv');
			if ($argv) {
			$process_info[] = 'argv='.(string)csprintf('%LR', $argv);
			}

			$controller = idx($context, 'controller');
			if ($controller) {
			$process_info[] = 'controller='.$controller;
			}

			$method = idx($context, 'method');
			if ($method) {
			$process_info[] = 'method='.$method;
			}

			$process_info = implode(', ', $process_info);

			$info[] = pht(
			'This lock was most recently acquired by a process (%s) '.
			'%s second(s) ago.',
			$process_info,
			new PhutilNumber($now - $acquired));

			$released = $last_log->getLockReleased();
			if ($released) {
			$info[] = pht(
			'This lock was released %s second(s) ago.',
			new PhutilNumber($now - $released));
			} else {
			$info[] = pht('There is no record of this lock being released.');
			}

			return implode(' ', $info);
			}

			return pht(
			'Found no records of processes acquiring or releasing this lock.');
			}

			}

Owner Packages

No Owners

phabricator/src/infrastructure/util/PhabricatorGlobalLock.php51cb7a3db9e8master

PhabricatorGlobalLock.phpView Options

Owner Packages

phabricator/src/infrastructure/util/PhabricatorGlobalLock.php
51cb7a3db9e8master

PhabricatorGlobalLock.php
View Options