diff --git a/src/applications/settings/setting/PhabricatorOlderInlinesSetting.php b/src/applications/settings/setting/PhabricatorOlderInlinesSetting.php index ac1f57f457..e7f7ab3dd1 100644 --- a/src/applications/settings/setting/PhabricatorOlderInlinesSetting.php +++ b/src/applications/settings/setting/PhabricatorOlderInlinesSetting.php @@ -1,43 +1,43 @@ pht('Enabled'), self::VALUE_GHOST_INLINES_DISABLED => pht('Disabled'), ); } } diff --git a/src/applications/settings/setting/PhabricatorTitleGlyphsSetting.php b/src/applications/settings/setting/PhabricatorTitleGlyphsSetting.php index 04efbcb0f0..e5947c4875 100644 --- a/src/applications/settings/setting/PhabricatorTitleGlyphsSetting.php +++ b/src/applications/settings/setting/PhabricatorTitleGlyphsSetting.php @@ -1,41 +1,42 @@ pht("Use Unicode Glyphs: \xE2\x9A\x99"), self::VALUE_TITLE_TEXT => pht('Use Plain Text: [Differential]'), ); } } diff --git a/src/applications/settings/setting/PhabricatorUnifiedDiffsSetting.php b/src/applications/settings/setting/PhabricatorUnifiedDiffsSetting.php index 5a94397418..50ac8b3042 100644 --- a/src/applications/settings/setting/PhabricatorUnifiedDiffsSetting.php +++ b/src/applications/settings/setting/PhabricatorUnifiedDiffsSetting.php @@ -1,43 +1,43 @@ pht('On Small Screens'), self::VALUE_ALWAYS_UNIFIED => pht('Always'), ); } } diff --git a/src/applications/subscriptions/command/PhabricatorSubscriptionsSubscribeEmailCommand.php b/src/applications/subscriptions/command/PhabricatorSubscriptionsSubscribeEmailCommand.php index 0c361ad2c6..226fba34e5 100644 --- a/src/applications/subscriptions/command/PhabricatorSubscriptionsSubscribeEmailCommand.php +++ b/src/applications/subscriptions/command/PhabricatorSubscriptionsSubscribeEmailCommand.php @@ -1,75 +1,74 @@ setViewer($viewer) ->setAllowedTypes( array( PhabricatorPeopleUserPHIDType::TYPECONST, PhabricatorProjectProjectPHIDType::TYPECONST, )) ->setObjectList(implode(' ', $argv)) ->setAllowPartialResults(true) ->execute(); $xactions = array(); $xactions[] = $object->getApplicationTransactionTemplate() ->setTransactionType(PhabricatorTransactions::TYPE_SUBSCRIBERS) ->setNewValue( array( '+' => array_fuse($subscriber_phids), )); return $xactions; } } diff --git a/src/applications/system/management/PhabricatorSystemRemoveDestroyWorkflow.php b/src/applications/system/management/PhabricatorSystemRemoveDestroyWorkflow.php index bd6b4e361a..b66ad82b80 100644 --- a/src/applications/system/management/PhabricatorSystemRemoveDestroyWorkflow.php +++ b/src/applications/system/management/PhabricatorSystemRemoveDestroyWorkflow.php @@ -1,181 +1,180 @@ setName('destroy') ->setSynopsis(pht('Permanently destroy objects.')) ->setExamples('**destroy** [__options__] __object__ ...') ->setArguments( array( array( 'name' => 'force', 'help' => pht('Destroy objects without prompting.'), ), array( 'name' => 'objects', 'wildcard' => true, ), )); } public function execute(PhutilArgumentParser $args) { $console = PhutilConsole::getConsole(); $object_names = $args->getArg('objects'); if (!$object_names) { throw new PhutilArgumentUsageException( pht('Specify one or more objects to destroy.')); } $object_query = id(new PhabricatorObjectQuery()) ->setViewer($this->getViewer()) ->withNames($object_names); $object_query->execute(); $named_objects = $object_query->getNamedResults(); foreach ($object_names as $object_name) { if (empty($named_objects[$object_name])) { throw new PhutilArgumentUsageException( pht('No such object "%s" exists!', $object_name)); } } foreach ($named_objects as $object_name => $object) { if (!($object instanceof PhabricatorDestructibleInterface)) { throw new PhutilArgumentUsageException( pht( 'Object "%s" can not be destroyed (it does not implement %s).', $object_name, 'PhabricatorDestructibleInterface')); } } $banner = <<writeOut("\n\n%s\n\n", $banner); $console->writeOut( "** %s ** %s\n\n%s\n\n". "** %s ** %s\n\n%s\n\n", pht('IMPORTANT'), pht('DATA WILL BE PERMANENTLY DESTROYED'), phutil_console_wrap( pht( 'Objects will be permanently destroyed. There is no way to '. 'undo this operation or ever retrieve this data unless you '. 'maintain external backups.')), pht('IMPORTANT'), pht('DELETING OBJECTS OFTEN BREAKS THINGS'), phutil_console_wrap( pht( 'Destroying objects may cause related objects to stop working, '. 'and may leave scattered references to objects which no longer '. 'exist. In most cases, it is much better to disable or archive '. 'objects instead of destroying them. This risk is greatest when '. 'deleting complex or highly connected objects like repositories, '. 'projects and users.'. "\n\n". 'These tattered edges are an expected consequence of destroying '. - 'objects, and the Phabricator upstream will not help you fix '. - 'them. We strongly recommend disabling or archiving objects '. - 'instead.'))); + 'objects, and the upstream will not help you fix them. We '. + 'strongly recommend disabling or archiving objects instead.'))); $phids = mpull($named_objects, 'getPHID'); $handles = PhabricatorUser::getOmnipotentUser()->loadHandles($phids); $console->writeOut( pht( 'These %s object(s) will be destroyed forever:', phutil_count($named_objects))."\n\n"); foreach ($named_objects as $object_name => $object) { $phid = $object->getPHID(); $console->writeOut( " - %s (%s) %s\n", $object_name, get_class($object), $handles[$phid]->getFullName()); } $force = $args->getArg('force'); if (!$force) { $ok = $console->confirm( pht( 'Are you absolutely certain you want to destroy these %s object(s)?', phutil_count($named_objects))); if (!$ok) { throw new PhutilArgumentUsageException( pht('Aborted, your objects are safe.')); } } $console->writeOut("%s\n", pht('Destroying objects...')); $notes = array(); foreach ($named_objects as $object_name => $object) { $console->writeOut( pht( "Destroying %s **%s**...\n", get_class($object), $object_name)); $engine = id(new PhabricatorDestructionEngine()) ->setCollectNotes(true); $engine->destroyObject($object); foreach ($engine->getNotes() as $note) { $notes[] = $note; } } $console->writeOut( "%s\n", pht( 'Permanently destroyed %s object(s).', phutil_count($named_objects))); if ($notes) { id(new PhutilConsoleList()) ->addItems($notes) ->draw(); } return 0; } } diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRef.php b/src/infrastructure/cluster/PhabricatorDatabaseRef.php index da9f5e0d5e..1eb232ad86 100644 --- a/src/infrastructure/cluster/PhabricatorDatabaseRef.php +++ b/src/infrastructure/cluster/PhabricatorDatabaseRef.php @@ -1,748 +1,748 @@ host = $host; return $this; } public function getHost() { return $this->host; } public function setPort($port) { $this->port = $port; return $this; } public function getPort() { return $this->port; } public function setUser($user) { $this->user = $user; return $this; } public function getUser() { return $this->user; } public function setPass(PhutilOpaqueEnvelope $pass) { $this->pass = $pass; return $this; } public function getPass() { return $this->pass; } public function setIsMaster($is_master) { $this->isMaster = $is_master; return $this; } public function getIsMaster() { return $this->isMaster; } public function setDisabled($disabled) { $this->disabled = $disabled; return $this; } public function getDisabled() { return $this->disabled; } public function setConnectionLatency($connection_latency) { $this->connectionLatency = $connection_latency; return $this; } public function getConnectionLatency() { return $this->connectionLatency; } public function setConnectionStatus($connection_status) { $this->connectionStatus = $connection_status; return $this; } public function getConnectionStatus() { if ($this->connectionStatus === null) { throw new PhutilInvalidStateException('queryAll'); } return $this->connectionStatus; } public function setConnectionMessage($connection_message) { $this->connectionMessage = $connection_message; return $this; } public function getConnectionMessage() { return $this->connectionMessage; } public function setReplicaStatus($replica_status) { $this->replicaStatus = $replica_status; return $this; } public function getReplicaStatus() { return $this->replicaStatus; } public function setReplicaMessage($replica_message) { $this->replicaMessage = $replica_message; return $this; } public function getReplicaMessage() { return $this->replicaMessage; } public function setReplicaDelay($replica_delay) { $this->replicaDelay = $replica_delay; return $this; } public function getReplicaDelay() { return $this->replicaDelay; } public function setIsIndividual($is_individual) { $this->isIndividual = $is_individual; return $this; } public function getIsIndividual() { return $this->isIndividual; } public function setIsDefaultPartition($is_default_partition) { $this->isDefaultPartition = $is_default_partition; return $this; } public function getIsDefaultPartition() { return $this->isDefaultPartition; } public function setUsePersistentConnections($use_persistent_connections) { $this->usePersistentConnections = $use_persistent_connections; return $this; } public function getUsePersistentConnections() { return $this->usePersistentConnections; } public function setApplicationMap(array $application_map) { $this->applicationMap = $application_map; return $this; } public function getApplicationMap() { return $this->applicationMap; } public function getPartitionStateForCommit() { $state = PhabricatorEnv::getEnvConfig('cluster.databases'); foreach ($state as $key => $value) { // Don't store passwords, since we don't care if they differ and // users may find it surprising. unset($state[$key]['pass']); } return phutil_json_encode($state); } public function setMasterRef(PhabricatorDatabaseRef $master_ref) { $this->masterRef = $master_ref; return $this; } public function getMasterRef() { return $this->masterRef; } public function addReplicaRef(PhabricatorDatabaseRef $replica_ref) { $this->replicaRefs[] = $replica_ref; return $this; } public function getReplicaRefs() { return $this->replicaRefs; } public function getDisplayName() { return $this->getRefKey(); } public function getRefKey() { $host = $this->getHost(); $port = $this->getPort(); if (strlen($port)) { return "{$host}:{$port}"; } return $host; } public static function getConnectionStatusMap() { return array( self::STATUS_OKAY => array( 'icon' => 'fa-exchange', 'color' => 'green', 'label' => pht('Okay'), ), self::STATUS_FAIL => array( 'icon' => 'fa-times', 'color' => 'red', 'label' => pht('Failed'), ), self::STATUS_AUTH => array( 'icon' => 'fa-key', 'color' => 'red', 'label' => pht('Invalid Credentials'), ), self::STATUS_REPLICATION_CLIENT => array( 'icon' => 'fa-eye-slash', 'color' => 'yellow', 'label' => pht('Missing Permission'), ), ); } public static function getReplicaStatusMap() { return array( self::REPLICATION_OKAY => array( 'icon' => 'fa-download', 'color' => 'green', 'label' => pht('Okay'), ), self::REPLICATION_MASTER_REPLICA => array( 'icon' => 'fa-database', 'color' => 'red', 'label' => pht('Replicating Master'), ), self::REPLICATION_REPLICA_NONE => array( 'icon' => 'fa-download', 'color' => 'red', 'label' => pht('Not A Replica'), ), self::REPLICATION_SLOW => array( 'icon' => 'fa-hourglass', 'color' => 'red', 'label' => pht('Slow Replication'), ), self::REPLICATION_NOT_REPLICATING => array( 'icon' => 'fa-exclamation-triangle', 'color' => 'red', 'label' => pht('Not Replicating'), ), ); } public static function getClusterRefs() { $cache = PhabricatorCaches::getRequestCache(); $refs = $cache->getKey(self::KEY_REFS); if (!$refs) { $refs = self::newRefs(); $cache->setKey(self::KEY_REFS, $refs); } return $refs; } public static function getLiveIndividualRef() { $cache = PhabricatorCaches::getRequestCache(); $ref = $cache->getKey(self::KEY_INDIVIDUAL); if (!$ref) { $ref = self::newIndividualRef(); $cache->setKey(self::KEY_INDIVIDUAL, $ref); } return $ref; } public static function newRefs() { $default_port = PhabricatorEnv::getEnvConfig('mysql.port'); $default_port = nonempty($default_port, 3306); $default_user = PhabricatorEnv::getEnvConfig('mysql.user'); $default_pass = PhabricatorEnv::getEnvConfig('mysql.pass'); $default_pass = phutil_string_cast($default_pass); $default_pass = new PhutilOpaqueEnvelope($default_pass); $config = PhabricatorEnv::getEnvConfig('cluster.databases'); return id(new PhabricatorDatabaseRefParser()) ->setDefaultPort($default_port) ->setDefaultUser($default_user) ->setDefaultPass($default_pass) ->newRefs($config); } public static function queryAll() { $refs = self::getActiveDatabaseRefs(); return self::queryRefs($refs); } private static function queryRefs(array $refs) { foreach ($refs as $ref) { $conn = $ref->newManagementConnection(); $t_start = microtime(true); $replica_status = false; try { $replica_status = queryfx_one($conn, 'SHOW SLAVE STATUS'); $ref->setConnectionStatus(self::STATUS_OKAY); } catch (AphrontAccessDeniedQueryException $ex) { $ref->setConnectionStatus(self::STATUS_REPLICATION_CLIENT); $ref->setConnectionMessage( pht( 'No permission to run "SHOW SLAVE STATUS". Grant this user '. - '"REPLICATION CLIENT" permission to allow Phabricator to '. + '"REPLICATION CLIENT" permission to allow this server to '. 'monitor replica health.')); } catch (AphrontInvalidCredentialsQueryException $ex) { $ref->setConnectionStatus(self::STATUS_AUTH); $ref->setConnectionMessage($ex->getMessage()); } catch (AphrontQueryException $ex) { $ref->setConnectionStatus(self::STATUS_FAIL); $class = get_class($ex); $message = $ex->getMessage(); $ref->setConnectionMessage( pht( '%s: %s', get_class($ex), $ex->getMessage())); } $t_end = microtime(true); $ref->setConnectionLatency($t_end - $t_start); if ($replica_status !== false) { $is_replica = (bool)$replica_status; if ($ref->getIsMaster() && $is_replica) { $ref->setReplicaStatus(self::REPLICATION_MASTER_REPLICA); $ref->setReplicaMessage( pht( 'This host has a "master" role, but is replicating data from '. 'another host ("%s")!', idx($replica_status, 'Master_Host'))); } else if (!$ref->getIsMaster() && !$is_replica) { $ref->setReplicaStatus(self::REPLICATION_REPLICA_NONE); $ref->setReplicaMessage( pht( 'This host has a "replica" role, but is not replicating data '. 'from a master (no output from "SHOW SLAVE STATUS").')); } else { $ref->setReplicaStatus(self::REPLICATION_OKAY); } if ($is_replica) { $latency = idx($replica_status, 'Seconds_Behind_Master'); if (!strlen($latency)) { $ref->setReplicaStatus(self::REPLICATION_NOT_REPLICATING); } else { $latency = (int)$latency; $ref->setReplicaDelay($latency); if ($latency > 30) { $ref->setReplicaStatus(self::REPLICATION_SLOW); $ref->setReplicaMessage( pht( 'This replica is lagging far behind the master. Data is at '. 'risk!')); } } } } } return $refs; } public function newManagementConnection() { return $this->newConnection( array( 'retries' => 0, 'timeout' => 2, )); } public function newApplicationConnection($database) { return $this->newConnection( array( 'database' => $database, )); } public function isSevered() { // If we only have an individual database, never sever our connection to // it, at least for now. It's possible that using the same severing rules // might eventually make sense to help alleviate load-related failures, // but we should wait for all the cluster stuff to stabilize first. if ($this->getIsIndividual()) { return false; } if ($this->didFailToConnect) { return true; } $record = $this->getHealthRecord(); $is_healthy = $record->getIsHealthy(); if (!$is_healthy) { return true; } return false; } public function isReachable(AphrontDatabaseConnection $connection) { $record = $this->getHealthRecord(); $should_check = $record->getShouldCheck(); if ($this->isSevered() && !$should_check) { return false; } $this->connectionException = null; try { $connection->openConnection(); $reachable = true; } catch (AphrontSchemaQueryException $ex) { // We get one of these if the database we're trying to select does not // exist. In this case, just re-throw the exception. This is expected // during first-time setup, when databases like "config" will not exist // yet. throw $ex; } catch (Exception $ex) { $this->connectionException = $ex; $reachable = false; } if ($should_check) { $record->didHealthCheck($reachable); } if (!$reachable) { $this->didFailToConnect = true; } return $reachable; } public function checkHealth() { $health = $this->getHealthRecord(); $should_check = $health->getShouldCheck(); if ($should_check) { // This does an implicit health update. $connection = $this->newManagementConnection(); $this->isReachable($connection); } return $this; } private function getHealthRecordCacheKey() { $host = $this->getHost(); $port = $this->getPort(); $key = self::KEY_HEALTH; return "{$key}({$host}, {$port})"; } public function getHealthRecord() { if (!$this->healthRecord) { $this->healthRecord = new PhabricatorClusterServiceHealthRecord( $this->getHealthRecordCacheKey()); } return $this->healthRecord; } public function getConnectionException() { return $this->connectionException; } public static function getActiveDatabaseRefs() { $refs = array(); foreach (self::getMasterDatabaseRefs() as $ref) { $refs[] = $ref; } foreach (self::getReplicaDatabaseRefs() as $ref) { $refs[] = $ref; } return $refs; } public static function getAllMasterDatabaseRefs() { $refs = self::getClusterRefs(); if (!$refs) { return array(self::getLiveIndividualRef()); } $masters = array(); foreach ($refs as $ref) { if ($ref->getIsMaster()) { $masters[] = $ref; } } return $masters; } public static function getMasterDatabaseRefs() { $refs = self::getAllMasterDatabaseRefs(); return self::getEnabledRefs($refs); } public function isApplicationHost($database) { return isset($this->applicationMap[$database]); } public function loadRawMySQLConfigValue($key) { $conn = $this->newManagementConnection(); try { $value = queryfx_one($conn, 'SELECT @@%C', $key); // NOTE: Although MySQL allows us to escape configuration values as if // they are column names, the escaping is included in the column name // of the return value: if we select "@@`x`", we get back a column named // "@@`x`", not "@@x" as we might expect. $value = head($value); } catch (AphrontQueryException $ex) { $value = null; } return $value; } public static function getMasterDatabaseRefForApplication($application) { $masters = self::getMasterDatabaseRefs(); $application_master = null; $default_master = null; foreach ($masters as $master) { if ($master->isApplicationHost($application)) { $application_master = $master; break; } if ($master->getIsDefaultPartition()) { $default_master = $master; } } if ($application_master) { $masters = array($application_master); } else if ($default_master) { $masters = array($default_master); } else { $masters = array(); } $masters = self::getEnabledRefs($masters); $master = head($masters); return $master; } public static function newIndividualRef() { $default_user = PhabricatorEnv::getEnvConfig('mysql.user'); $default_pass = new PhutilOpaqueEnvelope( PhabricatorEnv::getEnvConfig('mysql.pass')); $default_host = PhabricatorEnv::getEnvConfig('mysql.host'); $default_port = PhabricatorEnv::getEnvConfig('mysql.port'); return id(new self()) ->setUser($default_user) ->setPass($default_pass) ->setHost($default_host) ->setPort($default_port) ->setIsIndividual(true) ->setIsMaster(true) ->setIsDefaultPartition(true) ->setUsePersistentConnections(false); } public static function getAllReplicaDatabaseRefs() { $refs = self::getClusterRefs(); if (!$refs) { return array(); } $replicas = array(); foreach ($refs as $ref) { if ($ref->getIsMaster()) { continue; } $replicas[] = $ref; } return $replicas; } public static function getReplicaDatabaseRefs() { $refs = self::getAllReplicaDatabaseRefs(); return self::getEnabledRefs($refs); } private static function getEnabledRefs(array $refs) { foreach ($refs as $key => $ref) { if ($ref->getDisabled()) { unset($refs[$key]); } } return $refs; } public static function getReplicaDatabaseRefForApplication($application) { $replicas = self::getReplicaDatabaseRefs(); $application_replicas = array(); $default_replicas = array(); foreach ($replicas as $replica) { $master = $replica->getMasterRef(); if ($master->isApplicationHost($application)) { $application_replicas[] = $replica; } if ($master->getIsDefaultPartition()) { $default_replicas[] = $replica; } } if ($application_replicas) { $replicas = $application_replicas; } else { $replicas = $default_replicas; } $replicas = self::getEnabledRefs($replicas); // TODO: We may have multiple replicas to choose from, and could make // more of an effort to pick the "best" one here instead of always // picking the first one. Once we've picked one, we should try to use // the same replica for the rest of the request, though. return head($replicas); } private function newConnection(array $options) { // If we believe the database is unhealthy, don't spend as much time // trying to connect to it, since it's likely to continue to fail and // hammering it can only make the problem worse. $record = $this->getHealthRecord(); if ($record->getIsHealthy()) { $default_retries = 3; $default_timeout = 10; } else { $default_retries = 0; $default_timeout = 2; } $spec = $options + array( 'user' => $this->getUser(), 'pass' => $this->getPass(), 'host' => $this->getHost(), 'port' => $this->getPort(), 'database' => null, 'retries' => $default_retries, 'timeout' => $default_timeout, 'persistent' => $this->getUsePersistentConnections(), ); $is_cli = (php_sapi_name() == 'cli'); $use_persistent = false; if (!empty($spec['persistent']) && !$is_cli) { $use_persistent = true; } unset($spec['persistent']); $connection = self::newRawConnection($spec); // If configured, use persistent connections. See T11672 for details. if ($use_persistent) { $connection->setPersistent($use_persistent); } // Unless this is a script running from the CLI, prevent any query from // running for more than 30 seconds. See T10849 for details. if (!$is_cli) { $connection->setQueryTimeout(30); } return $connection; } public static function newRawConnection(array $options) { if (extension_loaded('mysqli')) { return new AphrontMySQLiDatabaseConnection($options); } else { return new AphrontMySQLDatabaseConnection($options); } } } diff --git a/src/infrastructure/cluster/PhabricatorDatabaseRefParser.php b/src/infrastructure/cluster/PhabricatorDatabaseRefParser.php index db498e8a82..83eaa096b8 100644 --- a/src/infrastructure/cluster/PhabricatorDatabaseRefParser.php +++ b/src/infrastructure/cluster/PhabricatorDatabaseRefParser.php @@ -1,221 +1,221 @@ defaultPort = $default_port; return $this; } public function getDefaultPort() { return $this->defaultPort; } public function setDefaultUser($default_user) { $this->defaultUser = $default_user; return $this; } public function getDefaultUser() { return $this->defaultUser; } public function setDefaultPass($default_pass) { $this->defaultPass = $default_pass; return $this; } public function getDefaultPass() { return $this->defaultPass; } public function newRefs(array $config) { $default_port = $this->getDefaultPort(); $default_user = $this->getDefaultUser(); $default_pass = $this->getDefaultPass(); $refs = array(); $master_count = 0; foreach ($config as $key => $server) { $host = $server['host']; $port = idx($server, 'port', $default_port); $user = idx($server, 'user', $default_user); $disabled = idx($server, 'disabled', false); $pass = idx($server, 'pass'); if ($pass) { $pass = new PhutilOpaqueEnvelope($pass); } else { $pass = clone $default_pass; } $role = $server['role']; $is_master = ($role == 'master'); $use_persistent = (bool)idx($server, 'persistent', false); $ref = id(new PhabricatorDatabaseRef()) ->setHost($host) ->setPort($port) ->setUser($user) ->setPass($pass) ->setDisabled($disabled) ->setIsMaster($is_master) ->setUsePersistentConnections($use_persistent); if ($is_master) { $master_count++; } $refs[$key] = $ref; } $is_partitioned = ($master_count > 1); if ($is_partitioned) { $default_ref = null; $partition_map = array(); foreach ($refs as $key => $ref) { if (!$ref->getIsMaster()) { continue; } $server = $config[$key]; $partition = idx($server, 'partition'); if (!is_array($partition)) { throw new Exception( pht( - 'Phabricator is configured with multiple master databases, '. + 'This server is configured with multiple master databases, '. 'but master "%s" is missing a "partition" configuration key to '. 'define application partitioning.', $ref->getRefKey())); } $application_map = array(); foreach ($partition as $application) { if ($application === 'default') { if ($default_ref) { throw new Exception( pht( 'Multiple masters (databases "%s" and "%s") specify that '. 'they are the "default" partition. Only one master may be '. 'the default.', $ref->getRefKey(), $default_ref->getRefKey())); } else { $default_ref = $ref; $ref->setIsDefaultPartition(true); } } else if (isset($partition_map[$application])) { throw new Exception( pht( 'Multiple masters (databases "%s" and "%s") specify that '. 'they are the partition for application "%s". Each '. 'application may be allocated to only one partition.', $partition_map[$application]->getRefKey(), $ref->getRefKey(), $application)); } else { // TODO: We should check that the application is valid, to // prevent typos in application names. However, we do not // currently have an efficient way to enumerate all of the valid // application database names. $partition_map[$application] = $ref; $application_map[$application] = $application; } } $ref->setApplicationMap($application_map); } } else { // If we only have one master, make it the default. foreach ($refs as $ref) { if ($ref->getIsMaster()) { $ref->setIsDefaultPartition(true); } } } $ref_map = array(); $master_keys = array(); foreach ($refs as $ref) { $ref_key = $ref->getRefKey(); if (isset($ref_map[$ref_key])) { throw new Exception( pht( 'Multiple configured databases have the same internal '. 'key, "%s". You may have listed a database multiple times.', $ref_key)); } else { $ref_map[$ref_key] = $ref; if ($ref->getIsMaster()) { $master_keys[] = $ref_key; } } } foreach ($refs as $key => $ref) { if ($ref->getIsMaster()) { continue; } $server = $config[$key]; $partition = idx($server, 'partition'); if ($partition !== null) { throw new Exception( pht( 'Database "%s" is configured as a replica, but specifies a '. '"partition". Only master databases may have a partition '. 'configuration. Replicas use the same configuration as the '. 'master they follow.', $ref->getRefKey())); } $master_key = idx($server, 'master'); if ($master_key === null) { if ($is_partitioned) { throw new Exception( pht( 'Database "%s" is configured as a replica, but does not '. 'specify which "master" it follows in configuration. Valid '. 'masters are: %s.', $ref->getRefKey(), implode(', ', $master_keys))); } else if ($master_keys) { $master_key = head($master_keys); } else { throw new Exception( pht( 'Database "%s" is configured as a replica, but there is no '. 'master configured.', $ref->getRefKey())); } } if (!isset($ref_map[$master_key])) { throw new Exception( pht( 'Database "%s" is configured as a replica and specifies a '. 'master ("%s"), but that master is not a valid master. Valid '. 'masters are: %s.', $ref->getRefKey(), $master_key, implode(', ', $master_keys))); } $master_ref = $ref_map[$master_key]; $ref->setMasterRef($ref_map[$master_key]); $master_ref->addReplicaRef($ref); } return array_values($refs); } } diff --git a/src/infrastructure/daemon/workers/management/PhabricatorWorkerManagementFloodWorkflow.php b/src/infrastructure/daemon/workers/management/PhabricatorWorkerManagementFloodWorkflow.php index fb067ddaac..5719f82b18 100644 --- a/src/infrastructure/daemon/workers/management/PhabricatorWorkerManagementFloodWorkflow.php +++ b/src/infrastructure/daemon/workers/management/PhabricatorWorkerManagementFloodWorkflow.php @@ -1,50 +1,50 @@ setName('flood') ->setExamples('**flood**') ->setSynopsis( pht( 'Flood the queue with test tasks. This command is intended for '. - 'use when developing and debugging Phabricator.')) + 'use during development and debugging.')) ->setArguments( array( array( 'name' => 'duration', 'param' => 'seconds', 'help' => pht( 'Queue tasks which require a specific amount of wall time to '. 'complete. By default, tasks complete as quickly as possible.'), 'default' => 0, ), )); } public function execute(PhutilArgumentParser $args) { $console = PhutilConsole::getConsole(); $duration = (float)$args->getArg('duration'); $console->writeOut( "%s\n", pht('Adding many test tasks to worker queue. Use ^C to exit.')); $n = 0; while (true) { PhabricatorWorker::scheduleTask( 'PhabricatorTestWorker', array( 'duration' => $duration, )); if (($n++ % 100) === 0) { $console->writeOut('.'); } } } } diff --git a/src/infrastructure/storage/lisk/PhabricatorLiskDAO.php b/src/infrastructure/storage/lisk/PhabricatorLiskDAO.php index 85adb4fc29..93f623338f 100644 --- a/src/infrastructure/storage/lisk/PhabricatorLiskDAO.php +++ b/src/infrastructure/storage/lisk/PhabricatorLiskDAO.php @@ -1,353 +1,353 @@ '; const CONFIG_APPLICATION_SERIALIZERS = 'phabricator/serializers'; /* -( Configuring Storage )------------------------------------------------ */ /** * @task config */ public static function pushStorageNamespace($namespace) { self::$namespaceStack[] = $namespace; } /** * @task config */ public static function popStorageNamespace() { array_pop(self::$namespaceStack); } /** * @task config */ public static function getDefaultStorageNamespace() { return PhabricatorEnv::getEnvConfig('storage.default-namespace'); } /** * @task config */ public static function getStorageNamespace() { $namespace = end(self::$namespaceStack); if (!strlen($namespace)) { $namespace = self::getDefaultStorageNamespace(); } if (!strlen($namespace)) { throw new Exception(pht('No storage namespace configured!')); } return $namespace; } public function setForcedStorageNamespace($namespace) { $this->forcedNamespace = $namespace; return $this; } /** * @task config */ protected function establishLiveConnection($mode) { $namespace = self::getStorageNamespace(); $database = $namespace.'_'.$this->getApplicationName(); $is_readonly = PhabricatorEnv::isReadOnly(); if ($is_readonly && ($mode != 'r')) { $this->raiseImproperWrite($database); } $connection = $this->newClusterConnection( $this->getApplicationName(), $database, $mode); // TODO: This should be testing if the mode is "r", but that would probably // break a lot of things. Perform a more narrow test for readonly mode // until we have greater certainty that this works correctly most of the // time. if ($is_readonly) { $connection->setReadOnly(true); } return $connection; } private function newClusterConnection($application, $database, $mode) { $master = PhabricatorDatabaseRef::getMasterDatabaseRefForApplication( $application); $master_exception = null; if ($master && !$master->isSevered()) { $connection = $master->newApplicationConnection($database); if ($master->isReachable($connection)) { return $connection; } else { if ($mode == 'w') { $this->raiseImpossibleWrite($database); } PhabricatorEnv::setReadOnly( true, PhabricatorEnv::READONLY_UNREACHABLE); $master_exception = $master->getConnectionException(); } } $replica = PhabricatorDatabaseRef::getReplicaDatabaseRefForApplication( $application); if ($replica) { $connection = $replica->newApplicationConnection($database); $connection->setReadOnly(true); if ($replica->isReachable($connection)) { if ($master_exception) { // If we ended up here as the result of a failover, log the // exception. This is seriously bad news even if we are able // to recover from it. $proxy_exception = new PhutilProxyException( pht( 'Failed to connect to master database ("%s"), failing over '. 'into read-only mode.', $database), $master_exception); phlog($proxy_exception); } return $connection; } } if (!$master && !$replica) { $this->raiseUnconfigured($database); } $this->raiseUnreachable($database, $master_exception); } private function raiseImproperWrite($database) { throw new PhabricatorClusterImproperWriteException( pht( 'Unable to establish a write-mode connection (to application '. - 'database "%s") because Phabricator is in read-only mode. Whatever '. + 'database "%s") because this server is in read-only mode. Whatever '. 'you are trying to do does not function correctly in read-only mode.', $database)); } private function raiseImpossibleWrite($database) { throw new PhabricatorClusterImpossibleWriteException( pht( 'Unable to connect to master database ("%s"). This is a severe '. 'failure; your request did not complete.', $database)); } private function raiseUnconfigured($database) { throw new Exception( pht( 'Unable to establish a connection to any database host '. '(while trying "%s"). No masters or replicas are configured.', $database)); } private function raiseUnreachable($database, Exception $proxy = null) { $message = pht( 'Unable to establish a connection to any database host '. '(while trying "%s"). All masters and replicas are completely '. 'unreachable.', $database); if ($proxy) { $proxy_message = pht( '%s: %s', get_class($proxy), $proxy->getMessage()); $message = $message."\n\n".$proxy_message; } throw new PhabricatorClusterStrandedException($message); } /** * @task config */ public function getTableName() { $str = 'phabricator'; $len = strlen($str); $class = strtolower(get_class($this)); if (!strncmp($class, $str, $len)) { $class = substr($class, $len); } $app = $this->getApplicationName(); if (!strncmp($class, $app, strlen($app))) { $class = substr($class, strlen($app)); } if (strlen($class)) { return $app.'_'.$class; } else { return $app; } } /** * @task config */ abstract public function getApplicationName(); protected function getDatabaseName() { if ($this->forcedNamespace) { $namespace = $this->forcedNamespace; } else { $namespace = self::getStorageNamespace(); } return $namespace.'_'.$this->getApplicationName(); } /** * Break a list of escaped SQL statement fragments (e.g., VALUES lists for * INSERT, previously built with @{function:qsprintf}) into chunks which will * fit under the MySQL 'max_allowed_packet' limit. * * If a statement is too large to fit within the limit, it is broken into * its own chunk (but might fail when the query executes). */ public static function chunkSQL( array $fragments, $limit = null) { if ($limit === null) { // NOTE: Hard-code this at 1MB for now, minus a 10% safety buffer. // Eventually we could query MySQL or let the user configure it. $limit = (int)((1024 * 1024) * 0.90); } $result = array(); $chunk = array(); $len = 0; $glue_len = strlen(', '); foreach ($fragments as $fragment) { if ($fragment instanceof PhutilQueryString) { $this_len = strlen($fragment->getUnmaskedString()); } else { $this_len = strlen($fragment); } if ($chunk) { // Chunks after the first also imply glue. $this_len += $glue_len; } if ($len + $this_len <= $limit) { $len += $this_len; $chunk[] = $fragment; } else { if ($chunk) { $result[] = $chunk; } $len = ($this_len - $glue_len); $chunk = array($fragment); } } if ($chunk) { $result[] = $chunk; } return $result; } protected function assertAttached($property) { if ($property === self::ATTACHABLE) { throw new PhabricatorDataNotAttachedException($this); } return $property; } protected function assertAttachedKey($value, $key) { $this->assertAttached($value); if (!array_key_exists($key, $value)) { throw new PhabricatorDataNotAttachedException($this); } return $value[$key]; } protected function detectEncodingForStorage($string) { return phutil_is_utf8($string) ? 'utf8' : null; } protected function getUTF8StringFromStorage($string, $encoding) { if ($encoding == 'utf8') { return $string; } if (function_exists('mb_detect_encoding')) { if (strlen($encoding)) { $try_encodings = array( $encoding, ); } else { // TODO: This is pretty much a guess, and probably needs to be // configurable in the long run. $try_encodings = array( 'JIS', 'EUC-JP', 'SJIS', 'ISO-8859-1', ); } $guess = mb_detect_encoding($string, $try_encodings); if ($guess) { return mb_convert_encoding($string, 'UTF-8', $guess); } } return phutil_utf8ize($string); } protected function willReadData(array &$data) { parent::willReadData($data); static $custom; if ($custom === null) { $custom = $this->getConfigOption(self::CONFIG_APPLICATION_SERIALIZERS); } if ($custom) { foreach ($custom as $key => $serializer) { $data[$key] = $serializer->willReadValue($data[$key]); } } } protected function willWriteData(array &$data) { static $custom; if ($custom === null) { $custom = $this->getConfigOption(self::CONFIG_APPLICATION_SERIALIZERS); } if ($custom) { foreach ($custom as $key => $serializer) { $data[$key] = $serializer->willWriteValue($data[$key]); } } parent::willWriteData($data); } }