diff --git a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php index 2b5fc79f62..6feda7363c 100644 --- a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php +++ b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php @@ -1,377 +1,378 @@ refs = $refs; return $this; } public function getRefs() { if (!$this->refs) { return PhabricatorDatabaseRef::getMasterDatabaseRefs(); } return $this->refs; } public function setAPIs(array $apis) { $map = array(); foreach ($apis as $api) { $map[$api->getRef()->getRefKey()] = $api; } $this->apis = $map; return $this; } private function getDatabaseNames(PhabricatorDatabaseRef $ref) { $api = $this->getAPI($ref); $patches = PhabricatorSQLPatchList::buildAllPatches(); return $api->getDatabaseList( $patches, $only_living = true); } private function getAPI(PhabricatorDatabaseRef $ref) { $key = $ref->getRefKey(); if (isset($this->apis[$key])) { return $this->apis[$key]; } return id(new PhabricatorStorageManagementAPI()) ->setUser($ref->getUser()) ->setHost($ref->getHost()) ->setPort($ref->getPort()) ->setNamespace(PhabricatorLiskDAO::getDefaultStorageNamespace()) ->setPassword($ref->getPass()); } public function loadActualSchemata() { $refs = $this->getRefs(); $schemata = array(); foreach ($refs as $ref) { $schema = $this->loadActualSchemaForServer($ref); $schemata[$schema->getRef()->getRefKey()] = $schema; } return $schemata; } private function loadActualSchemaForServer(PhabricatorDatabaseRef $ref) { $databases = $this->getDatabaseNames($ref); $conn = $ref->newManagementConnection(); $tables = queryfx_all( $conn, - 'SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_COLLATION + 'SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_COLLATION, ENGINE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA IN (%Ls)', $databases); $database_info = queryfx_all( $conn, 'SELECT SCHEMA_NAME, DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME IN (%Ls)', $databases); $database_info = ipull($database_info, null, 'SCHEMA_NAME'); // Find databases which exist, but which the user does not have permission // to see. $invisible_databases = array(); foreach ($databases as $database_name) { if (isset($database_info[$database_name])) { continue; } try { queryfx($conn, 'SHOW TABLES IN %T', $database_name); } catch (AphrontAccessDeniedQueryException $ex) { // This database exists, the user just doesn't have permission to // see it. $invisible_databases[] = $database_name; } catch (AphrontSchemaQueryException $ex) { // This database is legitimately missing. } } $sql = array(); foreach ($tables as $table) { $sql[] = qsprintf( $conn, '(TABLE_SCHEMA = %s AND TABLE_NAME = %s)', $table['TABLE_SCHEMA'], $table['TABLE_NAME']); } if ($sql) { $column_info = queryfx_all( $conn, 'SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, CHARACTER_SET_NAME, COLLATION_NAME, COLUMN_TYPE, IS_NULLABLE, EXTRA FROM INFORMATION_SCHEMA.COLUMNS WHERE (%Q)', '('.implode(') OR (', $sql).')'); $column_info = igroup($column_info, 'TABLE_SCHEMA'); } else { $column_info = array(); } // NOTE: Tables like KEY_COLUMN_USAGE and TABLE_CONSTRAINTS only contain // primary, unique, and foreign keys, so we can't use them here. We pull // indexes later on using SHOW INDEXES. $server_schema = id(new PhabricatorConfigServerSchema()) ->setRef($ref); $tables = igroup($tables, 'TABLE_SCHEMA'); foreach ($tables as $database_name => $database_tables) { $info = $database_info[$database_name]; $database_schema = id(new PhabricatorConfigDatabaseSchema()) ->setName($database_name) ->setCharacterSet($info['DEFAULT_CHARACTER_SET_NAME']) ->setCollation($info['DEFAULT_COLLATION_NAME']); $database_column_info = idx($column_info, $database_name, array()); $database_column_info = igroup($database_column_info, 'TABLE_NAME'); foreach ($database_tables as $table) { $table_name = $table['TABLE_NAME']; $table_schema = id(new PhabricatorConfigTableSchema()) ->setName($table_name) - ->setCollation($table['TABLE_COLLATION']); + ->setCollation($table['TABLE_COLLATION']) + ->setEngine($table['ENGINE']); $columns = idx($database_column_info, $table_name, array()); foreach ($columns as $column) { if (strpos($column['EXTRA'], 'auto_increment') === false) { $auto_increment = false; } else { $auto_increment = true; } $column_schema = id(new PhabricatorConfigColumnSchema()) ->setName($column['COLUMN_NAME']) ->setCharacterSet($column['CHARACTER_SET_NAME']) ->setCollation($column['COLLATION_NAME']) ->setColumnType($column['COLUMN_TYPE']) ->setNullable($column['IS_NULLABLE'] == 'YES') ->setAutoIncrement($auto_increment); $table_schema->addColumn($column_schema); } $key_parts = queryfx_all( $conn, 'SHOW INDEXES FROM %T.%T', $database_name, $table_name); $keys = igroup($key_parts, 'Key_name'); foreach ($keys as $key_name => $key_pieces) { $key_pieces = isort($key_pieces, 'Seq_in_index'); $head = head($key_pieces); // This handles string indexes which index only a prefix of a field. $column_names = array(); foreach ($key_pieces as $piece) { $name = $piece['Column_name']; if ($piece['Sub_part']) { $name = $name.'('.$piece['Sub_part'].')'; } $column_names[] = $name; } $key_schema = id(new PhabricatorConfigKeySchema()) ->setName($key_name) ->setColumnNames($column_names) ->setUnique(!$head['Non_unique']) ->setIndexType($head['Index_type']); $table_schema->addKey($key_schema); } $database_schema->addTable($table_schema); } $server_schema->addDatabase($database_schema); } foreach ($invisible_databases as $database_name) { $server_schema->addDatabase( id(new PhabricatorConfigDatabaseSchema()) ->setName($database_name) ->setAccessDenied(true)); } return $server_schema; } public function loadExpectedSchemata() { $refs = $this->getRefs(); $schemata = array(); foreach ($refs as $ref) { $schema = $this->loadExpectedSchemaForServer($ref); $schemata[$schema->getRef()->getRefKey()] = $schema; } return $schemata; } public function loadExpectedSchemaForServer(PhabricatorDatabaseRef $ref) { $databases = $this->getDatabaseNames($ref); $info = $this->getAPI($ref)->getCharsetInfo(); $specs = id(new PhutilClassMapQuery()) ->setAncestorClass('PhabricatorConfigSchemaSpec') ->execute(); $server_schema = id(new PhabricatorConfigServerSchema()) ->setRef($ref); foreach ($specs as $spec) { $spec ->setUTF8Charset( $info[PhabricatorStorageManagementAPI::CHARSET_DEFAULT]) ->setUTF8BinaryCollation( $info[PhabricatorStorageManagementAPI::COLLATE_TEXT]) ->setUTF8SortingCollation( $info[PhabricatorStorageManagementAPI::COLLATE_SORT]) ->setServer($server_schema) ->buildSchemata($server_schema); } return $server_schema; } public function buildComparisonSchemata( array $expect_servers, array $actual_servers) { $schemata = array(); foreach ($actual_servers as $key => $actual_server) { $schemata[$key] = $this->buildComparisonSchemaForServer( $expect_servers[$key], $actual_server); } return $schemata; } private function buildComparisonSchemaForServer( PhabricatorConfigServerSchema $expect, PhabricatorConfigServerSchema $actual) { $comp_server = $actual->newEmptyClone(); $all_databases = $actual->getDatabases() + $expect->getDatabases(); foreach ($all_databases as $database_name => $database_template) { $actual_database = $actual->getDatabase($database_name); $expect_database = $expect->getDatabase($database_name); $issues = $this->compareSchemata($expect_database, $actual_database); $comp_database = $database_template->newEmptyClone() ->setIssues($issues); if (!$actual_database) { $actual_database = $expect_database->newEmptyClone(); } if (!$expect_database) { $expect_database = $actual_database->newEmptyClone(); } $all_tables = $actual_database->getTables() + $expect_database->getTables(); foreach ($all_tables as $table_name => $table_template) { $actual_table = $actual_database->getTable($table_name); $expect_table = $expect_database->getTable($table_name); $issues = $this->compareSchemata($expect_table, $actual_table); $comp_table = $table_template->newEmptyClone() ->setIssues($issues); if (!$actual_table) { $actual_table = $expect_table->newEmptyClone(); } if (!$expect_table) { $expect_table = $actual_table->newEmptyClone(); } $all_columns = $actual_table->getColumns() + $expect_table->getColumns(); foreach ($all_columns as $column_name => $column_template) { $actual_column = $actual_table->getColumn($column_name); $expect_column = $expect_table->getColumn($column_name); $issues = $this->compareSchemata($expect_column, $actual_column); $comp_column = $column_template->newEmptyClone() ->setIssues($issues); $comp_table->addColumn($comp_column); } $all_keys = $actual_table->getKeys() + $expect_table->getKeys(); foreach ($all_keys as $key_name => $key_template) { $actual_key = $actual_table->getKey($key_name); $expect_key = $expect_table->getKey($key_name); $issues = $this->compareSchemata($expect_key, $actual_key); $comp_key = $key_template->newEmptyClone() ->setIssues($issues); $comp_table->addKey($comp_key); } $comp_database->addTable($comp_table); } $comp_server->addDatabase($comp_database); } return $comp_server; } private function compareSchemata( PhabricatorConfigStorageSchema $expect = null, PhabricatorConfigStorageSchema $actual = null) { $expect_is_key = ($expect instanceof PhabricatorConfigKeySchema); $actual_is_key = ($actual instanceof PhabricatorConfigKeySchema); if ($expect_is_key || $actual_is_key) { $missing_issue = PhabricatorConfigStorageSchema::ISSUE_MISSINGKEY; $surplus_issue = PhabricatorConfigStorageSchema::ISSUE_SURPLUSKEY; } else { $missing_issue = PhabricatorConfigStorageSchema::ISSUE_MISSING; $surplus_issue = PhabricatorConfigStorageSchema::ISSUE_SURPLUS; } if (!$expect && !$actual) { throw new Exception(pht('Can not compare two missing schemata!')); } else if ($expect && !$actual) { $issues = array($missing_issue); } else if ($actual && !$expect) { $issues = array($surplus_issue); } else { $issues = $actual->compareTo($expect); } return $issues; } } diff --git a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php index e29795b98b..d808f76657 100644 --- a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php +++ b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php @@ -1,390 +1,401 @@ '; public function setUTF8SortingCollation($utf8_sorting_collation) { $this->utf8SortingCollation = $utf8_sorting_collation; return $this; } public function getUTF8SortingCollation() { return $this->utf8SortingCollation; } public function setUTF8BinaryCollation($utf8_binary_collation) { $this->utf8BinaryCollation = $utf8_binary_collation; return $this; } public function getUTF8BinaryCollation() { return $this->utf8BinaryCollation; } public function setUTF8Charset($utf8_charset) { $this->utf8Charset = $utf8_charset; return $this; } public function getUTF8Charset() { return $this->utf8Charset; } public function setServer(PhabricatorConfigServerSchema $server) { $this->server = $server; return $this; } public function getServer() { return $this->server; } abstract public function buildSchemata(); protected function buildLiskObjectSchema(PhabricatorLiskDAO $object) { $this->buildRawSchema( $object->getApplicationName(), $object->getTableName(), $object->getSchemaColumns(), $object->getSchemaKeys()); } protected function buildRawSchema( $database_name, $table_name, array $columns, array $keys) { $database = $this->getDatabase($database_name); $table = $this->newTable($table_name); + $fulltext_engine = 'MyISAM'; foreach ($columns as $name => $type) { if ($type === null) { continue; } $details = $this->getDetailsForDataType($type); $column_type = $details['type']; $charset = $details['charset']; $collation = $details['collation']; $nullable = $details['nullable']; $auto = $details['auto']; $column = $this->newColumn($name) ->setDataType($type) ->setColumnType($column_type) ->setCharacterSet($charset) ->setCollation($collation) ->setNullable($nullable) ->setAutoIncrement($auto); + // If this table has any FULLTEXT fields, we expect it to use the best + // available FULLTEXT engine, which may not be InnoDB. + switch ($type) { + case 'fulltext': + case 'fulltext?': + $table->setEngine($fulltext_engine); + break; + } + $table->addColumn($column); } foreach ($keys as $key_name => $key_spec) { if ($key_spec === null) { // This is a subclass removing a key which Lisk expects. continue; } $key = $this->newKey($key_name) ->setColumnNames(idx($key_spec, 'columns', array())); $key->setUnique((bool)idx($key_spec, 'unique')); $key->setIndexType(idx($key_spec, 'type', 'BTREE')); $table->addKey($key); } $database->addTable($table); } protected function buildEdgeSchemata(PhabricatorLiskDAO $object) { $this->buildRawSchema( $object->getApplicationName(), PhabricatorEdgeConfig::TABLE_NAME_EDGE, array( 'src' => 'phid', 'type' => 'uint32', 'dst' => 'phid', 'dateCreated' => 'epoch', 'seq' => 'uint32', 'dataID' => 'id?', ), array( 'PRIMARY' => array( 'columns' => array('src', 'type', 'dst'), 'unique' => true, ), 'src' => array( 'columns' => array('src', 'type', 'dateCreated', 'seq'), ), 'key_dst' => array( 'columns' => array('dst', 'type', 'src'), 'unique' => true, ), )); $this->buildRawSchema( $object->getApplicationName(), PhabricatorEdgeConfig::TABLE_NAME_EDGEDATA, array( 'id' => 'auto', 'data' => 'text', ), array( 'PRIMARY' => array( 'columns' => array('id'), 'unique' => true, ), )); } protected function getDatabase($name) { $server = $this->getServer(); $database = $server->getDatabase($this->getNamespacedDatabase($name)); if (!$database) { $database = $this->newDatabase($name); $server->addDatabase($database); } return $database; } protected function newDatabase($name) { return id(new PhabricatorConfigDatabaseSchema()) ->setName($this->getNamespacedDatabase($name)) ->setCharacterSet($this->getUTF8Charset()) ->setCollation($this->getUTF8BinaryCollation()); } protected function getNamespacedDatabase($name) { $namespace = PhabricatorLiskDAO::getStorageNamespace(); return $namespace.'_'.$name; } protected function newTable($name) { return id(new PhabricatorConfigTableSchema()) ->setName($name) - ->setCollation($this->getUTF8BinaryCollation()); + ->setCollation($this->getUTF8BinaryCollation()) + ->setEngine('InnoDB'); } protected function newColumn($name) { return id(new PhabricatorConfigColumnSchema()) ->setName($name); } protected function newKey($name) { return id(new PhabricatorConfigKeySchema()) ->setName($name); } public function getMaximumByteLengthForDataType($data_type) { $info = $this->getDetailsForDataType($data_type); return idx($info, 'bytes'); } private function getDetailsForDataType($data_type) { $column_type = null; $charset = null; $collation = null; $auto = false; $bytes = null; // If the type ends with "?", make the column nullable. $nullable = false; if (preg_match('/\?$/', $data_type)) { $nullable = true; $data_type = substr($data_type, 0, -1); } // NOTE: MySQL allows fragments like "VARCHAR(32) CHARACTER SET binary", // but just interprets that to mean "VARBINARY(32)". The fragment is // totally disallowed in a MODIFY statement vs a CREATE TABLE statement. $is_binary = ($this->getUTF8Charset() == 'binary'); $matches = null; $pattern = '/^(fulltext|sort|text|char)(\d+)?\z/'; if (preg_match($pattern, $data_type, $matches)) { // Limit the permitted column lengths under the theory that it would // be nice to eventually reduce this to a small set of standard lengths. static $valid_types = array( 'text255' => true, 'text160' => true, 'text128' => true, 'text64' => true, 'text40' => true, 'text32' => true, 'text20' => true, 'text16' => true, 'text12' => true, 'text8' => true, 'text4' => true, 'text' => true, 'char3' => true, 'sort255' => true, 'sort128' => true, 'sort64' => true, 'sort32' => true, 'sort' => true, 'fulltext' => true, ); if (empty($valid_types[$data_type])) { throw new Exception(pht('Unknown column type "%s"!', $data_type)); } $type = $matches[1]; $size = idx($matches, 2); if ($size) { $bytes = $size; } switch ($type) { case 'text': if ($is_binary) { if ($size) { $column_type = 'varbinary('.$size.')'; } else { $column_type = 'longblob'; } } else { if ($size) { $column_type = 'varchar('.$size.')'; } else { $column_type = 'longtext'; } } break; case 'sort': if ($size) { $column_type = 'varchar('.$size.')'; } else { $column_type = 'longtext'; } break; case 'fulltext'; // MySQL (at least, under MyISAM) refuses to create a FULLTEXT index // on a LONGBLOB column. We'd also lose case insensitivity in search. // Force this column to utf8 collation. This will truncate results // with 4-byte UTF characters in their text, but work reasonably in // the majority of cases. $column_type = 'longtext'; break; case 'char': $column_type = 'char('.$size.')'; break; } switch ($type) { case 'text': case 'char': if ($is_binary) { // We leave collation and character set unspecified in order to // generate valid SQL. } else { $charset = $this->getUTF8Charset(); $collation = $this->getUTF8BinaryCollation(); } break; case 'sort': case 'fulltext': if ($is_binary) { $charset = 'utf8'; } else { $charset = $this->getUTF8Charset(); } $collation = $this->getUTF8SortingCollation(); break; } } else { switch ($data_type) { case 'auto': $column_type = 'int(10) unsigned'; $auto = true; break; case 'auto64': $column_type = 'bigint(20) unsigned'; $auto = true; break; case 'id': case 'epoch': case 'uint32': $column_type = 'int(10) unsigned'; break; case 'sint32': $column_type = 'int(10)'; break; case 'id64': case 'uint64': $column_type = 'bigint(20) unsigned'; break; case 'sint64': $column_type = 'bigint(20)'; break; case 'phid': case 'policy'; case 'hashpath64': case 'ipaddress': $column_type = 'varbinary(64)'; break; case 'bytes64': $column_type = 'binary(64)'; break; case 'bytes40': $column_type = 'binary(40)'; break; case 'bytes32': $column_type = 'binary(32)'; break; case 'bytes20': $column_type = 'binary(20)'; break; case 'bytes12': $column_type = 'binary(12)'; break; case 'bytes4': $column_type = 'binary(4)'; break; case 'bytes': $column_type = 'longblob'; break; case 'bool': $column_type = 'tinyint(1)'; break; case 'double': $column_type = 'double'; break; case 'date': $column_type = 'date'; break; default: $column_type = self::DATATYPE_UNKNOWN; $charset = self::DATATYPE_UNKNOWN; $collation = self::DATATYPE_UNKNOWN; break; } } return array( 'type' => $column_type, 'charset' => $charset, 'collation' => $collation, 'nullable' => $nullable, 'auto' => $auto, 'bytes' => $bytes, ); } } diff --git a/src/applications/config/schema/PhabricatorConfigStorageSchema.php b/src/applications/config/schema/PhabricatorConfigStorageSchema.php index 706be80568..0f8b9c4d63 100644 --- a/src/applications/config/schema/PhabricatorConfigStorageSchema.php +++ b/src/applications/config/schema/PhabricatorConfigStorageSchema.php @@ -1,228 +1,234 @@ getName() != $expect->getName()) { throw new Exception(pht('Names must match to compare schemata!')); } return $this->compareToSimilarSchema($expect); } public function setName($name) { $this->name = $name; return $this; } public function getName() { return $this->name; } public function setIssues(array $issues) { $this->issues = array_fuse($issues); return $this; } public function getIssues() { $issues = $this->issues; foreach ($this->getSubschemata() as $sub) { switch ($sub->getStatus()) { case self::STATUS_WARN: $issues[self::ISSUE_SUBWARN] = self::ISSUE_SUBWARN; break; case self::STATUS_FAIL: $issues[self::ISSUE_SUBFAIL] = self::ISSUE_SUBFAIL; break; } } return $issues; } public function getLocalIssues() { return $this->issues; } public function hasIssue($issue) { return (bool)idx($this->getIssues(), $issue); } public function getAllIssues() { $issues = $this->getIssues(); foreach ($this->getSubschemata() as $sub) { $issues += $sub->getAllIssues(); } return $issues; } public function getStatus() { $status = self::STATUS_OKAY; foreach ($this->getAllIssues() as $issue) { $issue_status = self::getIssueStatus($issue); $status = self::getStrongestStatus($status, $issue_status); } return $status; } public static function getIssueName($issue) { switch ($issue) { case self::ISSUE_MISSING: return pht('Missing'); case self::ISSUE_MISSINGKEY: return pht('Missing Key'); case self::ISSUE_SURPLUS: return pht('Surplus'); case self::ISSUE_SURPLUSKEY: return pht('Surplus Key'); case self::ISSUE_CHARSET: return pht('Better Character Set Available'); case self::ISSUE_COLLATION: return pht('Better Collation Available'); case self::ISSUE_COLUMNTYPE: return pht('Wrong Column Type'); case self::ISSUE_NULLABLE: return pht('Wrong Nullable Setting'); case self::ISSUE_KEYCOLUMNS: return pht('Key on Wrong Columns'); case self::ISSUE_UNIQUE: return pht('Key has Wrong Uniqueness'); case self::ISSUE_LONGKEY: return pht('Key is Too Long'); case self::ISSUE_SUBWARN: return pht('Subschemata Have Warnings'); case self::ISSUE_SUBFAIL: return pht('Subschemata Have Failures'); case self::ISSUE_AUTOINCREMENT: return pht('Column has Wrong Autoincrement'); case self::ISSUE_UNKNOWN: return pht('Column Has No Specification'); case self::ISSUE_ACCESSDENIED: return pht('Access Denied'); + case self::ISSUE_ENGINE: + return pht('Better Table Engine Available'); default: throw new Exception(pht('Unknown schema issue "%s"!', $issue)); } } public static function getIssueDescription($issue) { switch ($issue) { case self::ISSUE_MISSING: return pht('This schema is expected to exist, but does not.'); case self::ISSUE_MISSINGKEY: return pht('This key is expected to exist, but does not.'); case self::ISSUE_SURPLUS: return pht('This schema is not expected to exist.'); case self::ISSUE_SURPLUSKEY: return pht('This key is not expected to exist.'); case self::ISSUE_CHARSET: return pht('This schema can use a better character set.'); case self::ISSUE_COLLATION: return pht('This schema can use a better collation.'); case self::ISSUE_COLUMNTYPE: return pht('This schema can use a better column type.'); case self::ISSUE_NULLABLE: return pht('This schema has the wrong nullable setting.'); case self::ISSUE_KEYCOLUMNS: return pht('This key is on the wrong columns.'); case self::ISSUE_UNIQUE: return pht('This key has the wrong uniqueness setting.'); case self::ISSUE_LONGKEY: return pht('This key is too long for utf8mb4.'); case self::ISSUE_SUBWARN: return pht('Subschemata have setup warnings.'); case self::ISSUE_SUBFAIL: return pht('Subschemata have setup failures.'); case self::ISSUE_AUTOINCREMENT: return pht('This column has the wrong autoincrement setting.'); case self::ISSUE_UNKNOWN: return pht('This column is missing a type specification.'); + case self::ISSUE_ENGINE: + return pht('This table can use a better table engine.'); default: throw new Exception(pht('Unknown schema issue "%s"!', $issue)); } } public static function getIssueStatus($issue) { switch ($issue) { case self::ISSUE_MISSING: case self::ISSUE_ACCESSDENIED: case self::ISSUE_SURPLUS: case self::ISSUE_NULLABLE: case self::ISSUE_SUBFAIL: case self::ISSUE_UNKNOWN: return self::STATUS_FAIL; case self::ISSUE_SUBWARN: case self::ISSUE_COLUMNTYPE: case self::ISSUE_CHARSET: case self::ISSUE_COLLATION: case self::ISSUE_MISSINGKEY: case self::ISSUE_SURPLUSKEY: case self::ISSUE_UNIQUE: case self::ISSUE_KEYCOLUMNS: case self::ISSUE_LONGKEY: case self::ISSUE_AUTOINCREMENT: + case self::ISSUE_ENGINE: return self::STATUS_WARN; default: throw new Exception(pht('Unknown schema issue "%s"!', $issue)); } } public static function getStatusSeverity($status) { switch ($status) { case self::STATUS_FAIL: return 2; case self::STATUS_WARN: return 1; case self::STATUS_OKAY: return 0; default: throw new Exception(pht('Unknown schema status "%s"!', $status)); } } public static function getStrongestStatus($u, $v) { $u_sev = self::getStatusSeverity($u); $v_sev = self::getStatusSeverity($v); if ($u_sev >= $v_sev) { return $u; } else { return $v; } } } diff --git a/src/applications/config/schema/PhabricatorConfigTableSchema.php b/src/applications/config/schema/PhabricatorConfigTableSchema.php index b05e7e1e8e..870c8ebddb 100644 --- a/src/applications/config/schema/PhabricatorConfigTableSchema.php +++ b/src/applications/config/schema/PhabricatorConfigTableSchema.php @@ -1,83 +1,97 @@ getName(); if (isset($this->columns[$key])) { throw new Exception( pht('Trying to add duplicate column "%s"!', $key)); } $this->columns[$key] = $column; return $this; } public function addKey(PhabricatorConfigKeySchema $key) { $name = $key->getName(); if (isset($this->keys[$name])) { throw new Exception( pht('Trying to add duplicate key "%s"!', $name)); } $key->setTable($this); $this->keys[$name] = $key; return $this; } public function getColumns() { return $this->columns; } public function getColumn($key) { return idx($this->getColumns(), $key); } public function getKeys() { return $this->keys; } public function getKey($key) { return idx($this->getKeys(), $key); } protected function getSubschemata() { // NOTE: Keys and columns may have the same name, so make sure we return // everything. return array_merge( array_values($this->columns), array_values($this->keys)); } public function setCollation($collation) { $this->collation = $collation; return $this; } public function getCollation() { return $this->collation; } + public function setEngine($engine) { + $this->engine = $engine; + return $this; + } + + public function getEngine() { + return $this->engine; + } + protected function compareToSimilarSchema( PhabricatorConfigStorageSchema $expect) { $issues = array(); if ($this->getCollation() != $expect->getCollation()) { $issues[] = self::ISSUE_COLLATION; } + if ($this->getEngine() != $expect->getEngine()) { + $issues[] = self::ISSUE_ENGINE; + } + return $issues; } public function newEmptyClone() { $clone = clone $this; $clone->columns = array(); $clone->keys = array(); return $clone; } } diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php index 90dbe7a1d0..caea88a608 100644 --- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php @@ -1,1153 +1,1160 @@ apis = $apis; return $this; } final public function getAnyAPI() { return head($this->getAPIs()); } final public function getMasterAPIs() { $apis = $this->getAPIs(); $results = array(); foreach ($apis as $api) { if ($api->getRef()->getIsMaster()) { $results[] = $api; } } if (!$results) { throw new PhutilArgumentUsageException( pht( 'This command only operates on database masters, but the selected '. 'database hosts do not include any masters.')); } return $results; } final public function getSingleAPI() { $apis = $this->getAPIs(); if (count($apis) == 1) { return head($apis); } throw new PhutilArgumentUsageException( pht( 'Phabricator is configured in cluster mode, with multiple database '. 'hosts. Use "--host" to specify which host you want to operate on.')); } final public function getAPIs() { return $this->apis; } final protected function isDryRun() { return $this->dryRun; } final protected function setDryRun($dry_run) { $this->dryRun = $dry_run; return $this; } final protected function isForce() { return $this->force; } final protected function setForce($force) { $this->force = $force; return $this; } public function getPatches() { return $this->patches; } public function setPatches(array $patches) { assert_instances_of($patches, 'PhabricatorStoragePatch'); $this->patches = $patches; return $this; } protected function isReadOnlyWorkflow() { return false; } public function execute(PhutilArgumentParser $args) { $this->setDryRun($args->getArg('dryrun')); $this->setForce($args->getArg('force')); if (!$this->isReadOnlyWorkflow()) { if (PhabricatorEnv::isReadOnly()) { if ($this->isForce()) { PhabricatorEnv::setReadOnly(false, null); } else { throw new PhutilArgumentUsageException( pht( 'Phabricator is currently in read-only mode. Use --force to '. 'override this mode.')); } } } return $this->didExecute($args); } public function didExecute(PhutilArgumentParser $args) {} private function loadSchemata(PhabricatorStorageManagementAPI $api) { $query = id(new PhabricatorConfigSchemaQuery()); $ref = $api->getRef(); $ref_key = $ref->getRefKey(); $query->setAPIs(array($api)); $query->setRefs(array($ref)); $actual = $query->loadActualSchemata(); $expect = $query->loadExpectedSchemata(); $comp = $query->buildComparisonSchemata($expect, $actual); return array( $comp[$ref_key], $expect[$ref_key], $actual[$ref_key], ); } final protected function adjustSchemata( PhabricatorStorageManagementAPI $api, $unsafe) { $lock = $this->lock($api); try { $err = $this->doAdjustSchemata($api, $unsafe); } catch (Exception $ex) { $lock->unlock(); throw $ex; } $lock->unlock(); return $err; } final private function doAdjustSchemata( PhabricatorStorageManagementAPI $api, $unsafe) { $console = PhutilConsole::getConsole(); $console->writeOut( "%s\n", pht( 'Verifying database schemata on "%s"...', $api->getRef()->getRefKey())); list($adjustments, $errors) = $this->findAdjustments($api); if (!$adjustments) { $console->writeOut( "%s\n", pht('Found no adjustments for schemata.')); return $this->printErrors($errors, 0); } if (!$this->force && !$api->isCharacterSetAvailable('utf8mb4')) { $message = pht( "You have an old version of MySQL (older than 5.5) which does not ". "support the utf8mb4 character set. We strongly recomend upgrading to ". "5.5 or newer.\n\n". "If you apply adjustments now and later update MySQL to 5.5 or newer, ". "you'll need to apply adjustments again (and they will take a long ". "time).\n\n". "You can exit this workflow, update MySQL now, and then run this ". "workflow again. This is recommended, but may cause a lot of downtime ". "right now.\n\n". "You can exit this workflow, continue using Phabricator without ". "applying adjustments, update MySQL at a later date, and then run ". "this workflow again. This is also a good approach, and will let you ". "delay downtime until later.\n\n". "You can proceed with this workflow, and then optionally update ". "MySQL at a later date. After you do, you'll need to apply ". "adjustments again.\n\n". "For more information, see \"Managing Storage Adjustments\" in ". "the documentation."); $console->writeOut( "\n** %s **\n\n%s\n", pht('OLD MySQL VERSION'), phutil_console_wrap($message)); $prompt = pht('Continue with old MySQL version?'); if (!phutil_console_confirm($prompt, $default_no = true)) { return; } } $table = id(new PhutilConsoleTable()) ->addColumn('database', array('title' => pht('Database'))) ->addColumn('table', array('title' => pht('Table'))) ->addColumn('name', array('title' => pht('Name'))) ->addColumn('info', array('title' => pht('Issues'))); foreach ($adjustments as $adjust) { $info = array(); foreach ($adjust['issues'] as $issue) { $info[] = PhabricatorConfigStorageSchema::getIssueName($issue); } $table->addRow(array( 'database' => $adjust['database'], 'table' => idx($adjust, 'table'), 'name' => idx($adjust, 'name'), 'info' => implode(', ', $info), )); } $console->writeOut("\n\n"); $table->draw(); if ($this->dryRun) { $console->writeOut( "%s\n", pht('DRYRUN: Would apply adjustments.')); return 0; } else if ($this->didInitialize) { // If we just initialized the database, continue without prompting. This // is nicer for first-time setup and there's no reasonable reason any // user would ever answer "no" to the prompt against an empty schema. } else if (!$this->force) { $console->writeOut( "\n%s\n", pht( "Found %s adjustment(s) to apply, detailed above.\n\n". "You can review adjustments in more detail from the web interface, ". "in Config > Database Status. To better understand the adjustment ". "workflow, see \"Managing Storage Adjustments\" in the ". "documentation.\n\n". "MySQL needs to copy table data to make some adjustments, so these ". "migrations may take some time.", phutil_count($adjustments))); $prompt = pht('Apply these schema adjustments?'); if (!phutil_console_confirm($prompt, $default_no = true)) { return 1; } } $console->writeOut( "%s\n", pht('Applying schema adjustments...')); $conn = $api->getConn(null); if ($unsafe) { queryfx($conn, 'SET SESSION sql_mode = %s', ''); } else { queryfx($conn, 'SET SESSION sql_mode = %s', 'STRICT_ALL_TABLES'); } $failed = array(); // We make changes in several phases. $phases = array( // Drop surplus autoincrements. This allows us to drop primary keys on // autoincrement columns. 'drop_auto', // Drop all keys we're going to adjust. This prevents them from // interfering with column changes. 'drop_keys', // Apply all database, table, and column changes. 'main', // Restore adjusted keys. 'add_keys', // Add missing autoincrements. 'add_auto', ); $bar = id(new PhutilConsoleProgressBar()) ->setTotal(count($adjustments) * count($phases)); foreach ($phases as $phase) { foreach ($adjustments as $adjust) { try { switch ($adjust['kind']) { case 'database': if ($phase == 'main') { queryfx( $conn, 'ALTER DATABASE %T CHARACTER SET = %s COLLATE = %s', $adjust['database'], $adjust['charset'], $adjust['collation']); } break; case 'table': if ($phase == 'main') { queryfx( $conn, - 'ALTER TABLE %T.%T COLLATE = %s', + 'ALTER TABLE %T.%T COLLATE = %s, ENGINE = %s', $adjust['database'], $adjust['table'], - $adjust['collation']); + $adjust['collation'], + $adjust['engine']); } break; case 'column': $apply = false; $auto = false; $new_auto = idx($adjust, 'auto'); if ($phase == 'drop_auto') { if ($new_auto === false) { $apply = true; $auto = false; } } else if ($phase == 'main') { $apply = true; if ($new_auto === false) { $auto = false; } else { $auto = $adjust['is_auto']; } } else if ($phase == 'add_auto') { if ($new_auto === true) { $apply = true; $auto = true; } } if ($apply) { $parts = array(); if ($auto) { $parts[] = qsprintf( $conn, 'AUTO_INCREMENT'); } if ($adjust['charset']) { $parts[] = qsprintf( $conn, 'CHARACTER SET %Q COLLATE %Q', $adjust['charset'], $adjust['collation']); } queryfx( $conn, 'ALTER TABLE %T.%T MODIFY %T %Q %Q %Q', $adjust['database'], $adjust['table'], $adjust['name'], $adjust['type'], implode(' ', $parts), $adjust['nullable'] ? 'NULL' : 'NOT NULL'); } break; case 'key': if (($phase == 'drop_keys') && $adjust['exists']) { if ($adjust['name'] == 'PRIMARY') { $key_name = 'PRIMARY KEY'; } else { $key_name = qsprintf($conn, 'KEY %T', $adjust['name']); } queryfx( $conn, 'ALTER TABLE %T.%T DROP %Q', $adjust['database'], $adjust['table'], $key_name); } if (($phase == 'add_keys') && $adjust['keep']) { // Different keys need different creation syntax. Notable // special cases are primary keys and fulltext keys. if ($adjust['name'] == 'PRIMARY') { $key_name = 'PRIMARY KEY'; } else if ($adjust['indexType'] == 'FULLTEXT') { $key_name = qsprintf($conn, 'FULLTEXT %T', $adjust['name']); } else { if ($adjust['unique']) { $key_name = qsprintf( $conn, 'UNIQUE KEY %T', $adjust['name']); } else { $key_name = qsprintf( $conn, '/* NONUNIQUE */ KEY %T', $adjust['name']); } } queryfx( $conn, 'ALTER TABLE %T.%T ADD %Q (%Q)', $adjust['database'], $adjust['table'], $key_name, implode(', ', $adjust['columns'])); } break; default: throw new Exception( pht('Unknown schema adjustment kind "%s"!', $adjust['kind'])); } } catch (AphrontQueryException $ex) { $failed[] = array($adjust, $ex); } $bar->update(1); } } $bar->done(); if (!$failed) { $console->writeOut( "%s\n", pht('Completed applying all schema adjustments.')); $err = 0; } else { $table = id(new PhutilConsoleTable()) ->addColumn('target', array('title' => pht('Target'))) ->addColumn('error', array('title' => pht('Error'))); foreach ($failed as $failure) { list($adjust, $ex) = $failure; $pieces = array_select_keys( $adjust, array('database', 'table', 'name')); $pieces = array_filter($pieces); $target = implode('.', $pieces); $table->addRow( array( 'target' => $target, 'error' => $ex->getMessage(), )); } $console->writeOut("\n"); $table->draw(); $console->writeOut( "\n%s\n", pht('Failed to make some schema adjustments, detailed above.')); $console->writeOut( "%s\n", pht( 'For help troubleshooting adjustments, see "Managing Storage '. 'Adjustments" in the documentation.')); $err = 1; } return $this->printErrors($errors, $err); } private function findAdjustments( PhabricatorStorageManagementAPI $api) { list($comp, $expect, $actual) = $this->loadSchemata($api); $issue_charset = PhabricatorConfigStorageSchema::ISSUE_CHARSET; $issue_collation = PhabricatorConfigStorageSchema::ISSUE_COLLATION; $issue_columntype = PhabricatorConfigStorageSchema::ISSUE_COLUMNTYPE; $issue_surpluskey = PhabricatorConfigStorageSchema::ISSUE_SURPLUSKEY; $issue_missingkey = PhabricatorConfigStorageSchema::ISSUE_MISSINGKEY; $issue_columns = PhabricatorConfigStorageSchema::ISSUE_KEYCOLUMNS; $issue_unique = PhabricatorConfigStorageSchema::ISSUE_UNIQUE; $issue_longkey = PhabricatorConfigStorageSchema::ISSUE_LONGKEY; $issue_auto = PhabricatorConfigStorageSchema::ISSUE_AUTOINCREMENT; + $issue_engine = PhabricatorConfigStorageSchema::ISSUE_ENGINE; $adjustments = array(); $errors = array(); foreach ($comp->getDatabases() as $database_name => $database) { foreach ($this->findErrors($database) as $issue) { $errors[] = array( 'database' => $database_name, 'issue' => $issue, ); } $expect_database = $expect->getDatabase($database_name); $actual_database = $actual->getDatabase($database_name); if (!$expect_database || !$actual_database) { // If there's a real issue here, skip this stuff. continue; } if ($actual_database->getAccessDenied()) { // If we can't access the database, we can't access the tables either. continue; } $issues = array(); if ($database->hasIssue($issue_charset)) { $issues[] = $issue_charset; } if ($database->hasIssue($issue_collation)) { $issues[] = $issue_collation; } if ($issues) { $adjustments[] = array( 'kind' => 'database', 'database' => $database_name, 'issues' => $issues, 'charset' => $expect_database->getCharacterSet(), 'collation' => $expect_database->getCollation(), ); } foreach ($database->getTables() as $table_name => $table) { foreach ($this->findErrors($table) as $issue) { $errors[] = array( 'database' => $database_name, 'table' => $table_name, 'issue' => $issue, ); } $expect_table = $expect_database->getTable($table_name); $actual_table = $actual_database->getTable($table_name); if (!$expect_table || !$actual_table) { continue; } $issues = array(); if ($table->hasIssue($issue_collation)) { $issues[] = $issue_collation; } + if ($table->hasIssue($issue_engine)) { + $issues[] = $issue_engine; + } + if ($issues) { $adjustments[] = array( 'kind' => 'table', 'database' => $database_name, 'table' => $table_name, 'issues' => $issues, 'collation' => $expect_table->getCollation(), + 'engine' => $expect_table->getEngine(), ); } foreach ($table->getColumns() as $column_name => $column) { foreach ($this->findErrors($column) as $issue) { $errors[] = array( 'database' => $database_name, 'table' => $table_name, 'name' => $column_name, 'issue' => $issue, ); } $expect_column = $expect_table->getColumn($column_name); $actual_column = $actual_table->getColumn($column_name); if (!$expect_column || !$actual_column) { continue; } $issues = array(); if ($column->hasIssue($issue_collation)) { $issues[] = $issue_collation; } if ($column->hasIssue($issue_charset)) { $issues[] = $issue_charset; } if ($column->hasIssue($issue_columntype)) { $issues[] = $issue_columntype; } if ($column->hasIssue($issue_auto)) { $issues[] = $issue_auto; } if ($issues) { if ($expect_column->getCharacterSet() === null) { // For non-text columns, we won't be specifying a collation or // character set. $charset = null; $collation = null; } else { $charset = $expect_column->getCharacterSet(); $collation = $expect_column->getCollation(); } $adjustment = array( 'kind' => 'column', 'database' => $database_name, 'table' => $table_name, 'name' => $column_name, 'issues' => $issues, 'collation' => $collation, 'charset' => $charset, 'type' => $expect_column->getColumnType(), // NOTE: We don't adjust column nullability because it is // dangerous, so always use the current nullability. 'nullable' => $actual_column->getNullable(), // NOTE: This always stores the current value, because we have // to make these updates separately. 'is_auto' => $actual_column->getAutoIncrement(), ); if ($column->hasIssue($issue_auto)) { $adjustment['auto'] = $expect_column->getAutoIncrement(); } $adjustments[] = $adjustment; } } foreach ($table->getKeys() as $key_name => $key) { foreach ($this->findErrors($key) as $issue) { $errors[] = array( 'database' => $database_name, 'table' => $table_name, 'name' => $key_name, 'issue' => $issue, ); } $expect_key = $expect_table->getKey($key_name); $actual_key = $actual_table->getKey($key_name); $issues = array(); $keep_key = true; if ($key->hasIssue($issue_surpluskey)) { $issues[] = $issue_surpluskey; $keep_key = false; } if ($key->hasIssue($issue_missingkey)) { $issues[] = $issue_missingkey; } if ($key->hasIssue($issue_columns)) { $issues[] = $issue_columns; } if ($key->hasIssue($issue_unique)) { $issues[] = $issue_unique; } // NOTE: We can't really fix this, per se, but we may need to remove // the key to change the column type. In the best case, the new // column type won't be overlong and recreating the key really will // fix the issue. In the worst case, we get the right column type and // lose the key, which is still better than retaining the key having // the wrong column type. if ($key->hasIssue($issue_longkey)) { $issues[] = $issue_longkey; } if ($issues) { $adjustment = array( 'kind' => 'key', 'database' => $database_name, 'table' => $table_name, 'name' => $key_name, 'issues' => $issues, 'exists' => (bool)$actual_key, 'keep' => $keep_key, ); if ($keep_key) { $adjustment += array( 'columns' => $expect_key->getColumnNames(), 'unique' => $expect_key->getUnique(), 'indexType' => $expect_key->getIndexType(), ); } $adjustments[] = $adjustment; } } } } return array($adjustments, $errors); } private function findErrors(PhabricatorConfigStorageSchema $schema) { $result = array(); foreach ($schema->getLocalIssues() as $issue) { $status = PhabricatorConfigStorageSchema::getIssueStatus($issue); if ($status == PhabricatorConfigStorageSchema::STATUS_FAIL) { $result[] = $issue; } } return $result; } private function printErrors(array $errors, $default_return) { if (!$errors) { return $default_return; } $console = PhutilConsole::getConsole(); $table = id(new PhutilConsoleTable()) ->addColumn('target', array('title' => pht('Target'))) ->addColumn('error', array('title' => pht('Error'))); $any_surplus = false; $all_surplus = true; $any_access = false; $all_access = true; foreach ($errors as $error) { $pieces = array_select_keys( $error, array('database', 'table', 'name')); $pieces = array_filter($pieces); $target = implode('.', $pieces); $name = PhabricatorConfigStorageSchema::getIssueName($error['issue']); $issue = $error['issue']; if ($issue === PhabricatorConfigStorageSchema::ISSUE_SURPLUS) { $any_surplus = true; } else { $all_surplus = false; } if ($issue === PhabricatorConfigStorageSchema::ISSUE_ACCESSDENIED) { $any_access = true; } else { $all_access = false; } $table->addRow( array( 'target' => $target, 'error' => $name, )); } $console->writeOut("\n"); $table->draw(); $console->writeOut("\n"); $message = array(); if ($all_surplus) { $message[] = pht( 'You have surplus schemata (extra tables or columns which Phabricator '. 'does not expect). For information on resolving these '. 'issues, see the "Surplus Schemata" section in the "Managing Storage '. 'Adjustments" article in the documentation.'); } else if ($all_access) { $message[] = pht( 'The user you are connecting to MySQL with does not have the correct '. 'permissions, and can not access some databases or tables that it '. 'needs to be able to access. GRANT the user additional permissions.'); } else { $message[] = pht( 'The schemata have errors (detailed above) which the adjustment '. 'workflow can not fix.'); if ($any_access) { $message[] = pht( 'Some of these errors are caused by access control problems. '. 'The user you are connecting with does not have permission to see '. 'all of the database or tables that Phabricator uses. You need to '. 'GRANT the user more permission, or use a different user.'); } if ($any_surplus) { $message[] = pht( 'Some of these errors are caused by surplus schemata (extra '. 'tables or columns which Phabricator does not expect). These are '. 'not serious. For information on resolving these issues, see the '. '"Surplus Schemata" section in the "Managing Storage Adjustments" '. 'article in the documentation.'); } $message[] = pht( 'If you are not developing Phabricator itself, report this issue to '. 'the upstream.'); $message[] = pht( 'If you are developing Phabricator, these errors usually indicate '. 'that your schema specifications do not agree with the schemata your '. 'code actually builds.'); } $message = implode("\n\n", $message); if ($all_surplus) { $console->writeOut( "** %s **\n\n%s\n", pht('SURPLUS SCHEMATA'), phutil_console_wrap($message)); } else if ($all_access) { $console->writeOut( "** %s **\n\n%s\n", pht('ACCESS DENIED'), phutil_console_wrap($message)); } else { $console->writeOut( "** %s **\n\n%s\n", pht('SCHEMATA ERRORS'), phutil_console_wrap($message)); } return 2; } final protected function upgradeSchemata( array $apis, $apply_only = null, $no_quickstart = false, $init_only = false) { $locks = array(); foreach ($apis as $api) { $locks[] = $this->lock($api); } try { $this->doUpgradeSchemata($apis, $apply_only, $no_quickstart, $init_only); } catch (Exception $ex) { foreach ($locks as $lock) { $lock->unlock(); } throw $ex; } foreach ($locks as $lock) { $lock->unlock(); } } final private function doUpgradeSchemata( array $apis, $apply_only, $no_quickstart, $init_only) { $patches = $this->patches; $is_dryrun = $this->dryRun; $api_map = array(); foreach ($apis as $api) { $api_map[$api->getRef()->getRefKey()] = $api; } foreach ($api_map as $ref_key => $api) { $applied = $api->getAppliedPatches(); $needs_init = ($applied === null); if (!$needs_init) { continue; } if ($is_dryrun) { echo tsprintf( "%s\n", pht( 'DRYRUN: Storage on host "%s" does not exist yet, so it '. 'would be created.', $ref_key)); continue; } if ($apply_only) { throw new PhutilArgumentUsageException( pht( 'Storage on host "%s" has not been initialized yet. You must '. 'initialize storage before selectively applying patches.', $ref_key)); } // If we're initializing storage for the first time on any host, track // it so that we can give the user a nicer experience during the // subsequent adjustment phase. $this->didInitialize = true; $legacy = $api->getLegacyPatches($patches); if ($legacy || $no_quickstart || $init_only) { // If we have legacy patches, we can't quickstart. $api->createDatabase('meta_data'); $api->createTable( 'meta_data', 'patch_status', array( 'patch VARCHAR(255) NOT NULL PRIMARY KEY COLLATE utf8_general_ci', 'applied INT UNSIGNED NOT NULL', )); foreach ($legacy as $patch) { $api->markPatchApplied($patch); } } else { echo tsprintf( "%s\n", pht( 'Loading quickstart template onto "%s"...', $ref_key)); $root = dirname(phutil_get_library_root('phabricator')); $sql = $root.'/resources/sql/quickstart.sql'; $api->applyPatchSQL($sql); } } if ($init_only) { echo pht('Storage initialized.')."\n"; return 0; } $applied_map = array(); foreach ($api_map as $ref_key => $api) { $applied = $api->getAppliedPatches(); // If we still have nothing applied, this is a dry run and we didn't // actually initialize storage. Here, just do nothing. if ($applied === null) { if ($is_dryrun) { continue; } else { throw new Exception( pht( 'Database initialization on host "%s" applied no patches!', $ref_key)); } } $applied = array_fuse($applied); if ($apply_only) { if (isset($applied[$apply_only])) { if (!$this->force && !$is_dryrun) { echo phutil_console_wrap( pht( 'Patch "%s" has already been applied on host "%s". Are you '. 'sure you want to apply it again? This may put your storage '. 'in a state that the upgrade scripts can not automatically '. 'manage.', $apply_only, $ref_key)); if (!phutil_console_confirm(pht('Apply patch again?'))) { echo pht('Cancelled.')."\n"; return 1; } } // Mark this patch as not yet applied on this host. unset($applied[$apply_only]); } } $applied_map[$ref_key] = $applied; } // If we're applying only a specific patch, select just that patch. if ($apply_only) { $patches = array_select_keys($patches, array($apply_only)); } // Apply each patch to each database. We apply patches patch-by-patch, // not database-by-database: for each patch we apply it to every database, // then move to the next patch. // We must do this because ".php" patches may depend on ".sql" patches // being up to date on all masters, and that will work fine if we put each // patch on every host before moving on. If we try to bring database hosts // up to date one at a time we can end up in a big mess. $duration_map = array(); // First, find any global patches which have been applied to ANY database. // We are just going to mark these as applied without actually running // them. Otherwise, adding new empty masters to an existing cluster will // try to apply them against invalid states. foreach ($patches as $key => $patch) { if ($patch->getIsGlobalPatch()) { foreach ($applied_map as $ref_key => $applied) { if (isset($applied[$key])) { $duration_map[$key] = 1; } } } } while (true) { $applied_something = false; foreach ($patches as $key => $patch) { // First, check if any databases need this patch. We can just skip it // if it has already been applied everywhere. $need_patch = array(); foreach ($applied_map as $ref_key => $applied) { if (isset($applied[$key])) { continue; } $need_patch[] = $ref_key; } if (!$need_patch) { unset($patches[$key]); continue; } // Check if we can apply this patch yet. Before we can apply a patch, // all of the dependencies for the patch must have been applied on all // databases. Requiring that all databases stay in sync prevents one // database from racing ahead if it happens to get a patch that nothing // else has yet. $missing_patch = null; foreach ($patch->getAfter() as $after) { foreach ($applied_map as $ref_key => $applied) { if (isset($applied[$after])) { // This database already has the patch. We can apply it to // other databases but don't need to apply it here. continue; } $missing_patch = $after; break 2; } } if ($missing_patch) { if ($apply_only) { echo tsprintf( "%s\n", pht( 'Unable to apply patch "%s" because it depends on patch '. '"%s", which has not been applied on some hosts: %s.', $apply_only, $missing_patch, implode(', ', $need_patch))); return 1; } else { // Some databases are missing the dependencies, so keep trying // other patches instead. If everything goes right, we'll apply the // dependencies and then come back and apply this patch later. continue; } } $is_global = $patch->getIsGlobalPatch(); $patch_apis = array_select_keys($api_map, $need_patch); foreach ($patch_apis as $ref_key => $api) { if ($is_global) { // If this is a global patch which we previously applied, just // read the duration from the map without actually applying // the patch. $duration = idx($duration_map, $key); } else { $duration = null; } if ($duration === null) { if ($is_dryrun) { echo tsprintf( "%s\n", pht( 'DRYRUN: Would apply patch "%s" to host "%s".', $key, $ref_key)); } else { echo tsprintf( "%s\n", pht( 'Applying patch "%s" to host "%s"...', $key, $ref_key)); } $t_begin = microtime(true); $api->applyPatch($patch); $t_end = microtime(true); $duration = ($t_end - $t_begin); $duration_map[$key] = $duration; } // If we're explicitly reapplying this patch, we don't need to // mark it as applied. if (!isset($applied_map[$ref_key][$key])) { $api->markPatchApplied($key, ($t_end - $t_begin)); $applied_map[$ref_key][$key] = true; } } // We applied this everywhere, so we're done with the patch. unset($patches[$key]); $applied_something = true; } if (!$applied_something) { if ($patches) { throw new Exception( pht( 'Some patches could not be applied: %s', implode(', ', array_keys($patches)))); } else if (!$is_dryrun && !$apply_only) { echo pht( 'Storage is up to date. Use "%s" for details.', 'storage status')."\n"; } break; } } } final protected function getBareHostAndPort($host) { // Split out port information, since the command-line client requires a // separate flag for the port. $uri = new PhutilURI('mysql://'.$host); if ($uri->getPort()) { $port = $uri->getPort(); $bare_hostname = $uri->getDomain(); } else { $port = null; $bare_hostname = $host; } return array($bare_hostname, $port); } /** * Acquires a @{class:PhabricatorGlobalLock}. * * @return PhabricatorGlobalLock */ final protected function lock(PhabricatorStorageManagementAPI $api) { // Although we're holding this lock on different databases so it could // have the same name on each as far as the database is concerned, the // locks would be the same within this process. $ref_key = $api->getRef()->getRefKey(); $ref_hash = PhabricatorHash::digestForIndex($ref_key); $lock_name = 'adjust('.$ref_hash.')'; return PhabricatorGlobalLock::newLock($lock_name) ->useSpecificConnection($api->getConn(null)) ->lock(); } }