diff --git a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php index a0da8d4aea..7c65240b89 100644 --- a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php +++ b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php @@ -1,372 +1,362 @@ utf8Collation = $utf8_collation; return $this; } public function getUTF8Collation() { return $this->utf8Collation; } public function setUTF8Charset($utf8_charset) { $this->utf8Charset = $utf8_charset; return $this; } public function getUTF8Charset() { return $this->utf8Charset; } public function setServer(PhabricatorConfigServerSchema $server) { $this->server = $server; return $this; } public function getServer() { return $this->server; } abstract public function buildSchemata(); protected function buildLiskSchemata($base) { $objects = id(new PhutilSymbolLoader()) ->setAncestorClass($base) ->loadObjects(); foreach ($objects as $object) { if ($object->getConfigOption(LiskDAO::CONFIG_NO_TABLE)) { continue; } $this->buildLiskObjectSchema($object); } } protected function buildTransactionSchema( PhabricatorApplicationTransaction $xaction, PhabricatorApplicationTransactionComment $comment = null) { $this->buildLiskObjectSchema($xaction); if ($comment) { $this->buildLiskObjectSchema($comment); } } protected function buildCustomFieldSchemata( PhabricatorLiskDAO $storage, array $indexes) { $this->buildLiskObjectSchema($storage); foreach ($indexes as $index) { $this->buildLiskObjectSchema($index); } } private function buildLiskObjectSchema(PhabricatorLiskDAO $object) { $this->buildRawSchema( $object->getApplicationName(), $object->getTableName(), $object->getSchemaColumns(), $object->getSchemaKeys()); } protected function buildRawSchema( $database_name, $table_name, array $columns, array $keys) { $database = $this->getDatabase($database_name); $table = $this->newTable($table_name); foreach ($columns as $name => $type) { if ($type === null) { continue; } $details = $this->getDetailsForDataType($type); list($column_type, $charset, $collation, $nullable) = $details; $column = $this->newColumn($name) ->setDataType($type) ->setColumnType($column_type) ->setCharacterSet($charset) ->setCollation($collation) ->setNullable($nullable); $table->addColumn($column); } foreach ($keys as $key_name => $key_spec) { if ($key_spec === null) { // This is a subclass removing a key which Lisk expects. continue; } $key = $this->newKey($key_name) ->setColumnNames(idx($key_spec, 'columns', array())); $key->setUnique((bool)idx($key_spec, 'unique')); $table->addKey($key); } $database->addTable($table); } protected function buildEdgeSchemata(PhabricatorLiskDAO $object) { $this->buildRawSchema( $object->getApplicationName(), PhabricatorEdgeConfig::TABLE_NAME_EDGE, array( 'src' => 'phid', 'type' => 'uint32', 'dst' => 'phid', 'dateCreated' => 'epoch', 'seq' => 'uint32', 'dataID' => 'id?', ), array( 'PRIMARY' => array( 'columns' => array('src', 'type', 'dst'), 'unique' => true, ), 'src' => array( 'columns' => array('src', 'type', 'dateCreated', 'seq'), ), 'key_dst' => array( 'columns' => array('dst', 'type', 'src'), 'unique' => true, ), )); $this->buildRawSchema( $object->getApplicationName(), PhabricatorEdgeConfig::TABLE_NAME_EDGEDATA, array( 'id' => 'id', 'data' => 'text', ), array( 'PRIMARY' => array( 'columns' => array('id'), 'unique' => true, ), )); } public function buildCounterSchema(PhabricatorLiskDAO $object) { $this->buildRawSchema( $object->getApplicationName(), PhabricatorLiskDAO::COUNTER_TABLE_NAME, array( 'counterName' => 'text32', 'counterValue' => 'id64', ), array( 'PRIMARY' => array( 'columns' => array('counterName'), 'unique' => true, ), )); } protected function getDatabase($name) { $server = $this->getServer(); $database = $server->getDatabase($this->getNamespacedDatabase($name)); if (!$database) { $database = $this->newDatabase($name); $server->addDatabase($database); } return $database; } protected function newDatabase($name) { return id(new PhabricatorConfigDatabaseSchema()) ->setName($this->getNamespacedDatabase($name)) ->setCharacterSet($this->getUTF8Charset()) ->setCollation($this->getUTF8Collation()); } protected function getNamespacedDatabase($name) { $namespace = PhabricatorLiskDAO::getStorageNamespace(); return $namespace.'_'.$name; } protected function newTable($name) { return id(new PhabricatorConfigTableSchema()) ->setName($name) ->setCollation($this->getUTF8Collation()); } protected function newColumn($name) { return id(new PhabricatorConfigColumnSchema()) ->setName($name); } protected function newKey($name) { return id(new PhabricatorConfigKeySchema()) ->setName($name); } private function getDetailsForDataType($data_type) { $column_type = null; $charset = null; $collation = null; // If the type ends with "?", make the column nullable. $nullable = false; if (preg_match('/\?$/', $data_type)) { $nullable = true; $data_type = substr($data_type, 0, -1); } + // NOTE: MySQL allows fragments like "VARCHAR(32) CHARACTER SET binary", + // but just interprets that to mean "VARBINARY(32)". The fragment is + // totally disallowed in a MODIFY statement vs a CREATE TABLE statement. + switch ($data_type) { case 'id': case 'epoch': case 'uint32': $column_type = 'int(10) unsigned'; break; case 'sint32': $column_type = 'int(10)'; break; case 'id64': case 'uint64': $column_type = 'bigint(20) unsigned'; break; case 'sint64': $column_type = 'bigint(20)'; break; case 'phid': case 'policy'; - $column_type = 'varchar(64)'; - $charset = 'binary'; - $collation = 'binary'; + $column_type = 'varbinary(64)'; break; case 'bytes64': - $column_type = 'char(64)'; - $charset = 'binary'; - $collation = 'binary'; + $column_type = 'binary(64)'; break; case 'bytes40': - $column_type = 'char(40)'; - $charset = 'binary'; - $collation = 'binary'; + $column_type = 'binary(40)'; break; case 'bytes32': - $column_type = 'char(32)'; - $charset = 'binary'; - $collation = 'binary'; + $column_type = 'binary(32)'; break; case 'bytes20': - $column_type = 'char(20)'; - $charset = 'binary'; - $collation = 'binary'; + $column_type = 'binary(20)'; break; case 'bytes12': - $column_type = 'char(12)'; - $charset = 'binary'; - $collation = 'binary'; + $column_type = 'binary(12)'; break; case 'bytes4': - $column_type = 'char(4)'; - $charset = 'binary'; - $collation = 'binary'; + $column_type = 'binary(4)'; break; case 'bytes': $column_type = 'longblob'; break; case 'text255': $column_type = 'varchar(255)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8Collation(); break; case 'text160': $column_type = 'varchar(160)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8Collation(); break; case 'text128': $column_type = 'varchar(128)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8Collation(); break; case 'text80': $column_type = 'varchar(80)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8Collation(); break; case 'text64': $column_type = 'varchar(64)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8Collation(); break; case 'text40': $column_type = 'varchar(40)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8Collation(); break; case 'text32': $column_type = 'varchar(32)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8Collation(); break; case 'text20': $column_type = 'varchar(20)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8Collation(); break; case 'text16': $column_type = 'varchar(16)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8Collation(); break; case 'text12': $column_type = 'varchar(12)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8Collation(); break; case 'text8': $column_type = 'varchar(8)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8Collation(); break; case 'text4': $column_type = 'varchar(4)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8Collation(); break; case 'text': $column_type = 'longtext'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8Collation(); break; case 'bool': $column_type = 'tinyint(1)'; break; case 'double': $column_type = 'double'; break; case 'date': $column_type = 'date'; break; default: $column_type = pht(''); $charset = pht(''); $collation = pht(''); break; } return array($column_type, $charset, $collation, $nullable); } } diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php index e7326d38ce..41e1235584 100644 --- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php @@ -1,227 +1,331 @@ setName('adjust') ->setExamples('**adjust** [__options__]') ->setSynopsis( pht( 'Make schemata adjustments to correct issues with characters sets, '. 'collations, and keys.')); } public function execute(PhutilArgumentParser $args) { $this->requireAllPatchesApplied(); $this->adjustSchemata(); return 0; } private function requireAllPatchesApplied() { $api = $this->getAPI(); $applied = $api->getAppliedPatches(); if ($applied === null) { throw new PhutilArgumentUsageException( pht( 'You have not initialized the database yet. You must initialize '. 'the database before you can adjust schemata. Run `storage upgrade` '. 'to initialize the database.')); } $applied = array_fuse($applied); $patches = $this->getPatches(); $patches = mpull($patches, null, 'getFullKey'); $missing = array_diff_key($patches, $applied); if ($missing) { throw new PhutilArgumentUsageException( pht( 'You have not applied all available storage patches yet. You must '. 'apply all available patches before you can adjust schemata. '. 'Run `storage status` to show patch status, and `storage upgrade` '. 'to apply missing patches.')); } } private function loadSchemata() { $query = id(new PhabricatorConfigSchemaQuery()) ->setAPI($this->getAPI()); $actual = $query->loadActualSchema(); $expect = $query->loadExpectedSchema(); $comp = $query->buildComparisonSchema($expect, $actual); return array($comp, $expect, $actual); } private function adjustSchemata() { $console = PhutilConsole::getConsole(); $console->writeOut( "%s\n", pht('Verifying database schemata...')); $adjustments = $this->findAdjustments(); if (!$adjustments) { $console->writeOut( "%s\n", pht('Found no issues with schemata.')); return; } $table = id(new PhutilConsoleTable()) ->addColumn('database', array('title' => pht('Database'))) ->addColumn('table', array('title' => pht('Table'))) ->addColumn('name', array('title' => pht('Name'))) ->addColumn('info', array('title' => pht('Issues'))); foreach ($adjustments as $adjust) { $info = array(); foreach ($adjust['issues'] as $issue) { $info[] = PhabricatorConfigStorageSchema::getIssueName($issue); } $table->addRow(array( 'database' => $adjust['database'], 'table' => idx($adjust, 'table'), 'name' => idx($adjust, 'name'), 'info' => implode(', ', $info), )); } $console->writeOut("\n\n"); $table->draw(); $console->writeOut( "\n%s\n", pht( "Found %s issues(s) with schemata, detailed above.\n\n". "You can review issues in more detail from the web interface, ". "in Config > Database Status.\n\n". "MySQL needs to copy table data to make some adjustments, so these ". "migrations may take some time.". // TODO: Remove warning once this stabilizes. "\n\n". "WARNING: This workflow is new and unstable. If you continue, you ". "may unrecoverably destory data. Make sure you have a backup before ". "you proceed.", new PhutilNumber(count($adjustments)))); $prompt = pht('Fix these schema issues?'); if (!phutil_console_confirm($prompt, $default_no = true)) { return; } $console->writeOut( "%s\n", pht('Fixing schema issues...')); $api = $this->getAPI(); $conn = $api->getConn(null); + $failed = array(); + $bar = id(new PhutilConsoleProgressBar()) ->setTotal(count($adjustments)); foreach ($adjustments as $adjust) { - switch ($adjust['kind']) { - case 'database': - queryfx( - $conn, - 'ALTER DATABASE %T CHARACTER SET = %s COLLATE = %s', - $adjust['database'], - $adjust['charset'], - $adjust['collation']); - break; - case 'table': - queryfx( - $conn, - 'ALTER TABLE %T.%T COLLATE = %s', - $adjust['database'], - $adjust['table'], - $adjust['collation']); - break; - default: - throw new Exception( - pht('Unknown schema adjustment kind "%s"!', $adjust['kind'])); + try { + switch ($adjust['kind']) { + case 'database': + queryfx( + $conn, + 'ALTER DATABASE %T CHARACTER SET = %s COLLATE = %s', + $adjust['database'], + $adjust['charset'], + $adjust['collation']); + break; + case 'table': + queryfx( + $conn, + 'ALTER TABLE %T.%T COLLATE = %s', + $adjust['database'], + $adjust['table'], + $adjust['collation']); + break; + case 'column': + $parts = array(); + if ($adjust['charset']) { + $parts[] = qsprintf( + $conn, + 'CHARACTER SET %Q COLLATE %Q', + $adjust['charset'], + $adjust['collation']); + } + + queryfx( + $conn, + 'ALTER TABLE %T.%T MODIFY %T %Q %Q %Q', + $adjust['database'], + $adjust['table'], + $adjust['name'], + $adjust['type'], + implode(' ', $parts), + $adjust['nullable'] ? 'NULL' : 'NOT NULL'); + + break; + default: + throw new Exception( + pht('Unknown schema adjustment kind "%s"!', $adjust['kind'])); + } + } catch (AphrontQueryException $ex) { + $failed[] = array($adjust, $ex); } - $bar->update(1); } $bar->done(); + if (!$failed) { + $console->writeOut( + "%s\n", + pht('Completed fixing all schema issues.')); + return; + } + + $table = id(new PhutilConsoleTable()) + ->addColumn('target', array('title' => pht('Target'))) + ->addColumn('error', array('title' => pht('Error'))); + + foreach ($failed as $failure) { + list($adjust, $ex) = $failure; + + $pieces = array_select_keys($adjust, array('database', 'table', 'naeme')); + $pieces = array_filter($pieces); + $target = implode('.', $pieces); + + $table->addRow( + array( + 'target' => $target, + 'error' => $ex->getMessage(), + )); + } + + $console->writeOut("\n"); + $table->draw(); $console->writeOut( - "%s\n", - pht('Completed fixing all schema issues.')); + "\n%s\n", + pht('Failed to make some schema adjustments, detailed above.')); + + return 1; } private function findAdjustments() { list($comp, $expect, $actual) = $this->loadSchemata(); $issue_charset = PhabricatorConfigStorageSchema::ISSUE_CHARSET; $issue_collation = PhabricatorConfigStorageSchema::ISSUE_COLLATION; + $issue_columntype = PhabricatorConfigStorageSchema::ISSUE_COLUMNTYPE; $adjustments = array(); foreach ($comp->getDatabases() as $database_name => $database) { $expect_database = $expect->getDatabase($database_name); $actual_database = $actual->getDatabase($database_name); if (!$expect_database || !$actual_database) { // If there's a real issue here, skip this stuff. continue; } $issues = array(); if ($database->hasIssue($issue_charset)) { $issues[] = $issue_charset; } if ($database->hasIssue($issue_collation)) { $issues[] = $issue_collation; } if ($issues) { $adjustments[] = array( 'kind' => 'database', 'database' => $database_name, 'issues' => $issues, 'charset' => $expect_database->getCharacterSet(), 'collation' => $expect_database->getCollation(), ); } foreach ($database->getTables() as $table_name => $table) { $expect_table = $expect_database->getTable($table_name); $actual_table = $actual_database->getTable($table_name); if (!$expect_table || !$actual_table) { continue; } $issues = array(); if ($table->hasIssue($issue_collation)) { $issues[] = $issue_collation; } if ($issues) { $adjustments[] = array( 'kind' => 'table', 'database' => $database_name, 'table' => $table_name, 'issues' => $issues, 'collation' => $expect_table->getCollation(), ); } + + foreach ($table->getColumns() as $column_name => $column) { + $expect_column = $expect_table->getColumn($column_name); + $actual_column = $actual_table->getColumn($column_name); + + if (!$expect_column || !$actual_column) { + continue; + } + + $issues = array(); + if ($column->hasIssue($issue_collation)) { + $issues[] = $issue_collation; + } + if ($column->hasIssue($issue_charset)) { + $issues[] = $issue_charset; + } + if ($column->hasIssue($issue_columntype)) { + $issues[] = $issue_columntype; + } + + if ($issues) { + if ($expect_column->getCharacterSet() === null) { + // For non-text columns, we won't be specifying a collation or + // character set. + $charset = null; + $collation = null; + } else { + $charset = $expect_column->getCharacterSet(); + $collation = $expect_column->getCollation(); + } + + + $adjustments[] = array( + 'kind' => 'column', + 'database' => $database_name, + 'table' => $table_name, + 'name' => $column_name, + 'issues' => $issues, + 'collation' => $collation, + 'charset' => $charset, + 'type' => $expect_column->getColumnType(), + + // NOTE: We don't adjust column nullability because it is + // dangerous, so always use the current nullability. + 'nullable' => $actual_column->getNullable(), + ); + } + } } } return $adjustments; } }