diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php index fe7a7d1e61..f423da8b2e 100644 --- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php @@ -1,665 +1,64 @@ setName('adjust') ->setExamples('**adjust** [__options__]') ->setSynopsis( pht( 'Make schemata adjustments to correct issues with characters sets, '. 'collations, and keys.')) ->setArguments( array( array( 'name' => 'unsafe', 'help' => pht( 'Permit adjustments which truncate data. This option may '. 'destroy some data, but the lost data is usually not '. 'important (most commonly, the ends of very long object '. 'titles).'), ), )); } public function execute(PhutilArgumentParser $args) { $force = $args->getArg('force'); $unsafe = $args->getArg('unsafe'); + $dry_run = $args->getArg('dryrun'); $this->requireAllPatchesApplied(); - return $this->adjustSchemata($force, $unsafe); + return $this->adjustSchemata($force, $unsafe, $dry_run); } private function requireAllPatchesApplied() { $api = $this->getAPI(); $applied = $api->getAppliedPatches(); if ($applied === null) { throw new PhutilArgumentUsageException( pht( 'You have not initialized the database yet. You must initialize '. 'the database before you can adjust schemata. Run `storage upgrade` '. 'to initialize the database.')); } $applied = array_fuse($applied); $patches = $this->getPatches(); $patches = mpull($patches, null, 'getFullKey'); $missing = array_diff_key($patches, $applied); if ($missing) { throw new PhutilArgumentUsageException( pht( 'You have not applied all available storage patches yet. You must '. 'apply all available patches before you can adjust schemata. '. 'Run `storage status` to show patch status, and `storage upgrade` '. 'to apply missing patches.')); } } - private function loadSchemata() { - $query = id(new PhabricatorConfigSchemaQuery()) - ->setAPI($this->getAPI()); - - $actual = $query->loadActualSchema(); - $expect = $query->loadExpectedSchema(); - $comp = $query->buildComparisonSchema($expect, $actual); - - return array($comp, $expect, $actual); - } - - private function adjustSchemata($force, $unsafe) { - $console = PhutilConsole::getConsole(); - - $console->writeOut( - "%s\n", - pht('Verifying database schemata...')); - - list($adjustments, $errors) = $this->findAdjustments(); - $api = $this->getAPI(); - - if (!$adjustments) { - $console->writeOut( - "%s\n", - pht('Found no adjustments for schemata.')); - - return $this->printErrors($errors, 0); - } - - if (!$force && !$api->isCharacterSetAvailable('utf8mb4')) { - $message = pht( - "You have an old version of MySQL (older than 5.5) which does not ". - "support the utf8mb4 character set. If you apply adjustments now ". - "and later update MySQL to 5.5 or newer, you'll need to apply ". - "adjustments again (and they will take a long time).\n\n". - "You can exit this workflow, update MySQL now, and then run this ". - "workflow again. This is recommended, but may cause a lot of downtime ". - "right now.\n\n". - "You can exit this workflow, continue using Phabricator without ". - "applying adjustments, update MySQL at a later date, and then run ". - "this workflow again. This is also a good approach, and will let you ". - "delay downtime until later.\n\n". - "You can proceed with this workflow, and then optionally update ". - "MySQL at a later date. After you do, you'll need to apply ". - "adjustments again.\n\n". - "For more information, see \"Managing Storage Adjustments\" in ". - "the documentation."); - - $console->writeOut( - "\n** %s **\n\n%s\n", - pht('OLD MySQL VERSION'), - phutil_console_wrap($message)); - - $prompt = pht('Continue with old MySQL version?'); - if (!phutil_console_confirm($prompt, $default_no = true)) { - return; - } - } - - $table = id(new PhutilConsoleTable()) - ->addColumn('database', array('title' => pht('Database'))) - ->addColumn('table', array('title' => pht('Table'))) - ->addColumn('name', array('title' => pht('Name'))) - ->addColumn('info', array('title' => pht('Issues'))); - - foreach ($adjustments as $adjust) { - $info = array(); - foreach ($adjust['issues'] as $issue) { - $info[] = PhabricatorConfigStorageSchema::getIssueName($issue); - } - - $table->addRow(array( - 'database' => $adjust['database'], - 'table' => idx($adjust, 'table'), - 'name' => idx($adjust, 'name'), - 'info' => implode(', ', $info), - )); - } - - $console->writeOut("\n\n"); - - $table->draw(); - - if (!$force) { - $console->writeOut( - "\n%s\n", - pht( - "Found %s issues(s) with schemata, detailed above.\n\n". - "You can review issues in more detail from the web interface, ". - "in Config > Database Status. To better understand the adjustment ". - "workflow, see \"Managing Storage Adjustments\" in the ". - "documentation.\n\n". - "MySQL needs to copy table data to make some adjustments, so these ". - "migrations may take some time.", - new PhutilNumber(count($adjustments)))); - - $prompt = pht('Fix these schema issues?'); - if (!phutil_console_confirm($prompt, $default_no = true)) { - return; - } - } - - $console->writeOut( - "%s\n", - pht('Dropping caches, for faster migrations...')); - - $root = dirname(phutil_get_library_root('phabricator')); - $bin = $root.'/bin/cache'; - phutil_passthru('%s purge --purge-all', $bin); - - $console->writeOut( - "%s\n", - pht('Fixing schema issues...')); - - $conn = $api->getConn(null); - - if ($unsafe) { - queryfx($conn, 'SET SESSION sql_mode = %s', ''); - } else { - queryfx($conn, 'SET SESSION sql_mode = %s', 'STRICT_ALL_TABLES'); - } - - $failed = array(); - - // We make changes in several phases. - $phases = array( - // Drop surplus autoincrements. This allows us to drop primary keys on - // autoincrement columns. - 'drop_auto', - - // Drop all keys we're going to adjust. This prevents them from - // interfering with column changes. - 'drop_keys', - - // Apply all database, table, and column changes. - 'main', - - // Restore adjusted keys. - 'add_keys', - - // Add missing autoincrements. - 'add_auto', - ); - - $bar = id(new PhutilConsoleProgressBar()) - ->setTotal(count($adjustments) * count($phases)); - - foreach ($phases as $phase) { - foreach ($adjustments as $adjust) { - try { - switch ($adjust['kind']) { - case 'database': - if ($phase == 'main') { - queryfx( - $conn, - 'ALTER DATABASE %T CHARACTER SET = %s COLLATE = %s', - $adjust['database'], - $adjust['charset'], - $adjust['collation']); - } - break; - case 'table': - if ($phase == 'main') { - queryfx( - $conn, - 'ALTER TABLE %T.%T COLLATE = %s', - $adjust['database'], - $adjust['table'], - $adjust['collation']); - } - break; - case 'column': - $apply = false; - $auto = false; - $new_auto = idx($adjust, 'auto'); - if ($phase == 'drop_auto') { - if ($new_auto === false) { - $apply = true; - $auto = false; - } - } else if ($phase == 'main') { - $apply = true; - if ($new_auto === false) { - $auto = false; - } else { - $auto = $adjust['is_auto']; - } - } else if ($phase == 'add_auto') { - if ($new_auto === true) { - $apply = true; - $auto = true; - } - } - - if ($apply) { - $parts = array(); - - if ($auto) { - $parts[] = qsprintf( - $conn, - 'AUTO_INCREMENT'); - } - - if ($adjust['charset']) { - $parts[] = qsprintf( - $conn, - 'CHARACTER SET %Q COLLATE %Q', - $adjust['charset'], - $adjust['collation']); - } - - queryfx( - $conn, - 'ALTER TABLE %T.%T MODIFY %T %Q %Q %Q', - $adjust['database'], - $adjust['table'], - $adjust['name'], - $adjust['type'], - implode(' ', $parts), - $adjust['nullable'] ? 'NULL' : 'NOT NULL'); - } - break; - case 'key': - if (($phase == 'drop_keys') && $adjust['exists']) { - if ($adjust['name'] == 'PRIMARY') { - $key_name = 'PRIMARY KEY'; - } else { - $key_name = qsprintf($conn, 'KEY %T', $adjust['name']); - } - - queryfx( - $conn, - 'ALTER TABLE %T.%T DROP %Q', - $adjust['database'], - $adjust['table'], - $key_name); - } - - if (($phase == 'add_keys') && $adjust['keep']) { - // Different keys need different creation syntax. Notable - // special cases are primary keys and fulltext keys. - if ($adjust['name'] == 'PRIMARY') { - $key_name = 'PRIMARY KEY'; - } else if ($adjust['indexType'] == 'FULLTEXT') { - $key_name = qsprintf($conn, 'FULLTEXT %T', $adjust['name']); - } else { - if ($adjust['unique']) { - $key_name = qsprintf( - $conn, - 'UNIQUE KEY %T', - $adjust['name']); - } else { - $key_name = qsprintf( - $conn, - '/* NONUNIQUE */ KEY %T', - $adjust['name']); - } - } - - queryfx( - $conn, - 'ALTER TABLE %T.%T ADD %Q (%Q)', - $adjust['database'], - $adjust['table'], - $key_name, - implode(', ', $adjust['columns'])); - } - break; - default: - throw new Exception( - pht('Unknown schema adjustment kind "%s"!', $adjust['kind'])); - } - } catch (AphrontQueryException $ex) { - $failed[] = array($adjust, $ex); - } - $bar->update(1); - } - } - $bar->done(); - - if (!$failed) { - $console->writeOut( - "%s\n", - pht('Completed fixing all schema issues.')); - - $err = 0; - } else { - $table = id(new PhutilConsoleTable()) - ->addColumn('target', array('title' => pht('Target'))) - ->addColumn('error', array('title' => pht('Error'))); - - foreach ($failed as $failure) { - list($adjust, $ex) = $failure; - - $pieces = array_select_keys( - $adjust, - array('database', 'table', 'name')); - $pieces = array_filter($pieces); - $target = implode('.', $pieces); - - $table->addRow( - array( - 'target' => $target, - 'error' => $ex->getMessage(), - )); - } - - $console->writeOut("\n"); - $table->draw(); - $console->writeOut( - "\n%s\n", - pht('Failed to make some schema adjustments, detailed above.')); - $console->writeOut( - "%s\n", - pht( - 'For help troubleshooting adjustments, see "Managing Storage '. - 'Adjustments" in the documentation.')); - - $err = 1; - } - - return $this->printErrors($errors, $err); - } - - private function findAdjustments() { - list($comp, $expect, $actual) = $this->loadSchemata(); - - $issue_charset = PhabricatorConfigStorageSchema::ISSUE_CHARSET; - $issue_collation = PhabricatorConfigStorageSchema::ISSUE_COLLATION; - $issue_columntype = PhabricatorConfigStorageSchema::ISSUE_COLUMNTYPE; - $issue_surpluskey = PhabricatorConfigStorageSchema::ISSUE_SURPLUSKEY; - $issue_missingkey = PhabricatorConfigStorageSchema::ISSUE_MISSINGKEY; - $issue_columns = PhabricatorConfigStorageSchema::ISSUE_KEYCOLUMNS; - $issue_unique = PhabricatorConfigStorageSchema::ISSUE_UNIQUE; - $issue_longkey = PhabricatorConfigStorageSchema::ISSUE_LONGKEY; - $issue_auto = PhabricatorConfigStorageSchema::ISSUE_AUTOINCREMENT; - - $adjustments = array(); - $errors = array(); - foreach ($comp->getDatabases() as $database_name => $database) { - foreach ($this->findErrors($database) as $issue) { - $errors[] = array( - 'database' => $database_name, - 'issue' => $issue, - ); - } - - $expect_database = $expect->getDatabase($database_name); - $actual_database = $actual->getDatabase($database_name); - - if (!$expect_database || !$actual_database) { - // If there's a real issue here, skip this stuff. - continue; - } - - $issues = array(); - if ($database->hasIssue($issue_charset)) { - $issues[] = $issue_charset; - } - if ($database->hasIssue($issue_collation)) { - $issues[] = $issue_collation; - } - - if ($issues) { - $adjustments[] = array( - 'kind' => 'database', - 'database' => $database_name, - 'issues' => $issues, - 'charset' => $expect_database->getCharacterSet(), - 'collation' => $expect_database->getCollation(), - ); - } - - foreach ($database->getTables() as $table_name => $table) { - foreach ($this->findErrors($table) as $issue) { - $errors[] = array( - 'database' => $database_name, - 'table' => $table_name, - 'issue' => $issue, - ); - } - - $expect_table = $expect_database->getTable($table_name); - $actual_table = $actual_database->getTable($table_name); - - if (!$expect_table || !$actual_table) { - continue; - } - - $issues = array(); - if ($table->hasIssue($issue_collation)) { - $issues[] = $issue_collation; - } - - if ($issues) { - $adjustments[] = array( - 'kind' => 'table', - 'database' => $database_name, - 'table' => $table_name, - 'issues' => $issues, - 'collation' => $expect_table->getCollation(), - ); - } - - foreach ($table->getColumns() as $column_name => $column) { - foreach ($this->findErrors($column) as $issue) { - $errors[] = array( - 'database' => $database_name, - 'table' => $table_name, - 'name' => $column_name, - 'issue' => $issue, - ); - } - - $expect_column = $expect_table->getColumn($column_name); - $actual_column = $actual_table->getColumn($column_name); - - if (!$expect_column || !$actual_column) { - continue; - } - - $issues = array(); - if ($column->hasIssue($issue_collation)) { - $issues[] = $issue_collation; - } - if ($column->hasIssue($issue_charset)) { - $issues[] = $issue_charset; - } - if ($column->hasIssue($issue_columntype)) { - $issues[] = $issue_columntype; - } - if ($column->hasIssue($issue_auto)) { - $issues[] = $issue_auto; - } - - if ($issues) { - if ($expect_column->getCharacterSet() === null) { - // For non-text columns, we won't be specifying a collation or - // character set. - $charset = null; - $collation = null; - } else { - $charset = $expect_column->getCharacterSet(); - $collation = $expect_column->getCollation(); - } - - $adjustment = array( - 'kind' => 'column', - 'database' => $database_name, - 'table' => $table_name, - 'name' => $column_name, - 'issues' => $issues, - 'collation' => $collation, - 'charset' => $charset, - 'type' => $expect_column->getColumnType(), - - // NOTE: We don't adjust column nullability because it is - // dangerous, so always use the current nullability. - 'nullable' => $actual_column->getNullable(), - - // NOTE: This always stores the current value, because we have - // to make these updates separately. - 'is_auto' => $actual_column->getAutoIncrement(), - ); - - if ($column->hasIssue($issue_auto)) { - $adjustment['auto'] = $expect_column->getAutoIncrement(); - } - - $adjustments[] = $adjustment; - } - } - - foreach ($table->getKeys() as $key_name => $key) { - foreach ($this->findErrors($key) as $issue) { - $errors[] = array( - 'database' => $database_name, - 'table' => $table_name, - 'name' => $key_name, - 'issue' => $issue, - ); - } - - $expect_key = $expect_table->getKey($key_name); - $actual_key = $actual_table->getKey($key_name); - - $issues = array(); - $keep_key = true; - if ($key->hasIssue($issue_surpluskey)) { - $issues[] = $issue_surpluskey; - $keep_key = false; - } - - if ($key->hasIssue($issue_missingkey)) { - $issues[] = $issue_missingkey; - } - - if ($key->hasIssue($issue_columns)) { - $issues[] = $issue_columns; - } - - if ($key->hasIssue($issue_unique)) { - $issues[] = $issue_unique; - } - - // NOTE: We can't really fix this, per se, but we may need to remove - // the key to change the column type. In the best case, the new - // column type won't be overlong and recreating the key really will - // fix the issue. In the worst case, we get the right column type and - // lose the key, which is still better than retaining the key having - // the wrong column type. - if ($key->hasIssue($issue_longkey)) { - $issues[] = $issue_longkey; - } - - if ($issues) { - $adjustment = array( - 'kind' => 'key', - 'database' => $database_name, - 'table' => $table_name, - 'name' => $key_name, - 'issues' => $issues, - 'exists' => (bool)$actual_key, - 'keep' => $keep_key, - ); - - if ($keep_key) { - $adjustment += array( - 'columns' => $expect_key->getColumnNames(), - 'unique' => $expect_key->getUnique(), - 'indexType' => $expect_key->getIndexType(), - ); - } - - $adjustments[] = $adjustment; - } - } - } - } - - return array($adjustments, $errors); - } - - private function findErrors(PhabricatorConfigStorageSchema $schema) { - $result = array(); - foreach ($schema->getLocalIssues() as $issue) { - $status = PhabricatorConfigStorageSchema::getIssueStatus($issue); - if ($status == PhabricatorConfigStorageSchema::STATUS_FAIL) { - $result[] = $issue; - } - } - return $result; - } - - private function printErrors(array $errors, $default_return) { - if (!$errors) { - return $default_return; - } - - $console = PhutilConsole::getConsole(); - - $table = id(new PhutilConsoleTable()) - ->addColumn('target', array('title' => pht('Target'))) - ->addColumn('error', array('title' => pht('Error'))); - - foreach ($errors as $error) { - $pieces = array_select_keys( - $error, - array('database', 'table', 'name')); - $pieces = array_filter($pieces); - $target = implode('.', $pieces); - - $name = PhabricatorConfigStorageSchema::getIssueName($error['issue']); - - $table->addRow( - array( - 'target' => $target, - 'error' => $name, - )); - } - - $console->writeOut("\n"); - $table->draw(); - $console->writeOut("\n"); - - $message = pht( - "The schemata have serious errors (detailed above) which the adjustment ". - "workflow can not fix.\n\n". - "If you are not developing Phabricator itself, report this issue to ". - "the upstream.\n\n". - "If you are developing Phabricator, these errors usually indicate that ". - "your schema specifications do not agree with the schemata your code ". - "actually builds."); - - $console->writeOut( - "** %s **\n\n%s\n", - pht('SCHEMATA ERRORS'), - phutil_console_wrap($message)); - - return 2; - } - } diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php index 8479a4e925..f9c2ed902a 100644 --- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementUpgradeWorkflow.php @@ -1,193 +1,210 @@ setName('upgrade') ->setExamples('**upgrade** [__options__]') ->setSynopsis('Upgrade database schemata.') ->setArguments( array( array( 'name' => 'apply', 'param' => 'patch', - 'help' => 'Apply __patch__ explicitly. This is an advanced '. - 'feature for development and debugging; you should '. - 'not normally use this flag.', + 'help' => pht( + 'Apply __patch__ explicitly. This is an advanced feature for '. + 'development and debugging; you should not normally use this '. + 'flag. This skips adjustment.'), ), array( 'name' => 'no-quickstart', - 'help' => 'Build storage patch-by-patch from scatch, even if it '. - 'could be loaded from the quickstart template.', + 'help' => pht( + 'Build storage patch-by-patch from scatch, even if it could '. + 'be loaded from the quickstart template.'), ), array( 'name' => 'init-only', - 'help' => 'Initialize storage only; do not apply patches.', + 'help' => pht( + 'Initialize storage only; do not apply patches or adjustments.'), + ), + array( + 'name' => 'no-adjust', + 'help' => pht( + 'Do not apply storage adjustments after storage upgrades.'), ), )); } public function execute(PhutilArgumentParser $args) { $is_dry = $args->getArg('dryrun'); $is_force = $args->getArg('force'); $api = $this->getAPI(); $patches = $this->getPatches(); if (!$is_dry && !$is_force) { echo phutil_console_wrap( 'Before running storage upgrades, you should take down the '. 'Phabricator web interface and stop any running Phabricator '. 'daemons (you can disable this warning with --force).'); if (!phutil_console_confirm('Are you ready to continue?')) { echo "Cancelled.\n"; return 1; } } $apply_only = $args->getArg('apply'); if ($apply_only) { if (empty($patches[$apply_only])) { throw new PhutilArgumentUsageException( "--apply argument '{$apply_only}' is not a valid patch. Use ". "'storage status' to show patch status."); } } $no_quickstart = $args->getArg('no-quickstart'); $init_only = $args->getArg('init-only'); + $no_adjust = $args->getArg('no-adjust'); $applied = $api->getAppliedPatches(); if ($applied === null) { if ($is_dry) { echo "DRYRUN: Patch metadata storage doesn't exist yet, it would ". "be created.\n"; return 0; } if ($apply_only) { throw new PhutilArgumentUsageException( 'Storage has not been initialized yet, you must initialize storage '. 'before selectively applying patches.'); return 1; } $legacy = $api->getLegacyPatches($patches); if ($legacy || $no_quickstart || $init_only) { // If we have legacy patches, we can't quickstart. $api->createDatabase('meta_data'); $api->createTable( 'meta_data', 'patch_status', array( 'patch VARCHAR(255) NOT NULL PRIMARY KEY COLLATE utf8_general_ci', 'applied INT UNSIGNED NOT NULL', )); foreach ($legacy as $patch) { $api->markPatchApplied($patch); } } else { echo "Loading quickstart template...\n"; $root = dirname(phutil_get_library_root('phabricator')); $sql = $root.'/resources/sql/quickstart.sql'; $api->applyPatchSQL($sql); } } if ($init_only) { echo "Storage initialized.\n"; return 0; } $applied = $api->getAppliedPatches(); $applied = array_fuse($applied); $skip_mark = false; if ($apply_only) { if (isset($applied[$apply_only])) { unset($applied[$apply_only]); $skip_mark = true; if (!$is_force && !$is_dry) { echo phutil_console_wrap( "Patch '{$apply_only}' has already been applied. Are you sure ". "you want to apply it again? This may put your storage in a state ". "that the upgrade scripts can not automatically manage."); if (!phutil_console_confirm('Apply patch again?')) { echo "Cancelled.\n"; return 1; } } } } while (true) { $applied_something = false; foreach ($patches as $key => $patch) { if (isset($applied[$key])) { unset($patches[$key]); continue; } if ($apply_only && $apply_only != $key) { unset($patches[$key]); continue; } $can_apply = true; foreach ($patch->getAfter() as $after) { if (empty($applied[$after])) { if ($apply_only) { echo "Unable to apply patch '{$apply_only}' because it depends ". "on patch '{$after}', which has not been applied.\n"; return 1; } $can_apply = false; break; } } if (!$can_apply) { continue; } $applied_something = true; if ($is_dry) { echo "DRYRUN: Would apply patch '{$key}'.\n"; } else { echo "Applying patch '{$key}'...\n"; $api->applyPatch($patch); if (!$skip_mark) { $api->markPatchApplied($key); } } unset($patches[$key]); $applied[$key] = true; } if (!$applied_something) { if (count($patches)) { throw new Exception( 'Some patches could not be applied: '. implode(', ', array_keys($patches))); } else if (!$is_dry && !$apply_only) { echo "Storage is up to date. Use 'storage status' for details.\n"; } break; } } - return 0; + $console = PhutilConsole::getConsole(); + if ($no_adjust || $init_only || $apply_only) { + $console->writeOut( + "%s\n", + pht('Declining to apply storage adjustments.')); + return 0; + } else { + return $this->adjustSchemata($is_force, $unsafe = false, $is_dry); + } } } diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php index ccb96b0c54..87778dbe42 100644 --- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementWorkflow.php @@ -1,28 +1,635 @@ patches = $patches; return $this; } public function getPatches() { return $this->patches; } final public function setAPI(PhabricatorStorageManagementAPI $api) { $this->api = $api; return $this; } final public function getAPI() { return $this->api; } + private function loadSchemata() { + $query = id(new PhabricatorConfigSchemaQuery()) + ->setAPI($this->getAPI()); + + $actual = $query->loadActualSchema(); + $expect = $query->loadExpectedSchema(); + $comp = $query->buildComparisonSchema($expect, $actual); + + return array($comp, $expect, $actual); + } + + protected function adjustSchemata($force, $unsafe, $dry_run) { + $console = PhutilConsole::getConsole(); + + $console->writeOut( + "%s\n", + pht('Verifying database schemata...')); + + list($adjustments, $errors) = $this->findAdjustments(); + $api = $this->getAPI(); + + if (!$adjustments) { + $console->writeOut( + "%s\n", + pht('Found no adjustments for schemata.')); + + return $this->printErrors($errors, 0); + } + + if (!$force && !$api->isCharacterSetAvailable('utf8mb4')) { + $message = pht( + "You have an old version of MySQL (older than 5.5) which does not ". + "support the utf8mb4 character set. If you apply adjustments now ". + "and later update MySQL to 5.5 or newer, you'll need to apply ". + "adjustments again (and they will take a long time).\n\n". + "You can exit this workflow, update MySQL now, and then run this ". + "workflow again. This is recommended, but may cause a lot of downtime ". + "right now.\n\n". + "You can exit this workflow, continue using Phabricator without ". + "applying adjustments, update MySQL at a later date, and then run ". + "this workflow again. This is also a good approach, and will let you ". + "delay downtime until later.\n\n". + "You can proceed with this workflow, and then optionally update ". + "MySQL at a later date. After you do, you'll need to apply ". + "adjustments again.\n\n". + "For more information, see \"Managing Storage Adjustments\" in ". + "the documentation."); + + $console->writeOut( + "\n** %s **\n\n%s\n", + pht('OLD MySQL VERSION'), + phutil_console_wrap($message)); + + $prompt = pht('Continue with old MySQL version?'); + if (!phutil_console_confirm($prompt, $default_no = true)) { + return; + } + } + + $table = id(new PhutilConsoleTable()) + ->addColumn('database', array('title' => pht('Database'))) + ->addColumn('table', array('title' => pht('Table'))) + ->addColumn('name', array('title' => pht('Name'))) + ->addColumn('info', array('title' => pht('Issues'))); + + foreach ($adjustments as $adjust) { + $info = array(); + foreach ($adjust['issues'] as $issue) { + $info[] = PhabricatorConfigStorageSchema::getIssueName($issue); + } + + $table->addRow(array( + 'database' => $adjust['database'], + 'table' => idx($adjust, 'table'), + 'name' => idx($adjust, 'name'), + 'info' => implode(', ', $info), + )); + } + + $console->writeOut("\n\n"); + + $table->draw(); + + if ($dry_run) { + $console->writeOut( + "%s\n", + pht('DRYRUN: Would apply adjustments.')); + return 0; + } else if (!$force) { + $console->writeOut( + "\n%s\n", + pht( + "Found %s issues(s) with schemata, detailed above.\n\n". + "You can review issues in more detail from the web interface, ". + "in Config > Database Status. To better understand the adjustment ". + "workflow, see \"Managing Storage Adjustments\" in the ". + "documentation.\n\n". + "MySQL needs to copy table data to make some adjustments, so these ". + "migrations may take some time.", + new PhutilNumber(count($adjustments)))); + + $prompt = pht('Fix these schema issues?'); + if (!phutil_console_confirm($prompt, $default_no = true)) { + return 1; + } + } + + $console->writeOut( + "%s\n", + pht('Dropping caches, for faster migrations...')); + + $root = dirname(phutil_get_library_root('phabricator')); + $bin = $root.'/bin/cache'; + phutil_passthru('%s purge --purge-all', $bin); + + $console->writeOut( + "%s\n", + pht('Fixing schema issues...')); + + $conn = $api->getConn(null); + + if ($unsafe) { + queryfx($conn, 'SET SESSION sql_mode = %s', ''); + } else { + queryfx($conn, 'SET SESSION sql_mode = %s', 'STRICT_ALL_TABLES'); + } + + $failed = array(); + + // We make changes in several phases. + $phases = array( + // Drop surplus autoincrements. This allows us to drop primary keys on + // autoincrement columns. + 'drop_auto', + + // Drop all keys we're going to adjust. This prevents them from + // interfering with column changes. + 'drop_keys', + + // Apply all database, table, and column changes. + 'main', + + // Restore adjusted keys. + 'add_keys', + + // Add missing autoincrements. + 'add_auto', + ); + + $bar = id(new PhutilConsoleProgressBar()) + ->setTotal(count($adjustments) * count($phases)); + + foreach ($phases as $phase) { + foreach ($adjustments as $adjust) { + try { + switch ($adjust['kind']) { + case 'database': + if ($phase == 'main') { + queryfx( + $conn, + 'ALTER DATABASE %T CHARACTER SET = %s COLLATE = %s', + $adjust['database'], + $adjust['charset'], + $adjust['collation']); + } + break; + case 'table': + if ($phase == 'main') { + queryfx( + $conn, + 'ALTER TABLE %T.%T COLLATE = %s', + $adjust['database'], + $adjust['table'], + $adjust['collation']); + } + break; + case 'column': + $apply = false; + $auto = false; + $new_auto = idx($adjust, 'auto'); + if ($phase == 'drop_auto') { + if ($new_auto === false) { + $apply = true; + $auto = false; + } + } else if ($phase == 'main') { + $apply = true; + if ($new_auto === false) { + $auto = false; + } else { + $auto = $adjust['is_auto']; + } + } else if ($phase == 'add_auto') { + if ($new_auto === true) { + $apply = true; + $auto = true; + } + } + + if ($apply) { + $parts = array(); + + if ($auto) { + $parts[] = qsprintf( + $conn, + 'AUTO_INCREMENT'); + } + + if ($adjust['charset']) { + $parts[] = qsprintf( + $conn, + 'CHARACTER SET %Q COLLATE %Q', + $adjust['charset'], + $adjust['collation']); + } + + queryfx( + $conn, + 'ALTER TABLE %T.%T MODIFY %T %Q %Q %Q', + $adjust['database'], + $adjust['table'], + $adjust['name'], + $adjust['type'], + implode(' ', $parts), + $adjust['nullable'] ? 'NULL' : 'NOT NULL'); + } + break; + case 'key': + if (($phase == 'drop_keys') && $adjust['exists']) { + if ($adjust['name'] == 'PRIMARY') { + $key_name = 'PRIMARY KEY'; + } else { + $key_name = qsprintf($conn, 'KEY %T', $adjust['name']); + } + + queryfx( + $conn, + 'ALTER TABLE %T.%T DROP %Q', + $adjust['database'], + $adjust['table'], + $key_name); + } + + if (($phase == 'add_keys') && $adjust['keep']) { + // Different keys need different creation syntax. Notable + // special cases are primary keys and fulltext keys. + if ($adjust['name'] == 'PRIMARY') { + $key_name = 'PRIMARY KEY'; + } else if ($adjust['indexType'] == 'FULLTEXT') { + $key_name = qsprintf($conn, 'FULLTEXT %T', $adjust['name']); + } else { + if ($adjust['unique']) { + $key_name = qsprintf( + $conn, + 'UNIQUE KEY %T', + $adjust['name']); + } else { + $key_name = qsprintf( + $conn, + '/* NONUNIQUE */ KEY %T', + $adjust['name']); + } + } + + queryfx( + $conn, + 'ALTER TABLE %T.%T ADD %Q (%Q)', + $adjust['database'], + $adjust['table'], + $key_name, + implode(', ', $adjust['columns'])); + } + break; + default: + throw new Exception( + pht('Unknown schema adjustment kind "%s"!', $adjust['kind'])); + } + } catch (AphrontQueryException $ex) { + $failed[] = array($adjust, $ex); + } + $bar->update(1); + } + } + $bar->done(); + + if (!$failed) { + $console->writeOut( + "%s\n", + pht('Completed fixing all schema issues.')); + + $err = 0; + } else { + $table = id(new PhutilConsoleTable()) + ->addColumn('target', array('title' => pht('Target'))) + ->addColumn('error', array('title' => pht('Error'))); + + foreach ($failed as $failure) { + list($adjust, $ex) = $failure; + + $pieces = array_select_keys( + $adjust, + array('database', 'table', 'name')); + $pieces = array_filter($pieces); + $target = implode('.', $pieces); + + $table->addRow( + array( + 'target' => $target, + 'error' => $ex->getMessage(), + )); + } + + $console->writeOut("\n"); + $table->draw(); + $console->writeOut( + "\n%s\n", + pht('Failed to make some schema adjustments, detailed above.')); + $console->writeOut( + "%s\n", + pht( + 'For help troubleshooting adjustments, see "Managing Storage '. + 'Adjustments" in the documentation.')); + + $err = 1; + } + + return $this->printErrors($errors, $err); + } + + private function findAdjustments() { + list($comp, $expect, $actual) = $this->loadSchemata(); + + $issue_charset = PhabricatorConfigStorageSchema::ISSUE_CHARSET; + $issue_collation = PhabricatorConfigStorageSchema::ISSUE_COLLATION; + $issue_columntype = PhabricatorConfigStorageSchema::ISSUE_COLUMNTYPE; + $issue_surpluskey = PhabricatorConfigStorageSchema::ISSUE_SURPLUSKEY; + $issue_missingkey = PhabricatorConfigStorageSchema::ISSUE_MISSINGKEY; + $issue_columns = PhabricatorConfigStorageSchema::ISSUE_KEYCOLUMNS; + $issue_unique = PhabricatorConfigStorageSchema::ISSUE_UNIQUE; + $issue_longkey = PhabricatorConfigStorageSchema::ISSUE_LONGKEY; + $issue_auto = PhabricatorConfigStorageSchema::ISSUE_AUTOINCREMENT; + + $adjustments = array(); + $errors = array(); + foreach ($comp->getDatabases() as $database_name => $database) { + foreach ($this->findErrors($database) as $issue) { + $errors[] = array( + 'database' => $database_name, + 'issue' => $issue, + ); + } + + $expect_database = $expect->getDatabase($database_name); + $actual_database = $actual->getDatabase($database_name); + + if (!$expect_database || !$actual_database) { + // If there's a real issue here, skip this stuff. + continue; + } + + $issues = array(); + if ($database->hasIssue($issue_charset)) { + $issues[] = $issue_charset; + } + if ($database->hasIssue($issue_collation)) { + $issues[] = $issue_collation; + } + + if ($issues) { + $adjustments[] = array( + 'kind' => 'database', + 'database' => $database_name, + 'issues' => $issues, + 'charset' => $expect_database->getCharacterSet(), + 'collation' => $expect_database->getCollation(), + ); + } + + foreach ($database->getTables() as $table_name => $table) { + foreach ($this->findErrors($table) as $issue) { + $errors[] = array( + 'database' => $database_name, + 'table' => $table_name, + 'issue' => $issue, + ); + } + + $expect_table = $expect_database->getTable($table_name); + $actual_table = $actual_database->getTable($table_name); + + if (!$expect_table || !$actual_table) { + continue; + } + + $issues = array(); + if ($table->hasIssue($issue_collation)) { + $issues[] = $issue_collation; + } + + if ($issues) { + $adjustments[] = array( + 'kind' => 'table', + 'database' => $database_name, + 'table' => $table_name, + 'issues' => $issues, + 'collation' => $expect_table->getCollation(), + ); + } + + foreach ($table->getColumns() as $column_name => $column) { + foreach ($this->findErrors($column) as $issue) { + $errors[] = array( + 'database' => $database_name, + 'table' => $table_name, + 'name' => $column_name, + 'issue' => $issue, + ); + } + + $expect_column = $expect_table->getColumn($column_name); + $actual_column = $actual_table->getColumn($column_name); + + if (!$expect_column || !$actual_column) { + continue; + } + + $issues = array(); + if ($column->hasIssue($issue_collation)) { + $issues[] = $issue_collation; + } + if ($column->hasIssue($issue_charset)) { + $issues[] = $issue_charset; + } + if ($column->hasIssue($issue_columntype)) { + $issues[] = $issue_columntype; + } + if ($column->hasIssue($issue_auto)) { + $issues[] = $issue_auto; + } + + if ($issues) { + if ($expect_column->getCharacterSet() === null) { + // For non-text columns, we won't be specifying a collation or + // character set. + $charset = null; + $collation = null; + } else { + $charset = $expect_column->getCharacterSet(); + $collation = $expect_column->getCollation(); + } + + $adjustment = array( + 'kind' => 'column', + 'database' => $database_name, + 'table' => $table_name, + 'name' => $column_name, + 'issues' => $issues, + 'collation' => $collation, + 'charset' => $charset, + 'type' => $expect_column->getColumnType(), + + // NOTE: We don't adjust column nullability because it is + // dangerous, so always use the current nullability. + 'nullable' => $actual_column->getNullable(), + + // NOTE: This always stores the current value, because we have + // to make these updates separately. + 'is_auto' => $actual_column->getAutoIncrement(), + ); + + if ($column->hasIssue($issue_auto)) { + $adjustment['auto'] = $expect_column->getAutoIncrement(); + } + + $adjustments[] = $adjustment; + } + } + + foreach ($table->getKeys() as $key_name => $key) { + foreach ($this->findErrors($key) as $issue) { + $errors[] = array( + 'database' => $database_name, + 'table' => $table_name, + 'name' => $key_name, + 'issue' => $issue, + ); + } + + $expect_key = $expect_table->getKey($key_name); + $actual_key = $actual_table->getKey($key_name); + + $issues = array(); + $keep_key = true; + if ($key->hasIssue($issue_surpluskey)) { + $issues[] = $issue_surpluskey; + $keep_key = false; + } + + if ($key->hasIssue($issue_missingkey)) { + $issues[] = $issue_missingkey; + } + + if ($key->hasIssue($issue_columns)) { + $issues[] = $issue_columns; + } + + if ($key->hasIssue($issue_unique)) { + $issues[] = $issue_unique; + } + + // NOTE: We can't really fix this, per se, but we may need to remove + // the key to change the column type. In the best case, the new + // column type won't be overlong and recreating the key really will + // fix the issue. In the worst case, we get the right column type and + // lose the key, which is still better than retaining the key having + // the wrong column type. + if ($key->hasIssue($issue_longkey)) { + $issues[] = $issue_longkey; + } + + if ($issues) { + $adjustment = array( + 'kind' => 'key', + 'database' => $database_name, + 'table' => $table_name, + 'name' => $key_name, + 'issues' => $issues, + 'exists' => (bool)$actual_key, + 'keep' => $keep_key, + ); + + if ($keep_key) { + $adjustment += array( + 'columns' => $expect_key->getColumnNames(), + 'unique' => $expect_key->getUnique(), + 'indexType' => $expect_key->getIndexType(), + ); + } + + $adjustments[] = $adjustment; + } + } + } + } + + return array($adjustments, $errors); + } + + private function findErrors(PhabricatorConfigStorageSchema $schema) { + $result = array(); + foreach ($schema->getLocalIssues() as $issue) { + $status = PhabricatorConfigStorageSchema::getIssueStatus($issue); + if ($status == PhabricatorConfigStorageSchema::STATUS_FAIL) { + $result[] = $issue; + } + } + return $result; + } + + private function printErrors(array $errors, $default_return) { + if (!$errors) { + return $default_return; + } + + $console = PhutilConsole::getConsole(); + + $table = id(new PhutilConsoleTable()) + ->addColumn('target', array('title' => pht('Target'))) + ->addColumn('error', array('title' => pht('Error'))); + + foreach ($errors as $error) { + $pieces = array_select_keys( + $error, + array('database', 'table', 'name')); + $pieces = array_filter($pieces); + $target = implode('.', $pieces); + + $name = PhabricatorConfigStorageSchema::getIssueName($error['issue']); + + $table->addRow( + array( + 'target' => $target, + 'error' => $name, + )); + } + + $console->writeOut("\n"); + $table->draw(); + $console->writeOut("\n"); + + $message = pht( + "The schemata have serious errors (detailed above) which the adjustment ". + "workflow can not fix.\n\n". + "If you are not developing Phabricator itself, report this issue to ". + "the upstream.\n\n". + "If you are developing Phabricator, these errors usually indicate that ". + "your schema specifications do not agree with the schemata your code ". + "actually builds."); + + $console->writeOut( + "** %s **\n\n%s\n", + pht('SCHEMATA ERRORS'), + phutil_console_wrap($message)); + + return 2; + } + }