diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -4480,7 +4480,7 @@ 'PhabricatorConfigEntryQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', 'PhabricatorConfigFileSource' => 'PhabricatorConfigProxySource', 'PhabricatorConfigGroupController' => 'PhabricatorConfigController', - 'PhabricatorConfigIgnoreController' => 'PhabricatorApplicationsController', + 'PhabricatorConfigIgnoreController' => 'PhabricatorConfigController', 'PhabricatorConfigIssueListController' => 'PhabricatorConfigController', 'PhabricatorConfigIssueViewController' => 'PhabricatorConfigController', 'PhabricatorConfigJSONOptionType' => 'PhabricatorConfigOptionType', diff --git a/src/applications/config/controller/PhabricatorConfigDatabaseController.php b/src/applications/config/controller/PhabricatorConfigDatabaseController.php --- a/src/applications/config/controller/PhabricatorConfigDatabaseController.php +++ b/src/applications/config/controller/PhabricatorConfigDatabaseController.php @@ -62,4 +62,20 @@ } } + protected function buildHeaderWithDocumentationLink($title) { + + $doc_link = PhabricatorEnv::getDoclink('Managing Storage Adjustments'); + + return id(new PHUIHeaderView()) + ->setHeader($title) + ->addActionLink( + id(new PHUIButtonView()) + ->setTag('a') + ->setIcon( + id(new PHUIIconView()) + ->setIconFont('fa-book')) + ->setHref($doc_link) + ->setText(pht('Learn More'))); + } + } diff --git a/src/applications/config/controller/PhabricatorConfigDatabaseIssueController.php b/src/applications/config/controller/PhabricatorConfigDatabaseIssueController.php --- a/src/applications/config/controller/PhabricatorConfigDatabaseIssueController.php +++ b/src/applications/config/controller/PhabricatorConfigDatabaseIssueController.php @@ -133,10 +133,6 @@ $errors = array(); - $errors[] = pht( - 'IMPORTANT: This feature is in development and the information below '. - 'is not accurate! Ignore it for now. See T1191.'); - if (isset($counts[PhabricatorConfigStorageSchema::STATUS_FAIL])) { $errors[] = pht( 'Detected %s serious issue(s) with the schemata.', @@ -152,7 +148,7 @@ $title = pht('Database Issues'); $table_box = id(new PHUIObjectBoxView()) - ->setHeaderText($title) + ->setHeader($this->buildHeaderWithDocumentationLink($title)) ->setFormErrors($errors) ->appendChild($table); diff --git a/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php b/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php --- a/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php +++ b/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php @@ -166,7 +166,7 @@ $comp->getIssues()); $box = id(new PHUIObjectBoxView()) - ->setHeaderText($title) + ->setHeader($this->buildHeaderWithDocumentationLink($title)) ->addPropertyList($properties) ->appendChild($table); @@ -261,7 +261,7 @@ $database->getIssues()); $box = id(new PHUIObjectBoxView()) - ->setHeaderText($title) + ->setHeader($this->buildHeaderWithDocumentationLink($title)) ->addPropertyList($properties) ->appendChild($table); @@ -471,7 +471,7 @@ $table->getIssues()); $box = id(new PHUIObjectBoxView()) - ->setHeaderText($title) + ->setHeader($this->buildHeaderWithDocumentationLink($title)) ->addPropertyList($properties) ->appendChild($table_view) ->appendChild($keys_view); @@ -609,7 +609,7 @@ $column->getIssues()); $box = id(new PHUIObjectBoxView()) - ->setHeaderText($title) + ->setHeader($this->buildHeaderWithDocumentationLink($title)) ->addPropertyList($properties); return $this->buildResponse($title, $box); @@ -702,7 +702,7 @@ $key->getIssues()); $box = id(new PHUIObjectBoxView()) - ->setHeaderText($title) + ->setHeader($this->buildHeaderWithDocumentationLink($title)) ->addPropertyList($properties); return $this->buildResponse($title, $box); diff --git a/src/docs/user/configuration/storage_adjust.diviner b/src/docs/user/configuration/storage_adjust.diviner new file mode 100644 --- /dev/null +++ b/src/docs/user/configuration/storage_adjust.diviner @@ -0,0 +1,147 @@ +@title Managing Storage Adjustments +@group config + +Explains how to apply storage adjustments to the MySQL schemata. + +Overview +======== + +Phabricator uses a workflow called //storage adjustment// to make some minor +kinds of changes to the MySQL schema. This workflow compliments the //storage +upgrade// workflow, which makes major changes. + +You can perform storage adjustment by running: + + phabricator/ $ ./bin/storage adjust + +This document describes what adjustments are, how they relate to storage +upgrades, how to perform them, and how to troubleshoot issues with storage +adjustment. + + +Understanding Adjustments +=================== + +Storage adjustments make minor changes to the Phabricator MySQL schemata to +improve consistency, unicode handling, and performance. Changes covered by +adjustment include: + + - Character set and collation settings for columns, tables, and databases. + - Setting and removing "Auto Increment" on columns. + - Adding, removing, renaming and adjusting keys. + +Adjustment does not make major changes to the schemata, like creating or +removing columns or tables or migrating data. (Major changes are performed by +the upgrade workflow.) + +Adjustments are separate from upgrades primarily because adjustments depend on +the MySQL version, while upgrades do not. If you update MySQL, better collations +may become available, and the adjustment workflow will convert your schemata to +use them. + +All changes covered by adjustment are minor, and technically optional. However, +you are strongly encouraged to apply outstanding adjustments: if you do not, +you may encounter issues storing or sorting some unicode data, and may suffer +poor performance on some queries. + + +Reviewing Outstanding Adjustments +================================= + +There are two ways to review outstanding adjustments: you can use the web UI, +or you can use the CLI. + +To access the web UI, navigate to {nav Config > Database Status} or +{nav Config > Database Issues}. The //Database Status// panel provides a general +overview of all schemata. The //Database Issues// panel shows outstanding +issues. + +These interfaces report //Errors//, which are serious issues that can not be +resolved through adjustment, and //Warnings//, which are minor issues that the +adjustment workflow can resolve. + +You can also review adjustments from the CLI, by running: + + phabricator/ $ ./bin/storage adjust + +Before you're prompted to actually apply adjustments, you'll be given a list of +available adjustments. You can then make a choice to apply them. + + +Performing Adjustments +====================== + +To perform adjustments, run the `adjust` workflow: + + phabricator/ $ ./bin/storage adjust + +For details about flags, use: + + phabricator/ $ ./bin/storage help adjust + +You do not normally need to run this workflow manually: it will be run +automatically after you run the `upgrade` workflow. + + +History and Rationale +===================== + +The primary motivation for the adjustment workflow is MySQL's handling of +unicode character sets. Before MySQL 5.5, MySQL supports a character set called +`utf8`. However, this character set can not store 4-byte unicode characters +(including emoji). Inserting 4-byte characters into a `utf8` column truncates +the data. + +With MySQL 5.5, a new `utf8mb4` character set was introduced. This character +set can safely store 4-byte unicode characters. + +The adjustment workflow allows us to alter the schema to primarily use +`binary` character sets on older MySQL, and primarily use `utf8mb4` character +sets on newer MySQL. The net effect is that Phabricator works consistently and +can store 4-byte unicode characters regardless of the MySQL version. Under +newer MySQL, we can also take advantage of the better collation rules the +`utf8mb4` character set offers. + +The adjustment workflow was introduced in November 2014. If your install +predates its introduction, your first adjustment may take a long time (we must +convert all of the data out of `utf8` and into the appropriate character set). +If your install was set up after November 2014, adjustments should generally +be very minor and complete quickly, unless you perform a major MySQL update and +make new character sets available. + +If you plan to update MySQL from an older version to 5.5 or newer, it is +advisable to update first, then run the adjustment workflow. If you adjust +first, you'll need to adjust again after updating, so you'll end up spending +twice as much time performing schemata adjustments. + + +Troubleshooting +=============== + +When you apply adjustments, some adjustments may fail. The two most common +errors you may encounter are: + + - **#1406 Data Too Long**: Usually this is caused by a very long object name + (like a task title) which contains multibyte unicode characters. When the + column type is converted to `binary`, only the first part of the title still + fits in the column. Depending on what is failing, you may be able to find + the relevant object in the web UI and retitle it so the adjustment succeeds. + Alternatively, you can use `--unsafe` to force the adjustment to truncate + the title. This will destroy some data, but usually the data is not + important (just the end of very long titles). + - **#1366 Incorrect String Value**: This can occur when converting invalid + or truncated multibyte unicode characters to a unicode character set. + In both cases, the old value can not be represented under the new character + set. You may be able to identify the object and edit it to allow the + adjustment to proceed, or you can use the `--unsafe` flag to truncate the + data at the invalid character. Usually, the truncated data is not important. + +As with most commands, you can add the `--trace` flag to get more details about +what `bin/storage adjust` is doing. This may help you diagnose or understand any +issues you encounter, and this data is useful if you file reports in the +upstream. + +In general, adjustments are not critical. If you run into issues applying +adjustments, it is safe to file a task in the upstream describing the problem +you've encountered and continue using Phabricator normally until the issue can +be resolved. diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php --- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php @@ -79,6 +79,7 @@ pht('Verifying database schemata...')); $adjustments = $this->findAdjustments(); + $api = $this->getAPI(); if (!$adjustments) { $console->writeOut( @@ -87,6 +88,36 @@ return; } + if (!$force && !$api->isCharacterSetAvailable('utf8mb4')) { + $message = pht( + "You have an old version of MySQL (older than 5.5) which does not ". + "support the utf8mb4 character set. If you apply adjustments now ". + "and later update MySQL to 5.5 or newer, you'll need to apply ". + "adjustments again (and they will take a long time).\n\n". + "You can exit this workflow, update MySQL now, and then run this ". + "workflow again. This is recommended, but may cause a lot of downtime ". + "right now.\n\n". + "You can exit this workflow, continue using Phabricator without ". + "applying adjustments, update MySQL at a later date, and then run ". + "this workflow again. This is also a good approach, and will let you ". + "delay downtime until later.\n\n". + "You can proceed with this workflow, and then optionally update ". + "MySQL at a later date. After you do, you'll need to apply ". + "adjustments again.\n\n". + "For more information, see \"Managing Storage Adjustments\" in ". + "the documentation."); + + $console->writeOut( + "\n** %s **\n\n%s\n", + pht('OLD MySQL VERSION'), + phutil_console_wrap($message)); + + $prompt = pht('Continue with old MySQL version?'); + if (!phutil_console_confirm($prompt, $default_no = true)) { + return; + } + } + $table = id(new PhutilConsoleTable()) ->addColumn('database', array('title' => pht('Database'))) ->addColumn('table', array('title' => pht('Table'))) @@ -147,7 +178,6 @@ "%s\n", pht('Fixing schema issues...')); - $api = $this->getAPI(); $conn = $api->getConn(null); if ($unsafe) { @@ -344,15 +374,11 @@ $console->writeOut( "\n%s\n", pht('Failed to make some schema adjustments, detailed above.')); - - if (!$unsafe) { - $console->writeOut( - "%s\n", - pht( - 'Migrations which fail with certain types of errors (including '. - '"#1406 Data Too Long" and "#1366 Incorrect String Value") can be '. - 'forced to complete by running again with `--unsafe`.')); - } + $console->writeOut( + "%s\n", + pht( + 'For help troubleshooting adjustments, see "Managing Storage '. + 'Adjustments" in the documentation.')); return 1; }