diff --git a/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php b/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php index 4abd8fd388..e925fe3738 100644 --- a/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php +++ b/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php @@ -1,39 +1,39 @@ buildLiskSchemata('PhabricatorCacheDAO'); $this->buildRawSchema( 'cache', id(new PhabricatorKeyValueDatabaseCache())->getTableName(), array( - 'id' => 'id64', + 'id' => 'auto64', 'cacheKeyHash' => 'bytes12', 'cacheKey' => 'text128', 'cacheFormat' => 'text16', 'cacheData' => 'bytes', 'cacheCreated' => 'epoch', 'cacheExpires' => 'epoch?', ), array( 'PRIMARY' => array( 'columns' => array('id'), 'unique' => true, ), 'key_cacheKeyHash' => array( 'columns' => array('cacheKeyHash'), 'unique' => true, ), 'key_cacheCreated' => array( 'columns' => array('cacheCreated'), ), 'key_ttl' => array( 'columns' => array('cacheExpires'), ), )); } } diff --git a/src/applications/conduit/storage/PhabricatorConduitMethodCallLog.php b/src/applications/conduit/storage/PhabricatorConduitMethodCallLog.php index d9a4a57ffe..bf7a501c20 100644 --- a/src/applications/conduit/storage/PhabricatorConduitMethodCallLog.php +++ b/src/applications/conduit/storage/PhabricatorConduitMethodCallLog.php @@ -1,59 +1,59 @@ array( - 'id' => 'id64', + 'id' => 'auto64', 'connectionID' => 'id64?', 'method' => 'text64', 'error' => 'text255', 'duration' => 'uint64', 'callerPHID' => 'phid?', ), self::CONFIG_KEY_SCHEMA => array( 'key_date' => array( 'columns' => array('dateCreated'), ), 'key_method' => array( 'columns' => array('method'), ), 'key_callermethod' => array( 'columns' => array('callerPHID', 'method'), ), ), ) + parent::getConfiguration(); } /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, ); } public function getPolicy($capability) { return PhabricatorPolicies::POLICY_USER; } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { return false; } public function describeAutomaticCapability($capability) { return null; } } diff --git a/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php b/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php index 4bdb53554c..4d7ba636ff 100644 --- a/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php +++ b/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php @@ -1,738 +1,756 @@ database = idx($data, 'database'); $this->table = idx($data, 'table'); $this->column = idx($data, 'column'); $this->key = idx($data, 'key'); } public function processRequest() { $request = $this->getRequest(); $viewer = $request->getUser(); $query = $this->buildSchemaQuery(); $actual = $query->loadActualSchema(); $expect = $query->loadExpectedSchema(); $comp = $query->buildComparisonSchema($expect, $actual); if ($this->column) { return $this->renderColumn( $comp, $expect, $actual, $this->database, $this->table, $this->column); } else if ($this->key) { return $this->renderKey( $comp, $expect, $actual, $this->database, $this->table, $this->key); } else if ($this->table) { return $this->renderTable( $comp, $expect, $actual, $this->database, $this->table); } else if ($this->database) { return $this->renderDatabase( $comp, $expect, $actual, $this->database); } else { return $this->renderServer( $comp, $expect, $actual); } } private function buildResponse($title, $body) { $nav = $this->buildSideNavView(); $nav->selectFilter('database/'); $crumbs = $this->buildApplicationCrumbs(); if ($this->database) { $crumbs->addTextCrumb( pht('Database Status'), $this->getApplicationURI('database/')); if ($this->table) { $crumbs->addTextCrumb( $this->database, $this->getApplicationURI('database/'.$this->database.'/')); if ($this->column || $this->key) { $crumbs->addTextCrumb( $this->table, $this->getApplicationURI( 'database/'.$this->database.'/'.$this->table.'/')); if ($this->column) { $crumbs->addTextCrumb($this->column); } else { $crumbs->addTextCrumb($this->key); } } else { $crumbs->addTextCrumb($this->table); } } else { $crumbs->addTextCrumb($this->database); } } else { $crumbs->addTextCrumb(pht('Database Status')); } $nav->setCrumbs($crumbs); $nav->appendChild($body); return $this->buildApplicationPage( $nav, array( 'title' => $title, )); } private function renderServer( PhabricatorConfigServerSchema $comp, PhabricatorConfigServerSchema $expect, PhabricatorConfigServerSchema $actual) { $charset_issue = PhabricatorConfigStorageSchema::ISSUE_CHARSET; $collation_issue = PhabricatorConfigStorageSchema::ISSUE_COLLATION; $rows = array(); foreach ($comp->getDatabases() as $database_name => $database) { $actual_database = $actual->getDatabase($database_name); if ($actual_database) { $charset = $actual_database->getCharacterSet(); $collation = $actual_database->getCollation(); } else { $charset = null; $collation = null; } $status = $database->getStatus(); $issues = $database->getIssues(); $rows[] = array( $this->renderIcon($status), phutil_tag( 'a', array( 'href' => $this->getApplicationURI( '/database/'.$database_name.'/'), ), $database_name), $this->renderAttr($charset, $database->hasIssue($charset_issue)), $this->renderAttr($collation, $database->hasIssue($collation_issue)), ); } $table = id(new AphrontTableView($rows)) ->setHeaders( array( null, pht('Database'), pht('Charset'), pht('Collation'), )) ->setColumnClasses( array( null, 'wide pri', null, null, )); $title = pht('Database Status'); $properties = $this->buildProperties( array( ), $comp->getIssues()); $box = id(new PHUIObjectBoxView()) ->setHeaderText($title) ->addPropertyList($properties) ->appendChild($table); return $this->buildResponse($title, $box); } private function renderDatabase( PhabricatorConfigServerSchema $comp, PhabricatorConfigServerSchema $expect, PhabricatorConfigServerSchema $actual, $database_name) { $collation_issue = PhabricatorConfigStorageSchema::ISSUE_COLLATION; $database = $comp->getDatabase($database_name); if (!$database) { return new Aphront404Response(); } $rows = array(); foreach ($database->getTables() as $table_name => $table) { $status = $table->getStatus(); $rows[] = array( $this->renderIcon($status), phutil_tag( 'a', array( 'href' => $this->getApplicationURI( '/database/'.$database_name.'/'.$table_name.'/'), ), $table_name), $this->renderAttr( $table->getCollation(), $table->hasIssue($collation_issue)), ); } $table = id(new AphrontTableView($rows)) ->setHeaders( array( null, pht('Table'), pht('Collation'), )) ->setColumnClasses( array( null, 'wide pri', null, )); $title = pht('Database Status: %s', $database_name); $actual_database = $actual->getDatabase($database_name); if ($actual_database) { $actual_charset = $actual_database->getCharacterSet(); $actual_collation = $actual_database->getCollation(); } else { $actual_charset = null; $actual_collation = null; } $expect_database = $expect->getDatabase($database_name); if ($expect_database) { $expect_charset = $expect_database->getCharacterSet(); $expect_collation = $expect_database->getCollation(); } else { $expect_charset = null; $expect_collation = null; } $properties = $this->buildProperties( array( array( pht('Character Set'), $actual_charset, ), array( pht('Expected Character Set'), $expect_charset, ), array( pht('Collation'), $actual_collation, ), array( pht('Expected Collation'), $expect_collation, ), ), $database->getIssues()); $box = id(new PHUIObjectBoxView()) ->setHeaderText($title) ->addPropertyList($properties) ->appendChild($table); return $this->buildResponse($title, $box); } private function renderTable( PhabricatorConfigServerSchema $comp, PhabricatorConfigServerSchema $expect, PhabricatorConfigServerSchema $actual, $database_name, $table_name) { $type_issue = PhabricatorConfigStorageSchema::ISSUE_COLUMNTYPE; $charset_issue = PhabricatorConfigStorageSchema::ISSUE_CHARSET; $collation_issue = PhabricatorConfigStorageSchema::ISSUE_COLLATION; $nullable_issue = PhabricatorConfigStorageSchema::ISSUE_NULLABLE; $unique_issue = PhabricatorConfigStorageSchema::ISSUE_UNIQUE; $columns_issue = PhabricatorConfigStorageSchema::ISSUE_KEYCOLUMNS; $longkey_issue = PhabricatorConfigStorageSchema::ISSUE_LONGKEY; + $auto_issue = PhabricatorConfigStorageSchema::ISSUE_AUTOINCREMENT; $database = $comp->getDatabase($database_name); if (!$database) { return new Aphront404Response(); } $table = $database->getTable($table_name); if (!$table) { return new Aphront404Response(); } $actual_database = $actual->getDatabase($database_name); $actual_table = null; if ($actual_database) { $actual_table = $actual_database->getTable($table_name); } $expect_database = $expect->getDatabase($database_name); $expect_table = null; if ($expect_database) { $expect_table = $expect_database->getTable($table_name); } $rows = array(); foreach ($table->getColumns() as $column_name => $column) { $expect_column = null; if ($expect_table) { $expect_column = $expect_table->getColumn($column_name); } $status = $column->getStatus(); $data_type = null; if ($expect_column) { $data_type = $expect_column->getDataType(); } $rows[] = array( $this->renderIcon($status), phutil_tag( 'a', array( 'href' => $this->getApplicationURI( 'database/'. $database_name.'/'. $table_name.'/'. 'col/'. $column_name.'/'), ), $column_name), $data_type, $this->renderAttr( $column->getColumnType(), $column->hasIssue($type_issue)), $this->renderAttr( $this->renderBoolean($column->getNullable()), $column->hasIssue($nullable_issue)), + $this->renderAttr( + $this->renderBoolean($column->getAutoIncrement()), + $column->hasIssue($auto_issue)), $this->renderAttr( $column->getCharacterSet(), $column->hasIssue($charset_issue)), $this->renderAttr( $column->getCollation(), $column->hasIssue($collation_issue)), ); } $table_view = id(new AphrontTableView($rows)) ->setHeaders( array( null, pht('Column'), pht('Data Type'), pht('Column Type'), pht('Nullable'), + pht('Autoincrement'), pht('Character Set'), pht('Collation'), )) ->setColumnClasses( array( null, 'wide pri', null, null, null, + null, null )); $key_rows = array(); foreach ($table->getKeys() as $key_name => $key) { $expect_key = null; if ($expect_table) { $expect_key = $expect_table->getKey($key_name); } $status = $key->getStatus(); $size = 0; foreach ($key->getColumnNames() as $column_spec) { list($column_name, $prefix) = $key->getKeyColumnAndPrefix($column_spec); $column = $table->getColumn($column_name); if (!$column) { $size = 0; break; } $size += $column->getKeyByteLength($prefix); } $size_formatted = null; if ($size) { $size_formatted = $this->renderAttr( $size, $key->hasIssue($longkey_issue)); } $key_rows[] = array( $this->renderIcon($status), phutil_tag( 'a', array( 'href' => $this->getApplicationURI( 'database/'. $database_name.'/'. $table_name.'/'. 'key/'. $key_name.'/'), ), $key_name), $this->renderAttr( implode(', ', $key->getColumnNames()), $key->hasIssue($columns_issue)), $this->renderAttr( $this->renderBoolean($key->getUnique()), $key->hasIssue($unique_issue)), $size_formatted, ); } $keys_view = id(new AphrontTableView($key_rows)) ->setHeaders( array( null, pht('Key'), pht('Columns'), pht('Unique'), pht('Size'), )) ->setColumnClasses( array( null, 'wide pri', null, null, null, )); $title = pht('Database Status: %s.%s', $database_name, $table_name); if ($actual_table) { $actual_collation = $actual_table->getCollation(); } else { $actual_collation = null; } if ($expect_table) { $expect_collation = $expect_table->getCollation(); } else { $expect_collation = null; } $properties = $this->buildProperties( array( array( pht('Collation'), $actual_collation, ), array( pht('Expected Collation'), $expect_collation, ), ), $table->getIssues()); $box = id(new PHUIObjectBoxView()) ->setHeaderText($title) ->addPropertyList($properties) ->appendChild($table_view) ->appendChild($keys_view); return $this->buildResponse($title, $box); } private function renderColumn( PhabricatorConfigServerSchema $comp, PhabricatorConfigServerSchema $expect, PhabricatorConfigServerSchema $actual, $database_name, $table_name, $column_name) { $database = $comp->getDatabase($database_name); if (!$database) { return new Aphront404Response(); } $table = $database->getTable($table_name); if (!$table) { return new Aphront404Response(); } $column = $table->getColumn($column_name); if (!$column) { return new Aphront404Response(); } $actual_database = $actual->getDatabase($database_name); $actual_table = null; $actual_column = null; if ($actual_database) { $actual_table = $actual_database->getTable($table_name); if ($actual_table) { $actual_column = $actual_table->getColumn($column_name); } } $expect_database = $expect->getDatabase($database_name); $expect_table = null; $expect_column = null; if ($expect_database) { $expect_table = $expect_database->getTable($table_name); if ($expect_table) { $expect_column = $expect_table->getColumn($column_name); } } if ($actual_column) { $actual_coltype = $actual_column->getColumnType(); $actual_charset = $actual_column->getCharacterSet(); $actual_collation = $actual_column->getCollation(); $actual_nullable = $actual_column->getNullable(); + $actual_auto = $actual_column->getAutoIncrement(); } else { $actual_coltype = null; $actual_charset = null; $actual_collation = null; $actual_nullable = null; + $actual_auto = null; } if ($expect_column) { $data_type = $expect_column->getDataType(); $expect_coltype = $expect_column->getColumnType(); $expect_charset = $expect_column->getCharacterSet(); $expect_collation = $expect_column->getCollation(); $expect_nullable = $expect_column->getNullable(); + $expect_auto = $expect_column->getAutoIncrement(); } else { $data_type = null; $expect_coltype = null; $expect_charset = null; $expect_collation = null; $expect_nullable = null; + $expect_auto = null; } $title = pht( 'Database Status: %s.%s.%s', $database_name, $table_name, $column_name); $properties = $this->buildProperties( array( array( pht('Data Type'), $data_type, ), array( pht('Column Type'), $actual_coltype, ), array( pht('Expected Column Type'), $expect_coltype, ), array( pht('Character Set'), $actual_charset, ), array( pht('Expected Character Set'), $expect_charset, ), array( pht('Collation'), $actual_collation, ), array( pht('Expected Collation'), $expect_collation, ), array( pht('Nullable'), $this->renderBoolean($actual_nullable), ), array( pht('Expected Nullable'), $this->renderBoolean($expect_nullable), ), + array( + pht('Autoincrement'), + $this->renderBoolean($actual_auto), + ), + array( + pht('Expected Autoincrement'), + $this->renderBoolean($expect_auto), + ), ), $column->getIssues()); $box = id(new PHUIObjectBoxView()) ->setHeaderText($title) ->addPropertyList($properties); return $this->buildResponse($title, $box); } private function renderKey( PhabricatorConfigServerSchema $comp, PhabricatorConfigServerSchema $expect, PhabricatorConfigServerSchema $actual, $database_name, $table_name, $key_name) { $database = $comp->getDatabase($database_name); if (!$database) { return new Aphront404Response(); } $table = $database->getTable($table_name); if (!$table) { return new Aphront404Response(); } $key = $table->getKey($key_name); if (!$key) { return new Aphront404Response(); } $actual_database = $actual->getDatabase($database_name); $actual_table = null; $actual_key = null; if ($actual_database) { $actual_table = $actual_database->getTable($table_name); if ($actual_table) { $actual_key = $actual_table->getKey($key_name); } } $expect_database = $expect->getDatabase($database_name); $expect_table = null; $expect_key = null; if ($expect_database) { $expect_table = $expect_database->getTable($table_name); if ($expect_table) { $expect_key = $expect_table->getKey($key_name); } } if ($actual_key) { $actual_columns = $actual_key->getColumnNames(); $actual_unique = $actual_key->getUnique(); } else { $actual_columns = array(); $actual_unique = null; } if ($expect_key) { $expect_columns = $expect_key->getColumnNames(); $expect_unique = $expect_key->getUnique(); } else { $expect_columns = array(); $expect_unique = null; } $title = pht( 'Database Status: %s.%s (%s)', $database_name, $table_name, $key_name); $properties = $this->buildProperties( array( array( pht('Unique'), $this->renderBoolean($actual_unique), ), array( pht('Expected Unique'), $this->renderBoolean($expect_unique), ), array( pht('Columns'), implode(', ', $actual_columns), ), array( pht('Expected Columns'), implode(', ', $expect_columns), ), ), $key->getIssues()); $box = id(new PHUIObjectBoxView()) ->setHeaderText($title) ->addPropertyList($properties); return $this->buildResponse($title, $box); } private function buildProperties(array $properties, array $issues) { $view = id(new PHUIPropertyListView()) ->setUser($this->getRequest()->getUser()); foreach ($properties as $property) { list($key, $value) = $property; $view->addProperty($key, $value); } $status_view = new PHUIStatusListView(); if (!$issues) { $status_view->addItem( id(new PHUIStatusItemView()) ->setIcon(PHUIStatusItemView::ICON_ACCEPT, 'green') ->setTarget(pht('No Schema Issues'))); } else { foreach ($issues as $issue) { $note = PhabricatorConfigStorageSchema::getIssueDescription($issue); $status = PhabricatorConfigStorageSchema::getIssueStatus($issue); switch ($status) { case PhabricatorConfigStorageSchema::STATUS_WARN: $icon = PHUIStatusItemView::ICON_WARNING; $color = 'yellow'; break; case PhabricatorConfigStorageSchema::STATUS_FAIL: default: $icon = PHUIStatusItemView::ICON_REJECT; $color = 'red'; break; } $item = id(new PHUIStatusItemView()) ->setTarget(PhabricatorConfigStorageSchema::getIssueName($issue)) ->setIcon($icon, $color) ->setNote($note); $status_view->addItem($item); } } $view->addProperty(pht('Schema Status'), $status_view); return $view; } } diff --git a/src/applications/config/schema/PhabricatorConfigColumnSchema.php b/src/applications/config/schema/PhabricatorConfigColumnSchema.php index ebb6f41707..3aa5e07a4c 100644 --- a/src/applications/config/schema/PhabricatorConfigColumnSchema.php +++ b/src/applications/config/schema/PhabricatorConfigColumnSchema.php @@ -1,142 +1,156 @@ autoIncrement = $auto_increment; + return $this; + } + + public function getAutoIncrement() { + return $this->autoIncrement; + } public function setNullable($nullable) { $this->nullable = $nullable; return $this; } public function getNullable() { return $this->nullable; } public function setColumnType($column_type) { $this->columnType = $column_type; return $this; } public function getColumnType() { return $this->columnType; } protected function getSubschemata() { return array(); } public function setDataType($data_type) { $this->dataType = $data_type; return $this; } public function getDataType() { return $this->dataType; } public function setCollation($collation) { $this->collation = $collation; return $this; } public function getCollation() { return $this->collation; } public function setCharacterSet($character_set) { $this->characterSet = $character_set; return $this; } public function getCharacterSet() { return $this->characterSet; } public function getKeyByteLength($prefix = null) { $type = $this->getColumnType(); $matches = null; if (preg_match('/^(?:var)?char\((\d+)\)$/', $type, $matches)) { // For utf8mb4, each character requires 4 bytes. $size = (int)$matches[1]; if ($prefix && $prefix < $size) { $size = $prefix; } return $size * 4; } $matches = null; if (preg_match('/^(?:var)?binary\((\d+)\)$/', $type, $matches)) { // binary()/varbinary() store fixed-length binary data, so their size // is always the column size. $size = (int)$matches[1]; if ($prefix && $prefix < $size) { $size = $prefix; } return $size; } // The "long..." types are arbitrarily long, so just use a big number to // get the point across. In practice, these should always index only a // prefix. if ($type == 'longtext') { $size = (1 << 16); if ($prefix && $prefix < $size) { $size = $prefix; } return $size * 4; } if ($type == 'longblob') { $size = (1 << 16); if ($prefix && $prefix < $size) { $size = $prefix; } return $size * 1; } switch ($type) { case 'int(10) unsigned': return 4; } // TODO: Build this out to catch overlong indexes. return 0; } public function compareToSimilarSchema( PhabricatorConfigStorageSchema $expect) { $issues = array(); if ($this->getCharacterSet() != $expect->getCharacterSet()) { $issues[] = self::ISSUE_CHARSET; } if ($this->getCollation() != $expect->getCollation()) { $issues[] = self::ISSUE_COLLATION; } if ($this->getColumnType() != $expect->getColumnType()) { $issues[] = self::ISSUE_COLUMNTYPE; } if ($this->getNullable() !== $expect->getNullable()) { $issues[] = self::ISSUE_NULLABLE; } + if ($this->getAutoIncrement() !== $expect->getAutoIncrement()) { + $issues[] = self::ISSUE_AUTOINCREMENT; + } + return $issues; } public function newEmptyClone() { $clone = clone $this; return $clone; } } diff --git a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php index b99f40637f..60292b4a74 100644 --- a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php +++ b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php @@ -1,281 +1,288 @@ api = $api; return $this; } protected function getAPI() { if (!$this->api) { throw new Exception(pht('Call setAPI() before issuing a query!')); } return $this->api; } protected function getConn() { return $this->getAPI()->getConn(null); } private function getDatabaseNames() { $api = $this->getAPI(); $patches = PhabricatorSQLPatchList::buildAllPatches(); return $api->getDatabaseList( $patches, $only_living = true); } public function loadActualSchema() { $databases = $this->getDatabaseNames(); $conn = $this->getConn(); $tables = queryfx_all( $conn, 'SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_COLLATION FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA IN (%Ls)', $databases); $database_info = queryfx_all( $conn, 'SELECT SCHEMA_NAME, DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME IN (%Ls)', $databases); $database_info = ipull($database_info, null, 'SCHEMA_NAME'); $sql = array(); foreach ($tables as $table) { $sql[] = qsprintf( $conn, '(TABLE_SCHEMA = %s AND TABLE_NAME = %s)', $table['TABLE_SCHEMA'], $table['TABLE_NAME']); } if ($sql) { $column_info = queryfx_all( $conn, 'SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, CHARACTER_SET_NAME, - COLLATION_NAME, COLUMN_TYPE, IS_NULLABLE + COLLATION_NAME, COLUMN_TYPE, IS_NULLABLE, EXTRA FROM INFORMATION_SCHEMA.COLUMNS WHERE (%Q)', '('.implode(') OR (', $sql).')'); $column_info = igroup($column_info, 'TABLE_SCHEMA'); } else { $column_info = array(); } // NOTE: Tables like KEY_COLUMN_USAGE and TABLE_CONSTRAINTS only contain // primary, unique, and foreign keys, so we can't use them here. We pull // indexes later on using SHOW INDEXES. $server_schema = new PhabricatorConfigServerSchema(); $tables = igroup($tables, 'TABLE_SCHEMA'); foreach ($tables as $database_name => $database_tables) { $info = $database_info[$database_name]; $database_schema = id(new PhabricatorConfigDatabaseSchema()) ->setName($database_name) ->setCharacterSet($info['DEFAULT_CHARACTER_SET_NAME']) ->setCollation($info['DEFAULT_COLLATION_NAME']); $database_column_info = idx($column_info, $database_name, array()); $database_column_info = igroup($database_column_info, 'TABLE_NAME'); foreach ($database_tables as $table) { $table_name = $table['TABLE_NAME']; $table_schema = id(new PhabricatorConfigTableSchema()) ->setName($table_name) ->setCollation($table['TABLE_COLLATION']); $columns = idx($database_column_info, $table_name, array()); foreach ($columns as $column) { + if (strpos($column['EXTRA'], 'auto_increment') === false) { + $auto_increment = false; + } else { + $auto_increment = true; + } + $column_schema = id(new PhabricatorConfigColumnSchema()) ->setName($column['COLUMN_NAME']) ->setCharacterSet($column['CHARACTER_SET_NAME']) ->setCollation($column['COLLATION_NAME']) ->setColumnType($column['COLUMN_TYPE']) - ->setNullable($column['IS_NULLABLE'] == 'YES'); + ->setNullable($column['IS_NULLABLE'] == 'YES') + ->setAutoIncrement($auto_increment); $table_schema->addColumn($column_schema); } $key_parts = queryfx_all( $conn, 'SHOW INDEXES FROM %T.%T', $database_name, $table_name); $keys = igroup($key_parts, 'Key_name'); foreach ($keys as $key_name => $key_pieces) { $key_pieces = isort($key_pieces, 'Seq_in_index'); $head = head($key_pieces); // This handles string indexes which index only a prefix of a field. $column_names = array(); foreach ($key_pieces as $piece) { $name = $piece['Column_name']; if ($piece['Sub_part']) { $name = $name.'('.$piece['Sub_part'].')'; } $column_names[] = $name; } $key_schema = id(new PhabricatorConfigKeySchema()) ->setName($key_name) ->setColumnNames($column_names) ->setUnique(!$head['Non_unique']) ->setIndexType($head['Index_type']); $table_schema->addKey($key_schema); } $database_schema->addTable($table_schema); } $server_schema->addDatabase($database_schema); } return $server_schema; } public function loadExpectedSchema() { $databases = $this->getDatabaseNames(); $api = $this->getAPI(); $charset_info = $api->getCharsetInfo(); list($charset, $collate_text, $collate_sort) = $charset_info; $specs = id(new PhutilSymbolLoader()) ->setAncestorClass('PhabricatorConfigSchemaSpec') ->loadObjects(); $server_schema = new PhabricatorConfigServerSchema(); foreach ($specs as $spec) { $spec ->setUTF8Charset($charset) ->setUTF8BinaryCollation($collate_text) ->setUTF8SortingCollation($collate_sort) ->setServer($server_schema) ->buildSchemata($server_schema); } return $server_schema; } public function buildComparisonSchema( PhabricatorConfigServerSchema $expect, PhabricatorConfigServerSchema $actual) { $comp_server = $actual->newEmptyClone(); $all_databases = $actual->getDatabases() + $expect->getDatabases(); foreach ($all_databases as $database_name => $database_template) { $actual_database = $actual->getDatabase($database_name); $expect_database = $expect->getDatabase($database_name); $issues = $this->compareSchemata($expect_database, $actual_database); $comp_database = $database_template->newEmptyClone() ->setIssues($issues); if (!$actual_database) { $actual_database = $expect_database->newEmptyClone(); } if (!$expect_database) { $expect_database = $actual_database->newEmptyClone(); } $all_tables = $actual_database->getTables() + $expect_database->getTables(); foreach ($all_tables as $table_name => $table_template) { $actual_table = $actual_database->getTable($table_name); $expect_table = $expect_database->getTable($table_name); $issues = $this->compareSchemata($expect_table, $actual_table); $comp_table = $table_template->newEmptyClone() ->setIssues($issues); if (!$actual_table) { $actual_table = $expect_table->newEmptyClone(); } if (!$expect_table) { $expect_table = $actual_table->newEmptyClone(); } $all_columns = $actual_table->getColumns() + $expect_table->getColumns(); foreach ($all_columns as $column_name => $column_template) { $actual_column = $actual_table->getColumn($column_name); $expect_column = $expect_table->getColumn($column_name); $issues = $this->compareSchemata($expect_column, $actual_column); $comp_column = $column_template->newEmptyClone() ->setIssues($issues); $comp_table->addColumn($comp_column); } $all_keys = $actual_table->getKeys() + $expect_table->getKeys(); foreach ($all_keys as $key_name => $key_template) { $actual_key = $actual_table->getKey($key_name); $expect_key = $expect_table->getKey($key_name); $issues = $this->compareSchemata($expect_key, $actual_key); $comp_key = $key_template->newEmptyClone() ->setIssues($issues); $comp_table->addKey($comp_key); } $comp_database->addTable($comp_table); } $comp_server->addDatabase($comp_database); } return $comp_server; } private function compareSchemata( PhabricatorConfigStorageSchema $expect = null, PhabricatorConfigStorageSchema $actual = null) { $expect_is_key = ($expect instanceof PhabricatorConfigKeySchema); $actual_is_key = ($actual instanceof PhabricatorConfigKeySchema); if ($expect_is_key || $actual_is_key) { $missing_issue = PhabricatorConfigStorageSchema::ISSUE_MISSINGKEY; $surplus_issue = PhabricatorConfigStorageSchema::ISSUE_SURPLUSKEY; } else { $missing_issue = PhabricatorConfigStorageSchema::ISSUE_MISSING; $surplus_issue = PhabricatorConfigStorageSchema::ISSUE_SURPLUS; } if (!$expect && !$actual) { throw new Exception(pht('Can not compare two missing schemata!')); } else if ($expect && !$actual) { $issues = array($missing_issue); } else if ($actual && !$expect) { $issues = array($surplus_issue); } else { $issues = $actual->compareTo($expect); } return $issues; } } diff --git a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php index cd68a97c4a..6cdde09544 100644 --- a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php +++ b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php @@ -1,398 +1,408 @@ utf8SortingCollation = $utf8_sorting_collation; return $this; } public function getUTF8SortingCollation() { return $this->utf8SortingCollation; } public function setUTF8BinaryCollation($utf8_binary_collation) { $this->utf8BinaryCollation = $utf8_binary_collation; return $this; } public function getUTF8BinaryCollation() { return $this->utf8BinaryCollation; } public function setUTF8Charset($utf8_charset) { $this->utf8Charset = $utf8_charset; return $this; } public function getUTF8Charset() { return $this->utf8Charset; } public function setServer(PhabricatorConfigServerSchema $server) { $this->server = $server; return $this; } public function getServer() { return $this->server; } abstract public function buildSchemata(); protected function buildLiskSchemata($base) { $objects = id(new PhutilSymbolLoader()) ->setAncestorClass($base) ->loadObjects(); foreach ($objects as $object) { if ($object->getConfigOption(LiskDAO::CONFIG_NO_TABLE)) { continue; } $this->buildLiskObjectSchema($object); } } protected function buildTransactionSchema( PhabricatorApplicationTransaction $xaction, PhabricatorApplicationTransactionComment $comment = null) { $this->buildLiskObjectSchema($xaction); if ($comment) { $this->buildLiskObjectSchema($comment); } } protected function buildCustomFieldSchemata( PhabricatorLiskDAO $storage, array $indexes) { $this->buildLiskObjectSchema($storage); foreach ($indexes as $index) { $this->buildLiskObjectSchema($index); } } private function buildLiskObjectSchema(PhabricatorLiskDAO $object) { $this->buildRawSchema( $object->getApplicationName(), $object->getTableName(), $object->getSchemaColumns(), $object->getSchemaKeys()); } protected function buildRawSchema( $database_name, $table_name, array $columns, array $keys) { $database = $this->getDatabase($database_name); $table = $this->newTable($table_name); foreach ($columns as $name => $type) { if ($type === null) { continue; } $details = $this->getDetailsForDataType($type); - list($column_type, $charset, $collation, $nullable) = $details; + list($column_type, $charset, $collation, $nullable, $auto) = $details; $column = $this->newColumn($name) ->setDataType($type) ->setColumnType($column_type) ->setCharacterSet($charset) ->setCollation($collation) - ->setNullable($nullable); + ->setNullable($nullable) + ->setAutoIncrement($auto); $table->addColumn($column); } foreach ($keys as $key_name => $key_spec) { if ($key_spec === null) { // This is a subclass removing a key which Lisk expects. continue; } $key = $this->newKey($key_name) ->setColumnNames(idx($key_spec, 'columns', array())); $key->setUnique((bool)idx($key_spec, 'unique')); $key->setIndexType(idx($key_spec, 'type', 'BTREE')); $table->addKey($key); } $database->addTable($table); } protected function buildEdgeSchemata(PhabricatorLiskDAO $object) { $this->buildRawSchema( $object->getApplicationName(), PhabricatorEdgeConfig::TABLE_NAME_EDGE, array( 'src' => 'phid', 'type' => 'uint32', 'dst' => 'phid', 'dateCreated' => 'epoch', 'seq' => 'uint32', 'dataID' => 'id?', ), array( 'PRIMARY' => array( 'columns' => array('src', 'type', 'dst'), 'unique' => true, ), 'src' => array( 'columns' => array('src', 'type', 'dateCreated', 'seq'), ), 'key_dst' => array( 'columns' => array('dst', 'type', 'src'), 'unique' => true, ), )); $this->buildRawSchema( $object->getApplicationName(), PhabricatorEdgeConfig::TABLE_NAME_EDGEDATA, array( - 'id' => 'id', + 'id' => 'auto', 'data' => 'text', ), array( 'PRIMARY' => array( 'columns' => array('id'), 'unique' => true, ), )); } public function buildCounterSchema(PhabricatorLiskDAO $object) { $this->buildRawSchema( $object->getApplicationName(), PhabricatorLiskDAO::COUNTER_TABLE_NAME, array( 'counterName' => 'text32', 'counterValue' => 'id64', ), array( 'PRIMARY' => array( 'columns' => array('counterName'), 'unique' => true, ), )); } protected function getDatabase($name) { $server = $this->getServer(); $database = $server->getDatabase($this->getNamespacedDatabase($name)); if (!$database) { $database = $this->newDatabase($name); $server->addDatabase($database); } return $database; } protected function newDatabase($name) { return id(new PhabricatorConfigDatabaseSchema()) ->setName($this->getNamespacedDatabase($name)) ->setCharacterSet($this->getUTF8Charset()) ->setCollation($this->getUTF8BinaryCollation()); } protected function getNamespacedDatabase($name) { $namespace = PhabricatorLiskDAO::getStorageNamespace(); return $namespace.'_'.$name; } protected function newTable($name) { return id(new PhabricatorConfigTableSchema()) ->setName($name) ->setCollation($this->getUTF8BinaryCollation()); } protected function newColumn($name) { return id(new PhabricatorConfigColumnSchema()) ->setName($name); } protected function newKey($name) { return id(new PhabricatorConfigKeySchema()) ->setName($name); } private function getDetailsForDataType($data_type) { $column_type = null; $charset = null; $collation = null; + $auto = false; // If the type ends with "?", make the column nullable. $nullable = false; if (preg_match('/\?$/', $data_type)) { $nullable = true; $data_type = substr($data_type, 0, -1); } // NOTE: MySQL allows fragments like "VARCHAR(32) CHARACTER SET binary", // but just interprets that to mean "VARBINARY(32)". The fragment is // totally disallowed in a MODIFY statement vs a CREATE TABLE statement. switch ($data_type) { + case 'auto': + $column_type = 'int(10) unsigned'; + $auto = true; + break; + case 'auto64': + $column_type = 'bigint(20) unsigned'; + $auto = true; + break; case 'id': case 'epoch': case 'uint32': $column_type = 'int(10) unsigned'; break; case 'sint32': $column_type = 'int(10)'; break; case 'id64': case 'uint64': $column_type = 'bigint(20) unsigned'; break; case 'sint64': $column_type = 'bigint(20)'; break; case 'phid': case 'policy'; $column_type = 'varbinary(64)'; break; case 'bytes64': $column_type = 'binary(64)'; break; case 'bytes40': $column_type = 'binary(40)'; break; case 'bytes32': $column_type = 'binary(32)'; break; case 'bytes20': $column_type = 'binary(20)'; break; case 'bytes12': $column_type = 'binary(12)'; break; case 'bytes4': $column_type = 'binary(4)'; break; case 'bytes': $column_type = 'longblob'; break; case 'sort255': $column_type = 'varchar(255)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8SortingCollation(); break; case 'sort128': $column_type = 'varchar(128)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8SortingCollation(); break; case 'sort64': $column_type = 'varchar(64)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8SortingCollation(); break; case 'sort32': $column_type = 'varchar(32)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8SortingCollation(); break; case 'sort': $column_type = 'longtext'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8SortingCollation(); break; case 'text255': $column_type = 'varchar(255)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8BinaryCollation(); break; case 'text160': $column_type = 'varchar(160)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8BinaryCollation(); break; case 'text128': $column_type = 'varchar(128)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8BinaryCollation(); break; case 'text80': $column_type = 'varchar(80)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8BinaryCollation(); break; case 'text64': $column_type = 'varchar(64)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8BinaryCollation(); break; case 'text40': $column_type = 'varchar(40)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8BinaryCollation(); break; case 'text32': $column_type = 'varchar(32)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8BinaryCollation(); break; case 'text20': $column_type = 'varchar(20)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8BinaryCollation(); break; case 'text16': $column_type = 'varchar(16)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8BinaryCollation(); break; case 'text12': $column_type = 'varchar(12)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8BinaryCollation(); break; case 'text8': $column_type = 'varchar(8)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8BinaryCollation(); break; case 'text4': $column_type = 'varchar(4)'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8BinaryCollation(); break; case 'text': $column_type = 'longtext'; $charset = $this->getUTF8Charset(); $collation = $this->getUTF8BinaryCollation(); break; case 'bool': $column_type = 'tinyint(1)'; break; case 'double': $column_type = 'double'; break; case 'date': $column_type = 'date'; break; default: $column_type = pht(''); $charset = pht(''); $collation = pht(''); break; } - return array($column_type, $charset, $collation, $nullable); + return array($column_type, $charset, $collation, $nullable, $auto); } } diff --git a/src/applications/config/schema/PhabricatorConfigStorageSchema.php b/src/applications/config/schema/PhabricatorConfigStorageSchema.php index 849a4157e8..96d554629a 100644 --- a/src/applications/config/schema/PhabricatorConfigStorageSchema.php +++ b/src/applications/config/schema/PhabricatorConfigStorageSchema.php @@ -1,212 +1,218 @@ getName() != $expect->getName()) { throw new Exception(pht('Names must match to compare schemata!')); } return $this->compareToSimilarSchema($expect); } public function setName($name) { $this->name = $name; return $this; } public function getName() { return $this->name; } public function setIssues(array $issues) { $this->issues = array_fuse($issues); return $this; } public function getIssues() { $issues = $this->issues; foreach ($this->getSubschemata() as $sub) { switch ($sub->getStatus()) { case self::STATUS_WARN: $issues[self::ISSUE_SUBWARN] = self::ISSUE_SUBWARN; break; case self::STATUS_FAIL: $issues[self::ISSUE_SUBFAIL] = self::ISSUE_SUBFAIL; break; } } return $issues; } public function getLocalIssues() { return $this->issues; } public function hasIssue($issue) { return (bool)idx($this->getIssues(), $issue); } public function getAllIssues() { $issues = $this->getIssues(); foreach ($this->getSubschemata() as $sub) { $issues += $sub->getAllIssues(); } return $issues; } public function getStatus() { $status = self::STATUS_OKAY; foreach ($this->getAllIssues() as $issue) { $issue_status = self::getIssueStatus($issue); $status = self::getStrongestStatus($status, $issue_status); } return $status; } public static function getIssueName($issue) { switch ($issue) { case self::ISSUE_MISSING: return pht('Missing'); case self::ISSUE_MISSINGKEY: return pht('Missing Key'); case self::ISSUE_SURPLUS: return pht('Surplus'); case self::ISSUE_SURPLUSKEY: return pht('Surplus Key'); case self::ISSUE_CHARSET: return pht('Better Character Set Available'); case self::ISSUE_COLLATION: return pht('Better Collation Available'); case self::ISSUE_COLUMNTYPE: return pht('Wrong Column Type'); case self::ISSUE_NULLABLE: return pht('Wrong Nullable Setting'); case self::ISSUE_KEYCOLUMNS: return pht('Key on Wrong Columns'); case self::ISSUE_UNIQUE: return pht('Key has Wrong Uniqueness'); case self::ISSUE_LONGKEY: return pht('Key is Too Long'); case self::ISSUE_SUBWARN: return pht('Subschemata Have Warnings'); case self::ISSUE_SUBFAIL: return pht('Subschemata Have Failures'); + case self::ISSUE_AUTOINCREMENT: + return pht('Column has Wrong Autoincrement'); default: throw new Exception(pht('Unknown schema issue "%s"!', $issue)); } } public static function getIssueDescription($issue) { switch ($issue) { case self::ISSUE_MISSING: return pht('This schema is expected to exist, but does not.'); case self::ISSUE_MISSINGKEY: return pht('This key is expected to exist, but does not.'); case self::ISSUE_SURPLUS: return pht('This schema is not expected to exist.'); case self::ISSUE_SURPLUSKEY: return pht('This key is not expected to exist.'); case self::ISSUE_CHARSET: return pht('This schema can use a better character set.'); case self::ISSUE_COLLATION: return pht('This schema can use a better collation.'); case self::ISSUE_COLUMNTYPE: return pht('This schema can use a better column type.'); case self::ISSUE_NULLABLE: return pht('This schema has the wrong nullable setting.'); case self::ISSUE_KEYCOLUMNS: return pht('This key is on the wrong columns.'); case self::ISSUE_UNIQUE: return pht('This key has the wrong uniqueness setting.'); case self::ISSUE_LONGKEY: return pht('This key is too long for utf8mb4.'); case self::ISSUE_SUBWARN: return pht('Subschemata have setup warnings.'); case self::ISSUE_SUBFAIL: return pht('Subschemata have setup failures.'); + case self::ISSUE_AUTOINCREMENT: + return pht('This column has the wrong autoincrement setting.'); default: throw new Exception(pht('Unknown schema issue "%s"!', $issue)); } } public static function getIssueStatus($issue) { switch ($issue) { case self::ISSUE_MISSING: case self::ISSUE_SURPLUS: case self::ISSUE_NULLABLE: case self::ISSUE_SUBFAIL: return self::STATUS_FAIL; case self::ISSUE_SUBWARN: case self::ISSUE_COLUMNTYPE: case self::ISSUE_CHARSET: case self::ISSUE_COLLATION: case self::ISSUE_MISSINGKEY: case self::ISSUE_SURPLUSKEY: case self::ISSUE_UNIQUE: case self::ISSUE_KEYCOLUMNS: case self::ISSUE_LONGKEY: + case self::ISSUE_AUTOINCREMENT: return self::STATUS_WARN; default: throw new Exception(pht('Unknown schema issue "%s"!', $issue)); } } public static function getStatusSeverity($status) { switch ($status) { case self::STATUS_FAIL: return 2; case self::STATUS_WARN: return 1; case self::STATUS_OKAY: return 0; default: throw new Exception(pht('Unknown schema status "%s"!', $status)); } } public static function getStrongestStatus($u, $v) { $u_sev = self::getStatusSeverity($u); $v_sev = self::getStatusSeverity($v); if ($u_sev >= $v_sev) { return $u; } else { return $v; } } } diff --git a/src/applications/fact/storage/PhabricatorFactAggregate.php b/src/applications/fact/storage/PhabricatorFactAggregate.php index d6f2219939..2d0fe52872 100644 --- a/src/applications/fact/storage/PhabricatorFactAggregate.php +++ b/src/applications/fact/storage/PhabricatorFactAggregate.php @@ -1,25 +1,25 @@ array( - 'id' => 'id64', + 'id' => 'auto64', 'factType' => 'text32', 'valueX' => 'uint64', ), self::CONFIG_KEY_SCHEMA => array( 'factType' => array( 'columns' => array('factType', 'objectPHID'), 'unique' => true, ), ), ) + parent::getConfiguration(); } } diff --git a/src/applications/fact/storage/PhabricatorFactRaw.php b/src/applications/fact/storage/PhabricatorFactRaw.php index d4684b61db..5de2be7aaa 100644 --- a/src/applications/fact/storage/PhabricatorFactRaw.php +++ b/src/applications/fact/storage/PhabricatorFactRaw.php @@ -1,39 +1,39 @@ array( - 'id' => 'id64', + 'id' => 'auto64', 'factType' => 'text32', 'objectA' => 'phid', 'valueX' => 'sint64', 'valueY' => 'sint64', ), self::CONFIG_KEY_SCHEMA => array( 'objectPHID' => array( 'columns' => array('objectPHID'), ), 'factType' => array( 'columns' => array('factType', 'epoch'), ), 'factType_2' => array( 'columns' => array('factType', 'objectA'), ), ), ) + parent::getConfiguration(); } } diff --git a/src/applications/harbormaster/storage/HarbormasterSchemaSpec.php b/src/applications/harbormaster/storage/HarbormasterSchemaSpec.php index da3ab69d14..663b9e53ce 100644 --- a/src/applications/harbormaster/storage/HarbormasterSchemaSpec.php +++ b/src/applications/harbormaster/storage/HarbormasterSchemaSpec.php @@ -1,49 +1,49 @@ buildLiskSchemata('HarbormasterDAO'); $this->buildEdgeSchemata(new HarbormasterBuildable()); $this->buildCounterSchema(new HarbormasterBuildable()); $this->buildTransactionSchema( new HarbormasterBuildableTransaction()); $this->buildTransactionSchema( new HarbormasterBuildTransaction()); $this->buildTransactionSchema( new HarbormasterBuildPlanTransaction()); $this->buildTransactionSchema( new HarbormasterBuildStepTransaction()); $this->buildRawSchema( id(new HarbormasterBuildable())->getApplicationName(), 'harbormaster_buildlogchunk', array( - 'id' => 'id', + 'id' => 'auto', 'logID' => 'id', 'encoding' => 'text32', // T6203/NULLABILITY // Both the type and nullability of this column are crazily wrong. 'size' => 'uint32?', 'chunk' => 'bytes', ), array( 'PRIMARY' => array( 'columns' => array('id'), 'unique' => true, ), 'key_log' => array( 'columns' => array('logID'), ), )); } } diff --git a/src/applications/project/storage/PhabricatorProjectSchemaSpec.php b/src/applications/project/storage/PhabricatorProjectSchemaSpec.php index 0abd429f5e..536ac504f6 100644 --- a/src/applications/project/storage/PhabricatorProjectSchemaSpec.php +++ b/src/applications/project/storage/PhabricatorProjectSchemaSpec.php @@ -1,48 +1,48 @@ buildLiskSchemata('PhabricatorProjectDAO'); $this->buildEdgeSchemata(new PhabricatorProject()); $this->buildTransactionSchema( new PhabricatorProjectTransaction()); $this->buildCustomFieldSchemata( new PhabricatorProjectCustomFieldStorage(), array( new PhabricatorProjectCustomFieldNumericIndex(), new PhabricatorProjectCustomFieldStringIndex(), )); $this->buildTransactionSchema( new PhabricatorProjectColumnTransaction()); $this->buildRawSchema( id(new PhabricatorProject())->getApplicationName(), PhabricatorProject::TABLE_DATASOURCE_TOKEN, array( - 'id' => 'id', + 'id' => 'auto', 'projectID' => 'id', 'token' => 'text128', ), array( 'PRIMARY' => array( 'columns' => array('id'), 'unique' => true, ), 'token' => array( 'columns' => array('token', 'projectID'), 'unique' => true, ), 'projectID' => array( 'columns' => array('projectID'), ), )); } } diff --git a/src/applications/repository/storage/PhabricatorRepositorySchemaSpec.php b/src/applications/repository/storage/PhabricatorRepositorySchemaSpec.php index e2254eea90..40cdc70aa7 100644 --- a/src/applications/repository/storage/PhabricatorRepositorySchemaSpec.php +++ b/src/applications/repository/storage/PhabricatorRepositorySchemaSpec.php @@ -1,185 +1,185 @@ buildLiskSchemata('PhabricatorRepositoryDAO'); $this->buildEdgeSchemata(new PhabricatorRepository()); $this->buildTransactionSchema( new PhabricatorRepositoryTransaction()); $this->buildRawSchema( id(new PhabricatorRepository())->getApplicationName(), PhabricatorRepository::TABLE_BADCOMMIT, array( 'fullCommitName' => 'text64', 'description' => 'text', ), array( 'PRIMARY' => array( 'columns' => array('fullCommitName'), 'unique' => true, ), )); $this->buildRawSchema( id(new PhabricatorRepository())->getApplicationName(), PhabricatorRepository::TABLE_COVERAGE, array( - 'id' => 'id', + 'id' => 'auto', 'branchID' => 'id', 'commitID' => 'id', 'pathID' => 'id', 'coverage' => 'bytes', ), array( 'PRIMARY' => array( 'columns' => array('id'), 'unique' => true, ), 'key_path' => array( 'columns' => array('branchID', 'pathID', 'commitID'), ), )); $this->buildRawSchema( id(new PhabricatorRepository())->getApplicationName(), PhabricatorRepository::TABLE_FILESYSTEM, array( 'repositoryID' => 'id', 'parentID' => 'id', 'svnCommit' => 'uint32', 'pathID' => 'id', 'existed' => 'bool', 'fileType' => 'uint32', ), array( 'PRIMARY' => array( 'columns' => array('repositoryID', 'parentID', 'pathID', 'svnCommit'), 'unique' => true, ), 'repositoryID' => array( 'columns' => array('repositoryID', 'svnCommit'), ), )); $this->buildRawSchema( id(new PhabricatorRepository())->getApplicationName(), PhabricatorRepository::TABLE_LINTMESSAGE, array( - 'id' => 'id', + 'id' => 'auto', 'branchID' => 'id', 'path' => 'text', 'line' => 'uint32', 'authorPHID' => 'phid?', 'code' => 'text32', 'severity' => 'text16', 'name' => 'text255', 'description' => 'text', ), array( 'PRIMARY' => array( 'columns' => array('id'), 'unique' => true, ), 'branchID' => array( 'columns' => array('branchID', 'path(64)'), ), 'branchID_2' => array( 'columns' => array('branchID', 'code', 'path(64)'), ), 'key_author' => array( 'columns' => array('authorPHID'), ), )); $this->buildRawSchema( id(new PhabricatorRepository())->getApplicationName(), PhabricatorRepository::TABLE_PARENTS, array( - 'id' => 'id', + 'id' => 'auto', 'childCommitID' => 'id', 'parentCommitID' => 'id', ), array( 'PRIMARY' => array( 'columns' => array('id'), 'unique' => true, ), 'key_child' => array( 'columns' => array('childCommitID', 'parentCommitID'), 'unique' => true, ), 'key_parent' => array( 'columns' => array('parentCommitID'), ), )); $this->buildRawSchema( id(new PhabricatorRepository())->getApplicationName(), PhabricatorRepository::TABLE_PATH, array( - 'id' => 'id', + 'id' => 'auto', 'path' => 'text', 'pathHash' => 'bytes32', ), array( 'PRIMARY' => array( 'columns' => array('id'), 'unique' => true, ), 'pathHash' => array( 'columns' => array('pathHash'), 'unique' => true, ), )); $this->buildRawSchema( id(new PhabricatorRepository())->getApplicationName(), PhabricatorRepository::TABLE_PATHCHANGE, array( 'repositoryID' => 'id', 'pathID' => 'id', 'commitID' => 'id', 'targetPathID' => 'id?', 'targetCommitID' => 'id?', 'changeType' => 'uint32', 'fileType' => 'uint32', 'isDirect' => 'bool', 'commitSequence' => 'uint32', ), array( 'PRIMARY' => array( 'columns' => array('commitID', 'pathID'), 'unique' => true, ), 'repositoryID' => array( 'columns' => array('repositoryID', 'pathID', 'commitSequence'), ), )); $this->buildRawSchema( id(new PhabricatorRepository())->getApplicationName(), PhabricatorRepository::TABLE_SUMMARY, array( 'repositoryID' => 'id', 'size' => 'uint32', 'lastCommitID' => 'id', 'epoch' => 'epoch?', ), array( 'PRIMARY' => array( 'columns' => array('repositoryID'), 'unique' => true, ), 'key_epoch' => array( 'columns' => array('epoch'), ), )); } } diff --git a/src/applications/tokens/storage/PhabricatorTokenCount.php b/src/applications/tokens/storage/PhabricatorTokenCount.php index c4be4407f0..8380f8a1f1 100644 --- a/src/applications/tokens/storage/PhabricatorTokenCount.php +++ b/src/applications/tokens/storage/PhabricatorTokenCount.php @@ -1,27 +1,28 @@ self::IDS_MANUAL, self::CONFIG_TIMESTAMPS => false, self::CONFIG_COLUMN_SCHEMA => array( + 'id' => 'auto', 'tokenCount' => 'uint32', ), self::CONFIG_KEY_SCHEMA => array( 'key_objectPHID' => array( 'columns' => array('objectPHID'), 'unique' => true, ), 'key_count' => array( 'columns' => array('tokenCount'), ), ), ) + parent::getConfiguration(); } } diff --git a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php index 1159708b19..58f021f002 100644 --- a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php +++ b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerArchiveTask.php @@ -1,89 +1,94 @@ self::IDS_MANUAL, + ) + parent::getConfiguration(); + $config[self::CONFIG_COLUMN_SCHEMA] = array( 'result' => 'uint32', 'duration' => 'uint64', ) + $config[self::CONFIG_COLUMN_SCHEMA]; $config[self::CONFIG_KEY_SCHEMA] = array( 'dateCreated' => array( 'columns' => array('dateCreated'), ), 'leaseOwner' => array( 'columns' => array('leaseOwner', 'priority', 'id'), ), ); return $config; } public function save() { if ($this->getID() === null) { throw new Exception('Trying to archive a task with no ID.'); } $other = new PhabricatorWorkerActiveTask(); $conn_w = $this->establishConnection('w'); $this->openTransaction(); queryfx( $conn_w, 'DELETE FROM %T WHERE id = %d', $other->getTableName(), $this->getID()); $result = parent::insert(); $this->saveTransaction(); return $result; } public function delete() { $this->openTransaction(); if ($this->getDataID()) { $conn_w = $this->establishConnection('w'); $data_table = new PhabricatorWorkerTaskData(); queryfx( $conn_w, 'DELETE FROM %T WHERE id = %d', $data_table->getTableName(), $this->getDataID()); } $result = parent::delete(); $this->saveTransaction(); return $result; } public function unarchiveTask() { $this->openTransaction(); $active = id(new PhabricatorWorkerActiveTask()) ->setID($this->getID()) ->setTaskClass($this->getTaskClass()) ->setLeaseOwner(null) ->setLeaseExpires(0) ->setFailureCount(0) ->setDataID($this->getDataID()) ->setPriority($this->getPriority()) ->insert(); $this->setDataID(null); $this->delete(); $this->saveTransaction(); return $active; } } diff --git a/src/infrastructure/storage/lisk/LiskDAO.php b/src/infrastructure/storage/lisk/LiskDAO.php index cd76f29b52..95a500a2dc 100644 --- a/src/infrastructure/storage/lisk/LiskDAO.php +++ b/src/infrastructure/storage/lisk/LiskDAO.php @@ -1,1833 +1,1840 @@ setName('Sawyer') * ->setBreed('Pug') * ->save(); * * Note that **Lisk automatically builds getters and setters for all of your * object's protected properties** via @{method:__call}. If you want to add * custom behavior to your getters or setters, you can do so by overriding the * @{method:readField} and @{method:writeField} methods. * * Calling @{method:save} will persist the object to the database. After calling * @{method:save}, you can call @{method:getID} to retrieve the object's ID. * * To load objects by ID, use the @{method:load} method: * * $dog = id(new Dog())->load($id); * * This will load the Dog record with ID $id into $dog, or `null` if no such * record exists (@{method:load} is an instance method rather than a static * method because PHP does not support late static binding, at least until PHP * 5.3). * * To update an object, change its properties and save it: * * $dog->setBreed('Lab')->save(); * * To delete an object, call @{method:delete}: * * $dog->delete(); * * That's Lisk CRUD in a nutshell. * * = Queries = * * Often, you want to load a bunch of objects, or execute a more specialized * query. Use @{method:loadAllWhere} or @{method:loadOneWhere} to do this: * * $pugs = $dog->loadAllWhere('breed = %s', 'Pug'); * $sawyer = $dog->loadOneWhere('name = %s', 'Sawyer'); * * These methods work like @{function@libphutil:queryfx}, but only take half of * a query (the part after the WHERE keyword). Lisk will handle the connection, * columns, and object construction; you are responsible for the rest of it. * @{method:loadAllWhere} returns a list of objects, while * @{method:loadOneWhere} returns a single object (or `null`). * * There's also a @{method:loadRelatives} method which helps to prevent the 1+N * queries problem. * * = Managing Transactions = * * Lisk uses a transaction stack, so code does not generally need to be aware * of the transactional state of objects to implement correct transaction * semantics: * * $obj->openTransaction(); * $obj->save(); * $other->save(); * // ... * $other->openTransaction(); * $other->save(); * $another->save(); * if ($some_condition) { * $other->saveTransaction(); * } else { * $other->killTransaction(); * } * // ... * $obj->saveTransaction(); * * Assuming ##$obj##, ##$other## and ##$another## live on the same database, * this code will work correctly by establishing savepoints. * * Selects whose data are used later in the transaction should be included in * @{method:beginReadLocking} or @{method:beginWriteLocking} block. * * @task conn Managing Connections * @task config Configuring Lisk * @task load Loading Objects * @task info Examining Objects * @task save Writing Objects * @task hook Hooks and Callbacks * @task util Utilities * @task xaction Managing Transactions * @task isolate Isolation for Unit Testing */ abstract class LiskDAO { const CONFIG_IDS = 'id-mechanism'; const CONFIG_TIMESTAMPS = 'timestamps'; const CONFIG_AUX_PHID = 'auxiliary-phid'; const CONFIG_SERIALIZATION = 'col-serialization'; const CONFIG_BINARY = 'binary'; const CONFIG_COLUMN_SCHEMA = 'col-schema'; const CONFIG_KEY_SCHEMA = 'key-schema'; const CONFIG_NO_TABLE = 'no-table'; const SERIALIZATION_NONE = 'id'; const SERIALIZATION_JSON = 'json'; const SERIALIZATION_PHP = 'php'; const IDS_AUTOINCREMENT = 'ids-auto'; const IDS_COUNTER = 'ids-counter'; const IDS_MANUAL = 'ids-manual'; const COUNTER_TABLE_NAME = 'lisk_counter'; private static $processIsolationLevel = 0; private static $transactionIsolationLevel = 0; private $ephemeral = false; private static $connections = array(); private $inSet = null; protected $id; protected $phid; protected $dateCreated; protected $dateModified; /** * Build an empty object. * * @return obj Empty object. */ public function __construct() { $id_key = $this->getIDKey(); if ($id_key) { $this->$id_key = null; } } /* -( Managing Connections )----------------------------------------------- */ /** * Establish a live connection to a database service. This method should * return a new connection. Lisk handles connection caching and management; * do not perform caching deeper in the stack. * * @param string Mode, either 'r' (reading) or 'w' (reading and writing). * @return AphrontDatabaseConnection New database connection. * @task conn */ abstract protected function establishLiveConnection($mode); /** * Return a namespace for this object's connections in the connection cache. * Generally, the database name is appropriate. Two connections are considered * equivalent if they have the same connection namespace and mode. * * @return string Connection namespace for cache * @task conn */ abstract protected function getConnectionNamespace(); /** * Get an existing, cached connection for this object. * * @param mode Connection mode. * @return AprontDatabaseConnection|null Connection, if it exists in cache. * @task conn */ protected function getEstablishedConnection($mode) { $key = $this->getConnectionNamespace().':'.$mode; if (isset(self::$connections[$key])) { return self::$connections[$key]; } return null; } /** * Store a connection in the connection cache. * * @param mode Connection mode. * @param AphrontDatabaseConnection Connection to cache. * @return this * @task conn */ protected function setEstablishedConnection( $mode, AphrontDatabaseConnection $connection, $force_unique = false) { $key = $this->getConnectionNamespace().':'.$mode; if ($force_unique) { $key .= ':unique'; while (isset(self::$connections[$key])) { $key .= '!'; } } self::$connections[$key] = $connection; return $this; } /* -( Configuring Lisk )--------------------------------------------------- */ /** * Change Lisk behaviors, like ID configuration and timestamps. If you want * to change these behaviors, you should override this method in your child * class and change the options you're interested in. For example: * * public function getConfiguration() { * return array( * Lisk_DataAccessObject::CONFIG_EXAMPLE => true, * ) + parent::getConfiguration(); * } * * The available options are: * * CONFIG_IDS * Lisk objects need to have a unique identifying ID. The three mechanisms * available for generating this ID are IDS_AUTOINCREMENT (default, assumes * the ID column is an autoincrement primary key), IDS_MANUAL (you are taking * full responsibility for ID management), or IDS_COUNTER (see below). * * InnoDB does not persist the value of `auto_increment` across restarts, * and instead initializes it to `MAX(id) + 1` during startup. This means it * may reissue the same autoincrement ID more than once, if the row is deleted * and then the database is restarted. To avoid this, you can set an object to * use a counter table with IDS_COUNTER. This will generally behave like * IDS_AUTOINCREMENT, except that the counter value will persist across * restarts and inserts will be slightly slower. If a database stores any * DAOs which use this mechanism, you must create a table there with this * schema: * * CREATE TABLE lisk_counter ( * counterName VARCHAR(64) COLLATE utf8_bin PRIMARY KEY, * counterValue BIGINT UNSIGNED NOT NULL * ) ENGINE=InnoDB DEFAULT CHARSET=utf8; * * CONFIG_TIMESTAMPS * Lisk can automatically handle keeping track of a `dateCreated' and * `dateModified' column, which it will update when it creates or modifies * an object. If you don't want to do this, you may disable this option. * By default, this option is ON. * * CONFIG_AUX_PHID * This option can be enabled by being set to some truthy value. The meaning * of this value is defined by your PHID generation mechanism. If this option * is enabled, a `phid' property will be populated with a unique PHID when an * object is created (or if it is saved and does not currently have one). You * need to override generatePHID() and hook it into your PHID generation * mechanism for this to work. By default, this option is OFF. * * CONFIG_SERIALIZATION * You can optionally provide a column serialization map that will be applied * to values when they are written to the database. For example: * * self::CONFIG_SERIALIZATION => array( * 'complex' => self::SERIALIZATION_JSON, * ) * * This will cause Lisk to JSON-serialize the 'complex' field before it is * written, and unserialize it when it is read. * * CONFIG_BINARY * You can optionally provide a map of columns to a flag indicating that * they store binary data. These columns will not raise an error when * handling binary writes. * * CONFIG_COLUMN_SCHEMA * Provide a map of columns to schema column types. * * CONFIG_KEY_SCHEMA * Provide a map of key names to key specifications. * * CONFIG_NO_TABLE * Allows you to specify that this object does not actually have a table in * the database. * * @return dictionary Map of configuration options to values. * * @task config */ protected function getConfiguration() { return array( self::CONFIG_IDS => self::IDS_AUTOINCREMENT, self::CONFIG_TIMESTAMPS => true, ); } /** * Determine the setting of a configuration option for this class of objects. * * @param const Option name, one of the CONFIG_* constants. * @return mixed Option value, if configured (null if unavailable). * * @task config */ public function getConfigOption($option_name) { static $options = null; if (!isset($options)) { $options = $this->getConfiguration(); } return idx($options, $option_name); } /* -( Loading Objects )---------------------------------------------------- */ /** * Load an object by ID. You need to invoke this as an instance method, not * a class method, because PHP doesn't have late static binding (until * PHP 5.3.0). For example: * * $dog = id(new Dog())->load($dog_id); * * @param int Numeric ID identifying the object to load. * @return obj|null Identified object, or null if it does not exist. * * @task load */ public function load($id) { if (is_object($id)) { $id = (string)$id; } if (!$id || (!is_int($id) && !ctype_digit($id))) { return null; } return $this->loadOneWhere( '%C = %d', $this->getIDKeyForUse(), $id); } /** * Loads all of the objects, unconditionally. * * @return dict Dictionary of all persisted objects of this type, keyed * on object ID. * * @task load */ public function loadAll() { return $this->loadAllWhere('1 = 1'); } /** * Load all objects which match a WHERE clause. You provide everything after * the 'WHERE'; Lisk handles everything up to it. For example: * * $old_dogs = id(new Dog())->loadAllWhere('age > %d', 7); * * The pattern and arguments are as per queryfx(). * * @param string queryfx()-style SQL WHERE clause. * @param ... Zero or more conversions. * @return dict Dictionary of matching objects, keyed on ID. * * @task load */ public function loadAllWhere($pattern /* , $arg, $arg, $arg ... */) { $args = func_get_args(); $data = call_user_func_array( array($this, 'loadRawDataWhere'), $args); return $this->loadAllFromArray($data); } /** * Load a single object identified by a 'WHERE' clause. You provide * everything after the 'WHERE', and Lisk builds the first half of the * query. See loadAllWhere(). This method is similar, but returns a single * result instead of a list. * * @param string queryfx()-style SQL WHERE clause. * @param ... Zero or more conversions. * @return obj|null Matching object, or null if no object matches. * * @task load */ public function loadOneWhere($pattern /* , $arg, $arg, $arg ... */) { $args = func_get_args(); $data = call_user_func_array( array($this, 'loadRawDataWhere'), $args); if (count($data) > 1) { throw new AphrontCountQueryException( 'More than 1 result from loadOneWhere()!'); } $data = reset($data); if (!$data) { return null; } return $this->loadFromArray($data); } protected function loadRawDataWhere($pattern /* , $args... */) { $connection = $this->establishConnection('r'); $lock_clause = ''; if ($connection->isReadLocking()) { $lock_clause = 'FOR UPDATE'; } else if ($connection->isWriteLocking()) { $lock_clause = 'LOCK IN SHARE MODE'; } $args = func_get_args(); $args = array_slice($args, 1); $pattern = 'SELECT * FROM %T WHERE '.$pattern.' %Q'; array_unshift($args, $this->getTableName()); array_push($args, $lock_clause); array_unshift($args, $pattern); return call_user_func_array( array($connection, 'queryData'), $args); } /** * Reload an object from the database, discarding any changes to persistent * properties. This is primarily useful after entering a transaction but * before applying changes to an object. * * @return this * * @task load */ public function reload() { if (!$this->getID()) { throw new Exception("Unable to reload object that hasn't been loaded!"); } $result = $this->loadOneWhere( '%C = %d', $this->getIDKeyForUse(), $this->getID()); if (!$result) { throw new AphrontObjectMissingQueryException(); } return $this; } /** * Initialize this object's properties from a dictionary. Generally, you * load single objects with loadOneWhere(), but sometimes it may be more * convenient to pull data from elsewhere directly (e.g., a complicated * join via @{method:queryData}) and then load from an array representation. * * @param dict Dictionary of properties, which should be equivalent to * selecting a row from the table or calling * @{method:getProperties}. * @return this * * @task load */ public function loadFromArray(array $row) { static $valid_properties = array(); $map = array(); foreach ($row as $k => $v) { // We permit (but ignore) extra properties in the array because a // common approach to building the array is to issue a raw SELECT query // which may include extra explicit columns or joins. // This pathway is very hot on some pages, so we're inlining a cache // and doing some microoptimization to avoid a strtolower() call for each // assignment. The common path (assigning a valid property which we've // already seen) always incurs only one empty(). The second most common // path (assigning an invalid property which we've already seen) costs // an empty() plus an isset(). if (empty($valid_properties[$k])) { if (isset($valid_properties[$k])) { // The value is set but empty, which means it's false, so we've // already determined it's not valid. We don't need to check again. continue; } $valid_properties[$k] = $this->hasProperty($k); if (!$valid_properties[$k]) { continue; } } $map[$k] = $v; } $this->willReadData($map); foreach ($map as $prop => $value) { $this->$prop = $value; } $this->didReadData(); return $this; } /** * Initialize a list of objects from a list of dictionaries. Usually you * load lists of objects with @{method:loadAllWhere}, but sometimes that * isn't flexible enough. One case is if you need to do joins to select the * right objects: * * function loadAllWithOwner($owner) { * $data = $this->queryData( * 'SELECT d.* * FROM owner o * JOIN owner_has_dog od ON o.id = od.ownerID * JOIN dog d ON od.dogID = d.id * WHERE o.id = %d', * $owner); * return $this->loadAllFromArray($data); * } * * This is a lot messier than @{method:loadAllWhere}, but more flexible. * * @param list List of property dictionaries. * @return dict List of constructed objects, keyed on ID. * * @task load */ public function loadAllFromArray(array $rows) { $result = array(); $id_key = $this->getIDKey(); foreach ($rows as $row) { $obj = clone $this; if ($id_key && isset($row[$id_key])) { $result[$row[$id_key]] = $obj->loadFromArray($row); } else { $result[] = $obj->loadFromArray($row); } if ($this->inSet) { $this->inSet->addToSet($obj); } } return $result; } /** * This method helps to prevent the 1+N queries problem. It happens when you * execute a query for each row in a result set. Like in this code: * * COUNTEREXAMPLE, name=Easy to write but expensive to execute * $diffs = id(new DifferentialDiff())->loadAllWhere( * 'revisionID = %d', * $revision->getID()); * foreach ($diffs as $diff) { * $changesets = id(new DifferentialChangeset())->loadAllWhere( * 'diffID = %d', * $diff->getID()); * // Do something with $changesets. * } * * One can solve this problem by reading all the dependent objects at once and * assigning them later: * * COUNTEREXAMPLE, name=Cheaper to execute but harder to write and maintain * $diffs = id(new DifferentialDiff())->loadAllWhere( * 'revisionID = %d', * $revision->getID()); * $all_changesets = id(new DifferentialChangeset())->loadAllWhere( * 'diffID IN (%Ld)', * mpull($diffs, 'getID')); * $all_changesets = mgroup($all_changesets, 'getDiffID'); * foreach ($diffs as $diff) { * $changesets = idx($all_changesets, $diff->getID(), array()); * // Do something with $changesets. * } * * The method @{method:loadRelatives} abstracts this approach which allows * writing a code which is simple and efficient at the same time: * * name=Easy to write and cheap to execute * $diffs = $revision->loadRelatives(new DifferentialDiff(), 'revisionID'); * foreach ($diffs as $diff) { * $changesets = $diff->loadRelatives( * new DifferentialChangeset(), * 'diffID'); * // Do something with $changesets. * } * * This will load dependent objects for all diffs in the first call of * @{method:loadRelatives} and use this result for all following calls. * * The method supports working with set of sets, like in this code: * * $diffs = $revision->loadRelatives(new DifferentialDiff(), 'revisionID'); * foreach ($diffs as $diff) { * $changesets = $diff->loadRelatives( * new DifferentialChangeset(), * 'diffID'); * foreach ($changesets as $changeset) { * $hunks = $changeset->loadRelatives( * new DifferentialHunk(), * 'changesetID'); * // Do something with hunks. * } * } * * This code will execute just three queries - one to load all diffs, one to * load all their related changesets and one to load all their related hunks. * You can try to write an equivalent code without using this method as * a homework. * * The method also supports retrieving referenced objects, for example authors * of all diffs (using shortcut @{method:loadOneRelative}): * * foreach ($diffs as $diff) { * $author = $diff->loadOneRelative( * new PhabricatorUser(), * 'phid', * 'getAuthorPHID'); * // Do something with author. * } * * It is also possible to specify additional conditions for the `WHERE` * clause. Similarly to @{method:loadAllWhere}, you can specify everything * after `WHERE` (except `LIMIT`). Contrary to @{method:loadAllWhere}, it is * allowed to pass only a constant string (`%` doesn't have a special * meaning). This is intentional to avoid mistakes with using data from one * row in retrieving other rows. Example of a correct usage: * * $status = $author->loadOneRelative( * new PhabricatorCalendarEvent(), * 'userPHID', * 'getPHID', * '(UNIX_TIMESTAMP() BETWEEN dateFrom AND dateTo)'); * * @param LiskDAO Type of objects to load. * @param string Name of the column in target table. * @param string Method name in this table. * @param string Additional constraints on returned rows. It supports no * placeholders and requires putting the WHERE part into * parentheses. It's not possible to use LIMIT. * @return list Objects of type $object. * * @task load */ public function loadRelatives( LiskDAO $object, $foreign_column, $key_method = 'getID', $where = '') { if (!$this->inSet) { id(new LiskDAOSet())->addToSet($this); } $relatives = $this->inSet->loadRelatives( $object, $foreign_column, $key_method, $where); return idx($relatives, $this->$key_method(), array()); } /** * Load referenced row. See @{method:loadRelatives} for details. * * @param LiskDAO Type of objects to load. * @param string Name of the column in target table. * @param string Method name in this table. * @param string Additional constraints on returned rows. It supports no * placeholders and requires putting the WHERE part into * parentheses. It's not possible to use LIMIT. * @return LiskDAO Object of type $object or null if there's no such object. * * @task load */ final public function loadOneRelative( LiskDAO $object, $foreign_column, $key_method = 'getID', $where = '') { $relatives = $this->loadRelatives( $object, $foreign_column, $key_method, $where); if (!$relatives) { return null; } if (count($relatives) > 1) { throw new AphrontCountQueryException( 'More than 1 result from loadOneRelative()!'); } return reset($relatives); } final public function putInSet(LiskDAOSet $set) { $this->inSet = $set; return $this; } final protected function getInSet() { return $this->inSet; } /* -( Examining Objects )-------------------------------------------------- */ /** * Set unique ID identifying this object. You normally don't need to call this * method unless with `IDS_MANUAL`. * * @param mixed Unique ID. * @return this * @task save */ public function setID($id) { static $id_key = null; if ($id_key === null) { $id_key = $this->getIDKeyForUse(); } $this->$id_key = $id; return $this; } /** * Retrieve the unique ID identifying this object. This value will be null if * the object hasn't been persisted and you didn't set it manually. * * @return mixed Unique ID. * * @task info */ public function getID() { static $id_key = null; if ($id_key === null) { $id_key = $this->getIDKeyForUse(); } return $this->$id_key; } public function getPHID() { return $this->phid; } /** * Test if a property exists. * * @param string Property name. * @return bool True if the property exists. * @task info */ public function hasProperty($property) { return (bool)$this->checkProperty($property); } /** * Retrieve a list of all object properties. This list only includes * properties that are declared as protected, and it is expected that * all properties returned by this function should be persisted to the * database. * Properties that should not be persisted must be declared as private. * * @return dict Dictionary of normalized (lowercase) to canonical (original * case) property names. * * @task info */ protected function getAllLiskProperties() { static $properties = null; if (!isset($properties)) { $class = new ReflectionClass(get_class($this)); $properties = array(); foreach ($class->getProperties(ReflectionProperty::IS_PROTECTED) as $p) { $properties[strtolower($p->getName())] = $p->getName(); } $id_key = $this->getIDKey(); if ($id_key != 'id') { unset($properties['id']); } if (!$this->getConfigOption(self::CONFIG_TIMESTAMPS)) { unset($properties['datecreated']); unset($properties['datemodified']); } if ($id_key != 'phid' && !$this->getConfigOption(self::CONFIG_AUX_PHID)) { unset($properties['phid']); } } return $properties; } /** * Check if a property exists on this object. * * @return string|null Canonical property name, or null if the property * does not exist. * * @task info */ protected function checkProperty($property) { static $properties = null; if ($properties === null) { $properties = $this->getAllLiskProperties(); } $property = strtolower($property); if (empty($properties[$property])) { return null; } return $properties[$property]; } /** * Get or build the database connection for this object. * * @param string 'r' for read, 'w' for read/write. * @param bool True to force a new connection. The connection will not * be retrieved from or saved into the connection cache. * @return LiskDatabaseConnection Lisk connection object. * * @task info */ public function establishConnection($mode, $force_new = false) { if ($mode != 'r' && $mode != 'w') { throw new Exception("Unknown mode '{$mode}', should be 'r' or 'w'."); } if (self::shouldIsolateAllLiskEffectsToCurrentProcess()) { $mode = 'isolate-'.$mode; $connection = $this->getEstablishedConnection($mode); if (!$connection) { $connection = $this->establishIsolatedConnection($mode); $this->setEstablishedConnection($mode, $connection); } return $connection; } if (self::shouldIsolateAllLiskEffectsToTransactions()) { // If we're doing fixture transaction isolation, force the mode to 'w' // so we always get the same connection for reads and writes, and thus // can see the writes inside the transaction. $mode = 'w'; } // TODO: There is currently no protection on 'r' queries against writing. $connection = null; if (!$force_new) { if ($mode == 'r') { // If we're requesting a read connection but already have a write // connection, reuse the write connection so that reads can take place // inside transactions. $connection = $this->getEstablishedConnection('w'); } if (!$connection) { $connection = $this->getEstablishedConnection($mode); } } if (!$connection) { $connection = $this->establishLiveConnection($mode); if (self::shouldIsolateAllLiskEffectsToTransactions()) { $connection->openTransaction(); } $this->setEstablishedConnection( $mode, $connection, $force_unique = $force_new); } return $connection; } /** * Convert this object into a property dictionary. This dictionary can be * restored into an object by using @{method:loadFromArray} (unless you're * using legacy features with CONFIG_CONVERT_CAMELCASE, but in that case you * should just go ahead and die in a fire). * * @return dict Dictionary of object properties. * * @task info */ protected function getAllLiskPropertyValues() { $map = array(); foreach ($this->getAllLiskProperties() as $p) { // We may receive a warning here for properties we've implicitly added // through configuration; squelch it. $map[$p] = @$this->$p; } return $map; } /* -( Writing Objects )---------------------------------------------------- */ /** * Make an object read-only. * * Making an object ephemeral indicates that you will be changing state in * such a way that you would never ever want it to be written back to the * storage. */ public function makeEphemeral() { $this->ephemeral = true; return $this; } private function isEphemeralCheck() { if ($this->ephemeral) { throw new LiskEphemeralObjectException(); } } /** * Persist this object to the database. In most cases, this is the only * method you need to call to do writes. If the object has not yet been * inserted this will do an insert; if it has, it will do an update. * * @return this * * @task save */ public function save() { if ($this->shouldInsertWhenSaved()) { return $this->insert(); } else { return $this->update(); } } /** * Save this object, forcing the query to use REPLACE regardless of object * state. * * @return this * * @task save */ public function replace() { $this->isEphemeralCheck(); return $this->insertRecordIntoDatabase('REPLACE'); } /** * Save this object, forcing the query to use INSERT regardless of object * state. * * @return this * * @task save */ public function insert() { $this->isEphemeralCheck(); return $this->insertRecordIntoDatabase('INSERT'); } /** * Save this object, forcing the query to use UPDATE regardless of object * state. * * @return this * * @task save */ public function update() { $this->isEphemeralCheck(); $this->willSaveObject(); $data = $this->getAllLiskPropertyValues(); $this->willWriteData($data); $map = array(); foreach ($data as $k => $v) { $map[$k] = $v; } $conn = $this->establishConnection('w'); $binary = $this->getBinaryColumns(); foreach ($map as $key => $value) { if (!empty($binary[$key])) { $map[$key] = qsprintf($conn, '%C = %nB', $key, $value); } else { $map[$key] = qsprintf($conn, '%C = %ns', $key, $value); } } $map = implode(', ', $map); $id = $this->getID(); $conn->query( 'UPDATE %T SET %Q WHERE %C = '.(is_int($id) ? '%d' : '%s'), $this->getTableName(), $map, $this->getIDKeyForUse(), $id); // We can't detect a missing object because updating an object without // changing any values doesn't affect rows. We could jiggle timestamps // to catch this for objects which track them if we wanted. $this->didWriteData(); return $this; } /** * Delete this object, permanently. * * @return this * * @task save */ public function delete() { $this->isEphemeralCheck(); $this->willDelete(); $conn = $this->establishConnection('w'); $conn->query( 'DELETE FROM %T WHERE %C = %d', $this->getTableName(), $this->getIDKeyForUse(), $this->getID()); $this->didDelete(); return $this; } /** * Internal implementation of INSERT and REPLACE. * * @param const Either "INSERT" or "REPLACE", to force the desired mode. * * @task save */ protected function insertRecordIntoDatabase($mode) { $this->willSaveObject(); $data = $this->getAllLiskPropertyValues(); $conn = $this->establishConnection('w'); $id_mechanism = $this->getConfigOption(self::CONFIG_IDS); switch ($id_mechanism) { case self::IDS_AUTOINCREMENT: // If we are using autoincrement IDs, let MySQL assign the value for the // ID column, if it is empty. If the caller has explicitly provided a // value, use it. $id_key = $this->getIDKeyForUse(); if (empty($data[$id_key])) { unset($data[$id_key]); } break; case self::IDS_COUNTER: // If we are using counter IDs, assign a new ID if we don't already have // one. $id_key = $this->getIDKeyForUse(); if (empty($data[$id_key])) { $counter_name = $this->getTableName(); $id = self::loadNextCounterID($conn, $counter_name); $this->setID($id); $data[$id_key] = $id; } break; case self::IDS_MANUAL: break; default: throw new Exception('Unknown CONFIG_IDs mechanism!'); } $this->willWriteData($data); $columns = array_keys($data); $binary = $this->getBinaryColumns(); foreach ($data as $key => $value) { try { if (!empty($binary[$key])) { $data[$key] = qsprintf($conn, '%nB', $value); } else { $data[$key] = qsprintf($conn, '%ns', $value); } } catch (AphrontParameterQueryException $parameter_exception) { throw new PhutilProxyException( pht( "Unable to insert or update object of class %s, field '%s' ". "has a nonscalar value.", get_class($this), $key), $parameter_exception); } } $data = implode(', ', $data); $conn->query( '%Q INTO %T (%LC) VALUES (%Q)', $mode, $this->getTableName(), $columns, $data); // Only use the insert id if this table is using auto-increment ids if ($id_mechanism === self::IDS_AUTOINCREMENT) { $this->setID($conn->getInsertID()); } $this->didWriteData(); return $this; } /** * Method used to determine whether to insert or update when saving. * * @return bool true if the record should be inserted */ protected function shouldInsertWhenSaved() { $key_type = $this->getConfigOption(self::CONFIG_IDS); if ($key_type == self::IDS_MANUAL) { throw new Exception( 'You are using manual IDs. You must override the '. 'shouldInsertWhenSaved() method to properly detect '. 'when to insert a new record.'); } else { return !$this->getID(); } } /* -( Hooks and Callbacks )------------------------------------------------ */ /** * Retrieve the database table name. By default, this is the class name. * * @return string Table name for object storage. * * @task hook */ public function getTableName() { return get_class($this); } /** * Retrieve the primary key column, "id" by default. If you can not * reasonably name your ID column "id", override this method. * * @return string Name of the ID column. * * @task hook */ public function getIDKey() { return 'id'; } protected function getIDKeyForUse() { $id_key = $this->getIDKey(); if (!$id_key) { throw new Exception( 'This DAO does not have a single-part primary key. The method you '. 'called requires a single-part primary key.'); } return $id_key; } /** * Generate a new PHID, used by CONFIG_AUX_PHID. * * @return phid Unique, newly allocated PHID. * * @task hook */ protected function generatePHID() { throw new Exception( 'To use CONFIG_AUX_PHID, you need to overload '. 'generatePHID() to perform PHID generation.'); } /** * Hook to apply serialization or validation to data before it is written to * the database. See also @{method:willReadData}. * * @task hook */ protected function willWriteData(array &$data) { $this->applyLiskDataSerialization($data, false); } /** * Hook to perform actions after data has been written to the database. * * @task hook */ protected function didWriteData() {} /** * Hook to make internal object state changes prior to INSERT, REPLACE or * UPDATE. * * @task hook */ protected function willSaveObject() { $use_timestamps = $this->getConfigOption(self::CONFIG_TIMESTAMPS); if ($use_timestamps) { if (!$this->getDateCreated()) { $this->setDateCreated(time()); } $this->setDateModified(time()); } if ($this->getConfigOption(self::CONFIG_AUX_PHID) && !$this->getPHID()) { $this->setPHID($this->generatePHID()); } } /** * Hook to apply serialization or validation to data as it is read from the * database. See also @{method:willWriteData}. * * @task hook */ protected function willReadData(array &$data) { $this->applyLiskDataSerialization($data, $deserialize = true); } /** * Hook to perform an action on data after it is read from the database. * * @task hook */ protected function didReadData() {} /** * Hook to perform an action before the deletion of an object. * * @task hook */ protected function willDelete() {} /** * Hook to perform an action after the deletion of an object. * * @task hook */ protected function didDelete() {} /** * Reads the value from a field. Override this method for custom behavior * of @{method:getField} instead of overriding getField directly. * * @param string Canonical field name * @return mixed Value of the field * * @task hook */ protected function readField($field) { if (isset($this->$field)) { return $this->$field; } return null; } /** * Writes a value to a field. Override this method for custom behavior of * setField($value) instead of overriding setField directly. * * @param string Canonical field name * @param mixed Value to write * * @task hook */ protected function writeField($field, $value) { $this->$field = $value; } /* -( Manging Transactions )----------------------------------------------- */ /** * Increase transaction stack depth. * * @return this */ public function openTransaction() { $this->establishConnection('w')->openTransaction(); return $this; } /** * Decrease transaction stack depth, saving work. * * @return this */ public function saveTransaction() { $this->establishConnection('w')->saveTransaction(); return $this; } /** * Decrease transaction stack depth, discarding work. * * @return this */ public function killTransaction() { $this->establishConnection('w')->killTransaction(); return $this; } /** * Begins read-locking selected rows with SELECT ... FOR UPDATE, so that * other connections can not read them (this is an enormous oversimplification * of FOR UPDATE semantics; consult the MySQL documentation for details). To * end read locking, call @{method:endReadLocking}. For example: * * $beach->openTransaction(); * $beach->beginReadLocking(); * * $beach->reload(); * $beach->setGrainsOfSand($beach->getGrainsOfSand() + 1); * $beach->save(); * * $beach->endReadLocking(); * $beach->saveTransaction(); * * @return this * @task xaction */ public function beginReadLocking() { $this->establishConnection('w')->beginReadLocking(); return $this; } /** * Ends read-locking that began at an earlier @{method:beginReadLocking} call. * * @return this * @task xaction */ public function endReadLocking() { $this->establishConnection('w')->endReadLocking(); return $this; } /** * Begins write-locking selected rows with SELECT ... LOCK IN SHARE MODE, so * that other connections can not update or delete them (this is an * oversimplification of LOCK IN SHARE MODE semantics; consult the * MySQL documentation for details). To end write locking, call * @{method:endWriteLocking}. * * @return this * @task xaction */ public function beginWriteLocking() { $this->establishConnection('w')->beginWriteLocking(); return $this; } /** * Ends write-locking that began at an earlier @{method:beginWriteLocking} * call. * * @return this * @task xaction */ public function endWriteLocking() { $this->establishConnection('w')->endWriteLocking(); return $this; } /* -( Isolation )---------------------------------------------------------- */ /** * @task isolate */ public static function beginIsolateAllLiskEffectsToCurrentProcess() { self::$processIsolationLevel++; } /** * @task isolate */ public static function endIsolateAllLiskEffectsToCurrentProcess() { self::$processIsolationLevel--; if (self::$processIsolationLevel < 0) { throw new Exception( 'Lisk process isolation level was reduced below 0.'); } } /** * @task isolate */ public static function shouldIsolateAllLiskEffectsToCurrentProcess() { return (bool)self::$processIsolationLevel; } /** * @task isolate */ private function establishIsolatedConnection($mode) { $config = array(); return new AphrontIsolatedDatabaseConnection($config); } /** * @task isolate */ public static function beginIsolateAllLiskEffectsToTransactions() { if (self::$transactionIsolationLevel === 0) { self::closeAllConnections(); } self::$transactionIsolationLevel++; } /** * @task isolate */ public static function endIsolateAllLiskEffectsToTransactions() { self::$transactionIsolationLevel--; if (self::$transactionIsolationLevel < 0) { throw new Exception( 'Lisk transaction isolation level was reduced below 0.'); } else if (self::$transactionIsolationLevel == 0) { foreach (self::$connections as $key => $conn) { if ($conn) { $conn->killTransaction(); } } self::closeAllConnections(); } } /** * @task isolate */ public static function shouldIsolateAllLiskEffectsToTransactions() { return (bool)self::$transactionIsolationLevel; } public static function closeAllConnections() { self::$connections = array(); } /* -( Utilities )---------------------------------------------------------- */ /** * Applies configured serialization to a dictionary of values. * * @task util */ protected function applyLiskDataSerialization(array &$data, $deserialize) { $serialization = $this->getConfigOption(self::CONFIG_SERIALIZATION); if ($serialization) { foreach (array_intersect_key($serialization, $data) as $col => $format) { switch ($format) { case self::SERIALIZATION_NONE: break; case self::SERIALIZATION_PHP: if ($deserialize) { $data[$col] = unserialize($data[$col]); } else { $data[$col] = serialize($data[$col]); } break; case self::SERIALIZATION_JSON: if ($deserialize) { $data[$col] = json_decode($data[$col], true); } else { $data[$col] = json_encode($data[$col]); } break; default: throw new Exception("Unknown serialization format '{$format}'."); } } } } /** * Black magic. Builds implied get*() and set*() for all properties. * * @param string Method name. * @param list Argument vector. * @return mixed get*() methods return the property value. set*() methods * return $this. * @task util */ public function __call($method, $args) { // NOTE: PHP has a bug that static variables defined in __call() are shared // across all children classes. Call a different method to work around this // bug. return $this->call($method, $args); } /** * @task util */ final protected function call($method, $args) { // NOTE: This method is very performance-sensitive (many thousands of calls // per page on some pages), and thus has some silliness in the name of // optimizations. static $dispatch_map = array(); if ($method[0] === 'g') { if (isset($dispatch_map[$method])) { $property = $dispatch_map[$method]; } else { if (substr($method, 0, 3) !== 'get') { throw new Exception("Unable to resolve method '{$method}'!"); } $property = substr($method, 3); if (!($property = $this->checkProperty($property))) { throw new Exception("Bad getter call: {$method}"); } $dispatch_map[$method] = $property; } return $this->readField($property); } if ($method[0] === 's') { if (isset($dispatch_map[$method])) { $property = $dispatch_map[$method]; } else { if (substr($method, 0, 3) !== 'set') { throw new Exception("Unable to resolve method '{$method}'!"); } $property = substr($method, 3); $property = $this->checkProperty($property); if (!$property) { throw new Exception("Bad setter call: {$method}"); } $dispatch_map[$method] = $property; } $this->writeField($property, $args[0]); return $this; } throw new Exception("Unable to resolve method '{$method}'."); } /** * Warns against writing to undeclared property. * * @task util */ public function __set($name, $value) { phlog('Wrote to undeclared property '.get_class($this).'::$'.$name.'.'); $this->$name = $value; } /** * Increments a named counter and returns the next value. * * @param AphrontDatabaseConnection Database where the counter resides. * @param string Counter name to create or increment. * @return int Next counter value. * * @task util */ public static function loadNextCounterID( AphrontDatabaseConnection $conn_w, $counter_name) { // NOTE: If an insert does not touch an autoincrement row or call // LAST_INSERT_ID(), MySQL normally does not change the value of // LAST_INSERT_ID(). This can cause a counter's value to leak to a // new counter if the second counter is created after the first one is // updated. To avoid this, we insert LAST_INSERT_ID(1), to ensure the // LAST_INSERT_ID() is always updated and always set correctly after the // query completes. queryfx( $conn_w, 'INSERT INTO %T (counterName, counterValue) VALUES (%s, LAST_INSERT_ID(1)) ON DUPLICATE KEY UPDATE counterValue = LAST_INSERT_ID(counterValue + 1)', self::COUNTER_TABLE_NAME, $counter_name); return $conn_w->getInsertID(); } private function getBinaryColumns() { return $this->getConfigOption(self::CONFIG_BINARY); } public function getSchemaColumns() { $custom_map = $this->getConfigOption(self::CONFIG_COLUMN_SCHEMA); if (!$custom_map) { $custom_map = array(); } $serialization = $this->getConfigOption(self::CONFIG_SERIALIZATION); if (!$serialization) { $serialization = array(); } $serialization_map = array( self::SERIALIZATION_JSON => 'text', self::SERIALIZATION_PHP => 'bytes', ); $binary_map = $this->getBinaryColumns(); + $id_mechanism = $this->getConfigOption(self::CONFIG_IDS); + if ($id_mechanism == self::IDS_AUTOINCREMENT) { + $id_type = 'auto'; + } else { + $id_type = 'id'; + } + $builtin = array( - 'id' => 'id', + 'id' => $id_type, 'phid' => 'phid', 'viewPolicy' => 'policy', 'editPolicy' => 'policy', 'epoch' => 'epoch', 'dateCreated' => 'epoch', 'dateModified' => 'epoch', ); $map = array(); foreach ($this->getAllLiskProperties() as $property) { // First, use types specified explicitly in the table configuration. if (array_key_exists($property, $custom_map)) { $map[$property] = $custom_map[$property]; continue; } // If we don't have an explicit type, try a builtin type for the // column. $type = idx($builtin, $property); if ($type) { $map[$property] = $type; continue; } // If the column has serialization, we can infer the column type. if (isset($serialization[$property])) { $type = idx($serialization_map, $serialization[$property]); if ($type) { $map[$property] = $type; continue; } } if (isset($binary_map[$property])) { $map[$property] = 'bytes'; continue; } // If the column is named `somethingPHID`, infer it is a PHID. if (preg_match('/[a-z]PHID$/', $property)) { $map[$property] = 'phid'; continue; } // If the column is named `somethingID`, infer it is an ID. if (preg_match('/[a-z]ID$/', $property)) { $map[$property] = 'id'; continue; } // We don't know the type of this column. $map[$property] = ''; } return $map; } public function getSchemaKeys() { $custom_map = $this->getConfigOption(self::CONFIG_KEY_SCHEMA); if (!$custom_map) { $custom_map = array(); } $default_map = array(); foreach ($this->getAllLiskProperties() as $property) { switch ($property) { case 'id': $default_map['PRIMARY'] = array( 'columns' => array('id'), 'unique' => true, ); break; case 'phid': $default_map['key_phid'] = array( 'columns' => array('phid'), 'unique' => true, ); break; } } return $custom_map + $default_map; } } diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php index 4831a7e34b..e2df2581a8 100644 --- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementAdjustWorkflow.php @@ -1,463 +1,511 @@ setName('adjust') ->setExamples('**adjust** [__options__]') ->setSynopsis( pht( 'Make schemata adjustments to correct issues with characters sets, '. 'collations, and keys.')); } public function execute(PhutilArgumentParser $args) { $force = $args->getArg('force'); $this->requireAllPatchesApplied(); return $this->adjustSchemata($force); } private function requireAllPatchesApplied() { $api = $this->getAPI(); $applied = $api->getAppliedPatches(); if ($applied === null) { throw new PhutilArgumentUsageException( pht( 'You have not initialized the database yet. You must initialize '. 'the database before you can adjust schemata. Run `storage upgrade` '. 'to initialize the database.')); } $applied = array_fuse($applied); $patches = $this->getPatches(); $patches = mpull($patches, null, 'getFullKey'); $missing = array_diff_key($patches, $applied); if ($missing) { throw new PhutilArgumentUsageException( pht( 'You have not applied all available storage patches yet. You must '. 'apply all available patches before you can adjust schemata. '. 'Run `storage status` to show patch status, and `storage upgrade` '. 'to apply missing patches.')); } } private function loadSchemata() { $query = id(new PhabricatorConfigSchemaQuery()) ->setAPI($this->getAPI()); $actual = $query->loadActualSchema(); $expect = $query->loadExpectedSchema(); $comp = $query->buildComparisonSchema($expect, $actual); return array($comp, $expect, $actual); } private function adjustSchemata($force) { $console = PhutilConsole::getConsole(); $console->writeOut( "%s\n", pht('Verifying database schemata...')); $adjustments = $this->findAdjustments(); if (!$adjustments) { $console->writeOut( "%s\n", pht('Found no issues with schemata.')); return; } $table = id(new PhutilConsoleTable()) ->addColumn('database', array('title' => pht('Database'))) ->addColumn('table', array('title' => pht('Table'))) ->addColumn('name', array('title' => pht('Name'))) ->addColumn('info', array('title' => pht('Issues'))); foreach ($adjustments as $adjust) { $info = array(); foreach ($adjust['issues'] as $issue) { $info[] = PhabricatorConfigStorageSchema::getIssueName($issue); } $table->addRow(array( 'database' => $adjust['database'], 'table' => idx($adjust, 'table'), 'name' => idx($adjust, 'name'), 'info' => implode(', ', $info), )); } $console->writeOut("\n\n"); $table->draw(); if (!$force) { $console->writeOut( "\n%s\n", pht( "Found %s issues(s) with schemata, detailed above.\n\n". "You can review issues in more detail from the web interface, ". "in Config > Database Status.\n\n". "MySQL needs to copy table data to make some adjustments, so these ". "migrations may take some time.". // TODO: Remove warning once this stabilizes. "\n\n". "WARNING: This workflow is new and unstable. If you continue, you ". "may unrecoverably destory data. Make sure you have a backup before ". "you proceed.", new PhutilNumber(count($adjustments)))); $prompt = pht('Fix these schema issues?'); if (!phutil_console_confirm($prompt, $default_no = true)) { return; } } $console->writeOut( "%s\n", pht('Fixing schema issues...')); $api = $this->getAPI(); $conn = $api->getConn(null); $failed = array(); - // We make changes in three phases: - // - // Phase 0: Drop all keys which we're going to adjust. This prevents them - // from interfering with column changes. - // - // Phase 1: Apply all database, table, and column changes. - // - // Phase 2: Restore adjusted keys. - $phases = 3; + // We make changes in several phases. + $phases = array( + // Drop surplus autoincrements. This allows us to drop primary keys on + // autoincrement columns. + 'drop_auto', + + // Drop all keys we're going to adjust. This prevents them from + // interfering with column changes. + 'drop_keys', + + // Apply all database, table, and column changes. + 'main', + + // Restore adjusted keys. + 'add_keys', + + // Add missing autoincrements. + 'add_auto', + ); $bar = id(new PhutilConsoleProgressBar()) - ->setTotal(count($adjustments) * $phases); + ->setTotal(count($adjustments) * count($phases)); - for ($phase = 0; $phase < $phases; $phase++) { + foreach ($phases as $phase) { foreach ($adjustments as $adjust) { try { switch ($adjust['kind']) { case 'database': - if ($phase != 1) { - break; + if ($phase == 'main') { + queryfx( + $conn, + 'ALTER DATABASE %T CHARACTER SET = %s COLLATE = %s', + $adjust['database'], + $adjust['charset'], + $adjust['collation']); } - queryfx( - $conn, - 'ALTER DATABASE %T CHARACTER SET = %s COLLATE = %s', - $adjust['database'], - $adjust['charset'], - $adjust['collation']); break; case 'table': - if ($phase != 1) { - break; + if ($phase == 'main') { + queryfx( + $conn, + 'ALTER TABLE %T.%T COLLATE = %s', + $adjust['database'], + $adjust['table'], + $adjust['collation']); } - queryfx( - $conn, - 'ALTER TABLE %T.%T COLLATE = %s', - $adjust['database'], - $adjust['table'], - $adjust['collation']); break; case 'column': - if ($phase != 1) { - break; - } - $parts = array(); - if ($adjust['charset']) { - $parts[] = qsprintf( - $conn, - 'CHARACTER SET %Q COLLATE %Q', - $adjust['charset'], - $adjust['collation']); + $apply = false; + $auto = false; + $new_auto = idx($adjust, 'auto'); + if ($phase == 'drop_auto') { + if ($new_auto === false) { + $apply = true; + $auto = false; + } + } else if ($phase == 'main') { + $apply = true; + if ($new_auto === false) { + $auto = false; + } else { + $auto = $adjust['is_auto']; + } + } else if ($phase == 'add_auto') { + if ($new_auto === true) { + $apply = true; + $auto = true; + } } - queryfx( - $conn, - 'ALTER TABLE %T.%T MODIFY %T %Q %Q %Q', - $adjust['database'], - $adjust['table'], - $adjust['name'], - $adjust['type'], - implode(' ', $parts), - $adjust['nullable'] ? 'NULL' : 'NOT NULL'); + if ($apply) { + $parts = array(); + + if ($auto) { + $parts[] = qsprintf( + $conn, + 'AUTO_INCREMENT'); + } + + if ($adjust['charset']) { + $parts[] = qsprintf( + $conn, + 'CHARACTER SET %Q COLLATE %Q', + $adjust['charset'], + $adjust['collation']); + } + queryfx( + $conn, + 'ALTER TABLE %T.%T MODIFY %T %Q %Q %Q', + $adjust['database'], + $adjust['table'], + $adjust['name'], + $adjust['type'], + implode(' ', $parts), + $adjust['nullable'] ? 'NULL' : 'NOT NULL'); + } break; case 'key': if (($phase == 0) && $adjust['exists']) { if ($adjust['name'] == 'PRIMARY') { $key_name = 'PRIMARY KEY'; } else { $key_name = qsprintf($conn, 'KEY %T', $adjust['name']); } queryfx( $conn, 'ALTER TABLE %T.%T DROP %Q', $adjust['database'], $adjust['table'], $key_name); } if (($phase == 2) && $adjust['keep']) { // Different keys need different creation syntax. Notable // special cases are primary keys and fulltext keys. if ($adjust['name'] == 'PRIMARY') { $key_name = 'PRIMARY KEY'; } else if ($adjust['indexType'] == 'FULLTEXT') { $key_name = qsprintf($conn, 'FULLTEXT %T', $adjust['name']); } else { if ($adjust['unique']) { $key_name = qsprintf( $conn, 'UNIQUE KEY %T', $adjust['name']); } else { $key_name = qsprintf( $conn, '/* NONUNIQUE */ KEY %T', $adjust['name']); } } queryfx( $conn, 'ALTER TABLE %T.%T ADD %Q (%Q)', $adjust['database'], $adjust['table'], $key_name, implode(', ', $adjust['columns'])); } break; default: throw new Exception( pht('Unknown schema adjustment kind "%s"!', $adjust['kind'])); } } catch (AphrontQueryException $ex) { $failed[] = array($adjust, $ex); } $bar->update(1); } } $bar->done(); if (!$failed) { $console->writeOut( "%s\n", pht('Completed fixing all schema issues.')); return 0; } $table = id(new PhutilConsoleTable()) ->addColumn('target', array('title' => pht('Target'))) ->addColumn('error', array('title' => pht('Error'))); foreach ($failed as $failure) { list($adjust, $ex) = $failure; $pieces = array_select_keys($adjust, array('database', 'table', 'name')); $pieces = array_filter($pieces); $target = implode('.', $pieces); $table->addRow( array( 'target' => $target, 'error' => $ex->getMessage(), )); } $console->writeOut("\n"); $table->draw(); $console->writeOut( "\n%s\n", pht('Failed to make some schema adjustments, detailed above.')); return 1; } private function findAdjustments() { list($comp, $expect, $actual) = $this->loadSchemata(); $issue_charset = PhabricatorConfigStorageSchema::ISSUE_CHARSET; $issue_collation = PhabricatorConfigStorageSchema::ISSUE_COLLATION; $issue_columntype = PhabricatorConfigStorageSchema::ISSUE_COLUMNTYPE; $issue_surpluskey = PhabricatorConfigStorageSchema::ISSUE_SURPLUSKEY; $issue_missingkey = PhabricatorConfigStorageSchema::ISSUE_MISSINGKEY; $issue_columns = PhabricatorConfigStorageSchema::ISSUE_KEYCOLUMNS; $issue_unique = PhabricatorConfigStorageSchema::ISSUE_UNIQUE; $issue_longkey = PhabricatorConfigStorageSchema::ISSUE_LONGKEY; + $issue_auto = PhabricatorConfigStorageSchema::ISSUE_AUTOINCREMENT; $adjustments = array(); foreach ($comp->getDatabases() as $database_name => $database) { $expect_database = $expect->getDatabase($database_name); $actual_database = $actual->getDatabase($database_name); if (!$expect_database || !$actual_database) { // If there's a real issue here, skip this stuff. continue; } $issues = array(); if ($database->hasIssue($issue_charset)) { $issues[] = $issue_charset; } if ($database->hasIssue($issue_collation)) { $issues[] = $issue_collation; } if ($issues) { $adjustments[] = array( 'kind' => 'database', 'database' => $database_name, 'issues' => $issues, 'charset' => $expect_database->getCharacterSet(), 'collation' => $expect_database->getCollation(), ); } foreach ($database->getTables() as $table_name => $table) { $expect_table = $expect_database->getTable($table_name); $actual_table = $actual_database->getTable($table_name); if (!$expect_table || !$actual_table) { continue; } $issues = array(); if ($table->hasIssue($issue_collation)) { $issues[] = $issue_collation; } if ($issues) { $adjustments[] = array( 'kind' => 'table', 'database' => $database_name, 'table' => $table_name, 'issues' => $issues, 'collation' => $expect_table->getCollation(), ); } foreach ($table->getColumns() as $column_name => $column) { $expect_column = $expect_table->getColumn($column_name); $actual_column = $actual_table->getColumn($column_name); if (!$expect_column || !$actual_column) { continue; } $issues = array(); if ($column->hasIssue($issue_collation)) { $issues[] = $issue_collation; } if ($column->hasIssue($issue_charset)) { $issues[] = $issue_charset; } if ($column->hasIssue($issue_columntype)) { $issues[] = $issue_columntype; } + if ($column->hasIssue($issue_auto)) { + $issues[] = $issue_auto; + } if ($issues) { if ($expect_column->getCharacterSet() === null) { // For non-text columns, we won't be specifying a collation or // character set. $charset = null; $collation = null; } else { $charset = $expect_column->getCharacterSet(); $collation = $expect_column->getCollation(); } - - $adjustments[] = array( + $adjustment = array( 'kind' => 'column', 'database' => $database_name, 'table' => $table_name, 'name' => $column_name, 'issues' => $issues, 'collation' => $collation, 'charset' => $charset, 'type' => $expect_column->getColumnType(), // NOTE: We don't adjust column nullability because it is // dangerous, so always use the current nullability. 'nullable' => $actual_column->getNullable(), + + // NOTE: This always stores the current value, because we have + // to make these updates separately. + 'is_auto' => $actual_column->getAutoIncrement(), ); + + if ($column->hasIssue($issue_auto)) { + $adjustment['auto'] = $expect_column->getAutoIncrement(); + } + + $adjustments[] = $adjustment; } } foreach ($table->getKeys() as $key_name => $key) { $expect_key = $expect_table->getKey($key_name); $actual_key = $actual_table->getKey($key_name); $issues = array(); $keep_key = true; if ($key->hasIssue($issue_surpluskey)) { $issues[] = $issue_surpluskey; $keep_key = false; } if ($key->hasIssue($issue_missingkey)) { $issues[] = $issue_missingkey; } if ($key->hasIssue($issue_columns)) { $issues[] = $issue_columns; } if ($key->hasIssue($issue_unique)) { $issues[] = $issue_unique; } // NOTE: We can't really fix this, per se, but we may need to remove // the key to change the column type. In the best case, the new // column type won't be overlong and recreating the key really will // fix the issue. In the worst case, we get the right column type and // lose the key, which is still better than retaining the key having // the wrong column type. if ($key->hasIssue($issue_longkey)) { $issues[] = $issue_longkey; } if ($issues) { $adjustment = array( 'kind' => 'key', 'database' => $database_name, 'table' => $table_name, 'name' => $key_name, 'issues' => $issues, 'exists' => (bool)$actual_key, 'keep' => $keep_key, ); if ($keep_key) { $adjustment += array( 'columns' => $expect_key->getColumnNames(), 'unique' => $expect_key->getUnique(), 'indexType' => $expect_key->getIndexType(), ); } $adjustments[] = $adjustment; } } } } return $adjustments; } }