Page Menu
Home
Phabricator
Search
Configure Global Search
Log In
Files
F15416619
D18682.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
10 KB
Referenced Files
None
Subscribers
None
D18682.diff
View Options
diff --git a/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php b/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php
--- a/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php
+++ b/src/applications/cache/storage/PhabricatorCacheSchemaSpec.php
@@ -30,6 +30,9 @@
'key_ttl' => array(
'columns' => array('cacheExpires'),
),
+ ),
+ array(
+ 'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_CACHE,
));
}
diff --git a/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php b/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php
--- a/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php
+++ b/src/applications/config/controller/PhabricatorConfigDatabaseStatusController.php
@@ -261,6 +261,7 @@
$this->renderAttr(
$table->getCollation(),
$table->hasIssue($collation_issue)),
+ $table->getPersistenceTypeDisplayName(),
);
}
@@ -270,12 +271,14 @@
null,
pht('Table'),
pht('Collation'),
+ pht('Persistence'),
))
->setColumnClasses(
array(
null,
'wide pri',
null,
+ null,
));
$title = $database_name;
diff --git a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php
--- a/src/applications/config/schema/PhabricatorConfigSchemaQuery.php
+++ b/src/applications/config/schema/PhabricatorConfigSchemaQuery.php
@@ -338,6 +338,8 @@
$comp_table->addKey($comp_key);
}
+ $comp_table->setPersistenceType($expect_table->getPersistenceType());
+
$comp_database->addTable($comp_table);
}
$comp_server->addDatabase($comp_database);
diff --git a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php
--- a/src/applications/config/schema/PhabricatorConfigSchemaSpec.php
+++ b/src/applications/config/schema/PhabricatorConfigSchemaSpec.php
@@ -56,36 +56,52 @@
}
protected function buildFerretIndexSchema(PhabricatorFerretEngine $engine) {
+ $index_options = array(
+ 'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_INDEX,
+ );
+
$this->buildRawSchema(
$engine->getApplicationName(),
$engine->getDocumentTableName(),
$engine->getDocumentSchemaColumns(),
- $engine->getDocumentSchemaKeys());
+ $engine->getDocumentSchemaKeys(),
+ $index_options);
$this->buildRawSchema(
$engine->getApplicationName(),
$engine->getFieldTableName(),
$engine->getFieldSchemaColumns(),
- $engine->getFieldSchemaKeys());
+ $engine->getFieldSchemaKeys(),
+ $index_options);
$this->buildRawSchema(
$engine->getApplicationName(),
$engine->getNgramsTableName(),
$engine->getNgramsSchemaColumns(),
- $engine->getNgramsSchemaKeys());
+ $engine->getNgramsSchemaKeys(),
+ $index_options);
$this->buildRawSchema(
$engine->getApplicationName(),
$engine->getCommonNgramsTableName(),
$engine->getCommonNgramsSchemaColumns(),
- $engine->getCommonNgramsSchemaKeys());
+ $engine->getCommonNgramsSchemaKeys(),
+ $index_options);
}
protected function buildRawSchema(
$database_name,
$table_name,
array $columns,
- array $keys) {
+ array $keys,
+ array $options = array()) {
+
+ PhutilTypeSpec::checkMap(
+ $options,
+ array(
+ 'persistence' => 'optional string',
+ ));
+
$database = $this->getDatabase($database_name);
$table = $this->newTable($table_name);
@@ -144,6 +160,11 @@
$table->addKey($key);
}
+ $persistence_type = idx($options, 'persistence');
+ if ($persistence_type !== null) {
+ $table->setPersistenceType($persistence_type);
+ }
+
$database->addTable($table);
}
diff --git a/src/applications/config/schema/PhabricatorConfigTableSchema.php b/src/applications/config/schema/PhabricatorConfigTableSchema.php
--- a/src/applications/config/schema/PhabricatorConfigTableSchema.php
+++ b/src/applications/config/schema/PhabricatorConfigTableSchema.php
@@ -7,6 +7,11 @@
private $engine;
private $columns = array();
private $keys = array();
+ private $persistenceType = self::PERSISTENCE_DATA;
+
+ const PERSISTENCE_DATA = 'data';
+ const PERSISTENCE_CACHE = 'cache';
+ const PERSISTENCE_INDEX = 'index';
public function addColumn(PhabricatorConfigColumnSchema $column) {
$key = $column->getName();
@@ -45,6 +50,27 @@
return idx($this->getKeys(), $key);
}
+ public function setPersistenceType($persistence_type) {
+ $this->persistenceType = $persistence_type;
+ return $this;
+ }
+
+ public function getPersistenceType() {
+ return $this->persistenceType;
+ }
+
+ public function getPersistenceTypeDisplayName() {
+ $map = array(
+ self::PERSISTENCE_DATA => pht('Data'),
+ self::PERSISTENCE_CACHE => pht('Cache'),
+ self::PERSISTENCE_INDEX => pht('Index'),
+ );
+
+ $type = $this->getPersistenceType();
+
+ return idx($map, $type, $type);
+ }
+
protected function getSubschemata() {
// NOTE: Keys and columns may have the same name, so make sure we return
// everything.
diff --git a/src/applications/differential/storage/DifferentialSchemaSpec.php b/src/applications/differential/storage/DifferentialSchemaSpec.php
--- a/src/applications/differential/storage/DifferentialSchemaSpec.php
+++ b/src/applications/differential/storage/DifferentialSchemaSpec.php
@@ -21,6 +21,9 @@
'dateCreated' => array(
'columns' => array('dateCreated'),
),
+ ),
+ array(
+ 'persistence' => PhabricatorConfigTableSchema::PERSISTENCE_CACHE,
));
$this->buildRawSchema(
diff --git a/src/docs/user/configuration/configuring_backups.diviner b/src/docs/user/configuration/configuring_backups.diviner
--- a/src/docs/user/configuration/configuring_backups.diviner
+++ b/src/docs/user/configuration/configuring_backups.diviner
@@ -145,6 +145,24 @@
should also restrict access to the backups.
+Skipping Indexes
+================
+
+By default, `bin/storage dump` does not dump all of the data in the database:
+it skips some caches which can be rebuilt automatically and do not need to be
+backed up. Some of these caches are very large, so the size of the dump may
+be significantly smaller than the size of the databases.
+
+If you have a large amount of data, you can specify `--no-indexes` when taking
+a database dump to skip additional tables which contain search indexes. This
+will reduce the size (and increase the speed) of the backup. This is an
+advanced option which most installs will not benefit from.
+
+This index data can be rebuilt after a restore, but will not be rebuilt
+automatically. If you choose to use this flag, you must manually rebuild
+indexes after a restore (for details, see ((reindex))).
+
+
Next Steps
==========
diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php
--- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php
+++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php
@@ -30,6 +30,13 @@
'With __--output__, write a compressed file to disk instead '.
'of a plaintext file.'),
),
+ array(
+ 'name' => 'no-indexes',
+ 'help' => pht(
+ 'Do not dump data in rebuildable index tables. This means '.
+ 'backups are smaller and faster, but you will need to manually '.
+ 'rebuild indexes after performing a restore.'),
+ ),
array(
'name' => 'overwrite',
'help' => pht(
@@ -49,6 +56,8 @@
$console = PhutilConsole::getConsole();
+ $with_indexes = !$args->getArg('no-indexes');
+
$applied = $api->getAppliedPatches();
if ($applied === null) {
$namespace = $api->getNamespace();
@@ -65,18 +74,58 @@
$ref = $api->getRef();
$ref_key = $ref->getRefKey();
- $schemata_map = id(new PhabricatorConfigSchemaQuery())
+ $schemata_query = id(new PhabricatorConfigSchemaQuery())
->setAPIs(array($api))
- ->setRefs(array($ref))
- ->loadActualSchemata();
- $schemata = $schemata_map[$ref_key];
+ ->setRefs(array($ref));
+
+ $actual_map = $schemata_query->loadActualSchemata();
+ $expect_map = $schemata_query->loadExpectedSchemata();
+
+ $schemata = $actual_map[$ref_key];
+ $expect = $expect_map[$ref_key];
$targets = array();
foreach ($schemata->getDatabases() as $database_name => $database) {
+ $expect_database = $expect->getDatabase($database_name);
foreach ($database->getTables() as $table_name => $table) {
+
+ // NOTE: It's possible for us to find tables in these database which
+ // we don't expect to be there. For example, an older version of
+ // Phabricator may have had a table that was later dropped. We assume
+ // these are data tables and always dump them, erring on the side of
+ // caution.
+
+ $persistence = PhabricatorConfigTableSchema::PERSISTENCE_DATA;
+ if ($expect_database) {
+ $expect_table = $expect_database->getTable($table_name);
+ if ($expect_table) {
+ $persistence = $expect_table->getPersistenceType();
+ }
+ }
+
+ switch ($persistence) {
+ case PhabricatorConfigTableSchema::PERSISTENCE_CACHE:
+ // When dumping tables, leave the data in cache tables in the
+ // database. This will be automatically rebuild after the data
+ // is restored and does not need to be persisted in backups.
+ $with_data = false;
+ break;
+ case PhabricatorConfigTableSchema::PERSISTENCE_INDEX:
+ // When dumping tables, leave index data behind of the caller
+ // specified "--no-indexes". These tables can be rebuilt manually
+ // from other tables, but do not rebuild automatically.
+ $with_data = $with_indexes;
+ break;
+ case PhabricatorConfigTableSchema::PERSISTENCE_DATA:
+ default:
+ $with_data = true;
+ break;
+ }
+
$targets[] = array(
'database' => $database_name,
'table' => $table_name,
+ 'data' => $with_data,
);
}
}
@@ -147,6 +196,10 @@
foreach ($targets as $target) {
$target_argv = $argv;
+ if (!$target['data']) {
+ $target_argv[] = '--no-data';
+ }
+
if ($has_password) {
$commands[] = csprintf(
'mysqldump -p%P %Ls -- %R %R',
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Fri, Mar 21, 12:32 PM (2 w, 1 d ago)
Storage Engine
blob
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
7715460
Default Alt Text
D18682.diff (10 KB)
Attached To
Mode
D18682: Add "persistence" types (data, cache, or index) to tables, and tweak what "storage dump" dumps
Attached
Detach File
Event Timeline
Log In to Comment