diff --git a/src/applications/files/management/PhabricatorFilesManagementCompactWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementCompactWorkflow.php index 0c8e7153e5..5d8115389b 100644 --- a/src/applications/files/management/PhabricatorFilesManagementCompactWorkflow.php +++ b/src/applications/files/management/PhabricatorFilesManagementCompactWorkflow.php @@ -1,134 +1,118 @@ newIteratorArguments(); + $arguments[] = array( + 'name' => 'dry-run', + 'help' => pht('Show what would be compacted.'), + ); + $this ->setName('compact') ->setSynopsis( pht( 'Merge identical files to share the same storage. In some cases, '. 'this can repair files with missing data.')) - ->setArguments( - array( - array( - 'name' => 'dry-run', - 'help' => pht('Show what would be compacted.'), - ), - array( - 'name' => 'all', - 'help' => pht('Compact all files.'), - ), - array( - 'name' => 'names', - 'wildcard' => true, - ), - )); + ->setArguments($arguments); } public function execute(PhutilArgumentParser $args) { $console = PhutilConsole::getConsole(); $iterator = $this->buildIterator($args); - if (!$iterator) { - throw new PhutilArgumentUsageException( - pht( - 'Either specify a list of files to compact, or use `%s` '. - 'to compact all files.', - '--all')); - } - $is_dry_run = $args->getArg('dry-run'); foreach ($iterator as $file) { $monogram = $file->getMonogram(); $hash = $file->getContentHash(); if (!$hash) { $console->writeOut( "%s\n", pht('%s: No content hash.', $monogram)); continue; } // Find other files with the same content hash. We're going to point // them at the data for this file. $similar_files = id(new PhabricatorFile())->loadAllWhere( 'contentHash = %s AND id != %d AND (storageEngine != %s OR storageHandle != %s)', $hash, $file->getID(), $file->getStorageEngine(), $file->getStorageHandle()); if (!$similar_files) { $console->writeOut( "%s\n", pht('%s: No other files with the same content hash.', $monogram)); continue; } // Only compact files into this one if we can load the data. This // prevents us from breaking working files if we're missing some data. try { $data = $file->loadFileData(); } catch (Exception $ex) { $data = null; } if ($data === null) { $console->writeOut( "%s\n", pht( '%s: Unable to load file data; declining to compact.', $monogram)); continue; } foreach ($similar_files as $similar_file) { if ($is_dry_run) { $console->writeOut( "%s\n", pht( '%s: Would compact storage with %s.', $monogram, $similar_file->getMonogram())); continue; } $console->writeOut( "%s\n", pht( '%s: Compacting storage with %s.', $monogram, $similar_file->getMonogram())); $old_instance = null; try { $old_instance = $similar_file->instantiateStorageEngine(); $old_engine = $similar_file->getStorageEngine(); $old_handle = $similar_file->getStorageHandle(); } catch (Exception $ex) { // If the old stuff is busted, we just won't try to delete the // old data. phlog($ex); } $similar_file ->setStorageEngine($file->getStorageEngine()) ->setStorageHandle($file->getStorageHandle()) ->save(); if ($old_instance) { $similar_file->deleteFileDataIfUnused( $old_instance, $old_engine, $old_handle); } } } return 0; } } diff --git a/src/applications/files/management/PhabricatorFilesManagementCycleWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementCycleWorkflow.php index 6d574d633a..cbcfac2cc8 100644 --- a/src/applications/files/management/PhabricatorFilesManagementCycleWorkflow.php +++ b/src/applications/files/management/PhabricatorFilesManagementCycleWorkflow.php @@ -1,132 +1,118 @@ newIteratorArguments(); + $arguments[] = array( + 'name' => 'key', + 'param' => 'keyname', + 'help' => pht('Select a specific storage key to cycle to.'), + ); + $this ->setName('cycle') ->setSynopsis( pht('Cycle master key for encrypted files.')) - ->setArguments( - array( - array( - 'name' => 'key', - 'param' => 'keyname', - 'help' => pht('Select a specific storage key to cycle to.'), - ), - array( - 'name' => 'all', - 'help' => pht('Change encoding for all files.'), - ), - array( - 'name' => 'names', - 'wildcard' => true, - ), - )); + ->setArguments($arguments); } public function execute(PhutilArgumentParser $args) { $iterator = $this->buildIterator($args); - if (!$iterator) { - throw new PhutilArgumentUsageException( - pht( - 'Either specify a list of files to cycle, or use --all to cycle '. - 'all files.')); - } $format_map = PhabricatorFileStorageFormat::getAllFormats(); $engines = PhabricatorFileStorageEngine::loadAllEngines(); $key_name = $args->getArg('key'); $failed = array(); foreach ($iterator as $file) { $monogram = $file->getMonogram(); $engine_key = $file->getStorageEngine(); $engine = idx($engines, $engine_key); if (!$engine) { echo tsprintf( "%s\n", pht( '%s: Uses unknown storage engine "%s".', $monogram, $engine_key)); $failed[] = $file; continue; } if ($engine->isChunkEngine()) { echo tsprintf( "%s\n", pht( '%s: Stored as chunks, declining to cycle directly.', $monogram)); continue; } $format_key = $file->getStorageFormat(); if (empty($format_map[$format_key])) { echo tsprintf( "%s\n", pht( '%s: Uses unknown storage format "%s".', $monogram, $format_key)); $failed[] = $file; continue; } $format = clone $format_map[$format_key]; $format->setFile($file); if (!$format->canCycleMasterKey()) { echo tsprintf( "%s\n", pht( '%s: Storage format ("%s") does not support key cycling.', $monogram, $format->getStorageFormatName())); continue; } echo tsprintf( "%s\n", pht( '%s: Cycling master key.', $monogram)); try { if ($key_name) { $format->selectMasterKey($key_name); } $file->cycleMasterStorageKey($format); echo tsprintf( "%s\n", pht('Done.')); } catch (Exception $ex) { echo tsprintf( "%B\n", pht('Failed! %s', (string)$ex)); $failed[] = $file; } } if ($failed) { $monograms = mpull($failed, 'getMonogram'); echo tsprintf( "%s\n", pht('Failures: %s.', implode(', ', $monograms))); return 1; } return 0; } } diff --git a/src/applications/files/management/PhabricatorFilesManagementEncodeWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementEncodeWorkflow.php index 1d972326da..f7d299bc5f 100644 --- a/src/applications/files/management/PhabricatorFilesManagementEncodeWorkflow.php +++ b/src/applications/files/management/PhabricatorFilesManagementEncodeWorkflow.php @@ -1,151 +1,140 @@ newIteratorArguments(); + + $arguments[] = array( + 'name' => 'as', + 'param' => 'format', + 'help' => pht('Select the storage format to use.'), + ); + + $arguments[] = array( + 'name' => 'key', + 'param' => 'keyname', + 'help' => pht('Select a specific storage key.'), + ); + + $arguments[] = array( + 'name' => 'force', + 'help' => pht( + 'Re-encode files which are already stored in the target '. + 'encoding.'), + ); + $this ->setName('encode') ->setSynopsis( pht('Change the storage encoding of files.')) - ->setArguments( - array( - array( - 'name' => 'as', - 'param' => 'format', - 'help' => pht('Select the storage format to use.'), - ), - array( - 'name' => 'key', - 'param' => 'keyname', - 'help' => pht('Select a specific storage key.'), - ), - array( - 'name' => 'all', - 'help' => pht('Change encoding for all files.'), - ), - array( - 'name' => 'force', - 'help' => pht( - 'Re-encode files which are already stored in the target '. - 'encoding.'), - ), - array( - 'name' => 'names', - 'wildcard' => true, - ), - )); + ->setArguments($arguments); } public function execute(PhutilArgumentParser $args) { $iterator = $this->buildIterator($args); - if (!$iterator) { - throw new PhutilArgumentUsageException( - pht( - 'Either specify a list of files to encode, or use --all to '. - 'encode all files.')); - } $force = (bool)$args->getArg('force'); $format_list = PhabricatorFileStorageFormat::getAllFormats(); $format_list = array_keys($format_list); $format_list = implode(', ', $format_list); $format_key = $args->getArg('as'); if (!strlen($format_key)) { throw new PhutilArgumentUsageException( pht( 'Use --as to select a target encoding format. Available '. 'formats are: %s.', $format_list)); } $format = PhabricatorFileStorageFormat::getFormat($format_key); if (!$format) { throw new PhutilArgumentUsageException( pht( 'Storage format "%s" is not valid. Available formats are: %s.', $format_key, $format_list)); } $key_name = $args->getArg('key'); if (strlen($key_name)) { $format->selectMasterKey($key_name); } $engines = PhabricatorFileStorageEngine::loadAllEngines(); $failed = array(); foreach ($iterator as $file) { $monogram = $file->getMonogram(); $engine_key = $file->getStorageEngine(); $engine = idx($engines, $engine_key); if (!$engine) { echo tsprintf( "%s\n", pht( '%s: Uses unknown storage engine "%s".', $monogram, $engine_key)); $failed[] = $file; continue; } if ($engine->isChunkEngine()) { echo tsprintf( "%s\n", pht( '%s: Stored as chunks, no data to encode directly.', $monogram)); continue; } if (($file->getStorageFormat() == $format_key) && !$force) { echo tsprintf( "%s\n", pht( '%s: Already encoded in target format.', $monogram)); continue; } echo tsprintf( "%s\n", pht( '%s: Changing encoding from "%s" to "%s".', $monogram, $file->getStorageFormat(), $format_key)); try { $file->migrateToStorageFormat($format); echo tsprintf( "%s\n", pht('Done.')); } catch (Exception $ex) { echo tsprintf( "%B\n", pht('Failed! %s', (string)$ex)); $failed[] = $file; } } if ($failed) { $monograms = mpull($failed, 'getMonogram'); echo tsprintf( "%s\n", pht('Failures: %s.', implode(', ', $monograms))); return 1; } return 0; } } diff --git a/src/applications/files/management/PhabricatorFilesManagementIntegrityWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementIntegrityWorkflow.php index 344df71460..a30f4f970b 100644 --- a/src/applications/files/management/PhabricatorFilesManagementIntegrityWorkflow.php +++ b/src/applications/files/management/PhabricatorFilesManagementIntegrityWorkflow.php @@ -1,325 +1,317 @@ newIteratorArguments(); + + $arguments[] = array( + 'name' => 'strip', + 'help' => pht( + 'DANGEROUS. Strip integrity hashes from files. This makes '. + 'files vulnerable to corruption or tampering.'), + ); + + $arguments[] = array( + 'name' => 'corrupt', + 'help' => pht( + 'Corrupt integrity hashes for given files. This is intended '. + 'for debugging.'), + ); + + $arguments[] = array( + 'name' => 'compute', + 'help' => pht( + 'Compute and update integrity hashes for files which do not '. + 'yet have them.'), + ); + + $arguments[] = array( + 'name' => 'overwrite', + 'help' => pht( + 'DANGEROUS. Recompute and update integrity hashes, overwriting '. + 'invalid hashes. This may mark corrupt or dangerous files as '. + 'valid.'), + ); + + $arguments[] = array( + 'name' => 'force', + 'short' => 'f', + 'help' => pht( + 'Execute dangerous operations without prompting for '. + 'confirmation.'), + ); + + $this ->setName('integrity') ->setSynopsis(pht('Verify or recalculate file integrity hashes.')) - ->setArguments( - array( - array( - 'name' => 'all', - 'help' => pht('Affect all files.'), - ), - array( - 'name' => 'strip', - 'help' => pht( - 'DANGEROUS. Strip integrity hashes from files. This makes '. - 'files vulnerable to corruption or tampering.'), - ), - array( - 'name' => 'corrupt', - 'help' => pht( - 'Corrupt integrity hashes for given files. This is intended '. - 'for debugging.'), - ), - array( - 'name' => 'compute', - 'help' => pht( - 'Compute and update integrity hashes for files which do not '. - 'yet have them.'), - ), - array( - 'name' => 'overwrite', - 'help' => pht( - 'DANGEROUS. Recompute and update integrity hashes, overwriting '. - 'invalid hashes. This may mark corrupt or dangerous files as '. - 'valid.'), - ), - array( - 'name' => 'force', - 'short' => 'f', - 'help' => pht( - 'Execute dangerous operations without prompting for '. - 'confirmation.'), - ), - array( - 'name' => 'names', - 'wildcard' => true, - ), - )); + ->setArguments($arguments); } public function execute(PhutilArgumentParser $args) { $modes = array(); $is_strip = $args->getArg('strip'); if ($is_strip) { $modes[] = 'strip'; } $is_corrupt = $args->getArg('corrupt'); if ($is_corrupt) { $modes[] = 'corrupt'; } $is_compute = $args->getArg('compute'); if ($is_compute) { $modes[] = 'compute'; } $is_overwrite = $args->getArg('overwrite'); if ($is_overwrite) { $modes[] = 'overwrite'; } $is_verify = !$modes; if ($is_verify) { $modes[] = 'verify'; } if (count($modes) > 1) { throw new PhutilArgumentUsageException( pht( 'You have selected multiple operation modes (%s). Choose a '. 'single mode to operate in.', implode(', ', $modes))); } $is_force = $args->getArg('force'); if (!$is_force) { $prompt = null; if ($is_strip) { $prompt = pht( 'Stripping integrity hashes is dangerous and makes files '. 'vulnerable to corruption or tampering.'); } if ($is_corrupt) { $prompt = pht( 'Corrupting integrity hashes will prevent files from being '. 'accessed. This mode is intended only for development and '. 'debugging.'); } if ($is_overwrite) { $prompt = pht( 'Overwriting integrity hashes is dangerous and may mark files '. 'which have been corrupted or tampered with as safe.'); } if ($prompt) { $this->logWarn(pht('DANGEROUS'), $prompt); if (!phutil_console_confirm(pht('Continue anyway?'))) { throw new PhutilArgumentUsageException(pht('Aborted workflow.')); } } } $iterator = $this->buildIterator($args); - if (!$iterator) { - throw new PhutilArgumentUsageException( - pht( - 'Either specify a list of files to affect, or use "--all" to '. - 'affect all files.')); - } $failure_count = 0; $total_count = 0; foreach ($iterator as $file) { $total_count++; $display_name = $file->getMonogram(); $old_hash = $file->getIntegrityHash(); if ($is_strip) { if ($old_hash === null) { $this->logInfo( pht('SKIPPED'), pht( 'File "%s" does not have an integrity hash to strip.', $display_name)); } else { $file ->setIntegrityHash(null) ->save(); $this->logWarn( pht('STRIPPED'), pht( 'Stripped integrity hash for "%s".', $display_name)); } continue; } $need_hash = ($is_verify && $old_hash) || ($is_compute && ($old_hash === null)) || ($is_corrupt) || ($is_overwrite); if ($need_hash) { try { $new_hash = $file->newIntegrityHash(); } catch (Exception $ex) { $failure_count++; $this->logFail( pht('ERROR'), pht( 'Unable to compute integrity hash for file "%s": %s', $display_name, $ex->getMessage())); continue; } } else { $new_hash = null; } // NOTE: When running in "corrupt" mode, we only corrupt the hash if // we're able to compute a valid hash. Some files, like chunked files, // do not support integrity hashing so corrupting them would create an // unusual state. if ($is_corrupt) { if ($new_hash === null) { $this->logInfo( pht('IGNORED'), pht( 'Storage for file "%s" does not support integrity hashing.', $display_name)); } else { $file ->setIntegrityHash('') ->save(); $this->logWarn( pht('CORRUPTED'), pht( 'Corrupted integrity hash for file "%s".', $display_name)); } continue; } if ($is_verify) { if ($old_hash === null) { $this->logInfo( pht('NONE'), pht( 'File "%s" has no stored integrity hash.', $display_name)); } else if ($new_hash === null) { $failure_count++; $this->logWarn( pht('UNEXPECTED'), pht( 'Storage for file "%s" does not support integrity hashing, '. 'but the file has an integrity hash.', $display_name)); } else if (phutil_hashes_are_identical($old_hash, $new_hash)) { $this->logOkay( pht('VALID'), pht( 'File "%s" has a valid integrity hash.', $display_name)); } else { $failure_count++; $this->logFail( pht('MISMATCH'), pht( 'File "%s" has an invalid integrity hash!', $display_name)); } continue; } if ($is_compute) { if ($old_hash !== null) { $this->logInfo( pht('SKIP'), pht( 'File "%s" already has an integrity hash.', $display_name)); } else if ($new_hash === null) { $this->logInfo( pht('IGNORED'), pht( 'Storage for file "%s" does not support integrity hashing.', $display_name)); } else { $file ->setIntegrityHash($new_hash) ->save(); $this->logOkay( pht('COMPUTE'), pht( 'Computed and stored integrity hash for file "%s".', $display_name)); } continue; } if ($is_overwrite) { $same_hash = ($old_hash !== null) && ($new_hash !== null) && phutil_hashes_are_identical($old_hash, $new_hash); if ($new_hash === null) { $this->logInfo( pht('IGNORED'), pht( 'Storage for file "%s" does not support integrity hashing.', $display_name)); } else if ($same_hash) { $this->logInfo( pht('UNCHANGED'), pht( 'File "%s" already has the correct integrity hash.', $display_name)); } else { $file ->setIntegrityHash($new_hash) ->save(); $this->logOkay( pht('OVERWRITE'), pht( 'Overwrote integrity hash for file "%s".', $display_name)); } continue; } } if ($failure_count) { $this->logFail( pht('FAIL'), pht( 'Processed %s file(s), encountered %s error(s).', new PhutilNumber($total_count), new PhutilNumber($failure_count))); } else { $this->logOkay( pht('DONE'), pht( 'Processed %s file(s) with no errors.', new PhutilNumber($total_count))); } return 0; } } diff --git a/src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php index 58d5155aed..859ee0bf2d 100644 --- a/src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php +++ b/src/applications/files/management/PhabricatorFilesManagementMigrateWorkflow.php @@ -1,281 +1,266 @@ newIteratorArguments(); + + $arguments[] = array( + 'name' => 'engine', + 'param' => 'storage-engine', + 'help' => pht('Migrate to the named storage engine.'), + ); + + $arguments[] = array( + 'name' => 'dry-run', + 'help' => pht('Show what would be migrated.'), + ); + + $arguments[] = array( + 'name' => 'min-size', + 'param' => 'bytes', + 'help' => pht( + 'Do not migrate data for files which are smaller than a given '. + 'filesize.'), + ); + + $arguments[] = array( + 'name' => 'max-size', + 'param' => 'bytes', + 'help' => pht( + 'Do not migrate data for files which are larger than a given '. + 'filesize.'), + ); + + $arguments[] = array( + 'name' => 'copy', + 'help' => pht( + 'Copy file data instead of moving it: after migrating, do not '. + 'remove the old data even if it is no longer referenced.'), + ); + + $arguments[] = array( + 'name' => 'local-disk-source', + 'param' => 'path', + 'help' => pht( + 'When migrating from a local disk source, use the specified '. + 'path as the root directory.'), + ); + $this ->setName('migrate') ->setSynopsis(pht('Migrate files between storage engines.')) - ->setArguments( - array( - array( - 'name' => 'engine', - 'param' => 'storage_engine', - 'help' => pht('Migrate to the named storage engine.'), - ), - array( - 'name' => 'dry-run', - 'help' => pht('Show what would be migrated.'), - ), - array( - 'name' => 'min-size', - 'param' => 'bytes', - 'help' => pht( - 'Do not migrate data for files which are smaller than a given '. - 'filesize.'), - ), - array( - 'name' => 'max-size', - 'param' => 'bytes', - 'help' => pht( - 'Do not migrate data for files which are larger than a given '. - 'filesize.'), - ), - array( - 'name' => 'all', - 'help' => pht('Migrate all files.'), - ), - array( - 'name' => 'copy', - 'help' => pht( - 'Copy file data instead of moving it: after migrating, do not '. - 'remove the old data even if it is no longer referenced.'), - ), - array( - 'name' => 'names', - 'wildcard' => true, - ), - array( - 'name' => 'from-engine', - 'param' => 'engine', - 'help' => pht('Migrate files from the named storage engine.'), - ), - array( - 'name' => 'local-disk-source', - 'param' => 'path', - 'help' => pht( - 'When migrating from a local disk source, use the specified '. - 'path as the root directory.'), - ), - )); + ->setArguments($arguments); } public function execute(PhutilArgumentParser $args) { // See T13306. This flag allows you to import files from a backup of // local disk storage into some other engine. When the caller provides // the flag, we override the local disk engine configuration and treat // it as though it is configured to use the specified location. $local_disk_source = $args->getArg('local-disk-source'); if (strlen($local_disk_source)) { $path = Filesystem::resolvePath($local_disk_source); try { Filesystem::assertIsDirectory($path); } catch (FilesystemException $ex) { throw new PhutilArgumentUsageException( pht( 'The "--local-disk-source" argument must point to a valid, '. 'readable directory on local disk.')); } $env = PhabricatorEnv::beginScopedEnv(); $env->overrideEnvConfig('storage.local-disk.path', $path); } $target_key = $args->getArg('engine'); if (!$target_key) { throw new PhutilArgumentUsageException( pht( 'Specify an engine to migrate to with `%s`. '. 'Use `%s` to get a list of engines.', '--engine', 'files engines')); } $target_engine = PhabricatorFile::buildEngine($target_key); $iterator = $this->buildIterator($args); - if (!$iterator) { - throw new PhutilArgumentUsageException( - pht( - 'Either specify a list of files to migrate, or use `%s` '. - 'to migrate all files.', - '--all')); - } - $is_dry_run = $args->getArg('dry-run'); $min_size = (int)$args->getArg('min-size'); $max_size = (int)$args->getArg('max-size'); $is_copy = $args->getArg('copy'); $failed = array(); $engines = PhabricatorFileStorageEngine::loadAllEngines(); $total_bytes = 0; $total_files = 0; foreach ($iterator as $file) { $monogram = $file->getMonogram(); // See T7148. When we export data for an instance, we copy all the data // for Files from S3 into the database dump so that the database dump is // a complete, standalone archive of all the data. In the general case, // installs may have a similar process using "--copy" to create a more // complete backup. // When doing this, we may run into temporary files which have been // deleted between the time we took the original dump and the current // timestamp. These files can't be copied since the data no longer // exists: the daemons on the live install already deleted it. // Simply avoid this whole mess by declining to migrate expired temporary // files. They're as good as dead anyway. $ttl = $file->getTTL(); if ($ttl) { if ($ttl < PhabricatorTime::getNow()) { echo tsprintf( "%s\n", pht( '%s: Skipping expired temporary file.', $monogram)); continue; } } $engine_key = $file->getStorageEngine(); $engine = idx($engines, $engine_key); if (!$engine) { echo tsprintf( "%s\n", pht( '%s: Uses unknown storage engine "%s".', $monogram, $engine_key)); $failed[] = $file; continue; } if ($engine->isChunkEngine()) { echo tsprintf( "%s\n", pht( '%s: Stored as chunks, no data to migrate directly.', $monogram)); continue; } if ($engine_key === $target_key) { echo tsprintf( "%s\n", pht( '%s: Already stored in engine "%s".', $monogram, $target_key)); continue; } $byte_size = $file->getByteSize(); if ($min_size && ($byte_size < $min_size)) { echo tsprintf( "%s\n", pht( '%s: File size (%s) is smaller than minimum size (%s).', $monogram, phutil_format_bytes($byte_size), phutil_format_bytes($min_size))); continue; } if ($max_size && ($byte_size > $max_size)) { echo tsprintf( "%s\n", pht( '%s: File size (%s) is larger than maximum size (%s).', $monogram, phutil_format_bytes($byte_size), phutil_format_bytes($max_size))); continue; } if ($is_dry_run) { echo tsprintf( "%s\n", pht( '%s: (%s) Would migrate from "%s" to "%s" (dry run)...', $monogram, phutil_format_bytes($byte_size), $engine_key, $target_key)); } else { echo tsprintf( "%s\n", pht( '%s: (%s) Migrating from "%s" to "%s"...', $monogram, phutil_format_bytes($byte_size), $engine_key, $target_key)); } try { if ($is_dry_run) { // Do nothing, this is a dry run. } else { $file->migrateToEngine($target_engine, $is_copy); } $total_files += 1; $total_bytes += $byte_size; echo tsprintf( "%s\n", pht('Done.')); } catch (Exception $ex) { echo tsprintf( "%s\n", pht('Failed! %s', (string)$ex)); $failed[] = $file; throw $ex; } } echo tsprintf( "%s\n", pht( 'Total Migrated Files: %s', new PhutilNumber($total_files))); echo tsprintf( "%s\n", pht( 'Total Migrated Bytes: %s', phutil_format_bytes($total_bytes))); if ($is_dry_run) { echo tsprintf( "%s\n", pht( 'This was a dry run, so no real migrations were performed.')); } if ($failed) { $monograms = mpull($failed, 'getMonogram'); echo tsprintf( "%s\n", pht('Failures: %s.', implode(', ', $monograms))); return 1; } return 0; } } diff --git a/src/applications/files/management/PhabricatorFilesManagementRebuildWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementRebuildWorkflow.php index f7fc890ae4..e577a04d55 100644 --- a/src/applications/files/management/PhabricatorFilesManagementRebuildWorkflow.php +++ b/src/applications/files/management/PhabricatorFilesManagementRebuildWorkflow.php @@ -1,156 +1,144 @@ newIteratorArguments(); + + $arguments[] = array( + 'name' => 'dry-run', + 'help' => pht('Show what would be updated.'), + ); + + $arguments[] = array( + 'name' => 'rebuild-mime', + 'help' => pht('Rebuild MIME information.'), + ); + + $arguments[] = array( + 'name' => 'rebuild-dimensions', + 'help' => pht('Rebuild image dimension information.'), + ); + $this ->setName('rebuild') ->setSynopsis(pht('Rebuild metadata of old files.')) - ->setArguments( - array( - array( - 'name' => 'all', - 'help' => pht('Update all files.'), - ), - array( - 'name' => 'dry-run', - 'help' => pht('Show what would be updated.'), - ), - array( - 'name' => 'rebuild-mime', - 'help' => pht('Rebuild MIME information.'), - ), - array( - 'name' => 'rebuild-dimensions', - 'help' => pht('Rebuild image dimension information.'), - ), - array( - 'name' => 'names', - 'wildcard' => true, - ), - )); + ->setArguments($arguments); } public function execute(PhutilArgumentParser $args) { $console = PhutilConsole::getConsole(); $iterator = $this->buildIterator($args); - if (!$iterator) { - throw new PhutilArgumentUsageException( - pht( - 'Either specify a list of files to update, or use `%s` '. - 'to update all files.', - '--all')); - } $update = array( 'mime' => $args->getArg('rebuild-mime'), 'dimensions' => $args->getArg('rebuild-dimensions'), ); // If the user didn't select anything, rebuild everything. if (!array_filter($update)) { foreach ($update as $key => $ignored) { $update[$key] = true; } } $is_dry_run = $args->getArg('dry-run'); $failed = array(); foreach ($iterator as $file) { $fid = 'F'.$file->getID(); if ($update['mime']) { $tmp = new TempFile(); Filesystem::writeFile($tmp, $file->loadFileData()); $new_type = Filesystem::getMimeType($tmp); if ($new_type == $file->getMimeType()) { $console->writeOut( "%s\n", pht( '%s: Mime type not changed (%s).', $fid, $new_type)); } else { if ($is_dry_run) { $console->writeOut( "%s\n", pht( "%s: Would update Mime type: '%s' -> '%s'.", $fid, $file->getMimeType(), $new_type)); } else { $console->writeOut( "%s\n", pht( "%s: Updating Mime type: '%s' -> '%s'.", $fid, $file->getMimeType(), $new_type)); $file->setMimeType($new_type); $file->save(); } } } if ($update['dimensions']) { if (!$file->isViewableImage()) { $console->writeOut( "%s\n", pht('%s: Not an image file.', $fid)); continue; } $metadata = $file->getMetadata(); $image_width = idx($metadata, PhabricatorFile::METADATA_IMAGE_WIDTH); $image_height = idx($metadata, PhabricatorFile::METADATA_IMAGE_HEIGHT); if ($image_width && $image_height) { $console->writeOut( "%s\n", pht('%s: Image dimensions already exist.', $fid)); continue; } if ($is_dry_run) { $console->writeOut( "%s\n", pht('%s: Would update file dimensions (dry run)', $fid)); continue; } $console->writeOut( pht('%s: Updating metadata... ', $fid)); try { $file->updateDimensions(); $console->writeOut("%s\n", pht('Done.')); } catch (Exception $ex) { $console->writeOut("%s\n", pht('Failed!')); $console->writeErr("%s\n", (string)$ex); $failed[] = $file; } } } if ($failed) { $console->writeOut("**%s**\n", pht('Failures!')); $ids = array(); foreach ($failed as $file) { $ids[] = 'F'.$file->getID(); } $console->writeOut("%s\n", implode(', ', $ids)); return 1; } else { $console->writeOut("**%s**\n", pht('Success!')); return 0; } return 0; } } diff --git a/src/applications/files/management/PhabricatorFilesManagementWorkflow.php b/src/applications/files/management/PhabricatorFilesManagementWorkflow.php index 44d43dc66a..6debf9c711 100644 --- a/src/applications/files/management/PhabricatorFilesManagementWorkflow.php +++ b/src/applications/files/management/PhabricatorFilesManagementWorkflow.php @@ -1,72 +1,92 @@ 'all', + 'help' => pht('Operate on all files.'), + ), + array( + 'name' => 'names', + 'wildcard' => true, + ), + array( + 'name' => 'from-engine', + 'param' => 'storage-engine', + 'help' => pht('Operate on files stored in a specified engine.'), + ), + ); + } + protected function buildIterator(PhutilArgumentParser $args) { $viewer = $this->getViewer(); - $names = $args->getArg('names'); $is_all = $args->getArg('all'); + + $names = $args->getArg('names'); $from_engine = $args->getArg('from-engine'); $any_constraint = ($from_engine || $names); if (!$is_all && !$any_constraint) { throw new PhutilArgumentUsageException( pht( - 'Use "--all" to migrate all files, or choose files to migrate '. - 'with "--names" or "--from-engine".')); + 'Specify which files to operate on, or use "--all" to operate on '. + 'all files.')); } if ($is_all && $any_constraint) { throw new PhutilArgumentUsageException( pht( - 'You can not migrate all files with "--all" and also migrate only '. - 'a subset of files with "--from-engine" or "--names".')); + 'You can not operate on all files with "--all" and also operate '. + 'on a subset of files by naming them explicitly or using '. + 'constraint flags like "--from-engine".')); } // If we're migrating specific named files, convert the names into IDs // first. $ids = null; if ($names) { $files = $this->loadFilesWithNames($names); $ids = mpull($files, 'getID'); } $query = id(new PhabricatorFileQuery()) ->setViewer($viewer); if ($ids) { $query->withIDs($ids); } if ($from_engine) { $query->withStorageEngines(array($from_engine)); } return new PhabricatorQueryIterator($query); } protected function loadFilesWithNames(array $names) { $query = id(new PhabricatorObjectQuery()) ->setViewer($this->getViewer()) ->withNames($names) ->withTypes(array(PhabricatorFileFilePHIDType::TYPECONST)); $query->execute(); $files = $query->getNamedResults(); foreach ($names as $name) { if (empty($files[$name])) { throw new PhutilArgumentUsageException( pht( 'No file "%s" exists.', $name)); } } return array_values($files); } }