diff --git a/scripts/repository/reparse.php b/scripts/repository/reparse.php deleted file mode 100755 --- a/scripts/repository/reparse.php +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/env php -setSynopsis(<<parseStandardArguments(); -$args->parse( - array( - // what - array( - 'name' => 'revision', - 'wildcard' => true, - ), - array( - 'name' => 'all', - 'param' => 'callsign or phid', - 'help' => 'Reparse all commits in the specified repository. This '. - 'mode queues parsers into the task queue; you must run '. - 'taskmasters to actually do the parses. Use with '. - '__--force-local__ to run the tasks locally instead of '. - 'with taskmasters.', - ), - array( - 'name' => 'min-date', - 'param' => 'date', - 'help' => 'Must be used with __--all__, this will exclude commits '. - 'which are earlier than __date__.'. - "\n".$min_date_usage_examples, - ), - // which parts - array( - 'name' => 'message', - 'help' => 'Reparse commit messages.', - ), - array( - 'name' => 'change', - 'help' => 'Reparse changes.', - ), - array( - 'name' => 'herald', - 'help' => 'Reevaluate Herald rules (may send huge amounts of email!)', - ), - array( - 'name' => 'owners', - 'help' => 'Reevaluate related commits for owners packages (may '. - 'delete existing relationship entries between your '. - 'package and some old commits!)', - ), - array( - 'name' => 'harbormaster', - 'help' => 'EXPERIMENTAL. Execute Harbormaster.', - ), - // misc options - array( - 'name' => 'force', - 'short' => 'f', - 'help' => 'Act noninteractively, without prompting.', - ), - array( - 'name' => 'force-local', - 'help' => 'Only used with __--all__, use this to run the tasks '. - 'locally instead of deferring them to taskmaster daemons.', - ), - )); - -$all_from_repo = $args->getArg('all'); -$reparse_message = $args->getArg('message'); -$reparse_change = $args->getArg('change'); -$reparse_herald = $args->getArg('herald'); -$reparse_owners = $args->getArg('owners'); -$reparse_harbormaster = $args->getArg('harbormaster'); -$reparse_what = $args->getArg('revision'); -$force = $args->getArg('force'); -$force_local = $args->getArg('force-local'); -$min_date = $args->getArg('min-date'); - -if (!$all_from_repo && !$reparse_what) { - usage('Specify a commit or repository to reparse.'); -} - -if ($all_from_repo && $reparse_what) { - $commits = implode(', ', $reparse_what); - usage( - "Specify a commit or repository to reparse, not both:\n". - "All from repo: ".$all_from_repo."\n". - "Commit(s) to reparse: ".$commits); -} - -if (!$reparse_message && !$reparse_change && !$reparse_herald && - !$reparse_owners && !$reparse_harbormaster) { - usage('Specify what information to reparse with --message, --change, '. - '--herald, --harbormaster, and/or --owners'); -} - -$min_timestamp = false; -if ($min_date) { - $min_timestamp = strtotime($min_date); - - if (!$all_from_repo) { - usage( - "You must use --all if you specify --min-date\n". - "e.g.\n". - " ./reparse.php --all TEST --owners --min-date yesterday"); - } - - // previous to PHP 5.1.0 you would compare with -1, instead of false - if (false === $min_timestamp) { - usage( - "Supplied --min-date is not valid\n". - "Supplied value: '".$min_date."'\n". - $min_date_usage_examples); - } -} - -if ($reparse_owners && !$force) { - echo phutil_console_wrap( - 'You are about to recreate the relationship entries between the commits '. - 'and the packages they touch. This might delete some existing '. - 'relationship entries for some old commits.'); - - if (!phutil_console_confirm('Are you ready to continue?')) { - echo "Cancelled.\n"; - exit(1); - } -} - -$commits = array(); -if ($all_from_repo) { - $repository = id(new PhabricatorRepository())->loadOneWhere( - 'callsign = %s OR phid = %s', - $all_from_repo, - $all_from_repo); - if (!$repository) { - throw new Exception("Unknown repository {$all_from_repo}!"); - } - $constraint = ''; - if ($min_timestamp) { - echo "Excluding entries before UNIX timestamp: ".$min_timestamp."\n"; - $table = new PhabricatorRepositoryCommit(); - $conn_r = $table->establishConnection('r'); - $constraint = qsprintf( - $conn_r, - 'AND epoch >= %d', - $min_timestamp); - } - $commits = id(new PhabricatorRepositoryCommit())->loadAllWhere( - 'repositoryID = %d %Q', - $repository->getID(), - $constraint); - $callsign = $repository->getCallsign(); - if (!$commits) { - echo "No commits have been discovered in {$callsign} repository!\n"; - exit; - } -} else { - $commits = array(); - foreach ($reparse_what as $identifier) { - $matches = null; - if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $identifier, $matches)) { - throw new Exception("Can't parse commit identifier!"); - } - $callsign = $matches[1]; - $commit_identifier = $matches[2]; - $repository = id(new PhabricatorRepository())->loadOneWhere( - 'callsign = %s', - $callsign); - if (!$repository) { - throw new Exception("No repository with callsign '{$callsign}'!"); - } - $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere( - 'repositoryID = %d AND commitIdentifier = %s', - $repository->getID(), - $commit_identifier); - if (!$commit) { - throw new Exception( - "No matching commit '{$commit_identifier}' in repository ". - "'{$callsign}'. (For git and mercurial repositories, you must specify ". - "the entire commit hash.)"); - } - $commits[] = $commit; - } -} - -if ($all_from_repo && !$force_local) { - echo phutil_console_format( - '**NOTE**: This script will queue tasks to reparse the data. Once the '. - 'tasks have been queued, you need to run Taskmaster daemons to execute '. - 'them.'); - echo "\n\n"; - echo "QUEUEING TASKS (".number_format(count($commits))." Commits):\n"; -} - -$progress = new PhutilConsoleProgressBar(); -$progress->setTotal(count($commits)); - -$tasks = array(); -foreach ($commits as $commit) { - $classes = array(); - switch ($repository->getVersionControlSystem()) { - case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: - if ($reparse_message) { - $classes[] = 'PhabricatorRepositoryGitCommitMessageParserWorker'; - } - if ($reparse_change) { - $classes[] = 'PhabricatorRepositoryGitCommitChangeParserWorker'; - } - break; - case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: - if ($reparse_message) { - $classes[] = 'PhabricatorRepositoryMercurialCommitMessageParserWorker'; - } - if ($reparse_change) { - $classes[] = 'PhabricatorRepositoryMercurialCommitChangeParserWorker'; - } - break; - case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: - if ($reparse_message) { - $classes[] = 'PhabricatorRepositorySvnCommitMessageParserWorker'; - } - if ($reparse_change) { - $classes[] = 'PhabricatorRepositorySvnCommitChangeParserWorker'; - } - break; - } - - if ($reparse_herald) { - $classes[] = 'PhabricatorRepositoryCommitHeraldWorker'; - } - - if ($reparse_owners) { - $classes[] = 'PhabricatorRepositoryCommitOwnersWorker'; - } - - if ($reparse_harbormaster) { - $classes[] = 'HarbormasterRunnerWorker'; - } - - $spec = array( - 'commitID' => $commit->getID(), - 'only' => true, - ); - - if ($all_from_repo && !$force_local) { - foreach ($classes as $class) { - PhabricatorWorker::scheduleTask( - $class, - $spec, - array( - 'priority' => PhabricatorWorker::PRIORITY_IMPORT, - )); - } - } else { - foreach ($classes as $class) { - $worker = newv($class, array($spec)); - $worker->executeTask(); - } - } - - $progress->update(1); -} - -$progress->done(); - -function usage($message) { - echo phutil_console_format( - '**Usage Exception:** '.$message."\n". - "Use __--help__ to display full help\n"); - exit(1); -} diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2284,6 +2284,7 @@ 'PhabricatorRepositoryManagementParentsWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementParentsWorkflow.php', 'PhabricatorRepositoryManagementPullWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementPullWorkflow.php', 'PhabricatorRepositoryManagementRefsWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementRefsWorkflow.php', + 'PhabricatorRepositoryManagementReparseWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementReparseWorkflow.php', 'PhabricatorRepositoryManagementUpdateWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementUpdateWorkflow.php', 'PhabricatorRepositoryManagementWorkflow' => 'applications/repository/management/PhabricatorRepositoryManagementWorkflow.php', 'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'applications/repository/worker/commitchangeparser/PhabricatorRepositoryMercurialCommitChangeParserWorker.php', @@ -5524,6 +5525,7 @@ 'PhabricatorRepositoryManagementParentsWorkflow' => 'PhabricatorRepositoryManagementWorkflow', 'PhabricatorRepositoryManagementPullWorkflow' => 'PhabricatorRepositoryManagementWorkflow', 'PhabricatorRepositoryManagementRefsWorkflow' => 'PhabricatorRepositoryManagementWorkflow', + 'PhabricatorRepositoryManagementReparseWorkflow' => 'PhabricatorRepositoryManagementWorkflow', 'PhabricatorRepositoryManagementUpdateWorkflow' => 'PhabricatorRepositoryManagementWorkflow', 'PhabricatorRepositoryManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorRepositoryMercurialCommitChangeParserWorker' => 'PhabricatorRepositoryCommitChangeParserWorker', diff --git a/src/applications/diffusion/controller/DiffusionCommitEditController.php b/src/applications/diffusion/controller/DiffusionCommitEditController.php --- a/src/applications/diffusion/controller/DiffusionCommitEditController.php +++ b/src/applications/diffusion/controller/DiffusionCommitEditController.php @@ -67,6 +67,7 @@ ->setDatasource(new PhabricatorProjectDatasource())); $reason = $data->getCommitDetail('autocloseReason', false); + $reason = PhabricatorRepository::BECAUSE_AUTOCLOSE_FORCED; if ($reason !== false) { switch ($reason) { case PhabricatorRepository::BECAUSE_REPOSITORY_IMPORTING: @@ -78,6 +79,9 @@ case PhabricatorRepository::BECAUSE_NOT_ON_AUTOCLOSE_BRANCH: $desc = pht('No, Not On Autoclose Branch'); break; + case PhabricatorRepository::BECAUSE_AUTOCLOSE_FORCED: + $desc = pht('Yes, Forced Via bin/repository CLI Tool.'); + break; case null: $desc = pht('Yes'); break; diff --git a/src/applications/repository/management/PhabricatorRepositoryManagementReparseWorkflow.php b/src/applications/repository/management/PhabricatorRepositoryManagementReparseWorkflow.php new file mode 100644 --- /dev/null +++ b/src/applications/repository/management/PhabricatorRepositoryManagementReparseWorkflow.php @@ -0,0 +1,309 @@ +setName('reparse') + ->setExamples('**reparse** [options] __repository__') + ->setSynopsis(pht( + '**reparse** __what__ __which_parts__ [--trace] [--force]'."\n\n". + 'Rerun the Diffusion parser on specific commits and repositories. '. + 'Mostly useful for debugging changes to Diffusion.'."\n\n". + 'e.g. enqueue reparse owners in the TEST repo for all commits:'."\n". + 'repository reparse --all TEST --owners'."\n\n". + 'e.g. do same but exclude before yesterday (local time):'."\n". + 'repository reparse --all TEST --owners --min-date yesterday'."\n". + 'repository reparse --all TEST --owners --min-date "today -1 day".'. + "\n\n". + 'e.g. do same but exclude before 03/31/2013 (local time):'."\n". + 'repository reparse --all TEST --owners --min-date "03/31/2013"')) + ->setArguments( + array( + array( + 'name' => 'revision', + 'wildcard' => true, + ), + array( + 'name' => 'all', + 'param' => 'callsign or phid', + 'help' => pht( + 'Reparse all commits in the specified repository. This mode '. + 'queues parsers into the task queue; you must run taskmasters '. + 'to actually do the parses. Use with __--force-local__ to run '. + 'the tasks locally instead of with taskmasters.'), + ), + array( + 'name' => 'min-date', + 'param' => 'date', + 'help' => pht( + 'Must be used with __--all__, this will exclude commits which '. + 'are earlier than __date__.'."\n". + "Valid examples:\n". + " 'today', 'today 2pm', '-1 hour', '-2 hours', '-24 hours',\n". + " 'yesterday', 'today -1 day', 'yesterday 2pm', '2pm -1 day',\n". + " 'last Monday', 'last Monday 14:00', 'last Monday 2pm',\n". + " '31 March 2013', '31 Mar', '03/31', '03/31/2013',\n". + 'See __http://www.php.net/manual/en/datetime.formats.php__ for '. + 'more.'), + ), + array( + 'name' => 'message', + 'help' => pht('Reparse commit messages.'), + ), + array( + 'name' => 'change', + 'help' => pht('Reparse changes.'), + ), + array( + 'name' => 'herald', + 'help' => pht( + 'Reevaluate Herald rules (may send huge amounts of email!)'), + ), + array( + 'name' => 'owners', + 'help' => pht( + 'Reevaluate related commits for owners packages (may delete '. + 'existing relationship entries between your package and some '. + 'old commits!)'), + ), + array( + 'name' => 'force', + 'short' => 'f', + 'help' => pht('Act noninteractively, without prompting.'), + ), + array( + 'name' => 'force-local', + 'help' => pht( + 'Only used with __--all__, use this to run the tasks locally '. + 'instead of deferring them to taskmaster daemons.'), + ), + array( + 'name' => 'force-autoclose', + 'help' => pht( + 'Only used with __--message__, use this to make sure any '. + 'pertinent diffs are closed regardless of configuration.'), + ), + )); + + } + + public function execute(PhutilArgumentParser $args) { + $console = PhutilConsole::getConsole(); + + $all_from_repo = $args->getArg('all'); + $reparse_message = $args->getArg('message'); + $reparse_change = $args->getArg('change'); + $reparse_herald = $args->getArg('herald'); + $reparse_owners = $args->getArg('owners'); + $reparse_what = $args->getArg('revision'); + $force = $args->getArg('force'); + $force_local = $args->getArg('force-local'); + $min_date = $args->getArg('min-date'); + + if (!$all_from_repo && !$reparse_what) { + throw new PhutilArgumentUsageException( + pht('Specify a commit or repository to reparse.')); + } + + if ($all_from_repo && $reparse_what) { + $commits = implode(', ', $reparse_what); + throw new PhutilArgumentUsageException( + pht( + "Specify a commit or repository to reparse, not both:\n". + "All from repo: %s\n". + "Commit(s) to reparse: %s", + $all_from_repo, + $commits)); + } + + if (!$reparse_message && !$reparse_change && !$reparse_herald && + !$reparse_owners) { + throw new PhutilArgumentUsageException( + pht('Specify what information to reparse with --message, --change, '. + '--herald, and/or --owners')); + } + + $min_timestamp = false; + if ($min_date) { + $min_timestamp = strtotime($min_date); + + if (!$all_from_repo) { + throw new PhutilArgumentUsageException( + pht( + "You must use --all if you specify --min-date\n". + "e.g.\n". + " repository reparse --all TEST --owners --min-date yesterday")); + } + + // previous to PHP 5.1.0 you would compare with -1, instead of false + if (false === $min_timestamp) { + throw new PhutilArgumentUsageException( + pht( + "Supplied --min-date is not valid. See help for valid examples.\n". + "Supplied value: '%s'\n", + $min_date)); + } + } + + if ($reparse_owners && !$force) { + $console->writeOut("%s\n", pht( + 'You are about to recreate the relationship entries between the '. + 'commits and the packages they touch. This might delete some existing '. + 'relationship entries for some old commits.')); + + if (!phutil_console_confirm('Are you ready to continue?')) { + throw new PhutilArgumentUsageException(pht('Cancelled.')); + } + } + + $commits = array(); + if ($all_from_repo) { + $repository = id(new PhabricatorRepository())->loadOneWhere( + 'callsign = %s OR phid = %s', + $all_from_repo, + $all_from_repo); + if (!$repository) { + throw new PhutilArgumentUsageException( + pht('Unknown repository %s!', $all_from_repo)); + } + $constraint = ''; + if ($min_timestamp) { + $console->writeOut("%s\n", pht( + 'Excluding entries before UNIX timestamp: %s', + $min_timestamp)); + $table = new PhabricatorRepositoryCommit(); + $conn_r = $table->establishConnection('r'); + $constraint = qsprintf( + $conn_r, + 'AND epoch >= %d', + $min_timestamp); + } + $commits = id(new PhabricatorRepositoryCommit())->loadAllWhere( + 'repositoryID = %d %Q', + $repository->getID(), + $constraint); + $callsign = $repository->getCallsign(); + if (!$commits) { + throw new PhutilArgumentUsageException(pht( + "No commits have been discovered in %s repository!\n", + $callsign)); + } + } else { + $commits = array(); + foreach ($reparse_what as $identifier) { + $matches = null; + if (!preg_match('/r([A-Z]+)([a-z0-9]+)/', $identifier, $matches)) { + throw new PhutilArgumentUsageException(pht( + "Can't parse commit identifier: %s", + $identifier)); + } + $callsign = $matches[1]; + $commit_identifier = $matches[2]; + $repository = id(new PhabricatorRepository())->loadOneWhere( + 'callsign = %s', + $callsign); + if (!$repository) { + throw new PhutilArgumentUsageException(pht( + "No repository with callsign '%s'!", + $callsign)); + } + $commit = id(new PhabricatorRepositoryCommit())->loadOneWhere( + 'repositoryID = %d AND commitIdentifier = %s', + $repository->getID(), + $commit_identifier); + if (!$commit) { + throw new PhutilArgumentUsageException(pht( + "No matching commit '%s' in repository '%s'. ". + "(For git and mercurial repositories, you must specify the entire ". + "commit hash.)", + $commit_identifier, + $callsign)); + } + $commits[] = $commit; + } + } + + if ($all_from_repo && !$force_local) { + $console->writeOut("%s\n", pht( + '**NOTE**: This script will queue tasks to reparse the data. Once the '. + 'tasks have been queued, you need to run Taskmaster daemons to '. + 'execute them.'."\n\n". + "QUEUEING TASKS (%d Commits):", + number_format(count($commits)))); + } + + $progress = new PhutilConsoleProgressBar(); + $progress->setTotal(count($commits)); + + $tasks = array(); + foreach ($commits as $commit) { + $classes = array(); + switch ($repository->getVersionControlSystem()) { + case PhabricatorRepositoryType::REPOSITORY_TYPE_GIT: + if ($reparse_message) { + $classes[] = 'PhabricatorRepositoryGitCommitMessageParserWorker'; + } + if ($reparse_change) { + $classes[] = 'PhabricatorRepositoryGitCommitChangeParserWorker'; + } + break; + case PhabricatorRepositoryType::REPOSITORY_TYPE_MERCURIAL: + if ($reparse_message) { + $classes[] = + 'PhabricatorRepositoryMercurialCommitMessageParserWorker'; + } + if ($reparse_change) { + $classes[] = 'PhabricatorRepositoryMercurialCommitChangeParserWorker'; + } + break; + case PhabricatorRepositoryType::REPOSITORY_TYPE_SVN: + if ($reparse_message) { + $classes[] = 'PhabricatorRepositorySvnCommitMessageParserWorker'; + } + if ($reparse_change) { + $classes[] = 'PhabricatorRepositorySvnCommitChangeParserWorker'; + } + break; + } + + if ($reparse_herald) { + $classes[] = 'PhabricatorRepositoryCommitHeraldWorker'; + } + + if ($reparse_owners) { + $classes[] = 'PhabricatorRepositoryCommitOwnersWorker'; + } + + $spec = array( + 'commitID' => $commit->getID(), + 'only' => true, + 'forceAutoclose' => $args->getArg('force-autoclose'), + ); + + if ($all_from_repo && !$force_local) { + foreach ($classes as $class) { + PhabricatorWorker::scheduleTask( + $class, + $spec, + array( + 'priority' => PhabricatorWorker::PRIORITY_IMPORT, + )); + } + } else { + foreach ($classes as $class) { + $worker = newv($class, array($spec)); + $worker->executeTask(); + } + } + + $progress->update(1); + } + + $progress->done(); + + return 0; + } + +} diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -41,6 +41,7 @@ const BECAUSE_NOT_ON_AUTOCLOSE_BRANCH = 'auto/nobranch'; const BECAUSE_BRANCH_UNTRACKED = 'auto/notrack'; const BECAUSE_BRANCH_NOT_AUTOCLOSE = 'auto/noclose'; + const BECAUSE_AUTOCLOSE_FORCED = 'auto/forced'; protected $name; protected $callsign; diff --git a/src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php b/src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php --- a/src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php +++ b/src/applications/repository/worker/commitmessageparser/PhabricatorRepositoryCommitMessageParserWorker.php @@ -84,9 +84,15 @@ // aren't. Autoclose can be disabled for various reasons at the repository // or commit levels. - $autoclose_reason = $repository->shouldSkipAutocloseCommit($commit); + $force_autoclose = idx($this->getTaskData(), 'forceAutoclose', false); + if ($force_autoclose) { + $autoclose_reason = $repository::BECAUSE_AUTOCLOSE_FORCED; + } else { + $autoclose_reason = $repository->shouldSkipAutocloseCommit($commit); + } $data->setCommitDetail('autocloseReason', $autoclose_reason); - $should_autoclose = $repository->shouldAutocloseCommit($commit); + $should_autoclose = $force_autoclose || + $repository->shouldAutocloseCommit($commit); // When updating related objects, we'll act under an omnipotent user to