diff --git a/bin/garbage b/bin/garbage new file mode 120000 --- /dev/null +++ b/bin/garbage @@ -0,0 +1 @@ +../scripts/setup/manage_garbage.php \ No newline at end of file diff --git a/scripts/setup/manage_garbage.php b/scripts/setup/manage_garbage.php new file mode 100755 --- /dev/null +++ b/scripts/setup/manage_garbage.php @@ -0,0 +1,21 @@ +#!/usr/bin/env php +setTagline(pht('manage garbage colletors')); +$args->setSynopsis(<<parseStandardArguments(); + +$workflows = id(new PhutilClassMapQuery()) + ->setAncestorClass('PhabricatorGarbageCollectorManagementWorkflow') + ->execute(); +$workflows[] = new PhutilHelpArgumentWorkflow(); +$args->parseWorkflows($workflows); diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2197,7 +2197,9 @@ 'PhabricatorFundApplication' => 'applications/fund/application/PhabricatorFundApplication.php', 'PhabricatorGDSetupCheck' => 'applications/config/check/PhabricatorGDSetupCheck.php', 'PhabricatorGarbageCollector' => 'infrastructure/daemon/garbagecollector/PhabricatorGarbageCollector.php', - 'PhabricatorGarbageCollectorConfigOptions' => 'applications/config/option/PhabricatorGarbageCollectorConfigOptions.php', + 'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCollectWorkflow.php', + 'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementSetPolicyWorkflow.php', + 'PhabricatorGarbageCollectorManagementWorkflow' => 'infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementWorkflow.php', 'PhabricatorGestureUIExample' => 'applications/uiexample/examples/PhabricatorGestureUIExample.php', 'PhabricatorGitGraphStream' => 'applications/repository/daemon/PhabricatorGitGraphStream.php', 'PhabricatorGitHubAuthProvider' => 'applications/auth/provider/PhabricatorGitHubAuthProvider.php', @@ -6197,7 +6199,9 @@ 'PhabricatorFundApplication' => 'PhabricatorApplication', 'PhabricatorGDSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorGarbageCollector' => 'Phobject', - 'PhabricatorGarbageCollectorConfigOptions' => 'PhabricatorApplicationConfigOptions', + 'PhabricatorGarbageCollectorManagementCollectWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow', + 'PhabricatorGarbageCollectorManagementSetPolicyWorkflow' => 'PhabricatorGarbageCollectorManagementWorkflow', + 'PhabricatorGarbageCollectorManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorGestureUIExample' => 'PhabricatorUIExample', 'PhabricatorGitGraphStream' => 'PhabricatorRepositoryGraphStream', 'PhabricatorGitHubAuthProvider' => 'PhabricatorOAuth2AuthProvider', diff --git a/src/applications/auth/garbagecollector/PhabricatorAuthSessionGarbageCollector.php b/src/applications/auth/garbagecollector/PhabricatorAuthSessionGarbageCollector.php --- a/src/applications/auth/garbagecollector/PhabricatorAuthSessionGarbageCollector.php +++ b/src/applications/auth/garbagecollector/PhabricatorAuthSessionGarbageCollector.php @@ -6,14 +6,14 @@ const COLLECTORCONST = 'auth.sessions'; public function getCollectorName() { - return pht('Auth Sessions'); + return pht('Authentication Sessions'); } public function hasAutomaticPolicy() { return true; } - public function collectGarbage() { + protected function collectGarbage() { $session_table = new PhabricatorAuthSession(); $conn_w = $session_table->establishConnection('w'); diff --git a/src/applications/auth/garbagecollector/PhabricatorAuthTemporaryTokenGarbageCollector.php b/src/applications/auth/garbagecollector/PhabricatorAuthTemporaryTokenGarbageCollector.php --- a/src/applications/auth/garbagecollector/PhabricatorAuthTemporaryTokenGarbageCollector.php +++ b/src/applications/auth/garbagecollector/PhabricatorAuthTemporaryTokenGarbageCollector.php @@ -6,14 +6,14 @@ const COLLECTORCONST = 'auth.tokens'; public function getCollectorName() { - return pht('Auth Tokens'); + return pht('Authentication Tokens'); } public function hasAutomaticPolicy() { return true; } - public function collectGarbage() { + protected function collectGarbage() { $session_table = new PhabricatorAuthTemporaryToken(); $conn_w = $session_table->establishConnection('w'); diff --git a/src/applications/cache/garbagecollector/PhabricatorCacheGeneralGarbageCollector.php b/src/applications/cache/garbagecollector/PhabricatorCacheGeneralGarbageCollector.php --- a/src/applications/cache/garbagecollector/PhabricatorCacheGeneralGarbageCollector.php +++ b/src/applications/cache/garbagecollector/PhabricatorCacheGeneralGarbageCollector.php @@ -13,13 +13,7 @@ return phutil_units('30 days in seconds'); } - public function collectGarbage() { - $key = 'gcdaemon.ttl.general-cache'; - $ttl = PhabricatorEnv::getEnvConfig($key); - if ($ttl <= 0) { - return false; - } - + protected function collectGarbage() { $cache = new PhabricatorKeyValueDatabaseCache(); $conn_w = $cache->establishConnection('w'); @@ -28,7 +22,7 @@ 'DELETE FROM %T WHERE cacheCreated < %d ORDER BY cacheCreated ASC LIMIT 100', $cache->getTableName(), - time() - $ttl); + $this->getGarbageEpoch()); return ($conn_w->getAffectedRows() == 100); } diff --git a/src/applications/cache/garbagecollector/PhabricatorCacheMarkupGarbageCollector.php b/src/applications/cache/garbagecollector/PhabricatorCacheMarkupGarbageCollector.php --- a/src/applications/cache/garbagecollector/PhabricatorCacheMarkupGarbageCollector.php +++ b/src/applications/cache/garbagecollector/PhabricatorCacheMarkupGarbageCollector.php @@ -13,13 +13,7 @@ return phutil_units('30 days in seconds'); } - public function collectGarbage() { - $key = 'gcdaemon.ttl.markup-cache'; - $ttl = PhabricatorEnv::getEnvConfig($key); - if ($ttl <= 0) { - return false; - } - + protected function collectGarbage() { $table = new PhabricatorMarkupCache(); $conn_w = $table->establishConnection('w'); @@ -27,7 +21,7 @@ $conn_w, 'DELETE FROM %T WHERE dateCreated < %d LIMIT 100', $table->getTableName(), - time() - $ttl); + $this->getGarbageEpoch()); return ($conn_w->getAffectedRows() == 100); } diff --git a/src/applications/cache/garbagecollector/PhabricatorCacheTTLGarbageCollector.php b/src/applications/cache/garbagecollector/PhabricatorCacheTTLGarbageCollector.php --- a/src/applications/cache/garbagecollector/PhabricatorCacheTTLGarbageCollector.php +++ b/src/applications/cache/garbagecollector/PhabricatorCacheTTLGarbageCollector.php @@ -13,7 +13,7 @@ return true; } - public function collectGarbage() { + protected function collectGarbage() { $cache = new PhabricatorKeyValueDatabaseCache(); $conn_w = $cache->establishConnection('w'); @@ -22,7 +22,7 @@ 'DELETE FROM %T WHERE cacheExpires < %d ORDER BY cacheExpires ASC LIMIT 100', $cache->getTableName(), - time()); + PhabricatorTime::getNow()); return ($conn_w->getAffectedRows() == 100); } diff --git a/src/applications/conduit/garbagecollector/ConduitConnectionGarbageCollector.php b/src/applications/conduit/garbagecollector/ConduitConnectionGarbageCollector.php --- a/src/applications/conduit/garbagecollector/ConduitConnectionGarbageCollector.php +++ b/src/applications/conduit/garbagecollector/ConduitConnectionGarbageCollector.php @@ -13,21 +13,16 @@ return phutil_units('180 days in seconds'); } - public function collectGarbage() { - $key = 'gcdaemon.ttl.conduit-logs'; - $ttl = PhabricatorEnv::getEnvConfig($key); - if ($ttl <= 0) { - return false; - } - + protected function collectGarbage() { $table = new PhabricatorConduitConnectionLog(); $conn_w = $table->establishConnection('w'); + queryfx( $conn_w, 'DELETE FROM %T WHERE dateCreated < %d ORDER BY dateCreated ASC LIMIT 100', $table->getTableName(), - time() - $ttl); + $this->getGarbageEpoch()); return ($conn_w->getAffectedRows() == 100); } diff --git a/src/applications/conduit/garbagecollector/ConduitLogGarbageCollector.php b/src/applications/conduit/garbagecollector/ConduitLogGarbageCollector.php --- a/src/applications/conduit/garbagecollector/ConduitLogGarbageCollector.php +++ b/src/applications/conduit/garbagecollector/ConduitLogGarbageCollector.php @@ -13,21 +13,16 @@ return phutil_units('180 days in seconds'); } - public function collectGarbage() { - $key = 'gcdaemon.ttl.conduit-logs'; - $ttl = PhabricatorEnv::getEnvConfig($key); - if ($ttl <= 0) { - return false; - } - + protected function collectGarbage() { $table = new PhabricatorConduitMethodCallLog(); $conn_w = $table->establishConnection('w'); + queryfx( $conn_w, 'DELETE FROM %T WHERE dateCreated < %d ORDER BY dateCreated ASC LIMIT 100', $table->getTableName(), - time() - $ttl); + $this->getGarbageEpoch()); return ($conn_w->getAffectedRows() == 100); } diff --git a/src/applications/conduit/garbagecollector/ConduitTokenGarbageCollector.php b/src/applications/conduit/garbagecollector/ConduitTokenGarbageCollector.php --- a/src/applications/conduit/garbagecollector/ConduitTokenGarbageCollector.php +++ b/src/applications/conduit/garbagecollector/ConduitTokenGarbageCollector.php @@ -13,9 +13,10 @@ return true; } - public function collectGarbage() { + protected function collectGarbage() { $table = new PhabricatorConduitToken(); $conn_w = $table->establishConnection('w'); + queryfx( $conn_w, 'DELETE FROM %T WHERE expires <= %d diff --git a/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php b/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php --- a/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php +++ b/src/applications/config/check/PhabricatorExtraConfigSetupCheck.php @@ -176,6 +176,10 @@ 'Inbound mail addresses are now configured for each application '. 'in the Applications tool.'); + $gc_reason = pht( + 'Garbage collectors are now configured with "%s".', + 'bin/garbage set-policy'); + $ancient_config += array( 'phid.external-loaders' => pht( @@ -280,6 +284,14 @@ 'auth.login-message' => pht( 'This configuration option has been replaced with a modular '. 'handler. See T9346.'), + + 'gcdaemon.ttl.herald-transcripts' => $gc_reason, + 'gcdaemon.ttl.daemon-logs' => $gc_reason, + 'gcdaemon.ttl.differential-parse-cache' => $gc_reason, + 'gcdaemon.ttl.markup-cache' => $gc_reason, + 'gcdaemon.ttl.task-archive' => $gc_reason, + 'gcdaemon.ttl.general-cache' => $gc_reason, + 'gcdaemon.ttl.conduit-logs' => $gc_reason, ); return $ancient_config; diff --git a/src/applications/config/module/PhabricatorConfigCollectorsModule.php b/src/applications/config/module/PhabricatorConfigCollectorsModule.php --- a/src/applications/config/module/PhabricatorConfigCollectorsModule.php +++ b/src/applications/config/module/PhabricatorConfigCollectorsModule.php @@ -17,11 +17,13 @@ $collectors = msort($collectors, 'getCollectorConstant'); $rows = array(); + $rowc = array(); foreach ($collectors as $key => $collector) { + $class = null; if ($collector->hasAutomaticPolicy()) { $policy_view = phutil_tag('em', array(), pht('Automatic')); } else { - $policy = $collector->getDefaultRetentionPolicy(); + $policy = $collector->getRetentionPolicy(); if ($policy === null) { $policy_view = pht('Indefinite'); } else { @@ -30,8 +32,15 @@ '%s Day(s)', new PhutilNumber($days)); } + + $default = $collector->getDefaultRetentionPolicy(); + if ($policy !== $default) { + $class = 'highlighted'; + $policy_view = phutil_tag('strong', array(), $policy_view); + } } + $rowc[] = $class; $rows[] = array( $collector->getCollectorConstant(), $collector->getCollectorName(), @@ -40,6 +49,7 @@ } $table = id(new AphrontTableView($rows)) + ->setRowClasses($rowc) ->setHeaders( array( pht('Constant'), @@ -53,8 +63,16 @@ null, )); + $header = id(new PHUIHeaderView()) + ->setHeader(pht('Garbage Collectors')) + ->setSubheader( + pht( + 'Collectors with custom policies are highlighted. Use '. + '%s to change retention policies.', + phutil_tag('tt', array(), 'bin/garbage set-policy'))); + return id(new PHUIObjectBoxView()) - ->setHeaderText(pht('Garbage Collectors')) + ->setHeader($header) ->setTable($table); } diff --git a/src/applications/config/option/PhabricatorGarbageCollectorConfigOptions.php b/src/applications/config/option/PhabricatorGarbageCollectorConfigOptions.php deleted file mode 100644 --- a/src/applications/config/option/PhabricatorGarbageCollectorConfigOptions.php +++ /dev/null @@ -1,70 +0,0 @@ - array( - 30, - pht('Number of seconds to retain Herald transcripts for.'), - ), - 'gcdaemon.ttl.daemon-logs' => array( - 7, - pht('Number of seconds to retain Daemon logs for.'), - ), - 'gcdaemon.ttl.differential-parse-cache' => array( - 14, - pht('Number of seconds to retain Differential parse caches for.'), - ), - 'gcdaemon.ttl.markup-cache' => array( - 30, - pht('Number of seconds to retain Markup cache entries for.'), - ), - 'gcdaemon.ttl.task-archive' => array( - 14, - pht('Number of seconds to retain archived background tasks for.'), - ), - 'gcdaemon.ttl.general-cache' => array( - 30, - pht('Number of seconds to retain general cache entries for.'), - ), - 'gcdaemon.ttl.conduit-logs' => array( - 180, - pht('Number of seconds to retain Conduit call logs for.'), - ), - ); - - $result = array(); - foreach ($options as $key => $spec) { - list($default_days, $description) = $spec; - $result[] = $this - ->newOption($key, 'int', $default_days * (24 * 60 * 60)) - ->setDescription($description) - ->addExample((7 * 24 * 60 * 60), pht('Retain for 1 week')) - ->addExample((14 * 24 * 60 * 60), pht('Retain for 2 weeks')) - ->addExample((30 * 24 * 60 * 60), pht('Retain for 30 days')) - ->addExample((60 * 24 * 60 * 60), pht('Retain for 60 days')) - ->addExample(0, pht('Retain indefinitely')); - } - return $result; - } - -} diff --git a/src/applications/config/option/PhabricatorPHDConfigOptions.php b/src/applications/config/option/PhabricatorPHDConfigOptions.php --- a/src/applications/config/option/PhabricatorPHDConfigOptions.php +++ b/src/applications/config/option/PhabricatorPHDConfigOptions.php @@ -80,6 +80,17 @@ 'and the daemons. Primarily, this is a way to suppress the '. '"Daemons and Web Have Different Config" setup issue on a per '. 'config key basis.')), + $this->newOption('phd.garbage-collection', 'wild', array()) + ->setLocked(true) + ->setLockedMessage( + pht( + 'This option can not be edited from the web UI. Use %s to adjust '. + 'garbage collector policies.', + phutil_tag('tt', array(), 'bin/garbage set-policy'))) + ->setSummary(pht('Retention policies for garbage collection.')) + ->setDescription( + pht( + 'Customizes retention policies for garbage collectors.')), ); } diff --git a/src/applications/daemon/garbagecollector/PhabricatorDaemonLogEventGarbageCollector.php b/src/applications/daemon/garbagecollector/PhabricatorDaemonLogEventGarbageCollector.php --- a/src/applications/daemon/garbagecollector/PhabricatorDaemonLogEventGarbageCollector.php +++ b/src/applications/daemon/garbagecollector/PhabricatorDaemonLogEventGarbageCollector.php @@ -13,12 +13,7 @@ return phutil_units('7 days in seconds'); } - public function collectGarbage() { - $ttl = PhabricatorEnv::getEnvConfig('gcdaemon.ttl.daemon-logs'); - if ($ttl <= 0) { - return false; - } - + protected function collectGarbage() { $table = new PhabricatorDaemonLogEvent(); $conn_w = $table->establishConnection('w'); @@ -26,7 +21,7 @@ $conn_w, 'DELETE FROM %T WHERE epoch < %d LIMIT 100', $table->getTableName(), - time() - $ttl); + $this->getGarbageEpoch()); return ($conn_w->getAffectedRows() == 100); } diff --git a/src/applications/daemon/garbagecollector/PhabricatorDaemonLogGarbageCollector.php b/src/applications/daemon/garbagecollector/PhabricatorDaemonLogGarbageCollector.php --- a/src/applications/daemon/garbagecollector/PhabricatorDaemonLogGarbageCollector.php +++ b/src/applications/daemon/garbagecollector/PhabricatorDaemonLogGarbageCollector.php @@ -13,12 +13,7 @@ return phutil_units('7 days in seconds'); } - public function collectGarbage() { - $ttl = PhabricatorEnv::getEnvConfig('gcdaemon.ttl.daemon-logs'); - if ($ttl <= 0) { - return false; - } - + protected function collectGarbage() { $table = new PhabricatorDaemonLog(); $conn_w = $table->establishConnection('w'); @@ -26,7 +21,7 @@ $conn_w, 'DELETE FROM %T WHERE dateCreated < %d AND status != %s LIMIT 100', $table->getTableName(), - time() - $ttl, + $this->getGarbageEpoch(), PhabricatorDaemonLog::STATUS_RUNNING); return ($conn_w->getAffectedRows() == 100); diff --git a/src/applications/daemon/garbagecollector/PhabricatorDaemonTaskGarbageCollector.php b/src/applications/daemon/garbagecollector/PhabricatorDaemonTaskGarbageCollector.php --- a/src/applications/daemon/garbagecollector/PhabricatorDaemonTaskGarbageCollector.php +++ b/src/applications/daemon/garbagecollector/PhabricatorDaemonTaskGarbageCollector.php @@ -13,21 +13,15 @@ return phutil_units('14 days in seconds'); } - public function collectGarbage() { - $key = 'gcdaemon.ttl.task-archive'; - $ttl = PhabricatorEnv::getEnvConfig($key); - if ($ttl <= 0) { - return false; - } - + protected function collectGarbage() { $table = new PhabricatorWorkerArchiveTask(); $data_table = new PhabricatorWorkerTaskData(); $conn_w = $table->establishConnection('w'); $tasks = id(new PhabricatorWorkerArchiveTaskQuery()) - ->withDateCreatedBefore(time() - $ttl) + ->withDateCreatedBefore($this->getGarbageEpoch()) + ->setLimit(100) ->execute(); - if (!$tasks) { return false; } diff --git a/src/applications/differential/garbagecollector/DifferentialParseCacheGarbageCollector.php b/src/applications/differential/garbagecollector/DifferentialParseCacheGarbageCollector.php --- a/src/applications/differential/garbagecollector/DifferentialParseCacheGarbageCollector.php +++ b/src/applications/differential/garbagecollector/DifferentialParseCacheGarbageCollector.php @@ -13,13 +13,7 @@ return phutil_units('14 days in seconds'); } - public function collectGarbage() { - $key = 'gcdaemon.ttl.differential-parse-cache'; - $ttl = PhabricatorEnv::getEnvConfig($key); - if ($ttl <= 0) { - return false; - } - + protected function collectGarbage() { $table = new DifferentialChangeset(); $conn_w = $table->establishConnection('w'); @@ -27,7 +21,7 @@ $conn_w, 'DELETE FROM %T WHERE dateCreated < %d LIMIT 100', DifferentialChangeset::TABLE_CACHE, - time() - $ttl); + $this->getGarbageEpoch()); return ($conn_w->getAffectedRows() == 100); } diff --git a/src/applications/drydock/garbagecollector/DrydockLogGarbageCollector.php b/src/applications/drydock/garbagecollector/DrydockLogGarbageCollector.php --- a/src/applications/drydock/garbagecollector/DrydockLogGarbageCollector.php +++ b/src/applications/drydock/garbagecollector/DrydockLogGarbageCollector.php @@ -13,18 +13,15 @@ return phutil_units('30 days in seconds'); } - public function collectGarbage() { + protected function collectGarbage() { $log_table = new DrydockLog(); $conn_w = $log_table->establishConnection('w'); - $now = PhabricatorTime::getNow(); - $ttl = phutil_units('30 days in seconds'); - queryfx( $conn_w, 'DELETE FROM %T WHERE epoch <= %d LIMIT 100', $log_table->getTableName(), - $now - $ttl); + $this->getGarbageEpoch()); return ($conn_w->getAffectedRows() == 100); } diff --git a/src/applications/files/garbagecollector/PhabricatorFileTemporaryGarbageCollector.php b/src/applications/files/garbagecollector/PhabricatorFileTemporaryGarbageCollector.php --- a/src/applications/files/garbagecollector/PhabricatorFileTemporaryGarbageCollector.php +++ b/src/applications/files/garbagecollector/PhabricatorFileTemporaryGarbageCollector.php @@ -13,10 +13,10 @@ return true; } - public function collectGarbage() { + protected function collectGarbage() { $files = id(new PhabricatorFile())->loadAllWhere( 'ttl < %d LIMIT 100', - time()); + PhabricatorTime::getNow()); foreach ($files as $file) { $file->delete(); diff --git a/src/applications/herald/garbagecollector/HeraldTranscriptGarbageCollector.php b/src/applications/herald/garbagecollector/HeraldTranscriptGarbageCollector.php --- a/src/applications/herald/garbagecollector/HeraldTranscriptGarbageCollector.php +++ b/src/applications/herald/garbagecollector/HeraldTranscriptGarbageCollector.php @@ -13,12 +13,7 @@ return phutil_units('30 days in seconds'); } - public function collectGarbage() { - $ttl = PhabricatorEnv::getEnvConfig('gcdaemon.ttl.herald-transcripts'); - if ($ttl <= 0) { - return false; - } - + protected function collectGarbage() { $table = new HeraldTranscript(); $conn_w = $table->establishConnection('w'); @@ -33,7 +28,7 @@ WHERE garbageCollected = 0 AND time < %d LIMIT 100', $table->getTableName(), - time() - $ttl); + $this->getGarbageEpoch()); return ($conn_w->getAffectedRows() == 100); } diff --git a/src/applications/metamta/garbagecollector/MetaMTAMailReceivedGarbageCollector.php b/src/applications/metamta/garbagecollector/MetaMTAMailReceivedGarbageCollector.php --- a/src/applications/metamta/garbagecollector/MetaMTAMailReceivedGarbageCollector.php +++ b/src/applications/metamta/garbagecollector/MetaMTAMailReceivedGarbageCollector.php @@ -13,9 +13,7 @@ return phutil_units('90 days in seconds'); } - public function collectGarbage() { - $ttl = phutil_units('90 days in seconds'); - + protected function collectGarbage() { $table = new PhabricatorMetaMTAReceivedMail(); $conn_w = $table->establishConnection('w'); @@ -23,7 +21,7 @@ $conn_w, 'DELETE FROM %T WHERE dateCreated < %d LIMIT 100', $table->getTableName(), - time() - $ttl); + $this->getGarbageEpoch()); return ($conn_w->getAffectedRows() == 100); } diff --git a/src/applications/metamta/garbagecollector/MetaMTAMailSentGarbageCollector.php b/src/applications/metamta/garbagecollector/MetaMTAMailSentGarbageCollector.php --- a/src/applications/metamta/garbagecollector/MetaMTAMailSentGarbageCollector.php +++ b/src/applications/metamta/garbagecollector/MetaMTAMailSentGarbageCollector.php @@ -13,12 +13,10 @@ return phutil_units('90 days in seconds'); } - public function collectGarbage() { - $ttl = phutil_units('90 days in seconds'); - + protected function collectGarbage() { $mails = id(new PhabricatorMetaMTAMail())->loadAllWhere( 'dateCreated < %d LIMIT 100', - PhabricatorTime::getNow() - $ttl); + $this->getGarbageEpoch()); foreach ($mails as $mail) { $mail->delete(); diff --git a/src/applications/multimeter/garbagecollector/MultimeterEventGarbageCollector.php b/src/applications/multimeter/garbagecollector/MultimeterEventGarbageCollector.php --- a/src/applications/multimeter/garbagecollector/MultimeterEventGarbageCollector.php +++ b/src/applications/multimeter/garbagecollector/MultimeterEventGarbageCollector.php @@ -13,9 +13,7 @@ return phutil_units('90 days in seconds'); } - public function collectGarbage() { - $ttl = phutil_units('90 days in seconds'); - + protected function collectGarbage() { $table = new MultimeterEvent(); $conn_w = $table->establishConnection('w'); @@ -23,7 +21,7 @@ $conn_w, 'DELETE FROM %T WHERE epoch < %d LIMIT 100', $table->getTableName(), - PhabricatorTime::getNow() - $ttl); + $this->getGarbageEpoch()); return ($conn_w->getAffectedRows() == 100); } diff --git a/src/applications/notification/garbagecollector/FeedStoryNotificationGarbageCollector.php b/src/applications/notification/garbagecollector/FeedStoryNotificationGarbageCollector.php --- a/src/applications/notification/garbagecollector/FeedStoryNotificationGarbageCollector.php +++ b/src/applications/notification/garbagecollector/FeedStoryNotificationGarbageCollector.php @@ -13,9 +13,7 @@ return phutil_units('90 days in seconds'); } - public function collectGarbage() { - $ttl = 90 * 24 * 60 * 60; - + protected function collectGarbage() { $table = new PhabricatorFeedStoryNotification(); $conn_w = $table->establishConnection('w'); @@ -24,7 +22,7 @@ 'DELETE FROM %T WHERE chronologicalKey < (%d << 32) ORDER BY chronologicalKey ASC LIMIT 100', $table->getTableName(), - time() - $ttl); + $this->getGarbageEpoch()); return ($conn_w->getAffectedRows() == 100); } diff --git a/src/applications/people/garbagecollector/PeopleUserLogGarbageCollector.php b/src/applications/people/garbagecollector/PeopleUserLogGarbageCollector.php --- a/src/applications/people/garbagecollector/PeopleUserLogGarbageCollector.php +++ b/src/applications/people/garbagecollector/PeopleUserLogGarbageCollector.php @@ -13,9 +13,7 @@ return phutil_units('180 days in seconds'); } - public function collectGarbage() { - $ttl = phutil_units('180 days in seconds'); - + protected function collectGarbage() { $table = new PhabricatorUserLog(); $conn_w = $table->establishConnection('w'); @@ -23,7 +21,7 @@ $conn_w, 'DELETE FROM %T WHERE dateCreated < %d LIMIT 100', $table->getTableName(), - time() - $ttl); + $this->getGarbageEpoch()); return ($conn_w->getAffectedRows() == 100); } diff --git a/src/applications/system/garbagecollector/PhabricatorSystemActionGarbageCollector.php b/src/applications/system/garbagecollector/PhabricatorSystemActionGarbageCollector.php --- a/src/applications/system/garbagecollector/PhabricatorSystemActionGarbageCollector.php +++ b/src/applications/system/garbagecollector/PhabricatorSystemActionGarbageCollector.php @@ -13,9 +13,7 @@ return phutil_units('3 days in seconds'); } - public function collectGarbage() { - $ttl = phutil_units('3 days in seconds'); - + protected function collectGarbage() { $table = new PhabricatorSystemActionLog(); $conn_w = $table->establishConnection('w'); @@ -23,7 +21,7 @@ $conn_w, 'DELETE FROM %T WHERE epoch < %d LIMIT 100', $table->getTableName(), - time() - $ttl); + $this->getGarbageEpoch()); return ($conn_w->getAffectedRows() == 100); } diff --git a/src/applications/system/garbagecollector/PhabricatorSystemDestructionGarbageCollector.php b/src/applications/system/garbagecollector/PhabricatorSystemDestructionGarbageCollector.php --- a/src/applications/system/garbagecollector/PhabricatorSystemDestructionGarbageCollector.php +++ b/src/applications/system/garbagecollector/PhabricatorSystemDestructionGarbageCollector.php @@ -13,9 +13,7 @@ return phutil_units('90 days in seconds'); } - public function collectGarbage() { - $ttl = phutil_units('90 days in seconds'); - + protected function collectGarbage() { $table = new PhabricatorSystemDestructionLog(); $conn_w = $table->establishConnection('w'); @@ -23,7 +21,7 @@ $conn_w, 'DELETE FROM %T WHERE epoch < %d LIMIT 100', $table->getTableName(), - time() - $ttl); + $this->getGarbageEpoch()); return ($conn_w->getAffectedRows() == 100); } diff --git a/src/docs/user/configuration/managing_garbage.diviner b/src/docs/user/configuration/managing_garbage.diviner new file mode 100644 --- /dev/null +++ b/src/docs/user/configuration/managing_garbage.diviner @@ -0,0 +1,68 @@ +@title Managing Garbage Collection +@group config + +Understanding and configuring garbage collection. + +Overview +======== + +Phabricator generates various logs and caches during normal operation. Some of +these logs and caches are usually of very little use after some time has +passed, so they are deleted automatically (often after a month or two) in a +process called "garbage collection". + +Garbage collection is performed automatically by the daemons. You can review +all of the installed garbage collectors by browsing to {nav Config > Garbage +Collectors}. + + +Configuring Retention Policies +============================== + +You can reconfigure the data retention policies for most collectors. + +The default retention polcies should be suitable for most installs. However, +you might want to **decrease** retention to reduce the amount of disk space +used by some high-volume log that you don't find particularly interesting, or +to adhere to an organizational data retention policy. + +Alternatively, you might want to **increase** retention if you want to retain +some logs for a longer period of time, perhaps for auditing or analytic +purposes. + +You can review the current retention policies in +{nav Config > Garbage Collectors}. To change a policy, use +`bin/garbage set-policy` to select a new policy: + +``` +phabricator/ $ ./bin/garbage set-policy --collector cache.markup --days 7 +``` + +You can use `--days` to select how long data is retained for. You can also use +`--indefinite` to set an indefinite retention policy. This will stop the +garbage collector from cleaning up any data. Finally, you can use `--default` +to restore the default policy. + +Your changes should be reflected in the web UI immediately, and will take +effect in the actual collector **the next time the daemons are restarted**. + + +Troubleshooting +=============== + +You can manually run a collector with `bin/garbage collect`. + +``` +phabricator/ $ ./bin/garbage collect --collector cache.general +``` + +By using the `--trace` flag, you can inspect the operation of the collector +in detail. + + +Next Steps +========== + +Continue by: + + - exploring other daemon topics with @{article:Managing Daemons with phd}. diff --git a/src/infrastructure/daemon/garbagecollector/PhabricatorGarbageCollector.php b/src/infrastructure/daemon/garbagecollector/PhabricatorGarbageCollector.php --- a/src/infrastructure/daemon/garbagecollector/PhabricatorGarbageCollector.php +++ b/src/infrastructure/daemon/garbagecollector/PhabricatorGarbageCollector.php @@ -47,6 +47,28 @@ /** + * Get the effective retention policy. + * + * @return int|null Lifetime, or `null` for indefinite retention. + * @task info + */ + public function getRetentionPolicy() { + if ($this->hasAutomaticPolicy()) { + throw new Exception( + pht( + 'Can not get retention policy of collector with automatic '. + 'policy.')); + } + + $config = PhabricatorEnv::getEnvConfig('phd.garbage-collection'); + $const = $this->getCollectorConstant(); + + return idx($config, $const, $this->getDefaultRetentionPolicy()); + } + + + + /** * Get a unique string constant identifying this collector. * * @return string Collector constant. @@ -61,12 +83,60 @@ /** + * Run the collector. + * + * @return bool True if there is more garbage to collect. + * @task collect + */ + final public function runCollector() { + // Don't do anything if this collector is configured with an indefinite + // retention policy. + if (!$this->hasAutomaticPolicy()) { + $policy = $this->getRetentionPolicy(); + if (!$policy) { + return false; + } + } + + return $this->collectGarbage(); + } + + + /** * Collect garbage from whatever source this GC handles. * * @return bool True if there is more garbage to collect. * @task collect */ - abstract public function collectGarbage(); + abstract protected function collectGarbage(); + + + /** + * Get the most recent epoch timestamp that is considered garbage. + * + * Records older than this should be collected. + * + * @return int Most recent garbage timestamp. + * @task collect + */ + final protected function getGarbageEpoch() { + if ($this->hasAutomaticPolicy()) { + throw new Exception( + pht( + 'Can not get garbage epoch for a collector with an automatic '. + 'collection policy.')); + } + + $ttl = $this->getRetentionPolicy(); + if (!$ttl) { + throw new Exception( + pht( + 'Can not get garbage epoch for a collector with an indefinite '. + 'retention policy.')); + } + + return (PhabricatorTime::getNow() - $ttl); + } /** diff --git a/src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCollectWorkflow.php b/src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCollectWorkflow.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementCollectWorkflow.php @@ -0,0 +1,50 @@ +setName('collect') + ->setExamples('**collect** --collector __collector__') + ->setSynopsis( + pht('Run a garbage collector in the foreground.')) + ->setArguments( + array( + array( + 'name' => 'collector', + 'param' => 'const', + 'help' => pht( + 'Constant identifying the garbage collector to run.'), + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $collector = $this->getCollector($args->getArg('collector')); + + echo tsprintf( + "%s\n", + pht('Collecting "%s" garbage...', $collector->getCollectorName())); + + $any = false; + while (true) { + $more = $collector->runCollector(); + if ($more) { + $any = true; + } else { + break; + } + } + + if ($any) { + $message = pht('Finished collecting all the garbage.'); + } else { + $message = pht('Could not find any garbage to collect.'); + } + echo tsprintf("\n%s\n", $message); + + return 0; + } + +} diff --git a/src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementSetPolicyWorkflow.php b/src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementSetPolicyWorkflow.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementSetPolicyWorkflow.php @@ -0,0 +1,141 @@ +setName('set-policy') + ->setExamples( + "**set-policy** --collector __collector__ --days 30\n". + "**set-policy** --collector __collector__ --indefinite\n". + "**set-policy** --collector __collector__ --default") + ->setSynopsis( + pht( + 'Change retention policies for a garbage collector.')) + ->setArguments( + array( + array( + 'name' => 'collector', + 'param' => 'const', + 'help' => pht( + 'Constant identifying the garbage collector.'), + ), + array( + 'name' => 'indefinite', + 'help' => pht( + 'Set an indefinite retention policy.'), + ), + array( + 'name' => 'default', + 'help' => pht( + 'Use the default retention policy.'), + ), + array( + 'name' => 'days', + 'param' => 'count', + 'help' => pht( + 'Retain data for the specified number of days.'), + ), + )); + } + + public function execute(PhutilArgumentParser $args) { + $config_key = 'phd.garbage-collection'; + + $collector = $this->getCollector($args->getArg('collector')); + + $days = $args->getArg('days'); + $indefinite = $args->getArg('indefinite'); + $default = $args->getArg('default'); + + $count = 0; + if ($days !== null) { + $count++; + } + if ($indefinite) { + $count++; + } + if ($default) { + $count++; + } + + if (!$count) { + throw new PhutilArgumentUsageException( + pht( + 'Choose a policy with "%s", "%s" or "%s".', + '--days', + '--indefinite', + '--default')); + } + + if ($count > 1) { + throw new PhutilArgumentUsageException( + pht( + 'Options "%s", "%s" and "%s" represent mutually exclusive ways '. + 'to choose a policy. Specify only one.', + '--days', + '--indefinite', + '--default')); + } + + if ($days !== null) { + $days = (int)$days; + if ($days < 1) { + throw new PhutilArgumentUsageException( + pht( + 'Specify a positive number of days to retain data for.')); + } + } + + $collector_const = $collector->getCollectorConstant(); + $value = PhabricatorEnv::getEnvConfig($config_key); + + if ($days !== null) { + echo tsprintf( + "%s\n", + pht( + 'Setting retention policy for "%s" to %s day(s).', + $collector->getCollectorName(), + new PhutilNumber($days))); + + $value[$collector_const] = phutil_units($days.' days in seconds'); + } else if ($indefinite) { + echo tsprintf( + "%s\n", + pht( + 'Setting "%s" to be retained indefinitely.', + $collector->getCollectorName())); + + $value[$collector_const] = null; + } else { + echo tsprintf( + "%s\n", + pht( + 'Restoring "%s" to the default retention policy.', + $collector->getCollectorName())); + + unset($value[$collector_const]); + } + + id(new PhabricatorConfigLocalSource()) + ->setKeys( + array( + $config_key => $value, + )); + + echo tsprintf( + "%s\n", + pht( + 'Wrote new policy to local configuration.')); + + echo tsprintf( + "%s\n", + pht( + 'This change will take effect the next time the daemons are '. + 'restarted.')); + + return 0; + } + +} diff --git a/src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementWorkflow.php b/src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementWorkflow.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/daemon/garbagecollector/management/PhabricatorGarbageCollectorManagementWorkflow.php @@ -0,0 +1,32 @@ +garbageCollectors) { foreach ($this->garbageCollectors as $key => $collector) { - $more_garbage = $collector->collectGarbage(); + $more_garbage = $collector->runCollector(); if (!$more_garbage) { unset($this->garbageCollectors[$key]); } diff --git a/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php b/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php --- a/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php +++ b/src/infrastructure/internationalization/translation/PhabricatorUSEnglishTranslation.php @@ -1393,6 +1393,11 @@ '%s Days', ), + 'Setting retention policy for "%s" to %s day(s).' => array( + 'Setting retention policy for "%s" to one day.', + 'Setting retention policy for "%s" to %s days.', + ), + ); }