diff --git a/src/applications/daemon/controller/PhabricatorDaemonConsoleController.php b/src/applications/daemon/controller/PhabricatorDaemonConsoleController.php index b1b5d4c385..f4c5ef2241 100644 --- a/src/applications/daemon/controller/PhabricatorDaemonConsoleController.php +++ b/src/applications/daemon/controller/PhabricatorDaemonConsoleController.php @@ -1,271 +1,272 @@ getViewer(); $window_start = (time() - (60 * 15)); // Assume daemons spend about 250ms second in overhead per task acquiring // leases and doing other bookkeeping. This is probably an over-estimation, // but we'd rather show that utilization is too high than too low. $lease_overhead = 0.250; $completed = id(new PhabricatorWorkerArchiveTaskQuery()) ->withDateModifiedSince($window_start) ->execute(); $failed = id(new PhabricatorWorkerActiveTask())->loadAllWhere( 'failureTime > %d', $window_start); $usage_total = 0; $usage_start = PHP_INT_MAX; $completed_info = array(); foreach ($completed as $completed_task) { $class = $completed_task->getTaskClass(); if (empty($completed_info[$class])) { $completed_info[$class] = array( 'n' => 0, 'duration' => 0, ); } $completed_info[$class]['n']++; $duration = $completed_task->getDuration(); $completed_info[$class]['duration'] += $duration; // NOTE: Duration is in microseconds, but we're just using seconds to // compute utilization. $usage_total += $lease_overhead + ($duration / 1000000); $usage_start = min($usage_start, $completed_task->getDateModified()); } $completed_info = isort($completed_info, 'n'); $rows = array(); foreach ($completed_info as $class => $info) { $rows[] = array( $class, number_format($info['n']), number_format((int)($info['duration'] / $info['n'])).' us', ); } if ($failed) { // Add the time it takes to restart the daemons. This includes a guess // about other overhead of 2X. - $usage_total += PhutilDaemonOverseer::RESTART_WAIT * count($failed) * 2; + $restart_delay = PhutilDaemonHandle::getWaitBeforeRestart(); + $usage_total += $restart_delay * count($failed) * 2; foreach ($failed as $failed_task) { $usage_start = min($usage_start, $failed_task->getFailureTime()); } $rows[] = array( phutil_tag('em', array(), pht('Temporary Failures')), count($failed), null, ); } $logs = id(new PhabricatorDaemonLogQuery()) ->setViewer($viewer) ->withStatus(PhabricatorDaemonLogQuery::STATUS_ALIVE) ->setAllowStatusWrites(true) ->execute(); $taskmasters = 0; foreach ($logs as $log) { if ($log->getDaemon() == 'PhabricatorTaskmasterDaemon') { $taskmasters++; } } if ($taskmasters && $usage_total) { // Total number of wall-time seconds the daemons have been running since // the oldest event. For very short times round up to 15s so we don't // render any ridiculous numbers if you reload the page immediately after // restarting the daemons. $available_time = $taskmasters * max(15, (time() - $usage_start)); // Percentage of those wall-time seconds we can account for, which the // daemons spent doing work: $used_time = ($usage_total / $available_time); $rows[] = array( phutil_tag('em', array(), pht('Queue Utilization (Approximate)')), sprintf('%.1f%%', 100 * $used_time), null, ); } $completed_table = new AphrontTableView($rows); $completed_table->setNoDataString( pht('No tasks have completed in the last 15 minutes.')); $completed_table->setHeaders( array( pht('Class'), pht('Count'), pht('Avg'), )); $completed_table->setColumnClasses( array( 'wide', 'n', 'n', )); $completed_panel = new PHUIObjectBoxView(); $completed_panel->setHeaderText( pht('Recently Completed Tasks (Last 15m)')); $completed_panel->appendChild($completed_table); $daemon_table = new PhabricatorDaemonLogListView(); $daemon_table->setUser($viewer); $daemon_table->setDaemonLogs($logs); $daemon_panel = new PHUIObjectBoxView(); $daemon_panel->setHeaderText(pht('Active Daemons')); $daemon_panel->appendChild($daemon_table); $tasks = id(new PhabricatorWorkerLeaseQuery()) ->setSkipLease(true) ->withLeasedTasks(true) ->setLimit(100) ->execute(); $tasks_table = id(new PhabricatorDaemonTasksTableView()) ->setTasks($tasks) ->setNoDataString(pht('No tasks are leased by workers.')); $leased_panel = id(new PHUIObjectBoxView()) ->setHeaderText(pht('Leased Tasks')) ->appendChild($tasks_table); $task_table = new PhabricatorWorkerActiveTask(); $queued = queryfx_all( $task_table->establishConnection('r'), 'SELECT taskClass, count(*) N FROM %T GROUP BY taskClass ORDER BY N DESC', $task_table->getTableName()); $rows = array(); foreach ($queued as $row) { $rows[] = array( $row['taskClass'], number_format($row['N']), ); } $queued_table = new AphrontTableView($rows); $queued_table->setHeaders( array( pht('Class'), pht('Count'), )); $queued_table->setColumnClasses( array( 'wide', 'n', )); $queued_table->setNoDataString(pht('Task queue is empty.')); $queued_panel = new PHUIObjectBoxView(); $queued_panel->setHeaderText(pht('Queued Tasks')); $queued_panel->appendChild($queued_table); $upcoming = id(new PhabricatorWorkerLeaseQuery()) ->setLimit(10) ->setSkipLease(true) ->execute(); $upcoming_panel = id(new PHUIObjectBoxView()) ->setHeaderText(pht('Next In Queue')) ->appendChild( id(new PhabricatorDaemonTasksTableView()) ->setTasks($upcoming) ->setNoDataString(pht('Task queue is empty.'))); $triggers = id(new PhabricatorWorkerTriggerQuery()) ->setViewer($viewer) ->setOrder(PhabricatorWorkerTriggerQuery::ORDER_EXECUTION) ->needEvents(true) ->setLimit(10) ->execute(); $triggers_table = $this->buildTriggersTable($triggers); $triggers_panel = id(new PHUIObjectBoxView()) ->setHeaderText(pht('Upcoming Triggers')) ->appendChild($triggers_table); $crumbs = $this->buildApplicationCrumbs(); $crumbs->addTextCrumb(pht('Console')); $nav = $this->buildSideNavView(); $nav->selectFilter('/'); $nav->appendChild( array( $crumbs, $completed_panel, $daemon_panel, $queued_panel, $leased_panel, $upcoming_panel, $triggers_panel, )); return $this->buildApplicationPage( $nav, array( 'title' => pht('Console'), 'device' => false, )); } private function buildTriggersTable(array $triggers) { $viewer = $this->getViewer(); $rows = array(); foreach ($triggers as $trigger) { $event = $trigger->getEvent(); if ($event) { $last_epoch = $event->getLastEventEpoch(); $next_epoch = $event->getNextEventEpoch(); } else { $last_epoch = null; $next_epoch = null; } $rows[] = array( $trigger->getID(), $trigger->getClockClass(), $trigger->getActionClass(), $last_epoch ? phabricator_datetime($last_epoch, $viewer) : null, $next_epoch ? phabricator_datetime($next_epoch, $viewer) : null, ); } return id(new AphrontTableView($rows)) ->setNoDataString(pht('There are no upcoming event triggers.')) ->setHeaders( array( 'ID', 'Clock', 'Action', 'Last', 'Next', )) ->setColumnClasses( array( '', '', 'wide', 'date', 'date', )); } } diff --git a/src/applications/daemon/controller/PhabricatorDaemonLogViewController.php b/src/applications/daemon/controller/PhabricatorDaemonLogViewController.php index fa473bed8b..939d34801f 100644 --- a/src/applications/daemon/controller/PhabricatorDaemonLogViewController.php +++ b/src/applications/daemon/controller/PhabricatorDaemonLogViewController.php @@ -1,200 +1,200 @@ id = $data['id']; } public function processRequest() { $request = $this->getRequest(); $user = $request->getUser(); $log = id(new PhabricatorDaemonLogQuery()) ->setViewer($user) ->withIDs(array($this->id)) ->setAllowStatusWrites(true) ->executeOne(); if (!$log) { return new Aphront404Response(); } $events = id(new PhabricatorDaemonLogEvent())->loadAllWhere( 'logID = %d ORDER BY id DESC LIMIT 1000', $log->getID()); $crumbs = $this->buildApplicationCrumbs(); $crumbs->addTextCrumb(pht('Daemon %s', $log->getID())); $header = id(new PHUIHeaderView()) ->setHeader($log->getDaemon()); $tag = id(new PHUITagView()) ->setType(PHUITagView::TYPE_STATE); $status = $log->getStatus(); switch ($status) { case PhabricatorDaemonLog::STATUS_UNKNOWN: $tag->setBackgroundColor(PHUITagView::COLOR_ORANGE); $tag->setName(pht('Unknown')); break; case PhabricatorDaemonLog::STATUS_RUNNING: $tag->setBackgroundColor(PHUITagView::COLOR_GREEN); $tag->setName(pht('Running')); break; case PhabricatorDaemonLog::STATUS_DEAD: $tag->setBackgroundColor(PHUITagView::COLOR_RED); $tag->setName(pht('Dead')); break; case PhabricatorDaemonLog::STATUS_WAIT: $tag->setBackgroundColor(PHUITagView::COLOR_BLUE); $tag->setName(pht('Waiting')); break; case PhabricatorDaemonLog::STATUS_EXITING: $tag->setBackgroundColor(PHUITagView::COLOR_YELLOW); $tag->setName(pht('Exiting')); break; case PhabricatorDaemonLog::STATUS_EXITED: $tag->setBackgroundColor(PHUITagView::COLOR_GREY); $tag->setName(pht('Exited')); break; } $header->addTag($tag); $env_hash = PhabricatorEnv::calculateEnvironmentHash(); if ($log->getEnvHash() != $env_hash) { $tag = id(new PHUITagView()) ->setType(PHUITagView::TYPE_STATE) ->setBackgroundColor(PHUITagView::COLOR_YELLOW) ->setName(pht('Stale Config')); $header->addTag($tag); } $properties = $this->buildPropertyListView($log); $event_view = id(new PhabricatorDaemonLogEventsView()) ->setUser($user) ->setEvents($events); $event_panel = new AphrontPanelView(); $event_panel->setNoBackground(); $event_panel->appendChild($event_view); $object_box = id(new PHUIObjectBoxView()) ->setHeader($header) ->addPropertyList($properties); return $this->buildApplicationPage( array( $crumbs, $object_box, $event_panel, ), array( 'title' => pht('Daemon Log'), 'device' => false, )); } private function buildPropertyListView(PhabricatorDaemonLog $daemon) { $request = $this->getRequest(); $viewer = $request->getUser(); $view = id(new PHUIPropertyListView()) ->setUser($viewer); $id = $daemon->getID(); $c_epoch = $daemon->getDateCreated(); $u_epoch = $daemon->getDateModified(); $unknown_time = PhabricatorDaemonLogQuery::getTimeUntilUnknown(); $dead_time = PhabricatorDaemonLogQuery::getTimeUntilDead(); - $wait_time = PhutilDaemonOverseer::RESTART_WAIT; + $wait_time = PhutilDaemonHandle::getWaitBeforeRestart(); $details = null; $status = $daemon->getStatus(); switch ($status) { case PhabricatorDaemonLog::STATUS_RUNNING: $details = pht( 'This daemon is running normally and reported a status update '. 'recently (within %s).', phutil_format_relative_time($unknown_time)); break; case PhabricatorDaemonLog::STATUS_UNKNOWN: $details = pht( 'This daemon has not reported a status update recently (within %s). '. 'It may have exited abruptly. After %s, it will be presumed dead.', phutil_format_relative_time($unknown_time), phutil_format_relative_time($dead_time)); break; case PhabricatorDaemonLog::STATUS_DEAD: $details = pht( 'This daemon did not report a status update for %s. It is '. 'presumed dead. Usually, this indicates that the daemon was '. 'killed or otherwise exited abruptly with an error. You may '. 'need to restart it.', phutil_format_relative_time($dead_time)); break; case PhabricatorDaemonLog::STATUS_WAIT: $details = pht( 'This daemon is running normally and reported a status update '. 'recently (within %s). However, it encountered an error while '. 'doing work and is waiting a little while (%s) to resume '. 'processing. After encountering an error, daemons wait before '. 'resuming work to avoid overloading services.', phutil_format_relative_time($unknown_time), phutil_format_relative_time($wait_time)); break; case PhabricatorDaemonLog::STATUS_EXITING: $details = pht( 'This daemon is shutting down gracefully.'); break; case PhabricatorDaemonLog::STATUS_EXITED: $details = pht( 'This daemon exited normally and is no longer running.'); break; } $view->addProperty(pht('Status Details'), $details); $view->addProperty(pht('Daemon Class'), $daemon->getDaemon()); $view->addProperty(pht('Host'), $daemon->getHost()); $view->addProperty(pht('PID'), $daemon->getPID()); $view->addProperty(pht('Running as'), $daemon->getRunningAsUser()); $view->addProperty(pht('Started'), phabricator_datetime($c_epoch, $viewer)); $view->addProperty( pht('Seen'), pht( '%s ago (%s)', phutil_format_relative_time(time() - $u_epoch), phabricator_datetime($u_epoch, $viewer))); $argv = $daemon->getArgv(); if (is_array($argv)) { $argv = implode("\n", $argv); } $view->addProperty( pht('Argv'), phutil_tag( 'textarea', array( 'style' => 'width: 100%; height: 12em;', ), $argv)); $view->addProperty( pht('View Full Logs'), phutil_tag( 'tt', array(), "phabricator/ $ ./bin/phd log --id {$id}")); return $view; } } diff --git a/src/applications/daemon/event/PhabricatorDaemonEventListener.php b/src/applications/daemon/event/PhabricatorDaemonEventListener.php index 47c543907b..ada6ce92d3 100644 --- a/src/applications/daemon/event/PhabricatorDaemonEventListener.php +++ b/src/applications/daemon/event/PhabricatorDaemonEventListener.php @@ -1,120 +1,120 @@ listen(PhutilDaemonOverseer::EVENT_DID_LAUNCH); - $this->listen(PhutilDaemonOverseer::EVENT_DID_LOG); - $this->listen(PhutilDaemonOverseer::EVENT_DID_HEARTBEAT); - $this->listen(PhutilDaemonOverseer::EVENT_WILL_GRACEFUL); - $this->listen(PhutilDaemonOverseer::EVENT_WILL_EXIT); + $this->listen(PhutilDaemonHandle::EVENT_DID_LAUNCH); + $this->listen(PhutilDaemonHandle::EVENT_DID_LOG); + $this->listen(PhutilDaemonHandle::EVENT_DID_HEARTBEAT); + $this->listen(PhutilDaemonHandle::EVENT_WILL_GRACEFUL); + $this->listen(PhutilDaemonHandle::EVENT_WILL_EXIT); } public function handleEvent(PhutilEvent $event) { switch ($event->getType()) { - case PhutilDaemonOverseer::EVENT_DID_LAUNCH: + case PhutilDaemonHandle::EVENT_DID_LAUNCH: $this->handleLaunchEvent($event); break; - case PhutilDaemonOverseer::EVENT_DID_HEARTBEAT: + case PhutilDaemonHandle::EVENT_DID_HEARTBEAT: $this->handleHeartbeatEvent($event); break; - case PhutilDaemonOverseer::EVENT_DID_LOG: + case PhutilDaemonHandle::EVENT_DID_LOG: $this->handleLogEvent($event); break; - case PhutilDaemonOverseer::EVENT_WILL_GRACEFUL: + case PhutilDaemonHandle::EVENT_WILL_GRACEFUL: $this->handleGracefulEvent($event); break; - case PhutilDaemonOverseer::EVENT_WILL_EXIT: + case PhutilDaemonHandle::EVENT_WILL_EXIT: $this->handleExitEvent($event); break; } } private function handleLaunchEvent(PhutilEvent $event) { $id = $event->getValue('id'); $current_user = posix_getpwuid(posix_geteuid()); $daemon = id(new PhabricatorDaemonLog()) ->setDaemon($event->getValue('daemonClass')) ->setHost(php_uname('n')) ->setPID(getmypid()) ->setRunningAsUser($current_user['name']) ->setEnvHash(PhabricatorEnv::calculateEnvironmentHash()) ->setEnvInfo(PhabricatorEnv::calculateEnvironmentInfo()) ->setStatus(PhabricatorDaemonLog::STATUS_RUNNING) ->setArgv($event->getValue('argv')) ->setExplicitArgv($event->getValue('explicitArgv')) ->save(); $this->daemons[$id] = $daemon; } private function handleHeartbeatEvent(PhutilEvent $event) { $daemon = $this->getDaemon($event->getValue('id')); // Just update the timestamp. $daemon->save(); } private function handleLogEvent(PhutilEvent $event) { $daemon = $this->getDaemon($event->getValue('id')); // TODO: This is a bit awkward for historical reasons, clean it up after // removing Conduit. $message = $event->getValue('message'); $context = $event->getValue('context'); if (strlen($context) && $context !== $message) { $message = "({$context}) {$message}"; } $type = $event->getValue('type'); $message = phutil_utf8ize($message); id(new PhabricatorDaemonLogEvent()) ->setLogID($daemon->getID()) ->setLogType($type) ->setMessage((string)$message) ->setEpoch(time()) ->save(); switch ($type) { case 'WAIT': $current_status = PhabricatorDaemonLog::STATUS_WAIT; break; default: $current_status = PhabricatorDaemonLog::STATUS_RUNNING; break; } if ($current_status !== $daemon->getStatus()) { $daemon->setStatus($current_status)->save(); } } private function handleGracefulEvent(PhutilEvent $event) { $id = $event->getValue('id'); $daemon = $this->getDaemon($id); $daemon->setStatus(PhabricatorDaemonLog::STATUS_EXITING)->save(); } private function handleExitEvent(PhutilEvent $event) { $id = $event->getValue('id'); $daemon = $this->getDaemon($id); $daemon->setStatus(PhabricatorDaemonLog::STATUS_EXITED)->save(); unset($this->daemons[$id]); } private function getDaemon($id) { if (isset($this->daemons[$id])) { return $this->daemons[$id]; } throw new Exception("No such daemon '{$id}'!"); } } diff --git a/src/applications/daemon/query/PhabricatorDaemonLogQuery.php b/src/applications/daemon/query/PhabricatorDaemonLogQuery.php index 9994c21a71..7f411b27db 100644 --- a/src/applications/daemon/query/PhabricatorDaemonLogQuery.php +++ b/src/applications/daemon/query/PhabricatorDaemonLogQuery.php @@ -1,176 +1,176 @@ ids = $ids; return $this; } public function withoutIDs(array $ids) { $this->notIDs = $ids; return $this; } public function withStatus($status) { $this->status = $status; return $this; } public function withDaemonClasses(array $classes) { $this->daemonClasses = $classes; return $this; } public function setAllowStatusWrites($allow) { $this->allowStatusWrites = $allow; return $this; } protected function loadPage() { $table = new PhabricatorDaemonLog(); $conn_r = $table->establishConnection('r'); $data = queryfx_all( $conn_r, 'SELECT * FROM %T %Q %Q %Q', $table->getTableName(), $this->buildWhereClause($conn_r), $this->buildOrderClause($conn_r), $this->buildLimitClause($conn_r)); return $table->loadAllFromArray($data); } protected function willFilterPage(array $daemons) { $unknown_delay = PhabricatorDaemonLogQuery::getTimeUntilUnknown(); $dead_delay = PhabricatorDaemonLogQuery::getTimeUntilDead(); $status_running = PhabricatorDaemonLog::STATUS_RUNNING; $status_unknown = PhabricatorDaemonLog::STATUS_UNKNOWN; $status_wait = PhabricatorDaemonLog::STATUS_WAIT; $status_exiting = PhabricatorDaemonLog::STATUS_EXITING; $status_exited = PhabricatorDaemonLog::STATUS_EXITED; $status_dead = PhabricatorDaemonLog::STATUS_DEAD; $filter = array_fuse($this->getStatusConstants()); foreach ($daemons as $key => $daemon) { $status = $daemon->getStatus(); $seen = $daemon->getDateModified(); $is_running = ($status == $status_running) || ($status == $status_wait) || ($status == $status_exiting); // If we haven't seen the daemon recently, downgrade its status to // unknown. $unknown_time = ($seen + $unknown_delay); if ($is_running && ($unknown_time < time())) { $status = $status_unknown; } // If the daemon hasn't been seen in quite a while, assume it is dead. $dead_time = ($seen + $dead_delay); if (($status == $status_unknown) && ($dead_time < time())) { $status = $status_dead; } // If we changed the daemon's status, adjust it. if ($status != $daemon->getStatus()) { $daemon->setStatus($status); // ...and write it, if we're in a context where that's reasonable. if ($this->allowStatusWrites) { $guard = AphrontWriteGuard::beginScopedUnguardedWrites(); $daemon->save(); unset($guard); } } // If the daemon no longer matches the filter, get rid of it. if ($filter) { if (empty($filter[$daemon->getStatus()])) { unset($daemons[$key]); } } } return $daemons; } private function buildWhereClause(AphrontDatabaseConnection $conn_r) { $where = array(); if ($this->ids) { $where[] = qsprintf( $conn_r, 'id IN (%Ld)', $this->ids); } if ($this->notIDs) { $where[] = qsprintf( $conn_r, 'id NOT IN (%Ld)', $this->notIDs); } if ($this->getStatusConstants()) { $where[] = qsprintf( $conn_r, 'status IN (%Ls)', $this->getStatusConstants()); } if ($this->daemonClasses) { $where[] = qsprintf( $conn_r, 'daemon IN (%Ls)', $this->daemonClasses); } $where[] = $this->buildPagingClause($conn_r); return $this->formatWhereClause($where); } private function getStatusConstants() { $status = $this->status; switch ($status) { case self::STATUS_ALL: return array(); case self::STATUS_ALIVE: return array( PhabricatorDaemonLog::STATUS_UNKNOWN, PhabricatorDaemonLog::STATUS_RUNNING, PhabricatorDaemonLog::STATUS_WAIT, PhabricatorDaemonLog::STATUS_EXITING, ); default: throw new Exception("Unknown status '{$status}'!"); } } public function getQueryApplicationClass() { return 'PhabricatorDaemonsApplication'; } }