diff --git a/resources/sql/autopatches/20150430.multimeter.1.sql b/resources/sql/autopatches/20150430.multimeter.1.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20150430.multimeter.1.sql @@ -0,0 +1,14 @@ +CREATE TABLE {$NAMESPACE}_multimeter.multimeter_event ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + eventType INT UNSIGNED NOT NULL, + eventLabelID INT UNSIGNED NOT NULL, + resourceCost BIGINT NOT NULL, + sampleRate INT UNSIGNED NOT NULL, + eventContextID INT UNSIGNED NOT NULL, + eventHostID INT UNSIGNED NOT NULL, + eventViewerID INT UNSIGNED NOT NULL, + epoch INT UNSIGNED NOT NULL, + requestKey BINARY(12) NOT NULL, + KEY `key_request` (requestKey), + KEY `key_type` (eventType, epoch) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20150430.multimeter.2.host.sql b/resources/sql/autopatches/20150430.multimeter.2.host.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20150430.multimeter.2.host.sql @@ -0,0 +1,6 @@ +CREATE TABLE {$NAMESPACE}_multimeter.multimeter_host ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + name LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL, + nameHash BINARY(12) NOT NULL, + UNIQUE KEY `key_hash` (nameHash) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20150430.multimeter.3.viewer.sql b/resources/sql/autopatches/20150430.multimeter.3.viewer.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20150430.multimeter.3.viewer.sql @@ -0,0 +1,6 @@ +CREATE TABLE {$NAMESPACE}_multimeter.multimeter_viewer ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + name LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL, + nameHash BINARY(12) NOT NULL, + UNIQUE KEY `key_hash` (nameHash) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20150430.multimeter.4.context.sql b/resources/sql/autopatches/20150430.multimeter.4.context.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20150430.multimeter.4.context.sql @@ -0,0 +1,6 @@ +CREATE TABLE {$NAMESPACE}_multimeter.multimeter_context ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + name LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL, + nameHash BINARY(12) NOT NULL, + UNIQUE KEY `key_hash` (nameHash) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20150430.multimeter.5.label.sql b/resources/sql/autopatches/20150430.multimeter.5.label.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20150430.multimeter.5.label.sql @@ -0,0 +1,6 @@ +CREATE TABLE {$NAMESPACE}_multimeter.multimeter_label ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + name LONGTEXT COLLATE {$COLLATE_TEXT} NOT NULL, + nameHash BINARY(12) NOT NULL, + UNIQUE KEY `key_hash` (nameHash) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1080,6 +1080,17 @@ 'MetaMTAMailSentGarbageCollector' => 'applications/metamta/garbagecollector/MetaMTAMailSentGarbageCollector.php', 'MetaMTANotificationType' => 'applications/metamta/constants/MetaMTANotificationType.php', 'MetaMTAReceivedMailStatus' => 'applications/metamta/constants/MetaMTAReceivedMailStatus.php', + 'MultimeterContext' => 'applications/multimeter/storage/MultimeterContext.php', + 'MultimeterControl' => 'applications/multimeter/data/MultimeterControl.php', + 'MultimeterController' => 'applications/multimeter/controller/MultimeterController.php', + 'MultimeterDAO' => 'applications/multimeter/storage/MultimeterDAO.php', + 'MultimeterDimension' => 'applications/multimeter/storage/MultimeterDimension.php', + 'MultimeterEvent' => 'applications/multimeter/storage/MultimeterEvent.php', + 'MultimeterEventGarbageCollector' => 'applications/multimeter/garbagecollector/MultimeterEventGarbageCollector.php', + 'MultimeterHost' => 'applications/multimeter/storage/MultimeterHost.php', + 'MultimeterLabel' => 'applications/multimeter/storage/MultimeterLabel.php', + 'MultimeterSampleController' => 'applications/multimeter/controller/MultimeterSampleController.php', + 'MultimeterViewer' => 'applications/multimeter/storage/MultimeterViewer.php', 'NuanceConduitAPIMethod' => 'applications/nuance/conduit/NuanceConduitAPIMethod.php', 'NuanceController' => 'applications/nuance/controller/NuanceController.php', 'NuanceCreateItemConduitAPIMethod' => 'applications/nuance/conduit/NuanceCreateItemConduitAPIMethod.php', @@ -2062,6 +2073,7 @@ 'PhabricatorMetronomicTriggerClock' => 'infrastructure/daemon/workers/clock/PhabricatorMetronomicTriggerClock.php', 'PhabricatorMultiColumnUIExample' => 'applications/uiexample/examples/PhabricatorMultiColumnUIExample.php', 'PhabricatorMultiFactorSettingsPanel' => 'applications/settings/panel/PhabricatorMultiFactorSettingsPanel.php', + 'PhabricatorMultimeterApplication' => 'applications/multimeter/application/PhabricatorMultimeterApplication.php', 'PhabricatorMustVerifyEmailController' => 'applications/auth/controller/PhabricatorMustVerifyEmailController.php', 'PhabricatorMySQLConfigOptions' => 'applications/config/option/PhabricatorMySQLConfigOptions.php', 'PhabricatorMySQLFileStorageEngine' => 'applications/files/engine/PhabricatorMySQLFileStorageEngine.php', @@ -4381,6 +4393,16 @@ 'MetaMTAMailSentGarbageCollector' => 'PhabricatorGarbageCollector', 'MetaMTANotificationType' => 'MetaMTAConstants', 'MetaMTAReceivedMailStatus' => 'MetaMTAConstants', + 'MultimeterContext' => 'MultimeterDimension', + 'MultimeterController' => 'PhabricatorController', + 'MultimeterDAO' => 'PhabricatorLiskDAO', + 'MultimeterDimension' => 'MultimeterDAO', + 'MultimeterEvent' => 'MultimeterDAO', + 'MultimeterEventGarbageCollector' => 'PhabricatorGarbageCollector', + 'MultimeterHost' => 'MultimeterDimension', + 'MultimeterLabel' => 'MultimeterDimension', + 'MultimeterSampleController' => 'MultimeterController', + 'MultimeterViewer' => 'MultimeterDimension', 'NuanceConduitAPIMethod' => 'ConduitAPIMethod', 'NuanceController' => 'PhabricatorController', 'NuanceCreateItemConduitAPIMethod' => 'NuanceConduitAPIMethod', @@ -5443,6 +5465,7 @@ 'PhabricatorMetronomicTriggerClock' => 'PhabricatorTriggerClock', 'PhabricatorMultiColumnUIExample' => 'PhabricatorUIExample', 'PhabricatorMultiFactorSettingsPanel' => 'PhabricatorSettingsPanel', + 'PhabricatorMultimeterApplication' => 'PhabricatorApplication', 'PhabricatorMustVerifyEmailController' => 'PhabricatorAuthController', 'PhabricatorMySQLConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorMySQLFileStorageEngine' => 'PhabricatorFileStorageEngine', diff --git a/src/aphront/configuration/AphrontApplicationConfiguration.php b/src/aphront/configuration/AphrontApplicationConfiguration.php --- a/src/aphront/configuration/AphrontApplicationConfiguration.php +++ b/src/aphront/configuration/AphrontApplicationConfiguration.php @@ -58,8 +58,15 @@ * @phutil-external-symbol class PhabricatorStartup */ public static function runHTTPRequest(AphrontHTTPSink $sink) { + $multimeter = MultimeterControl::newInstance(); + $multimeter->setEventContext(''); + $multimeter->setEventViewer(''); + PhabricatorEnv::initializeWebEnvironment(); + $multimeter->setSampleRate( + PhabricatorEnv::getEnvConfig('debug.sample-rate')); + $debug_time_limit = PhabricatorEnv::getEnvConfig('debug.time-limit'); if ($debug_time_limit) { PhabricatorStartup::setDebugTimeLimit($debug_time_limit); @@ -135,6 +142,8 @@ $access_log->write(); + $multimeter->saveEvents(); + DarkConsoleXHProfPluginAPI::saveProfilerSample($access_log); // Add points to the rate limits for this request. diff --git a/src/applications/celerity/CelerityStaticResourceResponse.php b/src/applications/celerity/CelerityStaticResourceResponse.php --- a/src/applications/celerity/CelerityStaticResourceResponse.php +++ b/src/applications/celerity/CelerityStaticResourceResponse.php @@ -144,6 +144,9 @@ $uri = $this->getURI($map, $name); $type = $map->getResourceTypeForName($name); + $event_type = MultimeterEvent::TYPE_STATIC_RESOURCE; + MultimeterControl::getInstance()->newEvent($event_type, 'rsrc.'.$name, 1); + switch ($type) { case 'css': return phutil_tag( diff --git a/src/applications/config/option/PhabricatorDeveloperConfigOptions.php b/src/applications/config/option/PhabricatorDeveloperConfigOptions.php --- a/src/applications/config/option/PhabricatorDeveloperConfigOptions.php +++ b/src/applications/config/option/PhabricatorDeveloperConfigOptions.php @@ -118,6 +118,27 @@ "data to look at eventually). In development, it may be useful to ". "set it to 1 in order to debug performance problems.\n\n". "NOTE: You must install XHProf for profiling to work.")), + $this->newOption('debug.sample-rate', 'int', 1000) + ->setLocked(true) + ->addExample(0, pht('No performance sampling.')) + ->addExample(1, pht('Sample every request (slow).')) + ->addExample(1000, pht('Sample 0.1%% of requests.')) + ->setSummary(pht('Automatically sample some fraction of requests.')) + ->setDescription( + pht( + "The Multimeter application collects performance samples. You ". + "can use this data to help you understand what Phabricator is ". + "spending time and resources doing, and to identify problematic ". + "access patterns.". + "\n\n". + "This option controls how frequently sampling activates. Set it ". + "to some positive integer N to sample every 1 / N pages.". + "\n\n". + "For most installs, the default value (1 sample per 1000 pages) ". + "should collect enough data to be useful without requiring much ". + "storage or meaningfully impacting performance. If you're ". + "investigating performance issues, you can adjust the rate ". + "in order to collect more data.")), $this->newOption('phabricator.developer-mode', 'bool', false) ->setBoolOptions( array( diff --git a/src/applications/multimeter/application/PhabricatorMultimeterApplication.php b/src/applications/multimeter/application/PhabricatorMultimeterApplication.php new file mode 100644 --- /dev/null +++ b/src/applications/multimeter/application/PhabricatorMultimeterApplication.php @@ -0,0 +1,46 @@ + array( + '' => 'MultimeterSampleController', + ), + ); + } + +} diff --git a/src/applications/multimeter/controller/MultimeterController.php b/src/applications/multimeter/controller/MultimeterController.php new file mode 100644 --- /dev/null +++ b/src/applications/multimeter/controller/MultimeterController.php @@ -0,0 +1,70 @@ + new MultimeterLabel(), + 'eventViewerID' => new MultimeterViewer(), + 'eventHostID' => new MultimeterHost(), + 'eventContextID' => new MultimeterContext(), + ); + + $ids = array(); + foreach ($map as $key => $object) { + foreach ($rows as $row) { + $ids[$key][] = $row[$key]; + } + } + + foreach ($ids as $key => $list) { + $object = $map[$key]; + if (empty($this->dimensions[$key])) { + $this->dimensions[$key] = array(); + } + $this->dimensions[$key] += $object->loadAllWhere( + 'id IN (%Ld)', + $list); + } + } + + protected function getLabelDimension($id) { + if (empty($this->dimensions['eventLabelID'][$id])) { + return $this->newMissingDimension(new MultimeterLabel(), $id); + } + return $this->dimensions['eventLabelID'][$id]; + } + + protected function getViewerDimension($id) { + if (empty($this->dimensions['eventViewerID'][$id])) { + return $this->newMissingDimension(new MultimeterViewer(), $id); + } + return $this->dimensions['eventViewerID'][$id]; + } + + protected function getHostDimension($id) { + if (empty($this->dimensions['eventHostID'][$id])) { + return $this->newMissingDimension(new MultimeterHost(), $id); + } + return $this->dimensions['eventHostID'][$id]; + } + + protected function getContextDimension($id) { + if (empty($this->dimensions['eventContextID'][$id])) { + return $this->newMissingDimension(new MultimeterContext(), $id); + } + return $this->dimensions['eventContextID'][$id]; + } + + private function newMissingDimension(MultimeterDimension $dim, $id) { + $dim->setName(''); + return $dim; + } + +} diff --git a/src/applications/multimeter/controller/MultimeterSampleController.php b/src/applications/multimeter/controller/MultimeterSampleController.php new file mode 100644 --- /dev/null +++ b/src/applications/multimeter/controller/MultimeterSampleController.php @@ -0,0 +1,85 @@ +getViewer(); + + $table = new MultimeterEvent(); + $conn = $table->establishConnection('r'); + $data = queryfx_all( + $conn, + 'SELECT * FROM %T ORDER BY id DESC LIMIT 100', + $table->getTableName()); + + $this->loadDimensions($data); + + $rows = array(); + foreach ($data as $row) { + $rows[] = array( + $row['id'], + $row['requestKey'], + $this->getViewerDimension($row['eventViewerID'])->getName(), + $this->getContextDimension($row['eventContextID'])->getName(), + $this->getHostDimension($row['eventHostID'])->getName(), + MultimeterEvent::getEventTypeName($row['eventType']), + $this->getLabelDimension($row['eventLabelID'])->getName(), + MultimeterEvent::formatResourceCost( + $viewer, + $row['eventType'], + $row['resourceCost']), + $row['sampleRate'], + phabricator_datetime($row['epoch'], $viewer), + ); + } + + $table = id(new AphrontTableView($rows)) + ->setHeaders( + array( + pht('ID'), + pht('Request'), + pht('Viewer'), + pht('Context'), + pht('Host'), + pht('Type'), + pht('Label'), + pht('Cost'), + pht('Rate'), + pht('Epoch'), + )) + ->setColumnClasses( + array( + null, + null, + null, + null, + null, + null, + 'wide', + 'n', + 'n', + null, + )); + + $box = id(new PHUIObjectBoxView()) + ->setHeaderText(pht('Samples')) + ->appendChild($table); + + $crumbs = $this->buildApplicationCrumbs(); + $crumbs->addTextCrumb(pht('Samples')); + + return $this->buildApplicationPage( + array( + $crumbs, + $box, + ), + array( + 'title' => pht('Samples'), + )); + } + +} diff --git a/src/applications/multimeter/data/MultimeterControl.php b/src/applications/multimeter/data/MultimeterControl.php new file mode 100644 --- /dev/null +++ b/src/applications/multimeter/data/MultimeterControl.php @@ -0,0 +1,203 @@ +sampleRate !== 0) && ($this->pauseDepth == 0); + } + + public function setSampleRate($rate) { + if ($rate && (mt_rand(1, $rate) == $rate)) { + $sample_rate = $rate; + } else { + $sample_rate = 0; + } + + $this->sampleRate = $sample_rate; + + return; + } + + public function pauseMultimeter() { + $this->pauseDepth++; + return $this; + } + + public function unpauseMultimeter() { + if (!$this->pauseDepth) { + throw new Exception(pht('Trying to unpause an active multimeter!')); + } + $this->pauseDepth--; + return $this; + } + + + public function newEvent($type, $label, $cost) { + if (!$this->isActive()) { + return null; + } + + $event = id(new MultimeterEvent()) + ->setEventType($type) + ->setEventLabel($label) + ->setResourceCost($cost) + ->setEpoch(PhabricatorTime::getNow()); + + $this->events[] = $event; + + return $event; + } + + public function saveEvents() { + if (!$this->isActive()) { + return; + } + + $events = $this->events; + if (!$events) { + return; + } + + if ($this->sampleRate === null) { + throw new Exception(pht('Call setSampleRate() before saving events!')); + } + + // Don't sample any of this stuff. + $this->pauseMultimeter(); + + $use_scope = AphrontWriteGuard::isGuardActive(); + if ($use_scope) { + $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); + } else { + AphrontWriteGuard::allowDangerousUnguardedWrites(true); + } + + $caught = null; + try { + $this->writeEvents(); + } catch (Exception $ex) { + $caught = $ex; + } + + if ($use_scope) { + unset($unguarded); + } else { + AphrontWriteGuard::allowDangerousUnguardedWrites(false); + } + + $this->unpauseMultimeter(); + + if ($caught) { + throw $caught; + } + } + + private function writeEvents() { + $events = $this->events; + + $random = Filesystem::readRandomBytes(32); + $request_key = PhabricatorHash::digestForIndex($random); + + $host_id = $this->loadHostID(php_uname('n')); + $context_id = $this->loadEventContextID($this->eventContext); + $viewer_id = $this->loadEventViewerID($this->eventViewer); + $label_map = $this->loadEventLabelIDs(mpull($events, 'getEventLabel')); + + foreach ($events as $event) { + $event + ->setRequestKey($request_key) + ->setSampleRate($this->sampleRate) + ->setEventHostID($host_id) + ->setEventContextID($context_id) + ->setEventViewerID($viewer_id) + ->setEventLabelID($label_map[$event->getEventLabel()]) + ->save(); + } + } + + public function setEventContext($event_context) { + $this->eventContext = $event_context; + return $this; + } + + public function setEventViewer($viewer) { + $this->eventViewer = $viewer; + return $this; + } + + private function loadHostID($host) { + $map = $this->loadDimensionMap(new MultimeterHost(), array($host)); + return idx($map, $host); + } + + private function loadEventViewerID($viewer) { + $map = $this->loadDimensionMap(new MultimeterViewer(), array($viewer)); + return idx($map, $viewer); + } + + private function loadEventContextID($context) { + $map = $this->loadDimensionMap(new MultimeterContext(), array($context)); + return idx($map, $context); + } + + private function loadEventLabelIDs(array $labels) { + return $this->loadDimensionMap(new MultimeterLabel(), $labels); + } + + private function loadDimensionMap(MultimeterDimension $table, array $names) { + $hashes = array(); + foreach ($names as $name) { + $hashes[] = PhabricatorHash::digestForIndex($name); + } + + $objects = $table->loadAllWhere('nameHash IN (%Ls)', $hashes); + $map = mpull($objects, 'getID', 'getName'); + + $need = array(); + foreach ($names as $name) { + if (isset($map[$name])) { + continue; + } + $need[] = $name; + } + + foreach ($need as $name) { + $object = id(clone $table) + ->setName($name) + ->save(); + $map[$name] = $object->getID(); + } + + return $map; + } + +} diff --git a/src/applications/multimeter/garbagecollector/MultimeterEventGarbageCollector.php b/src/applications/multimeter/garbagecollector/MultimeterEventGarbageCollector.php new file mode 100644 --- /dev/null +++ b/src/applications/multimeter/garbagecollector/MultimeterEventGarbageCollector.php @@ -0,0 +1,21 @@ +establishConnection('w'); + + queryfx( + $conn_w, + 'DELETE FROM %T WHERE epoch < %d LIMIT 100', + $table->getTableName(), + PhabricatorTime::getNow() - $ttl); + + return ($conn_w->getAffectedRows() == 100); + } + +} diff --git a/src/applications/multimeter/storage/MultimeterContext.php b/src/applications/multimeter/storage/MultimeterContext.php new file mode 100644 --- /dev/null +++ b/src/applications/multimeter/storage/MultimeterContext.php @@ -0,0 +1,3 @@ +nameHash = PhabricatorHash::digestForIndex($name); + return parent::setName($name); + } + + protected function getConfiguration() { + return array( + self::CONFIG_TIMESTAMPS => false, + self::CONFIG_COLUMN_SCHEMA => array( + 'name' => 'text', + 'nameHash' => 'bytes12', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_hash' => array( + 'columns' => array('nameHash'), + 'unique' => true, + ), + ), + ) + parent::getConfiguration(); + } + +} diff --git a/src/applications/multimeter/storage/MultimeterEvent.php b/src/applications/multimeter/storage/MultimeterEvent.php new file mode 100644 --- /dev/null +++ b/src/applications/multimeter/storage/MultimeterEvent.php @@ -0,0 +1,71 @@ +eventLabel = $event_label; + return $this; + } + + public function getEventLabel() { + return $this->eventLabel; + } + + public static function getEventTypeName($type) { + switch ($type) { + case self::TYPE_STATIC_RESOURCE: + return pht('Static Resource'); + } + + return pht('Unknown ("%s")', $type); + } + + public static function formatResourceCost( + PhabricatorUser $viewer, + $type, + $cost) { + + switch ($type) { + case self::TYPE_STATIC_RESOURCE: + return pht('%s Req', new PhutilNumber($cost)); + } + + return pht('%s Unit(s)', new PhutilNumber($cost)); + } + + + protected function getConfiguration() { + return array( + self::CONFIG_TIMESTAMPS => false, + self::CONFIG_COLUMN_SCHEMA => array( + 'eventType' => 'uint32', + 'resourceCost' => 'sint64', + 'sampleRate' => 'uint32', + 'requestKey' => 'bytes12', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_request' => array( + 'columns' => array('requestKey'), + ), + 'key_type' => array( + 'columns' => array('eventType', 'epoch'), + ), + ), + ) + parent::getConfiguration(); + } + +} diff --git a/src/applications/multimeter/storage/MultimeterHost.php b/src/applications/multimeter/storage/MultimeterHost.php new file mode 100644 --- /dev/null +++ b/src/applications/multimeter/storage/MultimeterHost.php @@ -0,0 +1,3 @@ + array(), 'db.fund' => array(), 'db.almanac' => array(), + 'db.multimeter' => array(), '0000.legacy.sql' => array( 'legacy' => 0, ),