diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2916,6 +2916,7 @@ 'PhabricatorFactCursor' => 'applications/fact/storage/PhabricatorFactCursor.php', 'PhabricatorFactDAO' => 'applications/fact/storage/PhabricatorFactDAO.php', 'PhabricatorFactDaemon' => 'applications/fact/daemon/PhabricatorFactDaemon.php', + 'PhabricatorFactDatapointQuery' => 'applications/fact/query/PhabricatorFactDatapointQuery.php', 'PhabricatorFactDimension' => 'applications/fact/storage/PhabricatorFactDimension.php', 'PhabricatorFactEngine' => 'applications/fact/engine/PhabricatorFactEngine.php', 'PhabricatorFactEngineTestCase' => 'applications/fact/engine/__tests__/PhabricatorFactEngineTestCase.php', @@ -2928,6 +2929,7 @@ 'PhabricatorFactManagementListWorkflow' => 'applications/fact/management/PhabricatorFactManagementListWorkflow.php', 'PhabricatorFactManagementWorkflow' => 'applications/fact/management/PhabricatorFactManagementWorkflow.php', 'PhabricatorFactManiphestTaskEngine' => 'applications/fact/engine/PhabricatorFactManiphestTaskEngine.php', + 'PhabricatorFactObjectController' => 'applications/fact/controller/PhabricatorFactObjectController.php', 'PhabricatorFactObjectDimension' => 'applications/fact/storage/PhabricatorFactObjectDimension.php', 'PhabricatorFactRaw' => 'applications/fact/storage/PhabricatorFactRaw.php', 'PhabricatorFactUpdateIterator' => 'applications/fact/extract/PhabricatorFactUpdateIterator.php', @@ -8445,6 +8447,7 @@ 'PhabricatorFactCursor' => 'PhabricatorFactDAO', 'PhabricatorFactDAO' => 'PhabricatorLiskDAO', 'PhabricatorFactDaemon' => 'PhabricatorDaemon', + 'PhabricatorFactDatapointQuery' => 'Phobject', 'PhabricatorFactDimension' => 'PhabricatorFactDAO', 'PhabricatorFactEngine' => 'Phobject', 'PhabricatorFactEngineTestCase' => 'PhabricatorTestCase', @@ -8457,6 +8460,7 @@ 'PhabricatorFactManagementListWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementWorkflow' => 'PhabricatorManagementWorkflow', 'PhabricatorFactManiphestTaskEngine' => 'PhabricatorFactEngine', + 'PhabricatorFactObjectController' => 'PhabricatorFactController', 'PhabricatorFactObjectDimension' => 'PhabricatorFactDimension', 'PhabricatorFactRaw' => 'PhabricatorFactDAO', 'PhabricatorFactUpdateIterator' => 'PhutilBufferedIterator', diff --git a/src/applications/fact/application/PhabricatorFactApplication.php b/src/applications/fact/application/PhabricatorFactApplication.php --- a/src/applications/fact/application/PhabricatorFactApplication.php +++ b/src/applications/fact/application/PhabricatorFactApplication.php @@ -31,6 +31,7 @@ '/fact/' => array( '' => 'PhabricatorFactHomeController', 'chart/' => 'PhabricatorFactChartController', + 'object/(?[^/]+)/' => 'PhabricatorFactObjectController', ), ); } diff --git a/src/applications/fact/controller/PhabricatorFactObjectController.php b/src/applications/fact/controller/PhabricatorFactObjectController.php new file mode 100644 --- /dev/null +++ b/src/applications/fact/controller/PhabricatorFactObjectController.php @@ -0,0 +1,267 @@ +getViewer(); + + $phid = $request->getURIData('phid'); + $object = id(new PhabricatorObjectQuery()) + ->setViewer($viewer) + ->withNames(array($phid)) + ->executeOne(); + if (!$object) { + return new Aphront404Response(); + } + + $engines = PhabricatorFactEngine::loadAllEngines(); + foreach ($engines as $key => $engine) { + if (!$engine->supportsDatapointsForObject($object)) { + unset($engines[$key]); + } + } + + if (!$engines) { + return $this->newDialog() + ->setTitle(pht('No Engines')) + ->appendParagraph( + pht( + 'No fact engines support generating facts for this object.')) + ->addCancelButton($this->getApplicationURI()); + } + + $key_dimension = new PhabricatorFactKeyDimension(); + $object_phid = $object->getPHID(); + + $facts = array(); + $generated_datapoints = array(); + $timings = array(); + foreach ($engines as $key => $engine) { + $engine_facts = $engine->newFacts(); + $engine_facts = mpull($engine_facts, null, 'getKey'); + $facts[$key] = $engine_facts; + + $t_start = microtime(true); + $generated_datapoints[$key] = $engine->newDatapointsForObject($object); + $t_end = microtime(true); + + $timings[$key] = ($t_end - $t_start); + } + + $object_id = id(new PhabricatorFactObjectDimension()) + ->newDimensionID($object_phid, true); + + $stored_datapoints = id(new PhabricatorFactDatapointQuery()) + ->withFacts(array_mergev($facts)) + ->withObjectPHIDs(array($object_phid)) + ->needVectors(true) + ->execute(); + + $stored_groups = array(); + foreach ($stored_datapoints as $stored_datapoint) { + $stored_groups[$stored_datapoint['key']][] = $stored_datapoint; + } + + $stored_map = array(); + foreach ($engines as $key => $engine) { + $stored_map[$key] = array(); + foreach ($facts[$key] as $fact) { + $fact_datapoints = idx($stored_groups, $fact->getKey(), array()); + $fact_datapoints = igroup($fact_datapoints, 'vector'); + $stored_map[$key] += $fact_datapoints; + } + } + + $handle_phids = array(); + $handle_phids[] = $object->getPHID(); + foreach ($generated_datapoints as $key => $datapoint_set) { + foreach ($datapoint_set as $datapoint) { + $dimension_phid = $datapoint->getDimensionPHID(); + if ($dimension_phid !== null) { + $handle_phids[$dimension_phid] = $dimension_phid; + } + } + } + + foreach ($stored_map as $key => $stored_datapoints) { + foreach ($stored_datapoints as $vector_key => $datapoints) { + foreach ($datapoints as $datapoint) { + $dimension_phid = $datapoint['dimensionPHID']; + if ($dimension_phid !== null) { + $handle_phids[$dimension_phid] = $dimension_phid; + } + } + } + } + + $handles = $viewer->loadHandles($handle_phids); + + $dimension_map = id(new PhabricatorFactObjectDimension()) + ->newDimensionMap($handle_phids, true); + + $content = array(); + + $object_list = id(new PHUIPropertyListView()) + ->setViewer($viewer) + ->addProperty( + pht('Object'), + $handles[$object->getPHID()]->renderLink()); + + $total_cost = array_sum($timings); + $total_cost = pht('%sms', new PhutilNumber((int)(1000 * $total_cost))); + $object_list->addProperty(pht('Total Cost'), $total_cost); + + $object_box = id(new PHUIObjectBoxView()) + ->setHeaderText(pht('Fact Extraction Report')) + ->addPropertyList($object_list); + + $content[] = $object_box; + + $icon_fact = id(new PHUIIconView()) + ->setIcon('fa-line-chart green') + ->setTooltip(pht('Consistent Fact')); + + $icon_nodata = id(new PHUIIconView()) + ->setIcon('fa-question-circle-o violet') + ->setTooltip(pht('No Stored Datapoints')); + + $icon_new = id(new PHUIIconView()) + ->setIcon('fa-plus red') + ->setTooltip(pht('Not Stored')); + + $icon_surplus = id(new PHUIIconView()) + ->setIcon('fa-minus red') + ->setTooltip(pht('Not Generated')); + + foreach ($engines as $key => $engine) { + $rows = array(); + foreach ($generated_datapoints[$key] as $datapoint) { + $dimension_phid = $datapoint->getDimensionPHID(); + if ($dimension_phid !== null) { + $dimension = $handles[$datapoint->getDimensionPHID()]->renderLink(); + } else { + $dimension = null; + } + + $fact_key = $datapoint->getKey(); + + $fact = idx($facts[$key], $fact_key, null); + if ($fact) { + $fact_label = $fact->getName(); + } else { + $fact_label = $fact_key; + } + + $vector_key = $datapoint->newDatapointVector(); + if (isset($stored_map[$key][$vector_key])) { + unset($stored_map[$key][$vector_key]); + $icon = $icon_fact; + } else { + $icon = $icon_new; + } + + $rows[] = array( + $icon, + $fact_label, + $dimension, + $datapoint->getValue(), + phabricator_datetime($datapoint->getEpoch(), $viewer), + ); + } + + foreach ($stored_map[$key] as $vector_key => $datapoints) { + foreach ($datapoints as $datapoint) { + $dimension_phid = $datapoint['dimensionPHID']; + if ($dimension_phid !== null) { + $dimension = $handles[$dimension_phid]->renderLink(); + } else { + $dimension = null; + } + + $fact_key = $datapoint['key']; + $fact = idx($facts[$key], $fact_key, null); + if ($fact) { + $fact_label = $fact->getName(); + } else { + $fact_label = $fact_key; + } + + $rows[] = array( + $icon_surplus, + $fact_label, + $dimension, + $datapoint['value'], + phabricator_datetime($datapoint['epoch'], $viewer), + ); + } + } + + foreach ($facts[$key] as $fact) { + $has_any = id(new PhabricatorFactDatapointQuery()) + ->withFacts(array($fact)) + ->setLimit(1) + ->execute(); + if ($has_any) { + continue; + } + + if (!$has_any) { + $rows[] = array( + $icon_nodata, + $fact->getName(), + null, + null, + null, + ); + } + } + + $table = id(new AphrontTableView($rows)) + ->setHeaders( + array( + null, + pht('Fact'), + pht('Dimension'), + pht('Value'), + pht('Date'), + )) + ->setColumnClasses( + array( + '', + '', + '', + 'n wide right', + 'right', + )); + + $extraction_cost = $timings[$key]; + $extraction_cost = pht( + '%sms', + new PhutilNumber((int)(1000 * $extraction_cost))); + + $header = pht( + '%s (%s)', + get_class($engine), + $extraction_cost); + + $box = id(new PHUIObjectBoxView()) + ->setHeaderText($header) + ->setTable($table); + + $content[] = $box; + } + + $crumbs = $this->buildApplicationCrumbs() + ->addTextCrumb(pht('Chart')); + + $title = pht('Chart'); + + return $this->newPage() + ->setTitle($title) + ->setCrumbs($crumbs) + ->appendChild($content); + + } + +} diff --git a/src/applications/fact/query/PhabricatorFactDatapointQuery.php b/src/applications/fact/query/PhabricatorFactDatapointQuery.php new file mode 100644 --- /dev/null +++ b/src/applications/fact/query/PhabricatorFactDatapointQuery.php @@ -0,0 +1,181 @@ +facts = $facts; + return $this; + } + + public function withObjectPHIDs(array $object_phids) { + $this->objectPHIDs = $object_phids; + return $this; + } + + public function setLimit($limit) { + $this->limit = $limit; + return $this; + } + + public function needVectors($need) { + $this->needVectors = $need; + return $this; + } + + public function execute() { + $facts = mpull($this->facts, null, 'getKey'); + if (!$facts) { + throw new Exception(pht('Executing a fact query requires facts.')); + } + + $table_map = array(); + foreach ($facts as $fact) { + $datapoint = $fact->newDatapoint(); + $table = $datapoint->getTableName(); + + if (!isset($table_map[$table])) { + $table_map[$table] = array( + 'table' => $datapoint, + 'facts' => array(), + ); + } + + $table_map[$table]['facts'][] = $fact; + } + + $rows = array(); + foreach ($table_map as $spec) { + $rows[] = $this->executeWithTable($spec); + } + $rows = array_mergev($rows); + + $key_unmap = array_flip($this->keyMap); + $dimension_unmap = array_flip($this->dimensionMap); + + $groups = array(); + $need_phids = array(); + foreach ($rows as $row) { + $groups[$row['keyID']][] = $row; + + $object_id = $row['objectID']; + if (!isset($dimension_unmap[$object_id])) { + $need_phids[$object_id] = $object_id; + } + + $dimension_id = $row['dimensionID']; + if ($dimension_id && !isset($dimension_unmap[$dimension_id])) { + $need_phids[$dimension_id] = $dimension_id; + } + } + + $dimension_unmap += id(new PhabricatorFactObjectDimension()) + ->newDimensionUnmap($need_phids); + + $results = array(); + foreach ($groups as $key_id => $rows) { + $key = $key_unmap[$key_id]; + $fact = $facts[$key]; + $datapoint = $fact->newDatapoint(); + foreach ($rows as $row) { + $dimension_id = $row['dimensionID']; + if ($dimension_id) { + if (!isset($dimension_unmap[$dimension_id])) { + continue; + } else { + $dimension_phid = $dimension_unmap[$dimension_id]; + } + } else { + $dimension_phid = null; + } + + $object_id = $row['objectID']; + if (!isset($dimension_unmap[$object_id])) { + continue; + } else { + $object_phid = $dimension_unmap[$object_id]; + } + + $result = array( + 'key' => $key, + 'objectPHID' => $object_phid, + 'dimensionPHID' => $dimension_phid, + 'value' => (int)$row['value'], + 'epoch' => $row['epoch'], + ); + + if ($this->needVectors) { + $result['vector'] = $datapoint->newRawVector($result); + } + + $results[] = $result; + } + } + + return $results; + } + + private function executeWithTable(array $spec) { + $table = $spec['table']; + $facts = $spec['facts']; + $conn = $table->establishConnection('r'); + + $fact_keys = mpull($facts, 'getKey'); + $this->keyMap = id(new PhabricatorFactKeyDimension()) + ->newDimensionMap($fact_keys); + + if (!$this->keyMap) { + return array(); + } + + $where = array(); + + $where[] = qsprintf( + $conn, + 'keyID IN (%Ld)', + $this->keyMap); + + if ($this->objectPHIDs) { + $object_map = id(new PhabricatorFactObjectDimension()) + ->newDimensionMap($this->objectPHIDs); + if (!$object_map) { + return array(); + } + + $this->dimensionMap = $object_map; + + $where[] = qsprintf( + $conn, + 'objectID IN (%Ld)', + $this->dimensionMap); + } + + $where = '('.implode(') AND (', $where).')'; + + if ($this->limit) { + $limit = qsprintf( + $conn, + 'LIMIT %d', + $this->limit); + } else { + $limit = ''; + } + + return queryfx_all( + $conn, + 'SELECT keyID, objectID, dimensionID, value, epoch + FROM %T WHERE %Q %Q', + $table->getTableName(), + $where, + $limit); + } + +} diff --git a/src/applications/fact/storage/PhabricatorFactDimension.php b/src/applications/fact/storage/PhabricatorFactDimension.php --- a/src/applications/fact/storage/PhabricatorFactDimension.php +++ b/src/applications/fact/storage/PhabricatorFactDimension.php @@ -4,11 +4,30 @@ abstract protected function getDimensionColumnName(); - final public function newDimensionID($key) { - $map = $this->newDimensionMap(array($key)); + final public function newDimensionID($key, $create = false) { + $map = $this->newDimensionMap(array($key), $create); return idx($map, $key); } + final public function newDimensionUnmap(array $ids) { + if (!$ids) { + return array(); + } + + $conn = $this->establishConnection('r'); + $column = $this->getDimensionColumnName(); + + $rows = queryfx_all( + $conn, + 'SELECT id, %C FROM %T WHERE id IN (%Ld)', + $column, + $this->getTableName(), + $ids); + $rows = ipull($rows, $column, 'id'); + + return $rows; + } + final public function newDimensionMap(array $keys, $create = false) { if (!$keys) { return array(); @@ -52,14 +71,16 @@ $key); } - foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) { - queryfx( - $conn, - 'INSERT IGNORE INTO %T (%C) VALUES %Q', - $this->getTableName(), - $column, - $chunk); - } + $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); + foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) { + queryfx( + $conn, + 'INSERT IGNORE INTO %T (%C) VALUES %Q', + $this->getTableName(), + $column, + $chunk); + } + unset($unguarded); $rows = queryfx_all( $conn, diff --git a/src/applications/fact/storage/PhabricatorFactIntDatapoint.php b/src/applications/fact/storage/PhabricatorFactIntDatapoint.php --- a/src/applications/fact/storage/PhabricatorFactIntDatapoint.php +++ b/src/applications/fact/storage/PhabricatorFactIntDatapoint.php @@ -58,4 +58,30 @@ return $this->dimensionPHID; } + public function newDatapointVector() { + return $this->formatVector( + array( + $this->key, + $this->objectPHID, + $this->dimensionPHID, + $this->value, + $this->epoch, + )); + } + + public function newRawVector(array $spec) { + return $this->formatVector( + array( + $spec['key'], + $spec['objectPHID'], + $spec['dimensionPHID'], + $spec['value'], + $spec['epoch'], + )); + } + + private function formatVector(array $vector) { + return implode(':', $vector); + } + }