diff --git a/resources/sql/autopatches/20180218.fact.01.dim.key.sql b/resources/sql/autopatches/20180218.fact.01.dim.key.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20180218.fact.01.dim.key.sql @@ -0,0 +1,5 @@ +CREATE TABLE {$NAMESPACE}_fact.fact_keydimension ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + factKey VARCHAR(64) NOT NULL COLLATE {$COLLATE_TEXT}, + UNIQUE KEY `key_factkey` (factKey) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20180218.fact.02.dim.obj.sql b/resources/sql/autopatches/20180218.fact.02.dim.obj.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20180218.fact.02.dim.obj.sql @@ -0,0 +1,5 @@ +CREATE TABLE {$NAMESPACE}_fact.fact_objectdimension ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + objectPHID VARBINARY(64) NOT NULL, + UNIQUE KEY `key_object` (objectPHID) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/resources/sql/autopatches/20180218.fact.03.data.int.sql b/resources/sql/autopatches/20180218.fact.03.data.int.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20180218.fact.03.data.int.sql @@ -0,0 +1,8 @@ +CREATE TABLE {$NAMESPACE}_fact.fact_intdatapoint ( + id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + keyID INT UNSIGNED NOT NULL, + objectID INT UNSIGNED NOT NULL, + dimensionID INT UNSIGNED, + value BIGINT SIGNED NOT NULL, + epoch INT UNSIGNED NOT NULL +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2907,27 +2907,28 @@ 'PhabricatorExternalAccountsSettingsPanel' => 'applications/settings/panel/PhabricatorExternalAccountsSettingsPanel.php', 'PhabricatorExtraConfigSetupCheck' => 'applications/config/check/PhabricatorExtraConfigSetupCheck.php', 'PhabricatorFacebookAuthProvider' => 'applications/auth/provider/PhabricatorFacebookAuthProvider.php', + 'PhabricatorFact' => 'applications/fact/fact/PhabricatorFact.php', 'PhabricatorFactAggregate' => 'applications/fact/storage/PhabricatorFactAggregate.php', 'PhabricatorFactApplication' => 'applications/fact/application/PhabricatorFactApplication.php', 'PhabricatorFactChartController' => 'applications/fact/controller/PhabricatorFactChartController.php', 'PhabricatorFactController' => 'applications/fact/controller/PhabricatorFactController.php', - 'PhabricatorFactCountEngine' => 'applications/fact/engine/PhabricatorFactCountEngine.php', 'PhabricatorFactCursor' => 'applications/fact/storage/PhabricatorFactCursor.php', 'PhabricatorFactDAO' => 'applications/fact/storage/PhabricatorFactDAO.php', 'PhabricatorFactDaemon' => 'applications/fact/daemon/PhabricatorFactDaemon.php', + 'PhabricatorFactDimension' => 'applications/fact/storage/PhabricatorFactDimension.php', 'PhabricatorFactEngine' => 'applications/fact/engine/PhabricatorFactEngine.php', 'PhabricatorFactEngineTestCase' => 'applications/fact/engine/__tests__/PhabricatorFactEngineTestCase.php', 'PhabricatorFactHomeController' => 'applications/fact/controller/PhabricatorFactHomeController.php', - 'PhabricatorFactLastUpdatedEngine' => 'applications/fact/engine/PhabricatorFactLastUpdatedEngine.php', + 'PhabricatorFactIntDatapoint' => 'applications/fact/storage/PhabricatorFactIntDatapoint.php', + 'PhabricatorFactKeyDimension' => 'applications/fact/storage/PhabricatorFactKeyDimension.php', 'PhabricatorFactManagementAnalyzeWorkflow' => 'applications/fact/management/PhabricatorFactManagementAnalyzeWorkflow.php', 'PhabricatorFactManagementCursorsWorkflow' => 'applications/fact/management/PhabricatorFactManagementCursorsWorkflow.php', 'PhabricatorFactManagementDestroyWorkflow' => 'applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php', 'PhabricatorFactManagementListWorkflow' => 'applications/fact/management/PhabricatorFactManagementListWorkflow.php', - 'PhabricatorFactManagementStatusWorkflow' => 'applications/fact/management/PhabricatorFactManagementStatusWorkflow.php', 'PhabricatorFactManagementWorkflow' => 'applications/fact/management/PhabricatorFactManagementWorkflow.php', + 'PhabricatorFactManiphestTaskEngine' => 'applications/fact/engine/PhabricatorFactManiphestTaskEngine.php', + 'PhabricatorFactObjectDimension' => 'applications/fact/storage/PhabricatorFactObjectDimension.php', 'PhabricatorFactRaw' => 'applications/fact/storage/PhabricatorFactRaw.php', - 'PhabricatorFactSimpleSpec' => 'applications/fact/spec/PhabricatorFactSimpleSpec.php', - 'PhabricatorFactSpec' => 'applications/fact/spec/PhabricatorFactSpec.php', 'PhabricatorFactUpdateIterator' => 'applications/fact/extract/PhabricatorFactUpdateIterator.php', 'PhabricatorFavoritesApplication' => 'applications/favorites/application/PhabricatorFavoritesApplication.php', 'PhabricatorFavoritesController' => 'applications/favorites/controller/PhabricatorFavoritesController.php', @@ -3716,6 +3717,7 @@ 'PhabricatorPirateEnglishTranslation' => 'infrastructure/internationalization/translation/PhabricatorPirateEnglishTranslation.php', 'PhabricatorPlatformSite' => 'aphront/site/PhabricatorPlatformSite.php', 'PhabricatorPointsEditField' => 'applications/transactions/editfield/PhabricatorPointsEditField.php', + 'PhabricatorPointsFact' => 'applications/fact/fact/PhabricatorPointsFact.php', 'PhabricatorPolicies' => 'applications/policy/constants/PhabricatorPolicies.php', 'PhabricatorPolicy' => 'applications/policy/storage/PhabricatorPolicy.php', 'PhabricatorPolicyApplication' => 'applications/policy/application/PhabricatorPolicyApplication.php', @@ -8433,27 +8435,28 @@ 'PhabricatorExternalAccountsSettingsPanel' => 'PhabricatorSettingsPanel', 'PhabricatorExtraConfigSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorFacebookAuthProvider' => 'PhabricatorOAuth2AuthProvider', + 'PhabricatorFact' => 'Phobject', 'PhabricatorFactAggregate' => 'PhabricatorFactDAO', 'PhabricatorFactApplication' => 'PhabricatorApplication', 'PhabricatorFactChartController' => 'PhabricatorFactController', 'PhabricatorFactController' => 'PhabricatorController', - 'PhabricatorFactCountEngine' => 'PhabricatorFactEngine', 'PhabricatorFactCursor' => 'PhabricatorFactDAO', 'PhabricatorFactDAO' => 'PhabricatorLiskDAO', 'PhabricatorFactDaemon' => 'PhabricatorDaemon', + 'PhabricatorFactDimension' => 'PhabricatorFactDAO', 'PhabricatorFactEngine' => 'Phobject', 'PhabricatorFactEngineTestCase' => 'PhabricatorTestCase', 'PhabricatorFactHomeController' => 'PhabricatorFactController', - 'PhabricatorFactLastUpdatedEngine' => 'PhabricatorFactEngine', + 'PhabricatorFactIntDatapoint' => 'PhabricatorFactDAO', + 'PhabricatorFactKeyDimension' => 'PhabricatorFactDimension', 'PhabricatorFactManagementAnalyzeWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementCursorsWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementDestroyWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementListWorkflow' => 'PhabricatorFactManagementWorkflow', - 'PhabricatorFactManagementStatusWorkflow' => 'PhabricatorFactManagementWorkflow', 'PhabricatorFactManagementWorkflow' => 'PhabricatorManagementWorkflow', + 'PhabricatorFactManiphestTaskEngine' => 'PhabricatorFactEngine', + 'PhabricatorFactObjectDimension' => 'PhabricatorFactDimension', 'PhabricatorFactRaw' => 'PhabricatorFactDAO', - 'PhabricatorFactSimpleSpec' => 'PhabricatorFactSpec', - 'PhabricatorFactSpec' => 'Phobject', 'PhabricatorFactUpdateIterator' => 'PhutilBufferedIterator', 'PhabricatorFavoritesApplication' => 'PhabricatorApplication', 'PhabricatorFavoritesController' => 'PhabricatorController', @@ -9373,6 +9376,7 @@ 'PhabricatorPirateEnglishTranslation' => 'PhutilTranslation', 'PhabricatorPlatformSite' => 'PhabricatorSite', 'PhabricatorPointsEditField' => 'PhabricatorEditField', + 'PhabricatorPointsFact' => 'PhabricatorFact', 'PhabricatorPolicies' => 'PhabricatorPolicyConstants', 'PhabricatorPolicy' => array( 'PhabricatorPolicyDAO', diff --git a/src/applications/differential/application/PhabricatorDifferentialApplication.php b/src/applications/differential/application/PhabricatorDifferentialApplication.php --- a/src/applications/differential/application/PhabricatorDifferentialApplication.php +++ b/src/applications/differential/application/PhabricatorDifferentialApplication.php @@ -35,12 +35,6 @@ ); } - public function getFactObjectsForAnalysis() { - return array( - new DifferentialRevision(), - ); - } - public function getTitleGlyph() { return "\xE2\x9A\x99"; } diff --git a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php --- a/src/applications/diffusion/application/PhabricatorDiffusionApplication.php +++ b/src/applications/diffusion/application/PhabricatorDiffusionApplication.php @@ -39,12 +39,6 @@ ); } - public function getFactObjectsForAnalysis() { - return array( - new PhabricatorRepositoryCommit(), - ); - } - public function getRemarkupRules() { return array( new DiffusionCommitRemarkupRule(), diff --git a/src/applications/fact/controller/PhabricatorFactChartController.php b/src/applications/fact/controller/PhabricatorFactChartController.php --- a/src/applications/fact/controller/PhabricatorFactChartController.php +++ b/src/applications/fact/controller/PhabricatorFactChartController.php @@ -5,27 +5,32 @@ public function handleRequest(AphrontRequest $request) { $viewer = $request->getViewer(); - $table = new PhabricatorFactRaw(); - $conn_r = $table->establishConnection('r'); - $table_name = $table->getTableName(); - $series = $request->getStr('y1'); - $specs = PhabricatorFactSpec::newSpecsForFactTypes( - PhabricatorFactEngine::loadAllEngines(), - array($series)); - $spec = idx($specs, $series); + $facts = PhabricatorFact::getAllFacts(); + $fact = idx($facts, $series); + + if (!$fact) { + return new Aphront404Response(); + } + + $key_id = id(new PhabricatorFactKeyDimension()) + ->newDimensionID($fact->getKey()); + + $table = $fact->newDatapoint(); + $conn_r = $table->establishConnection('r'); + $table_name = $table->getTableName(); $data = queryfx_all( $conn_r, - 'SELECT valueX, epoch FROM %T WHERE factType = %s ORDER BY epoch ASC', + 'SELECT value, epoch FROM %T WHERE keyID = %d ORDER BY epoch ASC', $table_name, - $series); + $key_id); $points = array(); $sum = 0; foreach ($data as $key => $row) { - $sum += (int)$row['valueX']; + $sum += (int)$row['value']; $points[(int)$row['epoch']] = $sum; } @@ -71,7 +76,7 @@ )); $box = id(new PHUIObjectBoxView()) - ->setHeaderText(pht('Count of %s', $spec->getName())) + ->setHeaderText(pht('Count of %s', $fact->getName())) ->appendChild($chart); $crumbs = $this->buildApplicationCrumbs(); diff --git a/src/applications/fact/controller/PhabricatorFactHomeController.php b/src/applications/fact/controller/PhabricatorFactHomeController.php --- a/src/applications/fact/controller/PhabricatorFactHomeController.php +++ b/src/applications/fact/controller/PhabricatorFactHomeController.php @@ -15,45 +15,6 @@ return id(new AphrontRedirectResponse())->setURI($uri); } - $types = array( - '+N:*', - '+N:DREV', - 'updated', - ); - - $engines = PhabricatorFactEngine::loadAllEngines(); - $specs = PhabricatorFactSpec::newSpecsForFactTypes($engines, $types); - - $facts = id(new PhabricatorFactAggregate())->loadAllWhere( - 'factType IN (%Ls)', - $types); - - $rows = array(); - foreach ($facts as $fact) { - $spec = $specs[$fact->getFactType()]; - - $name = $spec->getName(); - $value = $spec->formatValueForDisplay($viewer, $fact->getValueX()); - - $rows[] = array($name, $value); - } - - $table = new AphrontTableView($rows); - $table->setHeaders( - array( - pht('Fact'), - pht('Value'), - )); - $table->setColumnClasses( - array( - 'wide', - 'n', - )); - - $panel = new PHUIObjectBoxView(); - $panel->setHeaderText(pht('Facts')); - $panel->setTable($table); - $chart_form = $this->buildChartForm(); $crumbs = $this->buildApplicationCrumbs(); @@ -64,46 +25,18 @@ return $this->newPage() ->setTitle($title) ->setCrumbs($crumbs) - ->appendChild(array( - $chart_form, - $panel, - )); - + ->appendChild( + array( + $chart_form, + )); } private function buildChartForm() { $request = $this->getRequest(); $viewer = $request->getUser(); - $table = new PhabricatorFactRaw(); - $conn_r = $table->establishConnection('r'); - $table_name = $table->getTableName(); - - $facts = queryfx_all( - $conn_r, - 'SELECT DISTINCT factType from %T', - $table_name); - - $specs = PhabricatorFactSpec::newSpecsForFactTypes( - PhabricatorFactEngine::loadAllEngines(), - ipull($facts, 'factType')); - - $options = array(); - foreach ($specs as $spec) { - if ($spec->getUnit() == PhabricatorFactSpec::UNIT_COUNT) { - $options[$spec->getType()] = $spec->getName(); - } - } - - if (!$options) { - return id(new PHUIInfoView()) - ->setSeverity(PHUIInfoView::SEVERITY_NODATA) - ->setTitle(pht('No Chartable Facts')) - ->appendChild(phutil_tag( - 'p', - array(), - pht('There are no facts that can be plotted yet.'))); - } + $specs = PhabricatorFact::getAllFacts(); + $options = mpull($specs, 'getName', 'getKey'); $form = id(new AphrontFormView()) ->setUser($viewer) diff --git a/src/applications/fact/daemon/PhabricatorFactDaemon.php b/src/applications/fact/daemon/PhabricatorFactDaemon.php --- a/src/applications/fact/daemon/PhabricatorFactDaemon.php +++ b/src/applications/fact/daemon/PhabricatorFactDaemon.php @@ -4,8 +4,6 @@ private $engines; - const RAW_FACT_BUFFER_LIMIT = 128; - protected function run() { $this->setEngines(PhabricatorFactEngine::loadAllEngines()); while (!$this->shouldExit()) { @@ -15,7 +13,6 @@ foreach ($iterators as $iterator_name => $iterator) { $this->processIteratorWithCursor($iterator_name, $iterator); } - $this->processAggregates(); $this->log(pht('Zzz...')); $this->sleep(60 * 5); @@ -72,59 +69,41 @@ public function processIterator($iterator) { $result = null; - $raw_facts = array(); + $datapoints = array(); foreach ($iterator as $key => $object) { $phid = $object->getPHID(); $this->log(pht('Processing %s...', $phid)); - $raw_facts[$phid] = $this->computeRawFacts($object); - if (count($raw_facts) > self::RAW_FACT_BUFFER_LIMIT) { - $this->updateRawFacts($raw_facts); - $raw_facts = array(); + $datapoints[$phid] = $this->newDatapoints($object); + if (count($datapoints) > 1024) { + $this->updateDatapoints($datapoints); + $datapoints = array(); } $result = $key; } - if ($raw_facts) { - $this->updateRawFacts($raw_facts); - $raw_facts = array(); + if ($datapoints) { + $this->updateDatapoints($datapoints); + $datapoints = array(); } return $result; } - public function processAggregates() { - $this->log(pht('Processing aggregates.')); - - $facts = $this->computeAggregateFacts(); - $this->updateAggregateFacts($facts); - } - - private function computeAggregateFacts() { + private function newDatapoints(PhabricatorLiskDAO $object) { $facts = array(); foreach ($this->engines as $engine) { - if (!$engine->shouldComputeAggregateFacts()) { + if (!$engine->supportsDatapointsForObject($object)) { continue; } - $facts[] = $engine->computeAggregateFacts(); - } - return array_mergev($facts); - } - - private function computeRawFacts(PhabricatorLiskDAO $object) { - $facts = array(); - foreach ($this->engines as $engine) { - if (!$engine->shouldComputeRawFactsForObject($object)) { - continue; - } - $facts[] = $engine->computeRawFactsForObject($object); + $facts[] = $engine->newDatapointsForObject($object); } return array_mergev($facts); } - private function updateRawFacts(array $map) { + private function updateDatapoints(array $map) { foreach ($map as $phid => $facts) { - assert_instances_of($facts, 'PhabricatorFactRaw'); + assert_instances_of($facts, 'PhabricatorFactIntDatapoint'); } $phids = array_keys($map); @@ -132,76 +111,79 @@ return; } - $table = new PhabricatorFactRaw(); + + $fact_keys = array(); + $objects = array(); + foreach ($map as $phid => $facts) { + foreach ($facts as $fact) { + $fact_keys[$fact->getKey()] = true; + + $object_phid = $fact->getObjectPHID(); + $objects[$object_phid] = $object_phid; + + $dimension_phid = $fact->getDimensionPHID(); + if ($dimension_phid !== null) { + $objects[$dimension_phid] = $dimension_phid; + } + } + } + + $key_map = id(new PhabricatorFactKeyDimension()) + ->newDimensionMap(array_keys($fact_keys)); + $object_map = id(new PhabricatorFactObjectDimension()) + ->newDimensionMap(array_keys($objects)); + + $table = new PhabricatorFactIntDatapoint(); $conn = $table->establishConnection('w'); $table_name = $table->getTableName(); $sql = array(); foreach ($map as $phid => $facts) { foreach ($facts as $fact) { + $key_id = $key_map[$fact->getKey()]; + $object_id = $object_map[$fact->getObjectPHID()]; + + $dimension_phid = $fact->getDimensionPHID(); + if ($dimension_phid !== null) { + $dimension_id = $object_map[$dimension_phid]; + } else { + $dimension_id = null; + } + $sql[] = qsprintf( $conn, - '(%s, %s, %s, %d, %d, %d)', - $fact->getFactType(), - $fact->getObjectPHID(), - $fact->getObjectA(), - $fact->getValueX(), - $fact->getValueY(), + '(%d, %d, %nd, %d, %d)', + $key_id, + $object_id, + $dimension_id, + $fact->getValue(), $fact->getEpoch()); } } + $rebuilt_ids = array_select_keys($object_map, $phids); + $table->openTransaction(); queryfx( $conn, - 'DELETE FROM %T WHERE objectPHID IN (%Ls)', + 'DELETE FROM %T WHERE objectID IN (%Ld)', $table_name, - $phids); + $rebuilt_ids); if ($sql) { - foreach (array_chunk($sql, 256) as $chunk) { + foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) { queryfx( $conn, 'INSERT INTO %T - (factType, objectPHID, objectA, valueX, valueY, epoch) + (keyID, objectID, dimensionID, value, epoch) VALUES %Q', $table_name, - implode(', ', $chunk)); + $chunk); } } $table->saveTransaction(); } - private function updateAggregateFacts(array $facts) { - if (!$facts) { - return; - } - - $table = new PhabricatorFactAggregate(); - $conn = $table->establishConnection('w'); - $table_name = $table->getTableName(); - - $sql = array(); - foreach ($facts as $fact) { - $sql[] = qsprintf( - $conn, - '(%s, %s, %d)', - $fact->getFactType(), - $fact->getObjectPHID(), - $fact->getValueX()); - } - - foreach (array_chunk($sql, 256) as $chunk) { - queryfx( - $conn, - 'INSERT INTO %T (factType, objectPHID, valueX) VALUES %Q - ON DUPLICATE KEY UPDATE valueX = VALUES(valueX)', - $table_name, - implode(', ', $chunk)); - } - - } - } diff --git a/src/applications/fact/engine/PhabricatorFactCountEngine.php b/src/applications/fact/engine/PhabricatorFactCountEngine.php deleted file mode 100644 --- a/src/applications/fact/engine/PhabricatorFactCountEngine.php +++ /dev/null @@ -1,86 +0,0 @@ -setName($name) - ->setUnit(PhabricatorFactSimpleSpec::UNIT_COUNT); - } - - if (!strncmp($type, 'N:', 2)) { - if ($type == 'N:*') { - $name = pht('Objects'); - } else { - $name = pht('Objects of type %s', substr($type, 2)); - } - $results[] = id(new PhabricatorFactSimpleSpec($type)) - ->setName($name) - ->setUnit(PhabricatorFactSimpleSpec::UNIT_COUNT); - } - - } - return $results; - } - - public function shouldComputeRawFactsForObject(PhabricatorLiskDAO $object) { - return true; - } - - public function computeRawFactsForObject(PhabricatorLiskDAO $object) { - $facts = array(); - - $phid = $object->getPHID(); - $type = phid_get_type($phid); - - foreach (array('N:*', 'N:'.$type) as $fact_type) { - $facts[] = id(new PhabricatorFactRaw()) - ->setFactType($fact_type) - ->setObjectPHID($phid) - ->setValueX(1) - ->setEpoch($object->getDateCreated()); - } - - return $facts; - } - - public function shouldComputeAggregateFacts() { - return true; - } - - public function computeAggregateFacts() { - $table = new PhabricatorFactRaw(); - $table_name = $table->getTableName(); - $conn = $table->establishConnection('r'); - - $counts = queryfx_all( - $conn, - 'SELECT factType, SUM(valueX) N FROM %T WHERE factType LIKE %> - GROUP BY factType', - $table_name, - 'N:'); - - $facts = array(); - foreach ($counts as $count) { - $facts[] = id(new PhabricatorFactAggregate()) - ->setFactType('+'.$count['factType']) - ->setValueX($count['N']); - } - - return $facts; - } - - -} diff --git a/src/applications/fact/engine/PhabricatorFactEngine.php b/src/applications/fact/engine/PhabricatorFactEngine.php --- a/src/applications/fact/engine/PhabricatorFactEngine.php +++ b/src/applications/fact/engine/PhabricatorFactEngine.php @@ -2,30 +2,37 @@ abstract class PhabricatorFactEngine extends Phobject { + private $factMap; + final public static function loadAllEngines() { return id(new PhutilClassMapQuery()) ->setAncestorClass(__CLASS__) ->execute(); } - public function getFactSpecs(array $fact_types) { - return array(); - } + abstract public function newFacts(); - public function shouldComputeRawFactsForObject(PhabricatorLiskDAO $object) { - return false; - } + abstract public function supportsDatapointsForObject( + PhabricatorLiskDAO $object); - public function computeRawFactsForObject(PhabricatorLiskDAO $object) { - return array(); - } + abstract public function newDatapointsForObject(PhabricatorLiskDAO $object); - public function shouldComputeAggregateFacts() { - return false; - } + final protected function getFact($key) { + if ($this->factMap === null) { + $facts = $this->newFacts(); + $facts = mpull($facts, null, 'getKey'); + $this->factMap = $facts; + } + + if (!isset($this->factMap[$key])) { + throw new Exception( + pht( + 'Unknown fact ("%s") for engine "%s".', + $key, + get_class($this))); + } - public function computeAggregateFacts() { - return array(); + return $this->factMap[$key]; } } diff --git a/src/applications/fact/engine/PhabricatorFactLastUpdatedEngine.php b/src/applications/fact/engine/PhabricatorFactLastUpdatedEngine.php deleted file mode 100644 --- a/src/applications/fact/engine/PhabricatorFactLastUpdatedEngine.php +++ /dev/null @@ -1,34 +0,0 @@ -setName(pht('Facts Last Updated')) - ->setUnit(PhabricatorFactSimpleSpec::UNIT_EPOCH); - } - } - return $results; - } - - public function shouldComputeAggregateFacts() { - return true; - } - - public function computeAggregateFacts() { - $facts = array(); - - $facts[] = id(new PhabricatorFactAggregate()) - ->setFactType('updated') - ->setValueX(time()); - - return $facts; - } - -} diff --git a/src/applications/fact/engine/PhabricatorFactManiphestTaskEngine.php b/src/applications/fact/engine/PhabricatorFactManiphestTaskEngine.php new file mode 100644 --- /dev/null +++ b/src/applications/fact/engine/PhabricatorFactManiphestTaskEngine.php @@ -0,0 +1,34 @@ +setKey('tasks.count.open'), + ); + } + + public function supportsDatapointsForObject(PhabricatorLiskDAO $object) { + return ($object instanceof ManiphestTask); + } + + public function newDatapointsForObject(PhabricatorLiskDAO $object) { + $datapoints = array(); + + $phid = $object->getPHID(); + $type = phid_get_type($phid); + + $datapoint = $this->getFact('tasks.count.open') + ->newDatapoint(); + + $datapoints[] = $datapoint + ->setObjectPHID($phid) + ->setValue(1) + ->setEpoch($object->getDateCreated()); + + return $datapoints; + } + +} diff --git a/src/applications/fact/fact/PhabricatorFact.php b/src/applications/fact/fact/PhabricatorFact.php new file mode 100644 --- /dev/null +++ b/src/applications/fact/fact/PhabricatorFact.php @@ -0,0 +1,40 @@ +newFacts(); + $facts = mpull($facts, null, 'getKey'); + $map += $facts; + } + + return $map; + } + + final public function setKey($key) { + $this->key = $key; + return $this; + } + + final public function getKey() { + return $this->key; + } + + final public function getName() { + return pht('Fact "%s"', $this->getKey()); + } + + final public function newDatapoint() { + return $this->newTemplateDatapoint() + ->setKey($this->getKey()); + } + + abstract protected function newTemplateDatapoint(); + +} diff --git a/src/applications/fact/fact/PhabricatorPointsFact.php b/src/applications/fact/fact/PhabricatorPointsFact.php new file mode 100644 --- /dev/null +++ b/src/applications/fact/fact/PhabricatorPointsFact.php @@ -0,0 +1,9 @@ +getArg('skip-aggregates')) { - $daemon->processAggregates(); - } - return 0; } diff --git a/src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php b/src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php --- a/src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php +++ b/src/applications/fact/management/PhabricatorFactManagementDestroyWorkflow.php @@ -23,8 +23,13 @@ } $tables = array(); - $tables[] = new PhabricatorFactRaw(); - $tables[] = new PhabricatorFactAggregate(); + $tables[] = new PhabricatorFactCursor(); + + $tables[] = new PhabricatorFactIntDatapoint(); + + $tables[] = new PhabricatorFactObjectDimension(); + $tables[] = new PhabricatorFactKeyDimension(); + foreach ($tables as $table) { $conn = $table->establishConnection('w'); $name = $table->getTableName(); diff --git a/src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php b/src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php deleted file mode 100644 --- a/src/applications/fact/management/PhabricatorFactManagementStatusWorkflow.php +++ /dev/null @@ -1,47 +0,0 @@ -setName('status') - ->setSynopsis(pht('Show status of fact data.')) - ->setArguments(array()); - } - - public function execute(PhutilArgumentParser $args) { - $console = PhutilConsole::getConsole(); - - $map = array( - 'raw' => new PhabricatorFactRaw(), - 'agg' => new PhabricatorFactAggregate(), - ); - - foreach ($map as $type => $table) { - $conn = $table->establishConnection('r'); - $name = $table->getTableName(); - - $row = queryfx_one( - $conn, - 'SELECT COUNT(*) N FROM %T', - $name); - - $n = $row['N']; - - switch ($type) { - case 'raw': - $desc = pht('There are %d raw fact(s) in storage.', $n); - break; - case 'agg': - $desc = pht('There are %d aggregate fact(s) in storage.', $n); - break; - } - - $console->writeOut("%s\n", $desc); - } - - return 0; - } - -} diff --git a/src/applications/fact/spec/PhabricatorFactSimpleSpec.php b/src/applications/fact/spec/PhabricatorFactSimpleSpec.php deleted file mode 100644 --- a/src/applications/fact/spec/PhabricatorFactSimpleSpec.php +++ /dev/null @@ -1,38 +0,0 @@ -type = $type; - } - - public function getType() { - return $this->type; - } - - public function setUnit($unit) { - $this->unit = $unit; - return $this; - } - - public function getUnit() { - return $this->unit; - } - - public function setName($name) { - $this->name = $name; - return $this; - } - - public function getName() { - if ($this->name !== null) { - return $this->name; - } - return parent::getName(); - } - -} diff --git a/src/applications/fact/spec/PhabricatorFactSpec.php b/src/applications/fact/spec/PhabricatorFactSpec.php deleted file mode 100644 --- a/src/applications/fact/spec/PhabricatorFactSpec.php +++ /dev/null @@ -1,53 +0,0 @@ -getFactSpecs($fact_types); - $specs = mpull($specs, null, 'getType'); - $map += $specs; - } - - foreach ($fact_types as $type) { - if (empty($map[$type])) { - $map[$type] = new PhabricatorFactSimpleSpec($type); - } - } - - return $map; - } - - abstract public function getType(); - - public function getUnit() { - return null; - } - - public function getName() { - return pht( - 'Fact (%s)', - $this->getType()); - } - - public function formatValueForDisplay(PhabricatorUser $user, $value) { - $unit = $this->getUnit(); - switch ($unit) { - case self::UNIT_COUNT: - return number_format($value); - case self::UNIT_EPOCH: - return phabricator_datetime($value, $user); - default: - return $value; - } - } - -} diff --git a/src/applications/fact/storage/PhabricatorFactDimension.php b/src/applications/fact/storage/PhabricatorFactDimension.php new file mode 100644 --- /dev/null +++ b/src/applications/fact/storage/PhabricatorFactDimension.php @@ -0,0 +1,85 @@ +newDimensionMap(array($key)); + return $map[$key]; + } + + final public function newDimensionMap(array $keys) { + if (!$keys) { + return array(); + } + + $conn = $this->establishConnection('r'); + $column = $this->getDimensionColumnName(); + + $rows = queryfx_all( + $conn, + 'SELECT id, %C FROM %T WHERE %C IN (%Ls)', + $column, + $this->getTableName(), + $column, + $keys); + $rows = ipull($rows, 'id', $column); + + $map = array(); + $need = array(); + foreach ($keys as $key) { + if (isset($rows[$key])) { + $map[$key] = (int)$rows[$key]; + } else { + $need[] = $key; + } + } + + if (!$need) { + return $map; + } + + $sql = array(); + foreach ($need as $key) { + $sql[] = qsprintf( + $conn, + '(%s)', + $key); + } + + foreach (PhabricatorLiskDAO::chunkSQL($sql) as $chunk) { + queryfx( + $conn, + 'INSERT IGNORE INTO %T (%C) VALUES %Q', + $this->getTableName(), + $column, + $chunk); + } + + $rows = queryfx_all( + $conn, + 'SELECT id, %C FROM %T WHERE %C IN (%Ls)', + $column, + $this->getTableName(), + $column, + $need); + $rows = ipull($rows, 'id', $column); + + foreach ($keys as $key) { + if (isset($rows[$key])) { + $map[$key] = (int)$rows[$key]; + } else { + throw new Exception( + pht( + 'Failed to load or generate dimension ID ("%s") for dimension '. + 'key "%s".', + get_class($this), + $key)); + } + } + + return $map; + } + +} diff --git a/src/applications/fact/storage/PhabricatorFactIntDatapoint.php b/src/applications/fact/storage/PhabricatorFactIntDatapoint.php new file mode 100644 --- /dev/null +++ b/src/applications/fact/storage/PhabricatorFactIntDatapoint.php @@ -0,0 +1,61 @@ + false, + self::CONFIG_COLUMN_SCHEMA => array( + 'id' => 'auto64', + 'dimensionID' => 'id?', + 'value' => 'sint64', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_dimension' => array( + 'columns' => array('keyID', 'dimensionID'), + ), + 'key_object' => array( + 'columns' => array('objectID'), + ), + ), + ) + parent::getConfiguration(); + } + + public function setKey($key) { + $this->key = $key; + return $this; + } + + public function getKey() { + return $this->key; + } + + public function setObjectPHID($object_phid) { + $this->objectPHID = $object_phid; + return $this; + } + + public function getObjectPHID() { + return $this->objectPHID; + } + + public function setDimensionPHID($dimension_phid) { + $this->dimensionPHID = $dimension_phid; + return $this; + } + + public function getDimensionPHID() { + return $this->dimensionPHID; + } + +} diff --git a/src/applications/fact/storage/PhabricatorFactKeyDimension.php b/src/applications/fact/storage/PhabricatorFactKeyDimension.php new file mode 100644 --- /dev/null +++ b/src/applications/fact/storage/PhabricatorFactKeyDimension.php @@ -0,0 +1,27 @@ + false, + self::CONFIG_COLUMN_SCHEMA => array( + 'factKey' => 'text64', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_factkey' => array( + 'columns' => array('factKey'), + 'unique' => true, + ), + ), + ) + parent::getConfiguration(); + } + + protected function getDimensionColumnName() { + return 'factKey'; + } + +} diff --git a/src/applications/fact/storage/PhabricatorFactObjectDimension.php b/src/applications/fact/storage/PhabricatorFactObjectDimension.php new file mode 100644 --- /dev/null +++ b/src/applications/fact/storage/PhabricatorFactObjectDimension.php @@ -0,0 +1,25 @@ + false, + self::CONFIG_COLUMN_SCHEMA => array(), + self::CONFIG_KEY_SCHEMA => array( + 'key_object' => array( + 'columns' => array('objectPHID'), + 'unique' => true, + ), + ), + ) + parent::getConfiguration(); + } + + protected function getDimensionColumnName() { + return 'objectPHID'; + } + +} diff --git a/src/applications/ponder/application/PhabricatorPonderApplication.php b/src/applications/ponder/application/PhabricatorPonderApplication.php --- a/src/applications/ponder/application/PhabricatorPonderApplication.php +++ b/src/applications/ponder/application/PhabricatorPonderApplication.php @@ -18,12 +18,6 @@ return 'fa-university'; } - public function getFactObjectsForAnalysis() { - return array( - new PonderQuestion(), - ); - } - public function getTitleGlyph() { return "\xE2\x97\xB3"; }