diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -22,6 +22,7 @@ 'AlmanacBindingTransactionQuery' => 'applications/almanac/query/AlmanacBindingTransactionQuery.php', 'AlmanacBindingViewController' => 'applications/almanac/controller/AlmanacBindingViewController.php', 'AlmanacBindingsSearchEngineAttachment' => 'applications/almanac/engineextension/AlmanacBindingsSearchEngineAttachment.php', + 'AlmanacCacheEngineExtension' => 'applications/almanac/engineextension/AlmanacCacheEngineExtension.php', 'AlmanacClusterDatabaseServiceType' => 'applications/almanac/servicetype/AlmanacClusterDatabaseServiceType.php', 'AlmanacClusterRepositoryServiceType' => 'applications/almanac/servicetype/AlmanacClusterRepositoryServiceType.php', 'AlmanacClusterServiceType' => 'applications/almanac/servicetype/AlmanacClusterServiceType.php', @@ -584,6 +585,7 @@ 'DiffusionBrowseQueryConduitAPIMethod' => 'applications/diffusion/conduit/DiffusionBrowseQueryConduitAPIMethod.php', 'DiffusionBrowseResultSet' => 'applications/diffusion/data/DiffusionBrowseResultSet.php', 'DiffusionBrowseTableView' => 'applications/diffusion/view/DiffusionBrowseTableView.php', + 'DiffusionCacheEngineExtension' => 'applications/diffusion/engineextension/DiffusionCacheEngineExtension.php', 'DiffusionCachedResolveRefsQuery' => 'applications/diffusion/query/DiffusionCachedResolveRefsQuery.php', 'DiffusionChangeController' => 'applications/diffusion/controller/DiffusionChangeController.php', 'DiffusionChangeHeraldFieldGroup' => 'applications/diffusion/herald/DiffusionChangeHeraldFieldGroup.php', @@ -2023,6 +2025,8 @@ 'PhabricatorBulkContentSource' => 'infrastructure/daemon/contentsource/PhabricatorBulkContentSource.php', 'PhabricatorBusyUIExample' => 'applications/uiexample/examples/PhabricatorBusyUIExample.php', 'PhabricatorCacheDAO' => 'applications/cache/storage/PhabricatorCacheDAO.php', + 'PhabricatorCacheEngine' => 'applications/system/engine/PhabricatorCacheEngine.php', + 'PhabricatorCacheEngineExtension' => 'applications/system/engine/PhabricatorCacheEngineExtension.php', 'PhabricatorCacheGeneralGarbageCollector' => 'applications/cache/garbagecollector/PhabricatorCacheGeneralGarbageCollector.php', 'PhabricatorCacheManagementPurgeWorkflow' => 'applications/cache/management/PhabricatorCacheManagementPurgeWorkflow.php', 'PhabricatorCacheManagementWorkflow' => 'applications/cache/management/PhabricatorCacheManagementWorkflow.php', @@ -4571,6 +4575,7 @@ 'AlmanacBindingTransactionQuery' => 'PhabricatorApplicationTransactionQuery', 'AlmanacBindingViewController' => 'AlmanacServiceController', 'AlmanacBindingsSearchEngineAttachment' => 'AlmanacSearchEngineAttachment', + 'AlmanacCacheEngineExtension' => 'PhabricatorCacheEngineExtension', 'AlmanacClusterDatabaseServiceType' => 'AlmanacClusterServiceType', 'AlmanacClusterRepositoryServiceType' => 'AlmanacClusterServiceType', 'AlmanacClusterServiceType' => 'AlmanacServiceType', @@ -5225,6 +5230,7 @@ 'DiffusionBrowseQueryConduitAPIMethod' => 'DiffusionQueryConduitAPIMethod', 'DiffusionBrowseResultSet' => 'Phobject', 'DiffusionBrowseTableView' => 'DiffusionView', + 'DiffusionCacheEngineExtension' => 'PhabricatorCacheEngineExtension', 'DiffusionCachedResolveRefsQuery' => 'DiffusionLowLevelQuery', 'DiffusionChangeController' => 'DiffusionController', 'DiffusionChangeHeraldFieldGroup' => 'HeraldFieldGroup', @@ -6883,6 +6889,8 @@ 'PhabricatorBulkContentSource' => 'PhabricatorContentSource', 'PhabricatorBusyUIExample' => 'PhabricatorUIExample', 'PhabricatorCacheDAO' => 'PhabricatorLiskDAO', + 'PhabricatorCacheEngine' => 'Phobject', + 'PhabricatorCacheEngineExtension' => 'Phobject', 'PhabricatorCacheGeneralGarbageCollector' => 'PhabricatorGarbageCollector', 'PhabricatorCacheManagementPurgeWorkflow' => 'PhabricatorCacheManagementWorkflow', 'PhabricatorCacheManagementWorkflow' => 'PhabricatorManagementWorkflow', diff --git a/src/applications/almanac/engineextension/AlmanacCacheEngineExtension.php b/src/applications/almanac/engineextension/AlmanacCacheEngineExtension.php new file mode 100644 --- /dev/null +++ b/src/applications/almanac/engineextension/AlmanacCacheEngineExtension.php @@ -0,0 +1,53 @@ +getViewer(); + + $results = array(); + foreach ($this->selectObjects($objects, 'AlmanacBinding') as $object) { + $results[] = $object->getServicePHID(); + $results[] = $object->getDevicePHID(); + $results[] = $object->getInterfacePHID(); + } + + $devices = $this->selectObjects($objects, 'AlmanacDevice'); + if ($devices) { + $interfaces = id(new AlmanacInterfaceQuery()) + ->setViewer($viewer) + ->withDevicePHIDs(mpull($devices, 'getPHID')) + ->execute(); + foreach ($interfaces as $interface) { + $results[] = $interface; + } + } + + foreach ($this->selectObjects($objects, 'AlmanacInterface') as $iface) { + $results[] = $iface->getDevicePHID(); + $results[] = $iface->getNetworkPHID(); + } + + foreach ($this->selectObjects($objects, 'AlmanacProperty') as $object) { + $results[] = $object->getObjectPHID(); + } + + return $results; + } + + public function deleteCaches( + PhabricatorCacheEngine $engine, + array $objects) { + return; + } + +} diff --git a/src/applications/diffusion/engineextension/DiffusionCacheEngineExtension.php b/src/applications/diffusion/engineextension/DiffusionCacheEngineExtension.php new file mode 100644 --- /dev/null +++ b/src/applications/diffusion/engineextension/DiffusionCacheEngineExtension.php @@ -0,0 +1,52 @@ +getViewer(); + $results = array(); + + // When an Almanac Service changes, update linked repositories. + + $services = $this->selectObjects($objects, 'AlmanacService'); + if ($services) { + $repositories = id(new PhabricatorRepositoryQuery()) + ->setViewer($viewer) + ->withAlmanacServicePHIDs(mpull($services, 'getPHID')) + ->execute(); + foreach ($repositories as $repository) { + $results[] = $repository; + } + } + + return $results; + } + + public function deleteCaches( + PhabricatorCacheEngine $engine, + array $objects) { + + $keys = array(); + $repositories = $this->selectObjects($objects, 'PhabricatorRepository'); + foreach ($repositories as $repository) { + $keys[] = $repository->getAlmanacServiceCacheKey(); + } + + $keys = array_filter($keys); + + if ($keys) { + $cache = PhabricatorCaches::getMutableStructureCache(); + $cache->deleteKeys($keys); + } + } + +} diff --git a/src/applications/repository/query/PhabricatorRepositoryQuery.php b/src/applications/repository/query/PhabricatorRepositoryQuery.php --- a/src/applications/repository/query/PhabricatorRepositoryQuery.php +++ b/src/applications/repository/query/PhabricatorRepositoryQuery.php @@ -12,6 +12,7 @@ private $uris; private $datasourceQuery; private $slugs; + private $almanacServicePHIDs; private $numericIdentifiers; private $callsignIdentifiers; @@ -134,6 +135,11 @@ return $this; } + public function withAlmanacServicePHIDs(array $phids) { + $this->almanacServicePHIDs = $phids; + return $this; + } + public function needCommitCounts($need_counts) { $this->needCommitCounts = $need_counts; return $this; @@ -659,6 +665,13 @@ $try_uris); } + if ($this->almanacServicePHIDs !== null) { + $where[] = qsprintf( + $conn, + 'r.almanacServicePHID IN (%Ls)', + $this->almanacServicePHIDs); + } + return $where; } diff --git a/src/applications/repository/storage/PhabricatorRepository.php b/src/applications/repository/storage/PhabricatorRepository.php --- a/src/applications/repository/storage/PhabricatorRepository.php +++ b/src/applications/repository/storage/PhabricatorRepository.php @@ -1865,21 +1865,25 @@ $never_proxy, array $protocols) { - $service = $this->loadAlmanacService(); - if (!$service) { + $cache_key = $this->getAlmanacServiceCacheKey(); + if (!$cache_key) { return null; } - $bindings = $service->getActiveBindings(); - if (!$bindings) { - throw new Exception( - pht( - 'The Almanac service for this repository is not bound to any '. - 'interfaces.')); + $cache = PhabricatorCaches::getMutableStructureCache(); + $uris = $cache->getKey($cache_key, false); + + // If we haven't built the cache yet, build it now. + if ($uris === false) { + $uris = $this->buildAlmanacServiceURIs(); + $cache->setKey($cache_key, $uris); } - $local_device = AlmanacKeys::getDeviceID(); + if ($uris === null) { + return null; + } + $local_device = AlmanacKeys::getDeviceID(); if ($never_proxy && !$local_device) { throw new Exception( pht( @@ -1890,10 +1894,8 @@ $protocol_map = array_fuse($protocols); - $uris = array(); - foreach ($bindings as $binding) { - $iface = $binding->getInterface(); - + $results = array(); + foreach ($uris as $uri) { // If we're never proxying this and it's locally satisfiable, return // `null` to tell the caller to handle it locally. If we're allowed to // proxy, we skip this check and may proxy the request to ourselves. @@ -1901,22 +1903,17 @@ // return `null`, and then the request will actually run.) if ($local_device && $never_proxy) { - if ($iface->getDevice()->getName() == $local_device) { + if ($uri['device'] == $local_device) { return null; } } - $uri = $this->getClusterRepositoryURIFromBinding($binding); - - $protocol = $uri->getProtocol(); - if (empty($protocol_map[$protocol])) { - continue; + if (isset($protocol_map[$uri['protocol']])) { + $results[] = new PhutilURI($uri['uri']); } - - $uris[] = $uri; } - if (!$uris) { + if (!$results) { throw new Exception( pht( 'The Almanac service for this repository is not bound to any '. @@ -1931,10 +1928,51 @@ 'Cluster hosts must correctly route their intracluster requests.')); } - shuffle($uris); - return head($uris); + shuffle($results); + return head($results); } + public function getAlmanacServiceCacheKey() { + $service_phid = $this->getAlmanacServicePHID(); + if (!$service_phid) { + return null; + } + + $repository_phid = $this->getPHID(); + return "diffusion.repository({$repository_phid}).service({$service_phid})"; + } + + private function buildAlmanacServiceURIs() { + $service = $this->loadAlmanacService(); + if (!$service) { + return null; + } + + $bindings = $service->getActiveBindings(); + if (!$bindings) { + throw new Exception( + pht( + 'The Almanac service for this repository is not bound to any '. + 'interfaces.')); + } + + $uris = array(); + foreach ($bindings as $binding) { + $iface = $binding->getInterface(); + + $uri = $this->getClusterRepositoryURIFromBinding($binding); + $protocol = $uri->getProtocol(); + $device_name = $iface->getDevice()->getName(); + + $uris[] = array( + 'protocol' => $protocol, + 'uri' => (string)$uri, + 'device' => $device_name, + ); + } + + return $uris; + } /** * Build a new Conduit client in order to make a service call to this diff --git a/src/applications/system/engine/PhabricatorCacheEngine.php b/src/applications/system/engine/PhabricatorCacheEngine.php new file mode 100644 --- /dev/null +++ b/src/applications/system/engine/PhabricatorCacheEngine.php @@ -0,0 +1,94 @@ +getPHID() => $object, + ); + + $new_objects = $objects; + while (true) { + $discovered_objects = array(); + $load = array(); + + $extensions = PhabricatorCacheEngineExtension::getAllExtensions(); + foreach ($extensions as $key => $extension) { + $discoveries = $extension->discoverLinkedObjects($this, $new_objects); + if (!is_array($discoveries)) { + throw new Exception( + pht( + 'Cache engine extension "%s" did not return a list of linked '. + 'objects.', + get_class($extension))); + } + + foreach ($discoveries as $discovery) { + if ($discovery === null) { + // This is allowed because it makes writing extensions a lot + // easier if they don't have to check that related PHIDs are + // actually set to something. + continue; + } + + $is_phid = is_string($discovery); + if ($is_phid) { + $phid = $discovery; + } else { + $phid = $discovery->getPHID(); + if (!$phid) { + throw new Exception( + pht( + 'Cache engine extension "%s" returned object (of class '. + '"%s") with no PHID.', + get_class($extension), + get_class($discovery))); + } + } + + if (isset($objects[$phid])) { + continue; + } + + if ($is_phid) { + $load[$phid] = $phid; + } else { + $objects[$phid] = $discovery; + $discovered_objects[$phid] = $discovery; + + // If another extension only knew about the PHID of this object, + // we don't need to load it any more. + unset($load[$phid]); + } + } + } + + if ($load) { + $load_objects = id(new PhabricatorObjectQuery()) + ->setViewer($this->getViewer()) + ->withPHIDs($load) + ->execute(); + foreach ($load_objects as $phid => $loaded_object) { + $objects[$phid] = $loaded_object; + $discovered_objects[$phid] = $loaded_object; + } + } + + // If we didn't find anything new to update, we're all set. + if (!$discovered_objects) { + break; + } + + $new_objects = $discovered_objects; + } + + foreach ($extensions as $extension) { + $extension->deleteCaches($this, $objects); + } + } + +} diff --git a/src/applications/system/engine/PhabricatorCacheEngineExtension.php b/src/applications/system/engine/PhabricatorCacheEngineExtension.php new file mode 100644 --- /dev/null +++ b/src/applications/system/engine/PhabricatorCacheEngineExtension.php @@ -0,0 +1,42 @@ +getPhobjectClassConstant('EXTENSIONKEY'); + } + + abstract public function getExtensionName(); + + public function discoverLinkedObjects( + PhabricatorCacheEngine $engine, + array $objects) { + return array(); + } + + public function deleteCaches( + PhabricatorCacheEngine $engine, + array $objects) { + return null; + } + + final public static function getAllExtensions() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getExtensionKey') + ->execute(); + } + + final public function selectObjects(array $objects, $class_name) { + $results = array(); + + foreach ($objects as $phid => $object) { + if ($object instanceof $class_name) { + $results[$phid] = $object; + } + } + + return $results; + } + +} diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -987,6 +987,10 @@ throw $ex; } + // If we need to perform cache engine updates, execute them now. + id(new PhabricatorCacheEngine()) + ->updateObject($object); + // Now that we've completely applied the core transaction set, try to apply // Herald rules. Herald rules are allowed to either take direct actions on // the database (like writing flags), or take indirect actions (like saving