diff --git a/src/applications/doorkeeper/bridge/DoorkeeperBridge.php b/src/applications/doorkeeper/bridge/DoorkeeperBridge.php index b09a1933c1..4a4ee2667b 100644 --- a/src/applications/doorkeeper/bridge/DoorkeeperBridge.php +++ b/src/applications/doorkeeper/bridge/DoorkeeperBridge.php @@ -1,51 +1,79 @@ throwOnMissingLink = $throw_on_missing_link; return $this; } final public function setViewer($viewer) { $this->viewer = $viewer; return $this; } final public function getViewer() { return $this->viewer; } final public function setContext($context) { $this->context = $context; return $this; } final public function getContextProperty($key, $default = null) { return idx($this->context, $key, $default); } public function isEnabled() { return true; } abstract public function canPullRef(DoorkeeperObjectRef $ref); abstract public function pullRefs(array $refs); public function fillObjectFromData(DoorkeeperExternalObject $obj, $result) { return; } public function didFailOnMissingLink() { if ($this->throwOnMissingLink) { throw new DoorkeeperMissingLinkException(); } return null; } + final protected function saveExternalObject( + DoorkeeperObjectRef $ref, + DoorkeeperExternalObject $obj) { + + $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); + try { + $obj->save(); + } catch (AphrontDuplicateKeyQueryException $ex) { + + // In various cases, we may race another process importing the same + // data. If we do, we'll collide on the object key. Load the object + // the other process created and use that. + $obj = id(new DoorkeeperExternalObjectQuery()) + ->setViewer($this->getViewer()) + ->withObjectKeys(array($ref->getObjectKey())) + ->executeOne(); + if (!$obj) { + throw new PhutilProxyException( + pht('Failed to load external object after collision.'), + $ex); + } + + $ref->attachExternalObject($obj); + } + unset($unguarded); + } + + } diff --git a/src/applications/doorkeeper/bridge/DoorkeeperBridgeAsana.php b/src/applications/doorkeeper/bridge/DoorkeeperBridgeAsana.php index 4e32239793..ec604e158e 100644 --- a/src/applications/doorkeeper/bridge/DoorkeeperBridgeAsana.php +++ b/src/applications/doorkeeper/bridge/DoorkeeperBridgeAsana.php @@ -1,129 +1,126 @@ getApplicationType() != self::APPTYPE_ASANA) { return false; } if ($ref->getApplicationDomain() != self::APPDOMAIN_ASANA) { return false; } $types = array( self::OBJTYPE_TASK => true, ); return isset($types[$ref->getObjectType()]); } public function pullRefs(array $refs) { $id_map = mpull($refs, 'getObjectID', 'getObjectKey'); $viewer = $this->getViewer(); $provider = PhabricatorAsanaAuthProvider::getAsanaProvider(); if (!$provider) { return; } $accounts = id(new PhabricatorExternalAccountQuery()) ->setViewer($viewer) ->withUserPHIDs(array($viewer->getPHID())) ->withAccountTypes(array($provider->getProviderType())) ->withAccountDomains(array($provider->getProviderDomain())) ->requireCapabilities( array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, )) ->execute(); if (!$accounts) { return $this->didFailOnMissingLink(); } // TODO: If the user has several linked Asana accounts, we just pick the // first one arbitrarily. We might want to try using all of them or do // something with more finesse. There's no UI way to link multiple accounts // right now so this is currently moot. $account = head($accounts); $token = $provider->getOAuthAccessToken($account); if (!$token) { return; } $template = id(new PhutilAsanaFuture()) ->setAccessToken($token); $futures = array(); foreach ($id_map as $key => $id) { $futures[$key] = id(clone $template) ->setRawAsanaQuery("tasks/{$id}"); } $results = array(); $failed = array(); foreach (new FutureIterator($futures) as $key => $future) { try { $results[$key] = $future->resolve(); } catch (Exception $ex) { if (($ex instanceof HTTPFutureResponseStatus) && ($ex->getStatusCode() == 404)) { // This indicates that the object has been deleted (or never existed, // or isn't visible to the current user) but it's a successful sync of // an object which isn't visible. } else { // This is something else, so consider it a synchronization failure. phlog($ex); $failed[$key] = $ex; } } } foreach ($refs as $ref) { $ref->setAttribute('name', pht('Asana Task %s', $ref->getObjectID())); $did_fail = idx($failed, $ref->getObjectKey()); if ($did_fail) { $ref->setSyncFailed(true); continue; } $result = idx($results, $ref->getObjectKey()); if (!$result) { continue; } $ref->setIsVisible(true); $ref->setAttribute('asana.data', $result); $ref->setAttribute('fullname', pht('Asana: %s', $result['name'])); $ref->setAttribute('title', $result['name']); $ref->setAttribute('description', $result['notes']); $obj = $ref->getExternalObject(); if ($obj->getID()) { continue; } $this->fillObjectFromData($obj, $result); - - $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); - $obj->save(); - unset($unguarded); + $this->saveExternalObject($ref, $obj); } } public function fillObjectFromData(DoorkeeperExternalObject $obj, $result) { $id = $result['id']; $uri = "https://app.asana.com/0/{$id}/{$id}"; $obj->setObjectURI($uri); } } diff --git a/src/applications/doorkeeper/bridge/DoorkeeperBridgeGitHubIssue.php b/src/applications/doorkeeper/bridge/DoorkeeperBridgeGitHubIssue.php index 0558b7d8f6..b5d7ddeea5 100644 --- a/src/applications/doorkeeper/bridge/DoorkeeperBridgeGitHubIssue.php +++ b/src/applications/doorkeeper/bridge/DoorkeeperBridgeGitHubIssue.php @@ -1,107 +1,104 @@ getObjectType() !== self::OBJTYPE_GITHUB_ISSUE) { return false; } return true; } public function pullRefs(array $refs) { $token = $this->getGitHubAccessToken(); if (!strlen($token)) { return null; } $template = id(new PhutilGitHubFuture()) ->setAccessToken($token); $futures = array(); $id_map = mpull($refs, 'getObjectID', 'getObjectKey'); foreach ($id_map as $key => $id) { list($user, $repository, $number) = $this->parseGitHubIssueID($id); $uri = "/repos/{$user}/{$repository}/issues/{$number}"; $data = array(); $futures[$key] = id(clone $template) ->setRawGitHubQuery($uri, $data); } $results = array(); $failed = array(); foreach (new FutureIterator($futures) as $key => $future) { try { $results[$key] = $future->resolve(); } catch (Exception $ex) { if (($ex instanceof HTTPFutureResponseStatus) && ($ex->getStatusCode() == 404)) { // TODO: Do we end up here for deleted objects and invisible // objects? } else { phlog($ex); $failed[$key] = $ex; } } } $viewer = $this->getViewer(); foreach ($refs as $ref) { $ref->setAttribute('name', pht('GitHub Issue %s', $ref->getObjectID())); $did_fail = idx($failed, $ref->getObjectKey()); if ($did_fail) { $ref->setSyncFailed(true); continue; } $result = idx($results, $ref->getObjectKey()); if (!$result) { continue; } $body = $result->getBody(); $ref->setIsVisible(true); $ref->setAttribute('api.raw', $body); $ref->setAttribute('name', $body['title']); $obj = $ref->getExternalObject(); $this->fillObjectFromData($obj, $result); - - $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); - $obj->save(); - unset($unguarded); + $this->saveExternalObject($ref, $obj); } } public function fillObjectFromData(DoorkeeperExternalObject $obj, $result) { $body = $result->getBody(); $uri = $body['html_url']; $obj->setObjectURI($uri); $title = idx($body, 'title'); $description = idx($body, 'body'); $created = idx($body, 'created_at'); $created = strtotime($created); $state = idx($body, 'state'); $obj->setProperty('task.title', $title); $obj->setProperty('task.description', $description); $obj->setProperty('task.created', $created); $obj->setProperty('task.state', $state); } } diff --git a/src/applications/doorkeeper/bridge/DoorkeeperBridgeGitHubUser.php b/src/applications/doorkeeper/bridge/DoorkeeperBridgeGitHubUser.php index 3470894e5c..ef31826eb3 100644 --- a/src/applications/doorkeeper/bridge/DoorkeeperBridgeGitHubUser.php +++ b/src/applications/doorkeeper/bridge/DoorkeeperBridgeGitHubUser.php @@ -1,120 +1,117 @@ getObjectType() !== self::OBJTYPE_GITHUB_USER) { return false; } return true; } public function pullRefs(array $refs) { $token = $this->getGitHubAccessToken(); if (!strlen($token)) { return null; } $template = id(new PhutilGitHubFuture()) ->setAccessToken($token); $futures = array(); $id_map = mpull($refs, 'getObjectID', 'getObjectKey'); foreach ($id_map as $key => $id) { // GitHub doesn't provide a way to query for users by ID directly, but we // can list all users, ordered by ID, starting at some particular ID, // with a page size of one, which will achieve the desired effect. $one_less = ($id - 1); $uri = "/users?since={$one_less}&per_page=1"; $data = array(); $futures[$key] = id(clone $template) ->setRawGitHubQuery($uri, $data); } $results = array(); $failed = array(); foreach (new FutureIterator($futures) as $key => $future) { try { $results[$key] = $future->resolve(); } catch (Exception $ex) { if (($ex instanceof HTTPFutureResponseStatus) && ($ex->getStatusCode() == 404)) { // TODO: Do we end up here for deleted objects and invisible // objects? } else { phlog($ex); $failed[$key] = $ex; } } } $viewer = $this->getViewer(); foreach ($refs as $ref) { $ref->setAttribute('name', pht('GitHub User %s', $ref->getObjectID())); $did_fail = idx($failed, $ref->getObjectKey()); if ($did_fail) { $ref->setSyncFailed(true); continue; } $result = idx($results, $ref->getObjectKey()); if (!$result) { continue; } $body = $result->getBody(); if (!is_array($body) || !count($body)) { $ref->setSyncFailed(true); continue; } $spec = head($body); if (!is_array($spec)) { $ref->setSyncFailed(true); continue; } // Because we're using a paging query to load each user, if a user (say, // user ID 123) does not exist for some reason, we might get the next // user (say, user ID 124) back. Make sure the user we got back is really // the user we expect. $id = idx($spec, 'id'); if ($id !== $ref->getObjectID()) { $ref->setSyncFailed(true); continue; } $ref->setIsVisible(true); $ref->setAttribute('api.raw', $spec); $ref->setAttribute('name', $spec['login']); $obj = $ref->getExternalObject(); $this->fillObjectFromData($obj, $spec); - - $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); - $obj->save(); - unset($unguarded); + $this->saveExternalObject($ref, $obj); } } public function fillObjectFromData(DoorkeeperExternalObject $obj, $spec) { $uri = $spec['html_url']; $obj->setObjectURI($uri); $login = $spec['login']; $obj->setDisplayName(pht('%s <%s>', $login, pht('GitHub'))); } } diff --git a/src/applications/doorkeeper/bridge/DoorkeeperBridgeJIRA.php b/src/applications/doorkeeper/bridge/DoorkeeperBridgeJIRA.php index c9b40d2e72..199f049dce 100644 --- a/src/applications/doorkeeper/bridge/DoorkeeperBridgeJIRA.php +++ b/src/applications/doorkeeper/bridge/DoorkeeperBridgeJIRA.php @@ -1,148 +1,145 @@ getApplicationType() != self::APPTYPE_JIRA) { return false; } $types = array( self::OBJTYPE_ISSUE => true, ); return isset($types[$ref->getObjectType()]); } public function pullRefs(array $refs) { $id_map = mpull($refs, 'getObjectID', 'getObjectKey'); $viewer = $this->getViewer(); $provider = PhabricatorJIRAAuthProvider::getJIRAProvider(); if (!$provider) { return; } $accounts = id(new PhabricatorExternalAccountQuery()) ->setViewer($viewer) ->withUserPHIDs(array($viewer->getPHID())) ->withAccountTypes(array($provider->getProviderType())) ->requireCapabilities( array( PhabricatorPolicyCapability::CAN_VIEW, PhabricatorPolicyCapability::CAN_EDIT, )) ->execute(); if (!$accounts) { return $this->didFailOnMissingLink(); } // TODO: When we support multiple JIRA instances, we need to disambiguate // issues (perhaps with additional configuration) or cast a wide net // (by querying all instances). For now, just query the one instance. $account = head($accounts); $futures = array(); foreach ($id_map as $key => $id) { $futures[$key] = $provider->newJIRAFuture( $account, 'rest/api/2/issue/'.phutil_escape_uri($id), 'GET'); } $results = array(); $failed = array(); foreach (new FutureIterator($futures) as $key => $future) { try { $results[$key] = $future->resolveJSON(); } catch (Exception $ex) { if (($ex instanceof HTTPFutureResponseStatus) && ($ex->getStatusCode() == 404)) { // This indicates that the object has been deleted (or never existed, // or isn't visible to the current user) but it's a successful sync of // an object which isn't visible. } else { // This is something else, so consider it a synchronization failure. phlog($ex); $failed[$key] = $ex; } } } foreach ($refs as $ref) { $ref->setAttribute('name', pht('JIRA %s', $ref->getObjectID())); $did_fail = idx($failed, $ref->getObjectKey()); if ($did_fail) { $ref->setSyncFailed(true); continue; } $result = idx($results, $ref->getObjectKey()); if (!$result) { continue; } $fields = idx($result, 'fields', array()); $ref->setIsVisible(true); $ref->setAttribute( 'fullname', pht('JIRA %s %s', $result['key'], idx($fields, 'summary'))); $ref->setAttribute('title', idx($fields, 'summary')); $ref->setAttribute('description', idx($result, 'description')); $obj = $ref->getExternalObject(); if ($obj->getID()) { continue; } $this->fillObjectFromData($obj, $result); - - $unguarded = AphrontWriteGuard::beginScopedUnguardedWrites(); - $obj->save(); - unset($unguarded); + $this->saveExternalObject($ref, $obj); } } public function fillObjectFromData(DoorkeeperExternalObject $obj, $result) { // Convert the "self" URI, which points at the REST endpoint, into a // browse URI. $self = idx($result, 'self'); $object_id = $obj->getObjectID(); $uri = self::getJIRAIssueBrowseURIFromJIRARestURI($self, $object_id); if ($uri !== null) { $obj->setObjectURI($uri); } } public static function getJIRAIssueBrowseURIFromJIRARestURI( $uri, $object_id) { $uri = new PhutilURI($uri); // The JIRA install might not be at the domain root, so we may need to // keep an initial part of the path, like "/jira/". Find the API specific // part of the URI, strip it off, then replace it with the web version. $path = $uri->getPath(); $pos = strrpos($path, 'rest/api/2/issue/'); if ($pos === false) { return null; } $path = substr($path, 0, $pos); $path = $path.'browse/'.$object_id; $uri->setPath($path); return (string)$uri; } }