diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1421,6 +1421,8 @@ 'NuanceController' => 'applications/nuance/controller/NuanceController.php', 'NuanceDAO' => 'applications/nuance/storage/NuanceDAO.php', 'NuanceGitHubEventItemType' => 'applications/nuance/item/NuanceGitHubEventItemType.php', + 'NuanceGitHubImportCursor' => 'applications/nuance/cursor/NuanceGitHubImportCursor.php', + 'NuanceGitHubIssuesImportCursor' => 'applications/nuance/cursor/NuanceGitHubIssuesImportCursor.php', 'NuanceGitHubRepositoryImportCursor' => 'applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php', 'NuanceGitHubRepositorySourceDefinition' => 'applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php', 'NuanceImportCursor' => 'applications/nuance/cursor/NuanceImportCursor.php', @@ -5677,7 +5679,9 @@ 'NuanceController' => 'PhabricatorController', 'NuanceDAO' => 'PhabricatorLiskDAO', 'NuanceGitHubEventItemType' => 'NuanceItemType', - 'NuanceGitHubRepositoryImportCursor' => 'NuanceImportCursor', + 'NuanceGitHubImportCursor' => 'NuanceImportCursor', + 'NuanceGitHubIssuesImportCursor' => 'NuanceGitHubImportCursor', + 'NuanceGitHubRepositoryImportCursor' => 'NuanceGitHubImportCursor', 'NuanceGitHubRepositorySourceDefinition' => 'NuanceSourceDefinition', 'NuanceImportCursor' => 'Phobject', 'NuanceImportCursorData' => array( diff --git a/src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php b/src/applications/nuance/cursor/NuanceGitHubImportCursor.php copy from src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php copy to src/applications/nuance/cursor/NuanceGitHubImportCursor.php --- a/src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php +++ b/src/applications/nuance/cursor/NuanceGitHubImportCursor.php @@ -1,11 +1,27 @@ = $now)) { $this->logInfo( pht( - 'Respecting "%s": waiting for %s second(s) to poll GitHub.', + 'Respecting "%s" or minimum poll delay: waiting for %s second(s) '. + 'to poll GitHub.', 'X-Poll-Interval', new PhutilNumber(1 + ($ttl - $now)))); @@ -36,7 +53,7 @@ return true; } - protected function pullDataFromSource() { + final protected function pullDataFromSource() { $viewer = $this->getViewer(); $now = PhabricatorTime::getNow(); @@ -51,10 +68,16 @@ $etag = null; $new_items = array(); $hit_known_items = false; - for ($page = 1; $page <= 10; $page++) { - $uri = "/repos/{$user}/{$repository}/events"; + + $max_page = $this->getMaximumPage(); + $page_size = $this->getPageSize(); + + for ($page = 1; $page <= $max_page; $page++) { + $uri = $this->getGitHubAPIEndpointURI($user, $repository); + $data = array( 'page' => $page, + 'per_page' => $page_size, ); $future = id(new PhutilGitHubFuture()) @@ -98,7 +121,7 @@ $records = $response->getBody(); foreach ($records as $record) { - $item = $this->newNuanceItemFromGitHubEvent($record); + $item = $this->newNuanceItemFromGitHubRecord($record); $item_key = $item->getItemKey(); $this->logInfo( @@ -133,7 +156,7 @@ break; } - if (count($records) < 30) { + if (count($records) < $page_size) { break; } } @@ -218,6 +241,8 @@ } $poll_interval = (int)$response->getHeaderValue('X-Poll-Interval'); + $poll_interval = max($this->getMinimumDelayBetweenPolls(), $poll_interval); + $poll_ttl = $start + $poll_interval; $this->setCursorProperty('github.poll.ttl', $poll_ttl); @@ -230,32 +255,4 @@ new PhutilNumber($poll_ttl - $now))); } - private function newNuanceItemFromGitHubEvent(array $record) { - $source = $this->getSource(); - - $id = $record['id']; - $item_key = "github.event.{$id}"; - - $container_key = null; - - $issue_id = idxv( - $record, - array( - 'payload', - 'issue', - 'id', - )); - if ($issue_id) { - $container_key = "github.issue.{$issue_id}"; - } - - return NuanceItem::initializeNewItem() - ->setStatus(NuanceItem::STATUS_IMPORTING) - ->setSourcePHID($source->getPHID()) - ->setItemType(NuanceGitHubEventItemType::ITEMTYPE) - ->setItemKey($item_key) - ->setItemContainerKey($container_key) - ->setItemProperty('api.raw', $record); - } - } diff --git a/src/applications/nuance/cursor/NuanceGitHubIssuesImportCursor.php b/src/applications/nuance/cursor/NuanceGitHubIssuesImportCursor.php new file mode 100644 --- /dev/null +++ b/src/applications/nuance/cursor/NuanceGitHubIssuesImportCursor.php @@ -0,0 +1,30 @@ +getSource(); + + $id = $record['id']; + $item_key = "github.issueevent.{$id}"; + + $container_key = null; + + return NuanceItem::initializeNewItem() + ->setStatus(NuanceItem::STATUS_IMPORTING) + ->setSourcePHID($source->getPHID()) + ->setItemType(NuanceGitHubEventItemType::ITEMTYPE) + ->setItemKey($item_key) + ->setItemContainerKey($container_key) + ->setItemProperty('api.type', 'issue') + ->setItemProperty('api.raw', $record); + } + +} diff --git a/src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php b/src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php --- a/src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php +++ b/src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php @@ -1,236 +1,23 @@ getCursorProperty('github.poll.ttl'); - if ($ttl && ($ttl >= $now)) { - $this->logInfo( - pht( - 'Respecting "%s": waiting for %s second(s) to poll GitHub.', - 'X-Poll-Interval', - new PhutilNumber(1 + ($ttl - $now)))); - - return false; - } - - // Respect GitHub's API rate limiting. If we've exceeded the rate limit, - // wait until it resets to try again. - $limit = $this->getCursorProperty('github.limit.ttl'); - if ($limit && ($limit >= $now)) { - $this->logInfo( - pht( - 'Respecting "%s": waiting for %s second(s) to poll GitHub.', - 'X-RateLimit-Reset', - new PhutilNumber(1 + ($limit - $now)))); - return false; - } - - return true; - } - - protected function pullDataFromSource() { - $viewer = $this->getViewer(); - $now = PhabricatorTime::getNow(); - - $source = $this->getSource(); - - $user = $source->getSourceProperty('github.user'); - $repository = $source->getSourceProperty('github.repository'); - $api_token = $source->getSourceProperty('github.token'); - - // This API only supports fetching 10 pages of 30 events each, for a total - // of 300 events. - $etag = null; - $new_items = array(); - $hit_known_items = false; - for ($page = 1; $page <= 10; $page++) { - $uri = "/repos/{$user}/{$repository}/events"; - $data = array( - 'page' => $page, - ); - - $future = id(new PhutilGitHubFuture()) - ->setAccessToken($api_token) - ->setRawGitHubQuery($uri, $data); - - if ($page == 1) { - $cursor_etag = $this->getCursorProperty('github.poll.etag'); - if ($cursor_etag) { - $future->addHeader('If-None-Match', $cursor_etag); - } - } - - $this->logInfo( - pht( - 'Polling GitHub Repository API endpoint "%s".', - $uri)); - $response = $future->resolve(); - - // Do this first: if we hit the rate limit, we get a response but the - // body isn't valid. - $this->updateRateLimits($response); - - if ($response->getStatus()->getStatusCode() == 304) { - $this->logInfo( - pht( - 'Received a 304 Not Modified from GitHub, no new events.')); - } - - // This means we hit a rate limit or a "Not Modified" because of the - // "ETag" header. In either case, we should bail out. - if ($response->getStatus()->isError()) { - $this->updatePolling($response, $now, false); - $this->getCursorData()->save(); - return false; - } - - if ($page == 1) { - $etag = $response->getHeaderValue('ETag'); - } - - $records = $response->getBody(); - foreach ($records as $record) { - $item = $this->newNuanceItemFromGitHubEvent($record); - $item_key = $item->getItemKey(); - - $this->logInfo( - pht( - 'Fetched event "%s".', - $item_key)); - - $new_items[$item->getItemKey()] = $item; - } - - if ($new_items) { - $existing = id(new NuanceItemQuery()) - ->setViewer($viewer) - ->withSourcePHIDs(array($source->getPHID())) - ->withItemKeys(array_keys($new_items)) - ->execute(); - $existing = mpull($existing, null, 'getItemKey'); - foreach ($new_items as $key => $new_item) { - if (isset($existing[$key])) { - unset($new_items[$key]); - $hit_known_items = true; - - $this->logInfo( - pht( - 'Event "%s" is previously known.', - $key)); - } - } - } - - if ($hit_known_items) { - break; - } - - if (count($records) < 30) { - break; - } - } - - // TODO: When we go through the whole queue without hitting anything we - // have seen before, we should record some sort of global event so we - // can tell the user when the bridging started or was interrupted? - if (!$hit_known_items) { - $already_polled = $this->getCursorProperty('github.polled'); - if ($already_polled) { - // TODO: This is bad: we missed some items, maybe because too much - // stuff happened too fast or the daemons were broken for a long - // time. - } else { - // TODO: This is OK, we're doing the initial import. - } - } - - if ($etag !== null) { - $this->updateETag($etag); - } - - $this->updatePolling($response, $now, true); - - // Reverse the new items so we insert them in chronological order. - $new_items = array_reverse($new_items); - - $source->openTransaction(); - foreach ($new_items as $new_item) { - $new_item->save(); - } - $this->getCursorData()->save(); - $source->saveTransaction(); - - foreach ($new_items as $new_item) { - $new_item->scheduleUpdate(); - } - - return false; + protected function getGitHubAPIEndpointURI($user, $repository) { + return "/repos/{$user}/{$repository}/events"; } - private function updateRateLimits(PhutilGitHubResponse $response) { - $remaining = $response->getHeaderValue('X-RateLimit-Remaining'); - $limit_reset = $response->getHeaderValue('X-RateLimit-Reset'); - $now = PhabricatorTime::getNow(); - - $limit_ttl = null; - if (strlen($remaining)) { - $remaining = (int)$remaining; - if (!$remaining) { - $limit_ttl = (int)$limit_reset; - } - } - - $this->setCursorProperty('github.limit.ttl', $limit_ttl); - - $this->logInfo( - pht( - 'This key has %s remaining API request(s), '. - 'limit resets in %s second(s).', - new PhutilNumber($remaining), - new PhutilNumber($limit_reset - $now))); - } - - private function updateETag($etag) { - - $this->setCursorProperty('github.poll.etag', $etag); - - $this->logInfo( - pht( - 'ETag for this request was "%s".', - $etag)); + protected function getMaximumPage() { + return 10; } - private function updatePolling( - PhutilGitHubResponse $response, - $start, - $success) { - - if ($success) { - $this->setCursorProperty('github.polled', true); - } - - $poll_interval = (int)$response->getHeaderValue('X-Poll-Interval'); - $poll_ttl = $start + $poll_interval; - $this->setCursorProperty('github.poll.ttl', $poll_ttl); - - $now = PhabricatorTime::getNow(); - - $this->logInfo( - pht( - 'Set API poll TTL to +%s second(s) (%s second(s) from now).', - new PhutilNumber($poll_interval), - new PhutilNumber($poll_ttl - $now))); + protected function getPageSize() { + return 30; } - private function newNuanceItemFromGitHubEvent(array $record) { + protected function newNuanceItemFromGitHubRecord(array $record) { $source = $this->getSource(); $id = $record['id']; @@ -255,6 +42,7 @@ ->setItemType(NuanceGitHubEventItemType::ITEMTYPE) ->setItemKey($item_key) ->setItemContainerKey($container_key) + ->setItemProperty('api.type', 'repository') ->setItemProperty('api.raw', $record); } diff --git a/src/applications/nuance/item/NuanceGitHubEventItemType.php b/src/applications/nuance/item/NuanceGitHubEventItemType.php --- a/src/applications/nuance/item/NuanceGitHubEventItemType.php +++ b/src/applications/nuance/item/NuanceGitHubEventItemType.php @@ -14,6 +14,27 @@ } public function getItemDisplayName(NuanceItem $item) { + $api_type = $item->getItemProperty('api.type'); + switch ($api_type) { + case 'issue': + return $this->getGitHubIssueAPIEventDisplayName($item); + case 'repository': + return $this->getGitHubRepositoryAPIEventDisplayName($item); + default: + return pht('GitHub Event (Unknown API Type "%s")', $api_type); + } + } + + private function getGitHubIssueAPIEventDisplayName(NuanceItem $item) { + $raw = $item->getItemProperty('api.raw', array()); + + $action = idxv($raw, array('event')); + $number = idxv($raw, array('issue', 'number')); + + return pht('GitHub Issue #%d (%s)', $number, $action); + } + + private function getGitHubRepositoryAPIEventDisplayName(NuanceItem $item) { $raw = $item->getItemProperty('api.raw', array()); $repo = idxv($raw, array('repo', 'name'), pht('')); diff --git a/src/applications/nuance/management/NuanceManagementImportWorkflow.php b/src/applications/nuance/management/NuanceManagementImportWorkflow.php --- a/src/applications/nuance/management/NuanceManagementImportWorkflow.php +++ b/src/applications/nuance/management/NuanceManagementImportWorkflow.php @@ -15,6 +15,11 @@ 'param' => 'source', 'help' => pht('Choose which source to import.'), ), + array( + 'name' => 'cursor', + 'param' => 'cursor', + 'help' => pht('Import only a particular cursor.'), + ), )); } @@ -40,6 +45,36 @@ $source->getName())); } + $select = $args->getArg('cursor'); + if (strlen($select)) { + if (empty($cursors[$select])) { + throw new PhutilArgumentUsageException( + pht( + 'This source ("%s") does not have a "%s" cursor. Available '. + 'cursors: %s.', + $source->getName(), + $select, + implode(', ', array_keys($cursors)))); + } else { + echo tsprintf( + "%s\n", + pht( + 'Importing cursor "%s" only.', + $select)); + $cursors = array_select_keys($cursors, array($select)); + } + } else { + echo tsprintf( + "%s\n", + pht( + 'Importing all cursors: %s.', + implode(', ', array_keys($cursors)))); + + echo tsprintf( + "%s\n", + pht('(Use --cursor to import only a particular cursor.)')); + } + foreach ($cursors as $cursor) { $cursor->importFromSource(); } diff --git a/src/applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php b/src/applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php --- a/src/applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php +++ b/src/applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php @@ -23,6 +23,8 @@ return array( id(new NuanceGitHubRepositoryImportCursor()) ->setCursorKey('events.repository'), + id(new NuanceGitHubIssuesImportCursor()) + ->setCursorKey('events.issues'), ); } diff --git a/src/applications/nuance/source/NuanceSourceDefinition.php b/src/applications/nuance/source/NuanceSourceDefinition.php --- a/src/applications/nuance/source/NuanceSourceDefinition.php +++ b/src/applications/nuance/source/NuanceSourceDefinition.php @@ -114,7 +114,7 @@ ->setCursorData($cursor_data); } - return $cursors; + return $map; } protected function newImportCursors() {