diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1421,6 +1421,8 @@ 'NuanceController' => 'applications/nuance/controller/NuanceController.php', 'NuanceCreateItemConduitAPIMethod' => 'applications/nuance/conduit/NuanceCreateItemConduitAPIMethod.php', 'NuanceDAO' => 'applications/nuance/storage/NuanceDAO.php', + 'NuanceGitHubRepositoryImportCursor' => 'applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php', + 'NuanceGitHubRepositorySourceDefinition' => 'applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php', 'NuanceImportCursor' => 'applications/nuance/cursor/NuanceImportCursor.php', 'NuanceImportCursorData' => 'applications/nuance/storage/NuanceImportCursorData.php', 'NuanceImportCursorDataQuery' => 'applications/nuance/query/NuanceImportCursorDataQuery.php', @@ -5668,6 +5670,8 @@ 'NuanceController' => 'PhabricatorController', 'NuanceCreateItemConduitAPIMethod' => 'NuanceConduitAPIMethod', 'NuanceDAO' => 'PhabricatorLiskDAO', + 'NuanceGitHubRepositoryImportCursor' => 'NuanceImportCursor', + 'NuanceGitHubRepositorySourceDefinition' => 'NuanceSourceDefinition', 'NuanceImportCursor' => 'Phobject', 'NuanceImportCursorData' => 'NuanceDAO', 'NuanceImportCursorDataQuery' => 'NuanceQuery', diff --git a/src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php b/src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php new file mode 100644 --- /dev/null +++ b/src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php @@ -0,0 +1,114 @@ +getCursorProperty('github.poll.ttl'); + if ($ttl && ($ttl >= $now)) { + $this->logInfo( + pht( + 'Respecting "%s": waiting for %s second(s) to poll GitHub.', + 'X-Poll-Interval', + new PhutilNumber(1 + ($ttl - $now)))); + + return false; + } + + // Respect GitHub's API rate limiting. If we've exceeded the rate limit, + // wait until it resets to try again. + $limit = $this->getCursorProperty('github.limit.ttl'); + if ($limit && ($limit >= $now)) { + $this->logInfo( + pht( + 'Respecting "%s": waiting for %s second(s) to poll GitHub.', + 'X-RateLimit-Reset', + new PhutilNumber(1 + ($limit - $now)))); + return false; + } + + return true; + } + + protected function pullDataFromSource() { + $source = $this->getSource(); + + $user = $source->getSourceProperty('github.user'); + $repository = $source->getSourceProperty('github.repository'); + $api_token = $source->getSourceProperty('github.token'); + + $uri = "/repos/{$user}/{$repository}/events"; + $data = array(); + + $future = id(new PhutilGitHubFuture()) + ->setAccessToken($api_token) + ->setRawGitHubQuery($uri, $data); + + $etag = $this->getCursorProperty('github.poll.etag'); + if ($etag) { + $future->addHeader('If-None-Match', $etag); + } + + $this->logInfo( + pht( + 'Polling GitHub Repository API endpoint "%s".', + $uri)); + $response = $future->resolve(); + + // Do this first: if we hit the rate limit, we get a response but the + // body isn't valid. + $this->updateRateLimits($response); + + // This means we hit a rate limit or a "Not Modified" because of the "ETag" + // header. In either case, we should bail out. + if ($response->getStatus()->isError()) { + // TODO: Save cursor data! + return false; + } + + $this->updateETag($response); + + var_dump($response->getBody()); + } + + private function updateRateLimits(PhutilGitHubResponse $response) { + $remaining = $response->getHeaderValue('X-RateLimit-Remaining'); + $limit_reset = $response->getHeaderValue('X-RateLimit-Reset'); + $now = PhabricatorTime::getNow(); + + $limit_ttl = null; + if (strlen($remaining)) { + $remaining = (int)$remaining; + if (!$remaining) { + $limit_ttl = (int)$limit_reset; + } + } + + $this->setCursorProperty('github.limit.ttl', $limit_ttl); + + $this->logInfo( + pht( + 'This key has %s remaining API request(s), '. + 'limit resets in %s second(s).', + new PhutilNumber($remaining), + new PhutilNumber($limit_reset - $now))); + } + + private function updateETag(PhutilGitHubResponse $response) { + $etag = $response->getHeaderValue('ETag'); + + $this->setCursorProperty('github.poll.etag', $etag); + + $this->logInfo( + pht( + 'ETag for this request was "%s".', + $etag)); + } + +} diff --git a/src/applications/nuance/cursor/NuanceImportCursor.php b/src/applications/nuance/cursor/NuanceImportCursor.php --- a/src/applications/nuance/cursor/NuanceImportCursor.php +++ b/src/applications/nuance/cursor/NuanceImportCursor.php @@ -2,9 +2,97 @@ abstract class NuanceImportCursor extends Phobject { + private $cursorData; + private $cursorKey; + private $source; + + abstract protected function shouldPullDataFromSource(); + abstract protected function pullDataFromSource(); + + final public function getCursorType() { + return $this->getPhobjectClassConstant('CURSORTYPE', 32); + } + + public function setCursorData(NuanceImportCursorData $cursor_data) { + $this->cursorData = $cursor_data; + return $this; + } + + public function getCursorData() { + return $this->cursorData; + } + + public function setSource($source) { + $this->source = $source; + return $this; + } + + public function getSource() { + return $this->source; + } + + public function setCursorKey($cursor_key) { + $this->cursorKey = $cursor_key; + return $this; + } + + public function getCursorKey() { + return $this->cursorKey; + } + final public function importFromSource() { - // TODO: Perhaps, do something. - return false; + if (!$this->shouldPullDataFromSource()) { + return false; + } + + $source = $this->getSource(); + $key = $this->getCursorKey(); + + $parts = array( + 'nsc', + $source->getID(), + PhabricatorHash::digestToLength($key, 20), + ); + $lock_name = implode('.', $parts); + + $lock = PhabricatorGlobalLock::newLock($lock_name); + $lock->lock(1); + + try { + $more_data = $this->pullDataFromSource(); + } catch (Exception $ex) { + $lock->unlock(); + throw $ex; + } + + $lock->unlock(); + + return $more_data; + } + + final public function newEmptyCursorData(NuanceSource $source) { + return id(new NuanceImportCursorData()) + ->setCursorKey($this->getCursorKey()) + ->setCursorType($this->getCursorType()) + ->setSourcePHID($source->getPHID()); + } + + final protected function logInfo($message) { + echo tsprintf( + " %s\n", + $this->getCursorKey(), + $message); + + return $this; + } + + final protected function getCursorProperty($key, $default = null) { + return $this->getCursorData()->getCursorProperty($key, $default); + } + + final protected function setCursorProperty($key, $value) { + $this->getCursorData()->setCursorProperty($key, $value); + return $this; } } diff --git a/src/applications/nuance/management/NuanceManagementImportWorkflow.php b/src/applications/nuance/management/NuanceManagementImportWorkflow.php --- a/src/applications/nuance/management/NuanceManagementImportWorkflow.php +++ b/src/applications/nuance/management/NuanceManagementImportWorkflow.php @@ -40,9 +40,9 @@ $source->getName())); } - echo tsprintf( - "%s\n", - pht('OK, but actual importing is not implemented yet.')); + foreach ($cursors as $cursor) { + $cursor->importFromSource(); + } return 0; } diff --git a/src/applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php b/src/applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php new file mode 100644 --- /dev/null +++ b/src/applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php @@ -0,0 +1,29 @@ +setCursorKey('events.repository'), + ); + } + +} diff --git a/src/applications/nuance/source/NuancePhabricatorFormSourceDefinition.php b/src/applications/nuance/source/NuancePhabricatorFormSourceDefinition.php --- a/src/applications/nuance/source/NuancePhabricatorFormSourceDefinition.php +++ b/src/applications/nuance/source/NuancePhabricatorFormSourceDefinition.php @@ -26,15 +26,6 @@ return $actions; } - public function updateItems() { - return null; - } - - public function renderView() {} - - public function renderListView() {} - - public function handleActionRequest(AphrontRequest $request) { $viewer = $request->getViewer(); diff --git a/src/applications/nuance/source/NuanceSourceDefinition.php b/src/applications/nuance/source/NuanceSourceDefinition.php --- a/src/applications/nuance/source/NuanceSourceDefinition.php +++ b/src/applications/nuance/source/NuanceSourceDefinition.php @@ -53,7 +53,66 @@ pht('This source has no input cursors.')); } - return $this->newImportCursors(); + $source = $this->getSource(); + $cursors = $this->newImportCursors(); + + $data = id(new NuanceImportCursorDataQuery()) + ->setViewer(PhabricatorUser::getOmnipotentUser()) + ->withSourcePHIDs(array($source->getPHID())) + ->execute(); + $data = mpull($data, 'getCursorKey'); + + $map = array(); + foreach ($cursors as $cursor) { + if (!($cursor instanceof NuanceImportCursor)) { + throw new Exception( + pht( + 'Source "%s" (of class "%s") returned an invalid value from '. + 'method "%s": all values must be objects of class "%s".', + $this->getName(), + get_class($this), + 'newImportCursors()', + 'NuanceImportCursor')); + } + + $key = $cursor->getCursorKey(); + if (!strlen($key)) { + throw new Exception( + pht( + 'Source "%s" (of class "%s") returned an import cursor with '. + 'a missing key from "%s". Each cursor must have a unique, '. + 'nonempty key.', + $this->getName(), + get_class($this), + 'newImportCursors()')); + } + + $other = idx($map, $key); + if ($other) { + throw new Exception( + pht( + 'Source "%s" (of class "%s") returned two cursors from method '. + '"%s" with the same key ("%s"). Each cursor must have a unique '. + 'key.', + $this->getName(), + get_class($this), + 'newImportCursors()', + $key)); + } + + $map[$key] = $cursor; + + $cursor->setSource($source); + + $cursor_data = idx($data, $key); + if (!$cursor_data) { + $cursor_data = $cursor->newEmptyCursorData($source); + } + + $cursor->setCursorData($cursor_data); + } + + return $cursors; } protected function newImportCursors() { @@ -79,21 +138,13 @@ */ abstract public function getSourceTypeConstant(); - /** - * Code to create and update @{class:NuanceItem}s and - * @{class:NuanceRequestor}s via daemons goes here. - * - * If that does not make sense for the @{class:NuanceSource} you are - * defining, simply return null. For example, - * @{class:NuancePhabricatorFormSourceDefinition} since these are one-way - * contact forms. - */ - abstract public function updateItems(); - - abstract public function renderView(); - - abstract public function renderListView(); + public function renderView() { + return null; + } + public function renderListView() { + return null; + } protected function newItemFromProperties( NuanceRequestor $requestor, diff --git a/src/applications/nuance/storage/NuanceImportCursorData.php b/src/applications/nuance/storage/NuanceImportCursorData.php --- a/src/applications/nuance/storage/NuanceImportCursorData.php +++ b/src/applications/nuance/storage/NuanceImportCursorData.php @@ -32,4 +32,13 @@ NuanceImportCursorPHIDType::TYPECONST); } + public function getCursorProperty($key, $default = null) { + return idx($this->properties, $key, $default); + } + + public function setCursorProperty($key, $value) { + $this->properties[$key] = $value; + return $this; + } + } diff --git a/src/applications/nuance/storage/NuanceSource.php b/src/applications/nuance/storage/NuanceSource.php --- a/src/applications/nuance/storage/NuanceSource.php +++ b/src/applications/nuance/storage/NuanceSource.php @@ -8,7 +8,7 @@ protected $name; protected $type; - protected $data; + protected $data = array(); protected $mailKey; protected $viewPolicy; protected $editPolicy; @@ -82,6 +82,15 @@ return $this; } + public function getSourceProperty($key, $default = null) { + return idx($this->data, $key, $default); + } + + public function setSourceProperty($key, $value) { + $this->data[$key] = $value; + return $this; + } + /* -( PhabricatorApplicationTransactionInterface )------------------------- */