Page MenuHomePhabricator
Files F15387247

D15439.diff
No OneTemporary
Actions

D15439.diff
View Options

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -1421,6 +1421,8 @@
'NuanceController' => 'applications/nuance/controller/NuanceController.php',
'NuanceCreateItemConduitAPIMethod' => 'applications/nuance/conduit/NuanceCreateItemConduitAPIMethod.php',
'NuanceDAO' => 'applications/nuance/storage/NuanceDAO.php',
+ 'NuanceGitHubRepositoryImportCursor' => 'applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php',
+ 'NuanceGitHubRepositorySourceDefinition' => 'applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php',
'NuanceImportCursor' => 'applications/nuance/cursor/NuanceImportCursor.php',
'NuanceImportCursorData' => 'applications/nuance/storage/NuanceImportCursorData.php',
'NuanceImportCursorDataQuery' => 'applications/nuance/query/NuanceImportCursorDataQuery.php',
@@ -5668,6 +5670,8 @@
'NuanceController' => 'PhabricatorController',
'NuanceCreateItemConduitAPIMethod' => 'NuanceConduitAPIMethod',
'NuanceDAO' => 'PhabricatorLiskDAO',
+ 'NuanceGitHubRepositoryImportCursor' => 'NuanceImportCursor',
+ 'NuanceGitHubRepositorySourceDefinition' => 'NuanceSourceDefinition',
'NuanceImportCursor' => 'Phobject',
'NuanceImportCursorData' => 'NuanceDAO',
'NuanceImportCursorDataQuery' => 'NuanceQuery',
diff --git a/src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php b/src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php
new file mode 100644
--- /dev/null
+++ b/src/applications/nuance/cursor/NuanceGitHubRepositoryImportCursor.php
@@ -0,0 +1,114 @@
+<?php
+
+final class NuanceGitHubRepositoryImportCursor
+ extends NuanceImportCursor {
+
+ const CURSORTYPE = 'github.repository';
+
+ protected function shouldPullDataFromSource() {
+ $now = PhabricatorTime::getNow();
+
+ // Respect GitHub's poll interval header. If we made a request recently,
+ // don't make another one until we've waited long enough.
+ $ttl = $this->getCursorProperty('github.poll.ttl');
+ if ($ttl && ($ttl >= $now)) {
+ $this->logInfo(
+ pht(
+ 'Respecting "%s": waiting for %s second(s) to poll GitHub.',
+ 'X-Poll-Interval',
+ new PhutilNumber(1 + ($ttl - $now))));
+
+ return false;
+ }
+
+ // Respect GitHub's API rate limiting. If we've exceeded the rate limit,
+ // wait until it resets to try again.
+ $limit = $this->getCursorProperty('github.limit.ttl');
+ if ($limit && ($limit >= $now)) {
+ $this->logInfo(
+ pht(
+ 'Respecting "%s": waiting for %s second(s) to poll GitHub.',
+ 'X-RateLimit-Reset',
+ new PhutilNumber(1 + ($limit - $now))));
+ return false;
+ }
+
+ return true;
+ }
+
+ protected function pullDataFromSource() {
+ $source = $this->getSource();
+
+ $user = $source->getSourceProperty('github.user');
+ $repository = $source->getSourceProperty('github.repository');
+ $api_token = $source->getSourceProperty('github.token');
+
+ $uri = "/repos/{$user}/{$repository}/events";
+ $data = array();
+
+ $future = id(new PhutilGitHubFuture())
+ ->setAccessToken($api_token)
+ ->setRawGitHubQuery($uri, $data);
+
+ $etag = $this->getCursorProperty('github.poll.etag');
+ if ($etag) {
+ $future->addHeader('If-None-Match', $etag);
+ }
+
+ $this->logInfo(
+ pht(
+ 'Polling GitHub Repository API endpoint "%s".',
+ $uri));
+ $response = $future->resolve();
+
+ // Do this first: if we hit the rate limit, we get a response but the
+ // body isn't valid.
+ $this->updateRateLimits($response);
+
+ // This means we hit a rate limit or a "Not Modified" because of the "ETag"
+ // header. In either case, we should bail out.
+ if ($response->getStatus()->isError()) {
+ // TODO: Save cursor data!
+ return false;
+ }
+
+ $this->updateETag($response);
+
+ var_dump($response->getBody());
+ }
+
+ private function updateRateLimits(PhutilGitHubResponse $response) {
+ $remaining = $response->getHeaderValue('X-RateLimit-Remaining');
+ $limit_reset = $response->getHeaderValue('X-RateLimit-Reset');
+ $now = PhabricatorTime::getNow();
+
+ $limit_ttl = null;
+ if (strlen($remaining)) {
+ $remaining = (int)$remaining;
+ if (!$remaining) {
+ $limit_ttl = (int)$limit_reset;
+ }
+ }
+
+ $this->setCursorProperty('github.limit.ttl', $limit_ttl);
+
+ $this->logInfo(
+ pht(
+ 'This key has %s remaining API request(s), '.
+ 'limit resets in %s second(s).',
+ new PhutilNumber($remaining),
+ new PhutilNumber($limit_reset - $now)));
+ }
+
+ private function updateETag(PhutilGitHubResponse $response) {
+ $etag = $response->getHeaderValue('ETag');
+
+ $this->setCursorProperty('github.poll.etag', $etag);
+
+ $this->logInfo(
+ pht(
+ 'ETag for this request was "%s".',
+ $etag));
+ }
+
+}
diff --git a/src/applications/nuance/cursor/NuanceImportCursor.php b/src/applications/nuance/cursor/NuanceImportCursor.php
--- a/src/applications/nuance/cursor/NuanceImportCursor.php
+++ b/src/applications/nuance/cursor/NuanceImportCursor.php
@@ -2,9 +2,97 @@
abstract class NuanceImportCursor extends Phobject {
+ private $cursorData;
+ private $cursorKey;
+ private $source;
+
+ abstract protected function shouldPullDataFromSource();
+ abstract protected function pullDataFromSource();
+
+ final public function getCursorType() {
+ return $this->getPhobjectClassConstant('CURSORTYPE', 32);
+ }
+
+ public function setCursorData(NuanceImportCursorData $cursor_data) {
+ $this->cursorData = $cursor_data;
+ return $this;
+ }
+
+ public function getCursorData() {
+ return $this->cursorData;
+ }
+
+ public function setSource($source) {
+ $this->source = $source;
+ return $this;
+ }
+
+ public function getSource() {
+ return $this->source;
+ }
+
+ public function setCursorKey($cursor_key) {
+ $this->cursorKey = $cursor_key;
+ return $this;
+ }
+
+ public function getCursorKey() {
+ return $this->cursorKey;
+ }
+
final public function importFromSource() {
- // TODO: Perhaps, do something.
- return false;
+ if (!$this->shouldPullDataFromSource()) {
+ return false;
+ }
+
+ $source = $this->getSource();
+ $key = $this->getCursorKey();
+
+ $parts = array(
+ 'nsc',
+ $source->getID(),
+ PhabricatorHash::digestToLength($key, 20),
+ );
+ $lock_name = implode('.', $parts);
+
+ $lock = PhabricatorGlobalLock::newLock($lock_name);
+ $lock->lock(1);
+
+ try {
+ $more_data = $this->pullDataFromSource();
+ } catch (Exception $ex) {
+ $lock->unlock();
+ throw $ex;
+ }
+
+ $lock->unlock();
+
+ return $more_data;
+ }
+
+ final public function newEmptyCursorData(NuanceSource $source) {
+ return id(new NuanceImportCursorData())
+ ->setCursorKey($this->getCursorKey())
+ ->setCursorType($this->getCursorType())
+ ->setSourcePHID($source->getPHID());
+ }
+
+ final protected function logInfo($message) {
+ echo tsprintf(
+ "<cursor:%s> %s\n",
+ $this->getCursorKey(),
+ $message);
+
+ return $this;
+ }
+
+ final protected function getCursorProperty($key, $default = null) {
+ return $this->getCursorData()->getCursorProperty($key, $default);
+ }
+
+ final protected function setCursorProperty($key, $value) {
+ $this->getCursorData()->setCursorProperty($key, $value);
+ return $this;
}
}
diff --git a/src/applications/nuance/management/NuanceManagementImportWorkflow.php b/src/applications/nuance/management/NuanceManagementImportWorkflow.php
--- a/src/applications/nuance/management/NuanceManagementImportWorkflow.php
+++ b/src/applications/nuance/management/NuanceManagementImportWorkflow.php
@@ -40,9 +40,9 @@
$source->getName()));
}
- echo tsprintf(
- "%s\n",
- pht('OK, but actual importing is not implemented yet.'));
+ foreach ($cursors as $cursor) {
+ $cursor->importFromSource();
+ }
return 0;
}
diff --git a/src/applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php b/src/applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php
new file mode 100644
--- /dev/null
+++ b/src/applications/nuance/source/NuanceGitHubRepositorySourceDefinition.php
@@ -0,0 +1,29 @@
+<?php
+
+final class NuanceGitHubRepositorySourceDefinition
+ extends NuanceSourceDefinition {
+
+ public function getName() {
+ return pht('GitHub Repository');
+ }
+
+ public function getSourceDescription() {
+ return pht('Import issues and pull requests from a GitHub repository.');
+ }
+
+ public function getSourceTypeConstant() {
+ return 'github.repository';
+ }
+
+ public function hasImportCursors() {
+ return true;
+ }
+
+ protected function newImportCursors() {
+ return array(
+ id(new NuanceGitHubRepositoryImportCursor())
+ ->setCursorKey('events.repository'),
+ );
+ }
+
+}
diff --git a/src/applications/nuance/source/NuancePhabricatorFormSourceDefinition.php b/src/applications/nuance/source/NuancePhabricatorFormSourceDefinition.php
--- a/src/applications/nuance/source/NuancePhabricatorFormSourceDefinition.php
+++ b/src/applications/nuance/source/NuancePhabricatorFormSourceDefinition.php
@@ -26,15 +26,6 @@
return $actions;
}
- public function updateItems() {
- return null;
- }
-
- public function renderView() {}
-
- public function renderListView() {}
-
-
public function handleActionRequest(AphrontRequest $request) {
$viewer = $request->getViewer();
diff --git a/src/applications/nuance/source/NuanceSourceDefinition.php b/src/applications/nuance/source/NuanceSourceDefinition.php
--- a/src/applications/nuance/source/NuanceSourceDefinition.php
+++ b/src/applications/nuance/source/NuanceSourceDefinition.php
@@ -53,7 +53,66 @@
pht('This source has no input cursors.'));
}
- return $this->newImportCursors();
+ $source = $this->getSource();
+ $cursors = $this->newImportCursors();
+
+ $data = id(new NuanceImportCursorDataQuery())
+ ->setViewer(PhabricatorUser::getOmnipotentUser())
+ ->withSourcePHIDs(array($source->getPHID()))
+ ->execute();
+ $data = mpull($data, 'getCursorKey');
+
+ $map = array();
+ foreach ($cursors as $cursor) {
+ if (!($cursor instanceof NuanceImportCursor)) {
+ throw new Exception(
+ pht(
+ 'Source "%s" (of class "%s") returned an invalid value from '.
+ 'method "%s": all values must be objects of class "%s".',
+ $this->getName(),
+ get_class($this),
+ 'newImportCursors()',
+ 'NuanceImportCursor'));
+ }
+
+ $key = $cursor->getCursorKey();
+ if (!strlen($key)) {
+ throw new Exception(
+ pht(
+ 'Source "%s" (of class "%s") returned an import cursor with '.
+ 'a missing key from "%s". Each cursor must have a unique, '.
+ 'nonempty key.',
+ $this->getName(),
+ get_class($this),
+ 'newImportCursors()'));
+ }
+
+ $other = idx($map, $key);
+ if ($other) {
+ throw new Exception(
+ pht(
+ 'Source "%s" (of class "%s") returned two cursors from method '.
+ '"%s" with the same key ("%s"). Each cursor must have a unique '.
+ 'key.',
+ $this->getName(),
+ get_class($this),
+ 'newImportCursors()',
+ $key));
+ }
+
+ $map[$key] = $cursor;
+
+ $cursor->setSource($source);
+
+ $cursor_data = idx($data, $key);
+ if (!$cursor_data) {
+ $cursor_data = $cursor->newEmptyCursorData($source);
+ }
+
+ $cursor->setCursorData($cursor_data);
+ }
+
+ return $cursors;
}
protected function newImportCursors() {
@@ -79,21 +138,13 @@
*/
abstract public function getSourceTypeConstant();
- /**
- * Code to create and update @{class:NuanceItem}s and
- * @{class:NuanceRequestor}s via daemons goes here.
- *
- * If that does not make sense for the @{class:NuanceSource} you are
- * defining, simply return null. For example,
- * @{class:NuancePhabricatorFormSourceDefinition} since these are one-way
- * contact forms.
- */
- abstract public function updateItems();
-
- abstract public function renderView();
-
- abstract public function renderListView();
+ public function renderView() {
+ return null;
+ }
+ public function renderListView() {
+ return null;
+ }
protected function newItemFromProperties(
NuanceRequestor $requestor,
diff --git a/src/applications/nuance/storage/NuanceImportCursorData.php b/src/applications/nuance/storage/NuanceImportCursorData.php
--- a/src/applications/nuance/storage/NuanceImportCursorData.php
+++ b/src/applications/nuance/storage/NuanceImportCursorData.php
@@ -32,4 +32,13 @@
NuanceImportCursorPHIDType::TYPECONST);
}
+ public function getCursorProperty($key, $default = null) {
+ return idx($this->properties, $key, $default);
+ }
+
+ public function setCursorProperty($key, $value) {
+ $this->properties[$key] = $value;
+ return $this;
+ }
+
}
diff --git a/src/applications/nuance/storage/NuanceSource.php b/src/applications/nuance/storage/NuanceSource.php
--- a/src/applications/nuance/storage/NuanceSource.php
+++ b/src/applications/nuance/storage/NuanceSource.php
@@ -8,7 +8,7 @@
protected $name;
protected $type;
- protected $data;
+ protected $data = array();
protected $mailKey;
protected $viewPolicy;
protected $editPolicy;
@@ -82,6 +82,15 @@
return $this;
}
+ public function getSourceProperty($key, $default = null) {
+ return idx($this->data, $key, $default);
+ }
+
+ public function setSourceProperty($key, $value) {
+ $this->data[$key] = $value;
+ return $this;
+ }
+
/* -( PhabricatorApplicationTransactionInterface )------------------------- */

File Metadata

Mime Type
text/plain
Expires
Sun, Mar 16, 2:05 AM (1 w, 3 d ago)
Storage Engine
blob
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
7390677
Default Alt Text
D15439.diff (13 KB)

Event Timeline

Phabricator · Built by Phacility