Page Menu
Home
Phabricator
Search
Configure Global Search
Log In
Files
F13281400
D15435.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
13 KB
Referenced Files
None
Subscribers
None
D15435.diff
View Options
diff --git a/resources/sql/autopatches/20160308.nuance.01.disabled.sql b/resources/sql/autopatches/20160308.nuance.01.disabled.sql
new file mode 100644
--- /dev/null
+++ b/resources/sql/autopatches/20160308.nuance.01.disabled.sql
@@ -0,0 +1,2 @@
+ALTER TABLE {$NAMESPACE}_nuance.nuance_source
+ ADD isDisabled BOOL NOT NULL;
diff --git a/resources/sql/autopatches/20160308.nuance.02.cursordata.sql b/resources/sql/autopatches/20160308.nuance.02.cursordata.sql
new file mode 100644
--- /dev/null
+++ b/resources/sql/autopatches/20160308.nuance.02.cursordata.sql
@@ -0,0 +1,12 @@
+CREATE TABLE {$NAMESPACE}_nuance.nuance_importcursordata (
+ id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
+ phid VARBINARY(64) NOT NULL,
+ sourcePHID VARBINARY(64) NOT NULL,
+ cursorKey VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT},
+ cursorType VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT},
+ properties LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT},
+ dateCreated INT UNSIGNED NOT NULL,
+ dateModified INT UNSIGNED NOT NULL,
+ UNIQUE KEY `key_phid` (phid),
+ UNIQUE KEY `key_source` (sourcePHID, cursorKey)
+) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT};
diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -1421,6 +1421,10 @@
'NuanceController' => 'applications/nuance/controller/NuanceController.php',
'NuanceCreateItemConduitAPIMethod' => 'applications/nuance/conduit/NuanceCreateItemConduitAPIMethod.php',
'NuanceDAO' => 'applications/nuance/storage/NuanceDAO.php',
+ 'NuanceImportCursor' => 'applications/nuance/cursor/NuanceImportCursor.php',
+ 'NuanceImportCursorData' => 'applications/nuance/storage/NuanceImportCursorData.php',
+ 'NuanceImportCursorDataQuery' => 'applications/nuance/query/NuanceImportCursorDataQuery.php',
+ 'NuanceImportCursorPHIDType' => 'applications/nuance/phid/NuanceImportCursorPHIDType.php',
'NuanceItem' => 'applications/nuance/storage/NuanceItem.php',
'NuanceItemEditController' => 'applications/nuance/controller/NuanceItemEditController.php',
'NuanceItemEditor' => 'applications/nuance/editor/NuanceItemEditor.php',
@@ -5661,6 +5665,10 @@
'NuanceController' => 'PhabricatorController',
'NuanceCreateItemConduitAPIMethod' => 'NuanceConduitAPIMethod',
'NuanceDAO' => 'PhabricatorLiskDAO',
+ 'NuanceImportCursor' => 'Phobject',
+ 'NuanceImportCursorData' => 'NuanceDAO',
+ 'NuanceImportCursorDataQuery' => 'NuanceQuery',
+ 'NuanceImportCursorPHIDType' => 'PhabricatorPHIDType',
'NuanceItem' => array(
'NuanceDAO',
'PhabricatorPolicyInterface',
diff --git a/src/applications/nuance/cursor/NuanceImportCursor.php b/src/applications/nuance/cursor/NuanceImportCursor.php
new file mode 100644
--- /dev/null
+++ b/src/applications/nuance/cursor/NuanceImportCursor.php
@@ -0,0 +1,10 @@
+<?php
+
+abstract class NuanceImportCursor extends Phobject {
+
+ final public function importFromSource() {
+ // TODO: Perhaps, do something.
+ return false;
+ }
+
+}
diff --git a/src/applications/nuance/phid/NuanceImportCursorPHIDType.php b/src/applications/nuance/phid/NuanceImportCursorPHIDType.php
new file mode 100644
--- /dev/null
+++ b/src/applications/nuance/phid/NuanceImportCursorPHIDType.php
@@ -0,0 +1,38 @@
+<?php
+
+final class NuanceImportCursorPHIDType extends PhabricatorPHIDType {
+
+ const TYPECONST = 'NUAC';
+
+ public function getTypeName() {
+ return pht('Import Cursor');
+ }
+
+ public function newObject() {
+ return new NuanceImportCursorData();
+ }
+
+ public function getPHIDTypeApplicationClass() {
+ return 'PhabricatorNuanceApplication';
+ }
+
+ protected function buildQueryForObjects(
+ PhabricatorObjectQuery $query,
+ array $phids) {
+
+ return id(new NuanceImportCursorDataQuery())
+ ->withPHIDs($phids);
+ }
+
+ public function loadHandles(
+ PhabricatorHandleQuery $query,
+ array $handles,
+ array $objects) {
+
+ $viewer = $query->getViewer();
+ foreach ($handles as $phid => $handle) {
+ $item = $objects[$phid];
+ }
+ }
+
+}
diff --git a/src/applications/nuance/query/NuanceSourceQuery.php b/src/applications/nuance/query/NuanceImportCursorDataQuery.php
copy from src/applications/nuance/query/NuanceSourceQuery.php
copy to src/applications/nuance/query/NuanceImportCursorDataQuery.php
--- a/src/applications/nuance/query/NuanceSourceQuery.php
+++ b/src/applications/nuance/query/NuanceImportCursorDataQuery.php
@@ -1,11 +1,11 @@
<?php
-final class NuanceSourceQuery
+final class NuanceImportCursorDataQuery
extends NuanceQuery {
private $ids;
private $phids;
- private $types;
+ private $sourcePHIDs;
public function withIDs(array $ids) {
$this->ids = $ids;
@@ -17,44 +17,27 @@
return $this;
}
- public function withTypes($types) {
- $this->types = $types;
+ public function withSourcePHIDs(array $source_phids) {
+ $this->sourcePHIDs = $source_phids;
return $this;
}
public function newResultObject() {
- return new NuanceSource();
+ return new NuanceImportCursorData();
}
protected function loadPage() {
return $this->loadStandardPage($this->newResultObject());
}
- protected function willFilterPage(array $sources) {
- $all_types = NuanceSourceDefinition::getAllDefinitions();
-
- foreach ($sources as $key => $source) {
- $definition = idx($all_types, $source->getType());
- if (!$definition) {
- $this->didRejectResult($source);
- unset($sources[$key]);
- continue;
- }
- $source->attachDefinition($definition);
- }
-
- return $sources;
- }
-
-
protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) {
$where = parent::buildWhereClauseParts($conn);
- if ($this->types !== null) {
+ if ($this->sourcePHIDs !== null) {
$where[] = qsprintf(
$conn,
- 'type IN (%Ls)',
- $this->types);
+ 'sourcePHID IN (%Ls)',
+ $this->sourcePHIDs);
}
if ($this->ids !== null) {
diff --git a/src/applications/nuance/query/NuanceSourceQuery.php b/src/applications/nuance/query/NuanceSourceQuery.php
--- a/src/applications/nuance/query/NuanceSourceQuery.php
+++ b/src/applications/nuance/query/NuanceSourceQuery.php
@@ -6,6 +6,8 @@
private $ids;
private $phids;
private $types;
+ private $isDisabled;
+ private $hasCursors;
public function withIDs(array $ids) {
$this->ids = $ids;
@@ -22,6 +24,16 @@
return $this;
}
+ public function withIsDisabled($disabled) {
+ $this->isDisabled = $disabled;
+ return $this;
+ }
+
+ public function withHasImportCursors($has_cursors) {
+ $this->hasCursors = $has_cursors;
+ return $this;
+ }
+
public function newResultObject() {
return new NuanceSource();
}
@@ -71,6 +83,44 @@
$this->phids);
}
+ if ($this->isDisabled !== null) {
+ $where[] = qsprintf(
+ $conn,
+ 'isDisabled = %d',
+ (int)$this->isDisabled);
+ }
+
+ if ($this->hasCursors !== null) {
+ $cursor_types = array();
+
+ $definitions = NuanceSourceDefinition::getAllDefinitions();
+ foreach ($definitions as $key => $definition) {
+ if ($definition->hasImportCursors()) {
+ $cursor_types[] = $key;
+ }
+ }
+
+ if ($this->hasCursors) {
+ if (!$cursor_types) {
+ throw new PhabricatorEmptyQueryException();
+ } else {
+ $where[] = qsprintf(
+ $conn,
+ 'type IN (%Ls)',
+ $cursor_types);
+ }
+ } else {
+ if (!$cursor_types) {
+ // Apply no constraint.
+ } else {
+ $where[] = qsprintf(
+ $conn,
+ 'type NOT IN (%Ls)',
+ $cursor_types);
+ }
+ }
+ }
+
return $where;
}
diff --git a/src/applications/nuance/source/NuanceSourceDefinition.php b/src/applications/nuance/source/NuanceSourceDefinition.php
--- a/src/applications/nuance/source/NuanceSourceDefinition.php
+++ b/src/applications/nuance/source/NuanceSourceDefinition.php
@@ -43,6 +43,23 @@
->execute();
}
+ public function hasImportCursors() {
+ return false;
+ }
+
+ final public function getImportCursors() {
+ if (!$this->hasImportCursors()) {
+ throw new Exception(
+ pht('This source has no input cursors.'));
+ }
+
+ return $this->newImportCursors();
+ }
+
+ protected function newImportCursors() {
+ throw new PhutilMethodNotImplementedException();
+ }
+
/**
* A human readable string like "Twitter" or "Phabricator Form".
*/
diff --git a/src/applications/nuance/storage/NuanceImportCursorData.php b/src/applications/nuance/storage/NuanceImportCursorData.php
new file mode 100644
--- /dev/null
+++ b/src/applications/nuance/storage/NuanceImportCursorData.php
@@ -0,0 +1,35 @@
+<?php
+
+final class NuanceImportCursorData
+ extends NuanceDAO {
+
+ protected $sourcePHID;
+ protected $cursorKey;
+ protected $cursorType;
+ protected $properties = array();
+
+ protected function getConfiguration() {
+ return array(
+ self::CONFIG_AUX_PHID => true,
+ self::CONFIG_SERIALIZATION => array(
+ 'properties' => self::SERIALIZATION_JSON,
+ ),
+ self::CONFIG_COLUMN_SCHEMA => array(
+ 'cursorType' => 'text32',
+ 'cursorKey' => 'text32',
+ ),
+ self::CONFIG_KEY_SCHEMA => array(
+ 'key_source' => array(
+ 'columns' => array('sourcePHID', 'cursorKey'),
+ 'unique' => true,
+ ),
+ ),
+ ) + parent::getConfiguration();
+ }
+
+ public function generatePHID() {
+ return PhabricatorPHID::generateNewPHID(
+ NuanceImportCursorPHIDType::TYPECONST);
+ }
+
+}
diff --git a/src/applications/nuance/storage/NuanceSource.php b/src/applications/nuance/storage/NuanceSource.php
--- a/src/applications/nuance/storage/NuanceSource.php
+++ b/src/applications/nuance/storage/NuanceSource.php
@@ -12,6 +12,7 @@
protected $viewPolicy;
protected $editPolicy;
protected $defaultQueuePHID;
+ protected $isDisabled;
private $definition = self::ATTACHABLE;
@@ -25,6 +26,7 @@
'name' => 'text255?',
'type' => 'text32',
'mailKey' => 'bytes20',
+ 'isDisabled' => 'bool',
),
self::CONFIG_KEY_SCHEMA => array(
'key_type' => array(
@@ -66,7 +68,8 @@
->setViewPolicy($view_policy)
->setEditPolicy($edit_policy)
->setType($definition->getSourceTypeConstant())
- ->attachDefinition($definition);
+ ->attachDefinition($definition)
+ ->setIsDisabled(0);
}
public function getDefinition() {
diff --git a/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php b/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php
--- a/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php
+++ b/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php
@@ -16,6 +16,10 @@
private $garbageCollectors;
private $nextCollection;
+ private $anyNuanceData;
+ private $nuanceSources;
+ private $nuanceCursors;
+
protected function run() {
// The trigger daemon is a low-level infrastructure daemon which schedules
@@ -99,6 +103,7 @@
$lock->unlock();
$sleep_duration = $this->getSleepDuration();
+ $sleep_duration = $this->runNuanceImportCursors($sleep_duration);
$sleep_duration = $this->runGarbageCollection($sleep_duration);
$this->sleep($sleep_duration);
} while (!$this->shouldExit());
@@ -379,4 +384,70 @@
return false;
}
+
+/* -( Nuance Importers )--------------------------------------------------- */
+
+
+ private function runNuanceImportCursors($duration) {
+ $run_until = (PhabricatorTime::getNow() + $duration);
+
+ do {
+ $more_data = $this->updateNuanceImportCursors();
+ if (!$more_data) {
+ break;
+ }
+ } while (PhabricatorTime::getNow() <= $run_until);
+
+ $remaining = max(0, $run_until - PhabricatorTime::getNow());
+
+ return $remaining;
+ }
+
+
+ private function updateNuanceImportCursors() {
+ $nuance_app = 'PhabricatorNuanceApplication';
+ if (!PhabricatorApplication::isClassInstalled($nuance_app)) {
+ return false;
+ }
+
+ // If we haven't loaded sources yet, load them first.
+ if (!$this->nuanceSources) {
+ $this->anyNuanceData = false;
+
+ $sources = id(new NuanceSourceQuery())
+ ->setViewer($this->getViewer())
+ ->withIsDisabled(false)
+ ->withHasImportCursors(true)
+ ->execute();
+ if (!$sources) {
+ return false;
+ }
+
+ $this->nuanceSources = array_reverse($sources);
+ }
+
+ // If we don't have any cursors, move to the next source and generate its
+ // cursors.
+ if (!$this->nuanceCursors) {
+ $source = array_pop($this->nuanceSources);
+ $cursors = $source->getImportCursors();
+ $this->nuanceCursors = array_reverse($cursors);
+ }
+
+ // Update the next cursor.
+ $cursor = array_pop($this->nuanceCursors);
+ if ($cursor) {
+ $more_data = $cursor->importFromSource();
+ if ($more_data) {
+ $this->anyNuanceData = true;
+ }
+ }
+
+ if (!$this->nuanceSources && !$this->nuanceCursors) {
+ return $this->anyNuanceData;
+ }
+
+ return true;
+ }
+
}
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Mon, Jun 3, 10:38 AM (3 w, 1 d ago)
Storage Engine
blob
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
6289328
Default Alt Text
D15435.diff (13 KB)
Attached To
Mode
D15435: Allow Nuances sources to provide import cursors
Attached
Detach File
Event Timeline
Log In to Comment