diff --git a/resources/sql/autopatches/20160308.nuance.01.disabled.sql b/resources/sql/autopatches/20160308.nuance.01.disabled.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20160308.nuance.01.disabled.sql @@ -0,0 +1,2 @@ +ALTER TABLE {$NAMESPACE}_nuance.nuance_source + ADD isDisabled BOOL NOT NULL; diff --git a/resources/sql/autopatches/20160308.nuance.02.cursordata.sql b/resources/sql/autopatches/20160308.nuance.02.cursordata.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20160308.nuance.02.cursordata.sql @@ -0,0 +1,12 @@ +CREATE TABLE {$NAMESPACE}_nuance.nuance_importcursordata ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + phid VARBINARY(64) NOT NULL, + sourcePHID VARBINARY(64) NOT NULL, + cursorKey VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT}, + cursorType VARCHAR(32) NOT NULL COLLATE {$COLLATE_TEXT}, + properties LONGTEXT NOT NULL COLLATE {$COLLATE_TEXT}, + dateCreated INT UNSIGNED NOT NULL, + dateModified INT UNSIGNED NOT NULL, + UNIQUE KEY `key_phid` (phid), + UNIQUE KEY `key_source` (sourcePHID, cursorKey) +) ENGINE=InnoDB, COLLATE {$COLLATE_TEXT}; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -1421,6 +1421,10 @@ 'NuanceController' => 'applications/nuance/controller/NuanceController.php', 'NuanceCreateItemConduitAPIMethod' => 'applications/nuance/conduit/NuanceCreateItemConduitAPIMethod.php', 'NuanceDAO' => 'applications/nuance/storage/NuanceDAO.php', + 'NuanceImportCursor' => 'applications/nuance/cursor/NuanceImportCursor.php', + 'NuanceImportCursorData' => 'applications/nuance/storage/NuanceImportCursorData.php', + 'NuanceImportCursorDataQuery' => 'applications/nuance/query/NuanceImportCursorDataQuery.php', + 'NuanceImportCursorPHIDType' => 'applications/nuance/phid/NuanceImportCursorPHIDType.php', 'NuanceItem' => 'applications/nuance/storage/NuanceItem.php', 'NuanceItemEditController' => 'applications/nuance/controller/NuanceItemEditController.php', 'NuanceItemEditor' => 'applications/nuance/editor/NuanceItemEditor.php', @@ -5661,6 +5665,10 @@ 'NuanceController' => 'PhabricatorController', 'NuanceCreateItemConduitAPIMethod' => 'NuanceConduitAPIMethod', 'NuanceDAO' => 'PhabricatorLiskDAO', + 'NuanceImportCursor' => 'Phobject', + 'NuanceImportCursorData' => 'NuanceDAO', + 'NuanceImportCursorDataQuery' => 'NuanceQuery', + 'NuanceImportCursorPHIDType' => 'PhabricatorPHIDType', 'NuanceItem' => array( 'NuanceDAO', 'PhabricatorPolicyInterface', diff --git a/src/applications/nuance/cursor/NuanceImportCursor.php b/src/applications/nuance/cursor/NuanceImportCursor.php new file mode 100644 --- /dev/null +++ b/src/applications/nuance/cursor/NuanceImportCursor.php @@ -0,0 +1,10 @@ +withPHIDs($phids); + } + + public function loadHandles( + PhabricatorHandleQuery $query, + array $handles, + array $objects) { + + $viewer = $query->getViewer(); + foreach ($handles as $phid => $handle) { + $item = $objects[$phid]; + } + } + +} diff --git a/src/applications/nuance/query/NuanceSourceQuery.php b/src/applications/nuance/query/NuanceImportCursorDataQuery.php copy from src/applications/nuance/query/NuanceSourceQuery.php copy to src/applications/nuance/query/NuanceImportCursorDataQuery.php --- a/src/applications/nuance/query/NuanceSourceQuery.php +++ b/src/applications/nuance/query/NuanceImportCursorDataQuery.php @@ -1,11 +1,11 @@ ids = $ids; @@ -17,44 +17,27 @@ return $this; } - public function withTypes($types) { - $this->types = $types; + public function withSourcePHIDs(array $source_phids) { + $this->sourcePHIDs = $source_phids; return $this; } public function newResultObject() { - return new NuanceSource(); + return new NuanceImportCursorData(); } protected function loadPage() { return $this->loadStandardPage($this->newResultObject()); } - protected function willFilterPage(array $sources) { - $all_types = NuanceSourceDefinition::getAllDefinitions(); - - foreach ($sources as $key => $source) { - $definition = idx($all_types, $source->getType()); - if (!$definition) { - $this->didRejectResult($source); - unset($sources[$key]); - continue; - } - $source->attachDefinition($definition); - } - - return $sources; - } - - protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { $where = parent::buildWhereClauseParts($conn); - if ($this->types !== null) { + if ($this->sourcePHIDs !== null) { $where[] = qsprintf( $conn, - 'type IN (%Ls)', - $this->types); + 'sourcePHID IN (%Ls)', + $this->sourcePHIDs); } if ($this->ids !== null) { diff --git a/src/applications/nuance/query/NuanceSourceQuery.php b/src/applications/nuance/query/NuanceSourceQuery.php --- a/src/applications/nuance/query/NuanceSourceQuery.php +++ b/src/applications/nuance/query/NuanceSourceQuery.php @@ -6,6 +6,8 @@ private $ids; private $phids; private $types; + private $isDisabled; + private $hasCursors; public function withIDs(array $ids) { $this->ids = $ids; @@ -22,6 +24,16 @@ return $this; } + public function withIsDisabled($disabled) { + $this->isDisabled = $disabled; + return $this; + } + + public function withHasImportCursors($has_cursors) { + $this->hasCursors = $has_cursors; + return $this; + } + public function newResultObject() { return new NuanceSource(); } @@ -71,6 +83,44 @@ $this->phids); } + if ($this->isDisabled !== null) { + $where[] = qsprintf( + $conn, + 'isDisabled = %d', + (int)$this->isDisabled); + } + + if ($this->hasCursors !== null) { + $cursor_types = array(); + + $definitions = NuanceSourceDefinition::getAllDefinitions(); + foreach ($definitions as $key => $definition) { + if ($definition->hasImportCursors()) { + $cursor_types[] = $key; + } + } + + if ($this->hasCursors) { + if (!$cursor_types) { + throw new PhabricatorEmptyQueryException(); + } else { + $where[] = qsprintf( + $conn, + 'type IN (%Ls)', + $cursor_types); + } + } else { + if (!$cursor_types) { + // Apply no constraint. + } else { + $where[] = qsprintf( + $conn, + 'type NOT IN (%Ls)', + $cursor_types); + } + } + } + return $where; } diff --git a/src/applications/nuance/source/NuanceSourceDefinition.php b/src/applications/nuance/source/NuanceSourceDefinition.php --- a/src/applications/nuance/source/NuanceSourceDefinition.php +++ b/src/applications/nuance/source/NuanceSourceDefinition.php @@ -43,6 +43,23 @@ ->execute(); } + public function hasImportCursors() { + return false; + } + + final public function getImportCursors() { + if (!$this->hasImportCursors()) { + throw new Exception( + pht('This source has no input cursors.')); + } + + return $this->newImportCursors(); + } + + protected function newImportCursors() { + throw new PhutilMethodNotImplementedException(); + } + /** * A human readable string like "Twitter" or "Phabricator Form". */ diff --git a/src/applications/nuance/storage/NuanceImportCursorData.php b/src/applications/nuance/storage/NuanceImportCursorData.php new file mode 100644 --- /dev/null +++ b/src/applications/nuance/storage/NuanceImportCursorData.php @@ -0,0 +1,35 @@ + true, + self::CONFIG_SERIALIZATION => array( + 'properties' => self::SERIALIZATION_JSON, + ), + self::CONFIG_COLUMN_SCHEMA => array( + 'cursorType' => 'text32', + 'cursorKey' => 'text32', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_source' => array( + 'columns' => array('sourcePHID', 'cursorKey'), + 'unique' => true, + ), + ), + ) + parent::getConfiguration(); + } + + public function generatePHID() { + return PhabricatorPHID::generateNewPHID( + NuanceImportCursorPHIDType::TYPECONST); + } + +} diff --git a/src/applications/nuance/storage/NuanceSource.php b/src/applications/nuance/storage/NuanceSource.php --- a/src/applications/nuance/storage/NuanceSource.php +++ b/src/applications/nuance/storage/NuanceSource.php @@ -12,6 +12,7 @@ protected $viewPolicy; protected $editPolicy; protected $defaultQueuePHID; + protected $isDisabled; private $definition = self::ATTACHABLE; @@ -25,6 +26,7 @@ 'name' => 'text255?', 'type' => 'text32', 'mailKey' => 'bytes20', + 'isDisabled' => 'bool', ), self::CONFIG_KEY_SCHEMA => array( 'key_type' => array( @@ -66,7 +68,8 @@ ->setViewPolicy($view_policy) ->setEditPolicy($edit_policy) ->setType($definition->getSourceTypeConstant()) - ->attachDefinition($definition); + ->attachDefinition($definition) + ->setIsDisabled(0); } public function getDefinition() { diff --git a/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php b/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php --- a/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php +++ b/src/infrastructure/daemon/workers/PhabricatorTriggerDaemon.php @@ -16,6 +16,10 @@ private $garbageCollectors; private $nextCollection; + private $anyNuanceData; + private $nuanceSources; + private $nuanceCursors; + protected function run() { // The trigger daemon is a low-level infrastructure daemon which schedules @@ -99,6 +103,7 @@ $lock->unlock(); $sleep_duration = $this->getSleepDuration(); + $sleep_duration = $this->runNuanceImportCursors($sleep_duration); $sleep_duration = $this->runGarbageCollection($sleep_duration); $this->sleep($sleep_duration); } while (!$this->shouldExit()); @@ -379,4 +384,70 @@ return false; } + +/* -( Nuance Importers )--------------------------------------------------- */ + + + private function runNuanceImportCursors($duration) { + $run_until = (PhabricatorTime::getNow() + $duration); + + do { + $more_data = $this->updateNuanceImportCursors(); + if (!$more_data) { + break; + } + } while (PhabricatorTime::getNow() <= $run_until); + + $remaining = max(0, $run_until - PhabricatorTime::getNow()); + + return $remaining; + } + + + private function updateNuanceImportCursors() { + $nuance_app = 'PhabricatorNuanceApplication'; + if (!PhabricatorApplication::isClassInstalled($nuance_app)) { + return false; + } + + // If we haven't loaded sources yet, load them first. + if (!$this->nuanceSources) { + $this->anyNuanceData = false; + + $sources = id(new NuanceSourceQuery()) + ->setViewer($this->getViewer()) + ->withIsDisabled(false) + ->withHasImportCursors(true) + ->execute(); + if (!$sources) { + return false; + } + + $this->nuanceSources = array_reverse($sources); + } + + // If we don't have any cursors, move to the next source and generate its + // cursors. + if (!$this->nuanceCursors) { + $source = array_pop($this->nuanceSources); + $cursors = $source->getImportCursors(); + $this->nuanceCursors = array_reverse($cursors); + } + + // Update the next cursor. + $cursor = array_pop($this->nuanceCursors); + if ($cursor) { + $more_data = $cursor->importFromSource(); + if ($more_data) { + $this->anyNuanceData = true; + } + } + + if (!$this->nuanceSources && !$this->nuanceCursors) { + return $this->anyNuanceData; + } + + return true; + } + }