diff --git a/resources/sql/autopatches/20150105.conpsearch.sql b/resources/sql/autopatches/20150105.conpsearch.sql new file mode 100644 --- /dev/null +++ b/resources/sql/autopatches/20150105.conpsearch.sql @@ -0,0 +1,14 @@ +CREATE TABLE {$NAMESPACE}_conpherence.conpherence_index ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY, + threadPHID VARBINARY(64) NOT NULL, + transactionPHID VARBINARY(64) NOT NULL, + previousTransactionPHID VARBINARY(64), + corpus longtext + CHARACTER SET {$CHARSET_FULLTEXT} + COLLATE {$COLLATE_FULLTEXT} + NOT NULL, + KEY `key_thread` (threadPHID), + UNIQUE KEY `key_transaction` (transactionPHID), + UNIQUE KEY `key_previous` (previousTransactionPHID), + FULLTEXT KEY `key_corpus` (corpus) +) ENGINE=MyISAM DEFAULT CHARSET={$CHARSET} COLLATE={$COLLATE_TEXT}; diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -230,7 +230,9 @@ 'ConpherenceDAO' => 'applications/conpherence/storage/ConpherenceDAO.php', 'ConpherenceEditor' => 'applications/conpherence/editor/ConpherenceEditor.php', 'ConpherenceFileWidgetView' => 'applications/conpherence/view/ConpherenceFileWidgetView.php', + 'ConpherenceFulltextQuery' => 'applications/conpherence/query/ConpherenceFulltextQuery.php', 'ConpherenceHovercardEventListener' => 'applications/conpherence/events/ConpherenceHovercardEventListener.php', + 'ConpherenceIndex' => 'applications/conpherence/storage/ConpherenceIndex.php', 'ConpherenceLayoutView' => 'applications/conpherence/view/ConpherenceLayoutView.php', 'ConpherenceListController' => 'applications/conpherence/controller/ConpherenceListController.php', 'ConpherenceMenuItemView' => 'applications/conpherence/view/ConpherenceMenuItemView.php', @@ -247,6 +249,7 @@ 'ConpherenceSchemaSpec' => 'applications/conpherence/storage/ConpherenceSchemaSpec.php', 'ConpherenceSettings' => 'applications/conpherence/constants/ConpherenceSettings.php', 'ConpherenceThread' => 'applications/conpherence/storage/ConpherenceThread.php', + 'ConpherenceThreadIndexer' => 'applications/conpherence/search/ConpherenceThreadIndexer.php', 'ConpherenceThreadListView' => 'applications/conpherence/view/ConpherenceThreadListView.php', 'ConpherenceThreadMailReceiver' => 'applications/conpherence/mail/ConpherenceThreadMailReceiver.php', 'ConpherenceThreadQuery' => 'applications/conpherence/query/ConpherenceThreadQuery.php', @@ -3290,7 +3293,9 @@ 'ConpherenceDAO' => 'PhabricatorLiskDAO', 'ConpherenceEditor' => 'PhabricatorApplicationTransactionEditor', 'ConpherenceFileWidgetView' => 'ConpherenceWidgetView', + 'ConpherenceFulltextQuery' => 'PhabricatorOffsetPagedQuery', 'ConpherenceHovercardEventListener' => 'PhabricatorEventListener', + 'ConpherenceIndex' => 'ConpherenceDAO', 'ConpherenceLayoutView' => 'AphrontView', 'ConpherenceListController' => 'ConpherenceController', 'ConpherenceMenuItemView' => 'AphrontTagView', @@ -3310,6 +3315,7 @@ 'ConpherenceDAO', 'PhabricatorPolicyInterface', ), + 'ConpherenceThreadIndexer' => 'PhabricatorSearchDocumentIndexer', 'ConpherenceThreadListView' => 'AphrontView', 'ConpherenceThreadMailReceiver' => 'PhabricatorObjectMailReceiver', 'ConpherenceThreadQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', @@ -5603,6 +5609,7 @@ 'PhabricatorSearchDeleteController' => 'PhabricatorSearchBaseController', 'PhabricatorSearchDocument' => 'PhabricatorSearchDAO', 'PhabricatorSearchDocumentField' => 'PhabricatorSearchDAO', + 'PhabricatorSearchDocumentIndexer' => 'Phobject', 'PhabricatorSearchDocumentQuery' => 'PhabricatorCursorPagedPolicyAwareQuery', 'PhabricatorSearchDocumentRelationship' => 'PhabricatorSearchDAO', 'PhabricatorSearchEditController' => 'PhabricatorSearchBaseController', diff --git a/src/applications/conpherence/editor/ConpherenceEditor.php b/src/applications/conpherence/editor/ConpherenceEditor.php --- a/src/applications/conpherence/editor/ConpherenceEditor.php +++ b/src/applications/conpherence/editor/ConpherenceEditor.php @@ -459,7 +459,23 @@ } protected function supportsSearch() { - return false; + return true; + } + + protected function getSearchContextParameter( + PhabricatorLiskDAO $object, + array $xactions) { + + $comment_phids = array(); + foreach ($xactions as $xaction) { + if ($xaction->hasComment()) { + $comment_phids[] = $xaction->getPHID(); + } + } + + return array( + 'commentPHIDs' => $comment_phids, + ); } } diff --git a/src/applications/conpherence/query/ConpherenceFulltextQuery.php b/src/applications/conpherence/query/ConpherenceFulltextQuery.php new file mode 100644 --- /dev/null +++ b/src/applications/conpherence/query/ConpherenceFulltextQuery.php @@ -0,0 +1,68 @@ +threadPHIDs = $phids; + return $this; + } + + public function withFulltext($fulltext) { + $this->fulltext = $fulltext; + return $this; + } + + public function execute() { + $table = new ConpherenceIndex(); + $conn_r = $table->establishConnection('r'); + + $rows = queryfx_all( + $conn_r, + 'SELECT threadPHID, transactionPHID, previousTransactionPHID + FROM %T i %Q %Q %Q', + $table->getTableName(), + $this->buildWhereClause($conn_r), + $this->buildOrderByClause($conn_r), + $this->buildLimitClause($conn_r)); + + return $rows; + } + + private function buildWhereClause($conn_r) { + $where = array(); + + if ($this->threadPHIDs !== null) { + $where[] = qsprintf( + $conn_r, + 'i.threadPHID IN (%Ls)', + $this->threadPHIDs); + } + + if (strlen($this->fulltext)) { + $where[] = qsprintf( + $conn_r, + 'MATCH(i.corpus) AGAINST (%s IN BOOLEAN MODE)', + $this->fulltext); + } + + return $this->formatWhereClause($where); + } + + private function buildOrderByClause(AphrontDatabaseConnection $conn_r) { + if (strlen($this->fulltext)) { + return qsprintf( + $conn_r, + 'ORDER BY MATCH(i.corpus) AGAINST (%s IN BOOLEAN MODE) DESC', + $this->fulltext); + } else { + return qsprintf( + $conn_r, + 'ORDER BY id DESC'); + } + } + +} diff --git a/src/applications/conpherence/search/ConpherenceThreadIndexer.php b/src/applications/conpherence/search/ConpherenceThreadIndexer.php new file mode 100644 --- /dev/null +++ b/src/applications/conpherence/search/ConpherenceThreadIndexer.php @@ -0,0 +1,89 @@ +setViewer($this->getViewer()) + ->withPHIDs(array($phid)) + ->executeOne(); + + if (!$object) { + throw new Exception(pht('No thread "%s" exists!', $phid)); + } + + return $object; + } + + protected function buildAbstractDocumentByPHID($phid) { + $thread = $this->loadDocumentByPHID($phid); + + // NOTE: We're explicitly not building a document here, only rebuilding + // the Conpherence search index. + + $context = nonempty($this->getContext(), array()); + $comment_phids = idx($context, 'commentPHIDs'); + + if (is_array($comment_phids) && !$comment_phids) { + // If this property is set, but empty, the transaction did not + // include any chat text. For example, a user might have left the + // conversation. + return null; + } + + $query = id(new ConpherenceTransactionQuery()) + ->setViewer($this->getViewer()) + ->withObjectPHIDs(array($thread->getPHID())) + ->withTransactionTypes(array(PhabricatorTransactions::TYPE_COMMENT)) + ->needComments(true); + + if ($comment_phids !== null) { + $query->withPHIDs($comment_phids); + } + + $xactions = $query->execute(); + + foreach ($xactions as $xaction) { + $this->indexComment($thread, $xaction); + } + + return null; + } + + private function indexComment( + ConpherenceThread $thread, + ConpherenceTransaction $xaction) { + + $previous = id(new ConpherenceTransactionQuery()) + ->setViewer($this->getViewer()) + ->withObjectPHIDs(array($thread->getPHID())) + ->withTransactionTypes(array(PhabricatorTransactions::TYPE_COMMENT)) + ->setAfterID($xaction->getID()) + ->setLimit(1) + ->executeOne(); + + $index = id(new ConpherenceIndex()) + ->setThreadPHID($thread->getPHID()) + ->setTransactionPHID($xaction->getPHID()) + ->setPreviousTransactionPHID($previous ? $previous->getPHID() : null) + ->setCorpus($xaction->getComment()->getContent()); + + queryfx( + $index->establishConnection('w'), + 'INSERT INTO %T + (threadPHID, transactionPHID, previousTransactionPHID, corpus) + VALUES (%s, %s, %ns, %s) + ON DUPLICATE KEY UPDATE corpus = VALUES(corpus)', + $index->getTableName(), + $index->getThreadPHID(), + $index->getTransactionPHID(), + $index->getPreviousTransactionPHID(), + $index->getCorpus()); + } + +} diff --git a/src/applications/conpherence/storage/ConpherenceIndex.php b/src/applications/conpherence/storage/ConpherenceIndex.php new file mode 100644 --- /dev/null +++ b/src/applications/conpherence/storage/ConpherenceIndex.php @@ -0,0 +1,38 @@ + false, + self::CONFIG_COLUMN_SCHEMA => array( + 'previousTransactionPHID' => 'phid?', + 'corpus' => 'fulltext', + ), + self::CONFIG_KEY_SCHEMA => array( + 'key_thread' => array( + 'columns' => array('threadPHID'), + ), + 'key_transaction' => array( + 'columns' => array('transactionPHID'), + 'unique' => true, + ), + 'key_previous' => array( + 'columns' => array('previousTransactionPHID'), + 'unique' => true, + ), + 'key_corpus' => array( + 'columns' => array('corpus'), + 'type' => 'FULLTEXT', + ), + ), + ) + parent::getConfiguration(); + } + +} diff --git a/src/applications/search/index/PhabricatorSearchDocumentIndexer.php b/src/applications/search/index/PhabricatorSearchDocumentIndexer.php --- a/src/applications/search/index/PhabricatorSearchDocumentIndexer.php +++ b/src/applications/search/index/PhabricatorSearchDocumentIndexer.php @@ -1,6 +1,17 @@ context = $context; + return $this; + } + + protected function getContext() { + return $this->context; + } abstract public function getIndexableObject(); abstract protected function buildAbstractDocumentByPHID($phid); @@ -30,9 +41,15 @@ return $object; } - public function indexDocumentByPHID($phid) { + public function indexDocumentByPHID($phid, $context) { try { + $this->setContext($context); + $document = $this->buildAbstractDocumentByPHID($phid); + if ($document === null) { + // This indexer doesn't build a document index, so we're done. + return $this; + } $object = $this->loadDocumentByPHID($phid); diff --git a/src/applications/search/index/PhabricatorSearchIndexer.php b/src/applications/search/index/PhabricatorSearchIndexer.php --- a/src/applications/search/index/PhabricatorSearchIndexer.php +++ b/src/applications/search/index/PhabricatorSearchIndexer.php @@ -2,25 +2,26 @@ final class PhabricatorSearchIndexer { - public function queueDocumentForIndexing($phid) { + public function queueDocumentForIndexing($phid, $context = null) { PhabricatorWorker::scheduleTask( 'PhabricatorSearchWorker', array( 'documentPHID' => $phid, + 'context' => $context, ), array( 'priority' => PhabricatorWorker::PRIORITY_IMPORT, )); } - public function indexDocumentByPHID($phid) { + public function indexDocumentByPHID($phid, $context) { $indexers = id(new PhutilSymbolLoader()) ->setAncestorClass('PhabricatorSearchDocumentIndexer') ->loadObjects(); foreach ($indexers as $indexer) { if ($indexer->shouldIndexDocumentByPHID($phid)) { - $indexer->indexDocumentByPHID($phid); + $indexer->indexDocumentByPHID($phid, $context); break; } } diff --git a/src/applications/search/management/PhabricatorSearchManagementIndexWorkflow.php b/src/applications/search/management/PhabricatorSearchManagementIndexWorkflow.php --- a/src/applications/search/management/PhabricatorSearchManagementIndexWorkflow.php +++ b/src/applications/search/management/PhabricatorSearchManagementIndexWorkflow.php @@ -114,16 +114,9 @@ } private function loadPHIDsByTypes($type) { - $indexer_symbols = id(new PhutilSymbolLoader()) - ->setAncestorClass('PhabricatorSearchDocumentIndexer') - ->setConcreteOnly(true) - ->setType('class') - ->selectAndLoadSymbols(); - - $indexers = array(); - foreach ($indexer_symbols as $symbol) { - $indexers[] = newv($symbol['name'], array()); - } + $indexers = id(new PhutilSymbolLoader()) + ->setAncestorClass('PhabricatorSearchObjectIndexer') + ->loadObjects(); $phids = array(); foreach ($indexers as $indexer) { diff --git a/src/applications/search/worker/PhabricatorSearchWorker.php b/src/applications/search/worker/PhabricatorSearchWorker.php --- a/src/applications/search/worker/PhabricatorSearchWorker.php +++ b/src/applications/search/worker/PhabricatorSearchWorker.php @@ -4,10 +4,12 @@ public function doWork() { $data = $this->getTaskData(); + $phid = idx($data, 'documentPHID'); + $context = idx($data, 'context'); id(new PhabricatorSearchIndexer()) - ->indexDocumentByPHID($phid); + ->indexDocumentByPHID($phid, $context); } } diff --git a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php --- a/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php +++ b/src/applications/transactions/editor/PhabricatorApplicationTransactionEditor.php @@ -800,7 +800,9 @@ if ($this->supportsSearch()) { id(new PhabricatorSearchIndexer()) - ->queueDocumentForIndexing($object->getPHID()); + ->queueDocumentForIndexing( + $object->getPHID(), + $this->getSearchContextParameter($object, $xactions)); } if ($this->shouldPublishFeedStory($object, $xactions)) { @@ -2355,6 +2357,15 @@ return false; } + /** + * @task search + */ + protected function getSearchContextParameter( + PhabricatorLiskDAO $object, + array $xactions) { + return null; + } + /* -( Herald Integration )-------------------------------------------------- */ diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php --- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementQuickstartWorkflow.php @@ -82,8 +82,8 @@ '{$NAMESPACE}', $dump); - // NOTE: This is a hack. We can not use `binary` for this column, because - // it is part of a fulltext index. + // NOTE: This is a hack. We can not use `binary` for these columns, because + // they are a part of a fulltext index. $old = $dump; $dump = preg_replace( '/`corpus` longtext CHARACTER SET .* COLLATE .*,/mi',