diff --git a/src/applications/config/check/PhabricatorMySQLSetupCheck.php b/src/applications/config/check/PhabricatorMySQLSetupCheck.php index 580ef9db67..d2318abcdd 100644 --- a/src/applications/config/check/PhabricatorMySQLSetupCheck.php +++ b/src/applications/config/check/PhabricatorMySQLSetupCheck.php @@ -1,402 +1,364 @@ executeRefChecks($ref); } } private function executeRefChecks(PhabricatorDatabaseRef $ref) { $max_allowed_packet = $ref->loadRawMySQLConfigValue('max_allowed_packet'); $host_name = $ref->getRefKey(); // This primarily supports setting the filesize limit for MySQL to 8MB, // which may produce a >16MB packet after escaping. $recommended_minimum = (32 * 1024 * 1024); if ($max_allowed_packet < $recommended_minimum) { $message = pht( 'On host "%s", MySQL is configured with a small "%s" (%d), which '. 'may cause some large writes to fail. The recommended minimum value '. 'for this setting is "%d".', $host_name, 'max_allowed_packet', $max_allowed_packet, $recommended_minimum); $this->newIssue('mysql.max_allowed_packet') ->setName(pht('Small MySQL "%s"', 'max_allowed_packet')) ->setMessage($message) ->setDatabaseRef($ref) ->addMySQLConfig('max_allowed_packet'); } $modes = $ref->loadRawMySQLConfigValue('sql_mode'); $modes = explode(',', $modes); if (!in_array('STRICT_ALL_TABLES', $modes)) { $summary = pht( 'MySQL is not in strict mode (on host "%s"), but using strict mode '. 'is strongly encouraged.', $host_name); $message = pht( "On database host \"%s\", the global %s is not set to %s. ". "It is strongly encouraged that you enable this mode when running ". "Phabricator.\n\n". "By default MySQL will silently ignore some types of errors, which ". "can cause data loss and raise security concerns. Enabling strict ". "mode makes MySQL raise an explicit error instead, and prevents this ". "entire class of problems from doing any damage.\n\n". "You can find more information about this mode (and how to configure ". "it) in the MySQL manual. Usually, it is sufficient to add this to ". "your %s file (in the %s section) and then restart %s:\n\n". "%s\n". "(Note that if you run other applications against the same database, ". "they may not work in strict mode. Be careful about enabling it in ". "these cases.)", $host_name, phutil_tag('tt', array(), 'sql_mode'), phutil_tag('tt', array(), 'STRICT_ALL_TABLES'), phutil_tag('tt', array(), 'my.cnf'), phutil_tag('tt', array(), '[mysqld]'), phutil_tag('tt', array(), 'mysqld'), phutil_tag('pre', array(), 'sql_mode=STRICT_ALL_TABLES')); $this->newIssue('mysql.mode') ->setName(pht('MySQL %s Mode Not Set', 'STRICT_ALL_TABLES')) ->setSummary($summary) ->setMessage($message) ->setDatabaseRef($ref) ->addMySQLConfig('sql_mode'); } if (in_array('ONLY_FULL_GROUP_BY', $modes)) { $summary = pht( 'MySQL is in ONLY_FULL_GROUP_BY mode (on host "%s"), but using this '. 'mode is strongly discouraged.', $host_name); $message = pht( "On database host \"%s\", the global %s is set to %s. ". "It is strongly encouraged that you disable this mode when running ". "Phabricator.\n\n". "With %s enabled, MySQL rejects queries for which the select list ". "or (as of MySQL 5.0.23) %s list refer to nonaggregated columns ". "that are not named in the %s clause. More importantly, Phabricator ". "does not work properly with this mode enabled.\n\n". "You can find more information about this mode (and how to configure ". "it) in the MySQL manual. Usually, it is sufficient to change the %s ". "in your %s file (in the %s section) and then restart %s:\n\n". "%s\n". "(Note that if you run other applications against the same database, ". "they may not work with %s. Be careful about enabling ". "it in these cases and consider migrating Phabricator to a different ". "database.)", $host_name, phutil_tag('tt', array(), 'sql_mode'), phutil_tag('tt', array(), 'ONLY_FULL_GROUP_BY'), phutil_tag('tt', array(), 'ONLY_FULL_GROUP_BY'), phutil_tag('tt', array(), 'HAVING'), phutil_tag('tt', array(), 'GROUP BY'), phutil_tag('tt', array(), 'sql_mode'), phutil_tag('tt', array(), 'my.cnf'), phutil_tag('tt', array(), '[mysqld]'), phutil_tag('tt', array(), 'mysqld'), phutil_tag('pre', array(), 'sql_mode=STRICT_ALL_TABLES'), phutil_tag('tt', array(), 'ONLY_FULL_GROUP_BY')); $this->newIssue('mysql.mode') ->setName(pht('MySQL %s Mode Set', 'ONLY_FULL_GROUP_BY')) ->setSummary($summary) ->setMessage($message) ->setDatabaseRef($ref) ->addMySQLConfig('sql_mode'); } $stopword_file = $ref->loadRawMySQLConfigValue('ft_stopword_file'); if ($this->shouldUseMySQLSearchEngine()) { if ($stopword_file === null) { $summary = pht( 'Your version of MySQL (on database host "%s") does not support '. 'configuration of a stopword file. You will not be able to find '. 'search results for common words.', $host_name); $message = pht( "Database host \"%s\" does not support the %s option. You will not ". "be able to find search results for common words. You can gain ". "access to this option by upgrading MySQL to a more recent ". "version.\n\n". "You can ignore this warning if you plan to configure ElasticSearch ". "later, or aren't concerned about searching for common words.", $host_name, phutil_tag('tt', array(), 'ft_stopword_file')); $this->newIssue('mysql.ft_stopword_file') ->setName(pht('MySQL %s Not Supported', 'ft_stopword_file')) ->setSummary($summary) ->setMessage($message) ->setDatabaseRef($ref) ->addMySQLConfig('ft_stopword_file'); } else if ($stopword_file == '(built-in)') { $root = dirname(phutil_get_library_root('phabricator')); $stopword_path = $root.'/resources/sql/stopwords.txt'; $stopword_path = Filesystem::resolvePath($stopword_path); $namespace = PhabricatorEnv::getEnvConfig('storage.default-namespace'); $summary = pht( 'MySQL (on host "%s") is using a default stopword file, which '. 'will prevent searching for many common words.', $host_name); $message = pht( "Database host \"%s\" is using the builtin stopword file for ". "building search indexes. This can make Phabricator's search ". "feature less useful.\n\n". "Stopwords are common words which are not indexed and thus can not ". "be searched for. The default stopword file has about 500 words, ". "including various words which you are likely to wish to search ". "for, such as 'various', 'likely', 'wish', and 'zero'.\n\n". "To make search more useful, you can use an alternate stopword ". "file with fewer words. Alternatively, if you aren't concerned ". "about searching for common words, you can ignore this warning. ". "If you later plan to configure ElasticSearch, you can also ignore ". "this warning: this stopword file only affects MySQL fulltext ". "indexes.\n\n". "To choose a different stopword file, add this to your %s file ". "(in the %s section) and then restart %s:\n\n". "%s\n". "(You can also use a different file if you prefer. The file ". "suggested above has about 50 of the most common English words.)\n\n". "Finally, run this command to rebuild indexes using the new ". "rules:\n\n". "%s", $host_name, phutil_tag('tt', array(), 'my.cnf'), phutil_tag('tt', array(), '[mysqld]'), phutil_tag('tt', array(), 'mysqld'), phutil_tag('pre', array(), 'ft_stopword_file='.$stopword_path), phutil_tag( 'pre', array(), "mysql> REPAIR TABLE {$namespace}_search.search_documentfield;")); $this->newIssue('mysql.ft_stopword_file') ->setName(pht('MySQL is Using Default Stopword File')) ->setSummary($summary) ->setMessage($message) ->setDatabaseRef($ref) ->addMySQLConfig('ft_stopword_file'); } } $min_len = $ref->loadRawMySQLConfigValue('ft_min_word_len'); if ($min_len >= 4) { if ($this->shouldUseMySQLSearchEngine()) { $namespace = PhabricatorEnv::getEnvConfig('storage.default-namespace'); $summary = pht( 'MySQL is configured (on host "%s") to only index words with at '. 'least %d characters.', $host_name, $min_len); $message = pht( "Database host \"%s\" is configured to use the default minimum word ". "length when building search indexes, which is 4. This means words ". "which are only 3 characters long will not be indexed and can not ". "be searched for.\n\n". "For example, you will not be able to find search results for words ". "like 'SMS', 'web', or 'DOS'.\n\n". "You can change this setting to 3 to allow these words to be ". "indexed. Alternatively, you can ignore this warning if you are ". "not concerned about searching for 3-letter words. If you later ". "plan to configure ElasticSearch, you can also ignore this warning: ". "only MySQL fulltext search is affected.\n\n". "To reduce the minimum word length to 3, add this to your %s file ". "(in the %s section) and then restart %s:\n\n". "%s\n". "Finally, run this command to rebuild indexes using the new ". "rules:\n\n". "%s", $host_name, phutil_tag('tt', array(), 'my.cnf'), phutil_tag('tt', array(), '[mysqld]'), phutil_tag('tt', array(), 'mysqld'), phutil_tag('pre', array(), 'ft_min_word_len=3'), phutil_tag( 'pre', array(), "mysql> REPAIR TABLE {$namespace}_search.search_documentfield;")); $this->newIssue('mysql.ft_min_word_len') ->setName(pht('MySQL is Using Default Minimum Word Length')) ->setSummary($summary) ->setMessage($message) ->setDatabaseRef($ref) ->addMySQLConfig('ft_min_word_len'); } } - $bool_syntax = $ref->loadRawMySQLConfigValue('ft_boolean_syntax'); - if ($bool_syntax != ' |-><()~*:""&^') { - if ($this->shouldUseMySQLSearchEngine()) { - $summary = pht( - 'MySQL (on host "%s") is configured to search on fulltext indexes '. - 'using "OR" by default. Using "AND" is usually the desired '. - 'behaviour.', - $host_name); - - $message = pht( - "Database host \"%s\" is configured to use the default Boolean ". - "search syntax when using fulltext indexes. This means searching ". - "for 'search words' will yield the query 'search OR words' ". - "instead of the desired 'search AND words'.\n\n". - "This might produce unexpected search results. \n\n". - "You can change this setting to a more sensible default. ". - "Alternatively, you can ignore this warning if ". - "using 'OR' is the desired behaviour. If you later plan ". - "to configure ElasticSearch, you can also ignore this warning: ". - "only MySQL fulltext search is affected.\n\n". - "To change this setting, add this to your %s file ". - "(in the %s section) and then restart %s:\n\n". - "%s\n", - $host_name, - phutil_tag('tt', array(), 'my.cnf'), - phutil_tag('tt', array(), '[mysqld]'), - phutil_tag('tt', array(), 'mysqld'), - phutil_tag('pre', array(), 'ft_boolean_syntax=\' |-><()~*:""&^\'')); - - $this->newIssue('mysql.ft_boolean_syntax') - ->setName(pht('MySQL is Using the Default Boolean Syntax')) - ->setSummary($summary) - ->setMessage($message) - ->setDatabaseRef($ref) - ->addMySQLConfig('ft_boolean_syntax'); - } - } - $innodb_pool = $ref->loadRawMySQLConfigValue('innodb_buffer_pool_size'); $innodb_bytes = phutil_parse_bytes($innodb_pool); $innodb_readable = phutil_format_bytes($innodb_bytes); // This is arbitrary and just trying to detect values that the user // probably didn't set themselves. The Mac OS X default is 128MB and // 40% of an AWS EC2 Micro instance is 245MB, so keeping it somewhere // between those two values seems like a reasonable approximation. $minimum_readable = '225MB'; $minimum_bytes = phutil_parse_bytes($minimum_readable); if ($innodb_bytes < $minimum_bytes) { $summary = pht( 'MySQL (on host "%s") is configured with a very small '. 'innodb_buffer_pool_size, which may impact performance.', $host_name); $message = pht( "Database host \"%s\" is configured with a very small %s (%s). ". "This may cause poor database performance and lock exhaustion.\n\n". "There are no hard-and-fast rules to setting an appropriate value, ". "but a reasonable starting point for a standard install is something ". "like 40%% of the total memory on the machine. For example, if you ". "have 4GB of RAM on the machine you have installed Phabricator on, ". "you might set this value to %s.\n\n". "You can read more about this option in the MySQL documentation to ". "help you make a decision about how to configure it for your use ". "case. There are no concerns specific to Phabricator which make it ". "different from normal workloads with respect to this setting.\n\n". "To adjust the setting, add something like this to your %s file (in ". "the %s section), replacing %s with an appropriate value for your ". "host and use case. Then restart %s:\n\n". "%s\n". "If you're satisfied with the current setting, you can safely ". "ignore this setup warning.", $host_name, phutil_tag('tt', array(), 'innodb_buffer_pool_size'), phutil_tag('tt', array(), $innodb_readable), phutil_tag('tt', array(), '1600M'), phutil_tag('tt', array(), 'my.cnf'), phutil_tag('tt', array(), '[mysqld]'), phutil_tag('tt', array(), '1600M'), phutil_tag('tt', array(), 'mysqld'), phutil_tag('pre', array(), 'innodb_buffer_pool_size=1600M')); $this->newIssue('mysql.innodb_buffer_pool_size') ->setName(pht('MySQL May Run Slowly')) ->setSummary($summary) ->setMessage($message) ->setDatabaseRef($ref) ->addMySQLConfig('innodb_buffer_pool_size'); } $conn = $ref->newManagementConnection(); $ok = PhabricatorStorageManagementAPI::isCharacterSetAvailableOnConnection( 'utf8mb4', $conn); if (!$ok) { $summary = pht( 'You are using an old version of MySQL (on host "%s"), and should '. 'upgrade.', $host_name); $message = pht( 'You are using an old version of MySQL (on host "%s") which has poor '. 'unicode support (it does not support the "utf8mb4" collation set). '. 'You will encounter limitations when working with some unicode data.'. "\n\n". 'We strongly recommend you upgrade to MySQL 5.5 or newer.', $host_name); $this->newIssue('mysql.utf8mb4') ->setName(pht('Old MySQL Version')) ->setSummary($summary) ->setDatabaseRef($ref) ->setMessage($message); } $info = queryfx_one( $conn, 'SELECT UNIX_TIMESTAMP() epoch'); $epoch = (int)$info['epoch']; $local = PhabricatorTime::getNow(); $delta = (int)abs($local - $epoch); if ($delta > 60) { $this->newIssue('mysql.clock') ->setName(pht('Major Web/Database Clock Skew')) ->setSummary( pht( 'This web host ("%s") is set to a very different time than a '. 'database host "%s".', php_uname('n'), $host_name)) ->setMessage( pht( 'A database host ("%s") and this web host ("%s") disagree on the '. 'current time by more than 60 seconds (absolute skew is %s '. 'seconds). Check that the current time is set correctly '. 'everywhere.', $host_name, php_uname('n'), new PhutilNumber($delta))); } } protected function shouldUseMySQLSearchEngine() { $search_engine = PhabricatorFulltextStorageEngine::loadEngine(); return ($search_engine instanceof PhabricatorMySQLFulltextStorageEngine); } } diff --git a/src/applications/conpherence/query/ConpherenceFulltextQuery.php b/src/applications/conpherence/query/ConpherenceFulltextQuery.php index cd1e202ebd..43de7455ae 100644 --- a/src/applications/conpherence/query/ConpherenceFulltextQuery.php +++ b/src/applications/conpherence/query/ConpherenceFulltextQuery.php @@ -1,81 +1,85 @@ threadPHIDs = $phids; return $this; } public function withPreviousTransactionPHIDs(array $phids) { $this->previousTransactionPHIDs = $phids; return $this; } public function withFulltext($fulltext) { $this->fulltext = $fulltext; return $this; } public function execute() { $table = new ConpherenceIndex(); $conn_r = $table->establishConnection('r'); $rows = queryfx_all( $conn_r, 'SELECT threadPHID, transactionPHID, previousTransactionPHID FROM %T i %Q %Q %Q', $table->getTableName(), $this->buildWhereClause($conn_r), $this->buildOrderByClause($conn_r), $this->buildLimitClause($conn_r)); return $rows; } protected function buildWhereClause(AphrontDatabaseConnection $conn_r) { $where = array(); if ($this->threadPHIDs !== null) { $where[] = qsprintf( $conn_r, 'i.threadPHID IN (%Ls)', $this->threadPHIDs); } if ($this->previousTransactionPHIDs !== null) { $where[] = qsprintf( $conn_r, 'i.previousTransactionPHID IN (%Ls)', $this->previousTransactionPHIDs); } if (strlen($this->fulltext)) { + $compiled_query = PhabricatorSearchDocument::newQueryCompiler() + ->setQuery($this->fulltext) + ->compileQuery(); + $where[] = qsprintf( $conn_r, 'MATCH(i.corpus) AGAINST (%s IN BOOLEAN MODE)', - $this->fulltext); + $compiled_query); } return $this->formatWhereClause($where); } private function buildOrderByClause(AphrontDatabaseConnection $conn_r) { if (strlen($this->fulltext)) { return qsprintf( $conn_r, 'ORDER BY MATCH(i.corpus) AGAINST (%s IN BOOLEAN MODE) DESC', $this->fulltext); } else { return qsprintf( $conn_r, 'ORDER BY id DESC'); } } } diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php index b7bef9a623..8937c124ea 100644 --- a/src/applications/search/controller/PhabricatorApplicationSearchController.php +++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php @@ -1,599 +1,601 @@ preface = $preface; return $this; } public function getPreface() { return $this->preface; } public function setQueryKey($query_key) { $this->queryKey = $query_key; return $this; } protected function getQueryKey() { return $this->queryKey; } public function setNavigation(AphrontSideNavFilterView $navigation) { $this->navigation = $navigation; return $this; } protected function getNavigation() { return $this->navigation; } public function setSearchEngine( PhabricatorApplicationSearchEngine $search_engine) { $this->searchEngine = $search_engine; return $this; } protected function getSearchEngine() { return $this->searchEngine; } protected function validateDelegatingController() { $parent = $this->getDelegatingController(); if (!$parent) { throw new Exception( pht('You must delegate to this controller, not invoke it directly.')); } $engine = $this->getSearchEngine(); if (!$engine) { throw new PhutilInvalidStateException('setEngine'); } $engine->setViewer($this->getRequest()->getUser()); $parent = $this->getDelegatingController(); } public function processRequest() { $this->validateDelegatingController(); $key = $this->getQueryKey(); if ($key == 'edit') { return $this->processEditRequest(); } else { return $this->processSearchRequest(); } } private function processSearchRequest() { $parent = $this->getDelegatingController(); $request = $this->getRequest(); $user = $request->getUser(); $engine = $this->getSearchEngine(); $nav = $this->getNavigation(); if (!$nav) { $nav = $this->buildNavigation(); } if ($request->isFormPost()) { $saved_query = $engine->buildSavedQueryFromRequest($request); $engine->saveQuery($saved_query); return id(new AphrontRedirectResponse())->setURI( $engine->getQueryResultsPageURI($saved_query->getQueryKey()).'#R'); } $named_query = null; $run_query = true; $query_key = $this->queryKey; if ($this->queryKey == 'advanced') { $run_query = false; $query_key = $request->getStr('query'); } else if (!strlen($this->queryKey)) { $found_query_data = false; if ($request->isHTTPGet() || $request->isQuicksand()) { // If this is a GET request and it has some query data, don't // do anything unless it's only before= or after=. We'll build and // execute a query from it below. This allows external tools to build // URIs like "/query/?users=a,b". $pt_data = $request->getPassthroughRequestData(); $exempt = array( 'before' => true, 'after' => true, 'nux' => true, 'overheated' => true, ); foreach ($pt_data as $pt_key => $pt_value) { if (isset($exempt[$pt_key])) { continue; } $found_query_data = true; break; } } if (!$found_query_data) { // Otherwise, there's no query data so just run the user's default // query for this application. $query_key = head_key($engine->loadEnabledNamedQueries()); } } if ($engine->isBuiltinQuery($query_key)) { $saved_query = $engine->buildSavedQueryFromBuiltin($query_key); $named_query = idx($engine->loadEnabledNamedQueries(), $query_key); } else if ($query_key) { $saved_query = id(new PhabricatorSavedQueryQuery()) ->setViewer($user) ->withQueryKeys(array($query_key)) ->executeOne(); if (!$saved_query) { return new Aphront404Response(); } $named_query = idx($engine->loadEnabledNamedQueries(), $query_key); } else { $saved_query = $engine->buildSavedQueryFromRequest($request); // Save the query to generate a query key, so "Save Custom Query..." and // other features like Maniphest's "Export..." work correctly. $engine->saveQuery($saved_query); } $nav->selectFilter( 'query/'.$saved_query->getQueryKey(), 'query/advanced'); $form = id(new AphrontFormView()) ->setUser($user) ->setAction($request->getPath()); $engine->buildSearchForm($form, $saved_query); $errors = $engine->getErrors(); if ($errors) { $run_query = false; } $submit = id(new AphrontFormSubmitControl()) ->setValue(pht('Execute Query')); if ($run_query && !$named_query && $user->isLoggedIn()) { $submit->addCancelButton( '/search/edit/'.$saved_query->getQueryKey().'/', pht('Save Custom Query...')); } // TODO: A "Create Dashboard Panel" action goes here somewhere once // we sort out T5307. $form->appendChild($submit); $body = array(); if ($this->getPreface()) { $body[] = $this->getPreface(); } if ($named_query) { $title = $named_query->getQueryName(); } else { $title = pht('Advanced Search'); } $header = id(new PHUIHeaderView()) ->setHeader($title) ->setProfileHeader(true); $box = id(new PHUIObjectBoxView()) ->setHeader($header) ->addClass('application-search-results'); if ($run_query || $named_query) { $box->setShowHide( pht('Edit Query'), pht('Hide Query'), $form, $this->getApplicationURI('query/advanced/?query='.$query_key), (!$named_query ? true : false)); } else { $box->setForm($form); } $body[] = $box; $more_crumbs = null; if ($run_query) { $exec_errors = array(); $box->setAnchor( id(new PhabricatorAnchorView()) ->setAnchorName('R')); try { $engine->setRequest($request); $query = $engine->buildQueryFromSavedQuery($saved_query); $pager = $engine->newPagerForSavedQuery($saved_query); $pager->readFromRequest($request); $objects = $engine->executeQuery($query, $pager); $force_nux = $request->getBool('nux'); if (!$objects || $force_nux) { $nux_view = $this->renderNewUserView($engine, $force_nux); } else { $nux_view = null; } $force_overheated = $request->getBool('overheated'); $is_overheated = $query->getIsOverheated() || $force_overheated; if ($nux_view) { $box->appendChild($nux_view); } else { $list = $engine->renderResults($objects, $saved_query); if (!($list instanceof PhabricatorApplicationSearchResultView)) { throw new Exception( pht( 'SearchEngines must render a "%s" object, but this engine '. '(of class "%s") rendered something else.', 'PhabricatorApplicationSearchResultView', get_class($engine))); } if ($list->getObjectList()) { $box->setObjectList($list->getObjectList()); } if ($list->getTable()) { $box->setTable($list->getTable()); } if ($list->getInfoView()) { $box->setInfoView($list->getInfoView()); } if ($list->getContent()) { $box->appendChild($list->getContent()); } if ($is_overheated) { $box->appendChild($this->newOverheatedView($objects)); } $result_header = $list->getHeader(); if ($result_header) { $box->setHeader($result_header); $header = $result_header; } $actions = $list->getActions(); if ($actions) { foreach ($actions as $action) { $header->addActionLink($action); } } $use_actions = $engine->newUseResultsActions($saved_query); // TODO: Eventually, modularize all this stuff. $builtin_use_actions = $this->newBuiltinUseActions(); if ($builtin_use_actions) { foreach ($builtin_use_actions as $builtin_use_action) { $use_actions[] = $builtin_use_action; } } if ($use_actions) { $use_dropdown = $this->newUseResultsDropdown( $saved_query, $use_actions); $header->addActionLink($use_dropdown); } $more_crumbs = $list->getCrumbs(); if ($pager->willShowPagingControls()) { $pager_box = id(new PHUIBoxView()) ->setColor(PHUIBoxView::GREY) ->addClass('application-search-pager') ->appendChild($pager); $body[] = $pager_box; } } } catch (PhabricatorTypeaheadInvalidTokenException $ex) { $exec_errors[] = pht( 'This query specifies an invalid parameter. Review the '. 'query parameters and correct errors.'); + } catch (PhutilSearchQueryCompilerSyntaxException $ex) { + $exec_errors[] = $ex->getMessage(); } // The engine may have encountered additional errors during rendering; // merge them in and show everything. foreach ($engine->getErrors() as $error) { $exec_errors[] = $error; } $errors = $exec_errors; } if ($errors) { $box->setFormErrors($errors, pht('Query Errors')); } $crumbs = $parent ->buildApplicationCrumbs() ->setBorder(true); if ($more_crumbs) { $query_uri = $engine->getQueryResultsPageURI($saved_query->getQueryKey()); $crumbs->addTextCrumb($title, $query_uri); foreach ($more_crumbs as $crumb) { $crumbs->addCrumb($crumb); } } else { $crumbs->addTextCrumb($title); } require_celerity_resource('application-search-view-css'); return $this->newPage() ->setApplicationMenu($this->buildApplicationMenu()) ->setTitle(pht('Query: %s', $title)) ->setCrumbs($crumbs) ->setNavigation($nav) ->addClass('application-search-view') ->appendChild($body); } private function processEditRequest() { $parent = $this->getDelegatingController(); $request = $this->getRequest(); $user = $request->getUser(); $engine = $this->getSearchEngine(); $nav = $this->getNavigation(); if (!$nav) { $nav = $this->buildNavigation(); } $named_queries = $engine->loadAllNamedQueries(); $list_id = celerity_generate_unique_node_id(); $list = new PHUIObjectItemListView(); $list->setUser($user); $list->setID($list_id); Javelin::initBehavior( 'search-reorder-queries', array( 'listID' => $list_id, 'orderURI' => '/search/order/'.get_class($engine).'/', )); foreach ($named_queries as $named_query) { $class = get_class($engine); $key = $named_query->getQueryKey(); $item = id(new PHUIObjectItemView()) ->setHeader($named_query->getQueryName()) ->setHref($engine->getQueryResultsPageURI($key)); if ($named_query->getIsBuiltin() && $named_query->getIsDisabled()) { $icon = 'fa-plus'; } else { $icon = 'fa-times'; } $item->addAction( id(new PHUIListItemView()) ->setIcon($icon) ->setHref('/search/delete/'.$key.'/'.$class.'/') ->setWorkflow(true)); if ($named_query->getIsBuiltin()) { if ($named_query->getIsDisabled()) { $item->addIcon('fa-times lightgreytext', pht('Disabled')); $item->setDisabled(true); } else { $item->addIcon('fa-lock lightgreytext', pht('Builtin')); } } else { $item->addAction( id(new PHUIListItemView()) ->setIcon('fa-pencil') ->setHref('/search/edit/'.$key.'/')); } $item->setGrippable(true); $item->addSigil('named-query'); $item->setMetadata( array( 'queryKey' => $named_query->getQueryKey(), )); $list->addItem($item); } $list->setNoDataString(pht('No saved queries.')); $crumbs = $parent ->buildApplicationCrumbs() ->addTextCrumb(pht('Saved Queries'), $engine->getQueryManagementURI()) ->setBorder(true); $nav->selectFilter('query/edit'); $header = id(new PHUIHeaderView()) ->setHeader(pht('Saved Queries')) ->setProfileHeader(true); $box = id(new PHUIObjectBoxView()) ->setHeader($header) ->setObjectList($list) ->addClass('application-search-results'); $nav->addClass('application-search-view'); require_celerity_resource('application-search-view-css'); return $this->newPage() ->setApplicationMenu($this->buildApplicationMenu()) ->setTitle(pht('Saved Queries')) ->setCrumbs($crumbs) ->setNavigation($nav) ->appendChild($box); } public function buildApplicationMenu() { $menu = $this->getDelegatingController() ->buildApplicationMenu(); if ($menu instanceof PHUIApplicationMenuView) { $menu->setSearchEngine($this->getSearchEngine()); } return $menu; } private function buildNavigation() { $viewer = $this->getViewer(); $engine = $this->getSearchEngine(); $nav = id(new AphrontSideNavFilterView()) ->setUser($viewer) ->setBaseURI(new PhutilURI($this->getApplicationURI())); $engine->addNavigationItems($nav->getMenu()); return $nav; } private function renderNewUserView( PhabricatorApplicationSearchEngine $engine, $force_nux) { // Don't render NUX if the user has clicked away from the default page. if (strlen($this->getQueryKey())) { return null; } // Don't put NUX in panels because it would be weird. if ($engine->isPanelContext()) { return null; } // Try to render the view itself first, since this should be very cheap // (just returning some text). $nux_view = $engine->renderNewUserView(); if (!$nux_view) { return null; } $query = $engine->newQuery(); if (!$query) { return null; } // Try to load any object at all. If we can, the application has seen some // use so we just render the normal view. if (!$force_nux) { $object = $query ->setViewer(PhabricatorUser::getOmnipotentUser()) ->setLimit(1) ->execute(); if ($object) { return null; } } return $nux_view; } private function newUseResultsDropdown( PhabricatorSavedQuery $query, array $dropdown_items) { $viewer = $this->getViewer(); $action_list = id(new PhabricatorActionListView()) ->setViewer($viewer); foreach ($dropdown_items as $dropdown_item) { $action_list->addAction($dropdown_item); } return id(new PHUIButtonView()) ->setTag('a') ->setHref('#') ->setText(pht('Use Results...')) ->setIcon('fa-road') ->setDropdownMenu($action_list); } private function newOverheatedView(array $results) { if ($results) { $message = pht( 'Most objects matching your query are not visible to you, so '. 'filtering results is taking a long time. Only some results are '. 'shown. Refine your query to find results more quickly.'); } else { $message = pht( 'Most objects matching your query are not visible to you, so '. 'filtering results is taking a long time. Refine your query to '. 'find results more quickly.'); } return id(new PHUIInfoView()) ->setSeverity(PHUIInfoView::SEVERITY_WARNING) ->setFlush(true) ->setTitle(pht('Query Overheated')) ->setErrors( array( $message, )); } private function newBuiltinUseActions() { $actions = array(); $is_dev = PhabricatorEnv::getEnvConfig('phabricator.developer-mode'); if ($is_dev) { $engine = $this->getSearchEngine(); $nux_uri = $engine->getQueryBaseURI(); $nux_uri = id(new PhutilURI($nux_uri)) ->setQueryParam('nux', true); $actions[] = id(new PhabricatorActionView()) ->setIcon('fa-bug') ->setName(pht('Developer: Show New User State')) ->setHref($nux_uri); } if ($is_dev) { $overheated_uri = $this->getRequest()->getRequestURI() ->setQueryParam('overheated', true); $actions[] = id(new PhabricatorActionView()) ->setIcon('fa-bug') ->setName(pht('Developer: Show Overheated State')) ->setHref($overheated_uri); } return $actions; } } diff --git a/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php b/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php index eafda99c77..d353fa62c8 100644 --- a/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php +++ b/src/applications/search/fulltextstorage/PhabricatorMySQLFulltextStorageEngine.php @@ -1,357 +1,366 @@ getPHID(); if (!$phid) { throw new Exception(pht('Document has no PHID!')); } $store = new PhabricatorSearchDocument(); $store->setPHID($doc->getPHID()); $store->setDocumentType($doc->getDocumentType()); $store->setDocumentTitle($doc->getDocumentTitle()); $store->setDocumentCreated($doc->getDocumentCreated()); $store->setDocumentModified($doc->getDocumentModified()); $store->replace(); $conn_w = $store->establishConnection('w'); $field_dao = new PhabricatorSearchDocumentField(); queryfx( $conn_w, 'DELETE FROM %T WHERE phid = %s', $field_dao->getTableName(), $phid); foreach ($doc->getFieldData() as $field) { list($ftype, $corpus, $aux_phid) = $field; queryfx( $conn_w, 'INSERT INTO %T (phid, phidType, field, auxPHID, corpus) '. 'VALUES (%s, %s, %s, %ns, %s)', $field_dao->getTableName(), $phid, $doc->getDocumentType(), $ftype, $aux_phid, $corpus); } $sql = array(); foreach ($doc->getRelationshipData() as $relationship) { list($rtype, $to_phid, $to_type, $time) = $relationship; $sql[] = qsprintf( $conn_w, '(%s, %s, %s, %s, %d)', $phid, $to_phid, $rtype, $to_type, $time); } $rship_dao = new PhabricatorSearchDocumentRelationship(); queryfx( $conn_w, 'DELETE FROM %T WHERE phid = %s', $rship_dao->getTableName(), $phid); if ($sql) { queryfx( $conn_w, 'INSERT INTO %T '. '(phid, relatedPHID, relation, relatedType, relatedTime) '. 'VALUES %Q', $rship_dao->getTableName(), implode(', ', $sql)); } } /** * Rebuild the PhabricatorSearchAbstractDocument that was used to index * an object out of the index itself. This is primarily useful for debugging, * as it allows you to inspect the search index representation of a * document. * * @param phid PHID of a document which exists in the search index. * @return null|PhabricatorSearchAbstractDocument Abstract document object * which corresponds to the original abstract document used to * build the document index. */ public function reconstructDocument($phid) { $dao_doc = new PhabricatorSearchDocument(); $dao_field = new PhabricatorSearchDocumentField(); $dao_relationship = new PhabricatorSearchDocumentRelationship(); $t_doc = $dao_doc->getTableName(); $t_field = $dao_field->getTableName(); $t_relationship = $dao_relationship->getTableName(); $doc = queryfx_one( $dao_doc->establishConnection('r'), 'SELECT * FROM %T WHERE phid = %s', $t_doc, $phid); if (!$doc) { return null; } $fields = queryfx_all( $dao_field->establishConnection('r'), 'SELECT * FROM %T WHERE phid = %s', $t_field, $phid); $relationships = queryfx_all( $dao_relationship->establishConnection('r'), 'SELECT * FROM %T WHERE phid = %s', $t_relationship, $phid); $adoc = id(new PhabricatorSearchAbstractDocument()) ->setPHID($phid) ->setDocumentType($doc['documentType']) ->setDocumentTitle($doc['documentTitle']) ->setDocumentCreated($doc['documentCreated']) ->setDocumentModified($doc['documentModified']); foreach ($fields as $field) { $adoc->addField( $field['field'], $field['corpus'], $field['auxPHID']); } foreach ($relationships as $relationship) { $adoc->addRelationship( $relationship['relation'], $relationship['relatedPHID'], $relationship['relatedType'], $relationship['relatedTime']); } return $adoc; } public function executeSearch(PhabricatorSavedQuery $query) { $where = array(); $join = array(); $order = 'ORDER BY documentCreated DESC'; $dao_doc = new PhabricatorSearchDocument(); $dao_field = new PhabricatorSearchDocumentField(); $t_doc = $dao_doc->getTableName(); $t_field = $dao_field->getTableName(); $conn_r = $dao_doc->establishConnection('r'); - $q = $query->getParameter('query'); + $raw_query = $query->getParameter('query'); + $q = $this->compileQuery($raw_query); if (strlen($q)) { $join[] = qsprintf( $conn_r, '%T field ON field.phid = document.phid', $t_field); $where[] = qsprintf( $conn_r, 'MATCH(corpus) AGAINST (%s IN BOOLEAN MODE)', $q); // When searching for a string, promote user listings above other // listings. $order = qsprintf( $conn_r, 'ORDER BY IF(documentType = %s, 0, 1) ASC, MAX(MATCH(corpus) AGAINST (%s)) DESC', 'USER', $q); $field = $query->getParameter('field'); if ($field) { $where[] = qsprintf( $conn_r, 'field.field = %s', $field); } } $exclude = $query->getParameter('exclude'); if ($exclude) { $where[] = qsprintf($conn_r, 'document.phid != %s', $exclude); } $types = $query->getParameter('types'); if ($types) { if (strlen($q)) { $where[] = qsprintf( $conn_r, 'field.phidType IN (%Ls)', $types); } $where[] = qsprintf( $conn_r, 'document.documentType IN (%Ls)', $types); } $join[] = $this->joinRelationship( $conn_r, $query, 'authorPHIDs', PhabricatorSearchRelationship::RELATIONSHIP_AUTHOR); $statuses = $query->getParameter('statuses', array()); $statuses = array_fuse($statuses); $open_rel = PhabricatorSearchRelationship::RELATIONSHIP_OPEN; $closed_rel = PhabricatorSearchRelationship::RELATIONSHIP_CLOSED; $include_open = !empty($statuses[$open_rel]); $include_closed = !empty($statuses[$closed_rel]); if ($include_open && !$include_closed) { $join[] = $this->joinRelationship( $conn_r, $query, 'statuses', $open_rel, true); } else if ($include_closed && !$include_open) { $join[] = $this->joinRelationship( $conn_r, $query, 'statuses', $closed_rel, true); } if ($query->getParameter('withAnyOwner')) { $join[] = $this->joinRelationship( $conn_r, $query, 'withAnyOwner', PhabricatorSearchRelationship::RELATIONSHIP_OWNER, true); } else if ($query->getParameter('withUnowned')) { $join[] = $this->joinRelationship( $conn_r, $query, 'withUnowned', PhabricatorSearchRelationship::RELATIONSHIP_UNOWNED, true); } else { $join[] = $this->joinRelationship( $conn_r, $query, 'ownerPHIDs', PhabricatorSearchRelationship::RELATIONSHIP_OWNER); } $join[] = $this->joinRelationship( $conn_r, $query, 'subscriberPHIDs', PhabricatorSearchRelationship::RELATIONSHIP_SUBSCRIBER); $join[] = $this->joinRelationship( $conn_r, $query, 'projectPHIDs', PhabricatorSearchRelationship::RELATIONSHIP_PROJECT); $join[] = $this->joinRelationship( $conn_r, $query, 'repository', PhabricatorSearchRelationship::RELATIONSHIP_REPOSITORY); $join = array_filter($join); foreach ($join as $key => $clause) { $join[$key] = ' JOIN '.$clause; } $join = implode(' ', $join); if ($where) { $where = 'WHERE '.implode(' AND ', $where); } else { $where = ''; } $offset = (int)$query->getParameter('offset', 0); $limit = (int)$query->getParameter('limit', 25); $hits = queryfx_all( $conn_r, 'SELECT document.phid FROM %T document %Q %Q GROUP BY document.phid %Q LIMIT %d, %d', $t_doc, $join, $where, $order, $offset, $limit); return ipull($hits, 'phid'); } protected function joinRelationship( AphrontDatabaseConnection $conn, PhabricatorSavedQuery $query, $field, $type, $is_existence = false) { $sql = qsprintf( $conn, '%T AS %C ON %C.phid = document.phid AND %C.relation = %s', id(new PhabricatorSearchDocumentRelationship())->getTableName(), $field, $field, $field, $type); if (!$is_existence) { $phids = $query->getParameter($field, array()); if (!$phids) { return null; } $sql .= qsprintf( $conn, ' AND %C.relatedPHID in (%Ls)', $field, $phids); } return $sql; } + private function compileQuery($raw_query) { + $compiler = PhabricatorSearchDocument::newQueryCompiler(); + + return $compiler + ->setQuery($raw_query) + ->compileQuery(); + } + public function indexExists() { return true; } } diff --git a/src/applications/search/storage/document/PhabricatorSearchDocument.php b/src/applications/search/storage/document/PhabricatorSearchDocument.php index 3e177c9813..28080944b3 100644 --- a/src/applications/search/storage/document/PhabricatorSearchDocument.php +++ b/src/applications/search/storage/document/PhabricatorSearchDocument.php @@ -1,40 +1,56 @@ false, self::CONFIG_IDS => self::IDS_MANUAL, self::CONFIG_COLUMN_SCHEMA => array( 'documentType' => 'text4', 'documentTitle' => 'text255', 'documentCreated' => 'epoch', 'documentModified' => 'epoch', ), self::CONFIG_KEY_SCHEMA => array( 'key_phid' => null, 'PRIMARY' => array( 'columns' => array('phid'), 'unique' => true, ), 'documentCreated' => array( 'columns' => array('documentCreated'), ), 'key_type' => array( 'columns' => array('documentType', 'documentCreated'), ), ), ) + parent::getConfiguration(); } public function getIDKey() { return 'phid'; } + public static function newQueryCompiler() { + $table = new self(); + $conn = $table->establishConnection('r'); + + $compiler = new PhutilSearchQueryCompiler(); + + $operators = queryfx_one( + $conn, + 'SELECT @@ft_boolean_syntax AS syntax'); + if ($operators) { + $compiler->setOperators($operators['syntax']); + } + + return $compiler; + } + }