diff --git a/src/applications/almanac/query/AlmanacInterfaceQuery.php b/src/applications/almanac/query/AlmanacInterfaceQuery.php index d5886761c1..5738108ffc 100644 --- a/src/applications/almanac/query/AlmanacInterfaceQuery.php +++ b/src/applications/almanac/query/AlmanacInterfaceQuery.php @@ -1,200 +1,211 @@ ids = $ids; return $this; } public function withPHIDs(array $phids) { $this->phids = $phids; return $this; } public function withNetworkPHIDs(array $phids) { $this->networkPHIDs = $phids; return $this; } public function withDevicePHIDs(array $phids) { $this->devicePHIDs = $phids; return $this; } public function withAddresses(array $addresses) { $this->addresses = $addresses; return $this; } public function newResultObject() { return new AlmanacInterface(); } protected function loadPage() { return $this->loadStandardPage($this->newResultObject()); } protected function willFilterPage(array $interfaces) { $network_phids = mpull($interfaces, 'getNetworkPHID'); $device_phids = mpull($interfaces, 'getDevicePHID'); $networks = id(new AlmanacNetworkQuery()) ->setParentQuery($this) ->setViewer($this->getViewer()) ->withPHIDs($network_phids) ->needProperties($this->getNeedProperties()) ->execute(); $networks = mpull($networks, null, 'getPHID'); $devices = id(new AlmanacDeviceQuery()) ->setParentQuery($this) ->setViewer($this->getViewer()) ->withPHIDs($device_phids) ->needProperties($this->getNeedProperties()) ->execute(); $devices = mpull($devices, null, 'getPHID'); foreach ($interfaces as $key => $interface) { $network = idx($networks, $interface->getNetworkPHID()); $device = idx($devices, $interface->getDevicePHID()); if (!$network || !$device) { $this->didRejectResult($interface); unset($interfaces[$key]); continue; } $interface->attachNetwork($network); $interface->attachDevice($device); } return $interfaces; } + protected function buildSelectClauseParts(AphrontDatabaseConnection $conn) { + $select = parent::buildSelectClauseParts($conn); + + if ($this->shouldJoinDeviceTable()) { + $select[] = qsprintf($conn, 'device.name'); + } + + return $select; + } + protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { $where = parent::buildWhereClauseParts($conn); if ($this->ids !== null) { $where[] = qsprintf( $conn, 'interface.id IN (%Ld)', $this->ids); } if ($this->phids !== null) { $where[] = qsprintf( $conn, 'interface.phid IN (%Ls)', $this->phids); } if ($this->networkPHIDs !== null) { $where[] = qsprintf( $conn, 'interface.networkPHID IN (%Ls)', $this->networkPHIDs); } if ($this->devicePHIDs !== null) { $where[] = qsprintf( $conn, 'interface.devicePHID IN (%Ls)', $this->devicePHIDs); } if ($this->addresses !== null) { $parts = array(); foreach ($this->addresses as $address) { $parts[] = qsprintf( $conn, '(interface.networkPHID = %s '. 'AND interface.address = %s '. 'AND interface.port = %d)', $address->getNetworkPHID(), $address->getAddress(), $address->getPort()); } $where[] = qsprintf($conn, '%LO', $parts); } return $where; } protected function buildJoinClauseParts(AphrontDatabaseConnection $conn) { $joins = parent::buildJoinClauseParts($conn); if ($this->shouldJoinDeviceTable()) { $joins[] = qsprintf( $conn, 'JOIN %T device ON device.phid = interface.devicePHID', id(new AlmanacDevice())->getTableName()); } return $joins; } protected function shouldGroupQueryResultRows() { if ($this->shouldJoinDeviceTable()) { return true; } return parent::shouldGroupQueryResultRows(); } private function shouldJoinDeviceTable() { $vector = $this->getOrderVector(); if ($vector->containsKey('name')) { return true; } return false; } protected function getPrimaryTableAlias() { return 'interface'; } public function getQueryApplicationClass() { return 'PhabricatorAlmanacApplication'; } public function getBuiltinOrders() { return array( 'name' => array( 'vector' => array('name', 'id'), 'name' => pht('Device Name'), ), ) + parent::getBuiltinOrders(); } public function getOrderableColumns() { return parent::getOrderableColumns() + array( 'name' => array( 'table' => 'device', 'column' => 'name', 'type' => 'string', 'reverse' => true, ), ); } - protected function getPagingValueMap($cursor, array $keys) { - $interface = $this->loadCursorObject($cursor); + protected function newPagingMapFromCursorObject( + PhabricatorQueryCursor $cursor, + array $keys) { - $map = array( - 'id' => $interface->getID(), - 'name' => $interface->getDevice()->getName(), - ); + $interface = $cursor->getObject(); - return $map; + return array( + 'id' => (int)$interface->getID(), + 'name' => $cursor->getRawRowProperty('device.name'), + ); } } diff --git a/src/applications/feed/query/PhabricatorFeedQuery.php b/src/applications/feed/query/PhabricatorFeedQuery.php index a35f14da57..8302af20c1 100644 --- a/src/applications/feed/query/PhabricatorFeedQuery.php +++ b/src/applications/feed/query/PhabricatorFeedQuery.php @@ -1,171 +1,175 @@ filterPHIDs = $phids; return $this; } public function withChronologicalKeys(array $keys) { $this->chronologicalKeys = $keys; return $this; } public function withEpochInRange($range_min, $range_max) { $this->rangeMin = $range_min; $this->rangeMax = $range_max; return $this; } public function newResultObject() { return new PhabricatorFeedStoryData(); } protected function loadPage() { // NOTE: We return raw rows from this method, which is a little unusual. return $this->loadStandardPageRows($this->newResultObject()); } protected function willFilterPage(array $data) { $stories = PhabricatorFeedStory::loadAllFromRows($data, $this->getViewer()); foreach ($stories as $key => $story) { if (!$story->isVisibleInFeed()) { unset($stories[$key]); } } return $stories; } protected function buildJoinClauseParts(AphrontDatabaseConnection $conn) { $joins = parent::buildJoinClauseParts($conn); // NOTE: We perform this join unconditionally (even if we have no filter // PHIDs) to omit rows which have no story references. These story data // rows are notifications or realtime alerts. $ref_table = new PhabricatorFeedStoryReference(); $joins[] = qsprintf( $conn, 'JOIN %T ref ON ref.chronologicalKey = story.chronologicalKey', $ref_table->getTableName()); return $joins; } protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { $where = parent::buildWhereClauseParts($conn); if ($this->filterPHIDs !== null) { $where[] = qsprintf( $conn, 'ref.objectPHID IN (%Ls)', $this->filterPHIDs); } if ($this->chronologicalKeys !== null) { // NOTE: We can't use "%d" to format these large integers on 32-bit // systems. Historically, we formatted these into integers in an // awkward way because MySQL could sometimes (?) fail to use the proper // keys if the values were formatted as strings instead of integers. // After the "qsprintf()" update to use PhutilQueryString, we can no // longer do this in a sneaky way. However, the MySQL key issue also // no longer appears to reproduce across several systems. So: just use // strings until problems turn up? $where[] = qsprintf( $conn, 'ref.chronologicalKey IN (%Ls)', $this->chronologicalKeys); } // NOTE: We may not have 64-bit PHP, so do the shifts in MySQL instead. // From EXPLAIN, it appears like MySQL is smart enough to compute the // result and make use of keys to execute the query. if ($this->rangeMin !== null) { $where[] = qsprintf( $conn, 'ref.chronologicalKey >= (%d << 32)', $this->rangeMin); } if ($this->rangeMax !== null) { $where[] = qsprintf( $conn, 'ref.chronologicalKey < (%d << 32)', $this->rangeMax); } return $where; } protected function buildGroupClause(AphrontDatabaseConnection $conn) { if ($this->filterPHIDs !== null) { return qsprintf($conn, 'GROUP BY ref.chronologicalKey'); } else { return qsprintf($conn, 'GROUP BY story.chronologicalKey'); } } protected function getDefaultOrderVector() { return array('key'); } public function getBuiltinOrders() { return array( 'newest' => array( 'vector' => array('key'), 'name' => pht('Creation (Newest First)'), 'aliases' => array('created'), ), 'oldest' => array( 'vector' => array('-key'), 'name' => pht('Creation (Oldest First)'), ), ); } public function getOrderableColumns() { $table = ($this->filterPHIDs ? 'ref' : 'story'); return array( 'key' => array( 'table' => $table, 'column' => 'chronologicalKey', 'type' => 'string', 'unique' => true, ), ); } - protected function getPagingValueMap($cursor, array $keys) { - return array( - 'key' => $cursor, - ); + protected function applyExternalCursorConstraintsToQuery( + PhabricatorCursorPagedPolicyAwareQuery $subquery, + $cursor) { + $subquery->withChronologicalKeys(array($cursor)); } - protected function getResultCursor($item) { - if ($item instanceof PhabricatorFeedStory) { - return $item->getChronologicalKey(); - } - return $item['chronologicalKey']; + protected function newExternalCursorStringForResult($object) { + return $object->getChronologicalKey(); + } + + protected function newPagingMapFromPartialObject($object) { + // This query is unusual, and the "object" is a raw result row. + return array( + 'key' => $object['chronologicalKey'], + ); } protected function getPrimaryTableAlias() { return 'story'; } public function getQueryApplicationClass() { return 'PhabricatorFeedApplication'; } } diff --git a/src/applications/maniphest/query/ManiphestTaskQuery.php b/src/applications/maniphest/query/ManiphestTaskQuery.php index 1033bfe333..9e58728cff 100644 --- a/src/applications/maniphest/query/ManiphestTaskQuery.php +++ b/src/applications/maniphest/query/ManiphestTaskQuery.php @@ -1,982 +1,1054 @@ authorPHIDs = $authors; return $this; } public function withIDs(array $ids) { $this->taskIDs = $ids; return $this; } public function withPHIDs(array $phids) { $this->taskPHIDs = $phids; return $this; } public function withOwners(array $owners) { if ($owners === array()) { throw new Exception(pht('Empty withOwners() constraint is not valid.')); } $no_owner = PhabricatorPeopleNoOwnerDatasource::FUNCTION_TOKEN; $any_owner = PhabricatorPeopleAnyOwnerDatasource::FUNCTION_TOKEN; foreach ($owners as $k => $phid) { if ($phid === $no_owner || $phid === null) { $this->noOwner = true; unset($owners[$k]); break; } if ($phid === $any_owner) { $this->anyOwner = true; unset($owners[$k]); break; } } if ($owners) { $this->ownerPHIDs = $owners; } return $this; } public function withStatus($status) { $this->status = $status; return $this; } public function withStatuses(array $statuses) { $this->statuses = $statuses; return $this; } public function withPriorities(array $priorities) { $this->priorities = $priorities; return $this; } public function withSubpriorities(array $subpriorities) { $this->subpriorities = $subpriorities; return $this; } public function withSubscribers(array $subscribers) { $this->subscriberPHIDs = $subscribers; return $this; } public function setGroupBy($group) { $this->groupBy = $group; switch ($this->groupBy) { case self::GROUP_NONE: $vector = array(); break; case self::GROUP_PRIORITY: $vector = array('priority'); break; case self::GROUP_OWNER: $vector = array('owner'); break; case self::GROUP_STATUS: $vector = array('status'); break; case self::GROUP_PROJECT: $vector = array('project'); break; } $this->setGroupVector($vector); return $this; } public function withOpenSubtasks($value) { $this->hasOpenSubtasks = $value; return $this; } public function withOpenParents($value) { $this->hasOpenParents = $value; return $this; } public function withParentTaskIDs(array $ids) { $this->parentTaskIDs = $ids; return $this; } public function withSubtaskIDs(array $ids) { $this->subtaskIDs = $ids; return $this; } public function withDateCreatedBefore($date_created_before) { $this->dateCreatedBefore = $date_created_before; return $this; } public function withDateCreatedAfter($date_created_after) { $this->dateCreatedAfter = $date_created_after; return $this; } public function withDateModifiedBefore($date_modified_before) { $this->dateModifiedBefore = $date_modified_before; return $this; } public function withDateModifiedAfter($date_modified_after) { $this->dateModifiedAfter = $date_modified_after; return $this; } public function withClosedEpochBetween($min, $max) { $this->closedEpochMin = $min; $this->closedEpochMax = $max; return $this; } public function withCloserPHIDs(array $phids) { $this->closerPHIDs = $phids; return $this; } public function needSubscriberPHIDs($bool) { $this->needSubscriberPHIDs = $bool; return $this; } public function needProjectPHIDs($bool) { $this->needProjectPHIDs = $bool; return $this; } public function withBridgedObjectPHIDs(array $phids) { $this->bridgedObjectPHIDs = $phids; return $this; } public function withSubtypes(array $subtypes) { $this->subtypes = $subtypes; return $this; } public function withColumnPHIDs(array $column_phids) { $this->columnPHIDs = $column_phids; return $this; } + public function withSpecificGroupByProjectPHID($project_phid) { + $this->specificGroupByProjectPHID = $project_phid; + return $this; + } + public function newResultObject() { return new ManiphestTask(); } protected function loadPage() { $task_dao = new ManiphestTask(); $conn = $task_dao->establishConnection('r'); $where = $this->buildWhereClause($conn); $group_column = qsprintf($conn, ''); switch ($this->groupBy) { case self::GROUP_PROJECT: $group_column = qsprintf( $conn, ', projectGroupName.indexedObjectPHID projectGroupPHID'); break; } $rows = queryfx_all( $conn, '%Q %Q FROM %T task %Q %Q %Q %Q %Q %Q', $this->buildSelectClause($conn), $group_column, $task_dao->getTableName(), $this->buildJoinClause($conn), $where, $this->buildGroupClause($conn), $this->buildHavingClause($conn), $this->buildOrderClause($conn), $this->buildLimitClause($conn)); switch ($this->groupBy) { case self::GROUP_PROJECT: $data = ipull($rows, null, 'id'); break; default: $data = $rows; break; } $data = $this->didLoadRawRows($data); $tasks = $task_dao->loadAllFromArray($data); switch ($this->groupBy) { case self::GROUP_PROJECT: $results = array(); foreach ($rows as $row) { $task = clone $tasks[$row['id']]; $task->attachGroupByProjectPHID($row['projectGroupPHID']); $results[] = $task; } $tasks = $results; break; } return $tasks; } protected function willFilterPage(array $tasks) { if ($this->groupBy == self::GROUP_PROJECT) { // We should only return project groups which the user can actually see. $project_phids = mpull($tasks, 'getGroupByProjectPHID'); $projects = id(new PhabricatorProjectQuery()) ->setViewer($this->getViewer()) ->withPHIDs($project_phids) ->execute(); $projects = mpull($projects, null, 'getPHID'); foreach ($tasks as $key => $task) { if (!$task->getGroupByProjectPHID()) { // This task is either not tagged with any projects, or only tagged // with projects which we're ignoring because they're being queried // for explicitly. continue; } if (empty($projects[$task->getGroupByProjectPHID()])) { unset($tasks[$key]); } } } return $tasks; } protected function didFilterPage(array $tasks) { $phids = mpull($tasks, 'getPHID'); if ($this->needProjectPHIDs) { $edge_query = id(new PhabricatorEdgeQuery()) ->withSourcePHIDs($phids) ->withEdgeTypes( array( PhabricatorProjectObjectHasProjectEdgeType::EDGECONST, )); $edge_query->execute(); foreach ($tasks as $task) { $project_phids = $edge_query->getDestinationPHIDs( array($task->getPHID())); $task->attachProjectPHIDs($project_phids); } } if ($this->needSubscriberPHIDs) { $subscriber_sets = id(new PhabricatorSubscribersQuery()) ->withObjectPHIDs($phids) ->execute(); foreach ($tasks as $task) { $subscribers = idx($subscriber_sets, $task->getPHID(), array()); $task->attachSubscriberPHIDs($subscribers); } } return $tasks; } protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { $where = parent::buildWhereClauseParts($conn); $where[] = $this->buildStatusWhereClause($conn); $where[] = $this->buildOwnerWhereClause($conn); if ($this->taskIDs !== null) { $where[] = qsprintf( $conn, 'task.id in (%Ld)', $this->taskIDs); } if ($this->taskPHIDs !== null) { $where[] = qsprintf( $conn, 'task.phid in (%Ls)', $this->taskPHIDs); } if ($this->statuses !== null) { $where[] = qsprintf( $conn, 'task.status IN (%Ls)', $this->statuses); } if ($this->authorPHIDs !== null) { $where[] = qsprintf( $conn, 'task.authorPHID in (%Ls)', $this->authorPHIDs); } if ($this->dateCreatedAfter) { $where[] = qsprintf( $conn, 'task.dateCreated >= %d', $this->dateCreatedAfter); } if ($this->dateCreatedBefore) { $where[] = qsprintf( $conn, 'task.dateCreated <= %d', $this->dateCreatedBefore); } if ($this->dateModifiedAfter) { $where[] = qsprintf( $conn, 'task.dateModified >= %d', $this->dateModifiedAfter); } if ($this->dateModifiedBefore) { $where[] = qsprintf( $conn, 'task.dateModified <= %d', $this->dateModifiedBefore); } if ($this->closedEpochMin !== null) { $where[] = qsprintf( $conn, 'task.closedEpoch >= %d', $this->closedEpochMin); } if ($this->closedEpochMax !== null) { $where[] = qsprintf( $conn, 'task.closedEpoch <= %d', $this->closedEpochMax); } if ($this->closerPHIDs !== null) { $where[] = qsprintf( $conn, 'task.closerPHID IN (%Ls)', $this->closerPHIDs); } if ($this->priorities !== null) { $where[] = qsprintf( $conn, 'task.priority IN (%Ld)', $this->priorities); } if ($this->bridgedObjectPHIDs !== null) { $where[] = qsprintf( $conn, 'task.bridgedObjectPHID IN (%Ls)', $this->bridgedObjectPHIDs); } if ($this->subtypes !== null) { $where[] = qsprintf( $conn, 'task.subtype IN (%Ls)', $this->subtypes); } if ($this->columnPHIDs !== null) { $viewer = $this->getViewer(); $columns = id(new PhabricatorProjectColumnQuery()) ->setParentQuery($this) ->setViewer($viewer) ->withPHIDs($this->columnPHIDs) ->execute(); if (!$columns) { throw new PhabricatorEmptyQueryException(); } // We must do board layout before we move forward because the column // positions may not yet exist otherwise. An example is that newly // created tasks may not yet be positioned in the backlog column. $projects = mpull($columns, 'getProject'); $projects = mpull($projects, null, 'getPHID'); // The board layout engine needs to know about every object that it's // going to be asked to do layout for. For now, we're just doing layout // on every object on the boards. In the future, we could do layout on a // smaller set of objects by using the constraints on this Query. For // example, if the caller is only asking for open tasks, we only need // to do layout on open tasks. // This fetches too many objects (every type of object tagged with the // project, not just tasks). We could narrow it by querying the edge // table on the Maniphest side, but there's currently no way to build // that query with EdgeQuery. $edge_query = id(new PhabricatorEdgeQuery()) ->withSourcePHIDs(array_keys($projects)) ->withEdgeTypes( array( PhabricatorProjectProjectHasObjectEdgeType::EDGECONST, )); $edge_query->execute(); $all_phids = $edge_query->getDestinationPHIDs(); // Since we overfetched PHIDs, filter out any non-tasks we got back. foreach ($all_phids as $key => $phid) { if (phid_get_type($phid) !== ManiphestTaskPHIDType::TYPECONST) { unset($all_phids[$key]); } } // If there are no tasks on the relevant boards, this query can't // possibly hit anything so we're all done. $task_phids = array_fuse($all_phids); if (!$task_phids) { throw new PhabricatorEmptyQueryException(); } // We know everything we need to know, so perform board layout. $engine = id(new PhabricatorBoardLayoutEngine()) ->setViewer($viewer) ->setFetchAllBoards(true) ->setBoardPHIDs(array_keys($projects)) ->setObjectPHIDs($task_phids) ->executeLayout(); // Find the tasks that are in the constraint columns after board layout // completes. $select_phids = array(); foreach ($columns as $column) { $in_column = $engine->getColumnObjectPHIDs( $column->getProjectPHID(), $column->getPHID()); foreach ($in_column as $phid) { $select_phids[$phid] = $phid; } } if (!$select_phids) { throw new PhabricatorEmptyQueryException(); } $where[] = qsprintf( $conn, 'task.phid IN (%Ls)', $select_phids); } + if ($this->specificGroupByProjectPHID !== null) { + $where[] = qsprintf( + $conn, + 'projectGroupName.indexedObjectPHID = %s', + $this->specificGroupByProjectPHID); + } + return $where; } private function buildStatusWhereClause(AphrontDatabaseConnection $conn) { static $map = array( self::STATUS_RESOLVED => ManiphestTaskStatus::STATUS_CLOSED_RESOLVED, self::STATUS_WONTFIX => ManiphestTaskStatus::STATUS_CLOSED_WONTFIX, self::STATUS_INVALID => ManiphestTaskStatus::STATUS_CLOSED_INVALID, self::STATUS_SPITE => ManiphestTaskStatus::STATUS_CLOSED_SPITE, self::STATUS_DUPLICATE => ManiphestTaskStatus::STATUS_CLOSED_DUPLICATE, ); switch ($this->status) { case self::STATUS_ANY: return null; case self::STATUS_OPEN: return qsprintf( $conn, 'task.status IN (%Ls)', ManiphestTaskStatus::getOpenStatusConstants()); case self::STATUS_CLOSED: return qsprintf( $conn, 'task.status IN (%Ls)', ManiphestTaskStatus::getClosedStatusConstants()); default: $constant = idx($map, $this->status); if (!$constant) { throw new Exception(pht("Unknown status query '%s'!", $this->status)); } return qsprintf( $conn, 'task.status = %s', $constant); } } private function buildOwnerWhereClause(AphrontDatabaseConnection $conn) { $subclause = array(); if ($this->noOwner) { $subclause[] = qsprintf( $conn, 'task.ownerPHID IS NULL'); } if ($this->anyOwner) { $subclause[] = qsprintf( $conn, 'task.ownerPHID IS NOT NULL'); } if ($this->ownerPHIDs !== null) { $subclause[] = qsprintf( $conn, 'task.ownerPHID IN (%Ls)', $this->ownerPHIDs); } if (!$subclause) { return qsprintf($conn, ''); } return qsprintf($conn, '%LO', $subclause); } protected function buildJoinClauseParts(AphrontDatabaseConnection $conn) { $open_statuses = ManiphestTaskStatus::getOpenStatusConstants(); $edge_table = PhabricatorEdgeConfig::TABLE_NAME_EDGE; $task_table = $this->newResultObject()->getTableName(); $parent_type = ManiphestTaskDependedOnByTaskEdgeType::EDGECONST; $subtask_type = ManiphestTaskDependsOnTaskEdgeType::EDGECONST; $joins = array(); if ($this->hasOpenParents !== null) { if ($this->hasOpenParents) { $join_type = qsprintf($conn, 'JOIN'); } else { $join_type = qsprintf($conn, 'LEFT JOIN'); } $joins[] = qsprintf( $conn, '%Q %T e_parent ON e_parent.src = task.phid AND e_parent.type = %d %Q %T parent ON e_parent.dst = parent.phid AND parent.status IN (%Ls)', $join_type, $edge_table, $parent_type, $join_type, $task_table, $open_statuses); } if ($this->hasOpenSubtasks !== null) { if ($this->hasOpenSubtasks) { $join_type = qsprintf($conn, 'JOIN'); } else { $join_type = qsprintf($conn, 'LEFT JOIN'); } $joins[] = qsprintf( $conn, '%Q %T e_subtask ON e_subtask.src = task.phid AND e_subtask.type = %d %Q %T subtask ON e_subtask.dst = subtask.phid AND subtask.status IN (%Ls)', $join_type, $edge_table, $subtask_type, $join_type, $task_table, $open_statuses); } if ($this->subscriberPHIDs !== null) { $joins[] = qsprintf( $conn, 'JOIN %T e_ccs ON e_ccs.src = task.phid '. 'AND e_ccs.type = %s '. 'AND e_ccs.dst in (%Ls)', PhabricatorEdgeConfig::TABLE_NAME_EDGE, PhabricatorObjectHasSubscriberEdgeType::EDGECONST, $this->subscriberPHIDs); } switch ($this->groupBy) { case self::GROUP_PROJECT: $ignore_group_phids = $this->getIgnoreGroupedProjectPHIDs(); if ($ignore_group_phids) { $joins[] = qsprintf( $conn, 'LEFT JOIN %T projectGroup ON task.phid = projectGroup.src AND projectGroup.type = %d AND projectGroup.dst NOT IN (%Ls)', $edge_table, PhabricatorProjectObjectHasProjectEdgeType::EDGECONST, $ignore_group_phids); } else { $joins[] = qsprintf( $conn, 'LEFT JOIN %T projectGroup ON task.phid = projectGroup.src AND projectGroup.type = %d', $edge_table, PhabricatorProjectObjectHasProjectEdgeType::EDGECONST); } $joins[] = qsprintf( $conn, 'LEFT JOIN %T projectGroupName ON projectGroup.dst = projectGroupName.indexedObjectPHID', id(new ManiphestNameIndex())->getTableName()); break; } if ($this->parentTaskIDs !== null) { $joins[] = qsprintf( $conn, 'JOIN %T e_has_parent ON e_has_parent.src = task.phid AND e_has_parent.type = %d JOIN %T has_parent ON e_has_parent.dst = has_parent.phid AND has_parent.id IN (%Ld)', $edge_table, $parent_type, $task_table, $this->parentTaskIDs); } if ($this->subtaskIDs !== null) { $joins[] = qsprintf( $conn, 'JOIN %T e_has_subtask ON e_has_subtask.src = task.phid AND e_has_subtask.type = %d JOIN %T has_subtask ON e_has_subtask.dst = has_subtask.phid AND has_subtask.id IN (%Ld)', $edge_table, $subtask_type, $task_table, $this->subtaskIDs); } $joins[] = parent::buildJoinClauseParts($conn); return $joins; } protected function buildGroupClause(AphrontDatabaseConnection $conn) { $joined_multiple_rows = ($this->hasOpenParents !== null) || ($this->hasOpenSubtasks !== null) || ($this->parentTaskIDs !== null) || ($this->subtaskIDs !== null) || $this->shouldGroupQueryResultRows(); $joined_project_name = ($this->groupBy == self::GROUP_PROJECT); // If we're joining multiple rows, we need to group the results by the // task IDs. if ($joined_multiple_rows) { if ($joined_project_name) { return qsprintf($conn, 'GROUP BY task.phid, projectGroup.dst'); } else { return qsprintf($conn, 'GROUP BY task.phid'); } } return qsprintf($conn, ''); } protected function buildHavingClauseParts(AphrontDatabaseConnection $conn) { $having = parent::buildHavingClauseParts($conn); if ($this->hasOpenParents !== null) { if (!$this->hasOpenParents) { $having[] = qsprintf( $conn, 'COUNT(parent.phid) = 0'); } } if ($this->hasOpenSubtasks !== null) { if (!$this->hasOpenSubtasks) { $having[] = qsprintf( $conn, 'COUNT(subtask.phid) = 0'); } } return $having; } /** * Return project PHIDs which we should ignore when grouping tasks by * project. For example, if a user issues a query like: * * Tasks tagged with all projects: Frontend, Bugs * * ...then we don't show "Frontend" or "Bugs" groups in the result set, since * they're meaningless as all results are in both groups. * * Similarly, for queries like: * * Tasks tagged with any projects: Public Relations * * ...we ignore the single project, as every result is in that project. (In * the case that there are several "any" projects, we do not ignore them.) * * @return list Project PHIDs which should be ignored in query * construction. */ private function getIgnoreGroupedProjectPHIDs() { // Maybe we should also exclude the "OPERATOR_NOT" PHIDs? It won't // impact the results, but we might end up with a better query plan. // Investigate this on real data? This is likely very rare. $edge_types = array( PhabricatorProjectObjectHasProjectEdgeType::EDGECONST, ); $phids = array(); $phids[] = $this->getEdgeLogicValues( $edge_types, array( PhabricatorQueryConstraint::OPERATOR_AND, )); $any = $this->getEdgeLogicValues( $edge_types, array( PhabricatorQueryConstraint::OPERATOR_OR, )); if (count($any) == 1) { $phids[] = $any; } return array_mergev($phids); } - protected function getResultCursor($result) { - $id = $result->getID(); - - if ($this->groupBy == self::GROUP_PROJECT) { - return rtrim($id.'.'.$result->getGroupByProjectPHID(), '.'); - } - - return $id; - } - public function getBuiltinOrders() { $orders = array( 'priority' => array( 'vector' => array('priority', 'id'), 'name' => pht('Priority'), 'aliases' => array(self::ORDER_PRIORITY), ), 'updated' => array( 'vector' => array('updated', 'id'), 'name' => pht('Date Updated (Latest First)'), 'aliases' => array(self::ORDER_MODIFIED), ), 'outdated' => array( 'vector' => array('-updated', '-id'), 'name' => pht('Date Updated (Oldest First)'), ), 'closed' => array( 'vector' => array('closed', 'id'), 'name' => pht('Date Closed (Latest First)'), ), 'title' => array( 'vector' => array('title', 'id'), 'name' => pht('Title'), 'aliases' => array(self::ORDER_TITLE), ), ) + parent::getBuiltinOrders(); // Alias the "newest" builtin to the historical key for it. $orders['newest']['aliases'][] = self::ORDER_CREATED; $orders = array_select_keys( $orders, array( 'priority', 'updated', 'outdated', 'newest', 'oldest', 'closed', 'title', )) + $orders; return $orders; } public function getOrderableColumns() { return parent::getOrderableColumns() + array( 'priority' => array( 'table' => 'task', 'column' => 'priority', 'type' => 'int', ), 'owner' => array( 'table' => 'task', 'column' => 'ownerOrdering', 'null' => 'head', 'reverse' => true, 'type' => 'string', ), 'status' => array( 'table' => 'task', 'column' => 'status', 'type' => 'string', 'reverse' => true, ), 'project' => array( 'table' => 'projectGroupName', 'column' => 'indexedObjectName', 'type' => 'string', 'null' => 'head', 'reverse' => true, ), 'title' => array( 'table' => 'task', 'column' => 'title', 'type' => 'string', 'reverse' => true, ), 'updated' => array( 'table' => 'task', 'column' => 'dateModified', 'type' => 'int', ), 'closed' => array( 'table' => 'task', 'column' => 'closedEpoch', 'type' => 'int', 'null' => 'tail', ), ); } - protected function getPagingValueMap($cursor, array $keys) { - $cursor_parts = explode('.', $cursor, 2); - $task_id = $cursor_parts[0]; - $group_id = idx($cursor_parts, 1); + protected function newPagingMapFromCursorObject( + PhabricatorQueryCursor $cursor, + array $keys) { - $task = $this->loadCursorObject($task_id); + $task = $cursor->getObject(); $map = array( - 'id' => $task->getID(), - 'priority' => $task->getPriority(), + 'id' => (int)$task->getID(), + 'priority' => (int)$task->getPriority(), 'owner' => $task->getOwnerOrdering(), 'status' => $task->getStatus(), 'title' => $task->getTitle(), - 'updated' => $task->getDateModified(), + 'updated' => (int)$task->getDateModified(), 'closed' => $task->getClosedEpoch(), ); - foreach ($keys as $key) { - switch ($key) { - case 'project': - $value = null; - if ($group_id) { - $paging_projects = id(new PhabricatorProjectQuery()) - ->setViewer($this->getViewer()) - ->withPHIDs(array($group_id)) - ->execute(); - if ($paging_projects) { - $value = head($paging_projects)->getName(); - } - } - $map[$key] = $value; - break; + if (isset($keys['project'])) { + $value = null; + + $group_phid = $task->getGroupByProjectPHID(); + if ($group_phid) { + $paging_projects = id(new PhabricatorProjectQuery()) + ->setViewer($this->getViewer()) + ->withPHIDs(array($group_phid)) + ->execute(); + if ($paging_projects) { + $value = head($paging_projects)->getName(); + } } + + $map['project'] = $value; } foreach ($keys as $key) { if ($this->isCustomFieldOrderKey($key)) { $map += $this->getPagingValueMapForCustomFields($task); break; } } return $map; } + protected function newExternalCursorStringForResult($object) { + $id = $object->getID(); + + if ($this->groupBy == self::GROUP_PROJECT) { + return rtrim($id.'.'.$object->getGroupByProjectPHID(), '.'); + } + + return $id; + } + + protected function newInternalCursorFromExternalCursor($cursor) { + list($task_id, $group_phid) = $this->parseCursor($cursor); + + $cursor_object = parent::newInternalCursorFromExternalCursor($cursor); + + if ($group_phid !== null) { + $project = id(new PhabricatorProjectQuery()) + ->setViewer($this->getViewer()) + ->withPHIDs(array($group_phid)) + ->execute(); + + if (!$project) { + $this->throwCursorException( + pht( + 'Group PHID ("%s") component of cursor ("%s") is not valid.', + $group_phid, + $cursor)); + } + + $cursor_object->getObject()->attachGroupByProjectPHID($group_phid); + } + + return $cursor_object; + } + + protected function applyExternalCursorConstraintsToQuery( + PhabricatorCursorPagedPolicyAwareQuery $subquery, + $cursor) { + list($task_id, $group_phid) = $this->parseCursor($cursor); + + $subquery->withIDs(array($task_id)); + + if ($group_phid) { + $subquery->setGroupBy(self::GROUP_PROJECT); + + // The subquery needs to return exactly one result. If a task is in + // several projects, the query may naturally return several results. + // Specify that we want only the particular instance of the task in + // the specified project. + $subquery->withSpecificGroupByProjectPHID($group_phid); + } + } + + + private function parseCursor($cursor) { + // Split a "123.PHID-PROJ-abcd" cursor into a "Task ID" part and a + // "Project PHID" part. + + $parts = explode('.', $cursor, 2); + + if (count($parts) < 2) { + $parts[] = null; + } + + if (!strlen($parts[1])) { + $parts[1] = null; + } + + return $parts; + } + protected function getPrimaryTableAlias() { return 'task'; } public function getQueryApplicationClass() { return 'PhabricatorManiphestApplication'; } } diff --git a/src/applications/phriction/query/PhrictionDocumentQuery.php b/src/applications/phriction/query/PhrictionDocumentQuery.php index 5f508ad804..f05d67c4f3 100644 --- a/src/applications/phriction/query/PhrictionDocumentQuery.php +++ b/src/applications/phriction/query/PhrictionDocumentQuery.php @@ -1,394 +1,398 @@ ids = $ids; return $this; } public function withPHIDs(array $phids) { $this->phids = $phids; return $this; } public function withSlugs(array $slugs) { $this->slugs = $slugs; return $this; } public function withDepths(array $depths) { $this->depths = $depths; return $this; } public function withSlugPrefix($slug_prefix) { $this->slugPrefix = $slug_prefix; return $this; } public function withStatuses(array $statuses) { $this->statuses = $statuses; return $this; } public function withParentPaths(array $paths) { $this->parentPaths = $paths; return $this; } public function withAncestorPaths(array $paths) { $this->ancestorPaths = $paths; return $this; } public function needContent($need_content) { $this->needContent = $need_content; return $this; } protected function loadPage() { return $this->loadStandardPage($this->newResultObject()); } public function newResultObject() { return new PhrictionDocument(); } protected function willFilterPage(array $documents) { if ($documents) { $ancestor_slugs = array(); foreach ($documents as $key => $document) { $document_slug = $document->getSlug(); foreach (PhabricatorSlug::getAncestry($document_slug) as $ancestor) { $ancestor_slugs[$ancestor][] = $key; } } if ($ancestor_slugs) { $table = new PhrictionDocument(); $conn_r = $table->establishConnection('r'); $ancestors = queryfx_all( $conn_r, 'SELECT * FROM %T WHERE slug IN (%Ls)', $document->getTableName(), array_keys($ancestor_slugs)); $ancestors = $table->loadAllFromArray($ancestors); $ancestors = mpull($ancestors, null, 'getSlug'); foreach ($ancestor_slugs as $ancestor_slug => $document_keys) { $ancestor = idx($ancestors, $ancestor_slug); foreach ($document_keys as $document_key) { $documents[$document_key]->attachAncestor( $ancestor_slug, $ancestor); } } } } // To view a Phriction document, you must also be able to view all of the // ancestor documents. Filter out documents which have ancestors that are // not visible. $document_map = array(); foreach ($documents as $document) { $document_map[$document->getSlug()] = $document; foreach ($document->getAncestors() as $key => $ancestor) { if ($ancestor) { $document_map[$key] = $ancestor; } } } $filtered_map = $this->applyPolicyFilter( $document_map, array(PhabricatorPolicyCapability::CAN_VIEW)); // Filter all of the documents where a parent is not visible. foreach ($documents as $document_key => $document) { // If the document itself is not visible, filter it. if (!isset($filtered_map[$document->getSlug()])) { $this->didRejectResult($documents[$document_key]); unset($documents[$document_key]); continue; } // If an ancestor exists but is not visible, filter the document. foreach ($document->getAncestors() as $ancestor_key => $ancestor) { if (!$ancestor) { continue; } if (!isset($filtered_map[$ancestor_key])) { $this->didRejectResult($documents[$document_key]); unset($documents[$document_key]); break; } } } if (!$documents) { return $documents; } if ($this->needContent) { $contents = id(new PhrictionContentQuery()) ->setViewer($this->getViewer()) ->setParentQuery($this) ->withPHIDs(mpull($documents, 'getContentPHID')) ->execute(); $contents = mpull($contents, null, 'getPHID'); foreach ($documents as $key => $document) { $content_phid = $document->getContentPHID(); if (empty($contents[$content_phid])) { unset($documents[$key]); continue; } $document->attachContent($contents[$content_phid]); } } return $documents; } + protected function buildSelectClauseParts(AphrontDatabaseConnection $conn) { + $select = parent::buildSelectClauseParts($conn); + + if ($this->shouldJoinContentTable()) { + $select[] = qsprintf($conn, 'c.title'); + } + + return $select; + } + protected function buildJoinClauseParts(AphrontDatabaseConnection $conn) { $joins = parent::buildJoinClauseParts($conn); - if ($this->getOrderVector()->containsKey('updated')) { + if ($this->shouldJoinContentTable()) { $content_dao = new PhrictionContent(); $joins[] = qsprintf( $conn, 'JOIN %T c ON d.contentPHID = c.phid', $content_dao->getTableName()); } return $joins; } + private function shouldJoinContentTable() { + return $this->getOrderVector()->containsKey('updated'); + } + protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { $where = parent::buildWhereClauseParts($conn); if ($this->ids !== null) { $where[] = qsprintf( $conn, 'd.id IN (%Ld)', $this->ids); } if ($this->phids !== null) { $where[] = qsprintf( $conn, 'd.phid IN (%Ls)', $this->phids); } if ($this->slugs !== null) { $where[] = qsprintf( $conn, 'd.slug IN (%Ls)', $this->slugs); } if ($this->statuses !== null) { $where[] = qsprintf( $conn, 'd.status IN (%Ls)', $this->statuses); } if ($this->slugPrefix !== null) { $where[] = qsprintf( $conn, 'd.slug LIKE %>', $this->slugPrefix); } if ($this->depths !== null) { $where[] = qsprintf( $conn, 'd.depth IN (%Ld)', $this->depths); } if ($this->parentPaths !== null || $this->ancestorPaths !== null) { $sets = array( array( 'paths' => $this->parentPaths, 'parents' => true, ), array( 'paths' => $this->ancestorPaths, 'parents' => false, ), ); $paths = array(); foreach ($sets as $set) { $set_paths = $set['paths']; if ($set_paths === null) { continue; } if (!$set_paths) { throw new PhabricatorEmptyQueryException( pht('No parent/ancestor paths specified.')); } $is_parents = $set['parents']; foreach ($set_paths as $path) { $path_normal = PhabricatorSlug::normalize($path); if ($path !== $path_normal) { throw new Exception( pht( 'Document path "%s" is not a valid path. The normalized '. 'form of this path is "%s".', $path, $path_normal)); } $depth = PhabricatorSlug::getDepth($path_normal); if ($is_parents) { $min_depth = $depth + 1; $max_depth = $depth + 1; } else { $min_depth = $depth + 1; $max_depth = null; } $paths[] = array( $path_normal, $min_depth, $max_depth, ); } } $path_clauses = array(); foreach ($paths as $path) { $parts = array(); list($prefix, $min, $max) = $path; // If we're getting children or ancestors of the root document, they // aren't actually stored with the leading "/" in the database, so // just skip this part of the clause. if ($prefix !== '/') { $parts[] = qsprintf( $conn, 'd.slug LIKE %>', $prefix); } if ($min !== null) { $parts[] = qsprintf( $conn, 'd.depth >= %d', $min); } if ($max !== null) { $parts[] = qsprintf( $conn, 'd.depth <= %d', $max); } if ($parts) { $path_clauses[] = qsprintf($conn, '%LA', $parts); } } if ($path_clauses) { $where[] = qsprintf($conn, '%LO', $path_clauses); } } return $where; } public function getBuiltinOrders() { return parent::getBuiltinOrders() + array( self::ORDER_HIERARCHY => array( 'vector' => array('depth', 'title', 'updated', 'id'), 'name' => pht('Hierarchy'), ), ); } public function getOrderableColumns() { return parent::getOrderableColumns() + array( 'depth' => array( 'table' => 'd', 'column' => 'depth', 'reverse' => true, 'type' => 'int', ), 'title' => array( 'table' => 'c', 'column' => 'title', 'reverse' => true, 'type' => 'string', ), 'updated' => array( 'table' => 'd', 'column' => 'editedEpoch', 'type' => 'int', 'unique' => false, ), ); } - protected function getPagingValueMap($cursor, array $keys) { - $document = $this->loadCursorObject($cursor); + protected function newPagingMapFromCursorObject( + PhabricatorQueryCursor $cursor, + array $keys) { + + $document = $cursor->getObject(); $map = array( - 'id' => $document->getID(), + 'id' => (int)$document->getID(), 'depth' => $document->getDepth(), - 'updated' => $document->getEditedEpoch(), + 'updated' => (int)$document->getEditedEpoch(), ); - foreach ($keys as $key) { - switch ($key) { - case 'title': - $map[$key] = $document->getContent()->getTitle(); - break; - } + if (isset($keys['title'])) { + $map['title'] = $cursor->getRawRowProperty('c.title'); } return $map; } - protected function willExecuteCursorQuery( - PhabricatorCursorPagedPolicyAwareQuery $query) { - $vector = $this->getOrderVector(); - - if ($vector->containsKey('title')) { - $query->needContent(true); - } - } - protected function getPrimaryTableAlias() { return 'd'; } public function getQueryApplicationClass() { return 'PhabricatorPhrictionApplication'; } } diff --git a/src/applications/repository/query/PhabricatorRepositoryQuery.php b/src/applications/repository/query/PhabricatorRepositoryQuery.php index 56c62cc8fb..e960cf888b 100644 --- a/src/applications/repository/query/PhabricatorRepositoryQuery.php +++ b/src/applications/repository/query/PhabricatorRepositoryQuery.php @@ -1,726 +1,701 @@ ids = $ids; return $this; } public function withPHIDs(array $phids) { $this->phids = $phids; return $this; } public function withCallsigns(array $callsigns) { $this->callsigns = $callsigns; return $this; } public function withIdentifiers(array $identifiers) { $identifiers = array_fuse($identifiers); $ids = array(); $callsigns = array(); $phids = array(); $monograms = array(); $slugs = array(); foreach ($identifiers as $identifier) { if (ctype_digit((string)$identifier)) { $ids[$identifier] = $identifier; continue; } if (preg_match('/^(r[A-Z]+|R[1-9]\d*)\z/', $identifier)) { $monograms[$identifier] = $identifier; continue; } $repository_type = PhabricatorRepositoryRepositoryPHIDType::TYPECONST; if (phid_get_type($identifier) === $repository_type) { $phids[$identifier] = $identifier; continue; } if (preg_match('/^[A-Z]+\z/', $identifier)) { $callsigns[$identifier] = $identifier; continue; } $slugs[$identifier] = $identifier; } $this->numericIdentifiers = $ids; $this->callsignIdentifiers = $callsigns; $this->phidIdentifiers = $phids; $this->monogramIdentifiers = $monograms; $this->slugIdentifiers = $slugs; return $this; } public function withStatus($status) { $this->status = $status; return $this; } public function withHosted($hosted) { $this->hosted = $hosted; return $this; } public function withTypes(array $types) { $this->types = $types; return $this; } public function withUUIDs(array $uuids) { $this->uuids = $uuids; return $this; } public function withURIs(array $uris) { $this->uris = $uris; return $this; } public function withDatasourceQuery($query) { $this->datasourceQuery = $query; return $this; } public function withSlugs(array $slugs) { $this->slugs = $slugs; return $this; } public function withAlmanacServicePHIDs(array $phids) { $this->almanacServicePHIDs = $phids; return $this; } public function needCommitCounts($need_counts) { $this->needCommitCounts = $need_counts; return $this; } public function needMostRecentCommits($need_commits) { $this->needMostRecentCommits = $need_commits; return $this; } public function needProjectPHIDs($need_phids) { $this->needProjectPHIDs = $need_phids; return $this; } public function needURIs($need_uris) { $this->needURIs = $need_uris; return $this; } public function needProfileImage($need) { $this->needProfileImage = $need; return $this; } public function getBuiltinOrders() { return array( 'committed' => array( 'vector' => array('committed', 'id'), 'name' => pht('Most Recent Commit'), ), 'name' => array( 'vector' => array('name', 'id'), 'name' => pht('Name'), ), 'callsign' => array( 'vector' => array('callsign'), 'name' => pht('Callsign'), ), 'size' => array( 'vector' => array('size', 'id'), 'name' => pht('Size'), ), ) + parent::getBuiltinOrders(); } public function getIdentifierMap() { if ($this->identifierMap === null) { throw new PhutilInvalidStateException('execute'); } return $this->identifierMap; } protected function willExecute() { $this->identifierMap = array(); } public function newResultObject() { return new PhabricatorRepository(); } protected function loadPage() { $table = $this->newResultObject(); $data = $this->loadStandardPageRows($table); $repositories = $table->loadAllFromArray($data); if ($this->needCommitCounts) { $sizes = ipull($data, 'size', 'id'); foreach ($repositories as $id => $repository) { $repository->attachCommitCount(nonempty($sizes[$id], 0)); } } if ($this->needMostRecentCommits) { $commit_ids = ipull($data, 'lastCommitID', 'id'); $commit_ids = array_filter($commit_ids); if ($commit_ids) { $commits = id(new DiffusionCommitQuery()) ->setViewer($this->getViewer()) ->withIDs($commit_ids) ->execute(); } else { $commits = array(); } foreach ($repositories as $id => $repository) { $commit = null; if (idx($commit_ids, $id)) { $commit = idx($commits, $commit_ids[$id]); } $repository->attachMostRecentCommit($commit); } } return $repositories; } protected function willFilterPage(array $repositories) { assert_instances_of($repositories, 'PhabricatorRepository'); // TODO: Denormalize repository status into the PhabricatorRepository // table so we can do this filtering in the database. foreach ($repositories as $key => $repo) { $status = $this->status; switch ($status) { case self::STATUS_OPEN: if (!$repo->isTracked()) { unset($repositories[$key]); } break; case self::STATUS_CLOSED: if ($repo->isTracked()) { unset($repositories[$key]); } break; case self::STATUS_ALL: break; default: throw new Exception("Unknown status '{$status}'!"); } // TODO: This should also be denormalized. $hosted = $this->hosted; switch ($hosted) { case self::HOSTED_PHABRICATOR: if (!$repo->isHosted()) { unset($repositories[$key]); } break; case self::HOSTED_REMOTE: if ($repo->isHosted()) { unset($repositories[$key]); } break; case self::HOSTED_ALL: break; default: throw new Exception(pht("Unknown hosted failed '%s'!", $hosted)); } } // Build the identifierMap if ($this->numericIdentifiers) { foreach ($this->numericIdentifiers as $id) { if (isset($repositories[$id])) { $this->identifierMap[$id] = $repositories[$id]; } } } if ($this->callsignIdentifiers) { $repository_callsigns = mpull($repositories, null, 'getCallsign'); foreach ($this->callsignIdentifiers as $callsign) { if (isset($repository_callsigns[$callsign])) { $this->identifierMap[$callsign] = $repository_callsigns[$callsign]; } } } if ($this->phidIdentifiers) { $repository_phids = mpull($repositories, null, 'getPHID'); foreach ($this->phidIdentifiers as $phid) { if (isset($repository_phids[$phid])) { $this->identifierMap[$phid] = $repository_phids[$phid]; } } } if ($this->monogramIdentifiers) { $monogram_map = array(); foreach ($repositories as $repository) { foreach ($repository->getAllMonograms() as $monogram) { $monogram_map[$monogram] = $repository; } } foreach ($this->monogramIdentifiers as $monogram) { if (isset($monogram_map[$monogram])) { $this->identifierMap[$monogram] = $monogram_map[$monogram]; } } } if ($this->slugIdentifiers) { $slug_map = array(); foreach ($repositories as $repository) { $slug = $repository->getRepositorySlug(); if ($slug === null) { continue; } $normal = phutil_utf8_strtolower($slug); $slug_map[$normal] = $repository; } foreach ($this->slugIdentifiers as $slug) { $normal = phutil_utf8_strtolower($slug); if (isset($slug_map[$normal])) { $this->identifierMap[$slug] = $slug_map[$normal]; } } } return $repositories; } protected function didFilterPage(array $repositories) { if ($this->needProjectPHIDs) { $type_project = PhabricatorProjectObjectHasProjectEdgeType::EDGECONST; $edge_query = id(new PhabricatorEdgeQuery()) ->withSourcePHIDs(mpull($repositories, 'getPHID')) ->withEdgeTypes(array($type_project)); $edge_query->execute(); foreach ($repositories as $repository) { $project_phids = $edge_query->getDestinationPHIDs( array( $repository->getPHID(), )); $repository->attachProjectPHIDs($project_phids); } } $viewer = $this->getViewer(); if ($this->needURIs) { $uris = id(new PhabricatorRepositoryURIQuery()) ->setViewer($viewer) ->withRepositories($repositories) ->execute(); $uri_groups = mgroup($uris, 'getRepositoryPHID'); foreach ($repositories as $repository) { $repository_uris = idx($uri_groups, $repository->getPHID(), array()); $repository->attachURIs($repository_uris); } } if ($this->needProfileImage) { $default = null; $file_phids = mpull($repositories, 'getProfileImagePHID'); $file_phids = array_filter($file_phids); if ($file_phids) { $files = id(new PhabricatorFileQuery()) ->setParentQuery($this) ->setViewer($this->getViewer()) ->withPHIDs($file_phids) ->execute(); $files = mpull($files, null, 'getPHID'); } else { $files = array(); } foreach ($repositories as $repository) { $file = idx($files, $repository->getProfileImagePHID()); if (!$file) { if (!$default) { $default = PhabricatorFile::loadBuiltin( $this->getViewer(), 'repo/code.png'); } $file = $default; } $repository->attachProfileImageFile($file); } } return $repositories; } protected function getPrimaryTableAlias() { return 'r'; } public function getOrderableColumns() { return parent::getOrderableColumns() + array( 'committed' => array( 'table' => 's', 'column' => 'epoch', 'type' => 'int', 'null' => 'tail', ), 'callsign' => array( 'table' => 'r', 'column' => 'callsign', 'type' => 'string', 'unique' => true, 'reverse' => true, 'null' => 'tail', ), 'name' => array( 'table' => 'r', 'column' => 'name', 'type' => 'string', 'reverse' => true, ), 'size' => array( 'table' => 's', 'column' => 'size', 'type' => 'int', 'null' => 'tail', ), ); } - protected function willExecuteCursorQuery( - PhabricatorCursorPagedPolicyAwareQuery $query) { - $vector = $this->getOrderVector(); + protected function newPagingMapFromCursorObject( + PhabricatorQueryCursor $cursor, + array $keys) { - if ($vector->containsKey('committed')) { - $query->needMostRecentCommits(true); - } - - if ($vector->containsKey('size')) { - $query->needCommitCounts(true); - } - } - - protected function getPagingValueMap($cursor, array $keys) { - $repository = $this->loadCursorObject($cursor); + $repository = $cursor->getObject(); $map = array( - 'id' => $repository->getID(), + 'id' => (int)$repository->getID(), 'callsign' => $repository->getCallsign(), 'name' => $repository->getName(), ); - foreach ($keys as $key) { - switch ($key) { - case 'committed': - $commit = $repository->getMostRecentCommit(); - if ($commit) { - $map[$key] = $commit->getEpoch(); - } else { - $map[$key] = null; - } - break; - case 'size': - $count = $repository->getCommitCount(); - if ($count) { - $map[$key] = $count; - } else { - $map[$key] = null; - } - break; - } + if (isset($keys['committed'])) { + $map['committed'] = $cursor->getRawRowProperty('epoch'); + } + + if (isset($keys['size'])) { + $map['size'] = $cursor->getRawRowProperty('size'); } return $map; } protected function buildSelectClauseParts(AphrontDatabaseConnection $conn) { $parts = parent::buildSelectClauseParts($conn); - $parts[] = qsprintf($conn, 'r.*'); - if ($this->shouldJoinSummaryTable()) { $parts[] = qsprintf($conn, 's.*'); } return $parts; } protected function buildJoinClauseParts(AphrontDatabaseConnection $conn) { $joins = parent::buildJoinClauseParts($conn); if ($this->shouldJoinSummaryTable()) { $joins[] = qsprintf( $conn, 'LEFT JOIN %T s ON r.id = s.repositoryID', PhabricatorRepository::TABLE_SUMMARY); } if ($this->shouldJoinURITable()) { $joins[] = qsprintf( $conn, 'LEFT JOIN %R uri ON r.phid = uri.repositoryPHID', new PhabricatorRepositoryURIIndex()); } return $joins; } protected function shouldGroupQueryResultRows() { if ($this->shouldJoinURITable()) { return true; } return parent::shouldGroupQueryResultRows(); } private function shouldJoinURITable() { return ($this->uris !== null); } private function shouldJoinSummaryTable() { if ($this->needCommitCounts) { return true; } if ($this->needMostRecentCommits) { return true; } $vector = $this->getOrderVector(); if ($vector->containsKey('committed')) { return true; } if ($vector->containsKey('size')) { return true; } return false; } protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { $where = parent::buildWhereClauseParts($conn); if ($this->ids !== null) { $where[] = qsprintf( $conn, 'r.id IN (%Ld)', $this->ids); } if ($this->phids !== null) { $where[] = qsprintf( $conn, 'r.phid IN (%Ls)', $this->phids); } if ($this->callsigns !== null) { $where[] = qsprintf( $conn, 'r.callsign IN (%Ls)', $this->callsigns); } if ($this->numericIdentifiers || $this->callsignIdentifiers || $this->phidIdentifiers || $this->monogramIdentifiers || $this->slugIdentifiers) { $identifier_clause = array(); if ($this->numericIdentifiers) { $identifier_clause[] = qsprintf( $conn, 'r.id IN (%Ld)', $this->numericIdentifiers); } if ($this->callsignIdentifiers) { $identifier_clause[] = qsprintf( $conn, 'r.callsign IN (%Ls)', $this->callsignIdentifiers); } if ($this->phidIdentifiers) { $identifier_clause[] = qsprintf( $conn, 'r.phid IN (%Ls)', $this->phidIdentifiers); } if ($this->monogramIdentifiers) { $monogram_callsigns = array(); $monogram_ids = array(); foreach ($this->monogramIdentifiers as $identifier) { if ($identifier[0] == 'r') { $monogram_callsigns[] = substr($identifier, 1); } else { $monogram_ids[] = substr($identifier, 1); } } if ($monogram_ids) { $identifier_clause[] = qsprintf( $conn, 'r.id IN (%Ld)', $monogram_ids); } if ($monogram_callsigns) { $identifier_clause[] = qsprintf( $conn, 'r.callsign IN (%Ls)', $monogram_callsigns); } } if ($this->slugIdentifiers) { $identifier_clause[] = qsprintf( $conn, 'r.repositorySlug IN (%Ls)', $this->slugIdentifiers); } $where[] = qsprintf($conn, '%LO', $identifier_clause); } if ($this->types) { $where[] = qsprintf( $conn, 'r.versionControlSystem IN (%Ls)', $this->types); } if ($this->uuids) { $where[] = qsprintf( $conn, 'r.uuid IN (%Ls)', $this->uuids); } if (strlen($this->datasourceQuery)) { // This handles having "rP" match callsigns starting with "P...". $query = trim($this->datasourceQuery); if (preg_match('/^r/', $query)) { $callsign = substr($query, 1); } else { $callsign = $query; } $where[] = qsprintf( $conn, 'r.name LIKE %> OR r.callsign LIKE %> OR r.repositorySlug LIKE %>', $query, $callsign, $query); } if ($this->slugs !== null) { $where[] = qsprintf( $conn, 'r.repositorySlug IN (%Ls)', $this->slugs); } if ($this->uris !== null) { $try_uris = $this->getNormalizedURIs(); $try_uris = array_fuse($try_uris); $where[] = qsprintf( $conn, 'uri.repositoryURI IN (%Ls)', $try_uris); } if ($this->almanacServicePHIDs !== null) { $where[] = qsprintf( $conn, 'r.almanacServicePHID IN (%Ls)', $this->almanacServicePHIDs); } return $where; } public function getQueryApplicationClass() { return 'PhabricatorDiffusionApplication'; } private function getNormalizedURIs() { $normalized_uris = array(); // Since we don't know which type of repository this URI is in the general // case, just generate all the normalizations. We could refine this in some // cases: if the query specifies VCS types, or the URI is a git-style URI // or an `svn+ssh` URI, we could deduce how to normalize it. However, this // would be more complicated and it's not clear if it matters in practice. $types = PhabricatorRepositoryURINormalizer::getAllURITypes(); foreach ($this->uris as $uri) { foreach ($types as $type) { $normalized_uri = new PhabricatorRepositoryURINormalizer($type, $uri); $normalized_uris[] = $normalized_uri->getNormalizedURI(); } } return array_unique($normalized_uris); } } diff --git a/src/infrastructure/edges/conduit/PhabricatorEdgeObject.php b/src/infrastructure/edges/conduit/PhabricatorEdgeObject.php index ec5f84e59f..3183185631 100644 --- a/src/infrastructure/edges/conduit/PhabricatorEdgeObject.php +++ b/src/infrastructure/edges/conduit/PhabricatorEdgeObject.php @@ -1,63 +1,76 @@ id = $row['id']; - $edge->src = $row['src']; - $edge->dst = $row['dst']; - $edge->type = $row['type']; + $edge->id = idx($row, 'id'); + $edge->src = idx($row, 'src'); + $edge->dst = idx($row, 'dst'); + $edge->type = idx($row, 'type'); + $edge->dateCreated = idx($row, 'dateCreated'); + $edge->sequence = idx($row, 'seq'); return $edge; } public function getID() { return $this->id; } public function getSourcePHID() { return $this->src; } public function getEdgeType() { return $this->type; } public function getDestinationPHID() { return $this->dst; } public function getPHID() { return null; } + public function getDateCreated() { + return $this->dateCreated; + } + + public function getSequence() { + return $this->sequence; + } + + /* -( PhabricatorPolicyInterface )----------------------------------------- */ public function getCapabilities() { return array( PhabricatorPolicyCapability::CAN_VIEW, ); } public function getPolicy($capability) { switch ($capability) { case PhabricatorPolicyCapability::CAN_VIEW: return PhabricatorPolicies::getMostOpenPolicy(); } } public function hasAutomaticCapability($capability, PhabricatorUser $viewer) { return false; } } diff --git a/src/infrastructure/edges/query/PhabricatorEdgeObjectQuery.php b/src/infrastructure/edges/query/PhabricatorEdgeObjectQuery.php index 1385215569..048a2a9fb4 100644 --- a/src/infrastructure/edges/query/PhabricatorEdgeObjectQuery.php +++ b/src/infrastructure/edges/query/PhabricatorEdgeObjectQuery.php @@ -1,163 +1,182 @@ sourcePHIDs = $source_phids; return $this; } public function withEdgeTypes(array $types) { $this->edgeTypes = $types; return $this; } public function withDestinationPHIDs(array $destination_phids) { $this->destinationPHIDs = $destination_phids; return $this; } protected function willExecute() { $source_phids = $this->sourcePHIDs; if (!$source_phids) { throw new Exception( pht( 'Edge object query must be executed with a nonempty list of '. 'source PHIDs.')); } $phid_item = null; $phid_type = null; foreach ($source_phids as $phid) { $this_type = phid_get_type($phid); if ($this_type == PhabricatorPHIDConstants::PHID_TYPE_UNKNOWN) { throw new Exception( pht( 'Source PHID "%s" in edge object query has unknown PHID type.', $phid)); } if ($phid_type === null) { $phid_type = $this_type; $phid_item = $phid; continue; } if ($phid_type !== $this_type) { throw new Exception( pht( 'Two source PHIDs ("%s" and "%s") have different PHID types '. '("%s" and "%s"). All PHIDs must be of the same type to execute '. 'an edge object query.', $phid_item, $phid, $phid_type, $this_type)); } } $this->sourcePHIDType = $phid_type; } protected function loadPage() { $type = $this->sourcePHIDType; $conn = PhabricatorEdgeConfig::establishConnection($type, 'r'); $table = PhabricatorEdgeConfig::TABLE_NAME_EDGE; $rows = $this->loadStandardPageRowsWithConnection($conn, $table); $result = array(); foreach ($rows as $row) { $result[] = PhabricatorEdgeObject::newFromRow($row); } return $result; } - protected function buildSelectClauseParts(AphrontDatabaseConnection $conn) { - $parts = parent::buildSelectClauseParts($conn); - - // TODO: This is hacky, because we don't have real IDs on this table. - $parts[] = qsprintf( - $conn, - 'CONCAT(dateCreated, %s, seq) AS id', - '_'); - - return $parts; - } - protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { $parts = parent::buildWhereClauseParts($conn); $parts[] = qsprintf( $conn, 'src IN (%Ls)', $this->sourcePHIDs); $parts[] = qsprintf( $conn, 'type IN (%Ls)', $this->edgeTypes); if ($this->destinationPHIDs !== null) { $parts[] = qsprintf( $conn, 'dst IN (%Ls)', $this->destinationPHIDs); } return $parts; } public function getQueryApplicationClass() { return null; } protected function getPrimaryTableAlias() { return 'edge'; } public function getOrderableColumns() { return array( 'dateCreated' => array( 'table' => 'edge', 'column' => 'dateCreated', 'type' => 'int', ), 'sequence' => array( 'table' => 'edge', 'column' => 'seq', 'type' => 'int', // TODO: This is not actually unique, but we're just doing our best // here. 'unique' => true, ), ); } protected function getDefaultOrderVector() { return array('dateCreated', 'sequence'); } - protected function getPagingValueMap($cursor, array $keys) { - $parts = explode('_', $cursor); + protected function newInternalCursorFromExternalCursor($cursor) { + list($epoch, $sequence) = $this->parseCursor($cursor); + + // Instead of actually loading an edge, we're just making a fake edge + // with the properties the cursor describes. + + $edge_object = PhabricatorEdgeObject::newFromRow( + array( + 'dateCreated' => $epoch, + 'seq' => $sequence, + )); + return id(new PhabricatorQueryCursor()) + ->setObject($edge_object); + } + + protected function newPagingMapFromPartialObject($object) { return array( - 'dateCreated' => $parts[0], - 'sequence' => $parts[1], + 'dateCreated' => $object->getDateCreated(), + 'sequence' => $object->getSequence(), ); } + protected function newExternalCursorStringForResult($object) { + return sprintf( + '%d_%d', + $object->getDateCreated(), + $object->getSequence()); + } + + private function parseCursor($cursor) { + if (!preg_match('/^\d+_\d+\z/', $cursor)) { + $this->throwCursorException( + pht( + 'Expected edge cursor in the form "0123_6789", got "%s".', + $cursor)); + } + + return explode('_', $cursor); + } + } diff --git a/src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php b/src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php index def502e11f..2fe70b1ceb 100644 --- a/src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php +++ b/src/infrastructure/query/policy/PhabricatorCursorPagedPolicyAwareQuery.php @@ -1,3063 +1,3040 @@ getID(); } protected function newInternalCursorFromExternalCursor($cursor) { - return $this->newInternalCursorObjectFromID($cursor); - } - - protected function newPagingMapFromCursorObject( - PhabricatorQueryCursor $cursor, - array $keys) { - - $object = $cursor->getObject(); - - return $this->newPagingMapFromPartialObject($object); - } - - protected function newPagingMapFromPartialObject($object) { - return array( - 'id' => (int)$object->getID(), - ); - } - - final protected function newInternalCursorObjectFromID($id) { $viewer = $this->getViewer(); $query = newv(get_class($this), array()); $query ->setParentQuery($this) - ->setViewer($viewer) - ->withIDs(array((int)$id)); + ->setViewer($viewer); // We're copying our order vector to the subquery so that the subquery // knows it should generate any supplemental information required by the // ordering. // For example, Phriction documents may be ordered by title, but the title // isn't a column in the "document" table: the query must JOIN the // "content" table to perform the ordering. Passing the ordering to the // subquery tells it that we need it to do that JOIN and attach relevant // paging information to the internal cursor object. // We only expect to load a single result, so the actual result order does // not matter. We only want the internal cursor for that result to look // like a cursor this parent query would generate. $query->setOrderVector($this->getOrderVector()); + $this->applyExternalCursorConstraintsToQuery($query, $cursor); + // We're executing the subquery normally to make sure the viewer can // actually see the object, and that it's a completely valid object which // passes all filtering and policy checks. You aren't allowed to use an // object you can't see as a cursor, since this can leak information. $result = $query->executeOne(); if (!$result) { - // TODO: Raise a more tailored exception here and make the UI a little - // prettier? - throw new Exception( + $this->throwCursorException( pht( 'Cursor "%s" does not identify a valid object in query "%s".', - $id, + $cursor, get_class($this))); } // Now that we made sure the viewer can actually see the object the // external cursor identifies, return the internal cursor the query // generated as a side effect while loading the object. return $query->getInternalCursorObject(); } + final protected function throwCursorException($message) { + // TODO: Raise a more tailored exception here and make the UI a little + // prettier? + throw new Exception($message); + } + + protected function applyExternalCursorConstraintsToQuery( + PhabricatorCursorPagedPolicyAwareQuery $subquery, + $cursor) { + $subquery->withIDs(array($cursor)); + } + + protected function newPagingMapFromCursorObject( + PhabricatorQueryCursor $cursor, + array $keys) { + + $object = $cursor->getObject(); + + return $this->newPagingMapFromPartialObject($object); + } + + protected function newPagingMapFromPartialObject($object) { + return array( + 'id' => (int)$object->getID(), + ); + } + + final private function getExternalCursorStringForResult($object) { $cursor = $this->newExternalCursorStringForResult($object); if (!is_string($cursor)) { throw new Exception( pht( 'Expected "newExternalCursorStringForResult()" in class "%s" to '. 'return a string, but got "%s".', get_class($this), phutil_describe_type($cursor))); } return $cursor; } final private function getExternalCursorString() { return $this->externalCursorString; } final private function setExternalCursorString($external_cursor) { $this->externalCursorString = $external_cursor; return $this; } final private function getIsQueryOrderReversed() { return $this->isQueryOrderReversed; } final private function setIsQueryOrderReversed($is_reversed) { $this->isQueryOrderReversed = $is_reversed; return $this; } final private function getInternalCursorObject() { return $this->internalCursorObject; } final private function setInternalCursorObject( PhabricatorQueryCursor $cursor) { $this->internalCursorObject = $cursor; return $this; } final private function getInternalCursorFromExternalCursor( $cursor_string) { $cursor_object = $this->newInternalCursorFromExternalCursor($cursor_string); if (!($cursor_object instanceof PhabricatorQueryCursor)) { throw new Exception( pht( 'Expected "newInternalCursorFromExternalCursor()" to return an '. 'object of class "PhabricatorQueryCursor", but got "%s" (in '. 'class "%s").', phutil_describe_type($cursor_object), get_class($this))); } return $cursor_object; } final private function getPagingMapFromCursorObject( PhabricatorQueryCursor $cursor, array $keys) { $map = $this->newPagingMapFromCursorObject($cursor, $keys); if (!is_array($map)) { throw new Exception( pht( 'Expected "newPagingMapFromCursorObject()" to return a map of '. 'paging values, but got "%s" (in class "%s").', phutil_describe_type($map), get_class($this))); } foreach ($keys as $key) { if (!array_key_exists($key, $map)) { throw new Exception( pht( 'Map returned by "newPagingMapFromCursorObject()" in class "%s" '. 'omits required key "%s".', get_class($this), $key)); } } return $map; } final protected function nextPage(array $page) { if (!$page) { return; } $cursor = id(new PhabricatorQueryCursor()) ->setObject(last($page)); + if ($this->rawCursorRow) { + $cursor->setRawRow($this->rawCursorRow); + } + $this->setInternalCursorObject($cursor); } final public function getFerretMetadata() { if (!$this->supportsFerretEngine()) { throw new Exception( pht( 'Unable to retrieve Ferret engine metadata, this class ("%s") does '. 'not support the Ferret engine.', get_class($this))); } return $this->ferretMetadata; } protected function loadStandardPage(PhabricatorLiskDAO $table) { $rows = $this->loadStandardPageRows($table); return $table->loadAllFromArray($rows); } protected function loadStandardPageRows(PhabricatorLiskDAO $table) { $conn = $table->establishConnection('r'); return $this->loadStandardPageRowsWithConnection( $conn, $table->getTableName()); } protected function loadStandardPageRowsWithConnection( AphrontDatabaseConnection $conn, $table_name) { $query = $this->buildStandardPageQuery($conn, $table_name); $rows = queryfx_all($conn, '%Q', $query); $rows = $this->didLoadRawRows($rows); return $rows; } protected function buildStandardPageQuery( AphrontDatabaseConnection $conn, $table_name) { $table_alias = $this->getPrimaryTableAlias(); if ($table_alias === null) { $table_alias = qsprintf($conn, ''); } else { $table_alias = qsprintf($conn, '%T', $table_alias); } return qsprintf( $conn, '%Q FROM %T %Q %Q %Q %Q %Q %Q %Q', $this->buildSelectClause($conn), $table_name, $table_alias, $this->buildJoinClause($conn), $this->buildWhereClause($conn), $this->buildGroupClause($conn), $this->buildHavingClause($conn), $this->buildOrderClause($conn), $this->buildLimitClause($conn)); } protected function didLoadRawRows(array $rows) { if ($this->ferretEngine) { foreach ($rows as $row) { $phid = $row['phid']; $metadata = id(new PhabricatorFerretMetadata()) ->setPHID($phid) ->setEngine($this->ferretEngine) ->setRelevance(idx($row, '_ft_rank')); $this->ferretMetadata[$phid] = $metadata; unset($row['_ft_rank']); } } - return $rows; - } + $this->rawCursorRow = last($rows); - /** - * Get the viewer for making cursor paging queries. - * - * NOTE: You should ONLY use this viewer to load cursor objects while - * building paging queries. - * - * Cursor paging can happen in two ways. First, the user can request a page - * like `/stuff/?after=33`, which explicitly causes paging. Otherwise, we - * can fall back to implicit paging if we filter some results out of a - * result list because the user can't see them and need to go fetch some more - * results to generate a large enough result list. - * - * In the first case, want to use the viewer's policies to load the object. - * This prevents an attacker from figuring out information about an object - * they can't see by executing queries like `/stuff/?after=33&order=name`, - * which would otherwise give them a hint about the name of the object. - * Generally, if a user can't see an object, they can't use it to page. - * - * In the second case, we need to load the object whether the user can see - * it or not, because we need to examine new results. For example, if a user - * loads `/stuff/` and we run a query for the first 100 items that they can - * see, but the first 100 rows in the database aren't visible, we need to - * be able to issue a query for the next 100 results. If we can't load the - * cursor object, we'll fail or issue the same query over and over again. - * So, generally, internal paging must bypass policy controls. - * - * This method returns the appropriate viewer, based on the context in which - * the paging is occurring. - * - * @return PhabricatorUser Viewer for executing paging queries. - */ - final protected function getPagingViewer() { - if ($this->internalPaging) { - return PhabricatorUser::getOmnipotentUser(); - } else { - return $this->getViewer(); - } + return $rows; } final protected function buildLimitClause(AphrontDatabaseConnection $conn) { if ($this->shouldLimitResults()) { $limit = $this->getRawResultLimit(); if ($limit) { return qsprintf($conn, 'LIMIT %d', $limit); } } return qsprintf($conn, ''); } protected function shouldLimitResults() { return true; } final protected function didLoadResults(array $results) { if ($this->getIsQueryOrderReversed()) { $results = array_reverse($results, $preserve_keys = true); } return $results; } final public function executeWithCursorPager(AphrontCursorPagerView $pager) { $limit = $pager->getPageSize(); $this->setLimit($limit + 1); if (strlen($pager->getAfterID())) { $this->setExternalCursorString($pager->getAfterID()); } else if ($pager->getBeforeID()) { $this->setExternalCursorString($pager->getBeforeID()); $this->setIsQueryOrderReversed(true); } $results = $this->execute(); $count = count($results); $sliced_results = $pager->sliceResults($results); if ($sliced_results) { // If we have results, generate external-facing cursors from the visible // results. This stops us from leaking any internal details about objects // which we loaded but which were not visible to the viewer. if ($pager->getBeforeID() || ($count > $limit)) { $last_object = last($sliced_results); $cursor = $this->getExternalCursorStringForResult($last_object); $pager->setNextPageID($cursor); } if ($pager->getAfterID() || ($pager->getBeforeID() && ($count > $limit))) { $head_object = head($sliced_results); $cursor = $this->getExternalCursorStringForResult($head_object); $pager->setPrevPageID($cursor); } } return $sliced_results; } /** * Return the alias this query uses to identify the primary table. * * Some automatic query constructions may need to be qualified with a table * alias if the query performs joins which make column names ambiguous. If * this is the case, return the alias for the primary table the query * uses; generally the object table which has `id` and `phid` columns. * * @return string Alias for the primary table. */ protected function getPrimaryTableAlias() { return null; } public function newResultObject() { return null; } /* -( Building Query Clauses )--------------------------------------------- */ /** * @task clauses */ protected function buildSelectClause(AphrontDatabaseConnection $conn) { $parts = $this->buildSelectClauseParts($conn); return $this->formatSelectClause($conn, $parts); } /** * @task clauses */ protected function buildSelectClauseParts(AphrontDatabaseConnection $conn) { $select = array(); $alias = $this->getPrimaryTableAlias(); if ($alias) { $select[] = qsprintf($conn, '%T.*', $alias); } else { $select[] = qsprintf($conn, '*'); } $select[] = $this->buildEdgeLogicSelectClause($conn); $select[] = $this->buildFerretSelectClause($conn); return $select; } /** * @task clauses */ protected function buildJoinClause(AphrontDatabaseConnection $conn) { $joins = $this->buildJoinClauseParts($conn); return $this->formatJoinClause($conn, $joins); } /** * @task clauses */ protected function buildJoinClauseParts(AphrontDatabaseConnection $conn) { $joins = array(); $joins[] = $this->buildEdgeLogicJoinClause($conn); $joins[] = $this->buildApplicationSearchJoinClause($conn); $joins[] = $this->buildNgramsJoinClause($conn); $joins[] = $this->buildFerretJoinClause($conn); return $joins; } /** * @task clauses */ protected function buildWhereClause(AphrontDatabaseConnection $conn) { $where = $this->buildWhereClauseParts($conn); return $this->formatWhereClause($conn, $where); } /** * @task clauses */ protected function buildWhereClauseParts(AphrontDatabaseConnection $conn) { $where = array(); $where[] = $this->buildPagingClause($conn); $where[] = $this->buildEdgeLogicWhereClause($conn); $where[] = $this->buildSpacesWhereClause($conn); $where[] = $this->buildNgramsWhereClause($conn); $where[] = $this->buildFerretWhereClause($conn); $where[] = $this->buildApplicationSearchWhereClause($conn); return $where; } /** * @task clauses */ protected function buildHavingClause(AphrontDatabaseConnection $conn) { $having = $this->buildHavingClauseParts($conn); return $this->formatHavingClause($conn, $having); } /** * @task clauses */ protected function buildHavingClauseParts(AphrontDatabaseConnection $conn) { $having = array(); $having[] = $this->buildEdgeLogicHavingClause($conn); return $having; } /** * @task clauses */ protected function buildGroupClause(AphrontDatabaseConnection $conn) { if (!$this->shouldGroupQueryResultRows()) { return qsprintf($conn, ''); } return qsprintf( $conn, 'GROUP BY %Q', $this->getApplicationSearchObjectPHIDColumn($conn)); } /** * @task clauses */ protected function shouldGroupQueryResultRows() { if ($this->shouldGroupEdgeLogicResultRows()) { return true; } if ($this->getApplicationSearchMayJoinMultipleRows()) { return true; } if ($this->shouldGroupNgramResultRows()) { return true; } if ($this->shouldGroupFerretResultRows()) { return true; } return false; } /* -( Paging )------------------------------------------------------------- */ /** * @task paging */ protected function buildPagingClause(AphrontDatabaseConnection $conn) { $orderable = $this->getOrderableColumns(); $vector = $this->getOrderVector(); // If we don't have a cursor object yet, it means we're trying to load // the first result page. We may need to build a cursor object from the // external string, or we may not need a paging clause yet. $cursor_object = $this->getInternalCursorObject(); if (!$cursor_object) { $external_cursor = $this->getExternalCursorString(); if ($external_cursor !== null) { $cursor_object = $this->getInternalCursorFromExternalCursor( $external_cursor); } } // If we still don't have a cursor object, this is the first result page // and we aren't paging it. We don't need to build a paging clause. if (!$cursor_object) { return qsprintf($conn, ''); } $reversed = $this->getIsQueryOrderReversed(); $keys = array(); foreach ($vector as $order) { $keys[] = $order->getOrderKey(); } + $keys = array_fuse($keys); $value_map = $this->getPagingMapFromCursorObject( $cursor_object, $keys); $columns = array(); foreach ($vector as $order) { $key = $order->getOrderKey(); $column = $orderable[$key]; $column['value'] = $value_map[$key]; // If the vector component is reversed, we need to reverse whatever the // order of the column is. if ($order->getIsReversed()) { $column['reverse'] = !idx($column, 'reverse', false); } $columns[] = $column; } return $this->buildPagingClauseFromMultipleColumns( $conn, $columns, array( 'reversed' => $reversed, )); } /** * Simplifies the task of constructing a paging clause across multiple * columns. In the general case, this looks like: * * A > a OR (A = a AND B > b) OR (A = a AND B = b AND C > c) * * To build a clause, specify the name, type, and value of each column * to include: * * $this->buildPagingClauseFromMultipleColumns( * $conn_r, * array( * array( * 'table' => 't', * 'column' => 'title', * 'type' => 'string', * 'value' => $cursor->getTitle(), * 'reverse' => true, * ), * array( * 'table' => 't', * 'column' => 'id', * 'type' => 'int', * 'value' => $cursor->getID(), * ), * ), * array( * 'reversed' => $is_reversed, * )); * * This method will then return a composable clause for inclusion in WHERE. * * @param AphrontDatabaseConnection Connection query will execute on. * @param list Column description dictionaries. * @param map Additional construction options. * @return string Query clause. * @task paging */ final protected function buildPagingClauseFromMultipleColumns( AphrontDatabaseConnection $conn, array $columns, array $options) { foreach ($columns as $column) { PhutilTypeSpec::checkMap( $column, array( 'table' => 'optional string|null', 'column' => 'string', 'value' => 'wild', 'type' => 'string', 'reverse' => 'optional bool', 'unique' => 'optional bool', 'null' => 'optional string|null', )); } PhutilTypeSpec::checkMap( $options, array( 'reversed' => 'optional bool', )); $is_query_reversed = idx($options, 'reversed', false); $clauses = array(); $accumulated = array(); $last_key = last_key($columns); foreach ($columns as $key => $column) { $type = $column['type']; $null = idx($column, 'null'); if ($column['value'] === null) { if ($null) { $value = null; } else { throw new Exception( pht( 'Column "%s" has null value, but does not specify a null '. 'behavior.', $key)); } } else { switch ($type) { case 'int': $value = qsprintf($conn, '%d', $column['value']); break; case 'float': $value = qsprintf($conn, '%f', $column['value']); break; case 'string': $value = qsprintf($conn, '%s', $column['value']); break; default: throw new Exception( pht( 'Column "%s" has unknown column type "%s".', $column['column'], $type)); } } $is_column_reversed = idx($column, 'reverse', false); $reverse = ($is_query_reversed xor $is_column_reversed); $clause = $accumulated; $table_name = idx($column, 'table'); $column_name = $column['column']; if ($table_name !== null) { $field = qsprintf($conn, '%T.%T', $table_name, $column_name); } else { $field = qsprintf($conn, '%T', $column_name); } $parts = array(); if ($null) { $can_page_if_null = ($null === 'head'); $can_page_if_nonnull = ($null === 'tail'); if ($reverse) { $can_page_if_null = !$can_page_if_null; $can_page_if_nonnull = !$can_page_if_nonnull; } $subclause = null; if ($can_page_if_null && $value === null) { $parts[] = qsprintf( $conn, '(%Q IS NOT NULL)', $field); } else if ($can_page_if_nonnull && $value !== null) { $parts[] = qsprintf( $conn, '(%Q IS NULL)', $field); } } if ($value !== null) { $parts[] = qsprintf( $conn, '%Q %Q %Q', $field, $reverse ? qsprintf($conn, '>') : qsprintf($conn, '<'), $value); } if ($parts) { $clause[] = qsprintf($conn, '%LO', $parts); } if ($clause) { $clauses[] = qsprintf($conn, '%LA', $clause); } if ($value === null) { $accumulated[] = qsprintf( $conn, '%Q IS NULL', $field); } else { $accumulated[] = qsprintf( $conn, '%Q = %Q', $field, $value); } } if ($clauses) { return qsprintf($conn, '%LO', $clauses); } return qsprintf($conn, ''); } /* -( Result Ordering )---------------------------------------------------- */ /** * Select a result ordering. * * This is a high-level method which selects an ordering from a predefined * list of builtin orders, as provided by @{method:getBuiltinOrders}. These * options are user-facing and not exhaustive, but are generally convenient * and meaningful. * * You can also use @{method:setOrderVector} to specify a low-level ordering * across individual orderable columns. This offers greater control but is * also more involved. * * @param string Key of a builtin order supported by this query. * @return this * @task order */ public function setOrder($order) { $aliases = $this->getBuiltinOrderAliasMap(); if (empty($aliases[$order])) { throw new Exception( pht( 'Query "%s" does not support a builtin order "%s". Supported orders '. 'are: %s.', get_class($this), $order, implode(', ', array_keys($aliases)))); } $this->builtinOrder = $aliases[$order]; $this->orderVector = null; return $this; } /** * Set a grouping order to apply before primary result ordering. * * This allows you to preface the query order vector with additional orders, * so you can effect "group by" queries while still respecting "order by". * * This is a high-level method which works alongside @{method:setOrder}. For * lower-level control over order vectors, use @{method:setOrderVector}. * * @param PhabricatorQueryOrderVector|list List of order keys. * @return this * @task order */ public function setGroupVector($vector) { $this->groupVector = $vector; $this->orderVector = null; return $this; } /** * Get builtin orders for this class. * * In application UIs, we want to be able to present users with a small * selection of meaningful order options (like "Order by Title") rather than * an exhaustive set of column ordering options. * * Meaningful user-facing orders are often really orders across multiple * columns: for example, a "title" ordering is usually implemented as a * "title, id" ordering under the hood. * * Builtin orders provide a mapping from convenient, understandable * user-facing orders to implementations. * * A builtin order should provide these keys: * * - `vector` (`list`): The actual order vector to use. * - `name` (`string`): Human-readable order name. * * @return map Map from builtin order keys to specification. * @task order */ public function getBuiltinOrders() { $orders = array( 'newest' => array( 'vector' => array('id'), 'name' => pht('Creation (Newest First)'), 'aliases' => array('created'), ), 'oldest' => array( 'vector' => array('-id'), 'name' => pht('Creation (Oldest First)'), ), ); $object = $this->newResultObject(); if ($object instanceof PhabricatorCustomFieldInterface) { $list = PhabricatorCustomField::getObjectFields( $object, PhabricatorCustomField::ROLE_APPLICATIONSEARCH); foreach ($list->getFields() as $field) { $index = $field->buildOrderIndex(); if (!$index) { continue; } $legacy_key = 'custom:'.$field->getFieldKey(); $modern_key = $field->getModernFieldKey(); $orders[$modern_key] = array( 'vector' => array($modern_key, 'id'), 'name' => $field->getFieldName(), 'aliases' => array($legacy_key), ); $orders['-'.$modern_key] = array( 'vector' => array('-'.$modern_key, '-id'), 'name' => pht('%s (Reversed)', $field->getFieldName()), ); } } if ($this->supportsFerretEngine()) { $orders['relevance'] = array( 'vector' => array('rank', 'fulltext-modified', 'id'), 'name' => pht('Relevance'), ); } return $orders; } public function getBuiltinOrderAliasMap() { $orders = $this->getBuiltinOrders(); $map = array(); foreach ($orders as $key => $order) { $keys = array(); $keys[] = $key; foreach (idx($order, 'aliases', array()) as $alias) { $keys[] = $alias; } foreach ($keys as $alias) { if (isset($map[$alias])) { throw new Exception( pht( 'Two builtin orders ("%s" and "%s") define the same key or '. 'alias ("%s"). Each order alias and key must be unique and '. 'identify a single order.', $key, $map[$alias], $alias)); } $map[$alias] = $key; } } return $map; } /** * Set a low-level column ordering. * * This is a low-level method which offers granular control over column * ordering. In most cases, applications can more easily use * @{method:setOrder} to choose a high-level builtin order. * * To set an order vector, specify a list of order keys as provided by * @{method:getOrderableColumns}. * * @param PhabricatorQueryOrderVector|list List of order keys. * @return this * @task order */ public function setOrderVector($vector) { $vector = PhabricatorQueryOrderVector::newFromVector($vector); $orderable = $this->getOrderableColumns(); // Make sure that all the components identify valid columns. $unique = array(); foreach ($vector as $order) { $key = $order->getOrderKey(); if (empty($orderable[$key])) { $valid = implode(', ', array_keys($orderable)); throw new Exception( pht( 'This query ("%s") does not support sorting by order key "%s". '. 'Supported orders are: %s.', get_class($this), $key, $valid)); } $unique[$key] = idx($orderable[$key], 'unique', false); } // Make sure that the last column is unique so that this is a strong // ordering which can be used for paging. $last = last($unique); if ($last !== true) { throw new Exception( pht( 'Order vector "%s" is invalid: the last column in an order must '. 'be a column with unique values, but "%s" is not unique.', $vector->getAsString(), last_key($unique))); } // Make sure that other columns are not unique; an ordering like "id, name" // does not make sense because only "id" can ever have an effect. array_pop($unique); foreach ($unique as $key => $is_unique) { if ($is_unique) { throw new Exception( pht( 'Order vector "%s" is invalid: only the last column in an order '. 'may be unique, but "%s" is a unique column and not the last '. 'column in the order.', $vector->getAsString(), $key)); } } $this->orderVector = $vector; return $this; } /** * Get the effective order vector. * * @return PhabricatorQueryOrderVector Effective vector. * @task order */ protected function getOrderVector() { if (!$this->orderVector) { if ($this->builtinOrder !== null) { $builtin_order = idx($this->getBuiltinOrders(), $this->builtinOrder); $vector = $builtin_order['vector']; } else { $vector = $this->getDefaultOrderVector(); } if ($this->groupVector) { $group = PhabricatorQueryOrderVector::newFromVector($this->groupVector); $group->appendVector($vector); $vector = $group; } $vector = PhabricatorQueryOrderVector::newFromVector($vector); // We call setOrderVector() here to apply checks to the default vector. // This catches any errors in the implementation. $this->setOrderVector($vector); } return $this->orderVector; } /** * @task order */ protected function getDefaultOrderVector() { return array('id'); } /** * @task order */ public function getOrderableColumns() { $cache = PhabricatorCaches::getRequestCache(); $class = get_class($this); $cache_key = 'query.orderablecolumns.'.$class; $columns = $cache->getKey($cache_key); if ($columns !== null) { return $columns; } $columns = array( 'id' => array( 'table' => $this->getPrimaryTableAlias(), 'column' => 'id', 'reverse' => false, 'type' => 'int', 'unique' => true, ), ); $object = $this->newResultObject(); if ($object instanceof PhabricatorCustomFieldInterface) { $list = PhabricatorCustomField::getObjectFields( $object, PhabricatorCustomField::ROLE_APPLICATIONSEARCH); foreach ($list->getFields() as $field) { $index = $field->buildOrderIndex(); if (!$index) { continue; } $digest = $field->getFieldIndex(); $key = $field->getModernFieldKey(); $columns[$key] = array( 'table' => 'appsearch_order_'.$digest, 'column' => 'indexValue', 'type' => $index->getIndexValueType(), 'null' => 'tail', 'customfield' => true, 'customfield.index.table' => $index->getTableName(), 'customfield.index.key' => $digest, ); } } if ($this->supportsFerretEngine()) { $columns['rank'] = array( 'table' => null, 'column' => '_ft_rank', 'type' => 'int', ); $columns['fulltext-created'] = array( 'table' => 'ft_doc', 'column' => 'epochCreated', 'type' => 'int', ); $columns['fulltext-modified'] = array( 'table' => 'ft_doc', 'column' => 'epochModified', 'type' => 'int', ); } $cache->setKey($cache_key, $columns); return $columns; } /** * @task order */ final protected function buildOrderClause( AphrontDatabaseConnection $conn, $for_union = false) { $orderable = $this->getOrderableColumns(); $vector = $this->getOrderVector(); $parts = array(); foreach ($vector as $order) { $part = $orderable[$order->getOrderKey()]; if ($order->getIsReversed()) { $part['reverse'] = !idx($part, 'reverse', false); } $parts[] = $part; } return $this->formatOrderClause($conn, $parts, $for_union); } /** * @task order */ protected function formatOrderClause( AphrontDatabaseConnection $conn, array $parts, $for_union = false) { $is_query_reversed = $this->getIsQueryOrderReversed(); $sql = array(); foreach ($parts as $key => $part) { $is_column_reversed = !empty($part['reverse']); $descending = true; if ($is_query_reversed) { $descending = !$descending; } if ($is_column_reversed) { $descending = !$descending; } $table = idx($part, 'table'); // When we're building an ORDER BY clause for a sequence of UNION // statements, we can't refer to tables from the subqueries. if ($for_union) { $table = null; } $column = $part['column']; if ($table !== null) { $field = qsprintf($conn, '%T.%T', $table, $column); } else { $field = qsprintf($conn, '%T', $column); } $null = idx($part, 'null'); if ($null) { switch ($null) { case 'head': $null_field = qsprintf($conn, '(%Q IS NULL)', $field); break; case 'tail': $null_field = qsprintf($conn, '(%Q IS NOT NULL)', $field); break; default: throw new Exception( pht( 'NULL value "%s" is invalid. Valid values are "head" and '. '"tail".', $null)); } if ($descending) { $sql[] = qsprintf($conn, '%Q DESC', $null_field); } else { $sql[] = qsprintf($conn, '%Q ASC', $null_field); } } if ($descending) { $sql[] = qsprintf($conn, '%Q DESC', $field); } else { $sql[] = qsprintf($conn, '%Q ASC', $field); } } return qsprintf($conn, 'ORDER BY %LQ', $sql); } /* -( Application Search )------------------------------------------------- */ /** * Constrain the query with an ApplicationSearch index, requiring field values * contain at least one of the values in a set. * * This constraint can build the most common types of queries, like: * * - Find users with shirt sizes "X" or "XL". * - Find shoes with size "13". * * @param PhabricatorCustomFieldIndexStorage Table where the index is stored. * @param string|list One or more values to filter by. * @return this * @task appsearch */ public function withApplicationSearchContainsConstraint( PhabricatorCustomFieldIndexStorage $index, $value) { $values = (array)$value; $data_values = array(); $constraint_values = array(); foreach ($values as $value) { if ($value instanceof PhabricatorQueryConstraint) { $constraint_values[] = $value; } else { $data_values[] = $value; } } $alias = 'appsearch_'.count($this->applicationSearchConstraints); $this->applicationSearchConstraints[] = array( 'type' => $index->getIndexValueType(), 'cond' => '=', 'table' => $index->getTableName(), 'index' => $index->getIndexKey(), 'alias' => $alias, 'value' => $values, 'data' => $data_values, 'constraints' => $constraint_values, ); return $this; } /** * Constrain the query with an ApplicationSearch index, requiring values * exist in a given range. * * This constraint is useful for expressing date ranges: * * - Find events between July 1st and July 7th. * * The ends of the range are inclusive, so a `$min` of `3` and a `$max` of * `5` will match fields with values `3`, `4`, or `5`. Providing `null` for * either end of the range will leave that end of the constraint open. * * @param PhabricatorCustomFieldIndexStorage Table where the index is stored. * @param int|null Minimum permissible value, inclusive. * @param int|null Maximum permissible value, inclusive. * @return this * @task appsearch */ public function withApplicationSearchRangeConstraint( PhabricatorCustomFieldIndexStorage $index, $min, $max) { $index_type = $index->getIndexValueType(); if ($index_type != 'int') { throw new Exception( pht( 'Attempting to apply a range constraint to a field with index type '. '"%s", expected type "%s".', $index_type, 'int')); } $alias = 'appsearch_'.count($this->applicationSearchConstraints); $this->applicationSearchConstraints[] = array( 'type' => $index->getIndexValueType(), 'cond' => 'range', 'table' => $index->getTableName(), 'index' => $index->getIndexKey(), 'alias' => $alias, 'value' => array($min, $max), 'data' => null, 'constraints' => null, ); return $this; } /** * Get the name of the query's primary object PHID column, for constructing * JOIN clauses. Normally (and by default) this is just `"phid"`, but it may * be something more exotic. * * See @{method:getPrimaryTableAlias} if the column needs to be qualified with * a table alias. * * @param AphrontDatabaseConnection Connection executing queries. * @return PhutilQueryString Column name. * @task appsearch */ protected function getApplicationSearchObjectPHIDColumn( AphrontDatabaseConnection $conn) { if ($this->getPrimaryTableAlias()) { return qsprintf($conn, '%T.phid', $this->getPrimaryTableAlias()); } else { return qsprintf($conn, 'phid'); } } /** * Determine if the JOINs built by ApplicationSearch might cause each primary * object to return multiple result rows. Generally, this means the query * needs an extra GROUP BY clause. * * @return bool True if the query may return multiple rows for each object. * @task appsearch */ protected function getApplicationSearchMayJoinMultipleRows() { foreach ($this->applicationSearchConstraints as $constraint) { $type = $constraint['type']; $value = $constraint['value']; $cond = $constraint['cond']; switch ($cond) { case '=': switch ($type) { case 'string': case 'int': if (count($value) > 1) { return true; } break; default: throw new Exception(pht('Unknown index type "%s"!', $type)); } break; case 'range': // NOTE: It's possible to write a custom field where multiple rows // match a range constraint, but we don't currently ship any in the // upstream and I can't immediately come up with cases where this // would make sense. break; default: throw new Exception(pht('Unknown constraint condition "%s"!', $cond)); } } return false; } /** * Construct a GROUP BY clause appropriate for ApplicationSearch constraints. * * @param AphrontDatabaseConnection Connection executing the query. * @return string Group clause. * @task appsearch */ protected function buildApplicationSearchGroupClause( AphrontDatabaseConnection $conn) { if ($this->getApplicationSearchMayJoinMultipleRows()) { return qsprintf( $conn, 'GROUP BY %Q', $this->getApplicationSearchObjectPHIDColumn($conn)); } else { return qsprintf($conn, ''); } } /** * Construct a JOIN clause appropriate for applying ApplicationSearch * constraints. * * @param AphrontDatabaseConnection Connection executing the query. * @return string Join clause. * @task appsearch */ protected function buildApplicationSearchJoinClause( AphrontDatabaseConnection $conn) { $joins = array(); foreach ($this->applicationSearchConstraints as $key => $constraint) { $table = $constraint['table']; $alias = $constraint['alias']; $index = $constraint['index']; $cond = $constraint['cond']; $phid_column = $this->getApplicationSearchObjectPHIDColumn($conn); switch ($cond) { case '=': // Figure out whether we need to do a LEFT JOIN or not. We need to // LEFT JOIN if we're going to select "IS NULL" rows. $join_type = qsprintf($conn, 'JOIN'); foreach ($constraint['constraints'] as $query_constraint) { $op = $query_constraint->getOperator(); if ($op === PhabricatorQueryConstraint::OPERATOR_NULL) { $join_type = qsprintf($conn, 'LEFT JOIN'); break; } } $joins[] = qsprintf( $conn, '%Q %T %T ON %T.objectPHID = %Q AND %T.indexKey = %s', $join_type, $table, $alias, $alias, $phid_column, $alias, $index); break; case 'range': list($min, $max) = $constraint['value']; if (($min === null) && ($max === null)) { // If there's no actual range constraint, just move on. break; } if ($min === null) { $constraint_clause = qsprintf( $conn, '%T.indexValue <= %d', $alias, $max); } else if ($max === null) { $constraint_clause = qsprintf( $conn, '%T.indexValue >= %d', $alias, $min); } else { $constraint_clause = qsprintf( $conn, '%T.indexValue BETWEEN %d AND %d', $alias, $min, $max); } $joins[] = qsprintf( $conn, 'JOIN %T %T ON %T.objectPHID = %Q AND %T.indexKey = %s AND (%Q)', $table, $alias, $alias, $phid_column, $alias, $index, $constraint_clause); break; default: throw new Exception(pht('Unknown constraint condition "%s"!', $cond)); } } $phid_column = $this->getApplicationSearchObjectPHIDColumn($conn); $orderable = $this->getOrderableColumns(); $vector = $this->getOrderVector(); foreach ($vector as $order) { $spec = $orderable[$order->getOrderKey()]; if (empty($spec['customfield'])) { continue; } $table = $spec['customfield.index.table']; $alias = $spec['table']; $key = $spec['customfield.index.key']; $joins[] = qsprintf( $conn, 'LEFT JOIN %T %T ON %T.objectPHID = %Q AND %T.indexKey = %s', $table, $alias, $alias, $phid_column, $alias, $key); } if ($joins) { return qsprintf($conn, '%LJ', $joins); } else { return qsprintf($conn, ''); } } /** * Construct a WHERE clause appropriate for applying ApplicationSearch * constraints. * * @param AphrontDatabaseConnection Connection executing the query. * @return list Where clause parts. * @task appsearch */ protected function buildApplicationSearchWhereClause( AphrontDatabaseConnection $conn) { $where = array(); foreach ($this->applicationSearchConstraints as $key => $constraint) { $alias = $constraint['alias']; $cond = $constraint['cond']; $type = $constraint['type']; $data_values = $constraint['data']; $constraint_values = $constraint['constraints']; $constraint_parts = array(); switch ($cond) { case '=': if ($data_values) { switch ($type) { case 'string': $constraint_parts[] = qsprintf( $conn, '%T.indexValue IN (%Ls)', $alias, $data_values); break; case 'int': $constraint_parts[] = qsprintf( $conn, '%T.indexValue IN (%Ld)', $alias, $data_values); break; default: throw new Exception(pht('Unknown index type "%s"!', $type)); } } if ($constraint_values) { foreach ($constraint_values as $value) { $op = $value->getOperator(); switch ($op) { case PhabricatorQueryConstraint::OPERATOR_NULL: $constraint_parts[] = qsprintf( $conn, '%T.indexValue IS NULL', $alias); break; case PhabricatorQueryConstraint::OPERATOR_ANY: $constraint_parts[] = qsprintf( $conn, '%T.indexValue IS NOT NULL', $alias); break; default: throw new Exception( pht( 'No support for applying operator "%s" against '. 'index of type "%s".', $op, $type)); } } } if ($constraint_parts) { $where[] = qsprintf($conn, '%LO', $constraint_parts); } break; } } return $where; } /* -( Integration with CustomField )--------------------------------------- */ /** * @task customfield */ protected function getPagingValueMapForCustomFields( PhabricatorCustomFieldInterface $object) { // We have to get the current field values on the cursor object. $fields = PhabricatorCustomField::getObjectFields( $object, PhabricatorCustomField::ROLE_APPLICATIONSEARCH); $fields->setViewer($this->getViewer()); $fields->readFieldsFromStorage($object); $map = array(); foreach ($fields->getFields() as $field) { $map['custom:'.$field->getFieldKey()] = $field->getValueForStorage(); } return $map; } /** * @task customfield */ protected function isCustomFieldOrderKey($key) { $prefix = 'custom:'; return !strncmp($key, $prefix, strlen($prefix)); } /* -( Ferret )------------------------------------------------------------- */ public function supportsFerretEngine() { $object = $this->newResultObject(); return ($object instanceof PhabricatorFerretInterface); } public function withFerretQuery( PhabricatorFerretEngine $engine, PhabricatorSavedQuery $query) { if (!$this->supportsFerretEngine()) { throw new Exception( pht( 'Query ("%s") does not support the Ferret fulltext engine.', get_class($this))); } $this->ferretEngine = $engine; $this->ferretQuery = $query; return $this; } public function getFerretTokens() { if (!$this->supportsFerretEngine()) { throw new Exception( pht( 'Query ("%s") does not support the Ferret fulltext engine.', get_class($this))); } return $this->ferretTokens; } public function withFerretConstraint( PhabricatorFerretEngine $engine, array $fulltext_tokens) { if (!$this->supportsFerretEngine()) { throw new Exception( pht( 'Query ("%s") does not support the Ferret fulltext engine.', get_class($this))); } if ($this->ferretEngine) { throw new Exception( pht( 'Query may not have multiple fulltext constraints.')); } if (!$fulltext_tokens) { return $this; } $this->ferretEngine = $engine; $this->ferretTokens = $fulltext_tokens; $current_function = $engine->getDefaultFunctionKey(); $table_map = array(); $idx = 1; foreach ($this->ferretTokens as $fulltext_token) { $raw_token = $fulltext_token->getToken(); $function = $raw_token->getFunction(); if ($function === null) { $function = $current_function; } $raw_field = $engine->getFieldForFunction($function); if (!isset($table_map[$function])) { $alias = 'ftfield_'.$idx++; $table_map[$function] = array( 'alias' => $alias, 'key' => $raw_field, ); } $current_function = $function; } // Join the title field separately so we can rank results. $table_map['rank'] = array( 'alias' => 'ft_rank', 'key' => PhabricatorSearchDocumentFieldType::FIELD_TITLE, ); $this->ferretTables = $table_map; return $this; } protected function buildFerretSelectClause(AphrontDatabaseConnection $conn) { $select = array(); if (!$this->supportsFerretEngine()) { return $select; } $vector = $this->getOrderVector(); if (!$vector->containsKey('rank')) { // We only need to SELECT the virtual "_ft_rank" column if we're // actually sorting the results by rank. return $select; } if (!$this->ferretEngine) { $select[] = qsprintf($conn, '0 _ft_rank'); return $select; } $engine = $this->ferretEngine; $stemmer = $engine->newStemmer(); $op_sub = PhutilSearchQueryCompiler::OPERATOR_SUBSTRING; $op_not = PhutilSearchQueryCompiler::OPERATOR_NOT; $table_alias = 'ft_rank'; $parts = array(); foreach ($this->ferretTokens as $fulltext_token) { $raw_token = $fulltext_token->getToken(); $value = $raw_token->getValue(); if ($raw_token->getOperator() == $op_not) { // Ignore "not" terms when ranking, since they aren't useful. continue; } if ($raw_token->getOperator() == $op_sub) { $is_substring = true; } else { $is_substring = false; } if ($is_substring) { $parts[] = qsprintf( $conn, 'IF(%T.rawCorpus LIKE %~, 2, 0)', $table_alias, $value); continue; } if ($raw_token->isQuoted()) { $is_quoted = true; $is_stemmed = false; } else { $is_quoted = false; $is_stemmed = true; } $term_constraints = array(); $term_value = $engine->newTermsCorpus($value); $parts[] = qsprintf( $conn, 'IF(%T.termCorpus LIKE %~, 2, 0)', $table_alias, $term_value); if ($is_stemmed) { $stem_value = $stemmer->stemToken($value); $stem_value = $engine->newTermsCorpus($stem_value); $parts[] = qsprintf( $conn, 'IF(%T.normalCorpus LIKE %~, 1, 0)', $table_alias, $stem_value); } } $parts[] = qsprintf($conn, '%d', 0); $sum = array_shift($parts); foreach ($parts as $part) { $sum = qsprintf( $conn, '%Q + %Q', $sum, $part); } $select[] = qsprintf( $conn, '%Q _ft_rank', $sum); return $select; } protected function buildFerretJoinClause(AphrontDatabaseConnection $conn) { if (!$this->ferretEngine) { return array(); } $op_sub = PhutilSearchQueryCompiler::OPERATOR_SUBSTRING; $op_not = PhutilSearchQueryCompiler::OPERATOR_NOT; $engine = $this->ferretEngine; $stemmer = $engine->newStemmer(); $ngram_table = $engine->getNgramsTableName(); $flat = array(); foreach ($this->ferretTokens as $fulltext_token) { $raw_token = $fulltext_token->getToken(); // If this is a negated term like "-pomegranate", don't join the ngram // table since we aren't looking for documents with this term. (We could // LEFT JOIN the table and require a NULL row, but this is probably more // trouble than it's worth.) if ($raw_token->getOperator() == $op_not) { continue; } $value = $raw_token->getValue(); $length = count(phutil_utf8v($value)); if ($raw_token->getOperator() == $op_sub) { $is_substring = true; } else { $is_substring = false; } // If the user specified a substring query for a substring which is // shorter than the ngram length, we can't use the ngram index, so // don't do a join. We'll fall back to just doing LIKE on the full // corpus. if ($is_substring) { if ($length < 3) { continue; } } if ($raw_token->isQuoted()) { $is_stemmed = false; } else { $is_stemmed = true; } if ($is_substring) { $ngrams = $engine->getSubstringNgramsFromString($value); } else { $terms_value = $engine->newTermsCorpus($value); $ngrams = $engine->getTermNgramsFromString($terms_value); // If this is a stemmed term, only look for ngrams present in both the // unstemmed and stemmed variations. if ($is_stemmed) { // Trim the boundary space characters so the stemmer recognizes this // is (or, at least, may be) a normal word and activates. $terms_value = trim($terms_value, ' '); $stem_value = $stemmer->stemToken($terms_value); $stem_ngrams = $engine->getTermNgramsFromString($stem_value); $ngrams = array_intersect($ngrams, $stem_ngrams); } } foreach ($ngrams as $ngram) { $flat[] = array( 'table' => $ngram_table, 'ngram' => $ngram, ); } } // Remove common ngrams, like "the", which occur too frequently in // documents to be useful in constraining the query. The best ngrams // are obscure sequences which occur in very few documents. if ($flat) { $common_ngrams = queryfx_all( $conn, 'SELECT ngram FROM %T WHERE ngram IN (%Ls)', $engine->getCommonNgramsTableName(), ipull($flat, 'ngram')); $common_ngrams = ipull($common_ngrams, 'ngram', 'ngram'); foreach ($flat as $key => $spec) { $ngram = $spec['ngram']; if (isset($common_ngrams[$ngram])) { unset($flat[$key]); continue; } // NOTE: MySQL discards trailing whitespace in CHAR(X) columns. $trim_ngram = rtrim($ngram, ' '); if (isset($common_ngrams[$trim_ngram])) { unset($flat[$key]); continue; } } } // MySQL only allows us to join a maximum of 61 tables per query. Each // ngram is going to cost us a join toward that limit, so if the user // specified a very long query string, just pick 16 of the ngrams // at random. if (count($flat) > 16) { shuffle($flat); $flat = array_slice($flat, 0, 16); } $alias = $this->getPrimaryTableAlias(); if ($alias) { $phid_column = qsprintf($conn, '%T.%T', $alias, 'phid'); } else { $phid_column = qsprintf($conn, '%T', 'phid'); } $document_table = $engine->getDocumentTableName(); $field_table = $engine->getFieldTableName(); $joins = array(); $joins[] = qsprintf( $conn, 'JOIN %T ft_doc ON ft_doc.objectPHID = %Q', $document_table, $phid_column); $idx = 1; foreach ($flat as $spec) { $table = $spec['table']; $ngram = $spec['ngram']; $alias = 'ftngram_'.$idx++; $joins[] = qsprintf( $conn, 'JOIN %T %T ON %T.documentID = ft_doc.id AND %T.ngram = %s', $table, $alias, $alias, $alias, $ngram); } foreach ($this->ferretTables as $table) { $alias = $table['alias']; $joins[] = qsprintf( $conn, 'JOIN %T %T ON ft_doc.id = %T.documentID AND %T.fieldKey = %s', $field_table, $alias, $alias, $alias, $table['key']); } return $joins; } protected function buildFerretWhereClause(AphrontDatabaseConnection $conn) { if (!$this->ferretEngine) { return array(); } $engine = $this->ferretEngine; $stemmer = $engine->newStemmer(); $table_map = $this->ferretTables; $op_sub = PhutilSearchQueryCompiler::OPERATOR_SUBSTRING; $op_not = PhutilSearchQueryCompiler::OPERATOR_NOT; $op_exact = PhutilSearchQueryCompiler::OPERATOR_EXACT; $where = array(); $current_function = 'all'; foreach ($this->ferretTokens as $fulltext_token) { $raw_token = $fulltext_token->getToken(); $value = $raw_token->getValue(); $function = $raw_token->getFunction(); if ($function === null) { $function = $current_function; } $current_function = $function; $table_alias = $table_map[$function]['alias']; $is_not = ($raw_token->getOperator() == $op_not); if ($raw_token->getOperator() == $op_sub) { $is_substring = true; } else { $is_substring = false; } // If we're doing exact search, just test the raw corpus. $is_exact = ($raw_token->getOperator() == $op_exact); if ($is_exact) { if ($is_not) { $where[] = qsprintf( $conn, '(%T.rawCorpus != %s)', $table_alias, $value); } else { $where[] = qsprintf( $conn, '(%T.rawCorpus = %s)', $table_alias, $value); } continue; } // If we're doing substring search, we just match against the raw corpus // and we're done. if ($is_substring) { if ($is_not) { $where[] = qsprintf( $conn, '(%T.rawCorpus NOT LIKE %~)', $table_alias, $value); } else { $where[] = qsprintf( $conn, '(%T.rawCorpus LIKE %~)', $table_alias, $value); } continue; } // Otherwise, we need to match against the term corpus and the normal // corpus, so that searching for "raw" does not find "strawberry". if ($raw_token->isQuoted()) { $is_quoted = true; $is_stemmed = false; } else { $is_quoted = false; $is_stemmed = true; } // Never stem negated queries, since this can exclude results users // did not mean to exclude and generally confuse things. if ($is_not) { $is_stemmed = false; } $term_constraints = array(); $term_value = $engine->newTermsCorpus($value); if ($is_not) { $term_constraints[] = qsprintf( $conn, '(%T.termCorpus NOT LIKE %~)', $table_alias, $term_value); } else { $term_constraints[] = qsprintf( $conn, '(%T.termCorpus LIKE %~)', $table_alias, $term_value); } if ($is_stemmed) { $stem_value = $stemmer->stemToken($value); $stem_value = $engine->newTermsCorpus($stem_value); $term_constraints[] = qsprintf( $conn, '(%T.normalCorpus LIKE %~)', $table_alias, $stem_value); } if ($is_not) { $where[] = qsprintf( $conn, '%LA', $term_constraints); } else if ($is_quoted) { $where[] = qsprintf( $conn, '(%T.rawCorpus LIKE %~ AND %LO)', $table_alias, $value, $term_constraints); } else { $where[] = qsprintf( $conn, '%LO', $term_constraints); } } if ($this->ferretQuery) { $query = $this->ferretQuery; $author_phids = $query->getParameter('authorPHIDs'); if ($author_phids) { $where[] = qsprintf( $conn, 'ft_doc.authorPHID IN (%Ls)', $author_phids); } $with_unowned = $query->getParameter('withUnowned'); $with_any = $query->getParameter('withAnyOwner'); if ($with_any && $with_unowned) { throw new PhabricatorEmptyQueryException( pht( 'This query matches only unowned documents owned by anyone, '. 'which is impossible.')); } $owner_phids = $query->getParameter('ownerPHIDs'); if ($owner_phids && !$with_any) { if ($with_unowned) { $where[] = qsprintf( $conn, 'ft_doc.ownerPHID IN (%Ls) OR ft_doc.ownerPHID IS NULL', $owner_phids); } else { $where[] = qsprintf( $conn, 'ft_doc.ownerPHID IN (%Ls)', $owner_phids); } } else if ($with_unowned) { $where[] = qsprintf( $conn, 'ft_doc.ownerPHID IS NULL'); } if ($with_any) { $where[] = qsprintf( $conn, 'ft_doc.ownerPHID IS NOT NULL'); } $rel_open = PhabricatorSearchRelationship::RELATIONSHIP_OPEN; $statuses = $query->getParameter('statuses'); $is_closed = null; if ($statuses) { $statuses = array_fuse($statuses); if (count($statuses) == 1) { if (isset($statuses[$rel_open])) { $is_closed = 0; } else { $is_closed = 1; } } } if ($is_closed !== null) { $where[] = qsprintf( $conn, 'ft_doc.isClosed = %d', $is_closed); } } return $where; } protected function shouldGroupFerretResultRows() { return (bool)$this->ferretTokens; } /* -( Ngrams )------------------------------------------------------------- */ protected function withNgramsConstraint( PhabricatorSearchNgrams $index, $value) { if (strlen($value)) { $this->ngrams[] = array( 'index' => $index, 'value' => $value, 'length' => count(phutil_utf8v($value)), ); } return $this; } protected function buildNgramsJoinClause(AphrontDatabaseConnection $conn) { $flat = array(); foreach ($this->ngrams as $spec) { $index = $spec['index']; $value = $spec['value']; $length = $spec['length']; if ($length >= 3) { $ngrams = $index->getNgramsFromString($value, 'query'); $prefix = false; } else if ($length == 2) { $ngrams = $index->getNgramsFromString($value, 'prefix'); $prefix = false; } else { $ngrams = array(' '.$value); $prefix = true; } foreach ($ngrams as $ngram) { $flat[] = array( 'table' => $index->getTableName(), 'ngram' => $ngram, 'prefix' => $prefix, ); } } // MySQL only allows us to join a maximum of 61 tables per query. Each // ngram is going to cost us a join toward that limit, so if the user // specified a very long query string, just pick 16 of the ngrams // at random. if (count($flat) > 16) { shuffle($flat); $flat = array_slice($flat, 0, 16); } $alias = $this->getPrimaryTableAlias(); if ($alias) { $id_column = qsprintf($conn, '%T.%T', $alias, 'id'); } else { $id_column = qsprintf($conn, '%T', 'id'); } $idx = 1; $joins = array(); foreach ($flat as $spec) { $table = $spec['table']; $ngram = $spec['ngram']; $prefix = $spec['prefix']; $alias = 'ngm'.$idx++; if ($prefix) { $joins[] = qsprintf( $conn, 'JOIN %T %T ON %T.objectID = %Q AND %T.ngram LIKE %>', $table, $alias, $alias, $id_column, $alias, $ngram); } else { $joins[] = qsprintf( $conn, 'JOIN %T %T ON %T.objectID = %Q AND %T.ngram = %s', $table, $alias, $alias, $id_column, $alias, $ngram); } } return $joins; } protected function buildNgramsWhereClause(AphrontDatabaseConnection $conn) { $where = array(); foreach ($this->ngrams as $ngram) { $index = $ngram['index']; $value = $ngram['value']; $column = $index->getColumnName(); $alias = $this->getPrimaryTableAlias(); if ($alias) { $column = qsprintf($conn, '%T.%T', $alias, $column); } else { $column = qsprintf($conn, '%T', $column); } $tokens = $index->tokenizeString($value); foreach ($tokens as $token) { $where[] = qsprintf( $conn, '%Q LIKE %~', $column, $token); } } return $where; } protected function shouldGroupNgramResultRows() { return (bool)$this->ngrams; } /* -( Edge Logic )--------------------------------------------------------- */ /** * Convenience method for specifying edge logic constraints with a list of * PHIDs. * * @param const Edge constant. * @param const Constraint operator. * @param list List of PHIDs. * @return this * @task edgelogic */ public function withEdgeLogicPHIDs($edge_type, $operator, array $phids) { $constraints = array(); foreach ($phids as $phid) { $constraints[] = new PhabricatorQueryConstraint($operator, $phid); } return $this->withEdgeLogicConstraints($edge_type, $constraints); } /** * @return this * @task edgelogic */ public function withEdgeLogicConstraints($edge_type, array $constraints) { assert_instances_of($constraints, 'PhabricatorQueryConstraint'); $constraints = mgroup($constraints, 'getOperator'); foreach ($constraints as $operator => $list) { foreach ($list as $item) { $this->edgeLogicConstraints[$edge_type][$operator][] = $item; } } $this->edgeLogicConstraintsAreValid = false; return $this; } /** * @task edgelogic */ public function buildEdgeLogicSelectClause(AphrontDatabaseConnection $conn) { $select = array(); $this->validateEdgeLogicConstraints(); foreach ($this->edgeLogicConstraints as $type => $constraints) { foreach ($constraints as $operator => $list) { $alias = $this->getEdgeLogicTableAlias($operator, $type); switch ($operator) { case PhabricatorQueryConstraint::OPERATOR_AND: if (count($list) > 1) { $select[] = qsprintf( $conn, 'COUNT(DISTINCT(%T.dst)) %T', $alias, $this->buildEdgeLogicTableAliasCount($alias)); } break; case PhabricatorQueryConstraint::OPERATOR_ANCESTOR: // This is tricky. We have a query which specifies multiple // projects, each of which may have an arbitrarily large number // of descendants. // Suppose the projects are "Engineering" and "Operations", and // "Engineering" has subprojects X, Y and Z. // We first use `FIELD(dst, X, Y, Z)` to produce a 0 if a row // is not part of Engineering at all, or some number other than // 0 if it is. // Then we use `IF(..., idx, NULL)` to convert the 0 to a NULL and // any other value to an index (say, 1) for the ancestor. // We build these up for every ancestor, then use `COALESCE(...)` // to select the non-null one, giving us an ancestor which this // row is a member of. // From there, we use `COUNT(DISTINCT(...))` to make sure that // each result row is a member of all ancestors. if (count($list) > 1) { $idx = 1; $parts = array(); foreach ($list as $constraint) { $parts[] = qsprintf( $conn, 'IF(FIELD(%T.dst, %Ls) != 0, %d, NULL)', $alias, (array)$constraint->getValue(), $idx++); } $parts = qsprintf($conn, '%LQ', $parts); $select[] = qsprintf( $conn, 'COUNT(DISTINCT(COALESCE(%Q))) %T', $parts, $this->buildEdgeLogicTableAliasAncestor($alias)); } break; default: break; } } } return $select; } /** * @task edgelogic */ public function buildEdgeLogicJoinClause(AphrontDatabaseConnection $conn) { $edge_table = PhabricatorEdgeConfig::TABLE_NAME_EDGE; $phid_column = $this->getApplicationSearchObjectPHIDColumn($conn); $joins = array(); foreach ($this->edgeLogicConstraints as $type => $constraints) { $op_null = PhabricatorQueryConstraint::OPERATOR_NULL; $has_null = isset($constraints[$op_null]); // If we're going to process an only() operator, build a list of the // acceptable set of PHIDs first. We'll only match results which have // no edges to any other PHIDs. $all_phids = array(); if (isset($constraints[PhabricatorQueryConstraint::OPERATOR_ONLY])) { foreach ($constraints as $operator => $list) { switch ($operator) { case PhabricatorQueryConstraint::OPERATOR_ANCESTOR: case PhabricatorQueryConstraint::OPERATOR_AND: case PhabricatorQueryConstraint::OPERATOR_OR: foreach ($list as $constraint) { $value = (array)$constraint->getValue(); foreach ($value as $v) { $all_phids[$v] = $v; } } break; } } } foreach ($constraints as $operator => $list) { $alias = $this->getEdgeLogicTableAlias($operator, $type); $phids = array(); foreach ($list as $constraint) { $value = (array)$constraint->getValue(); foreach ($value as $v) { $phids[$v] = $v; } } $phids = array_keys($phids); switch ($operator) { case PhabricatorQueryConstraint::OPERATOR_NOT: $joins[] = qsprintf( $conn, 'LEFT JOIN %T %T ON %Q = %T.src AND %T.type = %d AND %T.dst IN (%Ls)', $edge_table, $alias, $phid_column, $alias, $alias, $type, $alias, $phids); break; case PhabricatorQueryConstraint::OPERATOR_ANCESTOR: case PhabricatorQueryConstraint::OPERATOR_AND: case PhabricatorQueryConstraint::OPERATOR_OR: // If we're including results with no matches, we have to degrade // this to a LEFT join. We'll use WHERE to select matching rows // later. if ($has_null) { $join_type = qsprintf($conn, 'LEFT'); } else { $join_type = qsprintf($conn, ''); } $joins[] = qsprintf( $conn, '%Q JOIN %T %T ON %Q = %T.src AND %T.type = %d AND %T.dst IN (%Ls)', $join_type, $edge_table, $alias, $phid_column, $alias, $alias, $type, $alias, $phids); break; case PhabricatorQueryConstraint::OPERATOR_NULL: $joins[] = qsprintf( $conn, 'LEFT JOIN %T %T ON %Q = %T.src AND %T.type = %d', $edge_table, $alias, $phid_column, $alias, $alias, $type); break; case PhabricatorQueryConstraint::OPERATOR_ONLY: $joins[] = qsprintf( $conn, 'LEFT JOIN %T %T ON %Q = %T.src AND %T.type = %d AND %T.dst NOT IN (%Ls)', $edge_table, $alias, $phid_column, $alias, $alias, $type, $alias, $all_phids); break; } } } return $joins; } /** * @task edgelogic */ public function buildEdgeLogicWhereClause(AphrontDatabaseConnection $conn) { $where = array(); foreach ($this->edgeLogicConstraints as $type => $constraints) { $full = array(); $null = array(); $op_null = PhabricatorQueryConstraint::OPERATOR_NULL; $has_null = isset($constraints[$op_null]); foreach ($constraints as $operator => $list) { $alias = $this->getEdgeLogicTableAlias($operator, $type); switch ($operator) { case PhabricatorQueryConstraint::OPERATOR_NOT: case PhabricatorQueryConstraint::OPERATOR_ONLY: $full[] = qsprintf( $conn, '%T.dst IS NULL', $alias); break; case PhabricatorQueryConstraint::OPERATOR_AND: case PhabricatorQueryConstraint::OPERATOR_OR: if ($has_null) { $full[] = qsprintf( $conn, '%T.dst IS NOT NULL', $alias); } break; case PhabricatorQueryConstraint::OPERATOR_NULL: $null[] = qsprintf( $conn, '%T.dst IS NULL', $alias); break; } } if ($full && $null) { $where[] = qsprintf($conn, '(%LA OR %LA)', $full, $null); } else if ($full) { foreach ($full as $condition) { $where[] = $condition; } } else if ($null) { foreach ($null as $condition) { $where[] = $condition; } } } return $where; } /** * @task edgelogic */ public function buildEdgeLogicHavingClause(AphrontDatabaseConnection $conn) { $having = array(); foreach ($this->edgeLogicConstraints as $type => $constraints) { foreach ($constraints as $operator => $list) { $alias = $this->getEdgeLogicTableAlias($operator, $type); switch ($operator) { case PhabricatorQueryConstraint::OPERATOR_AND: if (count($list) > 1) { $having[] = qsprintf( $conn, '%T = %d', $this->buildEdgeLogicTableAliasCount($alias), count($list)); } break; case PhabricatorQueryConstraint::OPERATOR_ANCESTOR: if (count($list) > 1) { $having[] = qsprintf( $conn, '%T = %d', $this->buildEdgeLogicTableAliasAncestor($alias), count($list)); } break; } } } return $having; } /** * @task edgelogic */ public function shouldGroupEdgeLogicResultRows() { foreach ($this->edgeLogicConstraints as $type => $constraints) { foreach ($constraints as $operator => $list) { switch ($operator) { case PhabricatorQueryConstraint::OPERATOR_NOT: case PhabricatorQueryConstraint::OPERATOR_AND: case PhabricatorQueryConstraint::OPERATOR_OR: if (count($list) > 1) { return true; } break; case PhabricatorQueryConstraint::OPERATOR_ANCESTOR: // NOTE: We must always group query results rows when using an // "ANCESTOR" operator because a single task may be related to // two different descendants of a particular ancestor. For // discussion, see T12753. return true; case PhabricatorQueryConstraint::OPERATOR_NULL: case PhabricatorQueryConstraint::OPERATOR_ONLY: return true; } } } return false; } /** * @task edgelogic */ private function getEdgeLogicTableAlias($operator, $type) { return 'edgelogic_'.$operator.'_'.$type; } /** * @task edgelogic */ private function buildEdgeLogicTableAliasCount($alias) { return $alias.'_count'; } /** * @task edgelogic */ private function buildEdgeLogicTableAliasAncestor($alias) { return $alias.'_ancestor'; } /** * Select certain edge logic constraint values. * * @task edgelogic */ protected function getEdgeLogicValues( array $edge_types, array $operators) { $values = array(); $constraint_lists = $this->edgeLogicConstraints; if ($edge_types) { $constraint_lists = array_select_keys($constraint_lists, $edge_types); } foreach ($constraint_lists as $type => $constraints) { if ($operators) { $constraints = array_select_keys($constraints, $operators); } foreach ($constraints as $operator => $list) { foreach ($list as $constraint) { $value = (array)$constraint->getValue(); foreach ($value as $v) { $values[] = $v; } } } } return $values; } /** * Validate edge logic constraints for the query. * * @return this * @task edgelogic */ private function validateEdgeLogicConstraints() { if ($this->edgeLogicConstraintsAreValid) { return $this; } foreach ($this->edgeLogicConstraints as $type => $constraints) { foreach ($constraints as $operator => $list) { switch ($operator) { case PhabricatorQueryConstraint::OPERATOR_EMPTY: throw new PhabricatorEmptyQueryException( pht('This query specifies an empty constraint.')); } } } // This should probably be more modular, eventually, but we only do // project-based edge logic today. $project_phids = $this->getEdgeLogicValues( array( PhabricatorProjectObjectHasProjectEdgeType::EDGECONST, ), array( PhabricatorQueryConstraint::OPERATOR_AND, PhabricatorQueryConstraint::OPERATOR_OR, PhabricatorQueryConstraint::OPERATOR_NOT, PhabricatorQueryConstraint::OPERATOR_ANCESTOR, )); if ($project_phids) { $projects = id(new PhabricatorProjectQuery()) ->setViewer($this->getViewer()) ->setParentQuery($this) ->withPHIDs($project_phids) ->execute(); $projects = mpull($projects, null, 'getPHID'); foreach ($project_phids as $phid) { if (empty($projects[$phid])) { throw new PhabricatorEmptyQueryException( pht( 'This query is constrained by a project you do not have '. 'permission to see.')); } } } $op_and = PhabricatorQueryConstraint::OPERATOR_AND; $op_or = PhabricatorQueryConstraint::OPERATOR_OR; $op_ancestor = PhabricatorQueryConstraint::OPERATOR_ANCESTOR; foreach ($this->edgeLogicConstraints as $type => $constraints) { foreach ($constraints as $operator => $list) { switch ($operator) { case PhabricatorQueryConstraint::OPERATOR_ONLY: if (count($list) > 1) { throw new PhabricatorEmptyQueryException( pht( 'This query specifies only() more than once.')); } $have_and = idx($constraints, $op_and); $have_or = idx($constraints, $op_or); $have_ancestor = idx($constraints, $op_ancestor); if (!$have_and && !$have_or && !$have_ancestor) { throw new PhabricatorEmptyQueryException( pht( 'This query specifies only(), but no other constraints '. 'which it can apply to.')); } break; } } } $this->edgeLogicConstraintsAreValid = true; return $this; } /* -( Spaces )------------------------------------------------------------- */ /** * Constrain the query to return results from only specific Spaces. * * Pass a list of Space PHIDs, or `null` to represent the default space. Only * results in those Spaces will be returned. * * Queries are always constrained to include only results from spaces the * viewer has access to. * * @param list * @task spaces */ public function withSpacePHIDs(array $space_phids) { $object = $this->newResultObject(); if (!$object) { throw new Exception( pht( 'This query (of class "%s") does not implement newResultObject(), '. 'but must implement this method to enable support for Spaces.', get_class($this))); } if (!($object instanceof PhabricatorSpacesInterface)) { throw new Exception( pht( 'This query (of class "%s") returned an object of class "%s" from '. 'getNewResultObject(), but it does not implement the required '. 'interface ("%s"). Objects must implement this interface to enable '. 'Spaces support.', get_class($this), get_class($object), 'PhabricatorSpacesInterface')); } $this->spacePHIDs = $space_phids; return $this; } public function withSpaceIsArchived($archived) { $this->spaceIsArchived = $archived; return $this; } /** * Constrain the query to include only results in valid Spaces. * * This method builds part of a WHERE clause which considers the spaces the * viewer has access to see with any explicit constraint on spaces added by * @{method:withSpacePHIDs}. * * @param AphrontDatabaseConnection Database connection. * @return string Part of a WHERE clause. * @task spaces */ private function buildSpacesWhereClause(AphrontDatabaseConnection $conn) { $object = $this->newResultObject(); if (!$object) { return null; } if (!($object instanceof PhabricatorSpacesInterface)) { return null; } $viewer = $this->getViewer(); // If we have an omnipotent viewer and no formal space constraints, don't // emit a clause. This primarily enables older migrations to run cleanly, // without fataling because they try to match a `spacePHID` column which // does not exist yet. See T8743, T8746. if ($viewer->isOmnipotent()) { if ($this->spaceIsArchived === null && $this->spacePHIDs === null) { return null; } } // See T13240. If this query raises policy exceptions, don't filter objects // in the MySQL layer. We want them to reach the application layer so we // can reject them and raise an exception. if ($this->shouldRaisePolicyExceptions()) { return null; } $space_phids = array(); $include_null = false; $all = PhabricatorSpacesNamespaceQuery::getAllSpaces(); if (!$all) { // If there are no spaces at all, implicitly give the viewer access to // the default space. $include_null = true; } else { // Otherwise, give them access to the spaces they have permission to // see. $viewer_spaces = PhabricatorSpacesNamespaceQuery::getViewerSpaces( $viewer); foreach ($viewer_spaces as $viewer_space) { if ($this->spaceIsArchived !== null) { if ($viewer_space->getIsArchived() != $this->spaceIsArchived) { continue; } } $phid = $viewer_space->getPHID(); $space_phids[$phid] = $phid; if ($viewer_space->getIsDefaultNamespace()) { $include_null = true; } } } // If we have additional explicit constraints, evaluate them now. if ($this->spacePHIDs !== null) { $explicit = array(); $explicit_null = false; foreach ($this->spacePHIDs as $phid) { if ($phid === null) { $space = PhabricatorSpacesNamespaceQuery::getDefaultSpace(); } else { $space = idx($all, $phid); } if ($space) { $phid = $space->getPHID(); $explicit[$phid] = $phid; if ($space->getIsDefaultNamespace()) { $explicit_null = true; } } } // If the viewer can see the default space but it isn't on the explicit // list of spaces to query, don't match it. if ($include_null && !$explicit_null) { $include_null = false; } // Include only the spaces common to the viewer and the constraints. $space_phids = array_intersect_key($space_phids, $explicit); } if (!$space_phids && !$include_null) { if ($this->spacePHIDs === null) { throw new PhabricatorEmptyQueryException( pht('You do not have access to any spaces.')); } else { throw new PhabricatorEmptyQueryException( pht( 'You do not have access to any of the spaces this query '. 'is constrained to.')); } } $alias = $this->getPrimaryTableAlias(); if ($alias) { $col = qsprintf($conn, '%T.spacePHID', $alias); } else { $col = qsprintf($conn, 'spacePHID'); } if ($space_phids && $include_null) { return qsprintf( $conn, '(%Q IN (%Ls) OR %Q IS NULL)', $col, $space_phids, $col); } else if ($space_phids) { return qsprintf( $conn, '%Q IN (%Ls)', $col, $space_phids); } else { return qsprintf( $conn, '%Q IS NULL', $col); } } } diff --git a/src/infrastructure/query/policy/PhabricatorQueryCursor.php b/src/infrastructure/query/policy/PhabricatorQueryCursor.php index f2d183ff12..4ec1263130 100644 --- a/src/infrastructure/query/policy/PhabricatorQueryCursor.php +++ b/src/infrastructure/query/policy/PhabricatorQueryCursor.php @@ -1,17 +1,47 @@ object = $object; return $this; } public function getObject() { return $this->object; } + public function setRawRow(array $raw_row) { + $this->rawRow = $raw_row; + return $this; + } + + public function getRawRow() { + return $this->rawRow; + } + + public function getRawRowProperty($key) { + if (!is_array($this->rawRow)) { + throw new Exception( + pht( + 'Caller is trying to "getRawRowProperty()" with key "%s", but this '. + 'cursor has no raw row.', + $key)); + } + + if (!array_key_exists($key, $this->rawRow)) { + throw new Exception( + pht( + 'Caller is trying to access raw row property "%s", but the row '. + 'does not have this property.', + $key)); + } + + return $this->rawRow[$key]; + } + }