diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -2231,6 +2231,7 @@ 'PhabricatorBulkEngine' => 'applications/transactions/bulk/PhabricatorBulkEngine.php', 'PhabricatorBulkManagementMakeSilentWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementMakeSilentWorkflow.php', 'PhabricatorBulkManagementWorkflow' => 'applications/transactions/bulk/management/PhabricatorBulkManagementWorkflow.php', + 'PhabricatorCSVExportFormat' => 'infrastructure/export/PhabricatorCSVExportFormat.php', 'PhabricatorCacheDAO' => 'applications/cache/storage/PhabricatorCacheDAO.php', 'PhabricatorCacheEngine' => 'applications/system/engine/PhabricatorCacheEngine.php', 'PhabricatorCacheEngineExtension' => 'applications/system/engine/PhabricatorCacheEngineExtension.php', @@ -2844,6 +2845,7 @@ 'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php', 'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php', 'PhabricatorExportField' => 'infrastructure/export/PhabricatorExportField.php', + 'PhabricatorExportFormat' => 'infrastructure/export/PhabricatorExportFormat.php', 'PhabricatorExtendedPolicyInterface' => 'applications/policy/interface/PhabricatorExtendedPolicyInterface.php', 'PhabricatorExtendingPhabricatorConfigOptions' => 'applications/config/option/PhabricatorExtendingPhabricatorConfigOptions.php', 'PhabricatorExtensionsSetupCheck' => 'applications/config/check/PhabricatorExtensionsSetupCheck.php', @@ -3087,6 +3089,7 @@ 'PhabricatorInlineSummaryView' => 'infrastructure/diff/view/PhabricatorInlineSummaryView.php', 'PhabricatorInstructionsEditField' => 'applications/transactions/editfield/PhabricatorInstructionsEditField.php', 'PhabricatorIntConfigType' => 'applications/config/type/PhabricatorIntConfigType.php', + 'PhabricatorIntExportField' => 'infrastructure/export/PhabricatorIntExportField.php', 'PhabricatorInternalSetting' => 'applications/settings/setting/PhabricatorInternalSetting.php', 'PhabricatorInternationalizationManagementExtractWorkflow' => 'infrastructure/internationalization/management/PhabricatorInternationalizationManagementExtractWorkflow.php', 'PhabricatorInternationalizationManagementWorkflow' => 'infrastructure/internationalization/management/PhabricatorInternationalizationManagementWorkflow.php', @@ -3096,6 +3099,7 @@ 'PhabricatorIteratorFileUploadSource' => 'applications/files/uploadsource/PhabricatorIteratorFileUploadSource.php', 'PhabricatorJIRAAuthProvider' => 'applications/auth/provider/PhabricatorJIRAAuthProvider.php', 'PhabricatorJSONConfigType' => 'applications/config/type/PhabricatorJSONConfigType.php', + 'PhabricatorJSONExportFormat' => 'infrastructure/export/PhabricatorJSONExportFormat.php', 'PhabricatorJavelinLinter' => 'infrastructure/lint/linter/PhabricatorJavelinLinter.php', 'PhabricatorJiraIssueHasObjectEdgeType' => 'applications/doorkeeper/edge/PhabricatorJiraIssueHasObjectEdgeType.php', 'PhabricatorJumpNavHandler' => 'applications/search/engine/PhabricatorJumpNavHandler.php', @@ -4245,6 +4249,7 @@ 'PhabricatorTextAreaEditField' => 'applications/transactions/editfield/PhabricatorTextAreaEditField.php', 'PhabricatorTextConfigType' => 'applications/config/type/PhabricatorTextConfigType.php', 'PhabricatorTextEditField' => 'applications/transactions/editfield/PhabricatorTextEditField.php', + 'PhabricatorTextExportFormat' => 'infrastructure/export/PhabricatorTextExportFormat.php', 'PhabricatorTextListConfigType' => 'applications/config/type/PhabricatorTextListConfigType.php', 'PhabricatorTime' => 'infrastructure/time/PhabricatorTime.php', 'PhabricatorTimeFormatSetting' => 'applications/settings/setting/PhabricatorTimeFormatSetting.php', @@ -7564,6 +7569,7 @@ 'PhabricatorBulkEngine' => 'Phobject', 'PhabricatorBulkManagementMakeSilentWorkflow' => 'PhabricatorBulkManagementWorkflow', 'PhabricatorBulkManagementWorkflow' => 'PhabricatorManagementWorkflow', + 'PhabricatorCSVExportFormat' => 'PhabricatorExportFormat', 'PhabricatorCacheDAO' => 'PhabricatorLiskDAO', 'PhabricatorCacheEngine' => 'Phobject', 'PhabricatorCacheEngineExtension' => 'Phobject', @@ -8268,6 +8274,7 @@ 'PhabricatorExampleEventListener' => 'PhabricatorEventListener', 'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource', 'PhabricatorExportField' => 'Phobject', + 'PhabricatorExportFormat' => 'Phobject', 'PhabricatorExtendingPhabricatorConfigOptions' => 'PhabricatorApplicationConfigOptions', 'PhabricatorExtensionsSetupCheck' => 'PhabricatorSetupCheck', 'PhabricatorExternalAccount' => array( @@ -8551,6 +8558,7 @@ 'PhabricatorInlineSummaryView' => 'AphrontView', 'PhabricatorInstructionsEditField' => 'PhabricatorEditField', 'PhabricatorIntConfigType' => 'PhabricatorTextConfigType', + 'PhabricatorIntExportField' => 'PhabricatorExportField', 'PhabricatorInternalSetting' => 'PhabricatorSetting', 'PhabricatorInternationalizationManagementExtractWorkflow' => 'PhabricatorInternationalizationManagementWorkflow', 'PhabricatorInternationalizationManagementWorkflow' => 'PhabricatorManagementWorkflow', @@ -8560,6 +8568,7 @@ 'PhabricatorIteratorFileUploadSource' => 'PhabricatorFileUploadSource', 'PhabricatorJIRAAuthProvider' => 'PhabricatorOAuth1AuthProvider', 'PhabricatorJSONConfigType' => 'PhabricatorTextConfigType', + 'PhabricatorJSONExportFormat' => 'PhabricatorExportFormat', 'PhabricatorJavelinLinter' => 'ArcanistLinter', 'PhabricatorJiraIssueHasObjectEdgeType' => 'PhabricatorEdgeType', 'PhabricatorJumpNavHandler' => 'Phobject', @@ -9922,6 +9931,7 @@ 'PhabricatorTextAreaEditField' => 'PhabricatorEditField', 'PhabricatorTextConfigType' => 'PhabricatorConfigType', 'PhabricatorTextEditField' => 'PhabricatorEditField', + 'PhabricatorTextExportFormat' => 'PhabricatorExportFormat', 'PhabricatorTextListConfigType' => 'PhabricatorTextConfigType', 'PhabricatorTime' => 'Phobject', 'PhabricatorTimeFormatSetting' => 'PhabricatorSelectSetting', diff --git a/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php b/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php --- a/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php +++ b/src/applications/diffusion/query/DiffusionPullLogSearchEngine.php @@ -73,7 +73,7 @@ id(new PhabricatorStringExportField()) ->setKey('result') ->setLabel(pht('Result')), - id(new PhabricatorStringExportField()) + id(new PhabricatorIntExportField()) ->setKey('code') ->setLabel(pht('Code')), id(new PhabricatorEpochExportField()) diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php --- a/src/applications/search/controller/PhabricatorApplicationSearchController.php +++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php @@ -413,42 +413,80 @@ $filename = phutil_utf8_strtolower($filename); $filename = PhabricatorFile::normalizeFileName($filename); + $formats = PhabricatorExportFormat::getAllEnabledExportFormats(); + $format_options = mpull($formats, 'getExportFormatName'); + + $errors = array(); + + $e_format = null; if ($request->isFormPost()) { - $query = $engine->buildQueryFromSavedQuery($saved_query); + $format_key = $request->getStr('format'); + $format = idx($formats, $format_key); - // NOTE: We aren't reading the pager from the request. Exports always - // affect the entire result set. - $pager = $engine->newPagerForSavedQuery($saved_query); - $pager->setPageSize(0x7FFFFFFF); + if (!$format) { + $e_format = pht('Invalid'); + $errors[] = pht('Choose a valid export format.'); + } - $objects = $engine->executeQuery($query, $pager); + if (!$errors) { + $query = $engine->buildQueryFromSavedQuery($saved_query); - $extension = 'json'; - $mime_type = 'application/json'; - $filename = $filename.'.'.$extension; + // NOTE: We aren't reading the pager from the request. Exports always + // affect the entire result set. + $pager = $engine->newPagerForSavedQuery($saved_query); + $pager->setPageSize(0x7FFFFFFF); - $result = $engine->newExport($objects); - $result = id(new PhutilJSON()) - ->encodeAsList($result); + $objects = $engine->executeQuery($query, $pager); - $file = PhabricatorFile::newFromFileData( - $result, - array( - 'name' => $filename, - 'authorPHID' => $viewer->getPHID(), - 'ttl.relative' => phutil_units('15 minutes in seconds'), - 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE, - 'mime-type' => $mime_type, - )); + $extension = $format->getFileExtension(); + $mime_type = $format->getMIMEContentType(); + $filename = $filename.'.'.$extension; + + $format = clone $format; + $format->setViewer($viewer); - return $this->newDialog() - ->setTitle(pht('Download Results')) - ->appendParagraph( - pht('Click the download button to download the exported data.')) - ->addCancelButton($cancel_uri, pht('Done')) - ->setSubmitURI($file->getDownloadURI()) - ->setDisableWorkflowOnSubmit(true) - ->addSubmitButton(pht('Download Results')); + $export_data = $engine->newExport($objects); + + if (count($export_data) !== count($objects)) { + throw new Exception( + pht( + 'Search engine exported the wrong number of objects, expected '. + '%s but got %s.', + phutil_count($objects), + phutil_count($export_data))); + } + + $objects = array_values($objects); + $export_data = array_values($export_data); + + $field_list = $engine->newExportFieldList(); + $field_list = mpull($field_list, null, 'getKey'); + + for ($ii = 0; $ii < count($objects); $ii++) { + $format->addObject($objects[$ii], $field_list, $export_data[$ii]); + } + + $export_result = $format->newFileData(); + + $file = PhabricatorFile::newFromFileData( + $export_result, + array( + 'name' => $filename, + 'authorPHID' => $viewer->getPHID(), + 'ttl.relative' => phutil_units('15 minutes in seconds'), + 'viewPolicy' => PhabricatorPolicies::POLICY_NOONE, + 'mime-type' => $mime_type, + )); + + return $this->newDialog() + ->setTitle(pht('Download Results')) + ->appendParagraph( + pht('Click the download button to download the exported data.')) + ->addCancelButton($cancel_uri, pht('Done')) + ->setSubmitURI($file->getDownloadURI()) + ->setDisableWorkflowOnSubmit(true) + ->addSubmitButton(pht('Download Data')); + } } $export_form = id(new AphrontFormView()) @@ -457,13 +495,12 @@ id(new AphrontFormSelectControl()) ->setName('format') ->setLabel(pht('Format')) - ->setOptions( - array( - 'json' => 'JSON', - ))); + ->setError($e_format) + ->setOptions($format_options)); return $this->newDialog() ->setTitle(pht('Export Results')) + ->setErrors($errors) ->appendForm($export_form) ->addCancelButton($cancel_uri) ->addSubmitButton(pht('Continue')); @@ -826,7 +863,7 @@ $export_uri = $engine->getExportURI($query_key); $actions[] = id(new PhabricatorActionView()) ->setIcon('fa-download') - ->setName(pht('Export Results')) + ->setName(pht('Export Data')) ->setWorkflow(true) ->setHref($export_uri); } diff --git a/src/applications/search/engine/PhabricatorApplicationSearchEngine.php b/src/applications/search/engine/PhabricatorApplicationSearchEngine.php --- a/src/applications/search/engine/PhabricatorApplicationSearchEngine.php +++ b/src/applications/search/engine/PhabricatorApplicationSearchEngine.php @@ -1454,6 +1454,10 @@ return (bool)$fields; } + final public function newExportFieldList() { + return $this->newExportFields(); + } + protected function newExportFields() { return array(); } diff --git a/src/infrastructure/export/PhabricatorCSVExportFormat.php b/src/infrastructure/export/PhabricatorCSVExportFormat.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/export/PhabricatorCSVExportFormat.php @@ -0,0 +1,47 @@ + $field) { + $value = $map[$key]; + $value = $field->getTextValue($value); + + if (preg_match('/\s|,|\"/', $value)) { + $value = str_replace('"', '""', $value); + $value = '"'.$value.'"'; + } + + $values[] = $value; + } + + $this->rows[] = implode(',', $values); + } + + public function newFileData() { + return implode("\n", $this->rows); + } + +} diff --git a/src/infrastructure/export/PhabricatorEpochExportField.php b/src/infrastructure/export/PhabricatorEpochExportField.php --- a/src/infrastructure/export/PhabricatorEpochExportField.php +++ b/src/infrastructure/export/PhabricatorEpochExportField.php @@ -1,4 +1,27 @@ zone)) { + $this->zone = new DateTimeZone('UTC'); + } + + try { + $date = new DateTime('@'.$value); + } catch (Exception $ex) { + return null; + } + + $date->setTimezone($this->zone); + return $date->format('c'); + } + + public function getNaturalValue($value) { + return (int)$value; + } + +} diff --git a/src/infrastructure/export/PhabricatorExportField.php b/src/infrastructure/export/PhabricatorExportField.php --- a/src/infrastructure/export/PhabricatorExportField.php +++ b/src/infrastructure/export/PhabricatorExportField.php @@ -24,4 +24,12 @@ return $this->label; } + public function getTextValue($value) { + return (string)$this->getNaturalValue($value); + } + + public function getNaturalValue($value) { + return $value; + } + } diff --git a/src/infrastructure/export/PhabricatorExportFormat.php b/src/infrastructure/export/PhabricatorExportFormat.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/export/PhabricatorExportFormat.php @@ -0,0 +1,51 @@ +getPhobjectClassConstant('EXPORTKEY'); + } + + final public function setViewer(PhabricatorUser $viewer) { + $this->viewer = $viewer; + return $this; + } + + final public function getViewer() { + return $this->viewer; + } + + abstract public function getExportFormatName(); + abstract public function getMIMEContentType(); + abstract public function getFileExtension(); + + abstract public function addObject($object, array $fields, array $map); + abstract public function newFileData(); + + public function isExportFormatEnabled() { + return true; + } + + final public static function getAllExportFormats() { + return id(new PhutilClassMapQuery()) + ->setAncestorClass(__CLASS__) + ->setUniqueMethod('getExportFormatKey') + ->execute(); + } + + final public static function getAllEnabledExportFormats() { + $formats = self::getAllExportFormats(); + + foreach ($formats as $key => $format) { + if (!$format->isExportFormatEnabled()) { + unset($formats[$key]); + } + } + + return $formats; + } + +} diff --git a/src/infrastructure/export/PhabricatorIDExportField.php b/src/infrastructure/export/PhabricatorIDExportField.php --- a/src/infrastructure/export/PhabricatorIDExportField.php +++ b/src/infrastructure/export/PhabricatorIDExportField.php @@ -1,4 +1,10 @@ $field) { + $value = $map[$key]; + $value = $field->getNaturalValue($value); + + $values[$key] = $value; + } + + $this->objects[] = $values; + } + + public function newFileData() { + return id(new PhutilJSON()) + ->encodeAsList($this->objects); + } + +} diff --git a/src/infrastructure/export/PhabricatorTextExportFormat.php b/src/infrastructure/export/PhabricatorTextExportFormat.php new file mode 100644 --- /dev/null +++ b/src/infrastructure/export/PhabricatorTextExportFormat.php @@ -0,0 +1,43 @@ + $field) { + $value = $map[$key]; + $value = $field->getTextValue($value); + $value = addcslashes($value, "\0..\37\\\177..\377"); + + $values[] = $value; + } + + $this->rows[] = implode("\t", $values); + } + + public function newFileData() { + return implode("\n", $this->rows)."\n"; + } + +}