Page MenuHomePhabricator
Files F15393908

D18962.diff
No OneTemporary
Actions

D18962.diff
View Options

diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php
--- a/src/__phutil_library_map__.php
+++ b/src/__phutil_library_map__.php
@@ -2845,6 +2845,8 @@
'PhabricatorExampleEventListener' => 'infrastructure/events/PhabricatorExampleEventListener.php',
'PhabricatorExcelExportFormat' => 'infrastructure/export/format/PhabricatorExcelExportFormat.php',
'PhabricatorExecFutureFileUploadSource' => 'applications/files/uploadsource/PhabricatorExecFutureFileUploadSource.php',
+ 'PhabricatorExportEngine' => 'infrastructure/export/engine/PhabricatorExportEngine.php',
+ 'PhabricatorExportEngineBulkJobType' => 'infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php',
'PhabricatorExportEngineExtension' => 'infrastructure/export/engine/PhabricatorExportEngineExtension.php',
'PhabricatorExportField' => 'infrastructure/export/field/PhabricatorExportField.php',
'PhabricatorExportFormat' => 'infrastructure/export/format/PhabricatorExportFormat.php',
@@ -4419,6 +4421,7 @@
'PhabricatorWorkerManagementWorkflow' => 'infrastructure/daemon/workers/management/PhabricatorWorkerManagementWorkflow.php',
'PhabricatorWorkerPermanentFailureException' => 'infrastructure/daemon/workers/exception/PhabricatorWorkerPermanentFailureException.php',
'PhabricatorWorkerSchemaSpec' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerSchemaSpec.php',
+ 'PhabricatorWorkerSingleBulkJobType' => 'infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php',
'PhabricatorWorkerTask' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTask.php',
'PhabricatorWorkerTaskData' => 'infrastructure/daemon/workers/storage/PhabricatorWorkerTaskData.php',
'PhabricatorWorkerTaskDetailController' => 'applications/daemon/controller/PhabricatorWorkerTaskDetailController.php',
@@ -8286,6 +8289,8 @@
'PhabricatorExampleEventListener' => 'PhabricatorEventListener',
'PhabricatorExcelExportFormat' => 'PhabricatorExportFormat',
'PhabricatorExecFutureFileUploadSource' => 'PhabricatorFileUploadSource',
+ 'PhabricatorExportEngine' => 'Phobject',
+ 'PhabricatorExportEngineBulkJobType' => 'PhabricatorWorkerSingleBulkJobType',
'PhabricatorExportEngineExtension' => 'Phobject',
'PhabricatorExportField' => 'Phobject',
'PhabricatorExportFormat' => 'Phobject',
@@ -10154,6 +10159,7 @@
'PhabricatorWorkerManagementWorkflow' => 'PhabricatorManagementWorkflow',
'PhabricatorWorkerPermanentFailureException' => 'Exception',
'PhabricatorWorkerSchemaSpec' => 'PhabricatorConfigSchemaSpec',
+ 'PhabricatorWorkerSingleBulkJobType' => 'PhabricatorWorkerBulkJobType',
'PhabricatorWorkerTask' => 'PhabricatorWorkerDAO',
'PhabricatorWorkerTaskData' => 'PhabricatorWorkerDAO',
'PhabricatorWorkerTaskDetailController' => 'PhabricatorDaemonController',
diff --git a/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php b/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php
--- a/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php
+++ b/src/applications/daemon/controller/PhabricatorDaemonBulkJobViewController.php
@@ -71,18 +71,10 @@
$viewer = $this->getViewer();
$curtain = $this->newCurtainView($job);
- if ($job->isConfirming()) {
- $continue_uri = $job->getMonitorURI();
- } else {
- $continue_uri = $job->getDoneURI();
+ foreach ($job->getCurtainActions($viewer) as $action) {
+ $curtain->addAction($action);
}
- $curtain->addAction(
- id(new PhabricatorActionView())
- ->setHref($continue_uri)
- ->setIcon('fa-arrow-circle-o-right')
- ->setName(pht('Continue')));
-
return $curtain;
}
diff --git a/src/applications/search/controller/PhabricatorApplicationSearchController.php b/src/applications/search/controller/PhabricatorApplicationSearchController.php
--- a/src/applications/search/controller/PhabricatorApplicationSearchController.php
+++ b/src/applications/search/controller/PhabricatorApplicationSearchController.php
@@ -444,6 +444,17 @@
$format_key = head_key($format_options);
}
+ // Check if this is a large result set or not. If we're exporting a
+ // large amount of data, we'll build the actual export file in the daemons.
+
+ $threshold = 1000;
+ $query = $engine->buildQueryFromSavedQuery($saved_query);
+ $pager = $engine->newPagerForSavedQuery($saved_query);
+ $pager->setPageSize($threshold + 1);
+ $objects = $engine->executeQuery($query, $pager);
+ $object_count = count($objects);
+ $is_large_export = ($object_count > $threshold);
+
$errors = array();
$e_format = null;
@@ -475,59 +486,31 @@
if (!$errors) {
$this->writeExportFormatPreference($format_key);
- $query = $engine->buildQueryFromSavedQuery($saved_query);
-
- // NOTE: We aren't reading the pager from the request. Exports always
- // affect the entire result set.
- $pager = $engine->newPagerForSavedQuery($saved_query);
- $pager->setPageSize(0x7FFFFFFF);
-
- $objects = $engine->executeQuery($query, $pager);
-
- $extension = $format->getFileExtension();
- $mime_type = $format->getMIMEContentType();
- $filename = $filename.'.'.$extension;
-
- $format = id(clone $format)
+ $export_engine = id(new PhabricatorExportEngine())
->setViewer($viewer)
- ->setTitle($sheet_title);
+ ->setSearchEngine($engine)
+ ->setSavedQuery($saved_query)
+ ->setTitle($sheet_title)
+ ->setFilename($filename)
+ ->setExportFormat($format);
- $export_data = $engine->newExport($objects);
- $objects = array_values($objects);
+ if ($is_large_export) {
+ $job = $export_engine->newBulkJob($request);
- $field_list = $engine->newExportFieldList();
- $field_list = mpull($field_list, null, 'getKey');
-
- $format->addHeaders($field_list);
- for ($ii = 0; $ii < count($objects); $ii++) {
- $format->addObject($objects[$ii], $field_list, $export_data[$ii]);
+ return id(new AphrontRedirectResponse())
+ ->setURI($job->getMonitorURI());
+ } else {
+ $file = $export_engine->exportFile();
+
+ return $this->newDialog()
+ ->setTitle(pht('Download Results'))
+ ->appendParagraph(
+ pht('Click the download button to download the exported data.'))
+ ->addCancelButton($cancel_uri, pht('Done'))
+ ->setSubmitURI($file->getDownloadURI())
+ ->setDisableWorkflowOnSubmit(true)
+ ->addSubmitButton(pht('Download Data'));
}
-
- $export_result = $format->newFileData();
-
- // We have all the data in one big string and aren't actually
- // streaming it, but pretending that we are allows us to actviate
- // the chunk engine and store large files.
- $iterator = new ArrayIterator(array($export_result));
-
- $source = id(new PhabricatorIteratorFileUploadSource())
- ->setName($filename)
- ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)
- ->setMIMEType($mime_type)
- ->setRelativeTTL(phutil_units('60 minutes in seconds'))
- ->setAuthorPHID($viewer->getPHID())
- ->setIterator($iterator);
-
- $file = $source->uploadFile();
-
- return $this->newDialog()
- ->setTitle(pht('Download Results'))
- ->appendParagraph(
- pht('Click the download button to download the exported data.'))
- ->addCancelButton($cancel_uri, pht('Done'))
- ->setSubmitURI($file->getDownloadURI())
- ->setDisableWorkflowOnSubmit(true)
- ->addSubmitButton(pht('Download Data'));
}
}
diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php
--- a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php
+++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobType.php
@@ -25,4 +25,24 @@
->execute();
}
+ public function getCurtainActions(
+ PhabricatorUser $viewer,
+ PhabricatorWorkerBulkJob $job) {
+
+ if ($job->isConfirming()) {
+ $continue_uri = $job->getMonitorURI();
+ } else {
+ $continue_uri = $job->getDoneURI();
+ }
+
+ $continue = id(new PhabricatorActionView())
+ ->setHref($continue_uri)
+ ->setIcon('fa-arrow-circle-o-right')
+ ->setName(pht('Continue'));
+
+ return array(
+ $continue,
+ );
+ }
+
}
diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php
--- a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php
+++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerBulkJobWorker.php
@@ -77,6 +77,10 @@
pht('Job actor does not have permission to edit job.'));
}
+ // Allow the worker to fill user caches inline; bulk jobs occasionally
+ // need to access user preferences.
+ $actor->setAllowInlineCacheGeneration(true);
+
return $actor;
}
diff --git a/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/daemon/workers/bulk/PhabricatorWorkerSingleBulkJobType.php
@@ -0,0 +1,27 @@
+<?php
+
+/**
+ * An bulk job which can not be parallelized and executes only one task.
+ */
+abstract class PhabricatorWorkerSingleBulkJobType
+ extends PhabricatorWorkerBulkJobType {
+
+ public function getDescriptionForConfirm(PhabricatorWorkerBulkJob $job) {
+ return null;
+ }
+
+ public function getJobSize(PhabricatorWorkerBulkJob $job) {
+ return 1;
+ }
+
+ public function createTasks(PhabricatorWorkerBulkJob $job) {
+ $tasks = array();
+
+ $tasks[] = PhabricatorWorkerBulkTask::initializeNewTask(
+ $job,
+ $job->getPHID());
+
+ return $tasks;
+ }
+
+}
diff --git a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php
--- a/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php
+++ b/src/infrastructure/daemon/workers/storage/PhabricatorWorkerBulkJob.php
@@ -180,6 +180,10 @@
return $this->getJobImplementation()->getJobName($this);
}
+ public function getCurtainActions(PhabricatorUser $viewer) {
+ return $this->getJobImplementation()->getCurtainActions($viewer, $this);
+ }
+
/* -( PhabricatorPolicyInterface )----------------------------------------- */
diff --git a/src/infrastructure/export/engine/PhabricatorExportEngine.php b/src/infrastructure/export/engine/PhabricatorExportEngine.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/export/engine/PhabricatorExportEngine.php
@@ -0,0 +1,168 @@
+<?php
+
+final class PhabricatorExportEngine
+ extends Phobject {
+
+ private $viewer;
+ private $searchEngine;
+ private $savedQuery;
+ private $exportFormat;
+ private $filename;
+ private $title;
+
+ public function setViewer(PhabricatorUser $viewer) {
+ $this->viewer = $viewer;
+ return $this;
+ }
+
+ public function getViewer() {
+ return $this->viewer;
+ }
+
+ public function setSearchEngine(
+ PhabricatorApplicationSearchEngine $search_engine) {
+ $this->searchEngine = $search_engine;
+ return $this;
+ }
+
+ public function getSearchEngine() {
+ return $this->searchEngine;
+ }
+
+ public function setSavedQuery(PhabricatorSavedQuery $saved_query) {
+ $this->savedQuery = $saved_query;
+ return $this;
+ }
+
+ public function getSavedQuery() {
+ return $this->savedQuery;
+ }
+
+ public function setExportFormat(
+ PhabricatorExportFormat $export_format) {
+ $this->exportFormat = $export_format;
+ return $this;
+ }
+
+ public function getExportFormat() {
+ return $this->exportFormat;
+ }
+
+ public function setFilename($filename) {
+ $this->filename = $filename;
+ return $this;
+ }
+
+ public function getFilename() {
+ return $this->filename;
+ }
+
+ public function setTitle($title) {
+ $this->title = $title;
+ return $this;
+ }
+
+ public function getTitle() {
+ return $this->title;
+ }
+
+ public function newBulkJob(AphrontRequest $request) {
+ $viewer = $this->getViewer();
+ $engine = $this->getSearchEngine();
+ $saved_query = $this->getSavedQuery();
+ $format = $this->getExportFormat();
+
+ $params = array(
+ 'engineClass' => get_class($engine),
+ 'queryKey' => $saved_query->getQueryKey(),
+ 'formatKey' => $format->getExportFormatKey(),
+ 'title' => $this->getTitle(),
+ 'filename' => $this->getFilename(),
+ );
+
+ $job = PhabricatorWorkerBulkJob::initializeNewJob(
+ $viewer,
+ new PhabricatorExportEngineBulkJobType(),
+ $params);
+
+ // We queue these jobs directly into STATUS_WAITING without requiring
+ // a confirmation from the user.
+
+ $xactions = array();
+
+ $xactions[] = id(new PhabricatorWorkerBulkJobTransaction())
+ ->setTransactionType(PhabricatorWorkerBulkJobTransaction::TYPE_STATUS)
+ ->setNewValue(PhabricatorWorkerBulkJob::STATUS_WAITING);
+
+ $editor = id(new PhabricatorWorkerBulkJobEditor())
+ ->setActor($viewer)
+ ->setContentSourceFromRequest($request)
+ ->setContinueOnMissingFields(true)
+ ->applyTransactions($job, $xactions);
+
+ return $job;
+ }
+
+ public function exportFile() {
+ $viewer = $this->getViewer();
+ $engine = $this->getSearchEngine();
+ $saved_query = $this->getSavedQuery();
+ $format = $this->getExportFormat();
+ $title = $this->getTitle();
+ $filename = $this->getFilename();
+
+ $query = $engine->buildQueryFromSavedQuery($saved_query);
+
+ $extension = $format->getFileExtension();
+ $mime_type = $format->getMIMEContentType();
+ $filename = $filename.'.'.$extension;
+
+ $format = id(clone $format)
+ ->setViewer($viewer)
+ ->setTitle($title);
+
+ $field_list = $engine->newExportFieldList();
+ $field_list = mpull($field_list, null, 'getKey');
+ $format->addHeaders($field_list);
+
+ // Iterate over the query results in large page so we don't have to hold
+ // too much stuff in memory.
+ $page_size = 1000;
+ $page_cursor = null;
+ do {
+ $pager = $engine->newPagerForSavedQuery($saved_query);
+ $pager->setPageSize($page_size);
+
+ if ($page_cursor !== null) {
+ $pager->setAfterID($page_cursor);
+ }
+
+ $objects = $engine->executeQuery($query, $pager);
+ $objects = array_values($objects);
+ $page_cursor = $pager->getNextPageID();
+
+ $export_data = $engine->newExport($objects);
+ for ($ii = 0; $ii < count($objects); $ii++) {
+ $format->addObject($objects[$ii], $field_list, $export_data[$ii]);
+ }
+ } while ($pager->getHasMoreResults());
+
+ $export_result = $format->newFileData();
+
+ // We have all the data in one big string and aren't actually
+ // streaming it, but pretending that we are allows us to actviate
+ // the chunk engine and store large files.
+ $iterator = new ArrayIterator(array($export_result));
+
+ $source = id(new PhabricatorIteratorFileUploadSource())
+ ->setName($filename)
+ ->setViewPolicy(PhabricatorPolicies::POLICY_NOONE)
+ ->setMIMEType($mime_type)
+ ->setRelativeTTL(phutil_units('60 minutes in seconds'))
+ ->setAuthorPHID($viewer->getPHID())
+ ->setIterator($iterator);
+
+ return $source->uploadFile();
+ }
+
+}
diff --git a/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php b/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php
new file mode 100644
--- /dev/null
+++ b/src/infrastructure/export/engine/PhabricatorExportEngineBulkJobType.php
@@ -0,0 +1,118 @@
+<?php
+
+final class PhabricatorExportEngineBulkJobType
+ extends PhabricatorWorkerSingleBulkJobType {
+
+ public function getBulkJobTypeKey() {
+ return 'export';
+ }
+
+ public function getJobName(PhabricatorWorkerBulkJob $job) {
+ return pht('Data Export');
+ }
+
+ public function getCurtainActions(
+ PhabricatorUser $viewer,
+ PhabricatorWorkerBulkJob $job) {
+ $actions = array();
+
+ $file_phid = $job->getParameter('filePHID');
+ if (!$file_phid) {
+ $actions[] = id(new PhabricatorActionView())
+ ->setHref('#')
+ ->setIcon('fa-download')
+ ->setDisabled(true)
+ ->setName(pht('Exporting Data...'));
+ } else {
+ $file = id(new PhabricatorFileQuery())
+ ->setViewer($viewer)
+ ->withPHIDs(array($file_phid))
+ ->executeOne();
+ if (!$file) {
+ $actions[] = id(new PhabricatorActionView())
+ ->setHref('#')
+ ->setIcon('fa-download')
+ ->setDisabled(true)
+ ->setName(pht('Temporary File Expired'));
+ } else {
+ $actions[] = id(new PhabricatorActionView())
+ ->setRenderAsForm(true)
+ ->setHref($file->getDownloadURI())
+ ->setIcon('fa-download')
+ ->setName(pht('Download Data Export'));
+ }
+ }
+
+ return $actions;
+ }
+
+
+ public function runTask(
+ PhabricatorUser $actor,
+ PhabricatorWorkerBulkJob $job,
+ PhabricatorWorkerBulkTask $task) {
+
+ $engine_class = $job->getParameter('engineClass');
+ if (!is_subclass_of($engine_class, 'PhabricatorApplicationSearchEngine')) {
+ throw new Exception(
+ pht(
+ 'Unknown search engine class "%s".',
+ $engine_class));
+ }
+
+ $engine = newv($engine_class, array())
+ ->setViewer($actor);
+
+ $query_key = $job->getParameter('queryKey');
+ if ($engine->isBuiltinQuery($query_key)) {
+ $saved_query = $engine->buildSavedQueryFromBuiltin($query_key);
+ } else if ($query_key) {
+ $saved_query = id(new PhabricatorSavedQueryQuery())
+ ->setViewer($actor)
+ ->withQueryKeys(array($query_key))
+ ->executeOne();
+ } else {
+ $saved_query = null;
+ }
+
+ if (!$saved_query) {
+ throw new Exception(
+ pht(
+ 'Failed to load saved query ("%s").',
+ $query_key));
+ }
+
+ $format_key = $job->getParameter('formatKey');
+
+ $all_formats = PhabricatorExportFormat::getAllExportFormats();
+ $format = idx($all_formats, $format_key);
+ if (!$format) {
+ throw new Exception(
+ pht(
+ 'Unknown export format ("%s").',
+ $format_key));
+ }
+
+ if (!$format->isExportFormatEnabled()) {
+ throw new Exception(
+ pht(
+ 'Export format ("%s") is not enabled.',
+ $format_key));
+ }
+
+ $export_engine = id(new PhabricatorExportEngine())
+ ->setViewer($actor)
+ ->setTitle($job->getParameter('title'))
+ ->setFilename($job->getParameter('filename'))
+ ->setSearchEngine($engine)
+ ->setSavedQuery($saved_query)
+ ->setExportFormat($format);
+
+ $file = $export_engine->exportFile();
+
+ $job
+ ->setParameter('filePHID', $file->getPHID())
+ ->save();
+ }
+
+}

File Metadata

Mime Type
text/plain
Expires
Sun, Mar 16, 11:15 PM (2 w, 4 d ago)
Storage Engine
blob
Storage Format
Encrypted (AES-256-CBC)
Storage Handle
7634196
Default Alt Text
D18962.diff (19 KB)

Event Timeline

Phabricator · Built by Phacility