diff --git a/src/__phutil_library_map__.php b/src/__phutil_library_map__.php --- a/src/__phutil_library_map__.php +++ b/src/__phutil_library_map__.php @@ -3389,6 +3389,7 @@ 'PhabricatorMacroViewController' => 'applications/macro/controller/PhabricatorMacroViewController.php', 'PhabricatorMailAttachment' => 'applications/metamta/message/PhabricatorMailAttachment.php', 'PhabricatorMailConfigTestCase' => 'applications/metamta/storage/__tests__/PhabricatorMailConfigTestCase.php', + 'PhabricatorMailEmailEngine' => 'applications/metamta/engine/PhabricatorMailEmailEngine.php', 'PhabricatorMailEmailHeraldField' => 'applications/metamta/herald/PhabricatorMailEmailHeraldField.php', 'PhabricatorMailEmailHeraldFieldGroup' => 'applications/metamta/herald/PhabricatorMailEmailHeraldFieldGroup.php', 'PhabricatorMailEmailMessage' => 'applications/metamta/message/PhabricatorMailEmailMessage.php', @@ -3414,6 +3415,7 @@ 'PhabricatorMailManagementUnverifyWorkflow' => 'applications/metamta/management/PhabricatorMailManagementUnverifyWorkflow.php', 'PhabricatorMailManagementVolumeWorkflow' => 'applications/metamta/management/PhabricatorMailManagementVolumeWorkflow.php', 'PhabricatorMailManagementWorkflow' => 'applications/metamta/management/PhabricatorMailManagementWorkflow.php', + 'PhabricatorMailMessageEngine' => 'applications/metamta/engine/PhabricatorMailMessageEngine.php', 'PhabricatorMailMustEncryptHeraldAction' => 'applications/metamta/herald/PhabricatorMailMustEncryptHeraldAction.php', 'PhabricatorMailOutboundMailHeraldAdapter' => 'applications/metamta/herald/PhabricatorMailOutboundMailHeraldAdapter.php', 'PhabricatorMailOutboundRoutingHeraldAction' => 'applications/metamta/herald/PhabricatorMailOutboundRoutingHeraldAction.php', @@ -9221,6 +9223,7 @@ 'PhabricatorMacroViewController' => 'PhabricatorMacroController', 'PhabricatorMailAttachment' => 'Phobject', 'PhabricatorMailConfigTestCase' => 'PhabricatorTestCase', + 'PhabricatorMailEmailEngine' => 'PhabricatorMailMessageEngine', 'PhabricatorMailEmailHeraldField' => 'HeraldField', 'PhabricatorMailEmailHeraldFieldGroup' => 'HeraldFieldGroup', 'PhabricatorMailEmailMessage' => 'PhabricatorMailExternalMessage', @@ -9246,6 +9249,7 @@ 'PhabricatorMailManagementUnverifyWorkflow' => 'PhabricatorMailManagementWorkflow', 'PhabricatorMailManagementVolumeWorkflow' => 'PhabricatorMailManagementWorkflow', 'PhabricatorMailManagementWorkflow' => 'PhabricatorManagementWorkflow', + 'PhabricatorMailMessageEngine' => 'Phobject', 'PhabricatorMailMustEncryptHeraldAction' => 'HeraldAction', 'PhabricatorMailOutboundMailHeraldAdapter' => 'HeraldAdapter', 'PhabricatorMailOutboundRoutingHeraldAction' => 'HeraldAction', diff --git a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php --- a/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php +++ b/src/applications/metamta/controller/PhabricatorMetaMTAMailViewController.php @@ -187,9 +187,6 @@ ->setStacked(true); $headers = $mail->getDeliveredHeaders(); - if ($headers === null) { - $headers = $mail->generateHeaders(); - } // Sort headers by name. $headers = isort($headers, 0); diff --git a/src/applications/metamta/engine/PhabricatorMailEmailEngine.php b/src/applications/metamta/engine/PhabricatorMailEmailEngine.php new file mode 100644 --- /dev/null +++ b/src/applications/metamta/engine/PhabricatorMailEmailEngine.php @@ -0,0 +1,649 @@ +getMailer(); + $mail = $this->getMail(); + + $message = new PhabricatorMailEmailMessage(); + + $from_address = $this->newFromEmailAddress(); + $message->setFromAddress($from_address); + + $reply_address = $this->newReplyToEmailAddress(); + if ($reply_address) { + $message->setReplyToAddress($reply_address); + } + + $to_addresses = $this->newToEmailAddresses(); + $cc_addresses = $this->newCCEmailAddresses(); + + if (!$to_addresses && !$cc_addresses) { + $mail->setMessage( + pht( + 'Message has no valid recipients: all To/CC are disabled, '. + 'invalid, or configured not to receive this mail.')); + return null; + } + + // If this email describes a mail processing error, we rate limit outbound + // messages to each individual address. This prevents messes where + // something is stuck in a loop or dumps a ton of messages on us suddenly. + if ($mail->getIsErrorEmail()) { + $all_recipients = array(); + foreach ($to_addresses as $to_address) { + $all_recipients[] = $to_address->getAddress(); + } + foreach ($cc_addresses as $cc_address) { + $all_recipients[] = $cc_address->getAddress(); + } + if ($this->shouldRateLimitMail($all_recipients)) { + $mail->setMessage( + pht( + 'This is an error email, but one or more recipients have '. + 'exceeded the error email rate limit. Declining to deliver '. + 'message.')); + return null; + } + } + + // Some mailers require a valid "To:" in order to deliver mail. If we + // don't have any "To:", try to fill it in with a placeholder "To:". + // If that also fails, move the "Cc:" line to "To:". + if (!$to_addresses) { + $void_address = $this->newVoidEmailAddress(); + $cc_addresses = $to_addresses; + $to_addresses = array($void_address); + } + + $to_addresses = $this->getUniqueEmailAddresses($to_addresses); + $cc_addresses = $this->getUniqueEmailAddresses( + $cc_addresses, + $to_addresses); + + $message->setToAddresses($to_addresses); + $message->setCCAddresses($cc_addresses); + + $attachments = $this->newEmailAttachments(); + $message->setAttachments($attachments); + + $subject = $this->newEmailSubject(); + $message->setSubject($subject); + + $headers = $this->newEmailHeaders(); + foreach ($this->newEmailThreadingHeaders($mailer) as $threading_header) { + $headers[] = $threading_header; + } + + $stamps = $mail->getMailStamps(); + if ($stamps) { + $headers[] = $this->newEmailHeader( + 'X-Phabricator-Stamps', + implode(' ', $stamps)); + } + + $must_encrypt = $mail->getMustEncrypt(); + + $raw_body = $mail->getBody(); + $body = $raw_body; + if ($must_encrypt) { + $parts = array(); + + $encrypt_uri = $this->getMustEncryptURI(); + if (!strlen($encrypt_uri)) { + $encrypt_phid = $this->getRelatedPHID(); + if ($encrypt_phid) { + $encrypt_uri = urisprintf( + '/object/%s/', + $encrypt_phid); + } + } + + if (strlen($encrypt_uri)) { + $parts[] = pht( + 'This secure message is notifying you of a change to this object:'); + $parts[] = PhabricatorEnv::getProductionURI($encrypt_uri); + } + + $parts[] = pht( + 'The content for this message can only be transmitted over a '. + 'secure channel. To view the message content, follow this '. + 'link:'); + + $parts[] = PhabricatorEnv::getProductionURI($this->getURI()); + + $body = implode("\n\n", $parts); + } else { + $body = $raw_body; + } + + $body_limit = PhabricatorEnv::getEnvConfig('metamta.email-body-limit'); + if (strlen($body) > $body_limit) { + $body = id(new PhutilUTF8StringTruncator()) + ->setMaximumBytes($body_limit) + ->truncateString($body); + $body .= "\n"; + $body .= pht('(This email was truncated at %d bytes.)', $body_limit); + } + $message->setTextBody($body); + $body_limit -= strlen($body); + + // If we sent a different message body than we were asked to, record + // what we actually sent to make debugging and diagnostics easier. + if ($body !== $raw_body) { + $mail->setDeliveredBody($body); + } + + if ($must_encrypt) { + $send_html = false; + } else { + $send_html = $this->shouldSendHTML(); + } + + if ($send_html) { + $html_body = $mail->getHTMLBody(); + if (strlen($html_body)) { + // NOTE: We just drop the entire HTML body if it won't fit. Safely + // truncating HTML is hard, and we already have the text body to fall + // back to. + if (strlen($html_body) <= $body_limit) { + $message->setHTMLBody($html_body); + $body_limit -= strlen($html_body); + } + } + } + + // Pass the headers to the mailer, then save the state so we can show + // them in the web UI. If the mail must be encrypted, we remove headers + // which are not on a strict whitelist to avoid disclosing information. + $filtered_headers = $this->filterHeaders($headers, $must_encrypt); + $message->setHeaders($filtered_headers); + + $mail->setUnfilteredHeaders($headers); + $mail->setDeliveredHeaders($headers); + + if (PhabricatorEnv::getEnvConfig('phabricator.silent')) { + $mail->setMessage( + pht( + 'Phabricator is running in silent mode. See `%s` '. + 'in the configuration to change this setting.', + 'phabricator.silent')); + + return null; + } + + return $message; + } + +/* -( Message Components )------------------------------------------------- */ + + private function newFromEmailAddress() { + $from_address = $this->newDefaultEmailAddress(); + $mail = $this->getMail(); + + // If the mail content must be encrypted, always disguise the sender. + $must_encrypt = $mail->getMustEncrypt(); + if ($must_encrypt) { + return $from_address; + } + + // If we have a raw "From" address, use that. + $raw_from = $mail->getRawFrom(); + if ($raw_from) { + list($from_email, $from_name) = $raw_from; + return $this->newEmailAddress($from_email, $from_name); + } + + // Otherwise, use as much of the information for any sending entity as + // we can. + $from_phid = $mail->getFrom(); + + $actor = $this->getActor($from_phid); + if ($actor) { + $actor_email = $actor->getEmailAddress(); + $actor_name = $actor->getName(); + } else { + $actor_email = null; + $actor_name = null; + } + + $send_as_user = PhabricatorEnv::getEnvConfig('metamta.can-send-as-user'); + if ($send_as_user) { + if ($actor_email !== null) { + $from_address->setAddress($actor_email); + } + } + + if ($actor_name !== null) { + $from_address->setDisplayName($actor_name); + } + + return $from_address; + } + + private function newReplyToEmailAddress() { + $mail = $this->getMail(); + + $reply_raw = $mail->getReplyTo(); + if (!strlen($reply_raw)) { + return null; + } + + $reply_address = new PhutilEmailAddress($reply_raw); + + // If we have a sending object, change the display name. + $from_phid = $mail->getFrom(); + $actor = $this->getActor($from_phid); + if ($actor) { + $reply_address->setDisplayName($actor->getName()); + } + + // If we don't have a display name, fill in a default. + if (!strlen($reply_address->getDisplayName())) { + $reply_address->setDisplayName(pht('Phabricator')); + } + + return $reply_address; + } + + private function newToEmailAddresses() { + $mail = $this->getMail(); + + $phids = $mail->getToPHIDs(); + $addresses = $this->newEmailAddressesFromActorPHIDs($phids); + + foreach ($mail->getRawToAddresses() as $raw_address) { + $addresses[] = new PhutilEmailAddress($raw_address); + } + + return $addresses; + } + + private function newCCEmailAddresses() { + $mail = $this->getMail(); + $phids = $mail->getCcPHIDs(); + return $this->newEmailAddressesFromActorPHIDs($phids); + } + + private function newEmailAddressesFromActorPHIDs(array $phids) { + $mail = $this->getMail(); + $phids = $mail->expandRecipients($phids); + + $addresses = array(); + foreach ($phids as $phid) { + $actor = $this->getActor($phid); + if (!$actor) { + continue; + } + + if (!$actor->isDeliverable()) { + continue; + } + + $addresses[] = new PhutilEmailAddress($actor->getEmailAddress()); + } + + return $addresses; + } + + private function newEmailSubject() { + $mail = $this->getMail(); + + $is_threaded = (bool)$mail->getThreadID(); + $must_encrypt = $mail->getMustEncrypt(); + + $subject = array(); + + if ($is_threaded) { + if ($this->shouldAddRePrefix()) { + $subject[] = 'Re:'; + } + } + + $subject[] = trim($mail->getSubjectPrefix()); + + // If mail content must be encrypted, we replace the subject with + // a generic one. + if ($must_encrypt) { + $encrypt_subject = $mail->getMustEncryptSubject(); + if (!strlen($encrypt_subject)) { + $encrypt_subject = pht('Object Updated'); + } + $subject[] = $encrypt_subject; + } else { + $vary_prefix = $mail->getVarySubjectPrefix(); + if (strlen($vary_prefix)) { + if ($this->shouldVarySubject()) { + $subject[] = $vary_prefix; + } + } + + $subject[] = $mail->getSubject(); + } + + foreach ($subject as $key => $part) { + if (!strlen($part)) { + unset($subject[$key]); + } + } + + $subject = implode(' ', $subject); + return $subject; + } + + private function newEmailHeaders() { + $mail = $this->getMail(); + + $headers = array(); + + $headers[] = $this->newEmailHeader( + 'X-Phabricator-Sent-This-Message', + 'Yes'); + $headers[] = $this->newEmailHeader( + 'X-Mail-Transport-Agent', + 'MetaMTA'); + + // Some clients respect this to suppress OOF and other auto-responses. + $headers[] = $this->newEmailHeader( + 'X-Auto-Response-Suppress', + 'All'); + + $mailtags = $mail->getMailTags(); + if ($mailtags) { + $tag_header = array(); + foreach ($mailtags as $mailtag) { + $tag_header[] = '<'.$mailtag.'>'; + } + $tag_header = implode(', ', $tag_header); + $headers[] = $this->newEmailHeader( + 'X-Phabricator-Mail-Tags', + $tag_header); + } + + $value = $mail->getHeaders(); + foreach ($value as $pair) { + list($header_key, $header_value) = $pair; + + // NOTE: If we have \n in a header, SES rejects the email. + $header_value = str_replace("\n", ' ', $header_value); + $headers[] = $this->newEmailHeader($header_key, $header_value); + } + + $is_bulk = $mail->getIsBulk(); + if ($is_bulk) { + $headers[] = $this->newEmailHeader('Precedence', 'bulk'); + } + + if ($mail->getMustEncrypt()) { + $headers[] = $this->newEmailHeader('X-Phabricator-Must-Encrypt', 'Yes'); + } + + $related_phid = $mail->getRelatedPHID(); + if ($related_phid) { + $headers[] = $this->newEmailHeader('Thread-Topic', $related_phid); + } + + $headers[] = $this->newEmailHeader( + 'X-Phabricator-Mail-ID', + $mail->getID()); + + $unique = Filesystem::readRandomCharacters(16); + $headers[] = $this->newEmailHeader( + 'X-Phabricator-Send-Attempt', + $unique); + + return $headers; + } + + private function newEmailThreadingHeaders() { + $mailer = $this->getMailer(); + $mail = $this->getMail(); + + $headers = array(); + + $thread_id = $mail->getThreadID(); + if (!strlen($thread_id)) { + return $headers; + } + + $is_first = $mail->getIsFirstMessage(); + + // NOTE: Gmail freaks out about In-Reply-To and References which aren't in + // the form ""; this is also required by RFC 2822, + // although some clients are more liberal in what they accept. + $domain = $this->newMailDomain(); + $thread_id = '<'.$thread_id.'@'.$domain.'>'; + + if ($is_first && $mailer->supportsMessageIDHeader()) { + $headers[] = $this->newEmailHeader('Message-ID', $thread_id); + } else { + $in_reply_to = $thread_id; + $references = array($thread_id); + $parent_id = $mail->getParentMessageID(); + if ($parent_id) { + $in_reply_to = $parent_id; + // By RFC 2822, the most immediate parent should appear last + // in the "References" header, so this order is intentional. + $references[] = $parent_id; + } + $references = implode(' ', $references); + $headers[] = $this->newEmailHeader('In-Reply-To', $in_reply_to); + $headers[] = $this->newEmailHeader('References', $references); + } + $thread_index = $this->generateThreadIndex($thread_id, $is_first); + $headers[] = $this->newEmailHeader('Thread-Index', $thread_index); + + return $headers; + } + + private function newEmailAttachments() { + $mail = $this->getMail(); + + // If the mail content must be encrypted, don't add attachments. + $must_encrypt = $mail->getMustEncrypt(); + if ($must_encrypt) { + return array(); + } + + return $mail->getAttachments(); + } + +/* -( Preferences )-------------------------------------------------------- */ + + private function shouldAddRePrefix() { + $preferences = $this->getPreferences(); + + $value = $preferences->getSettingValue( + PhabricatorEmailRePrefixSetting::SETTINGKEY); + + return ($value == PhabricatorEmailRePrefixSetting::VALUE_RE_PREFIX); + } + + private function shouldVarySubject() { + $preferences = $this->getPreferences(); + + $value = $preferences->getSettingValue( + PhabricatorEmailVarySubjectsSetting::SETTINGKEY); + + return ($value == PhabricatorEmailVarySubjectsSetting::VALUE_VARY_SUBJECTS); + } + + private function shouldSendHTML() { + $preferences = $this->getPreferences(); + + $value = $preferences->getSettingValue( + PhabricatorEmailFormatSetting::SETTINGKEY); + + return ($value == PhabricatorEmailFormatSetting::VALUE_HTML_EMAIL); + } + + +/* -( Utilities )---------------------------------------------------------- */ + + private function newEmailHeader($name, $value) { + return id(new PhabricatorMailHeader()) + ->setName($name) + ->setValue($value); + } + + private function newEmailAddress($address, $name = null) { + $object = id(new PhutilEmailAddress()) + ->setAddress($address); + + if (strlen($name)) { + $object->setDisplayName($name); + } + + return $object; + } + + public function newDefaultEmailAddress() { + $raw_address = PhabricatorEnv::getEnvConfig('metamta.default-address'); + + if (!strlen($raw_address)) { + $domain = $this->newMailDomain(); + $raw_address = "noreply@{$domain}"; + } + + $address = new PhutilEmailAddress($raw_address); + + if (!strlen($address->getDisplayName())) { + $address->setDisplayName(pht('Phabricator')); + } + + return $address; + } + + public function newVoidEmailAddress() { + return $this->newDefaultEmailAddress(); + } + + private function newMailDomain() { + $domain = PhabricatorEnv::getEnvConfig('metamta.reply-handler-domain'); + if (strlen($domain)) { + return $domain; + } + + $install_uri = PhabricatorEnv::getURI('/'); + $install_uri = new PhutilURI($install_uri); + + return $install_uri->getDomain(); + } + + private function filterHeaders(array $headers, $must_encrypt) { + assert_instances_of($headers, 'PhabricatorMailHeader'); + + if (!$must_encrypt) { + return $headers; + } + + $whitelist = array( + 'In-Reply-To', + 'Message-ID', + 'Precedence', + 'References', + 'Thread-Index', + 'Thread-Topic', + + 'X-Mail-Transport-Agent', + 'X-Auto-Response-Suppress', + + 'X-Phabricator-Sent-This-Message', + 'X-Phabricator-Must-Encrypt', + 'X-Phabricator-Mail-ID', + 'X-Phabricator-Send-Attempt', + ); + + // NOTE: The major header we want to drop is "X-Phabricator-Mail-Tags". + // This header contains a significant amount of meaningful information + // about the object. + + $whitelist_map = array(); + foreach ($whitelist as $term) { + $whitelist_map[phutil_utf8_strtolower($term)] = true; + } + + foreach ($headers as $key => $header) { + $name = $header->getName(); + $name = phutil_utf8_strtolower($name); + + if (!isset($whitelist_map[$name])) { + unset($headers[$key]); + } + } + + return $headers; + } + + private function getUniqueEmailAddresses( + array $addresses, + array $exclude = array()) { + assert_instances_of($addresses, 'PhutilEmailAddress'); + assert_instances_of($exclude, 'PhutilEmailAddress'); + + $seen = array(); + + foreach ($exclude as $address) { + $seen[$address->getAddress()] = true; + } + + foreach ($addresses as $key => $address) { + $raw_address = $address->getAddress(); + + if (isset($seen[$raw_address])) { + unset($addresses[$key]); + continue; + } + + $seen[$raw_address] = true; + } + + return array_values($addresses); + } + + private function generateThreadIndex($seed, $is_first_mail) { + // When threading, Outlook ignores the 'References' and 'In-Reply-To' + // headers that most clients use. Instead, it uses a custom 'Thread-Index' + // header. The format of this header is something like this (from + // camel-exchange-folder.c in Evolution Exchange): + + /* A new post to a folder gets a 27-byte-long thread index. (The value + * is apparently unique but meaningless.) Each reply to a post gets a + * 32-byte-long thread index whose first 27 bytes are the same as the + * parent's thread index. Each reply to any of those gets a + * 37-byte-long thread index, etc. The Thread-Index header contains a + * base64 representation of this value. + */ + + // The specific implementation uses a 27-byte header for the first email + // a recipient receives, and a random 5-byte suffix (32 bytes total) + // thereafter. This means that all the replies are (incorrectly) siblings, + // but it would be very difficult to keep track of the entire tree and this + // gets us reasonable client behavior. + + $base = substr(md5($seed), 0, 27); + if (!$is_first_mail) { + // Not totally sure, but it seems like outlook orders replies by + // thread-index rather than timestamp, so to get these to show up in the + // right order we use the time as the last 4 bytes. + $base .= ' '.pack('N', time()); + } + + return base64_encode($base); + } + + private function shouldRateLimitMail(array $all_recipients) { + try { + PhabricatorSystemActionEngine::willTakeAction( + $all_recipients, + new PhabricatorMetaMTAErrorMailAction(), + 1); + return false; + } catch (PhabricatorSystemActionRateLimitException $ex) { + return true; + } + } + +} diff --git a/src/applications/metamta/engine/PhabricatorMailMessageEngine.php b/src/applications/metamta/engine/PhabricatorMailMessageEngine.php new file mode 100644 --- /dev/null +++ b/src/applications/metamta/engine/PhabricatorMailMessageEngine.php @@ -0,0 +1,55 @@ +mailer = $mailer; + return $this; + } + + final public function getMailer() { + return $this->mailer; + } + + final public function setMail(PhabricatorMetaMTAMail $mail) { + $this->mail = $mail; + return $this; + } + + final public function getMail() { + return $this->mail; + } + + final public function setActors(array $actors) { + assert_instances_of($actors, 'PhabricatorMetaMTAActor'); + $this->actors = $actors; + return $this; + } + + final public function getActors() { + return $this->actors; + } + + final public function getActor($phid) { + return idx($this->actors, $phid); + } + + final public function setPreferences( + PhabricatorUserPreferences $preferences) { + $this->preferences = $preferences; + return $this; + } + + final public function getPreferences() { + return $this->preferences; + } + +} diff --git a/src/applications/metamta/management/PhabricatorMailManagementShowOutboundWorkflow.php b/src/applications/metamta/management/PhabricatorMailManagementShowOutboundWorkflow.php --- a/src/applications/metamta/management/PhabricatorMailManagementShowOutboundWorkflow.php +++ b/src/applications/metamta/management/PhabricatorMailManagementShowOutboundWorkflow.php @@ -116,10 +116,6 @@ $headers = $message->getDeliveredHeaders(); $unfiltered = $message->getUnfilteredHeaders(); - if (!$unfiltered) { - $headers = $message->generateHeaders(); - $unfiltered = $headers; - } $header_map = array(); foreach ($headers as $header) { diff --git a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php --- a/src/applications/metamta/storage/PhabricatorMetaMTAMail.php +++ b/src/applications/metamta/storage/PhabricatorMetaMTAMail.php @@ -191,13 +191,17 @@ return $this; } + public function getHeaders() { + return $this->getParam('headers', array()); + } + public function addAttachment(PhabricatorMailAttachment $attachment) { $this->parameters['attachments'][] = $attachment->toDictionary(); return $this; } public function getAttachments() { - $dicts = $this->getParam('attachments'); + $dicts = $this->getParam('attachments', array()); $result = array(); foreach ($dicts as $dict) { @@ -256,11 +260,19 @@ return $this; } + public function getRawFrom() { + return $this->getParam('raw-from'); + } + public function setReplyTo($reply_to) { $this->setParam('reply-to', $reply_to); return $this; } + public function getReplyTo() { + return $this->getParam('reply-to'); + } + public function setSubject($subject) { $this->setParam('subject', $subject); return $this; @@ -271,11 +283,19 @@ return $this; } + public function getSubjectPrefix() { + return $this->getParam('subject-prefix'); + } + public function setVarySubjectPrefix($prefix) { $this->setParam('vary-subject-prefix', $prefix); return $this; } + public function getVarySubjectPrefix() { + return $this->getParam('vary-subject-prefix'); + } + public function setBody($body) { $this->setParam('body', $body); return $this; @@ -413,6 +433,10 @@ return $this; } + public function getIsBulk() { + return $this->getParam('is-bulk'); + } + /** * Use this method to set an ID used for message threading. MetaMTA will * set appropriate headers (Message-ID, In-Reply-To, References and @@ -429,6 +453,14 @@ return $this; } + public function getThreadID() { + return $this->getParam('thread-id'); + } + + public function getIsFirstMessage() { + return (bool)$this->getParam('is-first-message'); + } + /** * Save a newly created mail to the database. The mail will eventually be * delivered by the MetaMTA daemon. @@ -597,10 +629,6 @@ } } - foreach ($sorted as $mailer) { - $mailer->prepareForSend(); - } - return $sorted; } @@ -627,36 +655,51 @@ ->save(); } - $exceptions = array(); - foreach ($mailers as $template_mailer) { - $mailer = null; + $actors = $this->loadAllActors(); + + // If we're sending one mail to everyone, some recipients will be in + // "Cc" rather than "To". We'll move them to "To" later (or supply a + // dummy "To") but need to look for the recipient in either the + // "To" or "Cc" fields here. + $target_phid = head($this->getToPHIDs()); + if (!$target_phid) { + $target_phid = head($this->getCcPHIDs()); + } + $preferences = $this->loadPreferences($target_phid); + // Attach any files we're about to send to this message, so the recipients + // can view them. + $viewer = PhabricatorUser::getOmnipotentUser(); + $files = $this->loadAttachedFiles($viewer); + foreach ($files as $file) { + $file->attachToObject($this->getPHID()); + } + + $exceptions = array(); + foreach ($mailers as $mailer) { try { - $mailer = $this->buildMailer($template_mailer); + $message = id(new PhabricatorMailEmailEngine()) + ->setMailer($mailer) + ->setMail($this) + ->setActors($actors) + ->setPreferences($preferences) + ->newMessage($mailer); } catch (Exception $ex) { $exceptions[] = $ex; continue; } - if (!$mailer) { - // If we don't get a mailer back, that means the mail doesn't - // actually need to be sent (for example, because recipients have - // declined to receive the mail). Void it and return. + if (!$message) { + // If we don't get a message back, that means the mail doesn't actually + // need to be sent (for example, because recipients have declined to + // receive the mail). Void it and return. return $this ->setStatus(PhabricatorMailOutboundStatus::STATUS_VOID) ->save(); } try { - $ok = $mailer->send(); - if (!$ok) { - // TODO: At some point, we should clean this up and make all mailers - // throw. - throw new Exception( - pht( - 'Mail adapter encountered an unexpected, unspecified '. - 'failure.')); - } + $mailer->sendMessage($message); } catch (PhabricatorMetaMTAPermanentFailureException $ex) { // If any mailer raises a permanent failure, stop trying to send the // mail with other mailers. @@ -677,6 +720,19 @@ $this->setParam('mailer.key', $mailer_key); } + // Now that we sent the message, store the final deliverability outcomes + // and reasoning so we can explain why things happened the way they did. + $actor_list = array(); + foreach ($actors as $actor) { + $actor_list[$actor->getPHID()] = array( + 'deliverable' => $actor->isDeliverable(), + 'reasons' => $actor->getDeliverabilityReasons(), + ); + } + $this->setParam('actors.sent', $actor_list); + $this->setParam('routing.sent', $this->getParam('routing')); + $this->setParam('routingmap.sent', $this->getRoutingRuleMap()); + return $this ->setStatus(PhabricatorMailOutboundStatus::STATUS_SENT) ->save(); @@ -705,368 +761,6 @@ $exceptions); } - private function buildMailer(PhabricatorMailImplementationAdapter $mailer) { - $headers = $this->generateHeaders(); - - $params = $this->parameters; - - $actors = $this->loadAllActors(); - $deliverable_actors = $this->filterDeliverableActors($actors); - - $default_from = (string)$this->newDefaultEmailAddress(); - if (empty($params['from'])) { - $mailer->setFrom($default_from); - } - - $is_first = idx($params, 'is-first-message'); - unset($params['is-first-message']); - - $is_threaded = (bool)idx($params, 'thread-id'); - $must_encrypt = $this->getMustEncrypt(); - - $reply_to_name = idx($params, 'reply-to-name', ''); - unset($params['reply-to-name']); - - $add_cc = array(); - $add_to = array(); - - // If we're sending one mail to everyone, some recipients will be in - // "Cc" rather than "To". We'll move them to "To" later (or supply a - // dummy "To") but need to look for the recipient in either the - // "To" or "Cc" fields here. - $target_phid = head(idx($params, 'to', array())); - if (!$target_phid) { - $target_phid = head(idx($params, 'cc', array())); - } - - $preferences = $this->loadPreferences($target_phid); - - foreach ($params as $key => $value) { - switch ($key) { - case 'raw-from': - list($from_email, $from_name) = $value; - $mailer->setFrom($from_email, $from_name); - break; - case 'from': - // If the mail content must be encrypted, disguise the sender. - if ($must_encrypt) { - $mailer->setFrom($default_from, pht('Phabricator')); - break; - } - - $from = $value; - $actor_email = null; - $actor_name = null; - $actor = idx($actors, $from); - if ($actor) { - $actor_email = $actor->getEmailAddress(); - $actor_name = $actor->getName(); - } - $can_send_as_user = $actor_email && - PhabricatorEnv::getEnvConfig('metamta.can-send-as-user'); - - if ($can_send_as_user) { - $mailer->setFrom($actor_email, $actor_name); - } else { - $from_email = coalesce($actor_email, $default_from); - $from_name = coalesce($actor_name, pht('Phabricator')); - - if (empty($params['reply-to'])) { - $params['reply-to'] = $from_email; - $params['reply-to-name'] = $from_name; - } - - $mailer->setFrom($default_from, $from_name); - } - break; - case 'reply-to': - $mailer->addReplyTo($value, $reply_to_name); - break; - case 'to': - $to_phids = $this->expandRecipients($value); - $to_actors = array_select_keys($deliverable_actors, $to_phids); - $add_to = array_merge( - $add_to, - mpull($to_actors, 'getEmailAddress')); - break; - case 'raw-to': - $add_to = array_merge($add_to, $value); - break; - case 'cc': - $cc_phids = $this->expandRecipients($value); - $cc_actors = array_select_keys($deliverable_actors, $cc_phids); - $add_cc = array_merge( - $add_cc, - mpull($cc_actors, 'getEmailAddress')); - break; - case 'attachments': - $attached_viewer = PhabricatorUser::getOmnipotentUser(); - $files = $this->loadAttachedFiles($attached_viewer); - foreach ($files as $file) { - $file->attachToObject($this->getPHID()); - } - - // If the mail content must be encrypted, don't add attachments. - if ($must_encrypt) { - break; - } - - $value = $this->getAttachments(); - foreach ($value as $attachment) { - $mailer->addAttachment( - $attachment->getData(), - $attachment->getFilename(), - $attachment->getMimeType()); - } - break; - case 'subject': - $subject = array(); - - if ($is_threaded) { - if ($this->shouldAddRePrefix($preferences)) { - $subject[] = 'Re:'; - } - } - - $subject[] = trim(idx($params, 'subject-prefix')); - - // If mail content must be encrypted, we replace the subject with - // a generic one. - if ($must_encrypt) { - $encrypt_subject = $this->getMustEncryptSubject(); - if (!strlen($encrypt_subject)) { - $encrypt_subject = pht('Object Updated'); - } - $subject[] = $encrypt_subject; - } else { - $vary_prefix = idx($params, 'vary-subject-prefix'); - if ($vary_prefix != '') { - if ($this->shouldVarySubject($preferences)) { - $subject[] = $vary_prefix; - } - } - - $subject[] = $value; - } - - $mailer->setSubject(implode(' ', array_filter($subject))); - break; - case 'thread-id': - - // NOTE: Gmail freaks out about In-Reply-To and References which - // aren't in the form ""; this is also required - // by RFC 2822, although some clients are more liberal in what they - // accept. - $domain = $this->newMailDomain(); - $value = '<'.$value.'@'.$domain.'>'; - - if ($is_first && $mailer->supportsMessageIDHeader()) { - $headers[] = array('Message-ID', $value); - } else { - $in_reply_to = $value; - $references = array($value); - $parent_id = $this->getParentMessageID(); - if ($parent_id) { - $in_reply_to = $parent_id; - // By RFC 2822, the most immediate parent should appear last - // in the "References" header, so this order is intentional. - $references[] = $parent_id; - } - $references = implode(' ', $references); - $headers[] = array('In-Reply-To', $in_reply_to); - $headers[] = array('References', $references); - } - $thread_index = $this->generateThreadIndex($value, $is_first); - $headers[] = array('Thread-Index', $thread_index); - break; - default: - // Other parameters are handled elsewhere or are not relevant to - // constructing the message. - break; - } - } - - $stamps = $this->getMailStamps(); - if ($stamps) { - $headers[] = array('X-Phabricator-Stamps', implode(' ', $stamps)); - } - - $raw_body = idx($params, 'body', ''); - $body = $raw_body; - if ($must_encrypt) { - $parts = array(); - - $encrypt_uri = $this->getMustEncryptURI(); - if (!strlen($encrypt_uri)) { - $encrypt_phid = $this->getRelatedPHID(); - if ($encrypt_phid) { - $encrypt_uri = urisprintf( - '/object/%s/', - $encrypt_phid); - } - } - - if (strlen($encrypt_uri)) { - $parts[] = pht( - 'This secure message is notifying you of a change to this object:'); - $parts[] = PhabricatorEnv::getProductionURI($encrypt_uri); - } - - $parts[] = pht( - 'The content for this message can only be transmitted over a '. - 'secure channel. To view the message content, follow this '. - 'link:'); - - $parts[] = PhabricatorEnv::getProductionURI($this->getURI()); - - $body = implode("\n\n", $parts); - } else { - $body = $raw_body; - } - - $body_limit = PhabricatorEnv::getEnvConfig('metamta.email-body-limit'); - if (strlen($body) > $body_limit) { - $body = id(new PhutilUTF8StringTruncator()) - ->setMaximumBytes($body_limit) - ->truncateString($body); - $body .= "\n"; - $body .= pht('(This email was truncated at %d bytes.)', $body_limit); - } - $mailer->setBody($body); - $body_limit -= strlen($body); - - // If we sent a different message body than we were asked to, record - // what we actually sent to make debugging and diagnostics easier. - if ($body !== $raw_body) { - $this->setParam('body.sent', $body); - } - - if ($must_encrypt) { - $send_html = false; - } else { - $send_html = $this->shouldSendHTML($preferences); - } - - if ($send_html) { - $html_body = idx($params, 'html-body'); - if (strlen($html_body)) { - // NOTE: We just drop the entire HTML body if it won't fit. Safely - // truncating HTML is hard, and we already have the text body to fall - // back to. - if (strlen($html_body) <= $body_limit) { - $mailer->setHTMLBody($html_body); - $body_limit -= strlen($html_body); - } - } - } - - // Pass the headers to the mailer, then save the state so we can show - // them in the web UI. If the mail must be encrypted, we remove headers - // which are not on a strict whitelist to avoid disclosing information. - $filtered_headers = $this->filterHeaders($headers, $must_encrypt); - foreach ($filtered_headers as $header) { - list($header_key, $header_value) = $header; - $mailer->addHeader($header_key, $header_value); - } - $this->setParam('headers.unfiltered', $headers); - $this->setParam('headers.sent', $filtered_headers); - - // Save the final deliverability outcomes and reasoning so we can - // explain why things happened the way they did. - $actor_list = array(); - foreach ($actors as $actor) { - $actor_list[$actor->getPHID()] = array( - 'deliverable' => $actor->isDeliverable(), - 'reasons' => $actor->getDeliverabilityReasons(), - ); - } - $this->setParam('actors.sent', $actor_list); - - $this->setParam('routing.sent', $this->getParam('routing')); - $this->setParam('routingmap.sent', $this->getRoutingRuleMap()); - - if (!$add_to && !$add_cc) { - $this->setMessage( - pht( - 'Message has no valid recipients: all To/Cc are disabled, '. - 'invalid, or configured not to receive this mail.')); - - return null; - } - - if ($this->getIsErrorEmail()) { - $all_recipients = array_merge($add_to, $add_cc); - if ($this->shouldRateLimitMail($all_recipients)) { - $this->setMessage( - pht( - 'This is an error email, but one or more recipients have '. - 'exceeded the error email rate limit. Declining to deliver '. - 'message.')); - - return null; - } - } - - if (PhabricatorEnv::getEnvConfig('phabricator.silent')) { - $this->setMessage( - pht( - 'Phabricator is running in silent mode. See `%s` '. - 'in the configuration to change this setting.', - 'phabricator.silent')); - - return null; - } - - // Some mailers require a valid "To:" in order to deliver mail. If we don't - // have any "To:", fill it in with a placeholder "To:". This allows client - // rules based on whether the recipient is in "To:" or "CC:" to continue - // behaving in the same way. - if (!$add_to) { - $void_recipient = $this->newVoidEmailAddress(); - $add_to = array($void_recipient->getAddress()); - } - - $add_to = array_unique($add_to); - $add_cc = array_diff(array_unique($add_cc), $add_to); - - $mailer->addTos($add_to); - if ($add_cc) { - $mailer->addCCs($add_cc); - } - - return $mailer; - } - - private function generateThreadIndex($seed, $is_first_mail) { - // When threading, Outlook ignores the 'References' and 'In-Reply-To' - // headers that most clients use. Instead, it uses a custom 'Thread-Index' - // header. The format of this header is something like this (from - // camel-exchange-folder.c in Evolution Exchange): - - /* A new post to a folder gets a 27-byte-long thread index. (The value - * is apparently unique but meaningless.) Each reply to a post gets a - * 32-byte-long thread index whose first 27 bytes are the same as the - * parent's thread index. Each reply to any of those gets a - * 37-byte-long thread index, etc. The Thread-Index header contains a - * base64 representation of this value. - */ - - // The specific implementation uses a 27-byte header for the first email - // a recipient receives, and a random 5-byte suffix (32 bytes total) - // thereafter. This means that all the replies are (incorrectly) siblings, - // but it would be very difficult to keep track of the entire tree and this - // gets us reasonable client behavior. - - $base = substr(md5($seed), 0, 27); - if (!$is_first_mail) { - // Not totally sure, but it seems like outlook orders replies by - // thread-index rather than timestamp, so to get these to show up in the - // right order we use the time as the last 4 bytes. - $base .= ' '.pack('N', time()); - } - - return base64_encode($base); - } public static function shouldMailEachRecipient() { return PhabricatorEnv::getEnvConfig('metamta.one-mail-per-recipient'); @@ -1120,7 +814,7 @@ * recipients. * @return list Deaggregated list of mailable recipients. */ - private function expandRecipients(array $phids) { + public function expandRecipients(array $phids) { if ($this->recipientExpansionMap === null) { $all_phids = $this->getAllActorPHIDs(); $this->recipientExpansionMap = id(new PhabricatorMetaMTAMemberQuery()) @@ -1320,72 +1014,15 @@ return $actors; } - private function shouldRateLimitMail(array $all_recipients) { - try { - PhabricatorSystemActionEngine::willTakeAction( - $all_recipients, - new PhabricatorMetaMTAErrorMailAction(), - 1); - return false; - } catch (PhabricatorSystemActionRateLimitException $ex) { - return true; - } - } - - public function generateHeaders() { - $headers = array(); - - $headers[] = array('X-Phabricator-Sent-This-Message', 'Yes'); - $headers[] = array('X-Mail-Transport-Agent', 'MetaMTA'); - - // Some clients respect this to suppress OOF and other auto-responses. - $headers[] = array('X-Auto-Response-Suppress', 'All'); - - $mailtags = $this->getParam('mailtags'); - if ($mailtags) { - $tag_header = array(); - foreach ($mailtags as $mailtag) { - $tag_header[] = '<'.$mailtag.'>'; - } - $tag_header = implode(', ', $tag_header); - $headers[] = array('X-Phabricator-Mail-Tags', $tag_header); - } - - $value = $this->getParam('headers', array()); - foreach ($value as $pair) { - list($header_key, $header_value) = $pair; - - // NOTE: If we have \n in a header, SES rejects the email. - $header_value = str_replace("\n", ' ', $header_value); - $headers[] = array($header_key, $header_value); - } - - $is_bulk = $this->getParam('is-bulk'); - if ($is_bulk) { - $headers[] = array('Precedence', 'bulk'); - } - - if ($this->getMustEncrypt()) { - $headers[] = array('X-Phabricator-Must-Encrypt', 'Yes'); - } - - $related_phid = $this->getRelatedPHID(); - if ($related_phid) { - $headers[] = array('Thread-Topic', $related_phid); - } - - $headers[] = array('X-Phabricator-Mail-ID', $this->getID()); - - $unique = Filesystem::readRandomCharacters(16); - $headers[] = array('X-Phabricator-Send-Attempt', $unique); - - return $headers; - } - public function getDeliveredHeaders() { return $this->getParam('headers.sent'); } + public function setDeliveredHeaders(array $headers) { + $headers = $this->flattenHeaders($headers); + return $this->setParam('headers.sent', $headers); + } + public function getUnfilteredHeaders() { $unfiltered = $this->getParam('headers.unfiltered'); @@ -1399,6 +1036,25 @@ return $unfiltered; } + public function setUnfilteredHeaders(array $headers) { + $headers = $this->flattenHeaders($headers); + return $this->setParam('headers.unfiltered', $headers); + } + + private function flattenHeaders(array $headers) { + assert_instances_of($headers, 'PhabricatorMailHeader'); + + $list = array(); + foreach ($list as $header) { + $list[] = array( + $header->getName(), + $header->getValue(), + ); + } + + return $list; + } + public function getDeliveredActors() { return $this->getParam('actors.sent'); } @@ -1415,81 +1071,14 @@ return $this->getParam('body.sent'); } - private function filterHeaders(array $headers, $must_encrypt) { - if (!$must_encrypt) { - return $headers; - } - - $whitelist = array( - 'In-Reply-To', - 'Message-ID', - 'Precedence', - 'References', - 'Thread-Index', - 'Thread-Topic', - - 'X-Mail-Transport-Agent', - 'X-Auto-Response-Suppress', - - 'X-Phabricator-Sent-This-Message', - 'X-Phabricator-Must-Encrypt', - 'X-Phabricator-Mail-ID', - 'X-Phabricator-Send-Attempt', - ); - - // NOTE: The major header we want to drop is "X-Phabricator-Mail-Tags". - // This header contains a significant amount of meaningful information - // about the object. - - $whitelist_map = array(); - foreach ($whitelist as $term) { - $whitelist_map[phutil_utf8_strtolower($term)] = true; - } - - foreach ($headers as $key => $header) { - list($name, $value) = $header; - $name = phutil_utf8_strtolower($name); - - if (!isset($whitelist_map[$name])) { - unset($headers[$key]); - } - } - - return $headers; + public function setDeliveredBody($body) { + return $this->setParam('body.sent', $body); } public function getURI() { return '/mail/detail/'.$this->getID().'/'; } - private function newMailDomain() { - $domain = PhabricatorEnv::getEnvConfig('metamta.reply-handler-domain'); - if (strlen($domain)) { - return $domain; - } - - $install_uri = PhabricatorEnv::getURI('/'); - $install_uri = new PhutilURI($install_uri); - - return $install_uri->getDomain(); - } - - public function newDefaultEmailAddress() { - $raw_address = PhabricatorEnv::getEnvConfig('metamta.default-address'); - if (strlen($raw_address)) { - return new PhutilEmailAddress($raw_address); - } - - $domain = $this->newMailDomain(); - $address = "noreply@{$domain}"; - - return new PhutilEmailAddress($address); - } - - public function newVoidEmailAddress() { - return $this->newDefaultEmailAddress(); - } - /* -( Routing )------------------------------------------------------------ */ @@ -1578,27 +1167,6 @@ return PhabricatorUserPreferences::loadGlobalPreferences($viewer); } - private function shouldAddRePrefix(PhabricatorUserPreferences $preferences) { - $value = $preferences->getSettingValue( - PhabricatorEmailRePrefixSetting::SETTINGKEY); - - return ($value == PhabricatorEmailRePrefixSetting::VALUE_RE_PREFIX); - } - - private function shouldVarySubject(PhabricatorUserPreferences $preferences) { - $value = $preferences->getSettingValue( - PhabricatorEmailVarySubjectsSetting::SETTINGKEY); - - return ($value == PhabricatorEmailVarySubjectsSetting::VALUE_VARY_SUBJECTS); - } - - private function shouldSendHTML(PhabricatorUserPreferences $preferences) { - $value = $preferences->getSettingValue( - PhabricatorEmailFormatSetting::SETTINGKEY); - - return ($value == PhabricatorEmailFormatSetting::VALUE_HTML_EMAIL); - } - public function shouldRenderMailStampsInBody($viewer) { $preferences = $this->loadPreferences($viewer->getPHID()); $value = $preferences->getSettingValue(