diff --git a/src/applications/conpherence/editor/ConpherenceEditor.php b/src/applications/conpherence/editor/ConpherenceEditor.php
--- a/src/applications/conpherence/editor/ConpherenceEditor.php
+++ b/src/applications/conpherence/editor/ConpherenceEditor.php
@@ -235,6 +235,7 @@
     PhabricatorLiskDAO $object,
     PhabricatorApplicationTransaction $xaction) {
 
+    $make_author_recent_participant = true;
     switch ($xaction->getTransactionType()) {
       case PhabricatorTransactions::TYPE_COMMENT:
         $object->setMessageCount((int)$object->getMessageCount() + 1);
@@ -244,23 +245,38 @@
         break;
       case ConpherenceTransactionType::TYPE_PARTICIPANTS:
         if (!$this->getIsNewObject()) {
-          // if we added people, add them to the end of "recent" participants
           $old_map = array_fuse($xaction->getOldValue());
           $new_map = array_fuse($xaction->getNewValue());
+          // if we added people, add them to the end of "recent" participants
           $add = array_keys(array_diff_key($new_map, $old_map));
-          if ($add) {
+          // if we remove people, then definintely remove them from "recent"
+          // participants
+          $del = array_keys(array_diff_key($old_map, $new_map));
+          if ($add || $del) {
             $participants = $object->getRecentParticipantPHIDs();
-            $participants = array_merge($participants, $add);
+            if ($add) {
+              $participants = array_merge($participants, $add);
+            }
+            if ($del) {
+              $participants = array_diff($participants, $del);
+              $actor = $this->requireActor();
+              if (in_array($actor->getPHID(), $del)) {
+                $make_author_recent_participant = false;
+              }
+            }
             $participants = array_slice(array_unique($participants), 0, 10);
             $object->setRecentParticipantPHIDs($participants);
           }
         }
         break;
     }
-    $this->updateRecentParticipantPHIDs($object, $xaction);
+
+    if ($make_author_recent_participant) {
+      $this->makeAuthorMostRecentParticipant($object, $xaction);
+    }
   }
 
-  private function updateRecentParticipantPHIDs(
+  private function makeAuthorMostRecentParticipant(
     PhabricatorLiskDAO $object,
     PhabricatorApplicationTransaction $xaction) {