diff --git a/src/applications/aphlict/management/PhabricatorAphlictManagementWorkflow.php b/src/applications/aphlict/management/PhabricatorAphlictManagementWorkflow.php index 8d8e1edf77..c7213c1cad 100644 --- a/src/applications/aphlict/management/PhabricatorAphlictManagementWorkflow.php +++ b/src/applications/aphlict/management/PhabricatorAphlictManagementWorkflow.php @@ -1,470 +1,519 @@ debug = $debug; return $this; } protected function getLaunchArguments() { return array( array( 'name' => 'config', 'param' => 'file', 'help' => pht( 'Use a specific configuration file instead of the default '. 'configuration.'), ), ); } protected function parseLaunchArguments(PhutilArgumentParser $args) { $config_file = $args->getArg('config'); if ($config_file) { $full_path = Filesystem::resolvePath($config_file); $show_path = $full_path; } else { $root = dirname(dirname(phutil_get_library_root('phabricator'))); $try = array( 'phabricator/conf/aphlict/aphlict.custom.json', 'phabricator/conf/aphlict/aphlict.default.json', ); foreach ($try as $config) { $full_path = $root.'/'.$config; $show_path = $config; if (Filesystem::pathExists($full_path)) { break; } } } echo tsprintf( "%s\n", pht( 'Reading configuration from: %s', $show_path)); try { $data = Filesystem::readFile($full_path); } catch (Exception $ex) { throw new PhutilArgumentUsageException( pht( 'Failed to read configuration file. %s', $ex->getMessage())); } try { $data = phutil_json_decode($data); } catch (Exception $ex) { throw new PhutilArgumentUsageException( pht( 'Configuration file is not properly formatted JSON. %s', $ex->getMessage())); } try { PhutilTypeSpec::checkMap( $data, array( 'servers' => 'list', 'logs' => 'optional list', + 'cluster' => 'optional list', 'pidfile' => 'string', )); } catch (Exception $ex) { throw new PhutilArgumentUsageException( pht( 'Configuration file has improper configuration keys at top '. 'level. %s', $ex->getMessage())); } $servers = $data['servers']; $has_client = false; $has_admin = false; $port_map = array(); foreach ($servers as $index => $server) { PhutilTypeSpec::checkMap( $server, array( 'type' => 'string', 'port' => 'int', 'listen' => 'optional string|null', 'ssl.key' => 'optional string|null', 'ssl.cert' => 'optional string|null', 'ssl.chain' => 'optional string|null', )); $port = $server['port']; if (!isset($port_map[$port])) { $port_map[$port] = $index; } else { throw new PhutilArgumentUsageException( pht( 'Two servers (at indexes "%s" and "%s") both bind to the same '. 'port ("%s"). Each server must bind to a unique port.', $port_map[$port], $index, $port)); } $type = $server['type']; switch ($type) { case 'admin': $has_admin = true; break; case 'client': $has_client = true; break; default: throw new PhutilArgumentUsageException( pht( 'A specified server (at index "%s", on port "%s") has an '. 'invalid type ("%s"). Valid types are: admin, client.', $index, $port, $type)); } $ssl_key = idx($server, 'ssl.key'); $ssl_cert = idx($server, 'ssl.cert'); if (($ssl_key && !$ssl_cert) || ($ssl_cert && !$ssl_key)) { throw new PhutilArgumentUsageException( pht( 'A specified server (at index "%s", on port "%s") specifies '. 'only one of "%s" and "%s". Each server must specify neither '. '(to disable SSL) or specify both (to enable it).', $index, $port, 'ssl.key', 'ssl.cert')); } $ssl_chain = idx($server, 'ssl.chain'); if ($ssl_chain && (!$ssl_key && !$ssl_cert)) { throw new PhutilArgumentUsageException( pht( 'A specified server (at index "%s", on port "%s") specifies '. 'a value for "%s", but no value for "%s" or "%s". Servers '. 'should only provide an SSL chain if they also provide an SSL '. 'key and SSL certificate.', $index, $port, 'ssl.chain', 'ssl.key', 'ssl.cert')); } } if (!$servers) { throw new PhutilArgumentUsageException( pht( 'Configuration file does not specify any servers. This service '. 'will not be able to interact with the outside world if it does '. 'not listen on any ports. You must specify at least one "%s" '. 'server and at least one "%s" server.', 'admin', 'client')); } if (!$has_client) { throw new PhutilArgumentUsageException( pht( 'Configuration file does not specify any client servers. This '. 'service will be unable to transmit any notifications without a '. 'client server. You must specify at least one server with '. 'type "%s".', 'client')); } if (!$has_admin) { throw new PhutilArgumentUsageException( pht( 'Configuration file does not specify any administrative '. 'servers. This service will be unable to receive messages. '. 'You must specify at least one server with type "%s".', 'admin')); } - $logs = $data['logs']; + $logs = idx($data, 'logs', array()); foreach ($logs as $index => $log) { PhutilTypeSpec::checkMap( $log, array( 'path' => 'string', )); $path = $log['path']; try { $dir = dirname($path); if (!Filesystem::pathExists($dir)) { Filesystem::createDirectory($dir, 0755, true); } } catch (FilesystemException $ex) { throw new PhutilArgumentUsageException( pht( 'Failed to create directory "%s" for specified log file (with '. 'index "%s"). You should manually create this directory or '. 'choose a different logfile location. %s', $dir, $ex->getMessage())); } } + $peer_map = array(); + + $cluster = idx($data, 'cluster', array()); + foreach ($cluster as $index => $peer) { + PhutilTypeSpec::checkMap( + $peer, + array( + 'host' => 'string', + 'port' => 'int', + 'protocol' => 'string', + )); + + $host = $peer['host']; + $port = $peer['port']; + $protocol = $peer['protocol']; + + switch ($protocol) { + case 'http': + case 'https': + break; + default: + throw new PhutilArgumentUsageException( + pht( + 'Configuration file specifies cluster peer ("%s", at index '. + '"%s") with an invalid protocol, "%s". Valid protocols are '. + '"%s" or "%s".', + $host, + $index, + $protocol, + 'http', + 'https')); + } + + $peer_key = "{$host}:{$port}"; + if (!isset($peer_map[$peer_key])) { + $peer_map[$peer_key] = $index; + } else { + throw new PhutilArgumentUsageException( + pht( + 'Configuration file specifies cluster peer "%s" more than '. + 'once (at indexes "%s" and "%s"). Each peer must have a '. + 'unique host and port combination.', + $peer_key, + $peer_map[$peer_key], + $index)); + } + } + $this->configData = $data; $this->configPath = $full_path; $pid_path = $this->getPIDPath(); try { $dir = dirname($path); if (!Filesystem::pathExists($dir)) { Filesystem::createDirectory($dir, 0755, true); } } catch (FilesystemException $ex) { throw new PhutilArgumentUsageException( pht( 'Failed to create directory "%s" for specified PID file. You '. 'should manually create this directory or choose a different '. 'PID file location. %s', $dir, $ex->getMessage())); } } final public function getPIDPath() { return $this->configData['pidfile']; } final public function getPID() { $pid = null; if (Filesystem::pathExists($this->getPIDPath())) { $pid = (int)Filesystem::readFile($this->getPIDPath()); } return $pid; } final public function cleanup($signo = '?') { global $g_future; if ($g_future) { $g_future->resolveKill(); $g_future = null; } Filesystem::remove($this->getPIDPath()); exit(1); } public static function requireExtensions() { self::mustHaveExtension('pcntl'); self::mustHaveExtension('posix'); } private static function mustHaveExtension($ext) { if (!extension_loaded($ext)) { echo pht( "ERROR: The PHP extension '%s' is not installed. You must ". "install it to run Aphlict on this machine.", $ext)."\n"; exit(1); } $extension = new ReflectionExtension($ext); foreach ($extension->getFunctions() as $function) { $function = $function->name; if (!function_exists($function)) { echo pht( 'ERROR: The PHP function %s is disabled. You must '. 'enable it to run Aphlict on this machine.', $function.'()')."\n"; exit(1); } } } final protected function willLaunch() { $console = PhutilConsole::getConsole(); $pid = $this->getPID(); if ($pid) { throw new PhutilArgumentUsageException( pht( 'Unable to start notifications server because it is already '. 'running. Use `%s` to restart it.', 'aphlict restart')); } if (posix_getuid() == 0) { throw new PhutilArgumentUsageException( pht('The notification server should not be run as root.')); } // Make sure we can write to the PID file. if (!$this->debug) { Filesystem::writeFile($this->getPIDPath(), ''); } // First, start the server in configuration test mode with --test. This // will let us error explicitly if there are missing modules, before we // fork and lose access to the console. $test_argv = $this->getServerArgv(); $test_argv[] = '--test=true'; execx('%C', $this->getStartCommand($test_argv)); } private function getServerArgv() { $server_argv = array(); $server_argv[] = '--config='.$this->configPath; return $server_argv; } final protected function launch() { $console = PhutilConsole::getConsole(); if ($this->debug) { $console->writeOut( "%s\n", pht('Starting Aphlict server in foreground...')); } else { Filesystem::writeFile($this->getPIDPath(), getmypid()); } $command = $this->getStartCommand($this->getServerArgv()); if (!$this->debug) { declare(ticks = 1); pcntl_signal(SIGINT, array($this, 'cleanup')); pcntl_signal(SIGTERM, array($this, 'cleanup')); } register_shutdown_function(array($this, 'cleanup')); if ($this->debug) { $console->writeOut( "%s\n\n $ %s\n\n", pht('Launching server:'), $command); $err = phutil_passthru('%C', $command); $console->writeOut(">>> %s\n", pht('Server exited!')); exit($err); } else { while (true) { global $g_future; $g_future = new ExecFuture('exec %C', $command); $g_future->resolve(); // If the server exited, wait a couple of seconds and restart it. unset($g_future); sleep(2); } } } /* -( Commands )----------------------------------------------------------- */ final protected function executeStartCommand() { $console = PhutilConsole::getConsole(); $this->willLaunch(); $pid = pcntl_fork(); if ($pid < 0) { throw new Exception( pht( 'Failed to %s!', 'fork()')); } else if ($pid) { $console->writeErr("%s\n", pht('Aphlict Server started.')); exit(0); } // When we fork, the child process will inherit its parent's set of open // file descriptors. If the parent process of bin/aphlict is waiting for // bin/aphlict's file descriptors to close, it will be stuck waiting on // the daemonized process. (This happens if e.g. bin/aphlict is started // in another script using passthru().) fclose(STDOUT); fclose(STDERR); $this->launch(); return 0; } final protected function executeStopCommand() { $console = PhutilConsole::getConsole(); $pid = $this->getPID(); if (!$pid) { $console->writeErr("%s\n", pht('Aphlict is not running.')); return 0; } $console->writeErr("%s\n", pht('Stopping Aphlict Server (%s)...', $pid)); posix_kill($pid, SIGINT); $start = time(); do { if (!PhabricatorDaemonReference::isProcessRunning($pid)) { $console->writeOut( "%s\n", pht('Aphlict Server (%s) exited normally.', $pid)); $pid = null; break; } usleep(100000); } while (time() < $start + 5); if ($pid) { $console->writeErr("%s\n", pht('Sending %s a SIGKILL.', $pid)); posix_kill($pid, SIGKILL); unset($pid); } Filesystem::remove($this->getPIDPath()); return 0; } private function getNodeBinary() { if (Filesystem::binaryExists('nodejs')) { return 'nodejs'; } if (Filesystem::binaryExists('node')) { return 'node'; } throw new PhutilArgumentUsageException( pht( 'No `%s` or `%s` binary was found in %s. You must install '. 'Node.js to start the Aphlict server.', 'nodejs', 'node', '$PATH')); } private function getAphlictScriptPath() { $root = dirname(phutil_get_library_root('phabricator')); return $root.'/support/aphlict/server/aphlict_server.js'; } private function getStartCommand(array $server_argv) { return csprintf( '%s %s %Ls', $this->getNodeBinary(), $this->getAphlictScriptPath(), $server_argv); } } diff --git a/src/applications/notification/client/PhabricatorNotificationClient.php b/src/applications/notification/client/PhabricatorNotificationClient.php index 292fda7f49..ae8ac7eb34 100644 --- a/src/applications/notification/client/PhabricatorNotificationClient.php +++ b/src/applications/notification/client/PhabricatorNotificationClient.php @@ -1,32 +1,35 @@ loadServerStatus(); return; } return; } public static function tryToPostMessage(array $data) { $servers = PhabricatorNotificationServerRef::getEnabledAdminServers(); + + shuffle($servers); + foreach ($servers as $server) { try { $server->postMessage($data); return; } catch (Exception $ex) { // Just ignore any issues here. } } } } diff --git a/src/docs/user/cluster/cluster.diviner b/src/docs/user/cluster/cluster.diviner index 5cb1a2671e..0180c7ff42 100644 --- a/src/docs/user/cluster/cluster.diviner +++ b/src/docs/user/cluster/cluster.diviner @@ -1,251 +1,289 @@ @title Clustering Introduction @group cluster Guide to configuring Phabricator across multiple hosts for availability and performance. Overview ======== WARNING: This feature is a very early prototype; the features this document describes are mostly speculative fantasy. Phabricator can be configured to run on multiple hosts with redundant services to improve its availability and scalability, and make disaster recovery much easier. Clustering is more complex to setup and maintain than running everything on a single host, but greatly reduces the cost of recovering from hardware and network failures. Each Phabricator service has an array of clustering options that can be configured independently. Configuring a cluster is inherently complex, and this is an advanced feature aimed at installs with large userbases and experienced operations personnel who need this high degree of flexibility. The remainder of this document summarizes how to add redundancy to each service and where your efforts are likely to have the greatest impact. For additional guidance on setting up a cluster, see "Overlaying Services" and "Cluster Recipes" at the bottom of this document. Preparing for Clustering ======================== To begin deploying Phabricator in cluster mode, set up `cluster.addresses` in your configuration. This option should contain a list of network address blocks which are considered to be part of the cluster. Hosts in this list are allowed to bend (or even break) some of the security and policy rules when they make requests to other hosts in the cluster, so this list should be as small as possible. See "Cluster Whitelist Security" below for discussion. If you are deploying hardware in EC2, a reasonable approach is to launch a dedicated Phabricator VPC, whitelist the whole VPC as a Phabricator cluster, and then deploy only Phabricator services into that VPC. If you have additional auxiliary hosts which run builds and tests via Drydock, you should //not// include them in the cluster address definition. For more detailed discussion of the Drydock security model, see @{Drydock User Guide: Security}. Most other clustering features will not work until you define a cluster by configuring `cluster.addresses`. Cluster Whitelist Security ======================== When you configure `cluster.addresses`, you should keep the list of trusted cluster hosts as small as possible. Hosts on this list gain additional capabilities, including these: **Trusted HTTP Headers**: Normally, Phabricator distrusts the load balancer HTTP headers `X-Forwarded-For` and `X-Forwarded-Proto` because they may be client-controlled and can be set to arbitrary values by an attacker if no load balancer is deployed. In particular, clients can set `X-Forwarded-For` to any value and spoof traffic from arbitrary remotes. These headers are trusted when they are received from a host on the cluster address whitelist. This allows requests from cluster loadbalancers to be interpreted correctly by default without requiring additional custom code or configuration. **Intracluster HTTP**: Requests from cluster hosts are not required to use HTTPS, even if `security.require-https` is enabled, because it is common to terminate HTTPS on load balancers and use plain HTTP for requests within a cluster. **Special Authentication Mechanisms**: Cluster hosts are allowed to connect to other cluster hosts with "root credentials", and to impersonate any user account. The use of root credentials is required because the daemons must be able to bypass policies in order to function properly: they need to send mail about private conversations and import commits in private repositories. The ability to impersonate users is required because SSH nodes must receive, interpret, modify, and forward SSH traffic. They can not use the original credentials to do this because SSH authentication is asymmetric and they do not have the user's private key. Instead, they use root credentials and impersonate the user within the cluster. These mechanisms are still authenticated (and use asymmetric keys, like SSH does), so access to a host in the cluster address block does not mean that an attacker can immediately compromise the cluster. However, an over-broad cluster address whitelist may give an attacker who gains some access additional tools to escalate access. Note that if an attacker gains access to an actual cluster host, these extra powers are largely moot. Most cluster hosts must be able to connect to the master database to function properly, so the attacker will just do that and freely read or modify whatever data they want. Cluster: Databases ================= Configuring multiple database hosts is moderately complex, but normally has the highest impact on availability and resistance to data loss. This is usually the most important service to make redundant if your focus is on availability and disaster recovery. Configuring replicas allows Phabricator to run in read-only mode if you lose the master and to quickly promote the replica as a replacement. For details, see @{article:Cluster: Databases}. Cluster: Repositories ===================== Configuring multiple repository hosts is complex, but is required before you can add multiple daemon or web hosts. Repository replicas are important for availability if you host repositories on Phabricator, but less important if you host repositories elsewhere (instead, you should focus on making that service more available). The distributed nature of Git and Mercurial tend to mean that they are naturally somewhat resistant to data loss: every clone of a repository includes the entire history. +Repositories may become a scalability bottleneck, although this is rare unless +your install has an unusually heavy repository read volume. Slow clones/fetches +may hint at a repository capacity problem. Adding more repository hosts will +provide an approximately linear increase in capacity. + For details, see @{article:Cluster: Repositories}. Cluster: Daemons ================ Configuring multiple daemon hosts is straightforward, but you must configure repositories first. With daemons running on multiple hosts, you can transparently survive the loss of any subset of hosts without an interruption to daemon services, as long as at least one host remains alive. Daemons are stateless, so spreading daemons across multiple hosts provides no resistance to data loss. +Daemons can become a bottleneck, particularly if your install sees a large +volume of write traffic to repositories. If the daemon task queue has a +backlog, that hints at a capacity problem. If existing hosts have unused +resources, increase `phd.taskmasters` until they are fully utilized. From +there, adding more daemon hosts will provide an approximately linear increase +in capacity. + For details, see @{article:Cluster: Daemons}. Cluster: Web Servers ==================== Configuring multiple web hosts is straightforward, but you must configure repositories first. With multiple web hosts, you can transparently survive the loss of any subset of hosts as long as at least one host remains alive. Web hosts are stateless, -so putting multiple hosts in service provides no resistance to data loss. +so putting multiple hosts in service provides no resistance to data loss +because no data is at risk. + +Web hosts can become a bottleneck, particularly if you have a workload that is +heavily focused on reads from the web UI (like a public install with many +anonymous users). Slow responses to web requests may hint at a web capacity +problem. Adding more hosts will provide an approximately linear increase in +capacity. For details, see @{article:Cluster: Web Servers}. +Cluster: Notifications +====================== + +Configuring multiple notification hosts is simple and has no pre-requisites. + +With multiple notification hosts, you can survive the loss of any subset of +hosts as long as at least one host remains alive. Service may be breifly +disrupted directly after the incident which destroys the other hosts. + +Notifications are noncritical, so this normally has little practical impact +on service availability. Notifications are also stateless, so clustering this +service provides no resistance to data loss because no data is at risk. + +Notification delivery normally requires very few resources, so adding more +hosts is unlikely to have much impact on scalability. + +For details, see @{article:Cluster: Notifications}. + + Overlaying Services =================== Although hosts can run a single dedicated service type, certain groups of services work well together. Phabricator clusters usually do not need to be very large, so deploying a small number of hosts with multiple services is a good place to start. In planning a cluster, consider these blended host types: -**Everything**: Run HTTP, SSH, MySQL, repositories and daemons on a single -host. This is the starting point for single-node setups, and usually also the -best configuration when adding the second node. +**Everything**: Run HTTP, SSH, MySQL, notifications, repositories and daemons +on a single host. This is the starting point for single-node setups, and +usually also the best configuration when adding the second node. -**Everything Except Databases**: Run HTTP, SSH, repositories and daemons on one -host, and MySQL on a different host. MySQL uses many of the same resources that -other services use. It's also simpler to separate than other services, and -tends to benefit the most from dedicated hardware. +**Everything Except Databases**: Run HTTP, SSH, notifications, repositories and +daemons on one host, and MySQL on a different host. MySQL uses many of the same +resources that other services use. It's also simpler to separate than other +services, and tends to benefit the most from dedicated hardware. **Repositories and Daemons**: Run repositories and daemons on the same host. Repository hosts //must// run daemons, and it normally makes sense to completely overlay repositories and daemons. These services tend to use different resources (repositories are heavier on I/O and lighter on CPU/RAM; daemons are heavier on CPU/RAM and lighter on I/O). Repositories and daemons are also both less latency sensitive than other service types, so there's a wider margin of error for under provisioning them before performance is noticeably affected. These nodes tend to use system resources in a balanced way. Individual nodes in this class do not need to be particularly powerful. **Frontend Servers**: Run HTTP and SSH on the same host. These are easy to set up, stateless, and you can scale the pool up or down easily to meet demand. Routing both types of ingress traffic through the same initial tier can simplify load balancing. These nodes tend to need relatively little RAM. Cluster Recipes =============== This section provides some guidance on reasonable ways to scale up a cluster. The smallest possible cluster is **two hosts**. Run everything (web, ssh, -database, repositories, and daemons) on each host. One host will serve as the -master; the other will serve as a replica. +database, notifications, repositories, and daemons) on each host. One host will +serve as the master; the other will serve as a replica. Ideally, you should physically separate these hosts to reduce the chance that a natural disaster or infrastructure disruption could disable or destroy both hosts at the same time. From here, you can choose how you expand the cluster. To improve **scalability and performance**, separate loaded services onto dedicated hosts and then add more hosts of that type to increase capacity. If you have a two-node cluster, the best way to improve scalability by adding one host is likely to separate the master database onto its own host. Note that increasing scale may //decrease// availability by leaving you with too little capacity after a failure. If you have three hosts handling traffic and one datacenter fails, too much traffic may be sent to the single remaining host in the surviving datacenter. You can hedge against this by mirroring new hosts in other datacenters (for example, also separate the replica database onto its own host). After separating databases, separating repository + daemon nodes is likely -the next step. +the next step to consider. To improve **availability**, add another copy of everything you run in one datacenter to a new datacenter. For example, if you have a two-node cluster, the best way to improve availability is to run everything on a third host in a third datacenter. If you have a 6-node cluster with a web node, a database node and a repo + daemon node in two datacenters, add 3 more nodes to create a copy of each node in a third datacenter. You can continue adding hosts until you run out of hosts. Next Steps ========== Continue by: - learning how Phacility configures and operates a large, multi-tenant production cluster in ((cluster)). diff --git a/src/docs/user/cluster/cluster_notifications.diviner b/src/docs/user/cluster/cluster_notifications.diviner new file mode 100644 index 0000000000..f3837c869e --- /dev/null +++ b/src/docs/user/cluster/cluster_notifications.diviner @@ -0,0 +1,174 @@ +@title Cluster: Notifications +@group intro + +Configuring Phabricator to use multiple notification servers. + +Overview +======== + +WARNING: This feature is a very early prototype; the features this document +describes are mostly speculative fantasy. + +You can run multiple notification servers. The advantages of doing this +are: + + - you can completely survive the loss of any subset so long as one + remains standing; and + - performance and capacity may improve. + +This configuration is relatively simple, but has a small impact on availability +and does nothing to increase resitance to data loss. + + +Clustering Design Goals +======================= + +Notification clustering aims to restore service automatically after the loss +of some nodes. It does **not** attempt to guarantee that every message is +delivered. + +Notification messages provide timely information about events, but they are +never authoritative and never the only way for users to learn about events. +For example, if a notification about a task update is not delivered, the next +page you load will still show the notification in your notification menu. + +Generally, Phabricator works fine without notifications configured at all, so +clustering assumes that losing some messages during a disruption is acceptable. + + +How Clustering Works +==================== + +Notification clustering is very simple: notification servers relay every +message they receive to a list of peers. + +When you configure clustering, you'll run multiple servers and tell them that +the other servers exist. When any server receives a message, it retransmits it +to all the severs it knows about. + +When a server is lost, clients will automatically reconnect after a brief +delay. They may lose some notifications while their client is reconnecting, +but normally this should only last for a few seconds. + + +Configuring Aphlict +=================== + +To configure clustering on the server side, add a `cluster` key to your +Aphlict configuration file. For more details about configuring Aphlict, +see @{article:Notifications User Guide: Setup and Configuration}. + +The `cluster` key should contain a list of `"admin"` server locations. Every +message the server receives will be retransmitted to all nodes in the list. + +The server is smart enough to avoid sending messages in a cycle, and to avoid +sending messages to itself. You can safely list every server you run in the +configuration file, including the current server. + +You do not need to configure servers in an acyclic graph or only list //other// +servers: just list everything on every server and Aphlict will figure things +out from there. + +A simple example with two servers might look like this: + +```lang=json, name="aphlict.json (Cluster)" +{ + ... + "cluster": [ + { + "host": "notify001.mycompany.com", + "port": 22281, + "protocol": "http" + }, + { + "host": "notify002.mycompany.com", + "port": 22281, + "protocol": "http" + } + ] + ... +} +``` + + +Configuring Phabricator +======================= + +To configure clustering on the client side, add every service you run to +`notification.servers`. Generally, this will be twice as many entries as +you run actual servers, since each server runs a `"client"` service and an +`"admin"` service. + +A simple example with the two servers above (providing four total services) +might look like this: + +```lang=json, name="notification.servers (Cluster)" +[ + { + "type": "client", + "host": "notify001.mycompany.com", + "port": 22280, + "protocol": "https" + }, + { + "type": "client", + "host": "notify002.mycompany.com", + "port": 22280, + "protocol": "https" + }, + { + "type": "admin", + "host": "notify001.mycompany.com", + "port": 22281, + "protocol": "http" + }, + { + "type": "admin", + "host": "notify002.mycompany.com", + "port": 22281, + "protocol": "http" + } +] +``` + +If you put all of the `"client"` servers behind a load balancer, you would +just list the load balancer and let it handle pulling nodes in and out of +service. + +```lang=json, name="notification.servers (Cluster + Load Balancer)" +[ + { + "type": "client", + "host": "notify-lb.mycompany.com", + "port": 22280, + "protocol": "https" + }, + { + "type": "admin", + "host": "notify001.mycompany.com", + "port": 22281, + "protocol": "http" + }, + { + "type": "admin", + "host": "notify002.mycompany.com", + "port": 22281, + "protocol": "http" + } +] +``` + +Notification hosts do not need to run any additional services, although they +are free to do so. The notification server generally consumes few resources +and is resistant to most other loads on the machine, so it's reasonable to +overlay these on top of other services wherever it is convenient. + + +Next Steps +========== + +Continue by: + + - reviewing notification configuration with + @{article:Notifications User Guide: Setup and Configuration}; or + - returning to @{article:Clustering Introduction}. diff --git a/src/docs/user/configuration/notifications.diviner b/src/docs/user/configuration/notifications.diviner index 6a669eaef1..8fc7c1a437 100644 --- a/src/docs/user/configuration/notifications.diviner +++ b/src/docs/user/configuration/notifications.diviner @@ -1,264 +1,276 @@ @title Notifications User Guide: Setup and Configuration @group config Guide to setting up notifications. Overview ======== By default, Phabricator delivers information about events (like users creating tasks or commenting on code reviews) through email and in-application notifications. Phabricator can also be configured to deliver notifications in real time, by popping up a message in any open browser windows if something has happened or an object has been updated. To enable real-time notifications: - Configure and start the notification server, as described below. - Adjust `notification.servers` to point at it. This document describes the process in detail. Supported Browsers ================== Notifications are supported for browsers which support WebSockets. This covers most modern browsers (like Chrome, Firefox, Safari, and recent versions of Internet Explorer) and many mobile browsers. IE8 and IE9 do not support WebSockets, so real-time notifications won't work in those browsers. Installing Node and Modules =========================== The notification server uses Node.js, so you'll need to install it first. To install Node.js, follow the instructions on [[ http://nodejs.org | nodejs.org ]]. You will also need to install the `ws` module for Node. This needs to be installed into the notification server directory: phabricator/ $ cd support/aphlict/server/ phabricator/support/aphlict/server/ $ npm install ws Once Node.js and the `ws` module are installed, you're ready to start the server. Running the Aphlict Server ========================== After installing Node.js, you can control the notification server with the `bin/aphlict` command. To start the server: phabricator/ $ bin/aphlict start By default, the server must be able to listen on port `22280`. If you're using a host firewall (like a security group in EC2), make sure traffic can reach the server. The server configuration is controlled by a configuration file, which is separate from Phabricator's configuration settings. The default file can be found at `phabricator/conf/aphlict/aphlict.default.json`. To make adjustments to the default configuration, either copy this file to create `aphlict.custom.json` in the same directory (this file will be used if it exists) or specify a configuration file explicitly with the `--config` flag: phabricator/ $ bin/aphlict start --config path/to/config.json The configuration file has these settings: - `servers`: //Required list.// A list of servers to start. - `logs`: //Optional list.// A list of logs to write to. + - `cluster`: //Optional list.// A list of cluster peers. This is an advanced + feature. - `pidfile`: //Required string.// Path to a PID file. Each server in the `servers` list should be an object with these keys: - `type`: //Required string.// The type of server to start. Options are `admin` or `client`. Normally, you should run one of each. - `port`: //Required int.// The port this server should listen on. - `listen`: //Optional string.// Which interface to bind to. By default, the `admin` server is bound to `127.0.0.1` (so only other services on the local machine can connect to it), while the `client` server is bound to `0.0.0.0` (so any client can connect). - `ssl.key`: //Optional string.// If you want to use SSL on this port, the path to an SSL key. - `ssl.cert`: //Optional string.// If you want to use SSL on this port, the path to an SSL certificate. - `ssl.chain`: //Optional string.// If you have configured SSL on this port, an optional path to a certificate chain file. Each log in the `logs` list should be an object with these keys: - `path`: //Required string.// Path to the log file. +Each peer in the `cluster` list should be an object with these keys: + + - `host`: //Required string.// The peer host address. + - `port`: //Required int.// The peer port. + - `protocol`: //Required string.// The protocol to connect with, one of + `"http"` or `"https"`. + +Cluster configuration is an advanced topic and can be omitted for most +installs. For more information on how to configure a cluster, see +@{article:Clustering Introduction} and @{article:Cluster: Notifications}. + The defaults are appropriate for simple cases, but you may need to adjust them if you are running a more complex configuration. - Configuring Phabricator ======================= After starting the server, configure Phabricator to connect to it by adjusting `notification.servers`. This configuration option should have a list of servers that Phabricator should interact with. Normally, you'll list one client server and one admin server, like this: ```lang=json [ { "type": "client", "host": "phabricator.mycompany.com", "port": 22280, "protocol": "https" }, { "type": "admin", "host": "127.0.0.1", "port": 22281, "protocol": "http" } ] ``` This definition defines which services the user's browser will attempt to connect to. Most of the time, it will be very similar to the services defined in the Aphlict configuration. However, if you are sending traffic through a load balancer or terminating SSL somewhere before traffic reaches Aphlict, the services the browser connects to may need to have different hosts, ports or protocols than the underlying server listens on. Verifying Server Status ======================= After configuring `notification.servers`, navigate to {nav Config > Notification Servers} to verify that things are operational. Troubleshooting =============== You can run `aphlict` in the foreground to get output to your console: phabricator/ $ ./bin/aphlict debug Because the notification server uses WebSockets, your browser error console may also have information that is useful in figuring out what's wrong. The server also generates a log, by default in `/var/log/aphlict.log`. You can change this location by adjusting configuration. The log may contain information that is useful in resolving issues. SSL and HTTPS ============= If you serve Phabricator over HTTPS, you must also serve websockets over HTTPS. Browsers will refuse to connect to `ws://` websockets from HTTPS pages. If a client connects to Phabricator over HTTPS, Phabricator will automatically select an appropriate HTTPS service from `notification.servers` and instruct the browser to open a websocket connection with `wss://`. The simplest way to do this is configure Aphlict with an SSL key and certificate and let it terminate SSL directly. If you prefer not to do this, two other options are: - run the websocket through a websocket-capable loadbalancer and terminate SSL there; or - run the websocket through `nginx` over the same socket as the rest of your web traffic. See the next sections for more detail. Terminating SSL with a Load Balancer ==================================== If you want to terminate SSL in front of the notification server with a traditional load balancer or a similar device, do this: - Point `notification.servers` at your load balancer or reverse proxy, specifying that the protocol is `https`. - On the load balancer or proxy, terminate SSL and forward traffic to the Aphlict server. - In the Aphlict configuration, listen on the target port with `http`. Terminating SSL with Nginx ========================== If you use `nginx`, you can send websocket traffic to the same port as normal HTTP traffic and have `nginx` proxy it selectively based on the request path. This requires `nginx` 1.3 or greater. See the `nginx` documentation for details: > http://nginx.com/blog/websocket-nginx/ This is very complex, but allows you to support notifications without opening additional ports. An example `nginx` configuration might look something like this: ```lang=nginx, name=/etc/nginx/conf.d/connection_upgrade.conf map $http_upgrade $connection_upgrade { default upgrade; '' close; } ``` ```lang=nginx, name=/etc/nginx/conf.d/websocket_pool.conf upstream websocket_pool { ip_hash; server 127.0.0.1:22280; } ``` ```lang=nginx, name=/etc/nginx/sites-enabled/phabricator.example.com.conf server { server_name phabricator.example.com; root /path/to/phabricator/webroot; // ... location = /ws/ { proxy_pass http://websocket_pool; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; proxy_read_timeout 999999999; } } ``` With this approach, you should make these additional adjustments: **Phabricator Configuration**: The entry in `notification.servers` with type `"client"` should have these adjustments made: - Set `host` to the Phabricator host. - Set `port` to the standard HTTPS port (usually `443`). - Set `protocol` to `"https"`. - Set `path` to `/ws/`, so it matches the special `location` in your `nginx` config. You do not need to adjust the `"admin"` server. **Aphlict**: Your Aphlict configuration should make these adjustments to the `"client"` server: - The `protocol` should be `"http"`: `nginx` will send plain HTTP traffic to Aphlict. - Optionally, you can `listen` on `127.0.0.1` instead of `0.0.0.0`, because the server will no longer receive external traffic. diff --git a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php index 243aaecbba..268b32b462 100644 --- a/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php +++ b/src/infrastructure/storage/management/workflow/PhabricatorStorageManagementDumpWorkflow.php @@ -1,85 +1,87 @@ setName('dump') ->setExamples('**dump** [__options__]') ->setSynopsis(pht('Dump all data in storage to stdout.')) ->setArguments( array( array( 'name' => 'for-replica', 'help' => pht( 'Add __--dump-slave__ to the __mysqldump__ command, '. 'generating a CHANGE MASTER statement in the output.'), ), )); } public function didExecute(PhutilArgumentParser $args) { $api = $this->getAPI(); $patches = $this->getPatches(); $console = PhutilConsole::getConsole(); $applied = $api->getAppliedPatches(); if ($applied === null) { $namespace = $api->getNamespace(); $console->writeErr( pht( '**Storage Not Initialized**: There is no database storage '. 'initialized in this storage namespace ("%s"). Use '. '**%s** to initialize storage.', $namespace, './bin/storage upgrade')); return 1; } $databases = $api->getDatabaseList($patches, true); list($host, $port) = $this->getBareHostAndPort($api->getHost()); + $has_password = false; + $password = $api->getPassword(); if ($password) { if (strlen($password->openEnvelope())) { $has_password = true; } } $argv = array(); $argv[] = '--hex-blob'; $argv[] = '--single-transaction'; $argv[] = '--default-character-set=utf8'; if ($args->getArg('for-replica')) { $argv[] = '--dump-slave'; } $argv[] = '-u'; $argv[] = $api->getUser(); $argv[] = '-h'; $argv[] = $host; if ($port) { $argv[] = '--port'; $argv[] = $port; } $argv[] = '--databases'; foreach ($databases as $database) { $argv[] = $database; } if ($has_password) { $err = phutil_passthru('mysqldump -p%P %Ls', $password, $argv); } else { $err = phutil_passthru('mysqldump %Ls', $argv); } return $err; } } diff --git a/src/view/page/PhabricatorStandardPageView.php b/src/view/page/PhabricatorStandardPageView.php index 072e921bf6..08f0c0b3c4 100644 --- a/src/view/page/PhabricatorStandardPageView.php +++ b/src/view/page/PhabricatorStandardPageView.php @@ -1,848 +1,849 @@ showFooter = $show_footer; return $this; } public function getShowFooter() { return $this->showFooter; } public function setApplicationMenu($application_menu) { // NOTE: For now, this can either be a PHUIListView or a // PHUIApplicationMenuView. $this->applicationMenu = $application_menu; return $this; } public function getApplicationMenu() { return $this->applicationMenu; } public function setApplicationName($application_name) { $this->applicationName = $application_name; return $this; } public function setDisableConsole($disable) { $this->disableConsole = $disable; return $this; } public function getApplicationName() { return $this->applicationName; } public function setBaseURI($base_uri) { $this->baseURI = $base_uri; return $this; } public function getBaseURI() { return $this->baseURI; } public function setShowChrome($show_chrome) { $this->showChrome = $show_chrome; return $this; } public function getShowChrome() { return $this->showChrome; } public function addClass($class) { $this->classes[] = $class; return $this; } public function setPageObjectPHIDs(array $phids) { $this->pageObjects = $phids; return $this; } public function setShowDurableColumn($show) { $this->showDurableColumn = $show; return $this; } public function getShowDurableColumn() { $request = $this->getRequest(); if (!$request) { return false; } $viewer = $request->getUser(); if (!$viewer->isLoggedIn()) { return false; } $conpherence_installed = PhabricatorApplication::isClassInstalledForViewer( 'PhabricatorConpherenceApplication', $viewer); if (!$conpherence_installed) { return false; } if ($this->isQuicksandBlacklistURI()) { return false; } return true; } private function isQuicksandBlacklistURI() { $request = $this->getRequest(); if (!$request) { return false; } $patterns = $this->getQuicksandURIPatternBlacklist(); $path = $request->getRequestURI()->getPath(); foreach ($patterns as $pattern) { if (preg_match('(^'.$pattern.'$)', $path)) { return true; } } return false; } public function getDurableColumnVisible() { $column_key = PhabricatorUserPreferences::PREFERENCE_CONPHERENCE_COLUMN; return (bool)$this->getUserPreference($column_key, 0); } public function addQuicksandConfig(array $config) { $this->quicksandConfig = $config + $this->quicksandConfig; return $this; } public function getQuicksandConfig() { return $this->quicksandConfig; } public function setCrumbs(PHUICrumbsView $crumbs) { $this->crumbs = $crumbs; return $this; } public function getCrumbs() { return $this->crumbs; } public function setNavigation(AphrontSideNavFilterView $navigation) { $this->navigation = $navigation; return $this; } public function getNavigation() { return $this->navigation; } public function getTitle() { $glyph_key = PhabricatorUserPreferences::PREFERENCE_TITLES; if ($this->getUserPreference($glyph_key) == 'text') { $use_glyph = false; } else { $use_glyph = true; } $title = parent::getTitle(); $prefix = null; if ($use_glyph) { $prefix = $this->getGlyph(); } else { $application_name = $this->getApplicationName(); if (strlen($application_name)) { $prefix = '['.$application_name.']'; } } if (strlen($prefix)) { $title = $prefix.' '.$title; } return $title; } protected function willRenderPage() { parent::willRenderPage(); if (!$this->getRequest()) { throw new Exception( pht( 'You must set the %s to render a %s.', 'Request', __CLASS__)); } $console = $this->getConsole(); require_celerity_resource('phabricator-core-css'); require_celerity_resource('phabricator-zindex-css'); require_celerity_resource('phui-button-css'); require_celerity_resource('phui-spacing-css'); require_celerity_resource('phui-form-css'); require_celerity_resource('phabricator-standard-page-view'); require_celerity_resource('conpherence-durable-column-view'); require_celerity_resource('font-lato'); require_celerity_resource('font-aleo'); Javelin::initBehavior('workflow', array()); $request = $this->getRequest(); $user = null; if ($request) { $user = $request->getUser(); } if ($user) { $default_img_uri = celerity_get_resource_uri( 'rsrc/image/icon/fatcow/document_black.png'); $download_form = phabricator_form( $user, array( 'action' => '#', 'method' => 'POST', 'class' => 'lightbox-download-form', 'sigil' => 'download', ), phutil_tag( 'button', array(), pht('Download'))); Javelin::initBehavior( 'lightbox-attachments', array( 'defaultImageUri' => $default_img_uri, 'downloadForm' => $download_form, )); } Javelin::initBehavior('aphront-form-disable-on-submit'); Javelin::initBehavior('toggle-class', array()); Javelin::initBehavior('history-install'); Javelin::initBehavior('phabricator-gesture'); $current_token = null; if ($user) { $current_token = $user->getCSRFToken(); } Javelin::initBehavior( 'refresh-csrf', array( 'tokenName' => AphrontRequest::getCSRFTokenName(), 'header' => AphrontRequest::getCSRFHeaderName(), 'viaHeader' => AphrontRequest::getViaHeaderName(), 'current' => $current_token, )); Javelin::initBehavior('device'); Javelin::initBehavior( 'high-security-warning', $this->getHighSecurityWarningConfig()); if (PhabricatorEnv::isReadOnly()) { Javelin::initBehavior( 'read-only-warning', array( 'message' => PhabricatorEnv::getReadOnlyMessage(), 'uri' => PhabricatorEnv::getReadOnlyURI(), )); } if ($console) { require_celerity_resource('aphront-dark-console-css'); $headers = array(); if (DarkConsoleXHProfPluginAPI::isProfilerStarted()) { $headers[DarkConsoleXHProfPluginAPI::getProfilerHeader()] = 'page'; } if (DarkConsoleServicesPlugin::isQueryAnalyzerRequested()) { $headers[DarkConsoleServicesPlugin::getQueryAnalyzerHeader()] = true; } Javelin::initBehavior( 'dark-console', $this->getConsoleConfig()); // Change this to initBehavior when there is some behavior to initialize require_celerity_resource('javelin-behavior-error-log'); } if ($user) { $viewer = $user; } else { $viewer = new PhabricatorUser(); } $menu = id(new PhabricatorMainMenuView()) ->setUser($viewer); if ($this->getController()) { $menu->setController($this->getController()); } $application_menu = $this->getApplicationMenu(); if ($application_menu) { if ($application_menu instanceof PHUIApplicationMenuView) { $crumbs = $this->getCrumbs(); if ($crumbs) { $application_menu->setCrumbs($crumbs); } $application_menu = $application_menu->buildListView(); } $menu->setApplicationMenu($application_menu); } $this->menuContent = $menu->render(); } protected function getHead() { $monospaced = null; $request = $this->getRequest(); if ($request) { $user = $request->getUser(); if ($user) { $monospaced = $user->loadPreferences()->getPreference( PhabricatorUserPreferences::PREFERENCE_MONOSPACED); } } $response = CelerityAPI::getStaticResourceResponse(); $font_css = null; if (!empty($monospaced)) { // We can't print this normally because escaping quotation marks will // break the CSS. Instead, filter it strictly and then mark it as safe. $monospaced = new PhutilSafeHTML( PhabricatorUserPreferences::filterMonospacedCSSRule( $monospaced)); $font_css = hsprintf( '', $monospaced); } return hsprintf( '%s%s%s', parent::getHead(), $font_css, $response->renderSingleResource('javelin-magical-init', 'phabricator')); } public function setGlyph($glyph) { $this->glyph = $glyph; return $this; } public function getGlyph() { return $this->glyph; } protected function willSendResponse($response) { $request = $this->getRequest(); $response = parent::willSendResponse($response); $console = $request->getApplicationConfiguration()->getConsole(); if ($console) { $response = PhutilSafeHTML::applyFunction( 'str_replace', hsprintf(''), $console->render($request), $response); } return $response; } protected function getBody() { $user = null; $request = $this->getRequest(); if ($request) { $user = $request->getUser(); } $header_chrome = null; if ($this->getShowChrome()) { $header_chrome = $this->menuContent; } $classes = array(); $classes[] = 'main-page-frame'; $developer_warning = null; if (PhabricatorEnv::getEnvConfig('phabricator.developer-mode') && DarkConsoleErrorLogPluginAPI::getErrors()) { $developer_warning = phutil_tag_div( 'aphront-developer-error-callout', pht( 'This page raised PHP errors. Find them in DarkConsole '. 'or the error log.')); } // Render the "you have unresolved setup issues..." warning. $setup_warning = null; if ($user && $user->getIsAdmin()) { $open = PhabricatorSetupCheck::getOpenSetupIssueKeys(); if ($open) { $classes[] = 'page-has-warning'; $setup_warning = phutil_tag_div( 'setup-warning-callout', phutil_tag( 'a', array( 'href' => '/config/issue/', 'title' => implode(', ', $open), ), pht('You have %d unresolved setup issue(s)...', count($open)))); } } $main_page = phutil_tag( 'div', array( 'id' => 'phabricator-standard-page', 'class' => 'phabricator-standard-page', ), array( $developer_warning, $header_chrome, $setup_warning, phutil_tag( 'div', array( 'id' => 'phabricator-standard-page-body', 'class' => 'phabricator-standard-page-body', ), $this->renderPageBodyContent()), )); $durable_column = null; if ($this->getShowDurableColumn()) { $is_visible = $this->getDurableColumnVisible(); $durable_column = id(new ConpherenceDurableColumnView()) ->setSelectedConpherence(null) ->setUser($user) ->setQuicksandConfig($this->buildQuicksandConfig()) ->setVisible($is_visible) ->setInitialLoad(true); } Javelin::initBehavior('quicksand-blacklist', array( 'patterns' => $this->getQuicksandURIPatternBlacklist(), )); return phutil_tag( 'div', array( 'class' => implode(' ', $classes), ), array( $main_page, $durable_column, )); } private function renderPageBodyContent() { $console = $this->getConsole(); $body = parent::getBody(); $footer = $this->renderFooter(); $nav = $this->getNavigation(); if ($nav) { $crumbs = $this->getCrumbs(); if ($crumbs) { $nav->setCrumbs($crumbs); } $nav->appendChild($body); $nav->appendFooter($footer); $content = phutil_implode_html('', array($nav->render())); } else { $content = array(); $crumbs = $this->getCrumbs(); if ($crumbs) { $content[] = $crumbs; } $content[] = $body; $content[] = $footer; $content = phutil_implode_html('', $content); } return array( ($console ? hsprintf('') : null), $content, ); } protected function getTail() { $request = $this->getRequest(); $user = $request->getUser(); $tail = array( parent::getTail(), ); $response = CelerityAPI::getStaticResourceResponse(); if ($request->isHTTPS()) { $with_protocol = 'https'; } else { $with_protocol = 'http'; } $servers = PhabricatorNotificationServerRef::getEnabledClientServers( $with_protocol); if ($servers) { if ($user && $user->isLoggedIn()) { - // TODO: We could be smarter about selecting a server if there are - // multiple options available. + // TODO: We could tell the browser about all the servers and let it + // do random reconnects to improve reliability. + shuffle($servers); $server = head($servers); $client_uri = $server->getWebsocketURI(); Javelin::initBehavior( 'aphlict-listen', array( 'websocketURI' => (string)$client_uri, ) + $this->buildAphlictListenConfigData()); } } $tail[] = $response->renderHTMLFooter(); return $tail; } protected function getBodyClasses() { $classes = array(); if (!$this->getShowChrome()) { $classes[] = 'phabricator-chromeless-page'; } $agent = AphrontRequest::getHTTPHeader('User-Agent'); // Try to guess the device resolution based on UA strings to avoid a flash // of incorrectly-styled content. $device_guess = 'device-desktop'; if (preg_match('@iPhone|iPod|(Android.*Chrome/[.0-9]* Mobile)@', $agent)) { $device_guess = 'device-phone device'; } else if (preg_match('@iPad|(Android.*Chrome/)@', $agent)) { $device_guess = 'device-tablet device'; } $classes[] = $device_guess; if (preg_match('@Windows@', $agent)) { $classes[] = 'platform-windows'; } else if (preg_match('@Macintosh@', $agent)) { $classes[] = 'platform-mac'; } else if (preg_match('@X11@', $agent)) { $classes[] = 'platform-linux'; } if ($this->getRequest()->getStr('__print__')) { $classes[] = 'printable'; } if ($this->getRequest()->getStr('__aural__')) { $classes[] = 'audible'; } $classes[] = 'phui-theme-'.PhabricatorEnv::getEnvConfig('ui.header-color'); foreach ($this->classes as $class) { $classes[] = $class; } return implode(' ', $classes); } private function getConsole() { if ($this->disableConsole) { return null; } return $this->getRequest()->getApplicationConfiguration()->getConsole(); } private function getConsoleConfig() { $user = $this->getRequest()->getUser(); $headers = array(); if (DarkConsoleXHProfPluginAPI::isProfilerStarted()) { $headers[DarkConsoleXHProfPluginAPI::getProfilerHeader()] = 'page'; } if (DarkConsoleServicesPlugin::isQueryAnalyzerRequested()) { $headers[DarkConsoleServicesPlugin::getQueryAnalyzerHeader()] = true; } return array( // NOTE: We use a generic label here to prevent input reflection // and mitigate compression attacks like BREACH. See discussion in // T3684. 'uri' => pht('Main Request'), 'selected' => $user ? $user->getConsoleTab() : null, 'visible' => $user ? (int)$user->getConsoleVisible() : true, 'headers' => $headers, ); } private function getHighSecurityWarningConfig() { $user = $this->getRequest()->getUser(); $show = false; if ($user->hasSession()) { $hisec = ($user->getSession()->getHighSecurityUntil() - time()); if ($hisec > 0) { $show = true; } } return array( 'show' => $show, 'uri' => '/auth/session/downgrade/', 'message' => pht( 'Your session is in high security mode. When you '. 'finish using it, click here to leave.'), ); } private function renderFooter() { if (!$this->getShowChrome()) { return null; } if (!$this->getShowFooter()) { return null; } $items = PhabricatorEnv::getEnvConfig('ui.footer-items'); if (!$items) { return null; } $foot = array(); foreach ($items as $item) { $name = idx($item, 'name', pht('Unnamed Footer Item')); $href = idx($item, 'href'); if (!PhabricatorEnv::isValidURIForLink($href)) { $href = null; } if ($href !== null) { $tag = 'a'; } else { $tag = 'span'; } $foot[] = phutil_tag( $tag, array( 'href' => $href, ), $name); } $foot = phutil_implode_html(" \xC2\xB7 ", $foot); return phutil_tag( 'div', array( 'class' => 'phabricator-standard-page-footer grouped', ), $foot); } public function renderForQuicksand() { parent::willRenderPage(); $response = $this->renderPageBodyContent(); $response = $this->willSendResponse($response); $extra_config = $this->getQuicksandConfig(); return array( 'content' => hsprintf('%s', $response), ) + $this->buildQuicksandConfig() + $extra_config; } private function buildQuicksandConfig() { $viewer = $this->getRequest()->getUser(); $controller = $this->getController(); $dropdown_query = id(new AphlictDropdownDataQuery()) ->setViewer($viewer); $dropdown_query->execute(); $rendered_dropdowns = array(); $applications = array( 'PhabricatorHelpApplication', ); foreach ($applications as $application_class) { if (!PhabricatorApplication::isClassInstalledForViewer( $application_class, $viewer)) { continue; } $application = PhabricatorApplication::getByClass($application_class); $rendered_dropdowns[$application_class] = $application->buildMainMenuExtraNodes( $viewer, $controller); } $hisec_warning_config = $this->getHighSecurityWarningConfig(); $console_config = null; $console = $this->getConsole(); if ($console) { $console_config = $this->getConsoleConfig(); } $upload_enabled = false; if ($controller) { $upload_enabled = $controller->isGlobalDragAndDropUploadEnabled(); } $application_class = null; $application_search_icon = null; $controller = $this->getController(); if ($controller) { $application = $controller->getCurrentApplication(); if ($application) { $application_class = get_class($application); if ($application->getApplicationSearchDocumentTypes()) { $application_search_icon = $application->getIcon(); } } } return array( 'title' => $this->getTitle(), 'aphlictDropdownData' => array( $dropdown_query->getNotificationData(), $dropdown_query->getConpherenceData(), ), 'globalDragAndDrop' => $upload_enabled, 'aphlictDropdowns' => $rendered_dropdowns, 'hisecWarningConfig' => $hisec_warning_config, 'consoleConfig' => $console_config, 'applicationClass' => $application_class, 'applicationSearchIcon' => $application_search_icon, ) + $this->buildAphlictListenConfigData(); } private function buildAphlictListenConfigData() { $user = $this->getRequest()->getUser(); $subscriptions = $this->pageObjects; $subscriptions[] = $user->getPHID(); return array( 'pageObjects' => array_fill_keys($this->pageObjects, true), 'subscriptions' => $subscriptions, ); } private function getQuicksandURIPatternBlacklist() { $applications = PhabricatorApplication::getAllApplications(); $blacklist = array(); foreach ($applications as $application) { $blacklist[] = $application->getQuicksandURIPatternBlacklist(); } return array_mergev($blacklist); } private function getUserPreference($key, $default = null) { $request = $this->getRequest(); if (!$request) { return $default; } $user = $request->getUser(); if (!$user) { return $default; } return $user->loadPreferences()->getPreference($key, $default); } public function produceAphrontResponse() { $controller = $this->getController(); if (!$this->getApplicationMenu()) { $application_menu = $controller->buildApplicationMenu(); if ($application_menu) { $this->setApplicationMenu($application_menu); } } $viewer = $this->getUser(); if ($viewer && $viewer->getPHID()) { $object_phids = $this->pageObjects; foreach ($object_phids as $object_phid) { PhabricatorFeedStoryNotification::updateObjectNotificationViews( $viewer, $object_phid); } } if ($this->getRequest()->isQuicksand()) { $content = $this->renderForQuicksand(); $response = id(new AphrontAjaxResponse()) ->setContent($content); } else { $content = $this->render(); $response = id(new AphrontWebpageResponse()) ->setContent($content); } return $response; } } diff --git a/support/aphlict/server/aphlict_server.js b/support/aphlict/server/aphlict_server.js index 2c03875c7d..a4089ef54a 100644 --- a/support/aphlict/server/aphlict_server.js +++ b/support/aphlict/server/aphlict_server.js @@ -1,179 +1,199 @@ 'use strict'; var JX = require('./lib/javelin').JX; var http = require('http'); var https = require('https'); var util = require('util'); var fs = require('fs'); function parse_command_line_arguments(argv) { var args = { test: false, config: null }; for (var ii = 2; ii < argv.length; ii++) { var arg = argv[ii]; var matches = arg.match(/^--([^=]+)=(.*)$/); if (!matches) { throw new Error('Unknown argument "' + arg + '"!'); } if (!(matches[1] in args)) { throw new Error('Unknown argument "' + matches[1] + '"!'); } args[matches[1]] = matches[2]; } return args; } function parse_config(args) { var data = fs.readFileSync(args.config); return JSON.parse(data); } require('./lib/AphlictLog'); var debug = new JX.AphlictLog() .addConsole(console); var args = parse_command_line_arguments(process.argv); var config = parse_config(args); function set_exit_code(code) { process.on('exit', function() { process.exit(code); }); } process.on('uncaughtException', function(err) { var context = null; if (err.code == 'EACCES') { context = util.format( 'Unable to open file ("%s"). Check that permissions are set ' + 'correctly.', err.path); } var message = [ '\n<<< UNCAUGHT EXCEPTION! >>>', ]; if (context) { message.push(context); } message.push(err.stack); debug.log(message.join('\n\n')); set_exit_code(1); }); try { require('ws'); } catch (ex) { throw new Error( 'You need to install the Node.js "ws" module for websocket support. ' + 'See "Notifications User Guide: Setup and Configuration" in the ' + 'documentation for instructions. ' + ex.toString()); } // NOTE: Require these only after checking for the "ws" module, since they // depend on it. require('./lib/AphlictAdminServer'); require('./lib/AphlictClientServer'); - +require('./lib/AphlictPeerList'); +require('./lib/AphlictPeer'); var ii; var logs = config.logs || []; for (ii = 0; ii < logs.length; ii++) { debug.addLog(logs[ii].path); } var servers = []; for (ii = 0; ii < config.servers.length; ii++) { var spec = config.servers[ii]; spec.listen = spec.listen || '0.0.0.0'; if (spec['ssl.key']) { spec['ssl.key'] = fs.readFileSync(spec['ssl.key']); } if (spec['ssl.cert']){ spec['ssl.cert'] = fs.readFileSync(spec['ssl.cert']); } if (spec['ssl.chain']){ spec['ssl.chain'] = fs.readFileSync(spec['ssl.chain']); } servers.push(spec); } // If we're just doing a configuration test, exit here before starting any // servers. if (args.test) { debug.log('Configuration test OK.'); set_exit_code(0); return; } debug.log('Starting servers (service PID %d).', process.pid); for (ii = 0; ii < logs.length; ii++) { debug.log('Logging to "%s".', logs[ii].path); } var aphlict_servers = []; var aphlict_clients = []; var aphlict_admins = []; for (ii = 0; ii < servers.length; ii++) { var server = servers[ii]; var is_client = (server.type == 'client'); var http_server; if (server['ssl.key']) { var https_config = { key: server['ssl.key'], cert: server['ssl.cert'], }; if (server['ssl.chain']) { https_config.ca = server['ssl.chain']; } http_server = https.createServer(https_config); } else { http_server = http.createServer(); } var aphlict_server; if (is_client) { aphlict_server = new JX.AphlictClientServer(http_server); } else { aphlict_server = new JX.AphlictAdminServer(http_server); } aphlict_server.setLogger(debug); aphlict_server.listen(server.port, server.listen); debug.log( 'Started %s server (Port %d, %s).', server.type, server.port, server['ssl.key'] ? 'With SSL' : 'No SSL'); aphlict_servers.push(aphlict_server); if (is_client) { aphlict_clients.push(aphlict_server); } else { aphlict_admins.push(aphlict_server); } } +var peer_list = new JX.AphlictPeerList(); + +debug.log( + 'This server has fingerprint "%s".', + peer_list.getFingerprint()); + +var cluster = config.cluster || []; +for (ii = 0; ii < cluster.length; ii++) { + var peer = cluster[ii]; + + var peer_client = new JX.AphlictPeer() + .setHost(peer.host) + .setPort(peer.port) + .setProtocol(peer.protocol); + + peer_list.addPeer(peer_client); +} + for (ii = 0; ii < aphlict_admins.length; ii++) { var admin_server = aphlict_admins[ii]; admin_server.setClientServers(aphlict_clients); + admin_server.setPeerList(peer_list); } diff --git a/support/aphlict/server/lib/AphlictAdminServer.js b/support/aphlict/server/lib/AphlictAdminServer.js index 97dca15bde..3cac0be3b5 100644 --- a/support/aphlict/server/lib/AphlictAdminServer.js +++ b/support/aphlict/server/lib/AphlictAdminServer.js @@ -1,179 +1,197 @@ 'use strict'; var JX = require('./javelin').JX; require('./AphlictListenerList'); var http = require('http'); var url = require('url'); JX.install('AphlictAdminServer', { construct: function(server) { this._startTime = new Date().getTime(); this._messagesIn = 0; this._messagesOut = 0; server.on('request', JX.bind(this, this._onrequest)); this._server = server; this._clientServers = []; }, properties: { clientServers: null, logger: null, + peerList: null }, members: { _messagesIn: null, _messagesOut: null, _server: null, _startTime: null, getListenerLists: function(instance) { var clients = this.getClientServers(); var lists = []; for (var ii = 0; ii < clients.length; ii++) { lists.push(clients[ii].getListenerList(instance)); } return lists; }, log: function() { var logger = this.getLogger(); if (!logger) { return; } logger.log.apply(logger, arguments); return this; }, listen: function() { return this._server.listen.apply(this._server, arguments); }, _onrequest: function(request, response) { var self = this; var u = url.parse(request.url, true); var instance = u.query.instance || 'default'; // Publishing a notification. if (u.pathname == '/') { if (request.method == 'POST') { var body = ''; request.on('data', function(data) { body += data; }); request.on('end', function() { try { var msg = JSON.parse(body); self.log( 'Received notification (' + instance + '): ' + JSON.stringify(msg)); ++self._messagesIn; try { - self._transmit(instance, msg); - response.writeHead(200, {'Content-Type': 'text/plain'}); + self._transmit(instance, msg, response); } catch (err) { self.log( '<%s> Internal Server Error! %s', request.socket.remoteAddress, err); response.writeHead(500, 'Internal Server Error'); } } catch (err) { self.log( '<%s> Bad Request! %s', request.socket.remoteAddress, err); response.writeHead(400, 'Bad Request'); } finally { response.end(); } }); } else { response.writeHead(405, 'Method Not Allowed'); response.end(); } } else if (u.pathname == '/status/') { this._handleStatusRequest(request, response, instance); } else { response.writeHead(404, 'Not Found'); response.end(); } }, _handleStatusRequest: function(request, response, instance) { var active_count = 0; var total_count = 0; var lists = this.getListenerLists(instance); for (var ii = 0; ii < lists.length; ii++) { var list = lists[ii]; active_count += list.getActiveListenerCount(); total_count += list.getTotalListenerCount(); } var server_status = { 'instance': instance, 'uptime': (new Date().getTime() - this._startTime), 'clients.active': active_count, 'clients.total': total_count, 'messages.in': this._messagesIn, 'messages.out': this._messagesOut, 'version': 7 }; response.writeHead(200, {'Content-Type': 'application/json'}); response.write(JSON.stringify(server_status)); response.end(); }, /** * Transmits a message to all subscribed listeners. */ - _transmit: function(instance, message) { - var lists = this.getListenerLists(instance); + _transmit: function(instance, message, response) { + var peer_list = this.getPeerList(); - for (var ii = 0; ii < lists.length; ii++) { - var list = lists[ii]; - var listeners = list.getListeners(); - this._transmitToListeners(list, listeners, message); + message = peer_list.addFingerprint(message); + if (message) { + var lists = this.getListenerLists(instance); + + for (var ii = 0; ii < lists.length; ii++) { + var list = lists[ii]; + var listeners = list.getListeners(); + this._transmitToListeners(list, listeners, message); + } + + peer_list.broadcastMessage(instance, message); } + + // Respond to the caller with our fingerprint so it can stop sending + // us traffic we don't need to know about if it's a peer. In particular, + // this stops us from broadcasting messages to ourselves if we appear + // in the cluster list. + var receipt = { + fingerprint: this.getPeerList().getFingerprint() + }; + + response.writeHead(200, {'Content-Type': 'application/json'}); + response.write(JSON.stringify(receipt)); }, _transmitToListeners: function(list, listeners, message) { for (var ii = 0; ii < listeners.length; ii++) { var listener = listeners[ii]; if (!listener.isSubscribedToAny(message.subscribers)) { continue; } try { listener.writeMessage(message); ++this._messagesOut; this.log( '<%s> Wrote Message', listener.getDescription()); } catch (error) { list.removeListener(listener); this.log( '<%s> Write Error: %s', listener.getDescription(), error); } } } } }); diff --git a/support/aphlict/server/lib/AphlictPeer.js b/support/aphlict/server/lib/AphlictPeer.js new file mode 100644 index 0000000000..068977992c --- /dev/null +++ b/support/aphlict/server/lib/AphlictPeer.js @@ -0,0 +1,80 @@ +'use strict'; + +var JX = require('./javelin').JX; + +var http = require('http'); +var https = require('https'); + +JX.install('AphlictPeer', { + + construct: function() { + }, + + properties: { + host: null, + port: null, + protocol: null, + fingerprint: null + }, + + members: { + broadcastMessage: function(instance, message) { + var data; + try { + data = JSON.stringify(message); + } catch (error) { + return; + } + + // TODO: Maybe use "agent" stuff to pool connections? + + var options = { + hostname: this.getHost(), + port: this.getPort(), + method: 'POST', + path: '/?instance=' + instance, + headers: { + 'Content-Type': 'application/json', + 'Content-Length': data.length + } + }; + + var onresponse = JX.bind(this, this._onresponse); + + var request; + if (this.getProtocol() == 'https') { + request = https.request(options, onresponse); + } else { + request = http.request(options, onresponse); + } + + request.write(data); + request.end(); + }, + + _onresponse: function(response) { + var peer = this; + var data = ''; + + response.on('data', function(bytes) { + data += bytes; + }); + + response.on('end', function() { + var message; + try { + message = JSON.parse(data); + } catch (error) { + return; + } + + // If we got a valid receipt, update the fingerprint for this server. + var fingerprint = message.fingerprint; + if (fingerprint) { + peer.setFingerprint(fingerprint); + } + }); + } + } + +}); diff --git a/support/aphlict/server/lib/AphlictPeerList.js b/support/aphlict/server/lib/AphlictPeerList.js new file mode 100644 index 0000000000..9c8c707894 --- /dev/null +++ b/support/aphlict/server/lib/AphlictPeerList.js @@ -0,0 +1,86 @@ +'use strict'; + +var JX = require('./javelin').JX; + +JX.install('AphlictPeerList', { + + construct: function() { + this._peers = []; + + // Generate a new unique identify for this server. We just use this to + // identify messages we have already seen and figure out which peer is + // actually us, so we don't bounce messages around the cluster forever. + this._fingerprint = this._generateFingerprint(); + }, + + properties: { + }, + + members: { + _peers: null, + _fingerprint: null, + + addPeer: function(peer) { + this._peers.push(peer); + return this; + }, + + addFingerprint: function(message) { + var fingerprint = this.getFingerprint(); + + // Check if we've already touched this message. If we have, we do not + // broadcast it again. If we haven't, we add our fingerprint and then + // broadcast the modified version. + var touched = message.touched || []; + for (var ii = 0; ii < touched.length; ii++) { + if (touched[ii] == fingerprint) { + return null; + } + } + touched.push(fingerprint); + + message.touched = touched; + return message; + }, + + broadcastMessage: function(instance, message) { + var ii; + + var touches = {}; + var touched = message.touched; + for (ii = 0; ii < touched.length; ii++) { + touches[touched[ii]] = true; + } + + var peers = this._peers; + for (ii = 0; ii < peers.length; ii++) { + var peer = peers[ii]; + + // If we know the peer's fingerprint and it has already touched + // this message, don't broadcast it. + var fingerprint = peer.getFingerprint(); + if (fingerprint && touches[fingerprint]) { + continue; + } + + peer.broadcastMessage(instance, message); + } + }, + + getFingerprint: function() { + return this._fingerprint; + }, + + _generateFingerprint: function() { + var src = '23456789abcdefghjkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ'; + var len = 16; + var out = []; + for (var ii = 0; ii < len; ii++) { + var idx = Math.floor(Math.random() * src.length); + out.push(src[idx]); + } + return out.join(''); + } + } + +});