Changeset View
Changeset View
Standalone View
Standalone View
src/applications/diviner/workflow/DivinerGenerateWorkflow.php
| Show First 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | final class DivinerGenerateWorkflow extends DivinerWorkflow { | ||||
| public function execute(PhutilArgumentParser $args) { | public function execute(PhutilArgumentParser $args) { | ||||
| $book = $args->getArg('book'); | $book = $args->getArg('book'); | ||||
| if ($book) { | if ($book) { | ||||
| $books = array($book); | $books = array($book); | ||||
| } else { | } else { | ||||
| $cwd = getcwd(); | $cwd = getcwd(); | ||||
| $this->log(pht('FINDING DOCUMENTATION BOOKS')); | $this->log(pht('FINDING DOCUMENTATION BOOKS')); | ||||
| $books = id(new FileFinder($cwd)) | $books = id(new FileFinder($cwd)) | ||||
| ->withType('f') | ->withType('f') | ||||
| ->withSuffix('book') | ->withSuffix('book') | ||||
| ->find(); | ->find(); | ||||
| if (!$books) { | if (!$books) { | ||||
| throw new PhutilArgumentUsageException( | throw new PhutilArgumentUsageException( | ||||
| pht( | pht( | ||||
| "There are no Diviner '%s' files anywhere beneath the current ". | "There are no Diviner '%s' files anywhere beneath the current ". | ||||
| "directory. Use '%s' to specify a documentation book to generate.", | "directory. Use '%s' to specify a documentation book to generate.", | ||||
| '.book', | '.book', | ||||
| '--book <book>')); | '--book <book>')); | ||||
| } else { | } else { | ||||
| $this->log(pht('Found %s book(s).', new PhutilNumber(count($books)))); | $this->log(pht('Found %s book(s).', new PhutilNumber(count($books)))); | ||||
joshuaspence: Interestingly, with one book this is rendering as "Found 1 books." | |||||
| } | } | ||||
| } | } | ||||
| foreach ($books as $book) { | foreach ($books as $book) { | ||||
| $short_name = basename($book); | $short_name = basename($book); | ||||
| $this->log(pht('Generating book "%s"...', $short_name)); | $this->log(pht('Generating book "%s"...', $short_name)); | ||||
| $this->generateBook($book, $args); | $this->generateBook($book, $args); | ||||
| Show All 12 Lines | if ($args->getArg('clean')) { | ||||
| $this->log(pht('Done.')."\n"); | $this->log(pht('Done.')."\n"); | ||||
| } | } | ||||
| // The major challenge of documentation generation is one of dependency | // The major challenge of documentation generation is one of dependency | ||||
| // management. When regenerating documentation, we want to do the smallest | // management. When regenerating documentation, we want to do the smallest | ||||
| // amount of work we can, so that regenerating documentation after minor | // amount of work we can, so that regenerating documentation after minor | ||||
| // changes is quick. | // changes is quick. | ||||
| // | // | ||||
| // = ATOM CACHE = | // = Atom Cache = | ||||
| // | // | ||||
| // In the first stage, we find all the direct changes to source code since | // In the first stage, we find all the direct changes to source code since | ||||
| // the last run. This stage relies on two data structures: | // the last run. This stage relies on two data structures: | ||||
| // | // | ||||
| // - File Hash Map: `map<file_hash, node_hash>` | // - File Hash Map: `map<file_hash, node_hash>` | ||||
| // - Atom Map: `map<node_hash, true>` | // - Atom Map: `map<node_hash, true>` | ||||
| // | // | ||||
| // First, we hash all the source files in the project to detect any which | // First, we hash all the source files in the project to detect any which | ||||
| Show All 9 Lines | private function generateBook($book, PhutilArgumentParser $args) { | ||||
| // is the Atom Map. The node hash depends only on the definition of the atom | // is the Atom Map. The node hash depends only on the definition of the atom | ||||
| // and the atomizer implementation. It ends with an "N", for "node". | // and the atomizer implementation. It ends with an "N", for "node". | ||||
| // | // | ||||
| // (We need the Atom Map in addition to the File Hash Map because each file | // (We need the Atom Map in addition to the File Hash Map because each file | ||||
| // may have several atoms in it (e.g., multiple functions, or a class and | // may have several atoms in it (e.g., multiple functions, or a class and | ||||
| // its methods). The File Hash Map contains an exhaustive list of all atoms | // its methods). The File Hash Map contains an exhaustive list of all atoms | ||||
| // with type "file", but not child atoms of those top-level atoms.) | // with type "file", but not child atoms of those top-level atoms.) | ||||
| // | // | ||||
| // = GRAPH CACHE = | // = Graph Cache = | ||||
| // | // | ||||
| // We now know which atoms exist, and can compare the Atom Map to some | // We now know which atoms exist, and can compare the Atom Map to some | ||||
| // existing cache to figure out what has changed. However, this isn't | // existing cache to figure out what has changed. However, this isn't | ||||
| // sufficient to figure out which documentation actually needs to be | // sufficient to figure out which documentation actually needs to be | ||||
| // regenerated, because atoms depend on other atoms. For example, if `B | // regenerated, because atoms depend on other atoms. For example, if `B | ||||
| // extends A` and the definition for `A` changes, we need to regenerate the | // extends A` and the definition for `A` changes, we need to regenerate the | ||||
| // documentation in `B`. Similarly, if `X` links to `Y` and `Y` changes, we | // documentation in `B`. Similarly, if `X` links to `Y` and `Y` changes, we | ||||
| // should regenerate `X`. (In both these cases, the documentation for the | // should regenerate `X`. (In both these cases, the documentation for the | ||||
| ▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | private function generateBook($book, PhutilArgumentParser $args) { | ||||
| $this->buildGraphCache(); | $this->buildGraphCache(); | ||||
| $publisher_class = $args->getArg('publisher'); | $publisher_class = $args->getArg('publisher'); | ||||
| $symbols = id(new PhutilSymbolLoader()) | $symbols = id(new PhutilSymbolLoader()) | ||||
| ->setName($publisher_class) | ->setName($publisher_class) | ||||
| ->setConcreteOnly(true) | ->setConcreteOnly(true) | ||||
| ->setAncestorClass('DivinerPublisher') | ->setAncestorClass('DivinerPublisher') | ||||
| ->selectAndLoadSymbols(); | ->selectAndLoadSymbols(); | ||||
| if (!$symbols) { | if (!$symbols) { | ||||
| throw new Exception( | throw new PhutilArgumentUsageException( | ||||
| pht( | pht( | ||||
| "Publisher class '%s' must be a concrete subclass of %s.", | "Publisher class '%s' must be a concrete subclass of %s.", | ||||
| $publisher_class, | $publisher_class, | ||||
| 'DivinerPublisher')); | 'DivinerPublisher')); | ||||
| } | } | ||||
| $publisher = newv($publisher_class, array()); | $publisher = newv($publisher_class, array()); | ||||
| $this->publishDocumentation($args->getArg('clean'), $publisher); | $this->publishDocumentation($args->getArg('clean'), $publisher); | ||||
| } | } | ||||
| /* -( Atom Cache )--------------------------------------------------------- */ | /* -( Atom Cache )--------------------------------------------------------- */ | ||||
| private function buildAtomCache() { | private function buildAtomCache() { | ||||
| $this->log(pht('BUILDING ATOM CACHE')); | $this->log(pht('BUILDING ATOM CACHE')); | ||||
| $file_hashes = $this->findFilesInProject(); | $file_hashes = $this->findFilesInProject(); | ||||
| $this->log(pht('Found %d file(s) in project.', count($file_hashes))); | $this->log( | ||||
| pht( | |||||
| 'Found %s file(s) in project.', | |||||
| new PhutilNumber(count($file_hashes)))); | |||||
| $this->deleteDeadAtoms($file_hashes); | $this->deleteDeadAtoms($file_hashes); | ||||
| $atomize = $this->getFilesToAtomize($file_hashes); | $atomize = $this->getFilesToAtomize($file_hashes); | ||||
| $this->log(pht('Found %d unatomized, uncached file(s).', count($atomize))); | $this->log( | ||||
| pht( | |||||
| 'Found %s unatomized, uncached file(s).', | |||||
| new PhutilNumber(count($atomize)))); | |||||
| $file_atomizers = $this->getAtomizersForFiles($atomize); | $file_atomizers = $this->getAtomizersForFiles($atomize); | ||||
| $this->log(pht('Found %d file(s) to atomize.', count($file_atomizers))); | $this->log( | ||||
| pht( | |||||
| 'Found %s file(s) to atomize.', | |||||
| new PhutilNumber(count($file_atomizers)))); | |||||
| $futures = $this->buildAtomizerFutures($file_atomizers); | $futures = $this->buildAtomizerFutures($file_atomizers); | ||||
| $this->log(pht('Atomizing %d file(s).', count($file_atomizers))); | $this->log( | ||||
| pht( | |||||
| 'Atomizing %s file(s).', | |||||
| new PhutilNumber(count($file_atomizers)))); | |||||
| if ($futures) { | if ($futures) { | ||||
| $this->resolveAtomizerFutures($futures, $file_hashes); | $this->resolveAtomizerFutures($futures, $file_hashes); | ||||
| $this->log(pht('Atomization complete.')); | $this->log(pht('Atomization complete.')); | ||||
| } else { | } else { | ||||
| $this->log(pht('Atom cache is up to date, no files to atomize.')); | $this->log(pht('Atom cache is up to date, no files to atomize.')); | ||||
| } | } | ||||
| ▲ Show 20 Lines • Show All 124 Lines • ▼ Show 20 Lines | /* -( Atom Cache )--------------------------------------------------------- */ | ||||
| private function resolveAtomizerFutures(array $futures, array $file_hashes) { | private function resolveAtomizerFutures(array $futures, array $file_hashes) { | ||||
| assert_instances_of($futures, 'Future'); | assert_instances_of($futures, 'Future'); | ||||
| $atom_cache = $this->getAtomCache(); | $atom_cache = $this->getAtomCache(); | ||||
| $bar = id(new PhutilConsoleProgressBar()) | $bar = id(new PhutilConsoleProgressBar()) | ||||
| ->setTotal(count($futures)); | ->setTotal(count($futures)); | ||||
| $futures = id(new FutureIterator($futures)) | $futures = id(new FutureIterator($futures)) | ||||
| ->limit(4); | ->limit(4); | ||||
| foreach ($futures as $key => $future) { | foreach ($futures as $key => $future) { | ||||
| try { | try { | ||||
| $atoms = $future->resolveJSON(); | $atoms = $future->resolveJSON(); | ||||
| foreach ($atoms as $atom) { | foreach ($atoms as $atom) { | ||||
| if ($atom['type'] == DivinerAtom::TYPE_FILE) { | if ($atom['type'] == DivinerAtom::TYPE_FILE) { | ||||
| $file_hash = $file_hashes[$atom['file']]; | $file_hash = $file_hashes[$atom['file']]; | ||||
| $atom_cache->addFileHash($file_hash, $atom['hash']); | $atom_cache->addFileHash($file_hash, $atom['hash']); | ||||
| Show All 36 Lines | private function getDivinerAtomWorldVersion() { | ||||
| $version['atomizers'] = $atomizer_versions; | $version['atomizers'] = $atomizer_versions; | ||||
| return md5(serialize($version)); | return md5(serialize($version)); | ||||
| } | } | ||||
| /* -( Graph Cache )-------------------------------------------------------- */ | /* -( Graph Cache )-------------------------------------------------------- */ | ||||
| private function buildGraphCache() { | private function buildGraphCache() { | ||||
| $this->log(pht('BUILDING GRAPH CACHE')); | $this->log(pht('BUILDING GRAPH CACHE')); | ||||
| $atom_cache = $this->getAtomCache(); | $atom_cache = $this->getAtomCache(); | ||||
| $symbol_map = $atom_cache->getSymbolMap(); | $symbol_map = $atom_cache->getSymbolMap(); | ||||
| $atoms = $atom_cache->getAtomMap(); | $atoms = $atom_cache->getAtomMap(); | ||||
| $dirty_symbols = array(); | $dirty_symbols = array(); | ||||
| $dirty_nhashes = array(); | $dirty_nhashes = array(); | ||||
| $del_atoms = array_diff_key($symbol_map, $atoms); | $del_atoms = array_diff_key($symbol_map, $atoms); | ||||
| $this->log(pht('Found %d obsolete atom(s) in graph.', count($del_atoms))); | $this->log( | ||||
| pht( | |||||
| 'Found %s obsolete atom(s) in graph.', | |||||
| new PhutilNumber(count($del_atoms)))); | |||||
| foreach ($del_atoms as $nhash => $shash) { | foreach ($del_atoms as $nhash => $shash) { | ||||
| $atom_cache->deleteSymbol($nhash); | $atom_cache->deleteSymbol($nhash); | ||||
| $dirty_symbols[$shash] = true; | $dirty_symbols[$shash] = true; | ||||
| $atom_cache->deleteEdges($nhash); | $atom_cache->deleteEdges($nhash); | ||||
| $atom_cache->deleteGraph($nhash); | $atom_cache->deleteGraph($nhash); | ||||
| } | } | ||||
| $new_atoms = array_diff_key($atoms, $symbol_map); | $new_atoms = array_diff_key($atoms, $symbol_map); | ||||
| $this->log(pht('Found %d new atom(s) in graph.', count($new_atoms))); | $this->log( | ||||
| pht( | |||||
| 'Found %s new atom(s) in graph.', | |||||
| new PhutilNumber(count($new_atoms)))); | |||||
| foreach ($new_atoms as $nhash => $ignored) { | foreach ($new_atoms as $nhash => $ignored) { | ||||
| $shash = $this->computeSymbolHash($nhash); | $shash = $this->computeSymbolHash($nhash); | ||||
| $atom_cache->addSymbol($nhash, $shash); | $atom_cache->addSymbol($nhash, $shash); | ||||
| $dirty_symbols[$shash] = true; | $dirty_symbols[$shash] = true; | ||||
| $atom_cache->addEdges($nhash, $this->getEdges($nhash)); | $atom_cache->addEdges($nhash, $this->getEdges($nhash)); | ||||
| Show All 19 Lines | while ($symbol_stack) { | ||||
| $src_hash = $this->computeSymbolHash($edge); | $src_hash = $this->computeSymbolHash($edge); | ||||
| if (empty($dirty_symbols[$src_hash])) { | if (empty($dirty_symbols[$src_hash])) { | ||||
| $dirty_symbols[$src_hash] = true; | $dirty_symbols[$src_hash] = true; | ||||
| $symbol_stack[] = $src_hash; | $symbol_stack[] = $src_hash; | ||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| $this->log(pht('Found %d affected atoms.', count($dirty_nhashes))); | $this->log( | ||||
| pht( | |||||
| 'Found %s affected atoms.', | |||||
| new PhutilNumber(count($dirty_nhashes)))); | |||||
| foreach ($dirty_nhashes as $nhash => $ignored) { | foreach ($dirty_nhashes as $nhash => $ignored) { | ||||
| $atom_cache->addGraph($nhash, $this->computeGraphHash($nhash)); | $atom_cache->addGraph($nhash, $this->computeGraphHash($nhash)); | ||||
| } | } | ||||
| $this->log(pht('Writing graph cache.')); | $this->log(pht('Writing graph cache.')); | ||||
| $atom_cache->saveGraph(); | $atom_cache->saveGraph(); | ||||
| ▲ Show 20 Lines • Show All 72 Lines • Show Last 20 Lines | |||||
Interestingly, with one book this is rendering as "Found 1 books."