Index: scripts/symbols/import_project_symbols.php =================================================================== --- scripts/symbols/import_project_symbols.php +++ scripts/symbols/import_project_symbols.php @@ -1,6 +1,7 @@ #!/usr/bin/env php 'max-transaction', + 'param' => 'num-syms', + 'default' => '100000', + 'help' => 'Maximum number of symbols that should '. + 'be part of a single transaction', + ), + array( 'name' => 'more', 'wildcard' => true, ), @@ -53,6 +61,52 @@ $input = trim($input); $input = explode("\n", $input); + +function commit_symbols ($syms, $project, $no_purge) { + echo "Looking up path IDs...\n"; + $path_map = + PhabricatorRepositoryCommitChangeParserWorker::lookupOrCreatePaths( + ipull($syms, 'path')); + + $symbol = new PhabricatorRepositorySymbol(); + $conn_w = $symbol->establishConnection('w'); + + echo "Preparing queries...\n"; + $sql = array(); + foreach ($syms as $dict) { + $sql[] = qsprintf( + $conn_w, + '(%d, %s, %s, %s, %s, %d, %d)', + $project->getID(), + $dict['ctxt'], + $dict['name'], + $dict['type'], + $dict['lang'], + $dict['line'], + $path_map[$dict['path']]); + } + + if (!$no_purge) { + echo "Purging old syms...\n"; + queryfx($conn_w, + 'DELETE FROM %T WHERE arcanistProjectID = %d', + $symbol->getTableName(), + $project->getID()); + } + + echo "Loading ".number_format(count($sql))." syms...\n"; + foreach (array_chunk($sql, 128) as $chunk) { + queryfx($conn_w, + 'INSERT INTO %T + (arcanistProjectID, symbolContext, symbolName, symbolType, + symbolLanguage, lineNumber, pathID) VALUES %Q', + $symbol->getTableName(), + implode(', ', $chunk)); + } + +} + +$no_purge = $args->getArg('no-purge'); $symbols = array(); foreach ($input as $key => $line) { try { @@ -129,48 +183,26 @@ throw $e; } } -} -echo "Looking up path IDs...\n"; -$path_map = PhabricatorRepositoryCommitChangeParserWorker::lookupOrCreatePaths( - ipull($symbols, 'path')); - -$symbol = new PhabricatorRepositorySymbol(); -$conn_w = $symbol->establishConnection('w'); - -echo "Preparing queries...\n"; -$sql = array(); -foreach ($symbols as $dict) { - $sql[] = qsprintf( - $conn_w, - '(%d, %s, %s, %s, %s, %d, %d)', - $project->getID(), - $dict['ctxt'], - $dict['name'], - $dict['type'], - $dict['lang'], - $dict['line'], - $path_map[$dict['path']]); -} - -if (!$args->getArg('no-purge')) { - echo "Purging old symbols...\n"; - queryfx( - $conn_w, - 'DELETE FROM %T WHERE arcanistProjectID = %d', - $symbol->getTableName(), - $project->getID()); + if (count ($symbols) >= $args->getArg('max-transaction')) { + try { + echo "Committing {$args->getArg('max-transaction')} symbols....\n"; + commit_symbols($symbols, $project, $no_purge); + $no_purge = true; + unset($symbols); + $symbols = array(); + } catch (Exception $e) { + if ($args->getArg('ignore-errors')) { + continue; + } else { + throw $e; + } + } + } } -echo "Loading ".number_format(count($sql))." symbols...\n"; -foreach (array_chunk($sql, 128) as $chunk) { - queryfx( - $conn_w, - 'INSERT INTO %T - (arcanistProjectID, symbolContext, symbolName, symbolType, - symbolLanguage, lineNumber, pathID) VALUES %Q', - $symbol->getTableName(), - implode(', ', $chunk)); +if (count($symbols)) { + commit_symbols($symbols, $project, $args->getArg('no-purge')); } echo "Done.\n";