Index: src/aphront/console/DarkConsoleCore.php =================================================================== --- src/aphront/console/DarkConsoleCore.php +++ src/aphront/console/DarkConsoleCore.php @@ -74,9 +74,18 @@ $cache = new PhutilKeyValueCacheProfiler($cache); $cache->setProfiler(PhutilServiceProfiler::getInstance()); + // This encoding may fail if there are, e.g., database queries which + // include binary data. It would be a little cleaner to try to strip these, + // but just do something non-broken here if we end up with unrepresentable + // data. + $json = @json_encode($storage); + if (!$json) { + $json = '{}'; + } + $cache->setKeys( array( - 'darkconsole:'.$key => json_encode($storage), + 'darkconsole:'.$key => $json, ), $ttl = (60 * 60 * 6)); Index: src/applications/cache/PhabricatorKeyValueDatabaseCache.php =================================================================== --- src/applications/cache/PhabricatorKeyValueDatabaseCache.php +++ src/applications/cache/PhabricatorKeyValueDatabaseCache.php @@ -19,7 +19,7 @@ $sql[] = qsprintf( $conn_w, - '(%s, %s, %s, %s, %d, %nd)', + '(%s, %s, %s, %B, %d, %nd)', $hash, $key, $format, Index: src/applications/files/storage/PhabricatorFileStorageBlob.php =================================================================== --- src/applications/files/storage/PhabricatorFileStorageBlob.php +++ src/applications/files/storage/PhabricatorFileStorageBlob.php @@ -2,46 +2,17 @@ /** * Simple blob store DAO for @{class:PhabricatorMySQLFileStorageEngine}. - * - * @group file */ final class PhabricatorFileStorageBlob extends PhabricatorFileDAO { - // max_allowed_packet defaults to 1 MiB, escaping can make the data twice - // longer, query fits in the rest. - const CHUNK_SIZE = 5e5; protected $data; - private $fullData; - - protected function willWriteData(array &$data) { - parent::willWriteData($data); - - $this->fullData = $data['data']; - if (strlen($data['data']) > self::CHUNK_SIZE) { - $data['data'] = substr($data['data'], 0, self::CHUNK_SIZE); - $this->openTransaction(); - } - } - - protected function didWriteData() { - $size = self::CHUNK_SIZE; - $length = strlen($this->fullData); - if ($length > $size) { - $conn = $this->establishConnection('w'); - for ($offset = $size; $offset < $length; $offset += $size) { - queryfx( - $conn, - 'UPDATE %T SET data = CONCAT(data, %s) WHERE %C = %d', - $this->getTableName(), - substr($this->fullData, $offset, $size), - $this->getIDKeyForUse(), - $this->getID()); - } - $this->saveTransaction(); - } - - parent::didWriteData(); + public function getConfiguration() { + return array( + self::CONFIG_BINARY => array( + 'data' => true, + ), + ) + parent::getConfiguration(); } } Index: src/applications/passphrase/storage/PassphraseSecret.php =================================================================== --- src/applications/passphrase/storage/PassphraseSecret.php +++ src/applications/passphrase/storage/PassphraseSecret.php @@ -7,6 +7,9 @@ public function getConfiguration() { return array( self::CONFIG_TIMESTAMPS => false, + self::CONFIG_BINARY => array( + 'secretData' => true, + ), ) + parent::getConfiguration(); } Index: src/applications/repository/storage/PhabricatorRepositoryPushLog.php =================================================================== --- src/applications/repository/storage/PhabricatorRepositoryPushLog.php +++ src/applications/repository/storage/PhabricatorRepositoryPushLog.php @@ -59,6 +59,9 @@ return array( self::CONFIG_AUX_PHID => true, self::CONFIG_TIMESTAMPS => false, + self::CONFIG_BINARY => array( + 'refNameRaw' => true, + ), ) + parent::getConfiguration(); } Index: src/applications/repository/storage/PhabricatorRepositoryRefCursor.php =================================================================== --- src/applications/repository/storage/PhabricatorRepositoryRefCursor.php +++ src/applications/repository/storage/PhabricatorRepositoryRefCursor.php @@ -24,6 +24,9 @@ public function getConfiguration() { return array( self::CONFIG_TIMESTAMPS => false, + self::CONFIG_BINARY => array( + 'refNameRaw' => true, + ), ) + parent::getConfiguration(); } Index: src/infrastructure/__tests__/PhabricatorInfrastructureTestCase.php =================================================================== --- src/infrastructure/__tests__/PhabricatorInfrastructureTestCase.php +++ src/infrastructure/__tests__/PhabricatorInfrastructureTestCase.php @@ -77,5 +77,28 @@ $this->assertEqual($buf, $read->getBigData()); } + public function testRejectMySQLBMPQueries() { + $table = new HarbormasterScratchTable(); + $conn_r = $table->establishConnection('w'); + + $snowman = "\xE2\x98\x83"; + $gclef = "\xF0\x9D\x84\x9E"; + + qsprintf($conn_r, 'SELECT %B', $snowman); + qsprintf($conn_r, 'SELECT %s', $snowman); + qsprintf($conn_r, 'SELECT %B', $gclef); + + $caught = null; + try { + qsprintf($conn_r, 'SELECT %s', $gclef); + } catch (AphrontQueryCharacterSetException $ex) { + $caught = $ex; + } + + $this->assertEqual( + true, + ($caught instanceof AphrontQueryCharacterSetException)); + } + } Index: src/infrastructure/storage/__tests__/QueryFormattingTestCase.php =================================================================== --- src/infrastructure/storage/__tests__/QueryFormattingTestCase.php +++ src/infrastructure/storage/__tests__/QueryFormattingTestCase.php @@ -35,6 +35,23 @@ $this->assertEqual( 'NULL', qsprintf($conn_r, '%ns', null)); + + $this->assertEqual( + "'', ''", + qsprintf($conn_r, '%Ls', array('x', 'y'))); + + $this->assertEqual( + "''", + qsprintf($conn_r, '%B', null)); + + $this->assertEqual( + "NULL", + qsprintf($conn_r, '%nB', null)); + + $this->assertEqual( + "'', ''", + qsprintf($conn_r, '%LB', array('x', 'y'))); } + } Index: src/infrastructure/storage/lisk/LiskDAO.php =================================================================== --- src/infrastructure/storage/lisk/LiskDAO.php +++ src/infrastructure/storage/lisk/LiskDAO.php @@ -171,6 +171,7 @@ const CONFIG_AUX_PHID = 'auxiliary-phid'; const CONFIG_SERIALIZATION = 'col-serialization'; const CONFIG_PARTIAL_OBJECTS = 'partial-objects'; + const CONFIG_BINARY = 'binary'; const SERIALIZATION_NONE = 'id'; const SERIALIZATION_JSON = 'json'; @@ -356,6 +357,11 @@ * directly access or assign protected members of your class (use the getters * and setters). * + * CONFIG_BINARY + * You can optionally provide a map of columns to a flag indicating that + * they store binary data. These columns will not raise an error when + * handling binary writes. + * * @return dictionary Map of configuration options to values. * * @task config @@ -1148,9 +1154,14 @@ } $conn = $this->establishConnection('w'); + $binary = $this->getBinaryColumns(); foreach ($map as $key => $value) { - $map[$key] = qsprintf($conn, '%C = %ns', $key, $value); + if (!empty($binary[$key])) { + $map[$key] = qsprintf($conn, '%C = %nB', $key, $value); + } else { + $map[$key] = qsprintf($conn, '%C = %ns', $key, $value); + } } $map = implode(', ', $map); @@ -1242,10 +1253,15 @@ $this->willWriteData($data); $columns = array_keys($data); + $binary = $this->getBinaryColumns(); foreach ($data as $key => $value) { try { - $data[$key] = qsprintf($conn, '%ns', $value); + if (!empty($binary[$key])) { + $data[$key] = qsprintf($conn, '%nB', $value); + } else { + $data[$key] = qsprintf($conn, '%ns', $value); + } } catch (AphrontQueryParameterException $parameter_exception) { throw new PhutilProxyException( pht( @@ -1803,4 +1819,8 @@ return $conn_w->getInsertID(); } + private function getBinaryColumns() { + return $this->getConfigOption(self::CONFIG_BINARY); + } + }