Add configuration for database connection collation

matomo-org · Sep 6, 2024 · 29a4076 · 29a4076
1 parent 750fdce
commit 29a4076
Show file tree

Hide file tree

Showing 9 changed files with 175 additions and 7 deletions.
diff --git a/config/global.ini.php b/config/global.ini.php
@@ -45,6 +45,13 @@
 ; Matomo should work correctly without this setting but we recommend to have a charset set.
 charset = utf8
 
+; In some database setups the collation for the connection changes after a "SET NAMES" statement
+; is issued, unless a "COLLATE" argument is added.
+; If you encounter "Illegal mix of collation" errors, setting this config to the value matching
+; your existing database tables can help.
+; This setting will only be used if "charset" is also set.
+connection_collation =
+
 ; Database error codes to ignore during updates
 ;
 ;ignore_error_codes[] = 1105

diff --git a/core/Db.php b/core/Db.php
@@ -186,6 +186,7 @@ public static function createReaderDatabaseObject($dbConfig = null)
         $dbConfig['type'] = $masterDbConfig['type'];
         $dbConfig['tables_prefix'] = $masterDbConfig['tables_prefix'];
         $dbConfig['charset'] = $masterDbConfig['charset'];
+        $dbConfig['connection_collation'] = $masterDbConfig['connection_collation'] ?? null;
 
         $db = @Adapter::factory($dbConfig['adapter'], $dbConfig);
 

diff --git a/core/Tracker/Db/Mysqli.php b/core/Tracker/Db/Mysqli.php
@@ -26,6 +26,7 @@ class Mysqli extends Db
     protected $username;
     protected $password;
     protected $charset;
+    protected $connectionCollation;
     protected $activeTransaction = false;
 
     protected $enable_ssl;
@@ -57,11 +58,12 @@ public function __construct($dbInfo, $driverName = 'mysql')
             $this->port = (int)$dbInfo['port'];
             $this->socket = null;
         }
+
         $this->dbname = $dbInfo['dbname'];
         $this->username = $dbInfo['username'];
         $this->password = $dbInfo['password'];
-        $this->charset = isset($dbInfo['charset']) ? $dbInfo['charset'] : null;
-
+        $this->charset = $dbInfo['charset'] ?? null;
+        $this->connectionCollation = $dbInfo['connection_collation'] ?? null;
 
         if (!empty($dbInfo['enable_ssl'])) {
             $this->enable_ssl = $dbInfo['enable_ssl'];
@@ -133,8 +135,17 @@ public function connect()
             throw new DbException("Connect failed: " . mysqli_connect_error());
         }
 
-        if ($this->charset && !mysqli_set_charset($this->connection, $this->charset)) {
-            throw new DbException("Set Charset failed: " . mysqli_error($this->connection));
+        if ($this->charset && $this->connectionCollation) {
+            // mysqli_set_charset does not support setting a collation
+            $query = "SET NAMES '" . $this->charset . "' COLLATE '" . $this->connectionCollation . "'";
+
+            if (!mysqli_query($this->connection, $query)) {
+                throw new DbException("Set charset/connection collation failed: " . mysqli_error($this->connection));
+            }
+        } elseif ($this->charset) {
+            if (!mysqli_set_charset($this->connection, $this->charset)) {
+                throw new DbException("Set Charset failed: " . mysqli_error($this->connection));
+            }
         }
 
         $this->password = '';

diff --git a/core/Tracker/Db/Pdo/Mysql.php b/core/Tracker/Db/Pdo/Mysql.php
@@ -26,13 +26,33 @@ class Mysql extends Db
      * @var PDO
      */
     protected $connection = null;
+
+    /**
+     * @var string
+     */
     protected $dsn;
+
+    /**
+     * @var string
+     */
     private $username;
+
+    /**
+     * @var string
+     */
     private $password;
+
+    /**
+     * @var string|null
+     */
     protected $charset;
 
-    protected $mysqlOptions = array();
+    /**
+     * @var string|null
+     */
+    private $connectionCollation;
 
+    protected $mysqlOptions = [];
 
     protected $activeTransaction = false;
 
@@ -58,8 +78,11 @@ public function __construct($dbInfo, $driverName = 'mysql')
         if (isset($dbInfo['charset'])) {
             $this->charset = $dbInfo['charset'];
             $this->dsn .= ';charset=' . $this->charset;
-        }
 
+            if (!empty($dbInfo['connection_collation'])) {
+                $this->connectionCollation = $dbInfo['connection_collation'];
+            }
+        }
 
         if (isset($dbInfo['enable_ssl']) && $dbInfo['enable_ssl']) {
             if (!empty($dbInfo['ssl_key'])) {
@@ -409,6 +432,11 @@ private function establishConnection(): void
          */
         if (!empty($this->charset)) {
             $sql = "SET NAMES '" . $this->charset . "'";
+
+            if (!empty($this->connectionCollation)) {
+                $sql .= " COLLATE '" . $this->connectionCollation . "'";
+            }
+
             $this->connection->exec($sql);
         }
     }

diff --git a/libs/Zend/Db/Adapter/Mysqli.php b/libs/Zend/Db/Adapter/Mysqli.php
@@ -375,7 +375,12 @@ protected function _connect()
             throw new Zend_Db_Adapter_Mysqli_Exception(mysqli_connect_error());
         }
 
-        if (!empty($this->_config['charset'])) {
+        if (!empty($this->_config['charset']) && !empty($this->_config['connection_collation'])) {
+            // mysqli_set_charset does not support setting a collation
+            $query = "SET NAMES '" . $this->_config['charset'] . "' COLLATE '" . $this->_config['connection_collation'] . "'";
+
+            mysqli_query($this->_connection, $query);
+        } elseif (!empty($this->_config['charset'])) {
             mysqli_set_charset($this->_connection, $this->_config['charset']);
         }
     }

diff --git a/libs/Zend/Db/Adapter/Pdo/Mysql.php b/libs/Zend/Db/Adapter/Pdo/Mysql.php
@@ -103,6 +103,11 @@ protected function _connect()
 
         if (!empty($this->_config['charset'])) {
             $initCommand = "SET NAMES '" . $this->_config['charset'] . "'";
+
+            if (!empty($this->_config['connection_collation'])) {
+                $initCommand .= " COLLATE '" . $this->_config['connection_collation'] . "'";
+            }
+
             $this->_config['driver_options'][1002] = $initCommand; // 1002 = PDO::MYSQL_ATTR_INIT_COMMAND
         }
 

diff --git a/tests/PHPUnit/Integration/DbTest.php b/tests/PHPUnit/Integration/DbTest.php
@@ -257,6 +257,43 @@ public function testGetRowCount($adapter, $expectedClass)
         $this->assertEquals(1, $db->rowCount($result));
     }
 
+    /**
+     * @dataProvider getDbAdapter
+     */
+    public function testConnectionCollationDefault(string $adapter, string $expectedClass): void
+    {
+        Db::destroyDatabaseObject();
+
+        $config = Config::getInstance();
+        $config->database['adapter'] = $adapter;
+        $config->database['connection_collation'] = null;
+
+        $db = Db::get();
+        self::assertInstanceOf($expectedClass, $db);
+
+        // exact value depends on database used
+        $currentCollation = $db->fetchOne('SELECT @@collation_connection');
+        self::assertStringStartsWith('utf8', $currentCollation);
+    }
+
+    /**
+     * @dataProvider getDbAdapter
+     */
+    public function testConnectionCollationSetInConfig(string $adapter, string $expectedClass): void
+    {
+        Db::destroyDatabaseObject();
+
+        $config = Config::getInstance();
+        $config->database['adapter'] = $adapter;
+        $config->database['connection_collation'] = $config->database['charset'] . '_swedish_ci';
+
+        $db = Db::get();
+        self::assertInstanceOf($expectedClass, $db);
+
+        $currentCollation = $db->fetchOne('SELECT @@collation_connection');
+        self::assertSame($config->database['connection_collation'], $currentCollation);
+    }
+
     public function getDbAdapter()
     {
         return array(

diff --git a/tests/PHPUnit/Integration/Tracker/Db/MysqliTest.php b/tests/PHPUnit/Integration/Tracker/Db/MysqliTest.php
@@ -0,0 +1,53 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link    https://matomo.org
+ * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace PHPUnit\Integration\Tracker\Db;
+
+use Piwik\Config;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+use Piwik\Tracker;
+
+/**
+ * Tracker DB test
+ *
+ * @group Core
+ * @group TrackerDbTest
+ */
+class MysqliTest extends IntegrationTestCase
+{
+    public function setUp(): void
+    {
+        parent::setUp();
+
+        $config = Config::getInstance();
+        $config->database['adapter'] = 'MYSQLI';
+    }
+
+    public function testConnectionThrowsOnInvalidCharset(): void
+    {
+        self::expectException(Tracker\Db\DbException::class);
+        self::expectExceptionMessageMatches('/Set Charset failed/');
+
+        $config = Config::getInstance();
+        $config->database['charset'] = 'something really invalid';
+
+        Tracker\Db::connectPiwikTrackerDb();
+    }
+
+    public function testConnectionThrowsOnInvalidConnectionCollation(): void
+    {
+        self::expectException(Tracker\Db\DbException::class);
+        self::expectExceptionMessageMatches('/Set charset\/connection collation failed/');
+
+        $config = Config::getInstance();
+        $config->database['connection_collation'] = 'something really invalid';
+
+        Tracker\Db::connectPiwikTrackerDb();
+    }
+}
diff --git a/tests/PHPUnit/Integration/Tracker/DbTest.php b/tests/PHPUnit/Integration/Tracker/DbTest.php
@@ -192,6 +192,27 @@ public function testFetchAllNoMatch()
         $this->assertEquals(array(), $val);
     }
 
+    public function testConnectionCollationDefault(): void
+    {
+        $config = Config::getInstance();
+        $config->database['connection_collation'] = null;
+        $db = Tracker\Db::connectPiwikTrackerDb();
+
+        // exact value depends on database used
+        $currentCollation = $db->fetchOne('SELECT @@collation_connection');
+        self::assertStringStartsWith('utf8', $currentCollation);
+    }
+
+    public function testConnectionCollationSetInConfig(): void
+    {
+        $config = Config::getInstance();
+        $config->database['connection_collation'] = $config->database['charset'] . '_swedish_ci';
+        $db = Tracker\Db::connectPiwikTrackerDb();
+
+        $currentCollation = $db->fetchOne('SELECT @@collation_connection');
+        self::assertSame($config->database['connection_collation'], $currentCollation);
+    }
+
     private function insertRowId($value = '1')
     {
         $db = Tracker::getDatabase();