From 1a0acb0185de0e0e6711a97bac4aec8aa2a81e52 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 24 Jun 2017 06:07:38 +0000 Subject: [PATCH] switch from utf8 to utf8mb4 on the database level Signed-off-by: Janos SUTO --- etc/example.conf | 4 ++++ etc/sphinx.conf.in | 18 +++++++++--------- src/cfg.c | 1 + src/cfg.h | 1 + src/mysql.c | 8 ++++++-- util/db-mysql-root.sql.in | 2 +- util/db-mysql.sql | 26 +++++++++++++------------- webui/config.php | 1 + webui/system/database/mysql.php | 4 ++-- webui/system/database/sphinx.php | 2 +- 10 files changed, 39 insertions(+), 28 deletions(-) diff --git a/etc/example.conf b/etc/example.conf index 76b23cdc..16a7e775 100644 --- a/etc/example.conf +++ b/etc/example.conf @@ -195,6 +195,10 @@ memcached_to_db_interval=900 ; mysql stuff ; +// this can be either utf8 or utf8mb4. Make sure to match the value +// to the charset of the piler database! Also, make sure to set this +// value in sphinx.conf +mysqlcharset=utfmb4 ;mysqlhost=127.0.0.1 ;mysqlport=3306 mysqlsocket=/var/run/mysqld/mysqld.sock diff --git a/etc/sphinx.conf.in b/etc/sphinx.conf.in index 37c984ee..0af92020 100644 --- a/etc/sphinx.conf.in +++ b/etc/sphinx.conf.in @@ -17,7 +17,7 @@ source base source delta : base { - sql_query_pre = SET NAMES utf8 + sql_query_pre = SET NAMES utf8mb4 sql_query_pre = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index sql_query_post_index = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1) sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index \ @@ -28,37 +28,37 @@ source delta : base source main1 : base { - sql_query_pre = SET NAMES utf8 + sql_query_pre = SET NAMES utf8mb4 sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1 } source main2 : base { - sql_query_pre = SET NAMES utf8 + sql_query_pre = SET NAMES utf8mb4 sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1 } source main3 : base { - sql_query_pre = SET NAMES utf8 + sql_query_pre = SET NAMES utf8mb4 sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1 } source main4 : base { - sql_query_pre = SET NAMES utf8 + sql_query_pre = SET NAMES utf8mb4 sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1 } source dailydelta : base { - sql_query_pre = SET NAMES utf8 + sql_query_pre = SET NAMES utf8mb4 sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1 } source tag : base { - sql_query_pre = SET NAMES utf8 + sql_query_pre = SET NAMES utf8mb4 sql_query = SELECT `_id`, `id` AS iid, `uid`, `tag` FROM `tag` sql_attr_uint = iid @@ -68,7 +68,7 @@ source tag : base source note : base { - sql_query_pre = SET NAMES utf8 + sql_query_pre = SET NAMES utf8mb4 sql_query = SELECT `_id`, `id` AS iid, `uid`, `note` FROM `note` sql_attr_uint = iid @@ -79,7 +79,7 @@ source note : base source att : base { - sql_query_pre = SET NAMES utf8 + sql_query_pre = SET NAMES utf8mb4 sql_query = select a.id as aid, m.id as mid, a.name AS aname, a.size, REPLACE(REPLACE(m.`from`, '@', 'X'), '.', 'X') as `from`, REPLACE(REPLACE((select group_concat(rcpt.`to` ORDER BY `to` ASC SEPARATOR ' ') from rcpt where rcpt.id=mid group by rcpt.id), '@', 'X'), '.', 'X') as `to` from attachment a, metadata m where m.piler_id=a.piler_id sql_attr_uint = size diff --git a/src/cfg.c b/src/cfg.c index 53af84ad..f8e0df73 100644 --- a/src/cfg.c +++ b/src/cfg.c @@ -88,6 +88,7 @@ struct _parse_rule config_parse_rules[] = { "min_message_size", "integer", (void*) int_parser, offsetof(struct __config, min_message_size), "100", sizeof(int)}, { "min_word_len", "integer", (void*) int_parser, offsetof(struct __config, min_word_len), "1", sizeof(int)}, { "mmap_dedup_test", "integer", (void*) int_parser, offsetof(struct __config, mmap_dedup_test), "0", sizeof(int)}, + { "mysqlcharset", "string", (void*) string_parser, offsetof(struct __config, mysqlcharset), "utf8mb4", MAXVAL-1}, { "mysqlhost", "string", (void*) string_parser, offsetof(struct __config, mysqlhost), "", MAXVAL-1}, { "mysqlport", "integer", (void*) int_parser, offsetof(struct __config, mysqlport), "", sizeof(int)}, { "mysqlsocket", "string", (void*) string_parser, offsetof(struct __config, mysqlsocket), "/tmp/mysql.sock", MAXVAL-1}, diff --git a/src/cfg.h b/src/cfg.h index 18dc2190..29b81081 100644 --- a/src/cfg.h +++ b/src/cfg.h @@ -65,6 +65,7 @@ struct __config { // mysql stuff + char mysqlcharset[MAXVAL]; char mysqlhost[MAXVAL]; int mysqlport; char mysqlsocket[MAXVAL]; diff --git a/src/mysql.c b/src/mysql.c index d849b506..54124efe 100644 --- a/src/mysql.c +++ b/src/mysql.c @@ -11,6 +11,7 @@ int open_database(struct session_data *sdata, struct __config *cfg){ int rc=1; + char buf[BUFLEN]; mysql_init(&(sdata->mysql)); @@ -22,8 +23,11 @@ int open_database(struct session_data *sdata, struct __config *cfg){ return ERR; } - mysql_real_query(&(sdata->mysql), "SET NAMES utf8", strlen("SET NAMES utf8")); - mysql_real_query(&(sdata->mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8")); + snprintf(buf, sizeof(buf)-2, "SET NAMES %s", cfg->mysqlcharset); + mysql_real_query(&(sdata->mysql), buf, strlen(buf)); + + snprintf(buf, sizeof(buf)-2, "SET CHARACTER SET %s", cfg->mysqlcharset); + mysql_real_query(&(sdata->mysql), buf, strlen(buf)); return OK; } diff --git a/util/db-mysql-root.sql.in b/util/db-mysql-root.sql.in index be73e14c..38f614da 100644 --- a/util/db-mysql-root.sql.in +++ b/util/db-mysql-root.sql.in @@ -1,4 +1,4 @@ -create database MYSQL_DATABASE character set 'utf8'; +create database MYSQL_DATABASE character set 'utf8mb4'; grant all privileges on MYSQL_DATABASE.* to MYSQL_USERNAME@localhost identified by 'MYSQL_PASSWORD'; flush privileges; diff --git a/util/db-mysql.sql b/util/db-mysql.sql index f853ab93..2cd7143c 100644 --- a/util/db-mysql.sql +++ b/util/db-mysql.sql @@ -12,10 +12,10 @@ create table if not exists `sph_index` ( `to` text(8192) default null, `fromdomain` char(255) default null, `todomain` text(512) default null, - `subject` text(512) default null, + `subject` blob(512) default null, `arrived` int unsigned not null, `sent` int unsigned not null, - `body` text, + `body` mediumblob, `size` int default '0', `direction` int default 0, `folder` int default 0, @@ -29,7 +29,7 @@ create table if not exists `metadata` ( `id` bigint unsigned not null auto_increment, `from` varchar(128) not null, `fromdomain` varchar(64) not null, - `subject` text(512) default null, + `subject` blob(512) default null, `spam` tinyint(1) default 0, `arrived` int unsigned not null, `sent` int unsigned not null, @@ -117,7 +117,7 @@ create table if not exists `archiving_rule` ( `_size` char(2) default null, `size` int default 0, `attachment_name` varchar(128) default null, - `attachment_type` varchar(128) default null, + `attachment_type` varchar(64) default null, `_attachment_size` char(2) default null, `attachment_size` int default 0, `spam` tinyint(1) default -1, @@ -130,15 +130,15 @@ create table if not exists `archiving_rule` ( create table if not exists `retention_rule` ( `id` bigint unsigned not null auto_increment, - `domain` varchar(128) default null, - `from` varchar(128) default null, - `to` varchar(128) default null, + `domain` varchar(100) default null, + `from` varchar(100) default null, + `to` varchar(100) default null, `subject` varchar(128) default null, `body` varchar(128) default null, `_size` char(2) default null, `size` int default 0, - `attachment_name` varchar(128) default null, - `attachment_type` varchar(128) default null, + `attachment_name` varchar(100) default null, + `attachment_type` varchar(64) default null, `_attachment_size` char(2) default null, `attachment_size` int default 0, `spam` tinyint(1) default -1, @@ -151,15 +151,15 @@ create table if not exists `retention_rule` ( create table if not exists `folder_rule` ( `id` bigint unsigned not null auto_increment, - `domain` varchar(128) default null, - `from` varchar(128) default null, - `to` varchar(128) default null, + `domain` varchar(100) default null, + `from` varchar(100) default null, + `to` varchar(100) default null, `subject` varchar(128) default null, `body` varchar(128) default null, `_size` char(2) default null, `size` int default 0, `attachment_name` varchar(128) default null, - `attachment_type` varchar(128) default null, + `attachment_type` varchar(64) default null, `_attachment_size` char(2) default null, `attachment_size` int default 0, `spam` tinyint(1) default -1, diff --git a/webui/config.php b/webui/config.php index c2742f53..a95c8ee4 100644 --- a/webui/config.php +++ b/webui/config.php @@ -202,6 +202,7 @@ $config['DB_HOSTNAME'] = 'localhost'; $config['DB_USERNAME'] = 'piler'; $config['DB_PASSWORD'] = 'piler'; $config['DB_DATABASE'] = 'piler'; +$config['DB_CHARSET'] = 'utf8mb4'; $config['SPHINX_DRIVER'] = 'sphinx'; $config['SPHINX_DATABASE'] = 'sphinx'; diff --git a/webui/system/database/mysql.php b/webui/system/database/mysql.php index 82147d6e..6042e739 100644 --- a/webui/system/database/mysql.php +++ b/webui/system/database/mysql.php @@ -10,8 +10,8 @@ class MySQL { try { $this->link = new PDO("mysql:host=$hostname;dbname=$database", $username, $password, array( - PDO::MYSQL_ATTR_INIT_COMMAND => "SET NAMES utf8", - PDO::MYSQL_ATTR_INIT_COMMAND => "SET CHARACTER SET utf8" + PDO::MYSQL_ATTR_INIT_COMMAND => "SET NAMES " . DB_CHARSET, + PDO::MYSQL_ATTR_INIT_COMMAND => "SET CHARACTER SET " . DB_CHARSET ) ); diff --git a/webui/system/database/sphinx.php b/webui/system/database/sphinx.php index a4ec3e28..3b33d813 100644 --- a/webui/system/database/sphinx.php +++ b/webui/system/database/sphinx.php @@ -9,7 +9,7 @@ class Sphinx { list($host, $port) = explode(":", $hostname); try { - $this->link = new PDO("mysql:host=$host;port=$port;dbname=$database;charset=utf8", $username, $password); + $this->link = new PDO("mysql:host=$host;port=$port;dbname=$database;charset=" . DB_CHARSET, $username, $password); } catch(PDOException $exception) { exit('Error: ' . $exception->getMessage() . " on database: $database
");