switch from utf8 to utf8mb4 on the database level

Signed-off-by: Janos SUTO <sj@acts.hu>

etc/example.conf

@@ -195,6 +195,10 @@ memcached_to_db_interval=900
 ; mysql stuff
 ;
 
+// this can be either utf8 or utf8mb4. Make sure to match the value
+// to the charset of the piler database! Also, make sure to set this
+// value in sphinx.conf
+mysqlcharset=utfmb4
 ;mysqlhost=127.0.0.1
 ;mysqlport=3306
 mysqlsocket=/var/run/mysqld/mysqld.sock

etc/sphinx.conf.in

@@ -17,7 +17,7 @@ source base
 
 source delta : base
 {
-   sql_query_pre = SET NAMES utf8
+   sql_query_pre = SET NAMES utf8mb4
    sql_query_pre  = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index
    sql_query_post_index  = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
    sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index \
@@ -28,37 +28,37 @@ source delta : base
 
 source main1 : base
 {
-   sql_query_pre = SET NAMES utf8
+   sql_query_pre = SET NAMES utf8mb4
    sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
 }
 
 source main2 : base
 {
-   sql_query_pre = SET NAMES utf8
+   sql_query_pre = SET NAMES utf8mb4
    sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
 }
 
 source main3 : base
 {
-   sql_query_pre = SET NAMES utf8
+   sql_query_pre = SET NAMES utf8mb4
    sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
 }
 
 source main4 : base
 {
-   sql_query_pre = SET NAMES utf8
+   sql_query_pre = SET NAMES utf8mb4
    sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
 }
 
 source dailydelta : base
 {
-   sql_query_pre = SET NAMES utf8
+   sql_query_pre = SET NAMES utf8mb4
    sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
 }
 
 source tag : base
 {
-   sql_query_pre = SET NAMES utf8
+   sql_query_pre = SET NAMES utf8mb4
    sql_query  = SELECT `_id`, `id` AS iid, `uid`, `tag` FROM `tag`
 
    sql_attr_uint = iid
@@ -68,7 +68,7 @@ source tag : base
 
 source note : base
 {
-   sql_query_pre = SET NAMES utf8
+   sql_query_pre = SET NAMES utf8mb4
    sql_query  = SELECT `_id`, `id` AS iid, `uid`, `note` FROM `note`
 
    sql_attr_uint = iid
@@ -79,7 +79,7 @@ source note : base
 source att : base
 {
 
-   sql_query_pre = SET NAMES utf8
+   sql_query_pre = SET NAMES utf8mb4
    sql_query  =  select a.id as aid, m.id as mid, a.name AS aname, a.size, REPLACE(REPLACE(m.`from`, '@', 'X'), '.', 'X') as `from`, REPLACE(REPLACE((select group_concat(rcpt.`to` ORDER BY `to` ASC SEPARATOR ' ') from rcpt where rcpt.id=mid group by rcpt.id), '@', 'X'), '.', 'X') as `to` from attachment a, metadata m where m.piler_id=a.piler_id
 
    sql_attr_uint = size

src/cfg.c

@@ -88,6 +88,7 @@ struct _parse_rule config_parse_rules[] =
    { "min_message_size", "integer", (void*) int_parser, offsetof(struct __config, min_message_size), "100", sizeof(int)},
    { "min_word_len", "integer", (void*) int_parser, offsetof(struct __config, min_word_len), "1", sizeof(int)},
    { "mmap_dedup_test", "integer", (void*) int_parser, offsetof(struct __config, mmap_dedup_test), "0", sizeof(int)},
+   { "mysqlcharset", "string", (void*) string_parser, offsetof(struct __config, mysqlcharset), "utf8mb4", MAXVAL-1},
    { "mysqlhost", "string", (void*) string_parser, offsetof(struct __config, mysqlhost), "", MAXVAL-1},
    { "mysqlport", "integer", (void*) int_parser, offsetof(struct __config, mysqlport), "", sizeof(int)},
    { "mysqlsocket", "string", (void*) string_parser, offsetof(struct __config, mysqlsocket), "/tmp/mysql.sock", MAXVAL-1},

src/cfg.h

@@ -65,6 +65,7 @@ struct __config {
 
    // mysql stuff
 
+   char mysqlcharset[MAXVAL];
    char mysqlhost[MAXVAL];
    int mysqlport;
    char mysqlsocket[MAXVAL];

src/mysql.c

@@ -11,6 +11,7 @@
 
 int open_database(struct session_data *sdata, struct __config *cfg){
    int rc=1;
+   char buf[BUFLEN];
 
    mysql_init(&(sdata->mysql));
 
@@ -22,8 +23,11 @@ int open_database(struct session_data *sdata, struct __config *cfg){
       return ERR;
    }
 
-   mysql_real_query(&(sdata->mysql), "SET NAMES utf8", strlen("SET NAMES utf8"));
+   snprintf(buf, sizeof(buf)-2, "SET NAMES %s", cfg->mysqlcharset);
-   mysql_real_query(&(sdata->mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8"));
+   mysql_real_query(&(sdata->mysql), buf, strlen(buf));
+
+   snprintf(buf, sizeof(buf)-2, "SET CHARACTER SET %s", cfg->mysqlcharset);
+   mysql_real_query(&(sdata->mysql), buf, strlen(buf));
 
    return OK;
 }

util/db-mysql-root.sql.in

@@ -1,4 +1,4 @@
-create database MYSQL_DATABASE character set 'utf8';
+create database MYSQL_DATABASE character set 'utf8mb4';
 grant all privileges on MYSQL_DATABASE.* to MYSQL_USERNAME@localhost identified by 'MYSQL_PASSWORD';
 flush privileges;
 

util/db-mysql.sql

@@ -12,10 +12,10 @@ create table if not exists `sph_index` (
   `to` text(8192) default null,
   `fromdomain` char(255) default null,
   `todomain` text(512) default null,
-  `subject` text(512) default null,
+  `subject` blob(512) default null,
   `arrived` int unsigned not null,
   `sent` int unsigned not null,
-  `body` text,
+  `body` mediumblob,
   `size` int default '0',
   `direction` int default 0,
   `folder` int default 0,
@@ -29,7 +29,7 @@ create table if not exists `metadata` (
   `id` bigint unsigned not null auto_increment,
   `from` varchar(128) not null,
   `fromdomain` varchar(64) not null,
-  `subject` text(512) default null,
+  `subject` blob(512) default null,
   `spam` tinyint(1) default 0,
   `arrived` int unsigned not null,
   `sent` int unsigned not null,
@@ -117,7 +117,7 @@ create table if not exists `archiving_rule` (
    `_size` char(2) default null,
    `size` int default 0,
    `attachment_name` varchar(128) default null,
-   `attachment_type` varchar(128) default null,
+   `attachment_type` varchar(64) default null,
    `_attachment_size` char(2) default null,
    `attachment_size` int default 0,
    `spam` tinyint(1) default -1,
@@ -130,15 +130,15 @@ create table if not exists `archiving_rule` (
 
 create table if not exists `retention_rule` (
    `id` bigint unsigned not null auto_increment,
-   `domain` varchar(128) default null,
+   `domain` varchar(100) default null,
-   `from` varchar(128) default null,
+   `from` varchar(100) default null,
-   `to` varchar(128) default null,
+   `to` varchar(100) default null,
    `subject` varchar(128) default null,
    `body` varchar(128) default null,
    `_size` char(2) default null,
    `size` int default 0,
-   `attachment_name` varchar(128) default null,
+   `attachment_name` varchar(100) default null,
-   `attachment_type` varchar(128) default null,
+   `attachment_type` varchar(64) default null,
    `_attachment_size` char(2) default null,
    `attachment_size` int default 0,
    `spam` tinyint(1) default -1,
@@ -151,15 +151,15 @@ create table if not exists `retention_rule` (
 
 create table if not exists `folder_rule` (
    `id` bigint unsigned not null auto_increment,
-   `domain` varchar(128) default null,
+   `domain` varchar(100) default null,
-   `from` varchar(128) default null,
+   `from` varchar(100) default null,
-   `to` varchar(128) default null,
+   `to` varchar(100) default null,
    `subject` varchar(128) default null,
    `body` varchar(128) default null,
    `_size` char(2) default null,
    `size` int default 0,
    `attachment_name` varchar(128) default null,
-   `attachment_type` varchar(128) default null,
+   `attachment_type` varchar(64) default null,
    `_attachment_size` char(2) default null,
    `attachment_size` int default 0,
    `spam` tinyint(1) default -1,

webui/config.php

@@ -202,6 +202,7 @@ $config['DB_HOSTNAME'] = 'localhost';
 $config['DB_USERNAME'] = 'piler';
 $config['DB_PASSWORD'] = 'piler';
 $config['DB_DATABASE'] = 'piler';
+$config['DB_CHARSET'] = 'utf8mb4';
 
 $config['SPHINX_DRIVER'] = 'sphinx';
 $config['SPHINX_DATABASE'] = 'sphinx';

webui/system/database/mysql.php

@@ -10,8 +10,8 @@ class MySQL {
       try {
          $this->link = new PDO("mysql:host=$hostname;dbname=$database", $username, $password,
                                   array(
-                                     PDO::MYSQL_ATTR_INIT_COMMAND => "SET NAMES utf8",
+                                     PDO::MYSQL_ATTR_INIT_COMMAND => "SET NAMES " . DB_CHARSET,
-                                     PDO::MYSQL_ATTR_INIT_COMMAND => "SET CHARACTER SET utf8"
+                                     PDO::MYSQL_ATTR_INIT_COMMAND => "SET CHARACTER SET " . DB_CHARSET
                                   )
                               );
 

webui/system/database/sphinx.php

@@ -9,7 +9,7 @@ class Sphinx {
       list($host, $port) = explode(":", $hostname);
 
       try {
-         $this->link = new PDO("mysql:host=$host;port=$port;dbname=$database;charset=utf8", $username, $password);
+         $this->link = new PDO("mysql:host=$host;port=$port;dbname=$database;charset=" . DB_CHARSET, $username, $password);
       }
       catch(PDOException $exception) {
          exit('Error: ' . $exception->getMessage() . " on database: $database<br />");