bashclub/piler
1
0
Fork 0
mirror of https://bitbucket.org/jsuto/piler.git synced 2024-11-07 20:01:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

switch from utf8 to utf8mb4 on the database level

Browse Source 
Signed-off-by: Janos SUTO <sj@acts.hu>
This commit is contained in:
Janos SUTO 2017-06-24 06:07:38 +00:00
parent 44760a7e0f
commit 1a0acb0185
10 changed files with 39 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
etc/example.conf
View File

@ -195,6 +195,10 @@ memcached_to_db_interval=900
; mysql stuff
;
// this can be either utf8 or utf8mb4. Make sure to match the value
// to the charset of the piler database! Also, make sure to set this
// value in sphinx.conf
mysqlcharset=utfmb4
;mysqlhost=127.0.0.1
;mysqlport=3306
mysqlsocket=/var/run/mysqld/mysqld.sock

18
etc/sphinx.conf.in
View File

@ -17,7 +17,7 @@ source base
source delta : base
{
sql_query_pre = SET NAMES utf8
sql_query_pre = SET NAMES utf8mb4
sql_query_pre = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index
sql_query_post_index = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index \
@ -28,37 +28,37 @@ source delta : base
source main1 : base
{
sql_query_pre = SET NAMES utf8
sql_query_pre = SET NAMES utf8mb4
sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
}
source main2 : base
{
sql_query_pre = SET NAMES utf8
sql_query_pre = SET NAMES utf8mb4
sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
}
source main3 : base
{
sql_query_pre = SET NAMES utf8
sql_query_pre = SET NAMES utf8mb4
sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
}
source main4 : base
{
sql_query_pre = SET NAMES utf8
sql_query_pre = SET NAMES utf8mb4
sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
}
source dailydelta : base
{
sql_query_pre = SET NAMES utf8
sql_query_pre = SET NAMES utf8mb4
sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
}
source tag : base
{
sql_query_pre = SET NAMES utf8
sql_query_pre = SET NAMES utf8mb4
sql_query = SELECT `_id`, `id` AS iid, `uid`, `tag` FROM `tag`
sql_attr_uint = iid
@ -68,7 +68,7 @@ source tag : base
source note : base
{
sql_query_pre = SET NAMES utf8
sql_query_pre = SET NAMES utf8mb4
sql_query = SELECT `_id`, `id` AS iid, `uid`, `note` FROM `note`
sql_attr_uint = iid
@ -79,7 +79,7 @@ source note : base
source att : base
{
sql_query_pre = SET NAMES utf8
sql_query_pre = SET NAMES utf8mb4
sql_query = select a.id as aid, m.id as mid, a.name AS aname, a.size, REPLACE(REPLACE(m.`from`, '@', 'X'), '.', 'X') as `from`, REPLACE(REPLACE((select group_concat(rcpt.`to` ORDER BY `to` ASC SEPARATOR ' ') from rcpt where rcpt.id=mid group by rcpt.id), '@', 'X'), '.', 'X') as `to` from attachment a, metadata m where m.piler_id=a.piler_id
sql_attr_uint = size

1
src/cfg.c
View File

@ -88,6 +88,7 @@ struct _parse_rule config_parse_rules[] =
{ "min_message_size", "integer", (void*) int_parser, offsetof(struct __config, min_message_size), "100", sizeof(int)},
{ "min_word_len", "integer", (void*) int_parser, offsetof(struct __config, min_word_len), "1", sizeof(int)},
{ "mmap_dedup_test", "integer", (void*) int_parser, offsetof(struct __config, mmap_dedup_test), "0", sizeof(int)},
{ "mysqlcharset", "string", (void*) string_parser, offsetof(struct __config, mysqlcharset), "utf8mb4", MAXVAL-1},
{ "mysqlhost", "string", (void*) string_parser, offsetof(struct __config, mysqlhost), "", MAXVAL-1},
{ "mysqlport", "integer", (void*) int_parser, offsetof(struct __config, mysqlport), "", sizeof(int)},
{ "mysqlsocket", "string", (void*) string_parser, offsetof(struct __config, mysqlsocket), "/tmp/mysql.sock", MAXVAL-1},

1
src/cfg.h
View File

@ -65,6 +65,7 @@ struct __config {
// mysql stuff
char mysqlcharset[MAXVAL];
char mysqlhost[MAXVAL];
int mysqlport;
char mysqlsocket[MAXVAL];

8
src/mysql.c
View File

@ -11,6 +11,7 @@
int open_database(struct session_data *sdata, struct __config *cfg){
int rc=1;
char buf[BUFLEN];
mysql_init(&(sdata->mysql));
@ -22,8 +23,11 @@ int open_database(struct session_data *sdata, struct __config *cfg){
return ERR;
}
mysql_real_query(&(sdata->mysql), "SET NAMES utf8", strlen("SET NAMES utf8"));
mysql_real_query(&(sdata->mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8"));
snprintf(buf, sizeof(buf)-2, "SET NAMES %s", cfg->mysqlcharset);
mysql_real_query(&(sdata->mysql), buf, strlen(buf));
snprintf(buf, sizeof(buf)-2, "SET CHARACTER SET %s", cfg->mysqlcharset);
mysql_real_query(&(sdata->mysql), buf, strlen(buf));
return OK;
}

2
util/db-mysql-root.sql.in
View File

@ -1,4 +1,4 @@
create database MYSQL_DATABASE character set 'utf8';
create database MYSQL_DATABASE character set 'utf8mb4';
grant all privileges on MYSQL_DATABASE.* to MYSQL_USERNAME@localhost identified by 'MYSQL_PASSWORD';
flush privileges;

26
util/db-mysql.sql
View File

@ -12,10 +12,10 @@ create table if not exists `sph_index` (
`to` text(8192) default null,
`fromdomain` char(255) default null,
`todomain` text(512) default null,
`subject` text(512) default null,
`subject` blob(512) default null,
`arrived` int unsigned not null,
`sent` int unsigned not null,
`body` text,
`body` mediumblob,
`size` int default '0',
`direction` int default 0,
`folder` int default 0,
@ -29,7 +29,7 @@ create table if not exists `metadata` (
`id` bigint unsigned not null auto_increment,
`from` varchar(128) not null,
`fromdomain` varchar(64) not null,
`subject` text(512) default null,
`subject` blob(512) default null,
`spam` tinyint(1) default 0,
`arrived` int unsigned not null,
`sent` int unsigned not null,
@ -117,7 +117,7 @@ create table if not exists `archiving_rule` (
`_size` char(2) default null,
`size` int default 0,
`attachment_name` varchar(128) default null,
`attachment_type` varchar(128) default null,
`attachment_type` varchar(64) default null,
`_attachment_size` char(2) default null,
`attachment_size` int default 0,
`spam` tinyint(1) default -1,
@ -130,15 +130,15 @@ create table if not exists `archiving_rule` (
create table if not exists `retention_rule` (
`id` bigint unsigned not null auto_increment,
`domain` varchar(128) default null,
`from` varchar(128) default null,
`to` varchar(128) default null,
`domain` varchar(100) default null,
`from` varchar(100) default null,
`to` varchar(100) default null,
`subject` varchar(128) default null,
`body` varchar(128) default null,
`_size` char(2) default null,
`size` int default 0,
`attachment_name` varchar(128) default null,
`attachment_type` varchar(128) default null,
`attachment_name` varchar(100) default null,
`attachment_type` varchar(64) default null,
`_attachment_size` char(2) default null,
`attachment_size` int default 0,
`spam` tinyint(1) default -1,
@ -151,15 +151,15 @@ create table if not exists `retention_rule` (
create table if not exists `folder_rule` (
`id` bigint unsigned not null auto_increment,
`domain` varchar(128) default null,
`from` varchar(128) default null,
`to` varchar(128) default null,
`domain` varchar(100) default null,
`from` varchar(100) default null,
`to` varchar(100) default null,
`subject` varchar(128) default null,
`body` varchar(128) default null,
`_size` char(2) default null,
`size` int default 0,
`attachment_name` varchar(128) default null,
`attachment_type` varchar(128) default null,
`attachment_type` varchar(64) default null,
`_attachment_size` char(2) default null,
`attachment_size` int default 0,
`spam` tinyint(1) default -1,

1
webui/config.php
View File

@ -202,6 +202,7 @@ $config['DB_HOSTNAME'] = 'localhost';
$config['DB_USERNAME'] = 'piler';
$config['DB_PASSWORD'] = 'piler';
$config['DB_DATABASE'] = 'piler';
$config['DB_CHARSET'] = 'utf8mb4';
$config['SPHINX_DRIVER'] = 'sphinx';
$config['SPHINX_DATABASE'] = 'sphinx';

4
webui/system/database/mysql.php
View File

@ -10,8 +10,8 @@ class MySQL {
try {
$this->link = new PDO("mysql:host=$hostname;dbname=$database", $username, $password,
array(
PDO::MYSQL_ATTR_INIT_COMMAND => "SET NAMES utf8",
PDO::MYSQL_ATTR_INIT_COMMAND => "SET CHARACTER SET utf8"
PDO::MYSQL_ATTR_INIT_COMMAND => "SET NAMES " . DB_CHARSET,
PDO::MYSQL_ATTR_INIT_COMMAND => "SET CHARACTER SET " . DB_CHARSET
)
);

2
webui/system/database/sphinx.php
View File

@ -9,7 +9,7 @@ class Sphinx {
list($host, $port) = explode(":", $hostname);
try {
$this->link = new PDO("mysql:host=$host;port=$port;dbname=$database;charset=utf8", $username, $password);
$this->link = new PDO("mysql:host=$host;port=$port;dbname=$database;charset=" . DB_CHARSET, $username, $password);
}
catch(PDOException $exception) {
exit('Error: ' . $exception->getMessage() . " on database: $database<br />");