From fa52000e8d611a67743aef9453078f65f41580f8 Mon Sep 17 00:00:00 2001 From: SJ Date: Sat, 3 Dec 2011 23:05:00 +0100 Subject: [PATCH] 0.1.9 --- README | 3 + etc/sphinx.conf | 6 +- src/config.h | 3 +- src/message.c | 237 ++++++++++++++++++++++++++++++++-------------- src/store.c | 2 +- util/db-mysql.sql | 17 +++- 6 files changed, 186 insertions(+), 82 deletions(-) diff --git a/README b/README index 6613155b..dc804bbd 100644 --- a/README +++ b/README @@ -8,6 +8,9 @@ Features: locales debian alatt: dpkg-reconfigure locales +init: + indexer --all + searchd inditasa indexer delta1 --rotate diff --git a/etc/sphinx.conf b/etc/sphinx.conf index d8a88e8e..93f7f8ed 100644 --- a/etc/sphinx.conf +++ b/etc/sphinx.conf @@ -11,14 +11,13 @@ source main sql_pass = sphinx sql_query_pre = SET NAMES utf8 - sql_query = SELECT id, `from`, `to`, `subject`, `arrived`, `sent`, `body`, `size`, `attachments`, `piler_id` FROM sph_index \ + sql_query = SELECT id, `from`, `to`, `subject`, `arrived`, `sent`, `body`, `size`, `attachments` FROM sph_index \ WHERE id<=( SELECT max_doc_id FROM sph_counter WHERE counter_id=1 ) sql_attr_uint = size sql_attr_uint = arrived sql_attr_uint = sent sql_attr_uint = attachments - sql_attr_string = piler_id } source delta @@ -32,14 +31,13 @@ source delta sql_query_pre = SET NAMES utf8 sql_query_pre = REPLACE INTO sph_counter SELECT 1, MAX(id) FROM sph_index sql_query_post_index = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1) - sql_query = SELECT id, `from`, `to`, `subject`, `arrived`, `sent`, `body`, `size`, `attachments`, `piler_id` FROM sph_index \ + sql_query = SELECT id, `from`, `to`, `subject`, `arrived`, `sent`, `body`, `size`, `attachments` FROM sph_index \ WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1) sql_attr_uint = size sql_attr_uint = arrived sql_attr_uint = sent sql_attr_uint = attachments - sql_attr_string = piler_id } diff --git a/src/config.h b/src/config.h index 6a38416f..55378c7f 100644 --- a/src/config.h +++ b/src/config.h @@ -11,7 +11,7 @@ #define PROGNAME "piler" -#define VERSION "0.1.8" +#define VERSION "0.1.9" #define PROGINFO VERSION ", Janos SUTO \n\n" CONFIGURE_PARAMS "\n\nSend bugs/issues to https://jira.acts.hu:8443/\n" @@ -74,6 +74,7 @@ #define SQL_SPHINX_TABLE "sph_index" #define SQL_METADATA_TABLE "metadata" #define SQL_ATTACHMENT_TABLE "attachment" +#define SQL_RECIPIENT_TABLE "rcpt" #define SQL_ARCHIVING_RULE_TABLE "archiving_rule" #define SQL_COUNTER_TABLE "counter" diff --git a/src/message.c b/src/message.c index a59f17ca..9e7fb9f4 100644 --- a/src/message.c +++ b/src/message.c @@ -99,8 +99,6 @@ int is_body_digest_already_stored(struct session_data *sdata, struct _state *sta snprintf(s, SMALLBUFSIZE-1, "SELECT `bodydigest` FROM `%s` WHERE `bodydigest`='%s'", SQL_METADATA_TABLE, sdata->bodydigest); - //if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: check for body digest sql: *%s*", sdata->ttmpfile, s); - if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){ res = mysql_store_result(&(sdata->mysql)); if(res != NULL){ @@ -114,114 +112,210 @@ int is_body_digest_already_stored(struct session_data *sdata, struct _state *sta } -int hand_to_sphinx(struct session_data *sdata, struct _state *state, struct __config *cfg){ +int store_index_data(struct session_data *sdata, struct _state *state, uint64 id, struct __config *cfg){ int rc; - char *subj, s[BIGBUFSIZE+2*MAXBUFSIZE]; - - subj = state->b_subject; - if(*subj == ' ') subj++; - - snprintf(s, sizeof(s)-1, "INSERT INTO %s (`from`, `to`, `subject`, `body`, `arrived`, `sent`, `size`, `attachments`, `piler_id`) values('%s','%s','%s','%s',%ld,%ld,%d,%d,'%s')", SQL_SPHINX_TABLE, state->b_from, state->b_to, subj, state->b_body, sdata->now, sdata->sent, sdata->tot_len, state->n_attachments, sdata->ttmpfile); - - rc = mysql_real_query(&(sdata->mysql), s, strlen(s)); - - if(rc == 0) return OK; - - syslog(LOG_PRIORITY, "%s: sphinx sql error: *%s*", sdata->ttmpfile, mysql_error(&(sdata->mysql))); - - return ERR; -} - - -int store_meta_data(struct session_data *sdata, struct _state *state, struct __config *cfg){ - int i=0, rc, ret=ERR; - char *p, *subj, s[MAXBUFSIZE], s2[SMALLBUFSIZE]; + char *subj, s[SMALLBUFSIZE]; MYSQL_STMT *stmt; MYSQL_BIND bind[4]; unsigned long len[4]; - stmt = mysql_stmt_init(&(sdata->mysql)); - if(!stmt){ - if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_METADATA_TABLE); - goto ENDE_META; - } - subj = state->b_subject; if(*subj == ' ') subj++; - snprintf(s, MAXBUFSIZE-1, "INSERT INTO %s (`from`,`to`,`subject`,`arrived`,`sent`,`size`,`hlen`,`attachments`,`piler_id`,`message_id`,`digest`,`bodydigest`) VALUES(?,?,?,%ld,%ld,%d,%d,%d,'%s',?,'%s','%s')", SQL_METADATA_TABLE, sdata->now, sdata->sent, sdata->tot_len, sdata->hdr_len, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest); - if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: meta sql: *%s*", sdata->ttmpfile, s); + stmt = mysql_stmt_init(&(sdata->mysql)); + if(!stmt){ + if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_SPHINX_TABLE); + return ERR; + } + + + snprintf(s, sizeof(s)-1, "INSERT INTO %s (`id`, `from`, `to`, `subject`, `body`, `arrived`, `sent`, `size`, `attachments`) values(%llu,?,?,?,?,%ld,%ld,%d,%d)", SQL_SPHINX_TABLE, id, sdata->now, sdata->sent, sdata->tot_len, state->n_attachments); + if(mysql_stmt_prepare(stmt, s, strlen(s))){ - syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt)); - goto ENDE_META; + syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_stmt_error(stmt)); + return ERR; } - if(strlen(state->b_to) < 5){ - snprintf(s2, sizeof(s2)-1, "undisclosed-recipients"); - p = NULL; - goto LABEL1; - } - else p = state->b_to; + memset(bind, 0, sizeof(bind)); + bind[0].buffer_type = MYSQL_TYPE_STRING; + bind[0].buffer = state->b_from; + bind[0].is_null = 0; + len[0] = strlen(state->b_from); bind[0].length = &len[0]; + + bind[1].buffer_type = MYSQL_TYPE_STRING; + bind[1].buffer = state->b_to; + bind[1].is_null = 0; + len[1] = strlen(state->b_to); bind[1].length = &len[1]; + + bind[2].buffer_type = MYSQL_TYPE_STRING; + bind[2].buffer = subj; + bind[2].is_null = 0; + len[2] = strlen(subj); bind[2].length = &len[2]; + + bind[3].buffer_type = MYSQL_TYPE_STRING; + bind[3].buffer = state->b_body; + bind[3].is_null = 0; + len[3] = strlen(state->b_body); bind[3].length = &len[3]; + + + if(mysql_stmt_bind_param(stmt, bind)){ + syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_stmt_error(stmt)); + return ERR; + } + + + rc = mysql_stmt_execute(stmt); + + if(rc){ + syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute error: *%s*", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_error(&(sdata->mysql))); + return ERR; + } + + + return OK; +} + + +int store_recipients(struct session_data *sdata, char *to, uint64 id, struct __config *cfg){ + int rc, ret=OK; + char *p, s[SMALLBUFSIZE], puf[SMALLBUFSIZE]; + + MYSQL_STMT *stmt; + MYSQL_BIND bind[1]; + unsigned long len[1]; + + stmt = mysql_stmt_init(&(sdata->mysql)); + if(!stmt){ + if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_RECIPIENT_TABLE); + return ERR; + } + + snprintf(s, sizeof(s)-1, "INSERT INTO %s (`id`,`to`) VALUES('%llu',?)", SQL_RECIPIENT_TABLE, id); + + if(mysql_stmt_prepare(stmt, s, strlen(s))){ + syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_stmt_error(stmt)); + return ERR; + } + + p = to; do { - p = split_str(p, " ", s2, sizeof(s2)-1); - if(strlen(s2) > 5){ -LABEL1: + p = split_str(p, " ", puf, sizeof(puf)-1); - i++; + if(strlen(puf) > 5){ memset(bind, 0, sizeof(bind)); bind[0].buffer_type = MYSQL_TYPE_STRING; - bind[0].buffer = state->b_from; + bind[0].buffer = &puf[0]; bind[0].is_null = 0; - len[0] = strlen(state->b_from); bind[0].length = &len[0]; - - bind[1].buffer_type = MYSQL_TYPE_STRING; - bind[1].buffer = s2; - bind[1].is_null = 0; - len[1] = strlen(s2); bind[1].length = &len[1]; - - bind[2].buffer_type = MYSQL_TYPE_STRING; - bind[2].buffer = subj; - bind[2].is_null = 0; - len[2] = strlen(subj); bind[2].length = &len[2]; - - bind[3].buffer_type = MYSQL_TYPE_STRING; - bind[3].buffer = state->message_id; - bind[3].is_null = 0; - len[3] = strlen(state->message_id); bind[3].length = &len[3]; + len[0] = strlen(puf); bind[0].length = &len[0]; if(mysql_stmt_bind_param(stmt, bind)){ - syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt)); - goto ENDE_META; + syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_stmt_error(stmt)); + return ERR; } rc = mysql_stmt_execute(stmt); if(rc){ - syslog(LOG_PRIORITY, "%s: meta sql error: *%s*", sdata->ttmpfile, mysql_error(&(sdata->mysql))); - - ret = ERR_EXISTS; - goto ENDE_META; + syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute error: *%s*", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_error(&(sdata->mysql))); + ret = ERR; } } } while(p); - - if(i == 0) ret = ERR_EXISTS; - else ret = OK; + return ret; +} -ENDE_META: +int store_meta_data(struct session_data *sdata, struct _state *state, struct __config *cfg){ + int rc, ret=ERR; + char *subj, s[MAXBUFSIZE]; + + MYSQL_STMT *stmt; + MYSQL_BIND bind[4]; + unsigned long len[4]; + + my_ulonglong id=0; + + + stmt = mysql_stmt_init(&(sdata->mysql)); + if(!stmt){ + if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_METADATA_TABLE); + return ERR; + } + + subj = state->b_subject; + if(*subj == ' ') subj++; + + snprintf(s, MAXBUFSIZE-1, "INSERT INTO %s (`from`,`subject`,`arrived`,`sent`,`size`,`hlen`,`attachments`,`piler_id`,`message_id`,`digest`,`bodydigest`) VALUES(?,?,%ld,%ld,%d,%d,%d,'%s',?,'%s','%s')", SQL_METADATA_TABLE, sdata->now, sdata->sent, sdata->tot_len, sdata->hdr_len, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest); + + if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: meta sql: *%s*", sdata->ttmpfile, s); + + if(mysql_stmt_prepare(stmt, s, strlen(s))){ + syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt)); + return ERR; + } + + + + if(strlen(state->b_to) < 5){ + snprintf(state->b_to, SMALLBUFSIZE-1, "undisclosed-recipients"); + } + + memset(bind, 0, sizeof(bind)); + + bind[0].buffer_type = MYSQL_TYPE_STRING; + bind[0].buffer = state->b_from; + bind[0].is_null = 0; + len[0] = strlen(state->b_from); bind[0].length = &len[0]; + + bind[1].buffer_type = MYSQL_TYPE_STRING; + bind[1].buffer = subj; + bind[1].is_null = 0; + len[1] = strlen(subj); bind[1].length = &len[1]; + + bind[2].buffer_type = MYSQL_TYPE_STRING; + bind[2].buffer = state->message_id; + bind[2].is_null = 0; + len[2] = strlen(state->message_id); bind[2].length = &len[2]; + + if(mysql_stmt_bind_param(stmt, bind)){ + syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt)); + return ERR; + } + + + rc = mysql_stmt_execute(stmt); + + if(rc){ + syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute() error: *%s*", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_error(&(sdata->mysql))); + ret = ERR_EXISTS; + } + else { + id = mysql_stmt_insert_id(stmt); + rc = store_recipients(sdata, state->b_to, id, cfg); + + if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored recipients, rc=%d", sdata->ttmpfile, rc); + + if(rc == OK){ + rc = store_index_data(sdata, state, id, cfg); + + if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored indexdata, rc=%d", sdata->ttmpfile, rc); + + if(rc == OK) + ret = OK; + } + } return ret; } @@ -271,9 +365,6 @@ int processMessage(struct session_data *sdata, struct _state *state, struct __co if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored metadata, rc=%d", sdata->ttmpfile, rc); if(rc == ERR_EXISTS) return ERR_EXISTS; - rc = hand_to_sphinx(sdata, state, cfg); - if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored indexdata, rc=%d", sdata->ttmpfile, rc); - return OK; } diff --git a/src/store.c b/src/store.c index 6108816b..08d8a09f 100644 --- a/src/store.c +++ b/src/store.c @@ -119,7 +119,6 @@ int store_file(struct session_data *sdata, char *filename, int startpos, int len strncat(s, p, sizeof(s)-1); } - if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to store %d bytes for %s", sdata->ttmpfile, len, filename); p0 = strrchr(s, '/'); if(!p0) goto ENDE; *p0 = '\0'; @@ -150,6 +149,7 @@ int store_file(struct session_data *sdata, char *filename, int startpos, int len if(n == outlen){ ret = 1; + if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored '%s' %d/%d bytes", sdata->ttmpfile, filename, len, outlen); } else { syslog(LOG_PRIORITY, "%s: cannot write %d bytes (only %d)", sdata->ttmpfile, outlen, n); diff --git a/util/db-mysql.sql b/util/db-mysql.sql index 18852614..a5a7a539 100644 --- a/util/db-mysql.sql +++ b/util/db-mysql.sql @@ -7,7 +7,7 @@ create table `sph_counter` ( drop table if exists `sph_index`; create table `sph_index` ( - `id` int not null auto_increment, + `id` bigint not null, `from` char(255) default null, `to` text(512) default null, `subject` text(512) default null, @@ -16,7 +16,6 @@ create table `sph_index` ( `body` text, `size` int default '0', `attachments` int default 0, - `piler_id` char(36) not null, primary key (`id`) ) Engine=InnoDB; @@ -25,7 +24,6 @@ drop table if exists `metadata`; create table `metadata` ( `id` bigint unsigned not null auto_increment, `from` char(255) not null, - `to` text(2048) character set 'latin1' not null, `subject` text(512) default null, `arrived` int not null, `sent` int not null, @@ -44,6 +42,18 @@ create index metadata_idx on metadata(`piler_id`); create index metadata_idx2 on metadata(`message_id`); create index metadata_idx3 on metadata(`bodydigest`); + +drop table if exists `rcpt`; +create table `rcpt` ( + `id` bigint unsigned not null, + `to` char(64) not null, + unique(`id`,`to`) +) Engine=InnoDB; + +create index `rcpt_idx` on `rcpt`(`id`); +create index `rcpt_idx2` on `rcpt`(`to`); + + drop table if exists `attachment`; create table `attachment` ( `id` bigint unsigned not null auto_increment, @@ -60,6 +70,7 @@ create table `attachment` ( create index `attachment_idx` on `attachment`(`piler_id`); create index `attachment_idx2` on `attachment`(`sig`); + drop table if exists `archiving_rule`; create table `archiving_rule` ( `id` bigint unsigned not null auto_increment,