bashclub/piler
1
0
Fork 0
mirror of https://bitbucket.org/jsuto/piler.git synced 2024-11-08 00:41:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

added domain information to index table

Browse Source
This commit is contained in:
SJ 2012-01-14 09:53:26 +01:00
parent c74ed7195a
commit 14b9d0fe86
7 changed files with 66 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
etc/sphinx.conf
View File

@ -11,7 +11,7 @@ source main
sql_pass = sphinx
sql_query_pre = SET NAMES utf8
sql_query = SELECT id, `from`, `to`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `attachments` FROM sph_index \
sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `attachments` FROM sph_index \
WHERE id<=( SELECT max_doc_id FROM sph_counter WHERE counter_id=1 )
sql_attr_uint = size
@ -33,7 +33,7 @@ source delta
sql_query_pre = SET NAMES utf8
sql_query_pre = REPLACE INTO sph_counter SELECT 1, MAX(id) FROM sph_index
sql_query_post_index = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
sql_query = SELECT id, `from`, `to`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `attachments` FROM sph_index \
sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `attachments` FROM sph_index \
WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
sql_attr_uint = size

3
src/defs.h
View File

@ -139,11 +139,12 @@ struct _state {
struct list *boundaries;
struct list *rcpt;
struct list *rcpt_domain;
int n_attachments;
struct attachment attachments[MAX_ATTACHMENTS];
char b_from[SMALLBUFSIZE], b_to[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE];
char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_to[SMALLBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE];
};

74
src/message.c
View File

@ -117,8 +117,8 @@ int store_index_data(struct session_data *sdata, struct _state *state, uint64 id
char *subj, s[SMALLBUFSIZE];
MYSQL_STMT *stmt;
MYSQL_BIND bind[5];
unsigned long len[5];
MYSQL_BIND bind[7];
unsigned long len[7];
subj = state->b_subject;
if(*subj == ' ') subj++;
@ -131,7 +131,7 @@ int store_index_data(struct session_data *sdata, struct _state *state, uint64 id
}
snprintf(s, sizeof(s)-1, "INSERT INTO %s (`id`, `from`, `to`, `subject`, `body`, `arrived`, `sent`, `size`, `direction`, `attachments`, `attachment_types`) values(%llu,?,?,?,?,%ld,%ld,%d,%d,%d,?)", SQL_SPHINX_TABLE, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, state->n_attachments);
snprintf(s, sizeof(s)-1, "INSERT INTO %s (`id`, `from`, `to`, `fromdomain`, `todomain`, `subject`, `body`, `arrived`, `sent`, `size`, `direction`, `attachments`, `attachment_types`) values(%llu,?,?,?,?,?,?,%ld,%ld,%d,%d,%d,?)", SQL_SPHINX_TABLE, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, state->n_attachments);
if(mysql_stmt_prepare(stmt, s, strlen(s))){
@ -142,6 +142,8 @@ int store_index_data(struct session_data *sdata, struct _state *state, uint64 id
fix_email_address_for_sphinx(state->b_from);
fix_email_address_for_sphinx(state->b_to);
fix_email_address_for_sphinx(state->b_from_domain);
fix_email_address_for_sphinx(state->b_to_domain);
memset(bind, 0, sizeof(bind));
@ -157,19 +159,30 @@ int store_index_data(struct session_data *sdata, struct _state *state, uint64 id
len[1] = strlen(state->b_to); bind[1].length = &len[1];
bind[2].buffer_type = MYSQL_TYPE_STRING;
bind[2].buffer = subj;
bind[2].buffer = state->b_from_domain;
bind[2].is_null = 0;
len[2] = strlen(subj); bind[2].length = &len[2];
len[2] = strlen(state->b_from_domain); bind[2].length = &len[2];
bind[3].buffer_type = MYSQL_TYPE_STRING;
bind[3].buffer = state->b_body;
bind[3].buffer = state->b_to_domain;
bind[3].is_null = 0;
len[3] = strlen(state->b_body); bind[3].length = &len[3];
len[3] = strlen(state->b_to_domain); bind[3].length = &len[3];
bind[4].buffer_type = MYSQL_TYPE_STRING;
bind[4].buffer = sdata->attachments;
bind[4].buffer = subj;
bind[4].is_null = 0;
len[4] = strlen(sdata->attachments); bind[4].length = &len[4];
len[4] = strlen(subj); bind[4].length = &len[4];
bind[5].buffer_type = MYSQL_TYPE_STRING;
bind[5].buffer = state->b_body;
bind[5].is_null = 0;
len[5] = strlen(state->b_body); bind[5].length = &len[5];
bind[6].buffer_type = MYSQL_TYPE_STRING;
bind[6].buffer = sdata->attachments;
bind[6].is_null = 0;
len[6] = strlen(sdata->attachments); bind[6].length = &len[6];
if(mysql_stmt_bind_param(stmt, bind)){
@ -256,42 +269,9 @@ int store_recipients(struct session_data *sdata, char *to, uint64 id, struct __c
}
int update_verification_code(struct session_data *sdata, struct _state *state, uint64 id, struct __config *cfg){
char s[SMALLBUFSIZE], vcode[MAXBUFSIZE], digest[2*DIGEST_LENGTH+1];
MYSQL_RES *res;
MYSQL_ROW row;
snprintf(vcode, sizeof(vcode)-1, "%llu+%s%s%ld%ld%d%d%d%d%s%s%s+", id, state->b_from, state->message_id, sdata->now, sdata->sent, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
if(id > 2){
snprintf(s, SMALLBUFSIZE-1, "SELECT `vcode` FROM `%s` WHERE `id`=%llu OR `id`=%llu ORDER BY `id` ASC", SQL_METADATA_TABLE, id-1, id-2);
if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){
res = mysql_store_result(&(sdata->mysql));
if(res != NULL){
while((row = mysql_fetch_row(res))){
if(row){
if((char*)row[0]) strncat(vcode, (char*)row[0], sizeof(vcode)-1);
}
}
mysql_free_result(res);
}
}
}
digest_string(vcode, &digest[0]);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: verification data: %s, digest: %s", sdata->ttmpfile, vcode, digest);
snprintf(s, SMALLBUFSIZE-1, "UPDATE %s SET `vcode`='%s' WHERE id=%llu", SQL_METADATA_TABLE, digest, id);
return mysql_real_query(&(sdata->mysql), s, strlen(s));
}
int store_meta_data(struct session_data *sdata, struct _state *state, struct __config *cfg){
int rc, ret=ERR;
char *subj, *p, s[MAXBUFSIZE];
char *subj, *p, s[MAXBUFSIZE], vcode[2*DIGEST_LENGTH+1];
MYSQL_STMT *stmt;
MYSQL_BIND bind[4];
@ -309,7 +289,11 @@ int store_meta_data(struct session_data *sdata, struct _state *state, struct __c
subj = state->b_subject;
if(*subj == ' ') subj++;
snprintf(s, MAXBUFSIZE-1, "INSERT INTO %s (`from`,`fromdomain`,`subject`,`arrived`,`sent`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`digest`,`bodydigest`) VALUES(?,?,?,%ld,%ld,%d,%d,%d,%d,'%s',?,'%s','%s')", SQL_METADATA_TABLE, sdata->now, sdata->sent, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%d%d%d%d%s%s%s", id, subj, state->b_from, state->message_id, sdata->now, sdata->sent, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
digest_string(s, &vcode[0]);
snprintf(s, MAXBUFSIZE-1, "INSERT INTO %s (`from`,`fromdomain`,`subject`,`arrived`,`sent`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`digest`,`bodydigest`,`vcode`) VALUES(?,?,?,%ld,%ld,%d,%d,%d,%d,'%s',?,'%s','%s','%s')", SQL_METADATA_TABLE, sdata->now, sdata->sent, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest, vcode);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: meta sql: *%s*", sdata->ttmpfile, s);
@ -371,8 +355,6 @@ int store_meta_data(struct session_data *sdata, struct _state *state, struct __c
if(rc == OK){
rc = update_verification_code(sdata, state, id, cfg);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: updated verification code, rc=%d", sdata->ttmpfile, rc);
rc = store_index_data(sdata, state, id, cfg);

14
src/parser.c
View File

@ -55,6 +55,7 @@ void post_parse(struct session_data *sdata, struct _state *state, struct __confi
free_list(state->boundaries);
free_list(state->rcpt);
free_list(state->rcpt_domain);
trimBuffer(state->b_subject);
fixupEncodedHeaderLine(state->b_subject);
@ -95,7 +96,7 @@ void post_parse(struct session_data *sdata, struct _state *state, struct __confi
int parse_line(char *buf, struct _state *state, struct session_data *sdata, struct __config *cfg){
char *p, puf[SMALLBUFSIZE];
char *p, *q, puf[SMALLBUFSIZE];
int x, len, b64_len, boundary_line=0;
state->line_num++;
@ -401,6 +402,9 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, stru
if(state->message_state == MSG_FROM && does_it_seem_like_an_email_address(puf) == 1 && state->is_1st_header == 1 && state->b_from[0] == '\0' && strlen(state->b_from) < SMALLBUFSIZE-len-1){
memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
q = strchr(puf, '@');
if(q) memcpy(&(state->b_from_domain[strlen(state->b_from_domain)]), q+1, len);
if(is_email_address_on_my_domains(puf, cfg) == 1) sdata->internal_sender = 1;
}
else if((state->message_state == MSG_TO || state->message_state == MSG_CC) && state->is_1st_header == 1 && does_it_seem_like_an_email_address(puf) == 1 && strlen(state->b_to) < SMALLBUFSIZE-len-1){
@ -412,6 +416,14 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, stru
if(is_email_address_on_my_domains(puf, cfg) == 1) sdata->internal_recipient = 1;
else sdata->external_recipient = 1;
q = strchr(puf, '@');
if(q){
if(is_string_on_list(state->rcpt_domain, q+1) == 0){
append_list(&(state->rcpt_domain), q+1);
memcpy(&(state->b_to_domain[strlen(state->b_to_domain)]), q+1, strlen(q+1));
}
}
}
}
else if(state->message_state == MSG_BODY && strlen(state->b_body) < BIGBUFSIZE-len-1)

3
src/parser_utils.c
View File

@ -62,6 +62,7 @@ void init_state(struct _state *state){
state->boundaries = NULL;
state->rcpt = NULL;
state->rcpt_domain = NULL;
state->n_attachments = 0;
@ -74,7 +75,9 @@ void init_state(struct _state *state){
}
memset(state->b_from, 0, SMALLBUFSIZE);
memset(state->b_from_domain, 0, SMALLBUFSIZE);
memset(state->b_to, 0, SMALLBUFSIZE);
memset(state->b_to_domain, 0, SMALLBUFSIZE);
memset(state->b_subject, 0, MAXBUFSIZE);
memset(state->b_body, 0, BIGBUFSIZE);
}

4
src/test.c
View File

@ -70,8 +70,8 @@ int main(int argc, char **argv){
post_parse(&sdata, &state, &cfg);
printf("message-id: %s\n", state.message_id);
printf("from: *%s*\n", state.b_from);
printf("to: *%s*\n", state.b_to);
printf("from: *%s (%s)*\n", state.b_from, state.b_from_domain);
printf("to: *%s (%s)*\n", state.b_to, state.b_to_domain);
printf("subject: *%s*\n", state.b_subject);
//printf("body: *%s*\n", state.b_body);

16
util/db-mysql.sql
View File

@ -15,6 +15,8 @@ create table if not exists `sph_index` (
`id` bigint not null,
`from` char(255) default null,
`to` text(512) default null,
`fromdomain` char(255) default null,
`todomain` text(512) default null,
`subject` text(512) default null,
`arrived` int not null,
`sent` int not null,
@ -205,3 +207,17 @@ create table if not exists `domain` (
insert into `domain` (`domain`, `mapped`) values('local', 'local');
drop table if exists `audit`;
create table if not exists `audit` (
`id` bigint unsigned not null auto_increment,
`ts` int not null,
`email` char(128) not null,
`action` int not null,
`ipaddr` char(15) not null,
`meta_id` bigint unsigned not null,
`description` char(255) default null,
`vcode` char(64) default null,
primary key (`id`)
) ENGINE=InnoDB;