From 79cdeed1b6fda037798460ce9863d0cbaad53ecf Mon Sep 17 00:00:00 2001 From: SJ Date: Tue, 7 Feb 2012 17:21:23 +0100 Subject: [PATCH] parser and prepared statement improvements --- src/Makefile.in | 5 +++ src/attachment.c | 30 ++-------------- src/message.c | 88 +++++++++++++++++++++------------------------- src/parser.c | 46 ++++++++++++++++-------- src/parser.h | 1 + src/parser_utils.c | 7 ++++ src/piler.h | 4 ++- src/pilerget.c | 10 +----- src/pilerimport.c | 2 +- src/session.c | 2 +- 10 files changed, 94 insertions(+), 101 deletions(-) diff --git a/src/Makefile.in b/src/Makefile.in index cdec4d8a..9ea2c92e 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -85,6 +85,11 @@ install-piler: $(INSTALL) -m 0755 pilerget $(DESTDIR)$(bindir) $(INSTALL) -m 0755 pilerimport $(DESTDIR)$(bindir) + chown $(RUNNING_USER):$(RUNNING_GROUP) $(DESTDIR)$(bindir)/pilerget + chown $(RUNNING_USER):$(RUNNING_GROUP) $(DESTDIR)$(bindir)/pilerimport + chmod +s $(DESTDIR)$(bindir)/pilerget + chmod +s $(DESTDIR)$(bindir)/pilerimport + clean: rm -f *.o *.a libpiler.so* piler pilerconf pilerget pilerimport pilertest diff --git a/src/attachment.c b/src/attachment.c index 135c9cac..43693439 100644 --- a/src/attachment.c +++ b/src/attachment.c @@ -28,18 +28,9 @@ int store_attachments(struct session_data *sdata, struct _state *state, struct _ unsigned long len[7]; - stmt = mysql_stmt_init(&(sdata->mysql)); - if(!stmt){ - if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_ATTACHMENT_TABLE); - return rc; - } - snprintf(s, sizeof(s)-1, "INSERT INTO %s (`piler_id`,`attachment_id`,`sig`,`name`,`type`,`size`,`ptr`) VALUES(?,?,?,?,?,?,?)", SQL_ATTACHMENT_TABLE); - if(mysql_stmt_prepare(stmt, s, strlen(s))){ - syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_ATTACHMENT_TABLE, mysql_stmt_error(stmt)); - return rc; - } + if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) return rc; for(i=1; i<=state->n_attachments; i++){ @@ -153,17 +144,9 @@ int query_attachment_pointers(struct session_data *sdata, uint64 ptr, char *pile unsigned long len=0; - stmt = mysql_stmt_init(&(sdata->mysql)); - if(!stmt){ - goto ENDE; - } - snprintf(s, SMALLBUFSIZE-1, "SELECT `piler_id`, `attachment_id` FROM %s WHERE id=?", SQL_ATTACHMENT_TABLE); - - if(mysql_stmt_prepare(stmt, s, strlen(s))){ - goto ENDE; - } + if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) goto ENDE; memset(bind, 0, sizeof(bind)); @@ -234,17 +217,10 @@ int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr, str for(i=0; imysql)); - if(!stmt){ - goto ENDE; - } - snprintf(s, SMALLBUFSIZE-1, "SELECT `attachment_id`, `ptr` FROM %s WHERE piler_id=? ORDER BY attachment_id ASC", SQL_ATTACHMENT_TABLE); + if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) goto ENDE; - if(mysql_stmt_prepare(stmt, s, strlen(s))){ - goto ENDE; - } memset(bind, 0, sizeof(bind)); diff --git a/src/message.c b/src/message.c index 13e5520d..8879251c 100644 --- a/src/message.c +++ b/src/message.c @@ -17,7 +17,24 @@ #include -int is_existing_message_id(struct session_data *sdata, struct _state *state, struct __config *cfg){ +int prepare_a_mysql_statement(struct session_data *sdata, MYSQL_STMT **stmt, char *s){ + + *stmt = mysql_stmt_init(&(sdata->mysql)); + if(!*stmt){ + syslog(LOG_PRIORITY, "%s: mysql_stmt_init() error", sdata->ttmpfile); + return ERR; + } + + if(mysql_stmt_prepare(*stmt, s, strlen(s))){ + syslog(LOG_PRIORITY, "%s: mysql_stmt_prepare() error: %s => sql: %s", sdata->ttmpfile, mysql_stmt_error(*stmt), s); + return ERR; + } + + return OK; +} + + +int is_existing_message_id(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg){ int rc=0; char s[SMALLBUFSIZE]; MYSQL_STMT *stmt; @@ -26,18 +43,10 @@ int is_existing_message_id(struct session_data *sdata, struct _state *state, str unsigned long len=0; - stmt = mysql_stmt_init(&(sdata->mysql)); - if(!stmt){ - if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_METADATA_TABLE); - goto ENDE; - } - snprintf(s, SMALLBUFSIZE-1, "SELECT message_id FROM %s WHERE message_id=?", SQL_METADATA_TABLE); - if(mysql_stmt_prepare(stmt, s, strlen(s))){ - syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt)); - goto ENDE; - } + if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) goto ENDE; + memset(bind, 0, sizeof(bind)); @@ -125,20 +134,10 @@ int store_index_data(struct session_data *sdata, struct _state *state, uint64 id if(*subj == ' ') subj++; - stmt = mysql_stmt_init(&(sdata->mysql)); - if(!stmt){ - if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_SPHINX_TABLE); - return rc; - } - - snprintf(s, sizeof(s)-1, "INSERT INTO %s (`id`, `from`, `to`, `fromdomain`, `todomain`, `subject`, `body`, `arrived`, `sent`, `size`, `direction`, `attachments`, `attachment_types`) values(%llu,?,?,?,?,?,?,%ld,%ld,%d,%d,%d,?)", SQL_SPHINX_TABLE, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, state->n_attachments); + if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) return rc; - if(mysql_stmt_prepare(stmt, s, strlen(s))){ - syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_stmt_error(stmt)); - return rc; - } fix_email_address_for_sphinx(state->b_from); @@ -215,18 +214,11 @@ int store_recipients(struct session_data *sdata, char *to, uint64 id, struct __c MYSQL_BIND bind[2]; unsigned long len[2]; - stmt = mysql_stmt_init(&(sdata->mysql)); - if(!stmt){ - if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_RECIPIENT_TABLE); - return ERR; - } snprintf(s, sizeof(s)-1, "INSERT INTO %s (`id`,`to`,`todomain`) VALUES('%llu',?,?)", SQL_RECIPIENT_TABLE, id); - if(mysql_stmt_prepare(stmt, s, strlen(s))){ - syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_stmt_error(stmt)); - return ERR; - } + if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) return ERR; + p = to; do { @@ -278,7 +270,7 @@ CLOSE: int store_meta_data(struct session_data *sdata, struct _state *state, struct __config *cfg){ int rc, ret=ERR; - char *subj, *p, s[MAXBUFSIZE], vcode[2*DIGEST_LENGTH+1]; + char *subj, *p, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1]; MYSQL_STMT *stmt; MYSQL_BIND bind[4]; @@ -287,12 +279,6 @@ int store_meta_data(struct session_data *sdata, struct _state *state, struct __c my_ulonglong id=0; - stmt = mysql_stmt_init(&(sdata->mysql)); - if(!stmt){ - if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_METADATA_TABLE); - return ERR; - } - subj = state->b_subject; if(*subj == ' ') subj++; @@ -304,11 +290,19 @@ int store_meta_data(struct session_data *sdata, struct _state *state, struct __c if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: meta sql: *%s*", sdata->ttmpfile, s); - if(mysql_stmt_prepare(stmt, s, strlen(s))){ - syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt)); - return ERR; - } + if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) return ERR; + memset(s2, 0, sizeof(s2)); + + p = state->b_from; + do { + memset(s2, 0, sizeof(s2)); + p = split(p, ' ', s2, sizeof(s2)-1); + + if(s2[0] == '\0') continue; + + if(does_it_seem_like_an_email_address(s2) == 1){ break; } + } while(p); if(strlen(state->b_to) < 5){ @@ -318,9 +312,9 @@ int store_meta_data(struct session_data *sdata, struct _state *state, struct __c memset(bind, 0, sizeof(bind)); bind[0].buffer_type = MYSQL_TYPE_STRING; - bind[0].buffer = state->b_from; + bind[0].buffer = &s2[0]; bind[0].is_null = 0; - len[0] = strlen(state->b_from); bind[0].length = &len[0]; + len[0] = strlen(s2); bind[0].length = &len[0]; p = strchr(state->b_from, '@'); if(p && strlen(p) > 3){ @@ -362,8 +356,6 @@ int store_meta_data(struct session_data *sdata, struct _state *state, struct __c if(rc == OK){ - if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: updated verification code, rc=%d", sdata->ttmpfile, rc); - rc = store_index_data(sdata, state, id, cfg); if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored indexdata, rc=%d", sdata->ttmpfile, rc); @@ -380,12 +372,12 @@ CLOSE: } -int process_message(struct session_data *sdata, struct _state *state, struct __config *cfg){ +int process_message(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg){ int i, rc; /* discard if existing message_id */ - if(is_existing_message_id(sdata, state, cfg) == 1){ + if(is_existing_message_id(sdata, state, data, cfg) == 1){ return ERR_EXISTS; } diff --git a/src/parser.c b/src/parser.c index e900ed4f..5b98e2e4 100644 --- a/src/parser.c +++ b/src/parser.c @@ -91,7 +91,6 @@ void post_parse(struct session_data *sdata, struct _state *state, struct __confi if(state->b_to[len-1] == ' ') state->b_to[len-1] = '\0'; syslog(LOG_PRIORITY, "%s: from=%s, to=%s, subj=%s, message-id=%s", sdata->ttmpfile, state->b_from, state->b_to, state->b_subject, state->message_id); - } @@ -249,6 +248,10 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, stru } + if(state->is_1st_header == 1){ + fixupEncodedHeaderLine(buf); + } + /* Content-type: checking */ @@ -403,31 +406,46 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, stru len = strlen(puf); - if(state->message_state == MSG_FROM && does_it_seem_like_an_email_address(puf) == 1 && state->is_1st_header == 1 && state->b_from[0] == '\0' && strlen(state->b_from) < SMALLBUFSIZE-len-1){ + + if(state->message_state == MSG_FROM && state->is_1st_header == 1 && strlen(state->b_from) < SMALLBUFSIZE-len-1){ memcpy(&(state->b_from[strlen(state->b_from)]), puf, len); - q = strchr(puf, '@'); - if(q) memcpy(&(state->b_from_domain[strlen(state->b_from_domain)]), q+1, len); + if(does_it_seem_like_an_email_address(puf) == 1){ + q = strchr(puf, '@'); + if(q) memcpy(&(state->b_from_domain[strlen(state->b_from_domain)]), q+1, len); - if(is_email_address_on_my_domains(puf, cfg) == 1) sdata->internal_sender = 1; + if(is_email_address_on_my_domains(puf, cfg) == 1) sdata->internal_sender = 1; + + if(strlen(state->b_from) < SMALLBUFSIZE-len-1){ + split_email_address(puf); + memcpy(&(state->b_from[strlen(state->b_from)]), puf, len); + } + } } - else if((state->message_state == MSG_TO || state->message_state == MSG_CC) && state->is_1st_header == 1 && does_it_seem_like_an_email_address(puf) == 1 && strlen(state->b_to) < MAXBUFSIZE-len-1){ + else if((state->message_state == MSG_TO || state->message_state == MSG_CC) && state->is_1st_header == 1 && strlen(state->b_to) < MAXBUFSIZE-len-1){ if(is_string_on_list(state->rcpt, puf) == 0){ append_list(&(state->rcpt), puf); memcpy(&(state->b_to[strlen(state->b_to)]), puf, len); - if(is_email_address_on_my_domains(puf, cfg) == 1) sdata->internal_recipient = 1; - else sdata->external_recipient = 1; + if(does_it_seem_like_an_email_address(puf) == 1){ + if(is_email_address_on_my_domains(puf, cfg) == 1) sdata->internal_recipient = 1; + else sdata->external_recipient = 1; - q = strchr(puf, '@'); - if(q){ - if(is_string_on_list(state->rcpt_domain, q+1) == 0){ - append_list(&(state->rcpt_domain), q+1); - memcpy(&(state->b_to_domain[strlen(state->b_to_domain)]), q+1, strlen(q+1)); + q = strchr(puf, '@'); + if(q){ + if(is_string_on_list(state->rcpt_domain, q+1) == 0){ + append_list(&(state->rcpt_domain), q+1); + memcpy(&(state->b_to_domain[strlen(state->b_to_domain)]), q+1, strlen(q+1)); + } } - } + if(strlen(state->b_to) < MAXBUFSIZE-len-1){ + split_email_address(puf); + memcpy(&(state->b_to[strlen(state->b_to)]), puf, len); + } + + } } } else if(state->message_state == MSG_BODY && strlen(state->b_body) < BIGBUFSIZE-len-1) diff --git a/src/parser.h b/src/parser.h index 90999c5c..495f77e9 100644 --- a/src/parser.h +++ b/src/parser.h @@ -24,6 +24,7 @@ void markHTML(char *buf, struct _state *state); int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct _state *state); void translateLine(unsigned char *p, struct _state *state); void fix_email_address_for_sphinx(char *s); +void split_email_address(char *s); int does_it_seem_like_an_email_address(char *email); void reassembleToken(char *p); void degenerateToken(unsigned char *p); diff --git a/src/parser_utils.c b/src/parser_utils.c index a27eec65..cd330096 100644 --- a/src/parser_utils.c +++ b/src/parser_utils.c @@ -501,6 +501,13 @@ void fix_email_address_for_sphinx(char *s){ } +void split_email_address(char *s){ + for(; *s; s++){ + if(*s == '@' || *s == '.' || *s == '+' || *s == '-' || *s == '_') *s = ' '; + } +} + + int does_it_seem_like_an_email_address(char *email){ char *p; diff --git a/src/piler.h b/src/piler.h index c573b39d..9b2f3d6b 100644 --- a/src/piler.h +++ b/src/piler.h @@ -35,7 +35,7 @@ void digest_string(char *s, char *digest); int handle_smtp_session(int new_sd, struct __data *data, struct __config *cfg); -int process_message(struct session_data *sdata, struct _state *sstate, struct __config *cfg); +int process_message(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg); int store_file(struct session_data *sdata, char *filename, int startpos, int len, struct __config *cfg); int store_attachments(struct session_data *sdata, struct _state *state, struct __config *cfg); int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr, struct __config *cfg); @@ -48,6 +48,8 @@ void update_counters(struct session_data *sdata, struct __data *data, struct __c int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct __config *cfg); +int prepare_a_mysql_statement(struct session_data *sdata, MYSQL_STMT **stmt, char *s); + #endif /* _PILER_H */ diff --git a/src/pilerget.c b/src/pilerget.c index 95e15cac..3986f096 100644 --- a/src/pilerget.c +++ b/src/pilerget.c @@ -27,17 +27,9 @@ uint64 get_id_by_piler_id(struct session_data *sdata, char *digest, char *bodydi memset(digest, 0, 2*DIGEST_LENGTH+1); memset(bodydigest, 0, 2*DIGEST_LENGTH+1); - stmt = mysql_stmt_init(&(sdata->mysql)); - if(!stmt){ - goto ENDE; - } - snprintf(s, SMALLBUFSIZE-1, "SELECT `id`,`digest`,`bodydigest` FROM %s WHERE piler_id=?", SQL_METADATA_TABLE); - - if(mysql_stmt_prepare(stmt, s, strlen(s))){ - goto ENDE; - } + if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) goto ENDE; memset(bind, 0, sizeof(bind)); diff --git a/src/pilerimport.c b/src/pilerimport.c index d5f31cfe..2202eace 100644 --- a/src/pilerimport.c +++ b/src/pilerimport.c @@ -59,7 +59,7 @@ int import_message(char *filename, struct session_data *sdata, struct __data *da make_digests(sdata, cfg); - rc = process_message(sdata, &state, cfg); + rc = process_message(sdata, &state, data, cfg); ENDE: unlink(sdata->tmpframe); diff --git a/src/session.c b/src/session.c index a8621f61..b386a881 100644 --- a/src/session.c +++ b/src/session.c @@ -201,7 +201,7 @@ int handle_smtp_session(int new_sd, struct __data *data, struct __config *cfg){ counters.c_ignore++; } else { - inj = process_message(&sdata, &sstate, cfg); + inj = process_message(&sdata, &sstate, data, cfg); } }