parser and prepared statement improvements

src/Makefile.in

@@ -85,6 +85,11 @@ install-piler:
 	$(INSTALL) -m 0755 pilerget $(DESTDIR)$(bindir)
 	$(INSTALL) -m 0755 pilerimport $(DESTDIR)$(bindir)
 
+	chown $(RUNNING_USER):$(RUNNING_GROUP) $(DESTDIR)$(bindir)/pilerget
+	chown $(RUNNING_USER):$(RUNNING_GROUP) $(DESTDIR)$(bindir)/pilerimport
+	chmod +s $(DESTDIR)$(bindir)/pilerget
+	chmod +s $(DESTDIR)$(bindir)/pilerimport
+
 clean:
 	rm -f *.o *.a libpiler.so* piler pilerconf pilerget pilerimport pilertest
 

src/attachment.c

@@ -28,18 +28,9 @@ int store_attachments(struct session_data *sdata, struct _state *state, struct _
    unsigned long len[7];
 
 
-   stmt = mysql_stmt_init(&(sdata->mysql));
-   if(!stmt){
-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_ATTACHMENT_TABLE);
-      return rc;
-   }
-
    snprintf(s, sizeof(s)-1, "INSERT INTO %s (`piler_id`,`attachment_id`,`sig`,`name`,`type`,`size`,`ptr`) VALUES(?,?,?,?,?,?,?)", SQL_ATTACHMENT_TABLE);
 
-   if(mysql_stmt_prepare(stmt, s, strlen(s))){
+   if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) return rc;
-      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_ATTACHMENT_TABLE, mysql_stmt_error(stmt));
-      return rc;
-   }
 
 
    for(i=1; i<=state->n_attachments; i++){
@@ -153,17 +144,9 @@ int query_attachment_pointers(struct session_data *sdata, uint64 ptr, char *pile
    unsigned long len=0;
 
 
-   stmt = mysql_stmt_init(&(sdata->mysql));
-   if(!stmt){
-      goto ENDE;
-   }
-
    snprintf(s, SMALLBUFSIZE-1, "SELECT `piler_id`, `attachment_id` FROM %s WHERE id=?", SQL_ATTACHMENT_TABLE);
 
-
+   if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) goto ENDE;
-   if(mysql_stmt_prepare(stmt, s, strlen(s))){
-      goto ENDE;
-   }
 
    memset(bind, 0, sizeof(bind));
 
@@ -234,17 +217,10 @@ int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr, str
    for(i=0; i<MAX_ATTACHMENTS; i++) memset((char*)&ptr_arr[i], 0, sizeof(struct ptr_array));
 
 
-   stmt = mysql_stmt_init(&(sdata->mysql));
-   if(!stmt){
-      goto ENDE;
-   }
-
    snprintf(s, SMALLBUFSIZE-1, "SELECT `attachment_id`, `ptr` FROM %s WHERE piler_id=? ORDER BY attachment_id ASC", SQL_ATTACHMENT_TABLE);
 
+   if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) goto ENDE;
 
-   if(mysql_stmt_prepare(stmt, s, strlen(s))){
-      goto ENDE;
-   }
 
    memset(bind, 0, sizeof(bind));
 

src/message.c

@@ -17,7 +17,24 @@
 #include <zlib.h>
 
 
-int is_existing_message_id(struct session_data *sdata, struct _state *state, struct __config *cfg){
+int prepare_a_mysql_statement(struct session_data *sdata, MYSQL_STMT **stmt, char *s){
+
+   *stmt = mysql_stmt_init(&(sdata->mysql));
+   if(!*stmt){
+      syslog(LOG_PRIORITY, "%s: mysql_stmt_init() error", sdata->ttmpfile);
+      return ERR;
+   }
+
+   if(mysql_stmt_prepare(*stmt, s, strlen(s))){
+      syslog(LOG_PRIORITY, "%s: mysql_stmt_prepare() error: %s => sql: %s", sdata->ttmpfile, mysql_stmt_error(*stmt), s);
+      return ERR;
+   }
+
+   return OK; 
+}
+
+
+int is_existing_message_id(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg){
    int rc=0;
    char s[SMALLBUFSIZE];
    MYSQL_STMT *stmt;
@@ -26,18 +43,10 @@ int is_existing_message_id(struct session_data *sdata, struct _state *state, str
    unsigned long len=0;
 
 
-   stmt = mysql_stmt_init(&(sdata->mysql));
-   if(!stmt){
-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_METADATA_TABLE);
-      goto ENDE;
-   }
-
    snprintf(s, SMALLBUFSIZE-1, "SELECT message_id FROM %s WHERE message_id=?", SQL_METADATA_TABLE);
 
-   if(mysql_stmt_prepare(stmt, s, strlen(s))){
+   if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) goto ENDE;
-      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt));
+
-      goto ENDE;
-   }
 
    memset(bind, 0, sizeof(bind));
 
@@ -125,20 +134,10 @@ int store_index_data(struct session_data *sdata, struct _state *state, uint64 id
    if(*subj == ' ') subj++;
 
 
-   stmt = mysql_stmt_init(&(sdata->mysql));
-   if(!stmt){
-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_SPHINX_TABLE);
-      return rc;
-   }
-
-
    snprintf(s, sizeof(s)-1, "INSERT INTO %s (`id`, `from`, `to`, `fromdomain`, `todomain`, `subject`, `body`, `arrived`, `sent`, `size`, `direction`, `attachments`, `attachment_types`) values(%llu,?,?,?,?,?,?,%ld,%ld,%d,%d,%d,?)", SQL_SPHINX_TABLE, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, state->n_attachments);
 
+   if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) return rc;
 
-   if(mysql_stmt_prepare(stmt, s, strlen(s))){
-      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_stmt_error(stmt));
-      return rc;
-   }
 
 
    fix_email_address_for_sphinx(state->b_from);
@@ -215,18 +214,11 @@ int store_recipients(struct session_data *sdata, char *to, uint64 id, struct __c
    MYSQL_BIND bind[2];
    unsigned long len[2];
 
-   stmt = mysql_stmt_init(&(sdata->mysql));
-   if(!stmt){
-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_RECIPIENT_TABLE);
-      return ERR;
-   }
 
    snprintf(s, sizeof(s)-1, "INSERT INTO %s (`id`,`to`,`todomain`) VALUES('%llu',?,?)", SQL_RECIPIENT_TABLE, id);
 
-   if(mysql_stmt_prepare(stmt, s, strlen(s))){
+   if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) return ERR;
-      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_stmt_error(stmt));
+
-      return ERR;
-   }
 
    p = to;
    do {
@@ -278,7 +270,7 @@ CLOSE:
 
 int store_meta_data(struct session_data *sdata, struct _state *state, struct __config *cfg){
    int rc, ret=ERR;
-   char *subj, *p, s[MAXBUFSIZE], vcode[2*DIGEST_LENGTH+1];
+   char *subj, *p, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1];
 
    MYSQL_STMT *stmt;
    MYSQL_BIND bind[4];
@@ -287,12 +279,6 @@ int store_meta_data(struct session_data *sdata, struct _state *state, struct __c
    my_ulonglong id=0;
 
 
-   stmt = mysql_stmt_init(&(sdata->mysql));
-   if(!stmt){
-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_METADATA_TABLE);
-      return ERR;
-   }
-
    subj = state->b_subject;
    if(*subj == ' ') subj++;
 
@@ -304,11 +290,19 @@ int store_meta_data(struct session_data *sdata, struct _state *state, struct __c
 
    if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: meta sql: *%s*", sdata->ttmpfile, s);
 
-   if(mysql_stmt_prepare(stmt, s, strlen(s))){
+   if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) return ERR;
-      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt));
-      return ERR;
-   }
 
+   memset(s2, 0, sizeof(s2));
+
+   p = state->b_from;
+   do {
+      memset(s2, 0, sizeof(s2));
+      p = split(p, ' ', s2, sizeof(s2)-1);
+
+      if(s2[0] == '\0') continue;
+
+      if(does_it_seem_like_an_email_address(s2) == 1){ break; }
+   } while(p);
 
 
    if(strlen(state->b_to) < 5){
@@ -318,9 +312,9 @@ int store_meta_data(struct session_data *sdata, struct _state *state, struct __c
    memset(bind, 0, sizeof(bind));
 
    bind[0].buffer_type = MYSQL_TYPE_STRING;
-   bind[0].buffer = state->b_from;
+   bind[0].buffer = &s2[0];
    bind[0].is_null = 0;
-   len[0] = strlen(state->b_from); bind[0].length = &len[0];
+   len[0] = strlen(s2); bind[0].length = &len[0];
 
    p = strchr(state->b_from, '@');
    if(p && strlen(p) > 3){
@@ -362,8 +356,6 @@ int store_meta_data(struct session_data *sdata, struct _state *state, struct __c
 
       if(rc == OK){
 
-         if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: updated verification code, rc=%d", sdata->ttmpfile, rc);
-
          rc = store_index_data(sdata, state, id, cfg);
 
          if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored indexdata, rc=%d", sdata->ttmpfile, rc);
@@ -380,12 +372,12 @@ CLOSE:
 }
 
 
-int process_message(struct session_data *sdata, struct _state *state, struct __config *cfg){
+int process_message(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg){
    int i, rc;
 
    /* discard if existing message_id */
 
-   if(is_existing_message_id(sdata, state, cfg) == 1){
+   if(is_existing_message_id(sdata, state, data, cfg) == 1){
       return ERR_EXISTS;
    }
 

src/parser.c

@@ -91,7 +91,6 @@ void post_parse(struct session_data *sdata, struct _state *state, struct __confi
    if(state->b_to[len-1] == ' ') state->b_to[len-1] = '\0';
 
    syslog(LOG_PRIORITY, "%s: from=%s, to=%s, subj=%s, message-id=%s", sdata->ttmpfile, state->b_from, state->b_to, state->b_subject, state->message_id);
-
 }
 
 
@@ -249,6 +248,10 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, stru
 
    }
 
+   if(state->is_1st_header == 1){
+      fixupEncodedHeaderLine(buf);
+   }
+
 
    /* Content-type: checking */
 
@@ -403,20 +406,29 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, stru
 
       len = strlen(puf);
 
-      if(state->message_state == MSG_FROM && does_it_seem_like_an_email_address(puf) == 1 && state->is_1st_header == 1 && state->b_from[0] == '\0' && strlen(state->b_from) < SMALLBUFSIZE-len-1){
+
+      if(state->message_state == MSG_FROM && state->is_1st_header == 1 && strlen(state->b_from) < SMALLBUFSIZE-len-1){
          memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
 
+         if(does_it_seem_like_an_email_address(puf) == 1){
             q = strchr(puf, '@');
             if(q) memcpy(&(state->b_from_domain[strlen(state->b_from_domain)]), q+1, len);
 
             if(is_email_address_on_my_domains(puf, cfg) == 1) sdata->internal_sender = 1;
+
+            if(strlen(state->b_from) < SMALLBUFSIZE-len-1){
+               split_email_address(puf);
+               memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
             }
-      else if((state->message_state == MSG_TO || state->message_state == MSG_CC) && state->is_1st_header == 1 && does_it_seem_like_an_email_address(puf) == 1 && strlen(state->b_to) < MAXBUFSIZE-len-1){
+         }
+      }
+      else if((state->message_state == MSG_TO || state->message_state == MSG_CC) && state->is_1st_header == 1 && strlen(state->b_to) < MAXBUFSIZE-len-1){
 
          if(is_string_on_list(state->rcpt, puf) == 0){
             append_list(&(state->rcpt), puf);
             memcpy(&(state->b_to[strlen(state->b_to)]), puf, len);
 
+            if(does_it_seem_like_an_email_address(puf) == 1){
                if(is_email_address_on_my_domains(puf, cfg) == 1) sdata->internal_recipient = 1;
                else sdata->external_recipient = 1;
 
@@ -428,6 +440,12 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, stru
                   }
                }
 
+               if(strlen(state->b_to) < MAXBUFSIZE-len-1){
+                  split_email_address(puf);
+                  memcpy(&(state->b_to[strlen(state->b_to)]), puf, len);
+               }
+
+            }
          }
       }
       else if(state->message_state == MSG_BODY && strlen(state->b_body) < BIGBUFSIZE-len-1)

src/parser.h

@@ -24,6 +24,7 @@ void markHTML(char *buf, struct _state *state);
 int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct _state *state);
 void translateLine(unsigned char *p, struct _state *state);
 void fix_email_address_for_sphinx(char *s);
+void split_email_address(char *s);
 int does_it_seem_like_an_email_address(char *email);
 void reassembleToken(char *p);
 void degenerateToken(unsigned char *p);

src/parser_utils.c

@@ -501,6 +501,13 @@ void fix_email_address_for_sphinx(char *s){
 }
 
 
+void split_email_address(char *s){
+   for(; *s; s++){
+      if(*s == '@' || *s == '.' || *s == '+' || *s == '-' || *s == '_') *s = ' ';
+   }
+}
+
+
 int does_it_seem_like_an_email_address(char *email){
    char *p;
 

src/piler.h

@@ -35,7 +35,7 @@ void digest_string(char *s, char *digest);
 
 int handle_smtp_session(int new_sd, struct __data *data, struct __config *cfg);
 
-int process_message(struct session_data *sdata, struct _state *sstate, struct __config *cfg);
+int process_message(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg);
 int store_file(struct session_data *sdata, char *filename, int startpos, int len, struct __config *cfg);
 int store_attachments(struct session_data *sdata, struct _state *state, struct __config *cfg);
 int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr, struct __config *cfg);
@@ -48,6 +48,8 @@ void update_counters(struct session_data *sdata, struct __data *data, struct __c
 
 int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct __config *cfg);
 
+int prepare_a_mysql_statement(struct session_data *sdata, MYSQL_STMT **stmt, char *s);
+
 
 #endif /* _PILER_H */
 

src/pilerget.c

@@ -27,17 +27,9 @@ uint64 get_id_by_piler_id(struct session_data *sdata, char *digest, char *bodydi
    memset(digest, 0, 2*DIGEST_LENGTH+1);
    memset(bodydigest, 0, 2*DIGEST_LENGTH+1);
 
-   stmt = mysql_stmt_init(&(sdata->mysql));
-   if(!stmt){
-      goto ENDE;
-   }
-
    snprintf(s, SMALLBUFSIZE-1, "SELECT `id`,`digest`,`bodydigest` FROM %s WHERE piler_id=?", SQL_METADATA_TABLE);
 
-
+   if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) goto ENDE;
-   if(mysql_stmt_prepare(stmt, s, strlen(s))){
-      goto ENDE;
-   }
 
    memset(bind, 0, sizeof(bind));
 

src/pilerimport.c

@@ -59,7 +59,7 @@ int import_message(char *filename, struct session_data *sdata, struct __data *da
 
    make_digests(sdata, cfg);
 
-   rc = process_message(sdata, &state, cfg);
+   rc = process_message(sdata, &state, data, cfg);
 
 ENDE:
    unlink(sdata->tmpframe);

src/session.c

@@ -201,7 +201,7 @@ int handle_smtp_session(int new_sd, struct __data *data, struct __config *cfg){
                            counters.c_ignore++;
                         }
                         else {
-                           inj = process_message(&sdata, &sstate, cfg);
+                           inj = process_message(&sdata, &sstate, data, cfg);
                         }
 
                      }