0.1.9

README

@@ -8,6 +8,9 @@ Features:
 locales debian alatt: dpkg-reconfigure locales
 
 
+init:
+   indexer --all
+   searchd inditasa
 
 
 indexer delta1 --rotate

etc/sphinx.conf

@@ -11,14 +11,13 @@ source main
    sql_pass = sphinx
 
    sql_query_pre = SET NAMES utf8
-   sql_query = SELECT id, `from`, `to`, `subject`, `arrived`, `sent`, `body`, `size`, `attachments`, `piler_id` FROM sph_index \
+   sql_query = SELECT id, `from`, `to`, `subject`, `arrived`, `sent`, `body`, `size`, `attachments` FROM sph_index \
         WHERE id<=( SELECT max_doc_id FROM sph_counter WHERE counter_id=1 )
 
    sql_attr_uint = size
    sql_attr_uint = arrived
    sql_attr_uint = sent
    sql_attr_uint = attachments
-   sql_attr_string = piler_id
 }
 
 source delta
@@ -32,14 +31,13 @@ source delta
    sql_query_pre = SET NAMES utf8
    sql_query_pre  = REPLACE INTO sph_counter SELECT 1, MAX(id) FROM sph_index
    sql_query_post_index  = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
-   sql_query = SELECT id, `from`, `to`, `subject`, `arrived`, `sent`, `body`, `size`, `attachments`, `piler_id` FROM sph_index \
+   sql_query = SELECT id, `from`, `to`, `subject`, `arrived`, `sent`, `body`, `size`, `attachments` FROM sph_index \
         WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
 
    sql_attr_uint = size
    sql_attr_uint = arrived
    sql_attr_uint = sent
    sql_attr_uint = attachments
-   sql_attr_string = piler_id
 
 }
 

src/config.h

@@ -11,7 +11,7 @@
 
 #define PROGNAME "piler"
 
-#define VERSION "0.1.8"
+#define VERSION "0.1.9"
 
 #define PROGINFO VERSION ", Janos SUTO <sj@acts.hu>\n\n" CONFIGURE_PARAMS "\n\nSend bugs/issues to https://jira.acts.hu:8443/\n"
 
@@ -74,6 +74,7 @@
 #define SQL_SPHINX_TABLE "sph_index"
 #define SQL_METADATA_TABLE "metadata"
 #define SQL_ATTACHMENT_TABLE "attachment"
+#define SQL_RECIPIENT_TABLE "rcpt"
 #define SQL_ARCHIVING_RULE_TABLE "archiving_rule"
 #define SQL_COUNTER_TABLE "counter"
 

src/message.c

@@ -99,8 +99,6 @@ int is_body_digest_already_stored(struct session_data *sdata, struct _state *sta
 
    snprintf(s, SMALLBUFSIZE-1, "SELECT `bodydigest` FROM `%s` WHERE `bodydigest`='%s'", SQL_METADATA_TABLE, sdata->bodydigest);
 
-   //if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: check for body digest sql: *%s*", sdata->ttmpfile, s);
-
    if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){
       res = mysql_store_result(&(sdata->mysql));
       if(res != NULL){
@@ -114,114 +112,210 @@ int is_body_digest_already_stored(struct session_data *sdata, struct _state *sta
 }
 
 
-int hand_to_sphinx(struct session_data *sdata, struct _state *state, struct __config *cfg){
+int store_index_data(struct session_data *sdata, struct _state *state, uint64 id, struct __config *cfg){
    int rc;
-   char *subj, s[BIGBUFSIZE+2*MAXBUFSIZE];
+   char *subj, s[SMALLBUFSIZE];
-
-   subj = state->b_subject;
-   if(*subj == ' ') subj++;
-
-   snprintf(s, sizeof(s)-1, "INSERT INTO %s (`from`, `to`, `subject`, `body`, `arrived`, `sent`, `size`, `attachments`, `piler_id`) values('%s','%s','%s','%s',%ld,%ld,%d,%d,'%s')", SQL_SPHINX_TABLE, state->b_from, state->b_to, subj, state->b_body, sdata->now, sdata->sent, sdata->tot_len, state->n_attachments, sdata->ttmpfile);
-
-   rc = mysql_real_query(&(sdata->mysql), s, strlen(s));
-
-   if(rc == 0) return OK;
-
-   syslog(LOG_PRIORITY, "%s: sphinx sql error: *%s*", sdata->ttmpfile, mysql_error(&(sdata->mysql)));
-
-   return ERR;
-}
-
-
-int store_meta_data(struct session_data *sdata, struct _state *state, struct __config *cfg){
-   int i=0, rc, ret=ERR;
-   char *p, *subj, s[MAXBUFSIZE], s2[SMALLBUFSIZE];
 
    MYSQL_STMT *stmt;
    MYSQL_BIND bind[4];
    unsigned long len[4];
 
-   stmt = mysql_stmt_init(&(sdata->mysql));
-   if(!stmt){
-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_METADATA_TABLE);
-      goto ENDE_META;
-   }
-
    subj = state->b_subject;
    if(*subj == ' ') subj++;
 
-   snprintf(s, MAXBUFSIZE-1, "INSERT INTO %s (`from`,`to`,`subject`,`arrived`,`sent`,`size`,`hlen`,`attachments`,`piler_id`,`message_id`,`digest`,`bodydigest`) VALUES(?,?,?,%ld,%ld,%d,%d,%d,'%s',?,'%s','%s')", SQL_METADATA_TABLE, sdata->now, sdata->sent, sdata->tot_len, sdata->hdr_len, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
 
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: meta sql: *%s*", sdata->ttmpfile, s);
+   stmt = mysql_stmt_init(&(sdata->mysql));
+   if(!stmt){
+      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_SPHINX_TABLE);
+      return ERR;
+   }
+
+
+   snprintf(s, sizeof(s)-1, "INSERT INTO %s (`id`, `from`, `to`, `subject`, `body`, `arrived`, `sent`, `size`, `attachments`) values(%llu,?,?,?,?,%ld,%ld,%d,%d)", SQL_SPHINX_TABLE, id, sdata->now, sdata->sent, sdata->tot_len, state->n_attachments);
+
 
    if(mysql_stmt_prepare(stmt, s, strlen(s))){
-      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt));
+      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_stmt_error(stmt));
-      goto ENDE_META;
+      return ERR;
    }
 
 
 
-   if(strlen(state->b_to) < 5){
+   memset(bind, 0, sizeof(bind));
-      snprintf(s2, sizeof(s2)-1, "undisclosed-recipients");
-      p = NULL;
-      goto LABEL1;
-   }
-   else p = state->b_to;
 
+   bind[0].buffer_type = MYSQL_TYPE_STRING;
+   bind[0].buffer = state->b_from;
+   bind[0].is_null = 0;
+   len[0] = strlen(state->b_from); bind[0].length = &len[0];
+
+   bind[1].buffer_type = MYSQL_TYPE_STRING;
+   bind[1].buffer = state->b_to;
+   bind[1].is_null = 0;
+   len[1] = strlen(state->b_to); bind[1].length = &len[1];
+
+   bind[2].buffer_type = MYSQL_TYPE_STRING;
+   bind[2].buffer = subj;
+   bind[2].is_null = 0;
+   len[2] = strlen(subj); bind[2].length = &len[2];
+
+   bind[3].buffer_type = MYSQL_TYPE_STRING;
+   bind[3].buffer = state->b_body;
+   bind[3].is_null = 0;
+   len[3] = strlen(state->b_body); bind[3].length = &len[3];
+
+
+   if(mysql_stmt_bind_param(stmt, bind)){
+      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_stmt_error(stmt));
+      return ERR;
+   }
+
+
+   rc = mysql_stmt_execute(stmt);
+
+   if(rc){
+      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute error: *%s*", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_error(&(sdata->mysql)));
+      return ERR;
+   }
+
+
+   return OK;
+}
+
+
+int store_recipients(struct session_data *sdata, char *to, uint64 id, struct __config *cfg){
+   int rc, ret=OK;
+   char *p, s[SMALLBUFSIZE], puf[SMALLBUFSIZE];
+
+   MYSQL_STMT *stmt;
+   MYSQL_BIND bind[1];
+   unsigned long len[1];
+
+   stmt = mysql_stmt_init(&(sdata->mysql));
+   if(!stmt){
+      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_RECIPIENT_TABLE);
+      return ERR;
+   }
+
+   snprintf(s, sizeof(s)-1, "INSERT INTO %s (`id`,`to`) VALUES('%llu',?)", SQL_RECIPIENT_TABLE, id);
+
+   if(mysql_stmt_prepare(stmt, s, strlen(s))){
+      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_stmt_error(stmt));
+      return ERR;
+   }
+
+   p = to;
    do {
-      p = split_str(p, " ", s2, sizeof(s2)-1);
+      p = split_str(p, " ", puf, sizeof(puf)-1);
-      if(strlen(s2) > 5){
-LABEL1:
 
-         i++;
+      if(strlen(puf) > 5){
 
          memset(bind, 0, sizeof(bind));
 
          bind[0].buffer_type = MYSQL_TYPE_STRING;
-         bind[0].buffer = state->b_from;
+         bind[0].buffer = &puf[0];
          bind[0].is_null = 0;
-         len[0] = strlen(state->b_from); bind[0].length = &len[0];
+         len[0] = strlen(puf); bind[0].length = &len[0];
-
-         bind[1].buffer_type = MYSQL_TYPE_STRING;
-         bind[1].buffer = s2;
-         bind[1].is_null = 0;
-         len[1] = strlen(s2); bind[1].length = &len[1];
-
-         bind[2].buffer_type = MYSQL_TYPE_STRING;
-         bind[2].buffer = subj;
-         bind[2].is_null = 0;
-         len[2] = strlen(subj); bind[2].length = &len[2];
-
-         bind[3].buffer_type = MYSQL_TYPE_STRING;
-         bind[3].buffer = state->message_id;
-         bind[3].is_null = 0;
-         len[3] = strlen(state->message_id); bind[3].length = &len[3];
 
          if(mysql_stmt_bind_param(stmt, bind)){
-            syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt));
+            syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_stmt_error(stmt));
-            goto ENDE_META;
+            return ERR;
          }
 
 
          rc = mysql_stmt_execute(stmt);
 
          if(rc){
-            syslog(LOG_PRIORITY, "%s: meta sql error: *%s*", sdata->ttmpfile, mysql_error(&(sdata->mysql)));
+            syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute error: *%s*", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_error(&(sdata->mysql)));
-
+            ret = ERR;
-            ret = ERR_EXISTS;
-            goto ENDE_META;
          }
 
       }
 
    } while(p);
 
-
+   return ret;
-   if(i == 0) ret = ERR_EXISTS;
+}
-   else ret = OK;
 
 
-ENDE_META:
+int store_meta_data(struct session_data *sdata, struct _state *state, struct __config *cfg){
+   int rc, ret=ERR;
+   char *subj, s[MAXBUFSIZE];
+
+   MYSQL_STMT *stmt;
+   MYSQL_BIND bind[4];
+   unsigned long len[4];
+
+   my_ulonglong id=0;
+
+
+   stmt = mysql_stmt_init(&(sdata->mysql));
+   if(!stmt){
+      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_METADATA_TABLE);
+      return ERR;
+   }
+
+   subj = state->b_subject;
+   if(*subj == ' ') subj++;
+
+   snprintf(s, MAXBUFSIZE-1, "INSERT INTO %s (`from`,`subject`,`arrived`,`sent`,`size`,`hlen`,`attachments`,`piler_id`,`message_id`,`digest`,`bodydigest`) VALUES(?,?,%ld,%ld,%d,%d,%d,'%s',?,'%s','%s')", SQL_METADATA_TABLE, sdata->now, sdata->sent, sdata->tot_len, sdata->hdr_len, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
+
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: meta sql: *%s*", sdata->ttmpfile, s);
+
+   if(mysql_stmt_prepare(stmt, s, strlen(s))){
+      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt));
+      return ERR;
+   }
+
+
+
+   if(strlen(state->b_to) < 5){
+      snprintf(state->b_to, SMALLBUFSIZE-1, "undisclosed-recipients");
+   }
+
+   memset(bind, 0, sizeof(bind));
+
+   bind[0].buffer_type = MYSQL_TYPE_STRING;
+   bind[0].buffer = state->b_from;
+   bind[0].is_null = 0;
+   len[0] = strlen(state->b_from); bind[0].length = &len[0];
+
+   bind[1].buffer_type = MYSQL_TYPE_STRING;
+   bind[1].buffer = subj;
+   bind[1].is_null = 0;
+   len[1] = strlen(subj); bind[1].length = &len[1];
+
+   bind[2].buffer_type = MYSQL_TYPE_STRING;
+   bind[2].buffer = state->message_id;
+   bind[2].is_null = 0;
+   len[2] = strlen(state->message_id); bind[2].length = &len[2];
+
+   if(mysql_stmt_bind_param(stmt, bind)){
+      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt));
+      return ERR;
+   }
+
+
+   rc = mysql_stmt_execute(stmt);
+
+   if(rc){
+      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute() error: *%s*", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_error(&(sdata->mysql)));
+      ret = ERR_EXISTS;
+   }
+   else {
+      id = mysql_stmt_insert_id(stmt);
+      rc = store_recipients(sdata, state->b_to, id, cfg);
+
+      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored recipients, rc=%d", sdata->ttmpfile, rc);
+
+      if(rc == OK){
+         rc = store_index_data(sdata, state, id, cfg);
+
+         if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored indexdata, rc=%d", sdata->ttmpfile, rc);
+
+         if(rc == OK)
+            ret = OK;
+      }
+   }
 
    return ret;
 }
@@ -271,9 +365,6 @@ int processMessage(struct session_data *sdata, struct _state *state, struct __co
    if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored metadata, rc=%d",  sdata->ttmpfile, rc);
    if(rc == ERR_EXISTS) return ERR_EXISTS;
 
-   rc = hand_to_sphinx(sdata, state, cfg);
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored indexdata, rc=%d", sdata->ttmpfile, rc);
-
    return OK;
 }
 

src/store.c

@@ -119,7 +119,6 @@ int store_file(struct session_data *sdata, char *filename, int startpos, int len
       strncat(s, p, sizeof(s)-1);
    }
 
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to store %d bytes for %s", sdata->ttmpfile, len, filename);
 
    p0 = strrchr(s, '/'); if(!p0) goto ENDE;
    *p0 = '\0';
@@ -150,6 +149,7 @@ int store_file(struct session_data *sdata, char *filename, int startpos, int len
 
    if(n == outlen){
       ret = 1;
+      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored '%s' %d/%d bytes", sdata->ttmpfile, filename, len, outlen);
    }
    else {
       syslog(LOG_PRIORITY, "%s: cannot write %d bytes (only %d)", sdata->ttmpfile, outlen, n);

util/db-mysql.sql

@@ -7,7 +7,7 @@ create table `sph_counter` (
 
 drop table if exists `sph_index`;
 create table `sph_index` (
-  `id` int not null auto_increment,
+  `id` bigint not null,
   `from` char(255) default null,
   `to` text(512) default null,
   `subject` text(512) default null,
@@ -16,7 +16,6 @@ create table `sph_index` (
   `body` text,
   `size` int default '0',
   `attachments` int default 0,
-  `piler_id` char(36) not null,
   primary key (`id`)
 ) Engine=InnoDB;
 
@@ -25,7 +24,6 @@ drop table if exists `metadata`;
 create table `metadata` (
   `id` bigint unsigned not null auto_increment,
   `from` char(255) not null,
-  `to` text(2048) character set 'latin1' not null,
   `subject` text(512) default null,
   `arrived` int not null,
   `sent` int not null,
@@ -44,6 +42,18 @@ create index metadata_idx on metadata(`piler_id`);
 create index metadata_idx2 on metadata(`message_id`); 
 create index metadata_idx3 on metadata(`bodydigest`); 
 
+
+drop table if exists `rcpt`;
+create table `rcpt` (
+   `id` bigint unsigned not null,
+   `to` char(64) not null,
+   unique(`id`,`to`)
+) Engine=InnoDB;
+
+create index `rcpt_idx` on `rcpt`(`id`);
+create index `rcpt_idx2` on `rcpt`(`to`);
+
+
 drop table if exists `attachment`;
 create table `attachment` (
    `id` bigint unsigned not null auto_increment,
@@ -60,6 +70,7 @@ create table `attachment` (
 create index `attachment_idx` on `attachment`(`piler_id`);
 create index `attachment_idx2` on `attachment`(`sig`);
 
+
 drop table if exists `archiving_rule`;
 create table `archiving_rule` (
    `id` bigint unsigned not null auto_increment,