bashclub/piler
1
0
Fork 0
mirror of https://bitbucket.org/jsuto/piler.git synced 2024-12-24 18:50:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

0.1.9

Browse Source
This commit is contained in:
SJ 2011-12-03 23:05:00 +01:00
parent 8e9f3d066a
commit fa52000e8d
6 changed files with 186 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README
View File

@ -8,6 +8,9 @@ Features:
locales debian alatt: dpkg-reconfigure locales
init:
indexer --all
searchd inditasa
indexer delta1 --rotate

6
etc/sphinx.conf
View File

@ -11,14 +11,13 @@ source main
sql_pass = sphinx
sql_query_pre = SET NAMES utf8
sql_query = SELECT id, `from`, `to`, `subject`, `arrived`, `sent`, `body`, `size`, `attachments`, `piler_id` FROM sph_index \
sql_query = SELECT id, `from`, `to`, `subject`, `arrived`, `sent`, `body`, `size`, `attachments` FROM sph_index \
WHERE id<=( SELECT max_doc_id FROM sph_counter WHERE counter_id=1 )
sql_attr_uint = size
sql_attr_uint = arrived
sql_attr_uint = sent
sql_attr_uint = attachments
sql_attr_string = piler_id
}
source delta
@ -32,14 +31,13 @@ source delta
sql_query_pre = SET NAMES utf8
sql_query_pre = REPLACE INTO sph_counter SELECT 1, MAX(id) FROM sph_index
sql_query_post_index = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
sql_query = SELECT id, `from`, `to`, `subject`, `arrived`, `sent`, `body`, `size`, `attachments`, `piler_id` FROM sph_index \
sql_query = SELECT id, `from`, `to`, `subject`, `arrived`, `sent`, `body`, `size`, `attachments` FROM sph_index \
WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
sql_attr_uint = size
sql_attr_uint = arrived
sql_attr_uint = sent
sql_attr_uint = attachments
sql_attr_string = piler_id
}

3
src/config.h
View File

@ -11,7 +11,7 @@
#define PROGNAME "piler"
#define VERSION "0.1.8"
#define VERSION "0.1.9"
#define PROGINFO VERSION ", Janos SUTO <sj@acts.hu>\n\n" CONFIGURE_PARAMS "\n\nSend bugs/issues to https://jira.acts.hu:8443/\n"
@ -74,6 +74,7 @@
#define SQL_SPHINX_TABLE "sph_index"
#define SQL_METADATA_TABLE "metadata"
#define SQL_ATTACHMENT_TABLE "attachment"
#define SQL_RECIPIENT_TABLE "rcpt"
#define SQL_ARCHIVING_RULE_TABLE "archiving_rule"
#define SQL_COUNTER_TABLE "counter"

217
src/message.c
View File

@ -99,8 +99,6 @@ int is_body_digest_already_stored(struct session_data *sdata, struct _state *sta
snprintf(s, SMALLBUFSIZE-1, "SELECT `bodydigest` FROM `%s` WHERE `bodydigest`='%s'", SQL_METADATA_TABLE, sdata->bodydigest);
//if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: check for body digest sql: *%s*", sdata->ttmpfile, s);
if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){
res = mysql_store_result(&(sdata->mysql));
if(res != NULL){
@ -114,67 +112,35 @@ int is_body_digest_already_stored(struct session_data *sdata, struct _state *sta
}
int hand_to_sphinx(struct session_data *sdata, struct _state *state, struct __config *cfg){
int store_index_data(struct session_data *sdata, struct _state *state, uint64 id, struct __config *cfg){
int rc;
char *subj, s[BIGBUFSIZE+2*MAXBUFSIZE];
subj = state->b_subject;
if(*subj == ' ') subj++;
snprintf(s, sizeof(s)-1, "INSERT INTO %s (`from`, `to`, `subject`, `body`, `arrived`, `sent`, `size`, `attachments`, `piler_id`) values('%s','%s','%s','%s',%ld,%ld,%d,%d,'%s')", SQL_SPHINX_TABLE, state->b_from, state->b_to, subj, state->b_body, sdata->now, sdata->sent, sdata->tot_len, state->n_attachments, sdata->ttmpfile);
rc = mysql_real_query(&(sdata->mysql), s, strlen(s));
if(rc == 0) return OK;
syslog(LOG_PRIORITY, "%s: sphinx sql error: *%s*", sdata->ttmpfile, mysql_error(&(sdata->mysql)));
return ERR;
}
int store_meta_data(struct session_data *sdata, struct _state *state, struct __config *cfg){
int i=0, rc, ret=ERR;
char *p, *subj, s[MAXBUFSIZE], s2[SMALLBUFSIZE];
char *subj, s[SMALLBUFSIZE];
MYSQL_STMT *stmt;
MYSQL_BIND bind[4];
unsigned long len[4];
stmt = mysql_stmt_init(&(sdata->mysql));
if(!stmt){
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_METADATA_TABLE);
goto ENDE_META;
}
subj = state->b_subject;
if(*subj == ' ') subj++;
snprintf(s, MAXBUFSIZE-1, "INSERT INTO %s (`from`,`to`,`subject`,`arrived`,`sent`,`size`,`hlen`,`attachments`,`piler_id`,`message_id`,`digest`,`bodydigest`) VALUES(?,?,?,%ld,%ld,%d,%d,%d,'%s',?,'%s','%s')", SQL_METADATA_TABLE, sdata->now, sdata->sent, sdata->tot_len, sdata->hdr_len, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: meta sql: *%s*", sdata->ttmpfile, s);
stmt = mysql_stmt_init(&(sdata->mysql));
if(!stmt){
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_SPHINX_TABLE);
return ERR;
}
snprintf(s, sizeof(s)-1, "INSERT INTO %s (`id`, `from`, `to`, `subject`, `body`, `arrived`, `sent`, `size`, `attachments`) values(%llu,?,?,?,?,%ld,%ld,%d,%d)", SQL_SPHINX_TABLE, id, sdata->now, sdata->sent, sdata->tot_len, state->n_attachments);
if(mysql_stmt_prepare(stmt, s, strlen(s))){
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt));
goto ENDE_META;
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_stmt_error(stmt));
return ERR;
}
if(strlen(state->b_to) < 5){
snprintf(s2, sizeof(s2)-1, "undisclosed-recipients");
p = NULL;
goto LABEL1;
}
else p = state->b_to;
do {
p = split_str(p, " ", s2, sizeof(s2)-1);
if(strlen(s2) > 5){
LABEL1:
i++;
memset(bind, 0, sizeof(bind));
bind[0].buffer_type = MYSQL_TYPE_STRING;
@ -183,9 +149,9 @@ LABEL1:
len[0] = strlen(state->b_from); bind[0].length = &len[0];
bind[1].buffer_type = MYSQL_TYPE_STRING;
bind[1].buffer = s2;
bind[1].buffer = state->b_to;
bind[1].is_null = 0;
len[1] = strlen(s2); bind[1].length = &len[1];
len[1] = strlen(state->b_to); bind[1].length = &len[1];
bind[2].buffer_type = MYSQL_TYPE_STRING;
bind[2].buffer = subj;
@ -193,35 +159,163 @@ LABEL1:
len[2] = strlen(subj); bind[2].length = &len[2];
bind[3].buffer_type = MYSQL_TYPE_STRING;
bind[3].buffer = state->message_id;
bind[3].buffer = state->b_body;
bind[3].is_null = 0;
len[3] = strlen(state->message_id); bind[3].length = &len[3];
len[3] = strlen(state->b_body); bind[3].length = &len[3];
if(mysql_stmt_bind_param(stmt, bind)){
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt));
goto ENDE_META;
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_stmt_error(stmt));
return ERR;
}
rc = mysql_stmt_execute(stmt);
if(rc){
syslog(LOG_PRIORITY, "%s: meta sql error: *%s*", sdata->ttmpfile, mysql_error(&(sdata->mysql)));
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute error: *%s*", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_error(&(sdata->mysql)));
return ERR;
}
ret = ERR_EXISTS;
goto ENDE_META;
return OK;
}
int store_recipients(struct session_data *sdata, char *to, uint64 id, struct __config *cfg){
int rc, ret=OK;
char *p, s[SMALLBUFSIZE], puf[SMALLBUFSIZE];
MYSQL_STMT *stmt;
MYSQL_BIND bind[1];
unsigned long len[1];
stmt = mysql_stmt_init(&(sdata->mysql));
if(!stmt){
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_RECIPIENT_TABLE);
return ERR;
}
snprintf(s, sizeof(s)-1, "INSERT INTO %s (`id`,`to`) VALUES('%llu',?)", SQL_RECIPIENT_TABLE, id);
if(mysql_stmt_prepare(stmt, s, strlen(s))){
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_stmt_error(stmt));
return ERR;
}
p = to;
do {
p = split_str(p, " ", puf, sizeof(puf)-1);
if(strlen(puf) > 5){
memset(bind, 0, sizeof(bind));
bind[0].buffer_type = MYSQL_TYPE_STRING;
bind[0].buffer = &puf[0];
bind[0].is_null = 0;
len[0] = strlen(puf); bind[0].length = &len[0];
if(mysql_stmt_bind_param(stmt, bind)){
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_stmt_error(stmt));
return ERR;
}
rc = mysql_stmt_execute(stmt);
if(rc){
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute error: *%s*", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_error(&(sdata->mysql)));
ret = ERR;
}
}
} while(p);
if(i == 0) ret = ERR_EXISTS;
else ret = OK;
return ret;
}
ENDE_META:
int store_meta_data(struct session_data *sdata, struct _state *state, struct __config *cfg){
int rc, ret=ERR;
char *subj, s[MAXBUFSIZE];
MYSQL_STMT *stmt;
MYSQL_BIND bind[4];
unsigned long len[4];
my_ulonglong id=0;
stmt = mysql_stmt_init(&(sdata->mysql));
if(!stmt){
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_init() error", sdata->ttmpfile, SQL_METADATA_TABLE);
return ERR;
}
subj = state->b_subject;
if(*subj == ' ') subj++;
snprintf(s, MAXBUFSIZE-1, "INSERT INTO %s (`from`,`subject`,`arrived`,`sent`,`size`,`hlen`,`attachments`,`piler_id`,`message_id`,`digest`,`bodydigest`) VALUES(?,?,%ld,%ld,%d,%d,%d,'%s',?,'%s','%s')", SQL_METADATA_TABLE, sdata->now, sdata->sent, sdata->tot_len, sdata->hdr_len, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: meta sql: *%s*", sdata->ttmpfile, s);
if(mysql_stmt_prepare(stmt, s, strlen(s))){
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_prepare() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt));
return ERR;
}
if(strlen(state->b_to) < 5){
snprintf(state->b_to, SMALLBUFSIZE-1, "undisclosed-recipients");
}
memset(bind, 0, sizeof(bind));
bind[0].buffer_type = MYSQL_TYPE_STRING;
bind[0].buffer = state->b_from;
bind[0].is_null = 0;
len[0] = strlen(state->b_from); bind[0].length = &len[0];
bind[1].buffer_type = MYSQL_TYPE_STRING;
bind[1].buffer = subj;
bind[1].is_null = 0;
len[1] = strlen(subj); bind[1].length = &len[1];
bind[2].buffer_type = MYSQL_TYPE_STRING;
bind[2].buffer = state->message_id;
bind[2].is_null = 0;
len[2] = strlen(state->message_id); bind[2].length = &len[2];
if(mysql_stmt_bind_param(stmt, bind)){
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(stmt));
return ERR;
}
rc = mysql_stmt_execute(stmt);
if(rc){
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute() error: *%s*", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_error(&(sdata->mysql)));
ret = ERR_EXISTS;
}
else {
id = mysql_stmt_insert_id(stmt);
rc = store_recipients(sdata, state->b_to, id, cfg);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored recipients, rc=%d", sdata->ttmpfile, rc);
if(rc == OK){
rc = store_index_data(sdata, state, id, cfg);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored indexdata, rc=%d", sdata->ttmpfile, rc);
if(rc == OK)
ret = OK;
}
}
return ret;
}
@ -271,9 +365,6 @@ int processMessage(struct session_data *sdata, struct _state *state, struct __co
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored metadata, rc=%d", sdata->ttmpfile, rc);
if(rc == ERR_EXISTS) return ERR_EXISTS;
rc = hand_to_sphinx(sdata, state, cfg);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored indexdata, rc=%d", sdata->ttmpfile, rc);
return OK;
}

2
src/store.c
View File

@ -119,7 +119,6 @@ int store_file(struct session_data *sdata, char *filename, int startpos, int len
strncat(s, p, sizeof(s)-1);
}
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to store %d bytes for %s", sdata->ttmpfile, len, filename);
p0 = strrchr(s, '/'); if(!p0) goto ENDE;
*p0 = '\0';
@ -150,6 +149,7 @@ int store_file(struct session_data *sdata, char *filename, int startpos, int len
if(n == outlen){
ret = 1;
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored '%s' %d/%d bytes", sdata->ttmpfile, filename, len, outlen);
}
else {
syslog(LOG_PRIORITY, "%s: cannot write %d bytes (only %d)", sdata->ttmpfile, outlen, n);

17
util/db-mysql.sql
View File

@ -7,7 +7,7 @@ create table `sph_counter` (
drop table if exists `sph_index`;
create table `sph_index` (
`id` int not null auto_increment,
`id` bigint not null,
`from` char(255) default null,
`to` text(512) default null,
`subject` text(512) default null,
@ -16,7 +16,6 @@ create table `sph_index` (
`body` text,
`size` int default '0',
`attachments` int default 0,
`piler_id` char(36) not null,
primary key (`id`)
) Engine=InnoDB;
@ -25,7 +24,6 @@ drop table if exists `metadata`;
create table `metadata` (
`id` bigint unsigned not null auto_increment,
`from` char(255) not null,
`to` text(2048) character set 'latin1' not null,
`subject` text(512) default null,
`arrived` int not null,
`sent` int not null,
@ -44,6 +42,18 @@ create index metadata_idx on metadata(`piler_id`);
create index metadata_idx2 on metadata(`message_id`);
create index metadata_idx3 on metadata(`bodydigest`);
drop table if exists `rcpt`;
create table `rcpt` (
`id` bigint unsigned not null,
`to` char(64) not null,
unique(`id`,`to`)
) Engine=InnoDB;
create index `rcpt_idx` on `rcpt`(`id`);
create index `rcpt_idx2` on `rcpt`(`to`);
drop table if exists `attachment`;
create table `attachment` (
`id` bigint unsigned not null auto_increment,
@ -60,6 +70,7 @@ create table `attachment` (
create index `attachment_idx` on `attachment`(`piler_id`);
create index `attachment_idx2` on `attachment`(`sig`);
drop table if exists `archiving_rule`;
create table `archiving_rule` (
`id` bigint unsigned not null auto_increment,