bashclub/piler
1
0
Fork 0
mirror of https://bitbucket.org/jsuto/piler.git synced 2024-11-07 23:41:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

added folder support for piler daemon and pilerimport

Browse Source
This commit is contained in:
SJ 2012-08-23 10:23:58 +02:00
parent 572d258c2a
commit d0141507b0
11 changed files with 180 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/config.h
View File

@ -13,7 +13,7 @@
#define VERSION "0.1.20"
#define BUILD 687
#define BUILD 691
#define HOSTID "mailarchiver"
@ -76,6 +76,7 @@
#define SQL_SPHINX_TABLE "sph_index"
#define SQL_METADATA_TABLE "metadata"
#define SQL_ATTACHMENT_TABLE "attachment"
#define SQL_FOLDER_TABLE "folder"
#define SQL_RECIPIENT_TABLE "rcpt"
#define SQL_ARCHIVING_RULE_TABLE "archiving_rule"
#define SQL_RETENTION_RULE_TABLE "retention_rule"

2
src/defs.h
View File

@ -225,7 +225,7 @@ struct memcached_server {
struct __data {
char *folder;
int folder;
#ifdef HAVE_TRE
struct rule *archiving_rules;

98
src/import.c
View File

@ -118,3 +118,101 @@ ENDE:
return rc;
}
unsigned long get_folder_id(struct session_data *sdata, char *foldername){
unsigned long id=0;
char s[SMALLBUFSIZE];
MYSQL_STMT *stmt;
MYSQL_BIND bind[1];
unsigned long len[1];
snprintf(s, SMALLBUFSIZE-1, "SELECT `id` FROM %s WHERE `name`=?", SQL_FOLDER_TABLE);
if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) goto ENDE;
memset(bind, 0, sizeof(bind));
bind[0].buffer_type = MYSQL_TYPE_STRING;
bind[0].buffer = foldername;
bind[0].is_null = 0;
len[0] = strlen(foldername); bind[0].length = &len[0];
if(mysql_stmt_bind_param(stmt, bind)){
goto CLOSE;
}
if(mysql_stmt_execute(stmt)){
goto CLOSE;
}
memset(bind, 0, sizeof(bind));
bind[0].buffer_type = MYSQL_TYPE_LONG;
bind[0].buffer = (char *)&id;
bind[0].is_null = 0;
bind[0].length = 0;
if(mysql_stmt_bind_result(stmt, bind)){
goto CLOSE;
}
if(mysql_stmt_store_result(stmt)){
goto CLOSE;
}
mysql_stmt_fetch(stmt);
CLOSE:
mysql_stmt_close(stmt);
ENDE:
return id;
}
unsigned long add_new_folder(struct session_data *sdata, char *foldername){
unsigned long id=0;
char s[SMALLBUFSIZE];
MYSQL_STMT *stmt;
MYSQL_BIND bind[2];
unsigned long len[2];
snprintf(s, sizeof(s)-1, "INSERT INTO %s (`name`) VALUES(?)", SQL_FOLDER_TABLE);
if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) goto ENDE;
memset(bind, 0, sizeof(bind));
bind[0].buffer_type = MYSQL_TYPE_STRING;
bind[0].buffer = foldername;
bind[0].is_null = 0;
len[0] = strlen(foldername); bind[0].length = &len[0];
if(mysql_stmt_bind_param(stmt, bind)){
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_FOLDER_TABLE, mysql_stmt_error(stmt));
goto CLOSE;
}
if(mysql_stmt_execute(stmt)){
syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute error: *%s*", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_error(&(sdata->mysql)));
goto CLOSE;
}
id = mysql_stmt_insert_id(stmt);
CLOSE:
mysql_stmt_close(stmt);
ENDE:
return id;
}

12
src/message.c
View File

@ -122,7 +122,7 @@ int is_body_digest_already_stored(struct session_data *sdata, struct _state *sta
}
int store_index_data(struct session_data *sdata, struct _state *state, uint64 id, struct __config *cfg){
int store_index_data(struct session_data *sdata, struct _state *state, struct __data *data, uint64 id, struct __config *cfg){
int rc=ERR;
char *subj, s[SMALLBUFSIZE];
@ -134,7 +134,7 @@ int store_index_data(struct session_data *sdata, struct _state *state, uint64 id
if(*subj == ' ') subj++;
snprintf(s, sizeof(s)-1, "INSERT INTO %s (`id`, `from`, `to`, `fromdomain`, `todomain`, `subject`, `body`, `arrived`, `sent`, `size`, `direction`, `attachments`, `attachment_types`) values(%llu,?,?,?,?,?,?,%ld,%ld,%d,%d,%d,?)", SQL_SPHINX_TABLE, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, state->n_attachments);
snprintf(s, sizeof(s)-1, "INSERT INTO %s (`id`, `from`, `to`, `fromdomain`, `todomain`, `subject`, `body`, `arrived`, `sent`, `size`, `direction`, `folder`, `attachments`, `attachment_types`) values(%llu,?,?,?,?,?,?,%ld,%ld,%d,%d,%d,%d,?)", SQL_SPHINX_TABLE, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, data->folder, state->n_attachments);
if(prepare_a_mysql_statement(sdata, &stmt, s) == ERR) return rc;
@ -264,7 +264,7 @@ CLOSE:
}
int store_meta_data(struct session_data *sdata, struct _state *state, struct __config *cfg){
int store_meta_data(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg){
int rc, ret=ERR;
char *subj, *p, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1];
@ -356,7 +356,7 @@ int store_meta_data(struct session_data *sdata, struct _state *state, struct __c
if(rc == OK){
rc = store_index_data(sdata, state, id, cfg);
rc = store_index_data(sdata, state, data, id, cfg);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored indexdata, rc=%d", sdata->ttmpfile, rc);
@ -385,7 +385,7 @@ int process_message(struct session_data *sdata, struct _state *state, struct __d
/* check for existing body digest */
rc = is_body_digest_already_stored(sdata, state, cfg);
//rc = is_body_digest_already_stored(sdata, state, cfg);
/*
* TODO: check if the bodydigest were stored, then we should
@ -415,7 +415,7 @@ int process_message(struct session_data *sdata, struct _state *state, struct __d
sdata->retained += query_retain_period(data->retention_rules, state, sdata->tot_len, sdata->spam_message, cfg);
rc = store_meta_data(sdata, state, cfg);
rc = store_meta_data(sdata, state, data, cfg);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored metadata, rc=%d", sdata->ttmpfile, rc);
if(rc == ERR_EXISTS){

4
src/piler.c
View File

@ -288,7 +288,7 @@ void initialise_configuration(){
free_rule(data.archiving_rules);
free_rule(data.retention_rules);
data.folder = NULL;
data.folder = 0;
data.archiving_rules = NULL;
data.retention_rules = NULL;
@ -342,7 +342,7 @@ int main(int argc, char **argv){
(void) openlog(PROGNAME, LOG_PID, LOG_MAIL);
data.folder = NULL;
data.folder = 0;
data.archiving_rules = NULL;
data.retention_rules = NULL;

4
src/piler.h
View File

@ -52,8 +52,10 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct _
int prepare_a_mysql_statement(struct session_data *sdata, MYSQL_STMT **stmt, char *s);
int import_message(char *filename, struct session_data *sdata, struct __data *data, struct __config *cfg);
unsigned long get_folder_id(struct session_data *sdata, char *foldername);
unsigned long add_new_folder(struct session_data *sdata, char *foldername);
int store_index_data(struct session_data *sdata, struct _state *state, uint64 id, struct __config *cfg);
int store_index_data(struct session_data *sdata, struct _state *state, struct __data *data, uint64 id, struct __config *cfg);
#endif /* _PILER_H */

36
src/pilerimport.c
View File

@ -253,19 +253,17 @@ void usage(){
int main(int argc, char **argv){
int i, c, rc=0, n_mbox=0, tot_msgs=0;
char *configfile=CONFIG_FILE, *emlfile=NULL, *mboxdir=NULL, *mbox[MBOX_ARGS], *directory=NULL;
char *imapserver=NULL, *username=NULL, *password=NULL, *skiplist=SKIPLIST;
char *imapserver=NULL, *username=NULL, *password=NULL, *skiplist=SKIPLIST, *folder=NULL;
struct session_data sdata;
struct __config cfg;
struct __data data;
for(i=0; i<MBOX_ARGS; i++) mbox[i] = NULL;
data.folder = NULL;
data.folder = 0;
data.archiving_rules = NULL;
data.retention_rules = NULL;
while(1){
#ifdef _GNU_SOURCE
@ -280,16 +278,16 @@ int main(int argc, char **argv){
{"username", required_argument, 0, 'u' },
{"password", required_argument, 0, 'p' },
{"skiplist", required_argument, 0, 'x' },
{"folder", required_argument, 0, 'f' },
{"folder", required_argument, 0, 'F' },
{"help", no_argument, 0, 'h' },
{0,0,0,0}
};
int option_index = 0;
c = getopt_long(argc, argv, "c:m:M:e:d:i:u:p:x:f:h?", long_options, &option_index);
c = getopt_long(argc, argv, "c:m:M:e:d:i:u:p:x:F:h?", long_options, &option_index);
#else
c = getopt(argc, argv, "c:m:M:e:d:i:u:p:x:f:h?");
c = getopt(argc, argv, "c:m:M:e:d:i:u:p:x:F:h?");
#endif
if(c == -1) break;
@ -337,8 +335,8 @@ int main(int argc, char **argv){
skiplist = optarg;
break;
case 'f' :
data.folder = optarg;
case 'F' :
folder = optarg;
break;
case 'h' :
@ -356,7 +354,6 @@ int main(int argc, char **argv){
if(!mbox[0] && !mboxdir && !emlfile && !directory && !imapserver) usage();
cfg = read_config(configfile);
if(read_key(&cfg)){
@ -367,7 +364,7 @@ int main(int argc, char **argv){
mysql_init(&(sdata.mysql));
mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout);
if(mysql_real_connect(&(sdata.mysql), cfg.mysqlhost, cfg.mysqluser, cfg.mysqlpwd, cfg.mysqldb, cfg.mysqlport, cfg.mysqlsocket, 0) == 0){
printf("cant connect to mysql server\n");
printf("error: cant connect to mysql server\n");
return ERR;
}
@ -378,10 +375,26 @@ int main(int argc, char **argv){
(void) openlog("pilerimport", LOG_PID, LOG_MAIL);
if(folder){
data.folder = get_folder_id(&sdata, folder);
if(data.folder == 0){
data.folder = add_new_folder(&sdata, folder);
}
if(data.folder == 0){
printf("error: cannot get/add folder '%s'\n", folder);
mysql_close(&(sdata.mysql));
return 0;
}
}
load_rules(&sdata, &(data.archiving_rules), SQL_ARCHIVING_RULE_TABLE);
load_rules(&sdata, &(data.retention_rules), SQL_RETENTION_RULE_TABLE);
if(emlfile) rc = import_message(emlfile, &sdata, &data, &cfg);
if(mbox[0]){
for(i=0; i<n_mbox; i++){
rc = import_from_mailbox(mbox[i], &sdata, &data, &cfg);
@ -392,7 +405,6 @@ int main(int argc, char **argv){
if(imapserver && username && password) rc = import_from_imap_server(imapserver, username, password, &sdata, &data, skiplist, &cfg);
free_rule(data.archiving_rules);
free_rule(data.retention_rules);

28
src/reindex.c
View File

@ -41,7 +41,7 @@ void clean_exit(char *msg, int rc){
}
uint64 retrieve_email_by_metadata_id(struct session_data *sdata, uint64 from_id, uint64 to_id, struct __config *cfg){
uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct __data *data, uint64 from_id, uint64 to_id, struct __config *cfg){
MYSQL_RES *res;
MYSQL_ROW row;
FILE *f;
@ -86,7 +86,7 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, uint64 from_id,
sdata->now = strtoul(row[2], NULL, 10);
sdata->sent = strtoul(row[3], NULL, 10);
rc = store_index_data(sdata, &state, stored_id, cfg);
rc = store_index_data(sdata, &state, data, stored_id, cfg);
if(rc == OK) reindexed++;
else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename);
@ -113,13 +113,14 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, uint64 from_id,
int main(int argc, char **argv){
int c;
uint64 from_id=0, to_id=0, n=0;
char *configfile=CONFIG_FILE;
char *configfile=CONFIG_FILE, *folder=NULL;
struct session_data sdata;
struct __data data;
struct __config cfg;
while(1){
c = getopt(argc, argv, "c:f:t:phv?");
c = getopt(argc, argv, "c:f:t:F:phv?");
if(c == -1) break;
@ -137,6 +138,11 @@ int main(int argc, char **argv){
to_id = strtoull(optarg, NULL, 10);
break;
case 'F' :
folder = optarg;
break;
case 'p' :
progressbar = 1;
break;
@ -163,6 +169,18 @@ int main(int argc, char **argv){
return 1;
}
data.folder = 0;
data.archiving_rules = NULL;
data.retention_rules = NULL;
if(folder){
data.folder = get_folder_id(&sdata, folder);
if(data.folder == 0){
printf("error: could not get folder id for '%s'\n", folder);
return 0;
}
}
init_session_data(&sdata);
@ -176,7 +194,7 @@ int main(int argc, char **argv){
mysql_real_query(&(sdata.mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8"));
n = retrieve_email_by_metadata_id(&sdata, from_id, to_id, &cfg);
n = retrieve_email_by_metadata_id(&sdata, &data, from_id, to_id, &cfg);
printf("put %llu messages to %s table for reindexing\n", n, SQL_SPHINX_TABLE);

2
src/test.c
View File

@ -53,7 +53,7 @@ int main(int argc, char **argv){
printf("build: %d\n", get_build());
data.folder = NULL;
data.folder = 0;
data.archiving_rules = NULL;
data.retention_rules = NULL;

9
util/db-mysql.sql
View File

@ -23,6 +23,7 @@ create table if not exists `sph_index` (
`body` text,
`size` int default '0',
`direction` int default 0,
`folder` int default 0,
`attachments` int default 0,
`attachment_types` text(512) default null,
primary key (`id`)
@ -61,6 +62,14 @@ create index metadata_idx5 on metadata(`deleted`);
create index metadata_idx6 on metadata(`arrived`);
create index metadata_idx7 on metadata(`retained`);
drop table if exists `folder`;
create table if not exists `folder` (
`id` int not null auto_increment,
`parent_id` int default 0,
`name` char(64) not null unique,
primary key (`id`)
) Engine=InnoDB;
drop table if exists `rcpt`;
create table if not exists `rcpt` (

11
util/db-upgrade-0.20-vs-0.21.sql Normal file
View File

@ -0,0 +1,11 @@
alter table `sph_index` add column `folder` int default 0;
drop table if exists `folder`;
create table if not exists `folder` (
`id` int not null auto_increment,
`parent_id` int default 0,
`name` char(64) not null unique,
primary key (`id`)
) Engine=InnoDB;