From bafbbc7fa82302235c299e208c4e3842a6403622 Mon Sep 17 00:00:00 2001 From: SJ Date: Sat, 7 Jan 2012 23:26:47 +0100 Subject: [PATCH] added pilerexport for bulk export to EML, and simplified pilerget --- src/Makefile.in | 5 +- src/config.h | 1 + src/pilerexport.c | 309 ++++++++++++++++++++++++++++++++++++++++++++++ src/pilerget.c | 47 +------ util/db-mysql.sql | 3 +- 5 files changed, 319 insertions(+), 46 deletions(-) create mode 100644 src/pilerexport.c diff --git a/src/Makefile.in b/src/Makefile.in index d210b9f9..ffc1b149 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -33,7 +33,7 @@ MAKE = `which make` INSTALL = @INSTALL@ -all: libpiler.a piler pilerconf pilerget pilerimport test +all: libpiler.a piler pilerconf pilerget pilerimport pilerexport test install: install-piler @@ -54,6 +54,9 @@ pilerget: pilerget.c libpiler.a pilerimport: pilerimport.c libpiler.a $(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $^ -lpiler $(LIBS) $(LIBDIR) +pilerexport: pilerexport.c libpiler.a + $(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $^ -lpiler $(LIBS) $(LIBDIR) + pilerconf: pilerconf.c cfg.o misc.o tai.o $(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $^ $(LIBDIR) diff --git a/src/config.h b/src/config.h index a47352d2..10189e33 100644 --- a/src/config.h +++ b/src/config.h @@ -78,6 +78,7 @@ #define SQL_RECIPIENT_TABLE "rcpt" #define SQL_ARCHIVING_RULE_TABLE "archiving_rule" #define SQL_COUNTER_TABLE "counter" +#define SQL_MESSAGES_VIEW "messages" /* TRE stuff */ diff --git a/src/pilerexport.c b/src/pilerexport.c new file mode 100644 index 00000000..cec7ea0b --- /dev/null +++ b/src/pilerexport.c @@ -0,0 +1,309 @@ +/* + * pilerexport.c, SJ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +extern char *optarg; +extern int optind; + +char *query=NULL; +regex_t regexp; + + + +void usage(){ + printf("usage: .... \n"); + exit(0); +} + + +void clean_exit(char *msg, int rc){ + if(msg) printf("error: %s\n", msg); + + if(query) free(query); + + exit(rc); +} + + +int append_email_to_buffer(char **buffer, char *email){ + int len, arglen; + char *s=NULL, emailaddress[SMALLBUFSIZE]; + + snprintf(emailaddress, sizeof(emailaddress)-1, "'%s'", email); + arglen = strlen(emailaddress); + + if(!*buffer){ + *buffer = malloc(arglen+1); + memset(*buffer, 0, arglen+1); + memcpy(*buffer, emailaddress, arglen); + } + else { + len = strlen(*buffer); + s = realloc(*buffer, len + arglen+2); + if(!s){ + printf("malloc problem!\n"); + return 1; + } + + *buffer = s; + + memset(*buffer+len, 0, arglen+2); + strcat(*buffer, ","); + memcpy(*buffer+len+1, emailaddress, arglen); + } + + return 0; +} + + +int append_string_to_buffer(char **buffer, char *str){ + int len, arglen; + char *s=NULL; + + arglen = strlen(str); + + if(!*buffer){ + *buffer = malloc(arglen+1); + memset(*buffer, 0, arglen+1); + memcpy(*buffer, str, arglen); + } + else { + len = strlen(*buffer); + s = realloc(*buffer, len + arglen+1); + if(!s) return 1; + + *buffer = s; + + memset(*buffer+len, 0, arglen+1); + memcpy(*buffer+len, str, arglen); + } + + return 0; +} + + +int export_emails_matching_to_query(struct session_data *sdata, char *s, struct __config *cfg){ + MYSQL_RES *res; + MYSQL_ROW row; + FILE *f; + uint64 id; + char *digest=NULL, *bodydigest=NULL; + char filename[SMALLBUFSIZE]; + int rc=0; + + + rc = mysql_real_query(&(sdata->mysql), s, strlen(s)); + + if(rc == 0){ + res = mysql_store_result(&(sdata->mysql)); + if(res){ + while((row = mysql_fetch_row(res))){ + + id = strtoull(row[0], NULL, 10); + if(id > 0){ + snprintf(sdata->ttmpfile, SMALLBUFSIZE-1, "%s", (char*)row[1]); + digest = (char*)row[2]; + bodydigest = (char*)row[3]; + + + snprintf(filename, sizeof(filename)-1, "%llu.eml", id); + + f = fopen(filename, "w"); + if(f){ + rc = retrieve_email_from_archive(sdata, f, cfg); + fclose(f); + + snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename); + + make_digests(sdata, cfg); + + if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0) + printf("exported %s, verification: OK\n", filename); + else + printf("exported %s, verification: FAILED\n", filename); + + } + else printf("cannot open: %s\n", filename); + + } + } + mysql_free_result(res); + } + else rc = 1; + } + + return rc; +} + + +int main(int argc, char **argv){ + int i, rc, exportall=0, minsize=0, maxsize=0; + int where_condition=0; + size_t nmatch=0; + //unsigned long startdate=0, stopdate=0; + char *configfile=CONFIG_FILE; + char *to=NULL, *from=NULL; + char s[SMALLBUFSIZE]; + struct session_data sdata; + struct __config cfg; + + + if(regcomp(®exp, "^([\\+a-z0-9_\\.@\\-]+)$", REG_ICASE | REG_EXTENDED)){ + clean_exit("cannot compile rule!", 1); + } + + + while((i = getopt(argc, argv, "c:s:S:f:r:ah?")) > 0){ + + switch(i){ + + case 'c' : + configfile = optarg; + break; + + case 's' : + maxsize = atoi(optarg); + break; + + case 'S' : + minsize = atoi(optarg); + break; + + + case 'a' : + exportall = 1; + break; + + case 'f' : + + if(regexec(®exp, optarg, nmatch, NULL, 0)){ + printf("%s is not a valid email address\n", optarg); + break; + } + + rc = append_email_to_buffer(&from, optarg); + + break; + + case 'r' : + + if(regexec(®exp, optarg, nmatch, NULL, 0)){ + printf("%s is not a valid email address\n", optarg); + break; + } + + rc = append_email_to_buffer(&to, optarg); + + break; + + + + case 'h' : + case '?' : + usage(); + break; + + + default : + break; + } + } + + + regfree(®exp); + + + snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE ", SQL_MESSAGES_VIEW); + + rc = append_string_to_buffer(&query, s); + + if(from){ + rc += append_string_to_buffer(&query, "`from` IN ("); + rc += append_string_to_buffer(&query, from); + rc += append_string_to_buffer(&query, ")"); + + free(from); + + where_condition++; + } + + + if(to){ + if(where_condition) rc = append_string_to_buffer(&query, " AND "); + + rc += append_string_to_buffer(&query, "`to` IN ("); + rc += append_string_to_buffer(&query, to); + rc += append_string_to_buffer(&query, ")"); + + free(to); + + where_condition++; + } + + + if(minsize > 0){ + if(where_condition) rc = append_string_to_buffer(&query, " AND "); + snprintf(s, sizeof(s)-1, " `size` >= %d", minsize); + rc += append_string_to_buffer(&query, s); + } + + + if(maxsize > 0){ + if(where_condition) rc = append_string_to_buffer(&query, " AND "); + snprintf(s, sizeof(s)-1, " `size` <= %d", maxsize); + rc += append_string_to_buffer(&query, s); + } + + rc += append_string_to_buffer(&query, " ORDER BY id ASC"); + + if(rc) clean_exit("malloc problem building query", 1); + + printf("query: *%s*\n", query); + + + + + cfg = read_config(configfile); + + + if(read_key(&cfg)) clean_exit(ERR_READING_KEY, 1); + + + init_session_data(&sdata); + + + mysql_init(&(sdata.mysql)); + mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout); + if(mysql_real_connect(&(sdata.mysql), cfg.mysqlhost, cfg.mysqluser, cfg.mysqlpwd, cfg.mysqldb, cfg.mysqlport, cfg.mysqlsocket, 0) == 0){ + clean_exit("cannot connect to mysql server", 1); + } + + mysql_real_query(&(sdata.mysql), "SET NAMES utf8", strlen("SET NAMES utf8")); + mysql_real_query(&(sdata.mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8")); + + + rc = export_emails_matching_to_query(&sdata, query, &cfg); + + free(query); + + + mysql_close(&(sdata.mysql)); + + return 0; +} + + diff --git a/src/pilerget.c b/src/pilerget.c index bda8655a..c76c4534 100644 --- a/src/pilerget.c +++ b/src/pilerget.c @@ -97,9 +97,6 @@ ENDE: int main(int argc, char **argv){ int rc; - uint64 id; - char filename[SMALLBUFSIZE], digest[2*DIGEST_LENGTH+1], bodydigest[2*DIGEST_LENGTH+1]; - FILE *f; struct session_data sdata; struct __config cfg; @@ -130,47 +127,9 @@ int main(int argc, char **argv){ mysql_real_query(&(sdata.mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8")); - if(argv[1][0] == '-'){ - - memset(sdata.ttmpfile, 0, sizeof(sdata.ttmpfile)); - - while((rc = read(0, sdata.ttmpfile, RND_STR_LEN+1)) > 0){ - snprintf(sdata.filename, SMALLBUFSIZE-1, "%s", sdata.ttmpfile); - - trimBuffer(sdata.ttmpfile); - - id = get_id_by_piler_id(&sdata, &digest[0], &bodydigest[0], &cfg); - - if(id > 0){ - snprintf(filename, sizeof(filename)-1, "%llu.eml", id); - f = fopen(filename, "w"); - if(f){ - rc = retrieve_email_from_archive(&sdata, f, &cfg); - fclose(f); - - snprintf(sdata.ttmpfile, SMALLBUFSIZE-1, "%s", filename); - make_digests(&sdata, &cfg); - - if(strcmp(digest, sdata.digest) == 0 && strcmp(bodydigest, sdata.bodydigest) == 0) - printf("exported %s, verification: OK\n", sdata.ttmpfile); - else - printf("exported %s, verification: FAILED\n", sdata.ttmpfile); - - } - else printf("cannot open: %s\n", filename); - } - else printf("%s was not found in archive\n", sdata.ttmpfile); - - memset(sdata.ttmpfile, 0, sizeof(sdata.ttmpfile)); - - } - - } - else { - snprintf(sdata.ttmpfile, SMALLBUFSIZE-1, "%s", argv[1]); - snprintf(sdata.filename, SMALLBUFSIZE-1, "%s", sdata.ttmpfile); - rc = retrieve_email_from_archive(&sdata, stdout, &cfg); - } + snprintf(sdata.ttmpfile, SMALLBUFSIZE-1, "%s", argv[1]); + snprintf(sdata.filename, SMALLBUFSIZE-1, "%s", sdata.ttmpfile); + rc = retrieve_email_from_archive(&sdata, stdout, &cfg); mysql_close(&(sdata.mysql)); diff --git a/util/db-mysql.sql b/util/db-mysql.sql index c4f0bde9..b21e2cd9 100644 --- a/util/db-mysql.sql +++ b/util/db-mysql.sql @@ -64,7 +64,8 @@ create index `rcpt_idx2` on `rcpt`(`to`); drop view if exists `messages`; -create view `messages` AS select `metadata`.`id` AS `id`,`metadata`.`piler_id` AS `piler_id`,`metadata`.`from` AS `from`,`rcpt`.`to` AS `to`,`metadata`.`subject` AS `subject`, `metadata`.`size` AS `size`, `metadata`.`direction` AS `direction`, `metadata`.`arrived` AS `arrived` from (`metadata` join `rcpt`) where (`metadata`.`id` = `rcpt`.`id`); +create view `messages` AS select `metadata`.`id` AS `id`,`metadata`.`piler_id` AS `piler_id`,`metadata`.`from` AS `from`,`rcpt`.`to` AS `to`,`metadata`.`subject` AS `subject`, `metadata`.`size` AS `size`, `metadata`.`direction` AS `direction`, `metadata`.`arrived` AS `arrived`, `metadata`.`digest` AS `digest`, `metadata`.`bodydigest` AS `bodydigest` from (`metadata` join `rcpt`) where (`metadata`.`id` = `rcpt`.`id`); + drop table if exists `attachment`; create table if not exists `attachment` (