mirror of
https://bitbucket.org/jsuto/piler.git
synced 2024-12-24 19:10:13 +01:00
added pilerexport for bulk export to EML, and simplified pilerget
This commit is contained in:
SJ
parent
94660cf32f
commit
bafbbc7fa8
@ -33,7 +33,7 @@ MAKE = `which make`
|
||||
|
||||
INSTALL = @INSTALL@
|
||||
|
||||
all: libpiler.a piler pilerconf pilerget pilerimport test
|
||||
all: libpiler.a piler pilerconf pilerget pilerimport pilerexport test
|
||||
install: install-piler
|
||||
|
||||
|
||||
@ -54,6 +54,9 @@ pilerget: pilerget.c libpiler.a
|
||||
pilerimport: pilerimport.c libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $^ -lpiler $(LIBS) $(LIBDIR)
|
||||
|
||||
pilerexport: pilerexport.c libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $^ -lpiler $(LIBS) $(LIBDIR)
|
||||
|
||||
pilerconf: pilerconf.c cfg.o misc.o tai.o
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $^ $(LIBDIR)
|
||||
|
||||
|
||||
@ -78,6 +78,7 @@
|
||||
#define SQL_RECIPIENT_TABLE "rcpt"
|
||||
#define SQL_ARCHIVING_RULE_TABLE "archiving_rule"
|
||||
#define SQL_COUNTER_TABLE "counter"
|
||||
#define SQL_MESSAGES_VIEW "messages"
|
||||
|
||||
|
||||
/* TRE stuff */
|
||||
|
||||
309
src/pilerexport.c
Normal file
309
src/pilerexport.c
Normal file
@ -0,0 +1,309 @@
|
||||
/*
|
||||
* pilerexport.c, SJ
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
#include <time.h>
|
||||
#include <locale.h>
|
||||
#include <syslog.h>
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
extern char *optarg;
|
||||
extern int optind;
|
||||
|
||||
char *query=NULL;
|
||||
regex_t regexp;
|
||||
|
||||
|
||||
|
||||
void usage(){
|
||||
printf("usage: .... \n");
|
||||
exit(0);
|
||||
}
|
||||
|
||||
|
||||
void clean_exit(char *msg, int rc){
|
||||
if(msg) printf("error: %s\n", msg);
|
||||
|
||||
if(query) free(query);
|
||||
|
||||
exit(rc);
|
||||
}
|
||||
|
||||
|
||||
int append_email_to_buffer(char **buffer, char *email){
|
||||
int len, arglen;
|
||||
char *s=NULL, emailaddress[SMALLBUFSIZE];
|
||||
|
||||
snprintf(emailaddress, sizeof(emailaddress)-1, "'%s'", email);
|
||||
arglen = strlen(emailaddress);
|
||||
|
||||
if(!*buffer){
|
||||
*buffer = malloc(arglen+1);
|
||||
memset(*buffer, 0, arglen+1);
|
||||
memcpy(*buffer, emailaddress, arglen);
|
||||
}
|
||||
else {
|
||||
len = strlen(*buffer);
|
||||
s = realloc(*buffer, len + arglen+2);
|
||||
if(!s){
|
||||
printf("malloc problem!\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
*buffer = s;
|
||||
|
||||
memset(*buffer+len, 0, arglen+2);
|
||||
strcat(*buffer, ",");
|
||||
memcpy(*buffer+len+1, emailaddress, arglen);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int append_string_to_buffer(char **buffer, char *str){
|
||||
int len, arglen;
|
||||
char *s=NULL;
|
||||
|
||||
arglen = strlen(str);
|
||||
|
||||
if(!*buffer){
|
||||
*buffer = malloc(arglen+1);
|
||||
memset(*buffer, 0, arglen+1);
|
||||
memcpy(*buffer, str, arglen);
|
||||
}
|
||||
else {
|
||||
len = strlen(*buffer);
|
||||
s = realloc(*buffer, len + arglen+1);
|
||||
if(!s) return 1;
|
||||
|
||||
*buffer = s;
|
||||
|
||||
memset(*buffer+len, 0, arglen+1);
|
||||
memcpy(*buffer+len, str, arglen);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
int export_emails_matching_to_query(struct session_data *sdata, char *s, struct __config *cfg){
|
||||
MYSQL_RES *res;
|
||||
MYSQL_ROW row;
|
||||
FILE *f;
|
||||
uint64 id;
|
||||
char *digest=NULL, *bodydigest=NULL;
|
||||
char filename[SMALLBUFSIZE];
|
||||
int rc=0;
|
||||
|
||||
|
||||
rc = mysql_real_query(&(sdata->mysql), s, strlen(s));
|
||||
|
||||
if(rc == 0){
|
||||
res = mysql_store_result(&(sdata->mysql));
|
||||
if(res){
|
||||
while((row = mysql_fetch_row(res))){
|
||||
|
||||
id = strtoull(row[0], NULL, 10);
|
||||
if(id > 0){
|
||||
snprintf(sdata->ttmpfile, SMALLBUFSIZE-1, "%s", (char*)row[1]);
|
||||
digest = (char*)row[2];
|
||||
bodydigest = (char*)row[3];
|
||||
|
||||
|
||||
snprintf(filename, sizeof(filename)-1, "%llu.eml", id);
|
||||
|
||||
f = fopen(filename, "w");
|
||||
if(f){
|
||||
rc = retrieve_email_from_archive(sdata, f, cfg);
|
||||
fclose(f);
|
||||
|
||||
snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
|
||||
|
||||
make_digests(sdata, cfg);
|
||||
|
||||
if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0)
|
||||
printf("exported %s, verification: OK\n", filename);
|
||||
else
|
||||
printf("exported %s, verification: FAILED\n", filename);
|
||||
|
||||
}
|
||||
else printf("cannot open: %s\n", filename);
|
||||
|
||||
}
|
||||
}
|
||||
mysql_free_result(res);
|
||||
}
|
||||
else rc = 1;
|
||||
}
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char **argv){
|
||||
int i, rc, exportall=0, minsize=0, maxsize=0;
|
||||
int where_condition=0;
|
||||
size_t nmatch=0;
|
||||
//unsigned long startdate=0, stopdate=0;
|
||||
char *configfile=CONFIG_FILE;
|
||||
char *to=NULL, *from=NULL;
|
||||
char s[SMALLBUFSIZE];
|
||||
struct session_data sdata;
|
||||
struct __config cfg;
|
||||
|
||||
|
||||
if(regcomp(®exp, "^([\\+a-z0-9_\\.@\\-]+)$", REG_ICASE | REG_EXTENDED)){
|
||||
clean_exit("cannot compile rule!", 1);
|
||||
}
|
||||
|
||||
|
||||
while((i = getopt(argc, argv, "c:s:S:f:r:ah?")) > 0){
|
||||
|
||||
switch(i){
|
||||
|
||||
case 'c' :
|
||||
configfile = optarg;
|
||||
break;
|
||||
|
||||
case 's' :
|
||||
maxsize = atoi(optarg);
|
||||
break;
|
||||
|
||||
case 'S' :
|
||||
minsize = atoi(optarg);
|
||||
break;
|
||||
|
||||
|
||||
case 'a' :
|
||||
exportall = 1;
|
||||
break;
|
||||
|
||||
case 'f' :
|
||||
|
||||
if(regexec(®exp, optarg, nmatch, NULL, 0)){
|
||||
printf("%s is not a valid email address\n", optarg);
|
||||
break;
|
||||
}
|
||||
|
||||
rc = append_email_to_buffer(&from, optarg);
|
||||
|
||||
break;
|
||||
|
||||
case 'r' :
|
||||
|
||||
if(regexec(®exp, optarg, nmatch, NULL, 0)){
|
||||
printf("%s is not a valid email address\n", optarg);
|
||||
break;
|
||||
}
|
||||
|
||||
rc = append_email_to_buffer(&to, optarg);
|
||||
|
||||
break;
|
||||
|
||||
|
||||
|
||||
case 'h' :
|
||||
case '?' :
|
||||
usage();
|
||||
break;
|
||||
|
||||
|
||||
default :
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
regfree(®exp);
|
||||
|
||||
|
||||
snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE ", SQL_MESSAGES_VIEW);
|
||||
|
||||
rc = append_string_to_buffer(&query, s);
|
||||
|
||||
if(from){
|
||||
rc += append_string_to_buffer(&query, "`from` IN (");
|
||||
rc += append_string_to_buffer(&query, from);
|
||||
rc += append_string_to_buffer(&query, ")");
|
||||
|
||||
free(from);
|
||||
|
||||
where_condition++;
|
||||
}
|
||||
|
||||
|
||||
if(to){
|
||||
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
|
||||
|
||||
rc += append_string_to_buffer(&query, "`to` IN (");
|
||||
rc += append_string_to_buffer(&query, to);
|
||||
rc += append_string_to_buffer(&query, ")");
|
||||
|
||||
free(to);
|
||||
|
||||
where_condition++;
|
||||
}
|
||||
|
||||
|
||||
if(minsize > 0){
|
||||
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
|
||||
snprintf(s, sizeof(s)-1, " `size` >= %d", minsize);
|
||||
rc += append_string_to_buffer(&query, s);
|
||||
}
|
||||
|
||||
|
||||
if(maxsize > 0){
|
||||
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
|
||||
snprintf(s, sizeof(s)-1, " `size` <= %d", maxsize);
|
||||
rc += append_string_to_buffer(&query, s);
|
||||
}
|
||||
|
||||
rc += append_string_to_buffer(&query, " ORDER BY id ASC");
|
||||
|
||||
if(rc) clean_exit("malloc problem building query", 1);
|
||||
|
||||
printf("query: *%s*\n", query);
|
||||
|
||||
|
||||
|
||||
|
||||
cfg = read_config(configfile);
|
||||
|
||||
|
||||
if(read_key(&cfg)) clean_exit(ERR_READING_KEY, 1);
|
||||
|
||||
|
||||
init_session_data(&sdata);
|
||||
|
||||
|
||||
mysql_init(&(sdata.mysql));
|
||||
mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout);
|
||||
if(mysql_real_connect(&(sdata.mysql), cfg.mysqlhost, cfg.mysqluser, cfg.mysqlpwd, cfg.mysqldb, cfg.mysqlport, cfg.mysqlsocket, 0) == 0){
|
||||
clean_exit("cannot connect to mysql server", 1);
|
||||
}
|
||||
|
||||
mysql_real_query(&(sdata.mysql), "SET NAMES utf8", strlen("SET NAMES utf8"));
|
||||
mysql_real_query(&(sdata.mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8"));
|
||||
|
||||
|
||||
rc = export_emails_matching_to_query(&sdata, query, &cfg);
|
||||
|
||||
free(query);
|
||||
|
||||
|
||||
mysql_close(&(sdata.mysql));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@ -97,9 +97,6 @@ ENDE:
|
||||
|
||||
int main(int argc, char **argv){
|
||||
int rc;
|
||||
uint64 id;
|
||||
char filename[SMALLBUFSIZE], digest[2*DIGEST_LENGTH+1], bodydigest[2*DIGEST_LENGTH+1];
|
||||
FILE *f;
|
||||
struct session_data sdata;
|
||||
struct __config cfg;
|
||||
|
||||
@ -130,47 +127,9 @@ int main(int argc, char **argv){
|
||||
mysql_real_query(&(sdata.mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8"));
|
||||
|
||||
|
||||
if(argv[1][0] == '-'){
|
||||
|
||||
memset(sdata.ttmpfile, 0, sizeof(sdata.ttmpfile));
|
||||
|
||||
while((rc = read(0, sdata.ttmpfile, RND_STR_LEN+1)) > 0){
|
||||
snprintf(sdata.filename, SMALLBUFSIZE-1, "%s", sdata.ttmpfile);
|
||||
|
||||
trimBuffer(sdata.ttmpfile);
|
||||
|
||||
id = get_id_by_piler_id(&sdata, &digest[0], &bodydigest[0], &cfg);
|
||||
|
||||
if(id > 0){
|
||||
snprintf(filename, sizeof(filename)-1, "%llu.eml", id);
|
||||
f = fopen(filename, "w");
|
||||
if(f){
|
||||
rc = retrieve_email_from_archive(&sdata, f, &cfg);
|
||||
fclose(f);
|
||||
|
||||
snprintf(sdata.ttmpfile, SMALLBUFSIZE-1, "%s", filename);
|
||||
make_digests(&sdata, &cfg);
|
||||
|
||||
if(strcmp(digest, sdata.digest) == 0 && strcmp(bodydigest, sdata.bodydigest) == 0)
|
||||
printf("exported %s, verification: OK\n", sdata.ttmpfile);
|
||||
else
|
||||
printf("exported %s, verification: FAILED\n", sdata.ttmpfile);
|
||||
|
||||
}
|
||||
else printf("cannot open: %s\n", filename);
|
||||
}
|
||||
else printf("%s was not found in archive\n", sdata.ttmpfile);
|
||||
|
||||
memset(sdata.ttmpfile, 0, sizeof(sdata.ttmpfile));
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
else {
|
||||
snprintf(sdata.ttmpfile, SMALLBUFSIZE-1, "%s", argv[1]);
|
||||
snprintf(sdata.filename, SMALLBUFSIZE-1, "%s", sdata.ttmpfile);
|
||||
rc = retrieve_email_from_archive(&sdata, stdout, &cfg);
|
||||
}
|
||||
snprintf(sdata.ttmpfile, SMALLBUFSIZE-1, "%s", argv[1]);
|
||||
snprintf(sdata.filename, SMALLBUFSIZE-1, "%s", sdata.ttmpfile);
|
||||
rc = retrieve_email_from_archive(&sdata, stdout, &cfg);
|
||||
|
||||
|
||||
mysql_close(&(sdata.mysql));
|
||||
|
||||
@ -64,7 +64,8 @@ create index `rcpt_idx2` on `rcpt`(`to`);
|
||||
|
||||
|
||||
drop view if exists `messages`;
|
||||
create view `messages` AS select `metadata`.`id` AS `id`,`metadata`.`piler_id` AS `piler_id`,`metadata`.`from` AS `from`,`rcpt`.`to` AS `to`,`metadata`.`subject` AS `subject`, `metadata`.`size` AS `size`, `metadata`.`direction` AS `direction`, `metadata`.`arrived` AS `arrived` from (`metadata` join `rcpt`) where (`metadata`.`id` = `rcpt`.`id`);
|
||||
create view `messages` AS select `metadata`.`id` AS `id`,`metadata`.`piler_id` AS `piler_id`,`metadata`.`from` AS `from`,`rcpt`.`to` AS `to`,`metadata`.`subject` AS `subject`, `metadata`.`size` AS `size`, `metadata`.`direction` AS `direction`, `metadata`.`arrived` AS `arrived`, `metadata`.`digest` AS `digest`, `metadata`.`bodydigest` AS `bodydigest` from (`metadata` join `rcpt`) where (`metadata`.`id` = `rcpt`.`id`);
|
||||
|
||||
|
||||
drop table if exists `attachment`;
|
||||
create table if not exists `attachment` (
|
||||
|
||||
Loading…
Reference in New Issue
Block a user