mirror of
https://bitbucket.org/jsuto/piler.git
synced 2024-11-07 23:21:59 +01:00
improved import test
This commit is contained in:
SJ
parent
97ca851780
commit
aeb10573f4
@ -21,10 +21,11 @@ LIBDIR = -L. @LIBDIR@ @LDFLAGS@ -L../../src
|
||||
LIBS = @LIBS@ @mysql_libs@
|
||||
RUNNING_USER = @RUNNING_USER@
|
||||
RUNNING_GROUP = `@id_bin@ -gn $(RUNNING_USER)`
|
||||
OBJS = import_helper.o
|
||||
|
||||
INSTALL = @INSTALL@
|
||||
|
||||
all: parser debug
|
||||
all: $(OBJS) parser debug import
|
||||
|
||||
parser: parser.c ../src/libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $^ -lpiler $(LIBS) $(LIBDIR)
|
||||
@ -32,13 +33,18 @@ parser: parser.c ../src/libpiler.a
|
||||
debug: debug.c ../src/libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $^ -lpiler $(LIBS) $(LIBDIR)
|
||||
|
||||
import: import.c ../src/libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $^ -lpiler import_helper.o $(LIBS) $(LIBDIR)
|
||||
|
||||
|
||||
%.o: $(srcdir)/%.c
|
||||
$(CC) $(CFLAGS) -fPIC $(INCDIR) $(DEFS) -c $< -o $@
|
||||
|
||||
|
||||
install:
|
||||
|
||||
test:
|
||||
./parser
|
||||
|
||||
clean:
|
||||
rm -f parser debug
|
||||
rm -f parser debug import
|
||||
|
||||
distclean: clean
|
||||
rm -f Makefile
|
||||
|
||||
248
test/import.c
Normal file
248
test/import.c
Normal file
@ -0,0 +1,248 @@
|
||||
/*
|
||||
* pilerimport.c, SJ
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/socket.h>
|
||||
#include <fcntl.h>
|
||||
#include <dirent.h>
|
||||
#include <unistd.h>
|
||||
#include <time.h>
|
||||
#include <locale.h>
|
||||
#include <syslog.h>
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
extern char *optarg;
|
||||
extern int optind;
|
||||
|
||||
|
||||
int connect_to_imap_server(int sd, int *seq, char *imapserver, char *username, char *password);
|
||||
int list_folders(int sd, int *seq, char *folders, int foldersize);
|
||||
int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sdata, struct __data *data, struct __config *cfg);
|
||||
int import_message2(char *filename, struct session_data *sdata, struct __data *data, struct __config *cfg);
|
||||
|
||||
|
||||
int import_from_mailbox(char *mailbox, struct session_data *sdata, struct __data *data, struct __config *cfg){
|
||||
FILE *F, *f=NULL;
|
||||
int rc=ERR, tot_msgs=0, ret=OK;
|
||||
char buf[MAXBUFSIZE], fname[SMALLBUFSIZE];
|
||||
time_t t;
|
||||
|
||||
|
||||
F = fopen(mailbox, "r");
|
||||
if(!F){
|
||||
printf("cannot open mailbox: %s\n", mailbox);
|
||||
return rc;
|
||||
}
|
||||
|
||||
t = time(NULL);
|
||||
|
||||
while(fgets(buf, sizeof(buf)-1, F)){
|
||||
|
||||
if(buf[0] == 'F' && buf[1] == 'r' && buf[2] == 'o' && buf[3] == 'm' && buf[4] == ' '){
|
||||
tot_msgs++;
|
||||
if(f){
|
||||
fclose(f);
|
||||
rc = import_message2(fname, sdata, data, cfg);
|
||||
printf("processed: %7d\r", tot_msgs); fflush(stdout);
|
||||
|
||||
if(rc == ERR) ret = ERR;
|
||||
unlink(fname);
|
||||
}
|
||||
|
||||
snprintf(fname, sizeof(fname)-1, "%ld-%d", t, tot_msgs);
|
||||
f = fopen(fname, "w+");
|
||||
continue;
|
||||
}
|
||||
|
||||
if(f) fprintf(f, "%s", buf);
|
||||
}
|
||||
|
||||
if(f){
|
||||
fclose(f);
|
||||
rc = import_message2(fname, sdata, data, cfg);
|
||||
printf("processed: %7d\r", tot_msgs); fflush(stdout);
|
||||
|
||||
if(rc == ERR) ret = ERR;
|
||||
unlink(fname);
|
||||
}
|
||||
|
||||
fclose(F);
|
||||
|
||||
printf("\n");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
int import_from_maildir(char *directory, struct session_data *sdata, struct __data *data, struct __config *cfg){
|
||||
DIR *dir;
|
||||
struct dirent *de;
|
||||
int rc=ERR, ret=OK, tot_msgs=0;
|
||||
char fname[SMALLBUFSIZE];
|
||||
struct stat st;
|
||||
|
||||
dir = opendir(directory);
|
||||
if(!dir){
|
||||
printf("cannot open directory: %s\n", directory);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
|
||||
while((de = readdir(dir))){
|
||||
if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
|
||||
|
||||
snprintf(fname, sizeof(fname)-1, "%s/%s", directory, de->d_name);
|
||||
|
||||
if(stat(fname, &st) == 0){
|
||||
if(S_ISDIR(st.st_mode)){
|
||||
rc = import_from_maildir(fname, sdata, data, cfg);
|
||||
if(rc == ERR) ret = ERR;
|
||||
}
|
||||
else {
|
||||
|
||||
if(S_ISREG(st.st_mode)){
|
||||
rc = import_message2(fname, sdata, data, cfg);
|
||||
if(rc == OK) tot_msgs++;
|
||||
else ret = ERR;
|
||||
}
|
||||
else {
|
||||
printf("%s is not a file\n", fname);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
else {
|
||||
printf("cannot stat() %s\n", fname);
|
||||
}
|
||||
|
||||
}
|
||||
closedir(dir);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
int import_from_imap_server(char *imapserver, char *username, char *password, struct session_data *sdata, struct __data *data, struct __config *cfg){
|
||||
int rc=ERR, ret=OK, sd, seq=1;
|
||||
char *p, puf[MAXBUFSIZE];
|
||||
char folders[MAXBUFSIZE];
|
||||
|
||||
if((sd = socket(AF_INET, SOCK_STREAM, 0)) == -1){
|
||||
printf("cannot create socket\n");
|
||||
return ERR;
|
||||
}
|
||||
|
||||
if(connect_to_imap_server(sd, &seq, imapserver, username, password) == ERR){
|
||||
close(sd);
|
||||
return ERR;
|
||||
}
|
||||
|
||||
|
||||
list_folders(sd, &seq, &folders[0], sizeof(folders));
|
||||
|
||||
|
||||
p = &folders[0];
|
||||
do {
|
||||
memset(puf, 0, sizeof(puf));
|
||||
p = split(p, '\n', puf, sizeof(puf)-1);
|
||||
|
||||
printf("processing folder: %s... ", puf);
|
||||
|
||||
rc = process_imap_folder(sd, &seq, puf, sdata, data, cfg);
|
||||
if(rc == ERR) ret = ERR;
|
||||
|
||||
} while(p);
|
||||
|
||||
|
||||
close(sd);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
void usage(){
|
||||
printf("usage: pilerimport [-c <config file>] -e <eml file> | -m <mailbox file> | -d <directory> | -i <imap server> -u <imap username> -p <imap password>\n");
|
||||
exit(0);
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char **argv){
|
||||
int i, rc=0;
|
||||
char *configfile=CONFIG_FILE, *mailbox=NULL, *emlfile=NULL, *directory=NULL;
|
||||
char *imapserver=NULL, *username=NULL, *password=NULL;
|
||||
struct session_data sdata;
|
||||
struct __config cfg;
|
||||
struct __data data;
|
||||
|
||||
|
||||
while((i = getopt(argc, argv, "c:m:e:d:i:u:p:h?")) > 0){
|
||||
switch(i){
|
||||
|
||||
case 'c' :
|
||||
configfile = optarg;
|
||||
break;
|
||||
|
||||
case 'e' :
|
||||
emlfile = optarg;
|
||||
break;
|
||||
|
||||
case 'd' :
|
||||
directory = optarg;
|
||||
break;
|
||||
|
||||
case 'm' :
|
||||
mailbox = optarg;
|
||||
break;
|
||||
|
||||
case 'i' :
|
||||
imapserver = optarg;
|
||||
break;
|
||||
|
||||
case 'u' :
|
||||
username = optarg;
|
||||
break;
|
||||
|
||||
case 'p' :
|
||||
password = optarg;
|
||||
break;
|
||||
|
||||
case 'h' :
|
||||
case '?' :
|
||||
usage();
|
||||
break;
|
||||
|
||||
|
||||
default :
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
if(!mailbox && !emlfile && !directory && !imapserver) usage();
|
||||
|
||||
|
||||
cfg = read_config(configfile);
|
||||
|
||||
setlocale(LC_CTYPE, cfg.locale);
|
||||
|
||||
data.archiving_rules = NULL;
|
||||
data.retention_rules = NULL;
|
||||
|
||||
if(emlfile) rc = import_message2(emlfile, &sdata, &data, &cfg);
|
||||
if(mailbox) rc = import_from_mailbox(mailbox, &sdata, &data, &cfg);
|
||||
if(directory) rc = import_from_maildir(directory, &sdata, &data, &cfg);
|
||||
if(imapserver && username && password) rc = import_from_imap_server(imapserver, username, password, &sdata, &data, &cfg);
|
||||
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
97
test/import_helper.c
Normal file
97
test/import_helper.c
Normal file
@ -0,0 +1,97 @@
|
||||
/*
|
||||
* import.c, SJ
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <fcntl.h>
|
||||
#include <dirent.h>
|
||||
#include <unistd.h>
|
||||
#include <time.h>
|
||||
#include <locale.h>
|
||||
#include <syslog.h>
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
int import_message2(char *filename, struct session_data *sdata, struct __data *data, struct __config *cfg){
|
||||
int rc=ERR, i, fd;
|
||||
char *rule;
|
||||
struct stat st;
|
||||
struct _state state;
|
||||
|
||||
|
||||
init_session_data(sdata);
|
||||
|
||||
|
||||
if(strcmp(filename, "-") == 0){
|
||||
|
||||
if(read_from_stdin(sdata) == ERR){
|
||||
printf("error reading from stdin\n");
|
||||
return rc;
|
||||
}
|
||||
|
||||
snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", sdata->ttmpfile);
|
||||
|
||||
}
|
||||
else {
|
||||
|
||||
if(stat(filename, &st) != 0){
|
||||
printf("cannot stat() %s\n", filename);
|
||||
return rc;
|
||||
}
|
||||
|
||||
if(S_ISREG(st.st_mode) == 0){
|
||||
printf("%s is not a file\n", filename);
|
||||
return rc;
|
||||
}
|
||||
|
||||
fd = open(filename, O_RDONLY);
|
||||
if(fd == -1){
|
||||
printf("cannot open %s\n", filename);
|
||||
return rc;
|
||||
}
|
||||
close(fd);
|
||||
|
||||
snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
|
||||
|
||||
sdata->tot_len = st.st_size;
|
||||
}
|
||||
|
||||
|
||||
|
||||
sdata->sent = 0;
|
||||
|
||||
state = parse_message(sdata, 1, cfg);
|
||||
post_parse(sdata, &state, cfg);
|
||||
|
||||
if(sdata->sent > sdata->now) sdata->sent = sdata->now;
|
||||
if(sdata->sent == -1) sdata->sent = 0;
|
||||
|
||||
/* fat chances that you won't import emails before 1990.01.01 */
|
||||
|
||||
if(sdata->sent > 631148400) sdata->retained = sdata->sent;
|
||||
|
||||
rule = check_againt_ruleset(data->archiving_rules, &state, sdata->tot_len, sdata->spam_message);
|
||||
|
||||
if(rule){
|
||||
printf("discarding %s by archiving policy: %s\n", filename, rule);
|
||||
rc = OK;
|
||||
goto ENDE;
|
||||
}
|
||||
|
||||
make_digests(sdata, cfg);
|
||||
|
||||
ENDE:
|
||||
unlink(sdata->tmpframe);
|
||||
|
||||
if(strcmp(filename, "-") == 0) unlink(sdata->ttmpfile);
|
||||
|
||||
for(i=1; i<=state.n_attachments; i++) unlink(state.attachments[i].internalname);
|
||||
|
||||
return OK;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user