From 0dfccc1473bdec2e4f7ffda53fd86f628ca50b0b Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 28 May 2023 16:17:31 +0200 Subject: [PATCH] Fixed #1293 pilerimport syslogs stats Signed-off-by: Janos SUTO --- src/imap.c | 4 ++-- src/import.c | 22 ++++++++++++---------- src/import.h | 18 +++++++++--------- src/import_imap.c | 4 ++-- src/import_mailbox.c | 12 ++++++------ src/import_maildir.c | 6 +++--- src/import_pilerexport.c | 14 +++++++------- src/import_pop3.c | 4 ++-- src/pilerimport.c | 21 ++++++++++++++------- src/pop3.c | 4 ++-- 10 files changed, 59 insertions(+), 50 deletions(-) diff --git a/src/imap.c b/src/imap.c index ebd82dba..9f225180 100644 --- a/src/imap.c +++ b/src/imap.c @@ -313,7 +313,7 @@ void imap_expunge_message(struct data *data){ } -int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct config *cfg){ +int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){ int i, messages=0; messages = imap_select_cmd_on_folder(folder, data); @@ -330,7 +330,7 @@ int process_imap_folder(char *folder, struct session_data *sdata, struct data *d if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/messages); fflush(stdout); } if(data->import->dryrun == 0){ - int rc = import_message(sdata, data, cfg); + int rc = import_message(sdata, data, counters, cfg); if(data->import->remove_after_import == 1 && rc == OK){ imap_delete_message(data, i); diff --git a/src/import.c b/src/import.c index 6e3d3319..ceffa8d9 100644 --- a/src/import.c +++ b/src/import.c @@ -18,12 +18,11 @@ #include -int import_message(struct session_data *sdata, struct data *data, struct config *cfg){ +int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){ int rc=ERR; char *rule; struct stat st; struct parser_state state; - struct counters counters; if(data->import->delay > 0){ struct timespec req; @@ -122,24 +121,27 @@ int import_message(struct session_data *sdata, struct data *data, struct config switch(rc) { case OK: - bzero(&counters, sizeof(counters)); - counters.c_rcvd = 1; - counters.c_size += sdata->tot_len; - counters.c_stored_size = sdata->stored_len; - update_counters(sdata, data, &counters, cfg); + counters->c_rcvd++; + counters->c_size += sdata->tot_len; + counters->c_stored_size += sdata->stored_len; break; case ERR_EXISTS: rc = OK; - bzero(&counters, sizeof(counters)); - counters.c_duplicate = 1; - update_counters(sdata, data, &counters, cfg); + counters->c_duplicate++; if(data->quiet == 0) printf("duplicate: %s (duplicate id: %llu)\n", data->import->filename, sdata->duplicate_id); break; + case ERR_DISCARDED: + rc = OK; + + counters->c_ignore++; + + break; + default: printf("failed to import: %s (id: %s)\n", data->import->filename, sdata->ttmpfile); break; diff --git a/src/import.h b/src/import.h index 6f0f112d..dbdc46d0 100644 --- a/src/import.h +++ b/src/import.h @@ -6,23 +6,23 @@ #define _IMPORT_H -int import_message(struct session_data *sdata, struct data *data, struct config *cfg); +int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg); int update_import_table(struct session_data *sdata, struct data *data); -int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct config *cfg); -int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct config *cfg); -int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct config *cfg); -void import_from_pop3_server(struct session_data *sdata, struct data *data, struct config *cfg); -int import_from_imap_server(struct session_data *sdata, struct data *data, struct config *cfg); +int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg); +int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg); +int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg); +void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg); +int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg); int connect_to_pop3_server(struct data *data); -void process_pop3_emails(struct session_data *sdata, struct data *data, struct config *cfg); +void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg); int connect_to_imap_server(struct data *data); int list_folders(struct data *data); -int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct config *cfg); +int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg); void send_imap_close(struct data *data); -void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg); +void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg); #endif /* _IMPORT_H */ diff --git a/src/import_imap.c b/src/import_imap.c index 7804767e..1ab71333 100644 --- a/src/import_imap.c +++ b/src/import_imap.c @@ -22,7 +22,7 @@ #include -int import_from_imap_server(struct session_data *sdata, struct data *data, struct config *cfg){ +int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){ int i, rc=ERR, ret=OK, skipmatch; char port_string[8], puf[SMALLBUFSIZE]; struct addrinfo hints, *res; @@ -88,7 +88,7 @@ int import_from_imap_server(struct session_data *sdata, struct data *data, struc else { if(data->quiet == 0) printf("processing folder: %s... ", (char *)q->str); - if(process_imap_folder(q->str, sdata, data, cfg) == ERR) ret = ERR; + if(process_imap_folder(q->str, sdata, data, counters, cfg) == ERR) ret = ERR; } } diff --git a/src/import_mailbox.c b/src/import_mailbox.c index 376408fa..f0d74013 100644 --- a/src/import_mailbox.c +++ b/src/import_mailbox.c @@ -22,7 +22,7 @@ #include -int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct config *cfg){ +int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){ FILE *F, *f=NULL; int rc=ERR, tot_msgs=0, ret=OK; char buf[MAXBUFSIZE]; @@ -44,7 +44,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data * if(f){ fclose(f); f = NULL; - rc = import_message(sdata, data, cfg); + rc = import_message(sdata, data, counters, cfg); if(rc == ERR){ printf("error importing: '%s'\n", data->import->filename); ret = ERR; @@ -64,7 +64,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data * if(f){ fclose(f); - rc = import_message(sdata, data, cfg); + rc = import_message(sdata, data, counters, cfg); if(rc == ERR){ printf("ERROR: error importing: '%s'\n", data->import->filename); ret = ERR; @@ -80,7 +80,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data * } -int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct config *cfg){ +int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){ DIR *dir; struct dirent *de; int rc=ERR, ret=OK, i=0; @@ -103,7 +103,7 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct dat if(stat(fname, &st) == 0){ if(S_ISDIR(st.st_mode)){ folder = data->folder; - rc = import_mbox_from_dir(fname, sdata, data, cfg); + rc = import_mbox_from_dir(fname, sdata, data, counters, cfg); data->folder = folder; if(rc == ERR) ret = ERR; } @@ -126,7 +126,7 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct dat } - rc = import_from_mailbox(fname, sdata, data, cfg); + rc = import_from_mailbox(fname, sdata, data, counters, cfg); if(rc == OK) (data->import->tot_msgs)++; else ret = ERR; diff --git a/src/import_maildir.c b/src/import_maildir.c index ced1cafa..f4bd1e1c 100644 --- a/src/import_maildir.c +++ b/src/import_maildir.c @@ -22,7 +22,7 @@ #include -int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct config *cfg){ +int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg){ DIR *dir; struct dirent *de; int rc=ERR, ret=OK, i=0; @@ -46,7 +46,7 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir if(S_ISDIR(st.st_mode)){ folder = data->folder; snprintf(subdir, sizeof(subdir)-1, "%s", data->import->filename); - rc = import_from_maildir(sdata, data, subdir, cfg); + rc = import_from_maildir(sdata, data, subdir, counters, cfg); data->folder = folder; if(rc == ERR) ret = ERR; } @@ -76,7 +76,7 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir } - rc = import_message(sdata, data, cfg); + rc = import_message(sdata, data, counters, cfg); if(rc == OK) (data->import->tot_msgs)++; else if(rc == ERR){ diff --git a/src/import_pilerexport.c b/src/import_pilerexport.c index eb40692e..9f5add05 100644 --- a/src/import_pilerexport.c +++ b/src/import_pilerexport.c @@ -22,21 +22,21 @@ #include -void import_the_file(struct session_data *sdata, struct data *data, struct config *cfg){ +void import_the_file(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){ close(data->import->fd); data->import->fd = -1; - if(import_message(sdata, data, cfg) != ERR){ + if(import_message(sdata, data, counters, cfg) != ERR){ unlink(data->import->filename); } } -void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct config *cfg){ +void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){ if(!strcmp(buf, PILEREXPORT_BEGIN_MARK)){ if((*count) > 0){ - import_the_file(sdata, data, cfg); + import_the_file(sdata, data, counters, cfg); } (*count)++; @@ -57,7 +57,7 @@ void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *s } -void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg){ +void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){ int n, rc, nullbyte, savedlen=0, puflen; uint64 count=0; char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE]; @@ -96,7 +96,7 @@ void import_from_pilerexport(struct session_data *sdata, struct data *data, stru if(puflen > 0){ if(rc == OK){ - process_buffer(puf, puflen, &count, sdata, data, cfg); + process_buffer(puf, puflen, &count, sdata, data, counters, cfg); } else { snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf); @@ -109,6 +109,6 @@ void import_from_pilerexport(struct session_data *sdata, struct data *data, stru } while(n > 0); if(data->import->fd != -1){ - import_the_file(sdata, data, cfg); + import_the_file(sdata, data, counters, cfg); } } diff --git a/src/import_pop3.c b/src/import_pop3.c index a324ba22..70f78d50 100644 --- a/src/import_pop3.c +++ b/src/import_pop3.c @@ -22,7 +22,7 @@ #include -void import_from_pop3_server(struct session_data *sdata, struct data *data, struct config *cfg){ +void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){ int rc; char port_string[8]; struct addrinfo hints, *res; @@ -58,7 +58,7 @@ void import_from_pop3_server(struct session_data *sdata, struct data *data, stru goto ENDE_POP3; } - process_pop3_emails(sdata, data, cfg); + process_pop3_emails(sdata, data, counters, cfg); close_connection(data->net); diff --git a/src/pilerimport.c b/src/pilerimport.c index 711e89df..87cd3847 100644 --- a/src/pilerimport.c +++ b/src/pilerimport.c @@ -75,6 +75,9 @@ int main(int argc, char **argv){ struct data data; struct import import; struct net net; + struct counters counters; + + bzero(&counters, sizeof(counters)); for(i=0; ifilename[0] != '\0') import_message(&sdata, &data, &cfg); + if(data.import->filename[0] != '\0') import_message(&sdata, &data, &counters, &cfg); if(mbox[0]){ for(i=0; imboxdir) import_mbox_from_dir(data.import->mboxdir, &sdata, &data, &cfg); - if(directory) import_from_maildir(&sdata, &data, directory, &cfg); - if(imapserver) import_from_imap_server(&sdata, &data, &cfg); - if(pop3server) import_from_pop3_server(&sdata, &data, &cfg); - if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &cfg); + if(data.import->mboxdir) import_mbox_from_dir(data.import->mboxdir, &sdata, &data, &counters, &cfg); + if(directory) import_from_maildir(&sdata, &data, directory, &counters, &cfg); + if(imapserver) import_from_imap_server(&sdata, &data, &counters, &cfg); + if(pop3server) import_from_pop3_server(&sdata, &data, &counters, &cfg); + if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &counters, &cfg); clearrules(data.archiving_rules); clearrules(data.retention_rules); @@ -403,6 +406,10 @@ int main(int argc, char **argv){ clearhash(data.mydomains); + update_counters(&sdata, &data, &counters, &cfg); + + syslog(LOG_PRIORITY, "imported=%lld, duplicated=%lld, discarded=%lld", counters.c_rcvd, counters.c_duplicate, counters.c_ignore); + close_database(&sdata); if(cfg.rtindex) close_sphx(&sdata); diff --git a/src/pop3.c b/src/pop3.c index 94137fd3..d290c020 100644 --- a/src/pop3.c +++ b/src/pop3.c @@ -175,7 +175,7 @@ void pop3_delete_message(struct data *data, int i){ } -void process_pop3_emails(struct session_data *sdata, struct data *data, struct config *cfg){ +void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){ char buf[MAXBUFSIZE]; data->import->processed_messages = 0; @@ -191,7 +191,7 @@ void process_pop3_emails(struct session_data *sdata, struct data *data, struct c if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/data->import->total_messages); fflush(stdout); } if(data->import->dryrun == 0){ - int rc = import_message(sdata, data, cfg); + int rc = import_message(sdata, data, counters, cfg); if(data->import->remove_after_import == 1 && rc == OK){ pop3_delete_message(data, i);