bashclub/piler
1
0
Fork 0
mirror of https://bitbucket.org/jsuto/piler.git synced 2024-11-08 02:11:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fixed #1293 pilerimport syslogs stats

Browse Source 
Signed-off-by: Janos SUTO <sj@acts.hu>
This commit is contained in:
Janos SUTO 2023-05-28 16:17:31 +02:00
parent 5fbefb28a5
commit 0dfccc1473
10 changed files with 59 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/imap.c
View File

@ -313,7 +313,7 @@ void imap_expunge_message(struct data *data){
} }
int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct config *cfg){ int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
int i, messages=0; int i, messages=0;
messages = imap_select_cmd_on_folder(folder, data); messages = imap_select_cmd_on_folder(folder, data);
@ -330,7 +330,7 @@ int process_imap_folder(char *folder, struct session_data *sdata, struct data *d
if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/messages); fflush(stdout); } if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/messages); fflush(stdout); }
if(data->import->dryrun == 0){ if(data->import->dryrun == 0){
int rc = import_message(sdata, data, cfg); int rc = import_message(sdata, data, counters, cfg);
if(data->import->remove_after_import == 1 && rc == OK){ if(data->import->remove_after_import == 1 && rc == OK){
imap_delete_message(data, i); imap_delete_message(data, i);

22
src/import.c
View File

@ -18,12 +18,11 @@
#include <piler.h> #include <piler.h>
int import_message(struct session_data *sdata, struct data *data, struct config *cfg){ int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
int rc=ERR; int rc=ERR;
char *rule; char *rule;
struct stat st; struct stat st;
struct parser_state state; struct parser_state state;
struct counters counters;
if(data->import->delay > 0){ if(data->import->delay > 0){
struct timespec req; struct timespec req;
@ -122,24 +121,27 @@ int import_message(struct session_data *sdata, struct data *data, struct config
switch(rc) { switch(rc) {
case OK: case OK:
bzero(&counters, sizeof(counters)); counters->c_rcvd++;
counters.c_rcvd = 1; counters->c_size += sdata->tot_len;
counters.c_size += sdata->tot_len; counters->c_stored_size += sdata->stored_len;
counters.c_stored_size = sdata->stored_len;
update_counters(sdata, data, &counters, cfg);
break; break;
case ERR_EXISTS: case ERR_EXISTS:
rc = OK; rc = OK;
bzero(&counters, sizeof(counters)); counters->c_duplicate++;
counters.c_duplicate = 1;
update_counters(sdata, data, &counters, cfg);
if(data->quiet == 0) printf("duplicate: %s (duplicate id: %llu)\n", data->import->filename, sdata->duplicate_id); if(data->quiet == 0) printf("duplicate: %s (duplicate id: %llu)\n", data->import->filename, sdata->duplicate_id);
break; break;
case ERR_DISCARDED:
rc = OK;
counters->c_ignore++;
break;
default: default:
printf("failed to import: %s (id: %s)\n", data->import->filename, sdata->ttmpfile); printf("failed to import: %s (id: %s)\n", data->import->filename, sdata->ttmpfile);
break; break;

18
src/import.h
View File

@ -6,23 +6,23 @@
#define _IMPORT_H #define _IMPORT_H
int import_message(struct session_data *sdata, struct data *data, struct config *cfg); int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
int update_import_table(struct session_data *sdata, struct data *data); int update_import_table(struct session_data *sdata, struct data *data);
int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct config *cfg); int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg);
int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct config *cfg); int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct config *cfg); int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
void import_from_pop3_server(struct session_data *sdata, struct data *data, struct config *cfg); void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
int import_from_imap_server(struct session_data *sdata, struct data *data, struct config *cfg); int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
int connect_to_pop3_server(struct data *data); int connect_to_pop3_server(struct data *data);
void process_pop3_emails(struct session_data *sdata, struct data *data, struct config *cfg); void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
int connect_to_imap_server(struct data *data); int connect_to_imap_server(struct data *data);
int list_folders(struct data *data); int list_folders(struct data *data);
int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct config *cfg); int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
void send_imap_close(struct data *data); void send_imap_close(struct data *data);
void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg); void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
#endif /* _IMPORT_H */ #endif /* _IMPORT_H */

4
src/import_imap.c
View File

@ -22,7 +22,7 @@
#include <piler.h> #include <piler.h>
int import_from_imap_server(struct session_data *sdata, struct data *data, struct config *cfg){ int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
int i, rc=ERR, ret=OK, skipmatch; int i, rc=ERR, ret=OK, skipmatch;
char port_string[8], puf[SMALLBUFSIZE]; char port_string[8], puf[SMALLBUFSIZE];
struct addrinfo hints, *res; struct addrinfo hints, *res;
@ -88,7 +88,7 @@ int import_from_imap_server(struct session_data *sdata, struct data *data, struc
else { else {
if(data->quiet == 0) printf("processing folder: %s... ", (char *)q->str); if(data->quiet == 0) printf("processing folder: %s... ", (char *)q->str);
if(process_imap_folder(q->str, sdata, data, cfg) == ERR) ret = ERR; if(process_imap_folder(q->str, sdata, data, counters, cfg) == ERR) ret = ERR;
} }
} }

12
src/import_mailbox.c
View File

@ -22,7 +22,7 @@
#include <piler.h> #include <piler.h>
int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct config *cfg){ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
FILE *F, *f=NULL; FILE *F, *f=NULL;
int rc=ERR, tot_msgs=0, ret=OK; int rc=ERR, tot_msgs=0, ret=OK;
char buf[MAXBUFSIZE]; char buf[MAXBUFSIZE];
@ -44,7 +44,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *
if(f){ if(f){
fclose(f); fclose(f);
f = NULL; f = NULL;
rc = import_message(sdata, data, cfg); rc = import_message(sdata, data, counters, cfg);
if(rc == ERR){ if(rc == ERR){
printf("error importing: '%s'\n", data->import->filename); printf("error importing: '%s'\n", data->import->filename);
ret = ERR; ret = ERR;
@ -64,7 +64,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *
if(f){ if(f){
fclose(f); fclose(f);
rc = import_message(sdata, data, cfg); rc = import_message(sdata, data, counters, cfg);
if(rc == ERR){ if(rc == ERR){
printf("ERROR: error importing: '%s'\n", data->import->filename); printf("ERROR: error importing: '%s'\n", data->import->filename);
ret = ERR; ret = ERR;
@ -80,7 +80,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *
} }
int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct config *cfg){ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
DIR *dir; DIR *dir;
struct dirent *de; struct dirent *de;
int rc=ERR, ret=OK, i=0; int rc=ERR, ret=OK, i=0;
@ -103,7 +103,7 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct dat
if(stat(fname, &st) == 0){ if(stat(fname, &st) == 0){
if(S_ISDIR(st.st_mode)){ if(S_ISDIR(st.st_mode)){
folder = data->folder; folder = data->folder;
rc = import_mbox_from_dir(fname, sdata, data, cfg); rc = import_mbox_from_dir(fname, sdata, data, counters, cfg);
data->folder = folder; data->folder = folder;
if(rc == ERR) ret = ERR; if(rc == ERR) ret = ERR;
} }
@ -126,7 +126,7 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct dat
} }
rc = import_from_mailbox(fname, sdata, data, cfg); rc = import_from_mailbox(fname, sdata, data, counters, cfg);
if(rc == OK) (data->import->tot_msgs)++; if(rc == OK) (data->import->tot_msgs)++;
else ret = ERR; else ret = ERR;

6
src/import_maildir.c
View File

@ -22,7 +22,7 @@
#include <piler.h> #include <piler.h>
int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct config *cfg){ int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg){
DIR *dir; DIR *dir;
struct dirent *de; struct dirent *de;
int rc=ERR, ret=OK, i=0; int rc=ERR, ret=OK, i=0;
@ -46,7 +46,7 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir
if(S_ISDIR(st.st_mode)){ if(S_ISDIR(st.st_mode)){
folder = data->folder; folder = data->folder;
snprintf(subdir, sizeof(subdir)-1, "%s", data->import->filename); snprintf(subdir, sizeof(subdir)-1, "%s", data->import->filename);
rc = import_from_maildir(sdata, data, subdir, cfg); rc = import_from_maildir(sdata, data, subdir, counters, cfg);
data->folder = folder; data->folder = folder;
if(rc == ERR) ret = ERR; if(rc == ERR) ret = ERR;
} }
@ -76,7 +76,7 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir
} }
rc = import_message(sdata, data, cfg); rc = import_message(sdata, data, counters, cfg);
if(rc == OK) (data->import->tot_msgs)++; if(rc == OK) (data->import->tot_msgs)++;
else if(rc == ERR){ else if(rc == ERR){

14
src/import_pilerexport.c
View File

@ -22,21 +22,21 @@
#include <piler.h> #include <piler.h>
void import_the_file(struct session_data *sdata, struct data *data, struct config *cfg){ void import_the_file(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
close(data->import->fd); close(data->import->fd);
data->import->fd = -1; data->import->fd = -1;
if(import_message(sdata, data, cfg) != ERR){ if(import_message(sdata, data, counters, cfg) != ERR){
unlink(data->import->filename); unlink(data->import->filename);
} }
} }
void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct config *cfg){ void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
if(!strcmp(buf, PILEREXPORT_BEGIN_MARK)){ if(!strcmp(buf, PILEREXPORT_BEGIN_MARK)){
if((*count) > 0){ if((*count) > 0){
import_the_file(sdata, data, cfg); import_the_file(sdata, data, counters, cfg);
} }
(*count)++; (*count)++;
@ -57,7 +57,7 @@ void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *s
} }
void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg){ void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
int n, rc, nullbyte, savedlen=0, puflen; int n, rc, nullbyte, savedlen=0, puflen;
uint64 count=0; uint64 count=0;
char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE]; char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE];
@ -96,7 +96,7 @@ void import_from_pilerexport(struct session_data *sdata, struct data *data, stru
if(puflen > 0){ if(puflen > 0){
if(rc == OK){ if(rc == OK){
process_buffer(puf, puflen, &count, sdata, data, cfg); process_buffer(puf, puflen, &count, sdata, data, counters, cfg);
} }
else { else {
snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf); snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
@ -109,6 +109,6 @@ void import_from_pilerexport(struct session_data *sdata, struct data *data, stru
} while(n > 0); } while(n > 0);
if(data->import->fd != -1){ if(data->import->fd != -1){
import_the_file(sdata, data, cfg); import_the_file(sdata, data, counters, cfg);
} }
} }

4
src/import_pop3.c
View File

@ -22,7 +22,7 @@
#include <piler.h> #include <piler.h>
void import_from_pop3_server(struct session_data *sdata, struct data *data, struct config *cfg){ void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
int rc; int rc;
char port_string[8]; char port_string[8];
struct addrinfo hints, *res; struct addrinfo hints, *res;
@ -58,7 +58,7 @@ void import_from_pop3_server(struct session_data *sdata, struct data *data, stru
goto ENDE_POP3; goto ENDE_POP3;
} }
process_pop3_emails(sdata, data, cfg); process_pop3_emails(sdata, data, counters, cfg);
close_connection(data->net); close_connection(data->net);

21
src/pilerimport.c
View File

@ -75,6 +75,9 @@ int main(int argc, char **argv){
struct data data; struct data data;
struct import import; struct import import;
struct net net; struct net net;
struct counters counters;
bzero(&counters, sizeof(counters));
for(i=0; i<MBOX_ARGS; i++) mbox[i] = NULL; for(i=0; i<MBOX_ARGS; i++) mbox[i] = NULL;
@ -384,18 +387,18 @@ int main(int argc, char **argv){
load_mydomains(&sdata, &data, &cfg); load_mydomains(&sdata, &data, &cfg);
if(data.import->filename[0] != '\0') import_message(&sdata, &data, &cfg); if(data.import->filename[0] != '\0') import_message(&sdata, &data, &counters, &cfg);
if(mbox[0]){ if(mbox[0]){
for(i=0; i<n_mbox; i++){ for(i=0; i<n_mbox; i++){
import_from_mailbox(mbox[i], &sdata, &data, &cfg); import_from_mailbox(mbox[i], &sdata, &data, &counters, &cfg);
} }
} }
if(data.import->mboxdir) import_mbox_from_dir(data.import->mboxdir, &sdata, &data, &cfg); if(data.import->mboxdir) import_mbox_from_dir(data.import->mboxdir, &sdata, &data, &counters, &cfg);
if(directory) import_from_maildir(&sdata, &data, directory, &cfg); if(directory) import_from_maildir(&sdata, &data, directory, &counters, &cfg);
if(imapserver) import_from_imap_server(&sdata, &data, &cfg); if(imapserver) import_from_imap_server(&sdata, &data, &counters, &cfg);
if(pop3server) import_from_pop3_server(&sdata, &data, &cfg); if(pop3server) import_from_pop3_server(&sdata, &data, &counters, &cfg);
if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &cfg); if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &counters, &cfg);
clearrules(data.archiving_rules); clearrules(data.archiving_rules);
clearrules(data.retention_rules); clearrules(data.retention_rules);
@ -403,6 +406,10 @@ int main(int argc, char **argv){
clearhash(data.mydomains); clearhash(data.mydomains);
update_counters(&sdata, &data, &counters, &cfg);
syslog(LOG_PRIORITY, "imported=%lld, duplicated=%lld, discarded=%lld", counters.c_rcvd, counters.c_duplicate, counters.c_ignore);
close_database(&sdata); close_database(&sdata);
if(cfg.rtindex) close_sphx(&sdata); if(cfg.rtindex) close_sphx(&sdata);

4
src/pop3.c
View File

@ -175,7 +175,7 @@ void pop3_delete_message(struct data *data, int i){
} }
void process_pop3_emails(struct session_data *sdata, struct data *data, struct config *cfg){ void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
char buf[MAXBUFSIZE]; char buf[MAXBUFSIZE];
data->import->processed_messages = 0; data->import->processed_messages = 0;
@ -191,7 +191,7 @@ void process_pop3_emails(struct session_data *sdata, struct data *data, struct c
if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/data->import->total_messages); fflush(stdout); } if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/data->import->total_messages); fflush(stdout); }
if(data->import->dryrun == 0){ if(data->import->dryrun == 0){
int rc = import_message(sdata, data, cfg); int rc = import_message(sdata, data, counters, cfg);
if(data->import->remove_after_import == 1 && rc == OK){ if(data->import->remove_after_import == 1 && rc == OK){
pop3_delete_message(data, i); pop3_delete_message(data, i);