From 8a5e01c5c5b5e1d1a17b514b13dacfb971631068 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 8 Aug 2020 21:27:54 +0200 Subject: [PATCH 01/55] piler.c fixes Signed-off-by: Janos SUTO --- src/piler.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/piler.c b/src/piler.c index d0aa3310..46b99e18 100644 --- a/src/piler.c +++ b/src/piler.c @@ -106,7 +106,6 @@ int process_email(char *filename, struct session_data *sdata, struct data *data, char tmpbuf[SMALLBUFSIZE]; char *status=S_STATUS_UNDEF; char *arule; - char *rcpt; char *p; struct timezone tz; struct timeval tv1, tv2; @@ -135,7 +134,7 @@ int process_email(char *filename, struct session_data *sdata, struct data *data, post_parse(sdata, &parser_state, cfg); if(cfg->syslog_recipients == 1){ - rcpt = parser_state.b_to; + char *rcpt = parser_state.b_to; do { rcpt = split_str(rcpt, " ", tmpbuf, sizeof(tmpbuf)-1); @@ -162,9 +161,10 @@ int process_email(char *filename, struct session_data *sdata, struct data *data, syslog(LOG_PRIORITY, "%s: invalid message, hdr_len: %d", filename, sdata->hdr_len); rc = ERR; } - - rc = process_message(sdata, &parser_state, data, cfg); - unlink(parser_state.message_id_hash); + else { + rc = process_message(sdata, &parser_state, data, cfg); + unlink(parser_state.message_id_hash); + } } unlink(sdata->tmpframe); @@ -467,7 +467,7 @@ void initialise_configuration(){ int main(int argc, char **argv){ - int i, daemonise=0, dedupfd; + int i, daemonise=0; struct stat st; @@ -521,7 +521,7 @@ int main(int argc, char **argv){ if(stat(cfg.pidfile, &st) == 0) fatal(ERR_PID_FILE_EXISTS); if(cfg.mmap_dedup_test == 1){ - dedupfd = open(MESSAGE_ID_DEDUP_FILE, O_RDWR); + int dedupfd = open(MESSAGE_ID_DEDUP_FILE, O_RDWR); if(dedupfd == -1) fatal(ERR_OPEN_DEDUP_FILE); data.dedup = mmap(NULL, MAXCHILDREN*DIGEST_LENGTH*2, PROT_READ|PROT_WRITE, MAP_SHARED, dedupfd, 0); From 3e33bf99c8e0b366da0c29044d5dfc4236c4d95f Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 8 Aug 2020 21:28:09 +0200 Subject: [PATCH 02/55] piler-smtp.c fixes Signed-off-by: Janos SUTO --- src/piler-smtp.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/piler-smtp.c b/src/piler-smtp.c index b75abe34..41cbca46 100644 --- a/src/piler-smtp.c +++ b/src/piler-smtp.c @@ -52,14 +52,12 @@ void usage(){ void p_clean_exit(int sig){ - int i; - if(sig > 0) syslog(LOG_PRIORITY, "got signal: %d, %s", sig, strsignal(sig)); if(listenerfd != -1) close(listenerfd); if(sessions){ - for(i=0; i= LOG_DEBUG) syslog(LOG_PRIORITY, "%s @%ld", __func__, now); if(num_connections > 0){ - for(i=0; ilasttime >= cfg.smtp_timeout){ syslog(LOG_PRIORITY, "client %s timeout, lasttime: %ld", sessions[i]->remote_host, sessions[i]->lasttime); tear_down_session(sessions, sessions[i]->slot, &num_connections); @@ -128,8 +125,8 @@ void initialise_configuration(){ int main(int argc, char **argv){ - int listenerfd, client_sockfd; - int i, n, daemonise=0; + int client_sockfd; + int i, daemonise=0; int client_len = sizeof(struct sockaddr_storage); ssize_t readlen; struct sockaddr_storage client_address; @@ -220,7 +217,7 @@ int main(int argc, char **argv){ alarm(cfg.check_for_client_timeout_interval); for(;;){ - n = epoll_wait(efd, events, cfg.max_connections, -1); + int n = epoll_wait(efd, events, cfg.max_connections, -1); for(i=0; i Date: Sat, 8 Aug 2020 21:28:25 +0200 Subject: [PATCH 03/55] pilerexport.c fixes Signed-off-by: Janos SUTO --- src/pilerexport.c | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/src/pilerexport.c b/src/pilerexport.c index 9abd03aa..1b8de98a 100644 --- a/src/pilerexport.c +++ b/src/pilerexport.c @@ -87,15 +87,12 @@ unsigned long convert_time(char *yyyymmdd, int h, int m, int s){ tm.tm_mday = atoi(yyyymmdd); - - tm.tm_isdst = -1; - return mktime(&tm); } int append_email_to_buffer(char **buffer, char *email){ - int len, arglen; + int arglen; char *s=NULL, emailaddress[SMALLBUFSIZE]; snprintf(emailaddress, sizeof(emailaddress)-1, "'%s'", email); @@ -107,7 +104,7 @@ int append_email_to_buffer(char **buffer, char *email){ memcpy(*buffer, emailaddress, arglen); } else { - len = strlen(*buffer); + int len = strlen(*buffer); s = realloc(*buffer, len + arglen+2); if(!s){ printf("malloc problem!\n"); @@ -126,7 +123,7 @@ int append_email_to_buffer(char **buffer, char *email){ int append_string_to_buffer(char **buffer, char *str){ - int len, arglen; + int arglen; char *s=NULL; arglen = strlen(str); @@ -137,7 +134,7 @@ int append_string_to_buffer(char **buffer, char *str){ memcpy(*buffer, str, arglen); } else { - len = strlen(*buffer); + int len = strlen(*buffer); s = realloc(*buffer, len + arglen+1); if(!s) return 1; @@ -152,9 +149,7 @@ int append_string_to_buffer(char **buffer, char *str){ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *where_condition, uint64 last_id, int *num, struct config *cfg){ - MYSQL_RES *res; MYSQL_ROW row; - int rc=0; uint64 id=0; char s[SMALLBUFSIZE]; @@ -168,7 +163,7 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char * snprintf(s, sizeof(s)-1, "SELECT id FROM %s WHERE %s AND id > %llu ORDER BY id ASC LIMIT 0,%d", index_list, where_condition, last_id, max_matches); if(mysql_real_query(&(sdata2->mysql), s, strlen(s)) == 0){ - res = mysql_store_result(&(sdata2->mysql)); + MYSQL_RES *res = mysql_store_result(&(sdata2->mysql)); if(res != NULL){ while((row = mysql_fetch_row(res))){ id = strtoull(row[0], NULL, 10); @@ -194,12 +189,11 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char * uint64 get_total_found(struct session_data *sdata){ - MYSQL_RES *res; MYSQL_ROW row; uint64 total_found=0; if(mysql_real_query(&(sdata->mysql), "SHOW META LIKE 'total_found'", 28) == 0){ - res = mysql_store_result(&(sdata->mysql)); + MYSQL_RES *res = mysql_store_result(&(sdata->mysql)); if(res != NULL){ while((row = mysql_fetch_row(res))){ total_found = strtoull(row[1], NULL, 10); @@ -313,7 +307,7 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain if(startdate > 0){ if(where_condition) rc = append_string_to_buffer(&query, " AND "); - snprintf(s, sizeof(s)-1, " `sent` >= %ld", startdate); + snprintf(s, sizeof(s)-1, " `sent` >= %lu", startdate); rc += append_string_to_buffer(&query, s); where_condition++; @@ -322,10 +316,8 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain if(stopdate > 0){ if(where_condition) rc = append_string_to_buffer(&query, " AND "); - snprintf(s, sizeof(s)-1, " `sent` <= %ld", stopdate); + snprintf(s, sizeof(s)-1, " `sent` <= %lu", stopdate); rc += append_string_to_buffer(&query, s); - - where_condition++; } @@ -340,7 +332,6 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct uint64 id, n=0; char digest[SMALLBUFSIZE], bodydigest[SMALLBUFSIZE]; char filename[SMALLBUFSIZE]; - int rc=0; struct sql sql; if(prepare_sql_statement(sdata, &sql, s) == ERR) return ERR; @@ -410,7 +401,7 @@ ENDE: int main(int argc, char **argv){ - int c, minsize=0, maxsize=0; + int minsize=0, maxsize=0; size_t nmatch=0; unsigned long startdate=0, stopdate=0; char *configfile=CONFIG_FILE; @@ -451,9 +442,9 @@ int main(int argc, char **argv){ int option_index = 0; - c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?", long_options, &option_index); + int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?", long_options, &option_index); #else - c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?"); + int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?"); #endif if(c == -1) break; From 47838a059f9e959bb42e8c2f25f7f7e5d3424048 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 8 Aug 2020 21:29:41 +0200 Subject: [PATCH 04/55] pilerimport.c fixes Signed-off-by: Janos SUTO --- src/pilerimport.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pilerimport.c b/src/pilerimport.c index eaf7b87d..6b34d6f0 100644 --- a/src/pilerimport.c +++ b/src/pilerimport.c @@ -62,7 +62,7 @@ void usage(){ int main(int argc, char **argv){ - int i, c, n_mbox=0; + int i, n_mbox=0; char *configfile=CONFIG_FILE, *mbox[MBOX_ARGS], *directory=NULL; char puf[SMALLBUFSIZE], *imapserver=NULL, *pop3server=NULL; struct session_data sdata; @@ -145,9 +145,9 @@ int main(int argc, char **argv){ int option_index = 0; - c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:DRroqh?", long_options, &option_index); + int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:DRroqh?", long_options, &option_index); #else - c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:DRroqh?"); + int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:DRroqh?"); #endif if(c == -1) break; From 345fd715f5b1a1a1af65a7a6cd203c49bc6f9023 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 10 Aug 2020 20:58:34 +0200 Subject: [PATCH 05/55] cppcheck refactoring Signed-off-by: Janos SUTO --- configure | 2 +- configure.in | 2 +- cppcheck.sh | 7 +++ src/archive.c | 10 ++-- src/attachment.c | 3 +- src/base64.c | 78 --------------------------- src/bdat.c | 3 +- src/cfg.c | 19 ------- src/clamd.c | 6 +-- src/counters.c | 9 ++-- src/decoder.c | 21 ++------ src/decoder.h | 3 -- src/defs.h | 4 -- src/digest.c | 4 +- src/extract.c | 11 ++-- src/hash.c | 14 +++-- src/imap.c | 4 +- src/import.c | 5 +- src/import_imap.c | 2 +- src/list.c | 94 --------------------------------- src/list.h | 16 ------ src/memc.c | 53 ++----------------- src/message.c | 9 ---- src/misc.c | 62 +++------------------- src/misc.h | 3 -- src/mydomains.c | 5 +- src/parser.c | 27 +++++----- src/parser_utils.c | 52 ++++++++---------- src/pop3.c | 12 ++--- src/reindex.c | 15 +++--- src/rules.c | 36 ++++++------- src/session.c | 11 ++-- src/smtp.c | 9 ++-- src/stats.c | 4 +- src/store.c | 3 +- src/tai.c | 19 ------- src/test.c | 6 +-- suppressions.txt | 13 +++++ unit_tests/check_digest.c | 16 +++--- unit_tests/check_misc.c | 8 +-- unit_tests/check_parser_utils.c | 3 +- unit_tests/smtp.c | 10 ++-- 42 files changed, 169 insertions(+), 524 deletions(-) create mode 100755 cppcheck.sh delete mode 100644 src/base64.c delete mode 100644 src/list.c delete mode 100644 src/list.h create mode 100644 suppressions.txt diff --git a/configure b/configure index 639223d5..19428bde 100755 --- a/configure +++ b/configure @@ -4854,7 +4854,7 @@ echo; echo CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wuninitialized -Wno-format-truncation -g" LIBS="$antispam_libs $sunos_libs " -OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" +OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" ac_config_files="$ac_config_files Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile" diff --git a/configure.in b/configure.in index 388ac8e7..34bbebcc 100644 --- a/configure.in +++ b/configure.in @@ -537,7 +537,7 @@ echo; echo CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wuninitialized -Wno-format-truncation -g" LIBS="$antispam_libs $sunos_libs " -OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" +OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile]) AC_OUTPUT diff --git a/cppcheck.sh b/cppcheck.sh new file mode 100755 index 00000000..88794857 --- /dev/null +++ b/cppcheck.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +cppcheck -DHAVE_PDFTOTEXT -DHAVE_PPTHTML -DHAVE_TNEF -DHAVE_UNRTF -DHAVE_XLS2CSV -DHAVE_CATPPT -DHAVE_CATDOC -DHAVE_ZIP -D_GNU_SOURCE -DHAVE_DAEMON -DHAVE_TRE -DHAVE_CLAMD -DHAVE_LIBCLAMAV -DNEED_MYSQL --error-exitcode=1 --enable=all --suppressions-list=suppressions.txt --force src/ unit_tests/ diff --git a/src/archive.c b/src/archive.c index 9a0ffe97..9984c26f 100644 --- a/src/archive.c +++ b/src/archive.c @@ -54,7 +54,6 @@ void zerr(int ret){ int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){ int ret, pos=0; - unsigned have; z_stream strm; char *new_ptr; unsigned char out[REALLYBIGBUFSIZE]; @@ -96,7 +95,7 @@ int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){ return ret; } - have = REALLYBIGBUFSIZE - strm.avail_out; + unsigned have = REALLYBIGBUFSIZE - strm.avail_out; /* * write the uncompressed result either to stdout @@ -235,8 +234,8 @@ CLEANUP: int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct config *cfg){ - int i, attachments; - char *buffer=NULL, *saved_buffer, *p, filename[SMALLBUFSIZE], pointer[SMALLBUFSIZE]; + int attachments; + char *buffer=NULL, *saved_buffer, *p, filename[SMALLBUFSIZE]; struct ptr_array ptr_arr[MAX_ATTACHMENTS]; #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT struct stat st; @@ -270,7 +269,8 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct c if(buffer){ saved_buffer = buffer; - for(i=1; i<=attachments; i++){ + for(int i=1; i<=attachments; i++){ + char pointer[SMALLBUFSIZE]; snprintf(pointer, sizeof(pointer)-1, "ATTACHMENT_POINTER_%s.a%d_XXX_PILER", sdata->ttmpfile, i); p = strstr(buffer, pointer); diff --git a/src/attachment.c b/src/attachment.c index 70a156b6..f6d14f5d 100644 --- a/src/attachment.c +++ b/src/attachment.c @@ -17,7 +17,6 @@ int store_attachments(struct session_data *sdata, struct parser_state *state, struct config *cfg){ - uint64 id=0; int i, rc=1, found, affected_rows; struct sql sql, sql2; @@ -27,7 +26,7 @@ int store_attachments(struct session_data *sdata, struct parser_state *state, st for(i=1; i<=state->n_attachments; i++){ found = 0; - id = 0; + uint64 id = 0; if(state->attachments[i].size > 0){ diff --git a/src/base64.c b/src/base64.c deleted file mode 100644 index 82e69ff2..00000000 --- a/src/base64.c +++ /dev/null @@ -1,78 +0,0 @@ -/* - * base64.c, SJ - */ - -#include -#include - - -char base64_value(char c){ - static const char *base64_table = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - - if((int)c > 63) return '='; - - return base64_table[(int)c]; -} - - -void base64_encode_block(unsigned char *in, int inlen, char *out){ - char a, b, c, d, fragment; - - sprintf(out, "===="); - - if(inlen <= 0) return; - - fragment = *in & 0x3; - - a = *in >> 2; - - out[0] = base64_value(a); - - b = fragment << 4; - - if(inlen > 1) - b += *(in+1) >> 4; - - out[1] = base64_value(b); - - if(inlen == 1) return; - - - c = *(in+1) & 0xf; - c = c << 2; - - if(inlen > 2){ - fragment = *(in+2) & 0xfc; - c += fragment >> 6; - - d = *(in+2) & 0x3f; - out[3] = base64_value(d); - } - - out[2] = base64_value(c); -} - - -void base64_encode(unsigned char *in, int inlen, char *out, int outlen){ - int i=0, j, pos=0; - unsigned char buf[3]; - - memset(buf, 0, 3); - memset(out, 0, outlen); - - for(j=0; jfd == -1){ @@ -106,6 +104,7 @@ void process_bdat(struct smtp_session *session, char *readbuf, int readlen, stru move_email(session); + char buf[SMALLBUFSIZE]; snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile); send_smtp_response(session, buf); syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d, client=%s, fd=%d", session->ttmpfile, session->mailfrom, session->tot_len, session->remote_host, session->net.socket); diff --git a/src/cfg.c b/src/cfg.c index 820e5cd9..0c40b705 100644 --- a/src/cfg.c +++ b/src/cfg.c @@ -17,31 +17,12 @@ int string_parser(char *src, char *target, int limit){ return 0; }; -int multi_line_string_parser(char *src, char *target, unsigned int limit){ - if(strlen(src) > 0 && strlen(target) + strlen(src) + 3 < limit){ - strncat(target, src, limit-strlen(target)); - strncat(target, "\r\n", limit-strlen(target)); - - return 0; - } - - return 1; -}; - int int_parser(char *src, int *target){ *target = strtol(src, (char **) NULL, 10); return 0; }; - -int float_parser(char *src, float *target){ - *target = strtof(src, (char **) NULL); - - return 0; -}; - - struct _parse_rule { char *name; char *type; diff --git a/src/clamd.c b/src/clamd.c index 8f7f3f08..07b4a15b 100644 --- a/src/clamd.c +++ b/src/clamd.c @@ -20,7 +20,7 @@ int clamd_scan(char *tmpfile, struct config *cfg){ int s, n; - char *p, *q, buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE]; + char buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE]; struct sockaddr_un server; chmod(tmpfile, 0644); @@ -58,9 +58,9 @@ int clamd_scan(char *tmpfile, struct config *cfg){ if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: CLAMD DEBUG: %d %s", tmpfile, n, buf); if(strcasestr(buf, CLAMD_RESP_INFECTED)){ - p = strchr(buf, ' '); + char *p = strchr(buf, ' '); if(p){ - q = strrchr(p, ' '); + char *q = strrchr(p, ' '); if(q){ *q = '\0'; p++; diff --git a/src/counters.c b/src/counters.c index 35ad9402..f7030b30 100644 --- a/src/counters.c +++ b/src/counters.c @@ -51,13 +51,12 @@ struct counters load_counters(struct session_data *sdata){ void update_counters(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){ char buf[MAXBUFSIZE]; #ifdef HAVE_MEMCACHED - unsigned long long mc, rcvd; + unsigned long long mc; struct counters c; - char key[MAX_MEMCACHED_KEY_LEN]; unsigned int flags=0; #endif - if(counters->c_virus + counters->c_duplicate + counters->c_ignore + counters->c_size + counters->c_stored_size <= 0) return; + if(counters->c_virus + counters->c_duplicate + counters->c_ignore + counters->c_size + counters->c_stored_size == 0) return; #ifdef HAVE_MEMCACHED if(cfg->update_counters_to_memcached == 1){ @@ -65,7 +64,7 @@ void update_counters(struct session_data *sdata, struct data *data, struct count /* increment counters to memcached */ if(memcached_increment(&(data->memc), MEMCACHED_MSGS_RCVD, counters->c_rcvd, &mc) == MEMCACHED_SUCCESS){ - rcvd = mc; + unsigned long long rcvd = mc; if(counters->c_virus > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_VIRUS, counters->c_virus, &mc); if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_DUPLICATE, counters->c_duplicate, &mc); @@ -79,6 +78,8 @@ void update_counters(struct session_data *sdata, struct data *data, struct count snprintf(buf, MAXBUFSIZE-1, "%s %s %s %s %s %s %s", MEMCACHED_MSGS_RCVD, MEMCACHED_MSGS_VIRUS, MEMCACHED_MSGS_DUPLICATE, MEMCACHED_MSGS_IGNORE, MEMCACHED_MSGS_SIZE, MEMCACHED_MSGS_STORED_SIZE, MEMCACHED_COUNTERS_LAST_UPDATE); if(memcached_mget(&(data->memc), buf) == MEMCACHED_SUCCESS){ + char key[MAX_MEMCACHED_KEY_LEN]; + while((memcached_fetch_result(&(data->memc), &key[0], &buf[0], &flags))){ if(!strcmp(key, MEMCACHED_MSGS_RCVD)) c.c_rcvd = strtoull(buf, NULL, 10); else if(!strcmp(key, MEMCACHED_MSGS_VIRUS)) c.c_virus = strtoull(buf, NULL, 10); diff --git a/src/decoder.c b/src/decoder.c index 93c4fcc7..fc034ff1 100644 --- a/src/decoder.c +++ b/src/decoder.c @@ -98,20 +98,6 @@ inline void utf8_encode_char(unsigned char c, unsigned char *buf, int buflen, in } -void sanitiseBase64(char *s){ - char *p1; - - if(s == NULL) return; - - for(; *s; s++){ - if(b64[(unsigned int)(*s & 0xFF)] == 255){ - for(p1 = s; p1[0] != '\0'; p1++) - p1[0] = p1[1]; - } - } -} - - inline static void pack_4_into_3(char *s, char *s2){ int j, n[4], k1, k2; @@ -157,7 +143,7 @@ int decodeBase64(char *p){ int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){ - int i, len=0, decodedlen; + int i, len=0; char s[5], s2[3]; if(plen < 4 || plen > blen) @@ -166,7 +152,7 @@ int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){ for(i=0; iimport->filename, '/'); if(p) p++; else p = data->import->filename; + char newpath[SMALLBUFSIZE]; snprintf(newpath, sizeof(newpath)-2, "%s/%s", data->import->failed_folder, p); if(rename(data->import->filename, newpath)) diff --git a/src/import_imap.c b/src/import_imap.c index 79876c74..7804767e 100644 --- a/src/import_imap.c +++ b/src/import_imap.c @@ -73,7 +73,7 @@ int import_from_imap_server(struct session_data *sdata, struct data *data, struc q = data->imapfolders[i]; while(q != NULL){ - if(q && q->str && strlen(q->str) > 1){ + if(q->str && strlen(q->str) > 1){ skipmatch = 0; diff --git a/src/list.c b/src/list.c deleted file mode 100644 index 965d48b3..00000000 --- a/src/list.c +++ /dev/null @@ -1,94 +0,0 @@ -/* - * list.c, SJ - */ - -#include -#include -#include -#include -#include "list.h" -#include "config.h" - - -int append_list(struct list **list, char *p){ - struct list *q, *t, *u=NULL; - - q = *list; - - while(q){ - if(strcmp(q->s, p) == 0) - return 0; - - u = q; - q = q->r; - } - - t = create_list_item(p); - if(t){ - if(*list == NULL) - *list = t; - else if(u) - u->r = t; - - return 1; - } - - return -1; -} - - -struct list *create_list_item(char *s){ - struct list *h=NULL; - - if((h = malloc(sizeof(struct list))) == NULL) - return NULL; - - snprintf(h->s, SMALLBUFSIZE-1, "%s", s); - h->r = NULL; - - return h; -} - - -int is_string_on_list(struct list *list, char *s){ - struct list *p; - - p = list; - - while(p != NULL){ - if(strcmp(p->s, s) == 0) return 1; - p = p->r; - } - - return 0; -} - - -int is_item_on_string(struct list *list, char *s){ - struct list *p; - - p = list; - - while(p != NULL){ - if(strstr(s, p->s)) return 1; - p = p->r; - } - - return 0; -} - - -void free_list(struct list *list){ - struct list *p, *q; - - p = list; - - while(p != NULL){ - q = p->r; - - if(p) - free(p); - - p = q; - } -} diff --git a/src/list.h b/src/list.h deleted file mode 100644 index 56a413c7..00000000 --- a/src/list.h +++ /dev/null @@ -1,16 +0,0 @@ -/* - * list.h, SJ - */ - -#ifndef _LIST_H - #define _LIST_H - -#include "defs.h" - -int append_list(struct list **list, char *p); -struct list *create_list_item(char *s); -int is_string_on_list(struct list *list, char *s); -int is_item_on_string(struct list *list, char *s); -void free_list(struct list *list); - -#endif /* _LIST_H */ diff --git a/src/memc.c b/src/memc.c index c56c4db8..c04b84a0 100644 --- a/src/memc.c +++ b/src/memc.c @@ -59,7 +59,7 @@ void memcached_init(struct memcached_server *ptr, char *server_ip, int server_po int set_socket_options(struct memcached_server *ptr){ - int error, flag=1, flags, rval; + int error, flag=1, flags; struct timeval waittime; struct linger linger; @@ -128,8 +128,7 @@ int set_socket_options(struct memcached_server *ptr){ if((flags & O_NONBLOCK) == 0){ - rval = fcntl(ptr->fd, F_SETFL, flags | O_NONBLOCK); - if(rval == -1) return MEMCACHED_FAILURE; + if(fcntl(ptr->fd, F_SETFL, flags | O_NONBLOCK) == -1) return MEMCACHED_FAILURE; } return MEMCACHED_SUCCESS; @@ -223,7 +222,7 @@ int memcached_add(struct memcached_server *ptr, char *cmd, char *key, char *valu if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE; // cmd could be either 'add' or 'set' - snprintf(ptr->buf, MAXBUFSIZE-1, "%s %s %d %ld %d \r\n", cmd, key, flags, expiry, valuelen); + snprintf(ptr->buf, MAXBUFSIZE-1, "%s %s %u %lu %u \r\n", cmd, key, flags, expiry, valuelen); len = strlen(ptr->buf); strncat(ptr->buf, value, MAXBUFSIZE-strlen(ptr->buf)-1); @@ -263,52 +262,6 @@ int memcached_increment(struct memcached_server *ptr, char *key, unsigned long l } -char *memcached_get(struct memcached_server *ptr, char *key, unsigned int *len, unsigned int *flags){ - char *p; - - if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return NULL; - - snprintf(ptr->buf, MAXBUFSIZE, "get %s \r\n", key); - send(ptr->fd, ptr->buf, strlen(ptr->buf), 0); - - ptr->last_read_bytes = __recvtimeout(ptr->fd, ptr->buf, MAXBUFSIZE, ptr->rcv_timeout); - - if(ptr->last_read_bytes <= 0){ - memcached_shutdown(ptr); - return NULL; - } - - - if(ptr->last_read_bytes < 10) return NULL; - - if(strncmp("VALUE ", ptr->buf, 6)) return NULL; - - p = strchr(ptr->buf, '\r'); - if(!p) return NULL; - *p = '\0'; - ptr->result = p + 2; - - p = strrchr(ptr->buf + 6, ' '); - if(!p) return NULL; - *len = atoi(p+1); - *p = '\0'; - - - p = strrchr(ptr->buf + 6, ' '); - if(!p) return NULL; - *flags = atoi(p+1); - *p = '\0'; - - - p = strchr(ptr->result, '\r'); - if(!p) return NULL; - - *p = '\0'; - - return ptr->result; -} - - int memcached_mget(struct memcached_server *ptr, char *key){ if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE; diff --git a/src/message.c b/src/message.c index 6279af31..7700fcdb 100644 --- a/src/message.c +++ b/src/message.c @@ -136,15 +136,6 @@ int store_recipients(struct session_data *sdata, char *to, uint64 id, struct con } -void remove_recipients(struct session_data *sdata, uint64 id){ - char s[SMALLBUFSIZE]; - - snprintf(s, sizeof(s)-1, "DELETE FROM " SQL_RECIPIENT_TABLE " WHERE id=%llu", id); - - p_query(sdata, s); -} - - int store_folder_id(struct session_data *sdata, struct data *data, uint64 id, struct config *cfg){ int rc=ERR; struct sql sql; diff --git a/src/misc.c b/src/misc.c index 82703acb..d723f9ec 100644 --- a/src/misc.c +++ b/src/misc.c @@ -88,44 +88,6 @@ long tvdiff(struct timeval a, struct timeval b){ } -/* - * count a character in buffer - */ - -int countCharacterInBuffer(char *p, char c){ - int i=0; - - for(; *p; p++){ - if(*p == c) - i++; - } - - return i; -} - - -void replaceCharacterInBuffer(char *p, char from, char to){ - size_t i; - int k=0; - - for(i=0; i 0){ - p[k] = to; - k++; - } - } - else { - p[k] = p[i]; - k++; - } - - } - - p[k] = '\0'; -} - - /* * split a string by a character as delimiter */ @@ -181,10 +143,8 @@ char *split_str(char *row, char *what, char *s, int size){ r += strlen(what); } - if(s != NULL){ - strncpy(s, row, len); - s[len] = '\0'; - } + strncpy(s, row, len); + s[len] = '\0'; return r; } @@ -215,8 +175,7 @@ int trimBuffer(char *s){ int extract_verp_address(char *email){ - char *p, *p1, *p2; - char puf[SMALLBUFSIZE]; + char *p1; // a VERP address is like archive+user=domain.com@myarchive.local @@ -226,13 +185,14 @@ int extract_verp_address(char *email){ p1 = strchr(email, '+'); if(p1){ - p2 = strchr(p1, '@'); + char *p2 = strchr(p1, '@'); if(p2 && p2 > p1 + 2){ if(strchr(p1+1, '=')){ + char puf[SMALLBUFSIZE]; memset(puf, 0, sizeof(puf)); memcpy(&puf[0], p1+1, p2-p1-1); - p = strchr(puf, '='); + char *p = strchr(puf, '='); if(p) *p = '@'; strcpy(email, puf); } @@ -349,10 +309,10 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){ int readFromEntropyPool(int fd, void *_s, ssize_t n){ char *s = _s; - ssize_t res, pos = 0; + ssize_t pos = 0; while(n > pos){ - res = read(fd, s + pos, n - pos); + ssize_t res = read(fd, s + pos, n - pos); switch(res){ case -1: continue; case 0: return res; @@ -605,12 +565,6 @@ void strtolower(char *s){ } -void *get_in_addr(struct sockaddr *sa){ - if(sa->sa_family == AF_INET) return &(((struct sockaddr_in*)sa)->sin_addr); - return &(((struct sockaddr_in6*)sa)->sin6_addr); -} - - int make_socket_non_blocking(int fd){ int flags, s; diff --git a/src/misc.h b/src/misc.h index 7cd650c9..ed13c206 100644 --- a/src/misc.h +++ b/src/misc.h @@ -19,8 +19,6 @@ int get_build(); void get_extractor_list(); void __fatal(char *s); long tvdiff(struct timeval a, struct timeval b); -int countCharacterInBuffer(char *p, char c); -void replaceCharacterInBuffer(char *p, char from, char to); char *split(char *str, int ch, char *buf, int buflen, int *result); char *split_str(char *row, char *what, char *s, int size); int trimBuffer(char *s); @@ -42,7 +40,6 @@ void init_session_data(struct session_data *sdata, struct config *cfg); int read_from_stdin(struct session_data *sdata); void strtolower(char *s); -void *get_in_addr(struct sockaddr *sa); int make_socket_non_blocking(int fd); int create_and_bind(char *listen_addr, int listen_port); diff --git a/src/mydomains.c b/src/mydomains.c index a25bd25d..47f57fc2 100644 --- a/src/mydomains.c +++ b/src/mydomains.c @@ -11,7 +11,6 @@ void load_mydomains(struct session_data *sdata, struct data *data, struct config *cfg){ - int rc; char s[SMALLBUFSIZE]; struct sql sql; @@ -32,9 +31,7 @@ void load_mydomains(struct session_data *sdata, struct data *data, struct config p_store_results(&sql); while(p_fetch_results(&sql) == OK){ - rc = addnode(data->mydomains, s); - - if(rc == 0) syslog(LOG_PRIORITY, "failed to append mydomain: '%s'", s); + if(addnode(data->mydomains, s) == 0) syslog(LOG_PRIORITY, "failed to append mydomain: '%s'", s); else if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "added mydomain: '%s'", s); memset(s, 0, sizeof(s)); diff --git a/src/parser.c b/src/parser.c index 9882aad1..08fb682a 100644 --- a/src/parser.c +++ b/src/parser.c @@ -73,9 +73,7 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg){ - int i, rec=0; - unsigned int len; - char *p; + int i; clearhash(state->boundaries); clearhash(state->rcpt); @@ -105,12 +103,12 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest); - p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type); - len = strlen(p); + char *p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type); + unsigned int len = strlen(p); if(strlen(sdata->attachments) < SMALLBUFSIZE-len-1 && !strstr(sdata->attachments, p)) memcpy(&(sdata->attachments[strlen(sdata->attachments)]), p, len); if(state->attachments[i].dumped == 1){ - rec = 0; + int rec = 0; if(cfg->extract_attachments == 1 && state->bodylen < BIGBUFSIZE-1024) extract_attachment_content(sdata, state, state->attachments[i].aname, get_attachment_extractor_by_filename(state->attachments[i].filename), &rec, cfg); unlink(state->attachments[i].aname); @@ -153,15 +151,13 @@ void storno_attachment(struct parser_state *state){ void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned int abuffersize){ - int n64; - unsigned char b64buffer[MAXBUFSIZE]; - if(write(state->fd, abuffer, state->abufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__); if(state->b64fd != -1){ abuffer[state->abufpos] = '\0'; if(state->base64 == 1){ - n64 = base64_decode_attachment_buffer(abuffer, &b64buffer[0], sizeof(b64buffer)); + unsigned char b64buffer[MAXBUFSIZE]; + int n64 = base64_decode_attachment_buffer(abuffer, &b64buffer[0], sizeof(b64buffer)); if(write(state->b64fd, b64buffer, n64) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__); } else if(write(state->b64fd, abuffer, state->abufpos) == -1){ @@ -175,9 +171,8 @@ void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg){ - char *p, puf[SMALLBUFSIZE]; - char tmpbuf[MAXBUFSIZE]; - int writelen, boundary_line=0, result; + char *p; + int boundary_line=0; unsigned int len; if(cfg->debug == 1) printf("line: %s", buf); @@ -284,8 +279,9 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata syslog(LOG_PRIORITY, "%s: error opening %s", sdata->ttmpfile, state->attachments[state->n_attachments].internalname); } else { + char puf[SMALLBUFSIZE]; snprintf(puf, sizeof(puf)-1, "ATTACHMENT_POINTER_%s.a%d_XXX_PILER", sdata->ttmpfile, state->n_attachments); - writelen = strlen(puf); + int writelen = strlen(puf); if(writelen + state->writebufpos > writebuffersize-1){ if(write(state->mfd, writebuffer, state->writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__); state->writebufpos = 0; @@ -637,7 +633,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata /* encode the body if it's not utf-8 encoded */ if(state->message_state == MSG_BODY && state->utf8 != 1){ - result = utf8_encode(buf, strlen(buf), &tmpbuf[0], sizeof(tmpbuf), state->charset); + char tmpbuf[MAXBUFSIZE]; + int result = utf8_encode(buf, strlen(buf), &tmpbuf[0], sizeof(tmpbuf), state->charset); if(result == OK) snprintf(buf, MAXBUFSIZE-1, "%s", tmpbuf); } diff --git a/src/parser_utils.c b/src/parser_utils.c index e88d19fb..7481f2db 100644 --- a/src/parser_utils.c +++ b/src/parser_utils.c @@ -114,7 +114,7 @@ long get_local_timezone_offset(){ time_t parse_date_header(char *datestr){ - int n=0, len; + int n=0; long offset=0; time_t ts=0; char *p, *q, *r, *tz, s[SMALLBUFSIZE], tzh[4], tzm[3]; @@ -150,7 +150,7 @@ time_t parse_date_header(char *datestr){ do { p = split_str(p, " ", s, sizeof(s)-1); - len = strlen(s); + int len = strlen(s); if(len > 0){ n++; @@ -264,7 +264,7 @@ time_t parse_date_header(char *datestr){ int extract_boundary(char *p, struct parser_state *state){ - char *q, *q2; + char *q; p += strlen("boundary"); @@ -291,7 +291,7 @@ int extract_boundary(char *p, struct parser_state *state){ break; } - q2 = strchr(p, ';'); + char *q2 = strchr(p, ';'); if(q2) *q2 = '\0'; q = strrchr(p, '"'); @@ -315,14 +315,13 @@ int extract_boundary(char *p, struct parser_state *state){ void fixupEncodedHeaderLine(char *buf, int buflen){ - char *p, *q, *r, *s, *e, *end; + char *q, *r, *s, *e, *end; /* * I thought SMALLBUFSIZE would be enough for v, encoding and tmpbuf(2*), * but then I saw a 6-7000 byte long subject line, so I've switched to MAXBUFSIZE */ char v[MAXBUFSIZE], u[MAXBUFSIZE], puf[MAXBUFSIZE], encoding[MAXBUFSIZE], tmpbuf[2*MAXBUFSIZE]; int need_encoding, ret, prev_encoded=0, n_tokens=0; - int b64=0, qp=0; if(buflen < 5) return; @@ -333,7 +332,7 @@ void fixupEncodedHeaderLine(char *buf, int buflen){ do { q = split_str(q, " ", v, sizeof(v)-1); - p = v; + char *p = v; do { memset(u, 0, sizeof(u)); @@ -349,7 +348,7 @@ void fixupEncodedHeaderLine(char *buf, int buflen){ * Happy New Year! =?utf-8?q?=F0=9F=8E=86?= */ - b64 = qp = 0; + int b64=0, qp=0; memset(encoding, 0, sizeof(encoding)); r = strstr(p, "=?"); @@ -446,9 +445,9 @@ void fixupEncodedHeaderLine(char *buf, int buflen){ void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state){ int i=0; - char *p, puf[MAXBUFSIZE]; if(strlen(state->qpbuf) > 0){ + char puf[MAXBUFSIZE]; memset(puf, 0, sizeof(puf)); snprintf(puf, sizeof(puf)-1, "%s%s", state->qpbuf, buf); snprintf(buf, MAXBUFSIZE-1, "%s", puf); @@ -461,7 +460,7 @@ void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state){ } if(i == 1){ - p = strrchr(buf, ' '); + char *p = strrchr(buf, ' '); if(p){ memset(state->qpbuf, 0, MAX_TOKEN_LEN); if(strlen(p) < MAX_TOKEN_LEN-1){ @@ -475,9 +474,8 @@ void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state){ void fixupBase64EncodedLine(char *buf, struct parser_state *state){ - char *p, puf[MAXBUFSIZE]; - if(strlen(state->miscbuf) > 0){ + char puf[MAXBUFSIZE]; memset(puf, 0, sizeof(puf)); strncpy(puf, state->miscbuf, sizeof(puf)-strlen(puf)-1); strncat(puf, buf, sizeof(puf)-strlen(puf)-1); @@ -489,7 +487,7 @@ void fixupBase64EncodedLine(char *buf, struct parser_state *state){ } if(buf[strlen(buf)-1] != '\n'){ - p = strrchr(buf, ' '); + char *p = strrchr(buf, ' '); if(p){ memcpy(&(state->miscbuf[0]), p+1, MAX_TOKEN_LEN-1); *p = '\0'; @@ -576,7 +574,7 @@ void markHTML(char *buf, struct parser_state *state){ int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state){ - char *p, html[SMALLBUFSIZE]; + char html[SMALLBUFSIZE]; int len; if(pos == 0 && strncmp(htmlbuf, "style ", 6) == 0) state->style = 1; @@ -594,7 +592,7 @@ int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state) len = strlen(html); if(len > 8 && strchr(html, '=')){ - p = strstr(html, "cid:"); + char *p = strstr(html, "cid:"); if(p){ *(p+3) = '\0'; strncat(html, " ", SMALLBUFSIZE-1); @@ -697,11 +695,9 @@ int does_it_seem_like_an_email_address(char *email){ void add_recipient(char *email, unsigned int len, struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){ - char *q; - if(findnode(state->rcpt, email) == NULL){ - q = strchr(email, '@'); + char *q = strchr(email, '@'); /* skip any address matching ...@cfg->hostid, 2013.10.29, SJ */ if(q && strncmp(q+1, cfg->hostid, cfg->hostid_len) == 0){ @@ -769,7 +765,7 @@ void degenerateToken(unsigned char *p){ int i=1, d=0, dp=0; unsigned char *s; - /* quit if this the string does not end with a punctuation character */ + /* quit if the string does not end with a punctuation character */ if(!ispunct(*(p+strlen((char *)p)-1))) return; @@ -825,8 +821,7 @@ void fixURL(char *buf, int buflen){ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultbuflen){ - int extended=0; - char buf[SMALLBUFSIZE], puf[SMALLBUFSIZE], *p, *q, *encoding; + char buf[SMALLBUFSIZE], *p, *q; snprintf(buf, sizeof(buf)-1, "%s", s); @@ -862,6 +857,8 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb * */ + int extended=0; + p += strlen(name); if(*p == '*'){ extended = 1; @@ -889,7 +886,7 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb if(extended == 1){ - encoding = p; + char *encoding = p; q = strchr(p, '\''); if(q){ *q = '\0'; @@ -906,6 +903,7 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb snprintf(resultbuf, resultbuflen-2, "%s", p); } else { + char puf[SMALLBUFSIZE]; snprintf(puf, sizeof(puf)-1, "%s", p); fixupEncodedHeaderLine(puf, sizeof(puf)); @@ -919,8 +917,6 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb char *determine_attachment_type(char *filename, char *type){ - char *p; - if(strncasecmp(type, "text/", strlen("text/")) == 0) return "text,"; if(strncasecmp(type, "image/", strlen("image/")) == 0) return "image,"; if(strncasecmp(type, "audio/", strlen("audio/")) == 0) return "audio,"; @@ -951,7 +947,7 @@ char *determine_attachment_type(char *filename, char *type){ if(strncasecmp(type, "application/", 12) == 0){ - p = strrchr(filename, '.'); + char *p = strrchr(filename, '.'); if(p){ p++; @@ -1013,14 +1009,13 @@ char *get_attachment_extractor_by_filename(char *filename){ void parse_reference(struct parser_state *state, char *s){ - int len; char puf[SMALLBUFSIZE]; if(strlen(state->reference) > 10) return; do { s = split_str(s, " ", puf, sizeof(puf)-1); - len = strlen(puf); + int len = strlen(puf); if(len > 10 && len < SMALLBUFSIZE-1){ memcpy(&(state->reference[strlen(state->reference)]), puf, len); @@ -1046,12 +1041,11 @@ int base64_decode_attachment_buffer(char *p, unsigned char *b, int blen){ void fix_plus_sign_in_email_address(char *puf, char **at_sign, unsigned int *len){ - int n; char *r; r = strchr(puf, '+'); if(r){ - n = strlen(*at_sign); + int n = strlen(*at_sign); memmove(r, *at_sign, n); *(r+n) = '\0'; *len = strlen(puf); diff --git a/src/pop3.c b/src/pop3.c index ec3a5e2c..c8ab4194 100644 --- a/src/pop3.c +++ b/src/pop3.c @@ -63,7 +63,7 @@ int connect_to_pop3_server(struct data *data){ void get_number_of_total_messages(struct data *data){ - char *p, buf[MAXBUFSIZE]; + char buf[MAXBUFSIZE]; data->import->total_messages = 0; @@ -73,7 +73,7 @@ void get_number_of_total_messages(struct data *data){ recvtimeoutssl(data->net, buf, sizeof(buf)); if(strncmp(buf, "+OK ", 4) == 0){ - p = strchr(&buf[4], ' '); + char *p = strchr(&buf[4], ' '); if(p){ *p = '\0'; data->import->total_messages = atoi(&buf[4]); @@ -86,7 +86,7 @@ void get_number_of_total_messages(struct data *data){ int pop3_download_email(struct data *data, int i){ - int n, fd, pos=0, readlen=0, lastpos=0, nreads=0; + int n, fd, pos=0, lastpos=0, nreads=0; char *p, buf[MAXBUFSIZE]; char aggrbuf[3*MAXBUFSIZE]; @@ -108,7 +108,6 @@ int pop3_download_email(struct data *data, int i){ while((n = recvtimeoutssl(data->net, buf, sizeof(buf))) > 0){ nreads++; - readlen += n; if(nreads == 1){ @@ -167,7 +166,6 @@ void pop3_delete_message(struct data *data, int i){ void process_pop3_emails(struct session_data *sdata, struct data *data, struct config *cfg){ - int i=0, rc=ERR; char buf[MAXBUFSIZE]; data->import->processed_messages = 0; @@ -178,12 +176,12 @@ void process_pop3_emails(struct session_data *sdata, struct data *data, struct c if(data->import->total_messages <= 0) return; - for(i=data->import->start_position; i<=data->import->total_messages; i++){ + for(int i=data->import->start_position; i<=data->import->total_messages; i++){ if(pop3_download_email(data, i) == OK){ if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/data->import->total_messages); fflush(stdout); } if(data->import->dryrun == 0){ - rc = import_message(sdata, data, cfg); + int rc = import_message(sdata, data, cfg); if(data->import->remove_after_import == 1 && rc == OK){ pop3_delete_message(data, i); diff --git a/src/reindex.c b/src/reindex.c index b84a9891..2382d9f5 100644 --- a/src/reindex.c +++ b/src/reindex.c @@ -74,10 +74,7 @@ uint64 get_max_meta_id(struct session_data *sdata){ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *data, uint64 from_id, uint64 to_id, struct config *cfg){ - FILE *f; - char filename[SMALLBUFSIZE]; char s[SMALLBUFSIZE]; - int rc=0; uint64 stored_id=0, reindexed=0, delta; struct parser_state state; struct sql sql; @@ -111,12 +108,12 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da while(p_fetch_results(&sql) == OK){ if(stored_id > 0){ - + char filename[SMALLBUFSIZE]; snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id); - f = fopen(filename, "w"); + FILE *f = fopen(filename, "w"); if(f){ - rc = retrieve_email_from_archive(sdata, f, cfg); + int rc = retrieve_email_from_archive(sdata, f, cfg); fclose(f); if(rc){ @@ -162,7 +159,7 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da int main(int argc, char **argv){ - int c, all=0; + int all=0; uint64 from_id=0, to_id=0, n=0; char *configfile=CONFIG_FILE, *folder=NULL; struct session_data sdata; @@ -171,7 +168,7 @@ int main(int argc, char **argv){ while(1){ - c = getopt(argc, argv, "c:f:t:F:pahv?"); + int c = getopt(argc, argv, "c:f:t:F:pahv?"); if(c == -1) break; @@ -211,7 +208,7 @@ int main(int argc, char **argv){ } - if(all == 0 && (from_id <= 0 || to_id <= 0) ) usage(); + if(all == 0 && (from_id == 0 || to_id == 0) ) usage(); if(!can_i_write_directory(NULL)) __fatal("cannot write current directory!"); diff --git a/src/rules.c b/src/rules.c index d78472e2..ec667420 100644 --- a/src/rules.c +++ b/src/rules.c @@ -196,13 +196,14 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){ int count_match(struct rule *p, struct parser_state *state, int size, int spam){ int ismatch=0; - size_t nmatch=0; ismatch += check_spam_rule(spam, p->spam); ismatch += check_size_rule(size, p->size, p->_size); ismatch += check_attachment_rule(state, p); if(p->compiled == 1){ + size_t nmatch=0; + if(p->emptyfrom == 1){ ismatch += RULE_UNDEF; } @@ -338,7 +339,6 @@ int check_spam_rule(int is_spam, int spam){ int check_attachment_rule(struct parser_state *state, struct rule *rule){ int i; size_t nmatch=0; - int ismatch = 0; // If no attachment rule, then return RULE_UNDEF if(rule->emptyaname == 1 && rule->emptyatype == 1 && rule->attachment_size == 0) return RULE_UNDEF; @@ -348,7 +348,7 @@ int check_attachment_rule(struct parser_state *state, struct rule *rule){ for(i=1; i<=state->n_attachments; i++){ - ismatch = 0; + int ismatch = 0; if(rule->emptyaname == 0){ if(regexec(&(rule->attachment_name), state->attachments[i].filename, nmatch, NULL, 0) == 0) @@ -379,34 +379,32 @@ void initrules(struct node *xhash[]){ void clearrules(struct node *xhash[]){ - struct node *p, *q; + struct node *q; struct rule *rule; q = xhash[0]; while(q != NULL){ - p = q; + struct node *p = q; q = q->r; - if(p){ - if(p->str){ - rule = (struct rule*)p->str; + if(p->str){ + rule = (struct rule*)p->str; - regfree(&(rule->from)); - regfree(&(rule->to)); - regfree(&(rule->subject)); - regfree(&(rule->body)); - regfree(&(rule->attachment_name)); - regfree(&(rule->attachment_type)); + regfree(&(rule->from)); + regfree(&(rule->to)); + regfree(&(rule->subject)); + regfree(&(rule->body)); + regfree(&(rule->attachment_name)); + regfree(&(rule->attachment_type)); - free(rule->rulestr); + free(rule->rulestr); - if(rule->domain) free(rule->domain); + if(rule->domain) free(rule->domain); - free(rule); - } - free(p); + free(rule); } + free(p); } xhash[0] = NULL; diff --git a/src/session.c b/src/session.c index 8d1c7a79..b1cacb7e 100644 --- a/src/session.c +++ b/src/session.c @@ -30,7 +30,6 @@ int is_blocked_by_tcp_wrappers(int sd){ int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct config *cfg){ - char smtp_banner[SMALLBUFSIZE]; int slot; /* @@ -57,6 +56,8 @@ int start_new_session(struct smtp_session **sessions, int socket, int *num_conne sessions[slot] = malloc(sizeof(struct smtp_session)); if(sessions[slot]){ init_smtp_session(sessions[slot], slot, socket, cfg); + + char smtp_banner[SMALLBUFSIZE]; snprintf(smtp_banner, sizeof(smtp_banner)-1, SMTP_RESP_220_BANNER, cfg->hostid); send(socket, smtp_banner, strlen(smtp_banner), 0); @@ -233,14 +234,12 @@ void handle_data(struct smtp_session *session, char *readbuf, int readlen, struc void write_envelope_addresses(struct smtp_session *session, struct config *cfg){ - int i; - char *p, s[SMALLBUFSIZE]; - if(session->fd == -1) return; - for(i=0; inum_of_rcpt_to; i++){ - p = strchr(session->rcptto[i], '@'); + for(int i=0; inum_of_rcpt_to; i++){ + char *p = strchr(session->rcptto[i], '@'); if(p && strncmp(p+1, cfg->hostid, cfg->hostid_len)){ + char s[SMALLBUFSIZE]; snprintf(s, sizeof(s)-1, "X-Piler-Envelope-To: %s\n", session->rcptto[i]); if(write(session->fd, s, strlen(s)) == -1) syslog(LOG_PRIORITY, "ERROR: %s: cannot write envelope to address", session->ttmpfile); } diff --git a/src/smtp.c b/src/smtp.c index 4ce18239..bd40af02 100644 --- a/src/smtp.c +++ b/src/smtp.c @@ -72,15 +72,15 @@ void process_smtp_command(struct smtp_session *session, char *buf, struct config void process_data(struct smtp_session *session, char *buf, int buflen){ - int len=0, written=0, n_writes=0; - if(session->last_data_char == '\n' && strcmp(buf, ".\r\n") == 0){ process_command_period(session); } else { // write line to file + int written=0, n_writes=0; + while(written < buflen) { - len = write(session->fd, buf+written, buflen-written); + int len = write(session->fd, buf+written, buflen-written); n_writes++; if(len > 0){ @@ -98,7 +98,6 @@ void process_data(struct smtp_session *session, char *buf, int buflen){ void wait_for_ssl_accept(struct smtp_session *session){ int rc; - char ssl_error[SMALLBUFSIZE]; if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "waiting for ssl handshake"); @@ -115,6 +114,8 @@ void wait_for_ssl_accept(struct smtp_session *session){ } if(session->cfg->verbosity >= _LOG_DEBUG || session->net.use_ssl == 0){ + char ssl_error[SMALLBUFSIZE]; + ERR_error_string_n(ERR_get_error(), ssl_error, SMALLBUFSIZE); syslog(LOG_PRIORITY, "SSL_accept() result, rc=%d, errorcode: %d, error text: %s", rc, SSL_get_error(session->net.ssl, rc), ssl_error); diff --git a/src/stats.c b/src/stats.c index 68454a84..4b75db71 100644 --- a/src/stats.c +++ b/src/stats.c @@ -134,15 +134,15 @@ void sphinx_queries(struct session_data *sdata, struct stats *stats){ void count_error_emails(struct stats *stats){ DIR *dir; - struct dirent *de; struct stat st; - char buf[SMALLBUFSIZE]; dir = opendir(ERROR_DIR); if(dir){ + struct dirent *de; while((de = readdir(dir))){ if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue; + char buf[SMALLBUFSIZE]; snprintf(buf, sizeof(buf)-1, "%s/%s", ERROR_DIR, de->d_name); if(stat(buf, &st) == 0 && S_ISREG(st.st_mode)){ diff --git a/src/store.c b/src/store.c index dc3839a1..d903282d 100644 --- a/src/store.c +++ b/src/store.c @@ -213,12 +213,11 @@ ENDE: int remove_stored_message_files(struct session_data *sdata, struct parser_state *state, struct config *cfg){ - int i; char s[SMALLBUFSIZE]; if(state->n_attachments > 0){ - for(i=1; i<=state->n_attachments; i++){ + for(int i=1; i<=state->n_attachments; i++){ snprintf(s, sizeof(s)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, sdata->ttmpfile[8], sdata->ttmpfile[9], sdata->ttmpfile[10], sdata->ttmpfile[RND_STR_LEN-4], sdata->ttmpfile[RND_STR_LEN-3], sdata->ttmpfile[RND_STR_LEN-2], sdata->ttmpfile[RND_STR_LEN-1], sdata->ttmpfile, i); if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: unlinking %s", sdata->ttmpfile, s); diff --git a/src/tai.c b/src/tai.c index 344ee094..00f95465 100644 --- a/src/tai.c +++ b/src/tai.c @@ -4,7 +4,6 @@ #include #include "tai.h" -static char hex[16] = "0123456789abcdef"; void tai_pack(char *s, struct tai *t){ @@ -50,21 +49,3 @@ void taia_now(struct taia *t){ t->nano = 1000 * now.tv_usec + 500; t->atto = 0; } - - -void tai_timestamp(char *s){ - struct tai now; - char nowpack[TAI_PACK]; - int i; - - now.x = 4611686018427387914ULL + (uint64)time((long *) 0); - - tai_pack(nowpack, &now); - - for (i = 0;i < 8;++i) { - *(s+i*2) = hex[(nowpack[i] >> 4) & 15]; - *(s+i*2+1) = hex[nowpack[i] & 15]; - } - - *(s+2*TAI_PACK) = '\0'; -} diff --git a/src/test.c b/src/test.c index 497c77fb..91e2bcfc 100644 --- a/src/test.c +++ b/src/test.c @@ -28,7 +28,7 @@ void usage(){ int main(int argc, char **argv){ - int i, c; + int i; time_t retention_seconds=0; struct stat st; struct session_data sdata; @@ -57,9 +57,9 @@ int main(int argc, char **argv){ int option_index = 0; - c = getopt_long(argc, argv, "c:m:a:hv?", long_options, &option_index); + int c = getopt_long(argc, argv, "c:m:a:hv?", long_options, &option_index); #else - c = getopt(argc, argv, "c:m:a:hv?"); + int c = getopt(argc, argv, "c:m:a:hv?"); #endif if(c == -1) break; diff --git a/suppressions.txt b/suppressions.txt new file mode 100644 index 00000000..a50ff542 --- /dev/null +++ b/suppressions.txt @@ -0,0 +1,13 @@ +identicalConditionAfterEarlyExit:src/tokenizer.c:40 +invalidPrintfArgType_s:src/extract.c:175 +invalidPrintfArgType_s:src/misc.c:39 +invalidPrintfArgType_s:src/misc.c:43 +invalidPrintfArgType_s:src/misc.c:47 +invalidPrintfArgType_s:src/misc.c:51 +invalidPrintfArgType_s:src/misc.c:55 +invalidPrintfArgType_s:src/misc.c:59 +invalidPrintfArgType_s:src/misc.c:63 +redundantAssignment:src/imap.c:47 +unusedFunction:src/sig.c:33 +unusedFunction:src/sig.c:38 +unusedFunction:src/sig.c:44 diff --git a/unit_tests/check_digest.c b/unit_tests/check_digest.c index 23f490a9..9d5bf37d 100644 --- a/unit_tests/check_digest.c +++ b/unit_tests/check_digest.c @@ -5,7 +5,7 @@ #include "test.h" -struct digest_test tests[] = { +struct digest_test emls[] = { {"1.eml", "7fa4f06f7085986007454b374f6685e4cf838d9e9f8878f3cba89cfe98e29f56", "bbbafb22e2d4c035584a5024e9d4feaf32172559b4d7cacc8b7af4bd548da53e"}, {"2.eml", "668cb3b91b944af786667323442576b9813d65f3cd3bc33e9d5da303c79de038", "de90475409dd6ab24e80c1b7a987715c40fe8d28d91337b7f063b477159c7b3c"}, {"3.eml", "0d546d4cb4a8ce74ea5fd4cc51dbb4ebeaa7542f1c691817579da7eeab8d4771", "f585d011340d292ee52ddedb07cda662a8f1e46329d14a2ce92dca0604387bab"}, @@ -41,9 +41,9 @@ static void test_digest_file(){ unsigned int i; char digest[2*DIGEST_LENGTH+1]; - for(i=0; i= -3600, date_test[i].date_str); } diff --git a/unit_tests/smtp.c b/unit_tests/smtp.c index acb6337d..31914c74 100644 --- a/unit_tests/smtp.c +++ b/unit_tests/smtp.c @@ -27,10 +27,10 @@ void connect_to_smtp_server(char *server, int port, struct data *data){ char port_string[8], buf[MAXBUFSIZE]; struct addrinfo hints, *res; - data->net->socket = -1; - if(data == NULL) return; + data->net->socket = -1; + snprintf(port_string, sizeof(port_string)-1, "%d", port); memset(&hints, 0, sizeof(hints)); @@ -301,7 +301,7 @@ static void test_smtp_commands_period_command_in_its_own_packet(char *server, in int main(int argc, char **argv){ - int c, port=25; + int port=25; char *server=NULL; struct data data; struct net net; @@ -325,9 +325,9 @@ int main(int argc, char **argv){ int option_index = 0; - c = getopt_long(argc, argv, "c:s:p:t:lh?", long_options, &option_index); + int c = getopt_long(argc, argv, "c:s:p:t:lh?", long_options, &option_index); #else - c = getopt(argc, argv, "c:s:p:t:lh?"); + int c = getopt(argc, argv, "c:s:p:t:lh?"); #endif From e118feb3ab07a1cc25e03cc988c517294c422b63 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 10 Aug 2020 21:58:06 +0200 Subject: [PATCH 06/55] More refactoring Signed-off-by: Janos SUTO --- docker/Dockerfile | 10 ++++---- src/pilerexport.c | 39 +++++++++++-------------------- src/reindex.c | 58 +++++++++++++++++++++++------------------------ src/rules.c | 2 +- util/imapfetch.py | 1 - util/import.sh | 9 ++++---- util/purge.sh | 7 ++++-- 7 files changed, 58 insertions(+), 68 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e52adc52..baf3fd80 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 ARG PACKAGE @@ -17,13 +17,15 @@ ENV DEBIAN_FRONTEND="noninteractive" \ SPHINX_BIN_TARGZ="sphinx-3.1.1-bin.tar.gz" ADD "https://bitbucket.org/jsuto/piler/downloads/${PACKAGE}" "/${PACKAGE}" -ADD start.sh /start.sh +COPY start.sh /start.sh RUN apt-get update && \ apt-get -y --no-install-recommends install \ wget rsyslog openssl sysstat php7.2-cli php7.2-cgi php7.2-mysql php7.2-fpm php7.2-zip php7.2-ldap \ php7.2-gd php7.2-curl php7.2-xml catdoc unrtf poppler-utils nginx tnef sudo libodbc1 libpq5 libzip4 \ libtre5 libwrap0 cron libmariadb3 libmysqlclient-dev python python-mysqldb mariadb-server && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ service mysql start && mysqladmin -u root password ${MYSQL_ROOT_PASSWORD} && \ wget --no-check-certificate -q -O ${SPHINX_BIN_TARGZ} ${DOWNLOAD_URL}/generic-local/${SPHINX_BIN_TARGZ} && \ tar zxvf ${SPHINX_BIN_TARGZ} && \ @@ -31,8 +33,8 @@ RUN apt-get update && \ sed -i 's/mail.[iwe].*//' /etc/rsyslog.conf && \ sed -i '/session required pam_loginuid.so/c\#session required pam_loginuid.so' /etc/pam.d/cron && \ mkdir /etc/piler && \ - printf "[mysql]\nuser = piler\npassword = ${MYSQL_PILER_PASSWORD}\n" > /etc/piler/.my.cnf && \ - printf "[mysql]\nuser = root\npassword = ${MYSQL_ROOT_PASSWORD}\n" > /root/.my.cnf && \ + printf "[mysql]\nuser = piler\npassword = %s\n" ${MYSQL_PILER_PASSWORD} > /etc/piler/.my.cnf && \ + printf "[mysql]\nuser = root\npassword = %s\n" ${MYSQL_ROOT_PASSWORD} > /root/.my.cnf && \ echo "alias mysql='mysql --defaults-file=/etc/piler/.my.cnf'" > /root/.bashrc && \ echo "alias t='tail -f /var/log/syslog'" >> /root/.bashrc && \ dpkg -i $PACKAGE && \ diff --git a/src/pilerexport.c b/src/pilerexport.c index 1b8de98a..84362de6 100644 --- a/src/pilerexport.c +++ b/src/pilerexport.c @@ -22,6 +22,7 @@ extern int optind; int dryrun = 0; int exportall = 0; +int verification_status = 0; int rc = 0; char *query=NULL; int verbosity = 0; @@ -224,7 +225,6 @@ void export_emails_matching_id_list(struct session_data *sdata, struct session_d int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain, int minsize, int maxsize, unsigned long startdate, unsigned long stopdate){ - int where_condition=1; char s[SMALLBUFSIZE]; if(exportall == 1){ @@ -239,83 +239,69 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain rc = append_string_to_buffer(&query, s); if(from){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); rc += append_string_to_buffer(&query, "`from` IN ("); rc += append_string_to_buffer(&query, from); rc += append_string_to_buffer(&query, ")"); free(from); - - where_condition++; } if(to){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); rc += append_string_to_buffer(&query, "`to` IN ("); rc += append_string_to_buffer(&query, to); rc += append_string_to_buffer(&query, ")"); free(to); - - where_condition++; } if(fromdomain){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); rc += append_string_to_buffer(&query, "`fromdomain` IN ("); rc += append_string_to_buffer(&query, fromdomain); rc += append_string_to_buffer(&query, ")"); free(fromdomain); - - where_condition++; } if(todomain){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); rc += append_string_to_buffer(&query, "`todomain` IN ("); rc += append_string_to_buffer(&query, todomain); rc += append_string_to_buffer(&query, ")"); free(todomain); - - where_condition++; } if(minsize > 0){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); snprintf(s, sizeof(s)-1, " `size` >= %d", minsize); rc += append_string_to_buffer(&query, s); - - where_condition++; } if(maxsize > 0){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); snprintf(s, sizeof(s)-1, " `size` <= %d", maxsize); rc += append_string_to_buffer(&query, s); - - where_condition++; } if(startdate > 0){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); snprintf(s, sizeof(s)-1, " `sent` >= %lu", startdate); rc += append_string_to_buffer(&query, s); - - where_condition++; } if(stopdate > 0){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); snprintf(s, sizeof(s)-1, " `sent` <= %lu", stopdate); rc += append_string_to_buffer(&query, s); } @@ -374,9 +360,10 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0){ printf("exported: %10llu\r", n); fflush(stdout); } - else + else { printf("verification FAILED. %s\n", filename); - + verification_status = 1; + } } else printf("cannot open: %s\n", filename); } @@ -593,5 +580,5 @@ int main(int argc, char **argv){ close_database(&sdata); - return 0; + return verification_status; } diff --git a/src/reindex.c b/src/reindex.c index 2382d9f5..ad04b094 100644 --- a/src/reindex.c +++ b/src/reindex.c @@ -107,45 +107,43 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da while(p_fetch_results(&sql) == OK){ - if(stored_id > 0){ - char filename[SMALLBUFSIZE]; - snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id); + char filename[SMALLBUFSIZE]; + snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id); - FILE *f = fopen(filename, "w"); - if(f){ - int rc = retrieve_email_from_archive(sdata, f, cfg); - fclose(f); - - if(rc){ - printf("cannot retrieve: %s\n", filename); - unlink(filename); - continue; - } - - snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename); - - state = parse_message(sdata, 0, data, cfg); - post_parse(sdata, &state, cfg); - - rc = store_index_data(sdata, &state, data, stored_id, cfg); - - if(rc == OK) reindexed++; - else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename); + FILE *f = fopen(filename, "w"); + if(f){ + int rc = retrieve_email_from_archive(sdata, f, cfg); + fclose(f); + if(rc){ + printf("cannot retrieve: %s\n", filename); unlink(filename); - - if(progressbar){ - printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta)); - fflush(stdout); - } - + continue; + } + + snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename); + + state = parse_message(sdata, 0, data, cfg); + post_parse(sdata, &state, cfg); + + rc = store_index_data(sdata, &state, data, stored_id, cfg); + + if(rc == OK) reindexed++; + else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename); + + unlink(filename); + + if(progressbar){ + printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta)); + fflush(stdout); } - else printf("cannot open: %s\n", filename); } + else printf("cannot open: %s\n", filename); } + p_free_results(&sql); } diff --git a/src/rules.c b/src/rules.c index ec667420..37c8d3cf 100644 --- a/src/rules.c +++ b/src/rules.c @@ -240,7 +240,7 @@ char *check_against_ruleset(struct node *xhash[], struct parser_state *state, in if(q->str){ p = q->str; - if(p && count_match(p, state, size, spam) > 0){ + if(count_match(p, state, size, spam) > 0){ return p->rulestr; } } diff --git a/util/imapfetch.py b/util/imapfetch.py index ed9b7287..f9c98b3d 100755 --- a/util/imapfetch.py +++ b/util/imapfetch.py @@ -3,7 +3,6 @@ import argparse import imaplib -import pprint import re opts = {} diff --git a/util/import.sh b/util/import.sh index 102c0248..33528061 100755 --- a/util/import.sh +++ b/util/import.sh @@ -1,10 +1,13 @@ #!/bin/bash +set -o nounset +set -o errexit +set -o pipefail + export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin -PRIORITY=mail.error TMPFILE=/var/run/piler/import.tmp -if [ -f $TMPFILE ]; then exit 1; fi +if [[ -f $TMPFILE ]]; then exit 1; fi date > $TMPFILE @@ -17,5 +20,3 @@ trap finish EXIT cd /var/piler/imap pilerimport -G >/dev/null - - diff --git a/util/purge.sh b/util/purge.sh index 29ed8bad..7bb2bc20 100644 --- a/util/purge.sh +++ b/util/purge.sh @@ -1,11 +1,14 @@ #!/bin/bash +set -o nounset +set -o errexit +set -o pipefail + export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/libexec/piler:/usr/local/libexec/piler -PRIORITY=mail.error TMPFILE=/var/run/piler/purge.tmp PURGE_BEACON=/var/piler/stat/purge -if [ -f $TMPFILE ]; then exit 1; fi +if [[ -f $TMPFILE ]]; then exit 1; fi date > $TMPFILE From 027d301745ec9195c49e94f5edd0ac8800a0923b Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 10 Aug 2020 22:14:48 +0200 Subject: [PATCH 07/55] updated pipeline Signed-off-by: Janos SUTO --- bitbucket-pipelines.yml | 49 ++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 468c3398..9f0506e1 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -1,22 +1,35 @@ -# This is a sample build configuration for all languages. -# Check our guides at https://confluence.atlassian.com/x/VYk8Lw for more examples. -# Only use spaces to indent your .yml configuration. -# ----- -# You can specify a custom docker image from Docker Hub as your build environment. +image: gcc:6.5 -image: sutoj/builder:bionic clone: - depth: 5 + depth: full # SonarCloud scanner needs the full history to assign issues properly -pipelines: - default: - - step: +definitions: + caches: + sonar: ~/.sonar/cache # Caching SonarCloud artifacts will speed up your build + steps: + - step: &build-test-sonarcloud + name: Build, test and analyze on SonarCloud + caches: + - sonar script: - - service mysql start - - ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --with-database=mariadb - - make clean all install - - mysql -u piler -ppiler123 piler1 < /usr/share/piler/db-mysql.sql - - cd unit_tests - - ./run.sh - - cd .. - - phpunit + - export SONAR_SCANNER_VERSION=4.4.0.2170 + - export SONAR_SCANNER_HOME=$HOME/.sonar/sonar-scanner-$SONAR_SCANNER_VERSION-linux + - export BW_OUTPUT=$HOME/.sonar/bw-output + - mkdir -p $BW_OUTPUT + - curl --create-dirs -sSLo $HOME/.sonar/sonar-scanner.zip https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-$SONAR_SCANNER_VERSION-linux.zip + - unzip -o $HOME/.sonar/sonar-scanner.zip -d $HOME/.sonar/ + - export PATH=$SONAR_SCANNER_HOME/bin:$PATH + - export SONAR_SCANNER_OPTS="-server" + - curl --create-dirs -sSLo $HOME/.sonar/build-wrapper-linux-x86.zip https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip + - unzip -o $HOME/.sonar/sonar/build-wrapper-linux-x86.zip -d $HOME/.sonar/ + - export PATH=$HOME/.sonar/sonar/build-wrapper-linux-x86:$PATH + - build-wrapper-linux-x86-64 --out-dir $BW_OUTPUT + - sonar-scanner -Dsonar.cfamily.build-wrapper-output=$BW_OUTPUT + +pipelines: # More info here: https://confluence.atlassian.com/bitbucket/configure-bitbucket-pipelines-yml-792298910.html + branches: + master: + - step: *build-test-sonarcloud + pull-requests: + '**': + - step: *build-test-sonarcloud From e62e87fe72089ea1f4bae07abaf36054e2eaab78 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 10 Aug 2020 22:16:10 +0200 Subject: [PATCH 08/55] pipeline fix Signed-off-by: Janos SUTO --- bitbucket-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 9f0506e1..7d449612 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -23,7 +23,7 @@ definitions: - curl --create-dirs -sSLo $HOME/.sonar/build-wrapper-linux-x86.zip https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip - unzip -o $HOME/.sonar/sonar/build-wrapper-linux-x86.zip -d $HOME/.sonar/ - export PATH=$HOME/.sonar/sonar/build-wrapper-linux-x86:$PATH - - build-wrapper-linux-x86-64 --out-dir $BW_OUTPUT + - build-wrapper-linux-x86-64 --out-dir $BW_OUTPUT make clean all - sonar-scanner -Dsonar.cfamily.build-wrapper-output=$BW_OUTPUT pipelines: # More info here: https://confluence.atlassian.com/bitbucket/configure-bitbucket-pipelines-yml-792298910.html From 38031b731b1b983dc4950bfde8af3b75f431eeb4 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 10 Aug 2020 22:26:53 +0200 Subject: [PATCH 09/55] Revert "pipeline fix" This reverts commit e62e87fe72089ea1f4bae07abaf36054e2eaab78. --- bitbucket-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 7d449612..9f0506e1 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -23,7 +23,7 @@ definitions: - curl --create-dirs -sSLo $HOME/.sonar/build-wrapper-linux-x86.zip https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip - unzip -o $HOME/.sonar/sonar/build-wrapper-linux-x86.zip -d $HOME/.sonar/ - export PATH=$HOME/.sonar/sonar/build-wrapper-linux-x86:$PATH - - build-wrapper-linux-x86-64 --out-dir $BW_OUTPUT make clean all + - build-wrapper-linux-x86-64 --out-dir $BW_OUTPUT - sonar-scanner -Dsonar.cfamily.build-wrapper-output=$BW_OUTPUT pipelines: # More info here: https://confluence.atlassian.com/bitbucket/configure-bitbucket-pipelines-yml-792298910.html From cbe283357b7462116f5d26330090960a0e9c27ac Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 10 Aug 2020 22:27:02 +0200 Subject: [PATCH 10/55] Revert "updated pipeline" This reverts commit 027d301745ec9195c49e94f5edd0ac8800a0923b. --- bitbucket-pipelines.yml | 49 +++++++++++++++-------------------------- 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 9f0506e1..468c3398 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -1,35 +1,22 @@ -image: gcc:6.5 +# This is a sample build configuration for all languages. +# Check our guides at https://confluence.atlassian.com/x/VYk8Lw for more examples. +# Only use spaces to indent your .yml configuration. +# ----- +# You can specify a custom docker image from Docker Hub as your build environment. +image: sutoj/builder:bionic clone: - depth: full # SonarCloud scanner needs the full history to assign issues properly + depth: 5 -definitions: - caches: - sonar: ~/.sonar/cache # Caching SonarCloud artifacts will speed up your build - steps: - - step: &build-test-sonarcloud - name: Build, test and analyze on SonarCloud - caches: - - sonar +pipelines: + default: + - step: script: - - export SONAR_SCANNER_VERSION=4.4.0.2170 - - export SONAR_SCANNER_HOME=$HOME/.sonar/sonar-scanner-$SONAR_SCANNER_VERSION-linux - - export BW_OUTPUT=$HOME/.sonar/bw-output - - mkdir -p $BW_OUTPUT - - curl --create-dirs -sSLo $HOME/.sonar/sonar-scanner.zip https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-$SONAR_SCANNER_VERSION-linux.zip - - unzip -o $HOME/.sonar/sonar-scanner.zip -d $HOME/.sonar/ - - export PATH=$SONAR_SCANNER_HOME/bin:$PATH - - export SONAR_SCANNER_OPTS="-server" - - curl --create-dirs -sSLo $HOME/.sonar/build-wrapper-linux-x86.zip https://sonarcloud.io/static/cpp/build-wrapper-linux-x86.zip - - unzip -o $HOME/.sonar/sonar/build-wrapper-linux-x86.zip -d $HOME/.sonar/ - - export PATH=$HOME/.sonar/sonar/build-wrapper-linux-x86:$PATH - - build-wrapper-linux-x86-64 --out-dir $BW_OUTPUT - - sonar-scanner -Dsonar.cfamily.build-wrapper-output=$BW_OUTPUT - -pipelines: # More info here: https://confluence.atlassian.com/bitbucket/configure-bitbucket-pipelines-yml-792298910.html - branches: - master: - - step: *build-test-sonarcloud - pull-requests: - '**': - - step: *build-test-sonarcloud + - service mysql start + - ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --with-database=mariadb + - make clean all install + - mysql -u piler -ppiler123 piler1 < /usr/share/piler/db-mysql.sql + - cd unit_tests + - ./run.sh + - cd .. + - phpunit From d9ed819b2b5c6daf94a1923c561334ebd6fea8ad Mon Sep 17 00:00:00 2001 From: Kevin Lieser Date: Mon, 10 Aug 2020 21:04:55 +0000 Subject: [PATCH 11/55] [BUGFIX] Render multiple mail parts in mail view instead of only the last part --- webui/model/search/message.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webui/model/search/message.php b/webui/model/search/message.php index 0178f8f6..670f2245 100644 --- a/webui/model/search/message.php +++ b/webui/model/search/message.php @@ -185,10 +185,10 @@ class ModelSearchMessage extends Model { $body = Piler_Mime_Decode::fixMimeBodyPart($parts[$i]['headers'], $parts[$i]['body']); if($parts[$i]['headers']['content-type']['type'] == 'text/html') { - $this->message['text/html'] = $purifier->purify($body); + $this->message['text/html'] .= $purifier->purify($body); } else { - $this->message['text/plain'] = $body; + $this->message['text/plain'] .= $body; } } From f514d68bd1d602dd3de8e75a76e7661ef43bf6dc Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 11 Aug 2020 20:36:18 +0200 Subject: [PATCH 12/55] Added a separator to searching for attachment names Signed-off-by: Janos SUTO --- src/parser_utils.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/parser_utils.c b/src/parser_utils.c index 7481f2db..45e03048 100644 --- a/src/parser_utils.c +++ b/src/parser_utils.c @@ -1063,8 +1063,13 @@ void fill_attachment_name_buf(struct parser_state *state, char *buf){ int len = strlen(p); - if(len + state->anamepos < SMALLBUFSIZE-2){ + if(len + state->anamepos < SMALLBUFSIZE-3){ memcpy(&(state->attachment_name_buf[state->anamepos]), p, len); state->anamepos += len; + + // add a trailing separator semicolon to make sure there's separation + // with the next item + state->attachment_name_buf[state->anamepos] = ';'; + state->anamepos++; } } From 137671d3fae1d79a2304dbeb357dabbea8faaf06 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Wed, 12 Aug 2020 12:05:07 +0200 Subject: [PATCH 13/55] Fixed counters in test cases to match the new number of test emails Signed-off-by: Janos SUTO --- tests/cases/05-smtp.inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cases/05-smtp.inc b/tests/cases/05-smtp.inc index e6e26230..52a06208 100644 --- a/tests/cases/05-smtp.inc +++ b/tests/cases/05-smtp.inc @@ -17,10 +17,10 @@ case1() { "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus" --socket --no-counter - wait_until_emails_are_processed "piler1" 2999 + wait_until_emails_are_processed "piler1" 3000 docker exec "piler1" su piler -c /usr/libexec/piler/indexer.delta.sh 2>/dev/null - count_status_values 2999 2891 108 0 + count_status_values 3000 2892 108 0 test_retrieved_messages_are_the_same "piler1" "piler" From a38f757ca9ca72b66e2dfd8084c1c9d892297c63 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 17 Aug 2020 10:41:31 +0200 Subject: [PATCH 14/55] Released 1.3.9 Signed-off-by: Janos SUTO --- RELEASE_NOTES | 19 +++++++++++++++++++ VERSION | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 8006e008..5b84e188 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,3 +1,22 @@ +1.3.9: +------ + +- Added a separator to searching for attachment names +- [BUGFIX] Render multiple mail parts in mail view instead of only the last part +- Use TLS v1.2 with openssl 1.0.x for connecting remote pop3/imap servers +- Instant search results to the gui when the search page loads +- Support sphinx-3.3.1, introduced sphinx strict mode variable +- GUI domain fixes +- gcc 9 fixes +- Fix permission on sphinx data dir to 700 +- pilerpurge.py should honor the mysqlhost value +- Password change enabled by default +- Health page fixes +- GUI mime parser fixes +- Start/stop script fix +- Optimized search page for mobile devices, set $config['ENABLE_MOBILE_PREVIEW'] = 1; in config-site.php to enable it + + 1.3.0: ----- diff --git a/VERSION b/VERSION index e05cb332..d4c4950a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.8 +1.3.9 From ee39e64ffb3c97194c1deea6b791209fca2cf164 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 17 Aug 2020 10:42:01 +0200 Subject: [PATCH 15/55] Refactored init.d/rc.piler.in Signed-off-by: Janos SUTO --- init.d/rc.piler.in | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/init.d/rc.piler.in b/init.d/rc.piler.in index 7db00536..b280a367 100644 --- a/init.d/rc.piler.in +++ b/init.d/rc.piler.in @@ -57,11 +57,10 @@ case "$1" in ;; status) - if check_status; - then - exit 0 - else - exit 1 + if check_status; then + exit 0 + else + exit 1 fi ;; From f2e9dc834328c48e89283c8fa66d5d1a0750ac84 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 17 Aug 2020 20:32:42 +0200 Subject: [PATCH 16/55] Made strict sphinx schema the default Signed-off-by: Janos SUTO --- config.php.in | 2 +- etc/sphinx.conf.in | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config.php.in b/config.php.in index f7903b79..146b2d18 100644 --- a/config.php.in +++ b/config.php.in @@ -238,7 +238,7 @@ $config['SPHINX_MAIN_INDEX'] = 'main1,dailydelta1,delta1'; $config['SPHINX_ATTACHMENT_INDEX'] = 'att1'; $config['SPHINX_TAG_INDEX'] = 'tag1'; $config['SPHINX_NOTE_INDEX'] = 'note1'; -$config['SPHINX_STRICT_SCHEMA'] = 0; +$config['SPHINX_STRICT_SCHEMA'] = 1; $config['RELOAD_COMMAND'] = 'sudo -n /etc/init.d/rc.piler reload'; $config['PILERIMPORT_IMAP_COMMAND'] = '/usr/local/bin/pilerimport -d /var/piler/imap -q -r'; diff --git a/etc/sphinx.conf.in b/etc/sphinx.conf.in index cb0a8a62..a4b21591 100644 --- a/etc/sphinx.conf.in +++ b/etc/sphinx.conf.in @@ -1,7 +1,7 @@ #!/usr/bin/php Date: Wed, 9 Sep 2020 21:20:59 +0200 Subject: [PATCH 17/55] Added folder query to imap and pop3 authentication Signed-off-by: Janos SUTO --- webui/model/user/auth.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/webui/model/user/auth.php b/webui/model/user/auth.php index 313ae4f3..e102e231 100644 --- a/webui/model/user/auth.php +++ b/webui/model/user/auth.php @@ -394,6 +394,8 @@ class ModelUserAuth extends Model { $data = $this->fix_user_data($username, $username, $emails, 0); + $data['folders'] = $this->model_folder_folder->get_folder_id_array_for_user($data['uid'], 0); + $this->is_ga_code_needed($username); $session->set("auth_data", $data); @@ -429,6 +431,8 @@ class ModelUserAuth extends Model { $data = $this->fix_user_data($username, $username, $emails, 0); + $data['folders'] = $this->model_folder_folder->get_folder_id_array_for_user($data['uid'], 0); + $this->is_ga_code_needed($username); $session = Registry::get('session'); From 6ce7efd1ed9e40d846994701b0736800a5ded360 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 17 Sep 2020 22:44:41 +0200 Subject: [PATCH 18/55] Exclude 42+ email addresses from sphinx query Signed-off-by: Janos SUTO --- config.php.in | 1 + webui/system/misc.php | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/config.php.in b/config.php.in index 146b2d18..6eb8e6c1 100644 --- a/config.php.in +++ b/config.php.in @@ -239,6 +239,7 @@ $config['SPHINX_ATTACHMENT_INDEX'] = 'att1'; $config['SPHINX_TAG_INDEX'] = 'tag1'; $config['SPHINX_NOTE_INDEX'] = 'note1'; $config['SPHINX_STRICT_SCHEMA'] = 1; +$config['MAX_EMAIL_LEN'] = 41; $config['RELOAD_COMMAND'] = 'sudo -n /etc/init.d/rc.piler reload'; $config['PILERIMPORT_IMAP_COMMAND'] = '/usr/local/bin/pilerimport -d /var/piler/imap -q -r'; diff --git a/webui/system/misc.php b/webui/system/misc.php index 0b516085..0056eaa1 100644 --- a/webui/system/misc.php +++ b/webui/system/misc.php @@ -142,6 +142,16 @@ function checkemail($email, $domains) { function validemail($email = '') { if($email == '') { return 0; } + // sphinxsearch supports tokens up to 41 characters long + // If there's a longer token in the query, then sphinx + // reports a query error even if the query is itself correct + // So the workaround is to get rid of these email addresses + if(strlen($email) > MAX_EMAIL_LEN) { + $msg = sprintf("discarding email %s: longer than %d", $email, MAX_EMAIL_LEN); + syslog(LOG_INFO, $msg); + return 0; + } + if(preg_match("/@local$/", $email)) { return 1; } if(preg_match('/^[_a-zA-Z0-9-]+(\.[_a-zA-Z0-9-]+)*@[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)*(\.[a-zA-Z]{2,10})$/', $email)) { From 2a51a440340f1ed504bed29220f4dcfa8cc097a5 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 17 Sep 2020 22:45:47 +0200 Subject: [PATCH 19/55] validemail() to model/user/user.php Signed-off-by: Janos SUTO --- webui/model/user/user.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui/model/user/user.php b/webui/model/user/user.php index 03a3a196..fe111f4c 100644 --- a/webui/model/user/user.php +++ b/webui/model/user/user.php @@ -84,7 +84,7 @@ class ModelUserUser extends Model { if(isset($query->rows)) { foreach ($query->rows as $q) { - if(!in_array($q['email'], $data)) { array_push($data, $q['email']); } + if(validemail($q['email']) && !in_array($q['email'], $data)) { array_push($data, $q['email']); } } } From 6bb5c52d56f8d1b40aa7f29e7bcc12e4eb84e12a Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 17 Sep 2020 22:48:37 +0200 Subject: [PATCH 20/55] Added support for uninvention Signed-off-by: Janos SUTO --- config.php.in | 8 +++++++- webui/model/user/auth.php | 5 +++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/config.php.in b/config.php.in index 6eb8e6c1..020769dd 100644 --- a/config.php.in +++ b/config.php.in @@ -115,7 +115,11 @@ $config['LDAP_MAIL_ATTR'] = 'proxyAddresses'; //$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'memberOfGroup'; //$config['LDAP_MAIL_ATTR'] = 'mail'; - +// Uninvention specific settings +//$config['LDAP_MAIL_ATTR'] = 'mailPrimaryAddress'; +//$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'person'; +//$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'person'; +//$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mailAlternativeAddress'; // enable single sign-on (disabled by default) @@ -311,6 +315,8 @@ $config['DELIMITER'] = "\t"; $config['TRACKING_CODE'] = ''; +$mailattrs = ["mail", "mailalternateaddress", "proxyaddresses", "zimbraMailForwardingAddress", "member", "memberOfGroup", "othermailbox", "mailprimaryaddress", "mailalternativeaddress"]; + $langs = array( 'cz', 'de', diff --git a/webui/model/user/auth.php b/webui/model/user/auth.php index e102e231..d9cc383f 100644 --- a/webui/model/user/auth.php +++ b/webui/model/user/auth.php @@ -293,12 +293,13 @@ class ModelUserAuth extends Model { public function get_email_array_from_ldap_attr($e = array()) { - $data = array(); + global $mailattrs; + $data = []; foreach($e as $a) { if(LOG_LEVEL >= DEBUG) { syslog(LOG_INFO, "checking ldap entry dn: " . $a['dn'] . ", cn: " . $a['cn']); } - foreach (array("mail", "mailalternateaddress", "proxyaddresses", "zimbraMailForwardingAddress", "member", "memberOfGroup", "othermailbox") as $mailattr) { + foreach ($mailattrs as $mailattr) { if(isset($a[$mailattr])) { if(is_array($a[$mailattr])) { From f7a1318b0320fb75dd4c12d2f2aa8931e41a311b Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 17 Sep 2020 22:49:13 +0200 Subject: [PATCH 21/55] Added -Wimplicit-fallthrough=2 to CFLAGS Signed-off-by: Janos SUTO --- configure | 2 +- configure.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 19428bde..016ca71b 100755 --- a/configure +++ b/configure @@ -4852,7 +4852,7 @@ if test $? -eq 1; then echo "the user \"$RUNNING_USER\" does not exists, please echo; echo -CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wuninitialized -Wno-format-truncation -g" +CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wimplicit-fallthrough=2 -Wuninitialized -Wno-format-truncation -g" LIBS="$antispam_libs $sunos_libs " OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" diff --git a/configure.in b/configure.in index 34bbebcc..554eaee2 100644 --- a/configure.in +++ b/configure.in @@ -535,7 +535,7 @@ if test $? -eq 1; then echo "the user \"$RUNNING_USER\" does not exists, please echo; echo -CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wuninitialized -Wno-format-truncation -g" +CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wimplicit-fallthrough=2 -Wuninitialized -Wno-format-truncation -g" LIBS="$antispam_libs $sunos_libs " OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" From 39fd0899b083faeecd137d904bb1370b458e3ce3 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 15 Oct 2020 14:43:32 +0200 Subject: [PATCH 22/55] Fixed docker setup to listen on 127.0.0.1:80 Signed-off-by: Janos SUTO --- tests/setup.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/setup.inc b/tests/setup.inc index 1df725b1..2cf689ed 100644 --- a/tests/setup.inc +++ b/tests/setup.inc @@ -22,7 +22,7 @@ launch_containers() { docker run -d --net=piler "${DOCKER_LIMIT[@]}" --name piler1 \ -e PACKAGE="$PACKAGE" \ -e PILER_HOST="cust1.acts.hu" \ - -p 80:80 -p 25:25 \ + -p 127.0.0.1:80:80 -p 25:25 \ -v "${PACKAGE_DIR}:/data:ro" \ -v "${CONFIG_DIR}/11-aaaa.conf:/etc/rsyslog.d/11-aaaa.conf:ro" \ "$docker_image" From 4a131f3058766ecdeb3021f5478746f58938c2c8 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 17 Oct 2020 20:03:07 +0200 Subject: [PATCH 23/55] TSA fixes Signed-off-by: Janos SUTO --- util/sign.php | 38 ++++++------- webui/system/helper/TrustedTimestamps.php | 65 +++++++++++++---------- 2 files changed, 55 insertions(+), 48 deletions(-) diff --git a/util/sign.php b/util/sign.php index 25d69502..37a508fb 100644 --- a/util/sign.php +++ b/util/sign.php @@ -1,11 +1,11 @@ $start_id, - STOP_ID => $stop_id, - COUNT => $count, - HASH_VALUE => sha1($s) - ); + return [ + START_ID => $start_id, + STOP_ID => $stop_id, + COUNT => $count, + HASH_VALUE => sha1($s) + ]; } diff --git a/webui/system/helper/TrustedTimestamps.php b/webui/system/helper/TrustedTimestamps.php index 6ad00d65..5a218e9f 100644 --- a/webui/system/helper/TrustedTimestamps.php +++ b/webui/system/helper/TrustedTimestamps.php @@ -32,7 +32,7 @@ class TrustedTimestamps { if (strlen($hash) !== 40) throw new Exception("Invalid Hash."); - + $outfilepath = self::createTempFile(); $cmd = OPENSSL_BINARY . " ts -query -digest ".escapeshellarg($hash)." -cert -out ".escapeshellarg($outfilepath); @@ -41,7 +41,7 @@ class TrustedTimestamps if ($retcode !== 0) throw new Exception("OpenSSL does not seem to be installed: ".implode(", ", $retarray)); - + if (count($retarray) > 0 && stripos($retarray[0], "openssl:Error") !== false) throw new Exception("There was an error with OpenSSL. Is version >= 0.99 installed?: ".implode(", ", $retarray)); @@ -72,14 +72,14 @@ class TrustedTimestamps $binary_response_string = curl_exec($ch); $status = curl_getinfo($ch, CURLINFO_HTTP_CODE); curl_close($ch); - + if ($status != 200 || !strlen($binary_response_string)) throw new Exception("The request failed"); - + $base64_response_string = base64_encode($binary_response_string); - + $response_time = self::getTimestampFromAnswer ($base64_response_string); - + return array("response_string" => $base64_response_string, "response_time" => $response_time); } @@ -97,13 +97,13 @@ class TrustedTimestamps $responsefile = self::createTempFile($binary_response_string); $cmd = OPENSSL_BINARY . " ts -reply -in ".escapeshellarg($responsefile)." -text"; - + $retarray = array(); exec($cmd." 2>&1", $retarray, $retcode); - + if ($retcode !== 0) throw new Exception("The reply failed: ".implode(", ", $retarray)); - + $matches = array(); $response_time = 0; @@ -119,13 +119,13 @@ class TrustedTimestamps if (preg_match("~^Time\sstamp\:\s(.*)~", $retline, $matches)) { $response_time = strtotime($matches[1]); - break; + break; } } if (!$response_time) throw new Exception("The Timestamp was not found"); - + return $response_time; } @@ -141,25 +141,25 @@ class TrustedTimestamps { if (strlen($hash) !== 40) throw new Exception("Invalid Hash"); - + $binary_response_string = base64_decode($base64_response_string); - + if (!strlen($binary_response_string)) - throw new Exception("There was no response-string"); - + throw new Exception("There was no response-string"); + if (!intval($response_time)) throw new Exception("There is no valid response-time given"); - + if (!file_exists($tsa_cert_file)) throw new Exception("The TSA-Certificate could not be found"); - + $responsefile = self::createTempFile($binary_response_string); $cmd = OPENSSL_BINARY . " ts -verify -digest ".escapeshellarg($hash)." -in ".escapeshellarg($responsefile)." -CAfile ".escapeshellarg($tsa_cert_file); - + $retarray = array(); exec($cmd." 2>&1", $retarray, $retcode); - + /* * just 2 "normal" cases: * 1) Everything okay -> retcode 0 + retarray[0] == "Verification: OK" @@ -167,14 +167,23 @@ class TrustedTimestamps * * every other case (Certificate not found / invalid / openssl is not installed / ts command not known) * are being handled the same way -> retcode 1 + any retarray NOT containing "message imprint mismatch" + * + * For openssl 1.1.x it's 2 lines actually: + * + * Using configuration from /usr/lib/ssl/openssl.cnf + * Verification: OK + * */ - - if ($retcode === 0 && strtolower(trim($retarray[0])) == "verification: ok") - { - if (self::getTimestampFromAnswer ($base64_response_string) != $response_time) - throw new Exception("The responsetime of the request was changed"); - - return true; + + if ($retcode === 0) { + foreach ($retarray as $line) { + if(strtolower(trim($line)) == "verification: ok") { + if (self::getTimestampFromAnswer ($base64_response_string) != $response_time) + throw new Exception("The responsetime of the request was changed"); + + return true; + } + } } foreach ($retarray as $retline) @@ -198,12 +207,10 @@ class TrustedTimestamps if (!file_exists($tempfilename)) throw new Exception("Tempfile could not be created"); - + if (!empty($str) && !file_put_contents($tempfilename, $str)) throw new Exception("Could not write to tempfile"); return $tempfilename; } } - -?> From 427114b89202e3658664540e2eb5d30feeba659b Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 17 Oct 2020 20:33:00 +0200 Subject: [PATCH 24/55] More TSA fixes Signed-off-by: Janos SUTO --- config.php.in | 1 + util/sign.php | 9 ++++++--- webui/system/helper/TrustedTimestamps.php | 7 ++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/config.php.in b/config.php.in index 020769dd..94aed8ca 100644 --- a/config.php.in +++ b/config.php.in @@ -226,6 +226,7 @@ $config['TSA_URL'] = ''; $config['TSA_PUBLIC_KEY_FILE'] = ''; $config['TSA_START_ID'] = 1; $config['TSA_STAMP_REQUEST_UNIT_SIZE'] = 10000; +$config['TSA_VERIFY_CERTIFICATE'] = true; $config['DB_DRIVER'] = 'mysql'; $config['DB_PREFIX'] = ''; diff --git a/util/sign.php b/util/sign.php index 37a508fb..2903121f 100644 --- a/util/sign.php +++ b/util/sign.php @@ -82,9 +82,12 @@ if(MODE == 'time' && $data[COUNT] < 1) { exit; } -$requestfile_path = TrustedTimestamps::createRequestfile($data[HASH_VALUE]); - -$response = TrustedTimestamps::signRequestfile($requestfile_path, TSA_URL); +try { + $requestfile_path = TrustedTimestamps::createRequestfile($data[HASH_VALUE]); + $response = TrustedTimestamps::signRequestfile($requestfile_path, TSA_URL); +} catch(Exception $e) { + die("Error: " . $e->getMessage() . "\n"); +} $data[RESPONSE_STRING] = $response[RESPONSE_STRING]; $data[RESPONSE_TIME] = $response[RESPONSE_TIME]; diff --git a/webui/system/helper/TrustedTimestamps.php b/webui/system/helper/TrustedTimestamps.php index 5a218e9f..9dc702e5 100644 --- a/webui/system/helper/TrustedTimestamps.php +++ b/webui/system/helper/TrustedTimestamps.php @@ -69,12 +69,17 @@ class TrustedTimestamps curl_setopt($ch, CURLOPT_POSTFIELDS, file_get_contents($requestfile_path)); curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/timestamp-query')); curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)"); + curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, TSA_VERIFY_CERTIFICATE); + $binary_response_string = curl_exec($ch); + + $error = curl_error($ch); $status = curl_getinfo($ch, CURLINFO_HTTP_CODE); + curl_close($ch); if ($status != 200 || !strlen($binary_response_string)) - throw new Exception("The request failed"); + throw new Exception("The request failed. Status: $status, error: $error"); $base64_response_string = base64_encode($binary_response_string); From 7ca0986abbf8d152195a77eb6d657f33f9be123b Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 17 Oct 2020 20:33:34 +0200 Subject: [PATCH 25/55] Fixed inline image url Signed-off-by: Janos SUTO --- webui/view/theme/default/templates/message/view.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webui/view/theme/default/templates/message/view.tpl b/webui/view/theme/default/templates/message/view.tpl index 100ce9aa..84b8a6b2 100644 --- a/webui/view/theme/default/templates/message/view.tpl +++ b/webui/view/theme/default/templates/message/view.tpl @@ -38,7 +38,7 @@ |   - @@ -81,7 +81,7 @@ -

+

From 4682697453a61b0f27acb3cfdafe0aef96fd4f80 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 17 Oct 2020 20:47:56 +0200 Subject: [PATCH 26/55] Fixed test case to sign the archived messages Signed-off-by: Janos SUTO --- tests/cases/05-smtp.inc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/cases/05-smtp.inc b/tests/cases/05-smtp.inc index 52a06208..94a46aed 100644 --- a/tests/cases/05-smtp.inc +++ b/tests/cases/05-smtp.inc @@ -25,6 +25,8 @@ case1() { test_retrieved_messages_are_the_same "piler1" "piler" run_05_sphinx_tests + + docker exec "piler1" su piler -c 'php /usr/libexec/piler/sign.php --webui /var/piler/www --mode time' } From d60b53c2432886bdfe98185a7efe123413335ddc Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 24 Oct 2020 18:46:53 +0200 Subject: [PATCH 27/55] Removed "aname" search prefix Signed-off-by: Janos SUTO --- webui/controller/search/helper.php | 3 +-- webui/model/search/search.php | 21 +-------------------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/webui/controller/search/helper.php b/webui/controller/search/helper.php index 18b78f9c..fc5775a7 100644 --- a/webui/controller/search/helper.php +++ b/webui/controller/search/helper.php @@ -8,7 +8,6 @@ class ControllerSearchHelper extends Controller { 'date2' => '', 'direction' => '', 'size' => '', - 'aname' => '', 'attachment_type' => '', 'tag' => '', 'note' => '', @@ -50,7 +49,7 @@ class ControllerSearchHelper extends Controller { if($this->request->post['searchtype'] == 'expert'){ - if(isset($this->request->post['search']) && preg_match("/(from|to|subject|body|direction|d|size|date1|date2|attachment|a|aname|tag|note|id)\:/", $this->request->post['search'])) { + if(isset($this->request->post['search']) && preg_match("/(from|to|subject|body|direction|d|size|date1|date2|attachment|a|tag|note|id)\:/", $this->request->post['search'])) { $this->a = $this->model_search_search->preprocess_post_expert_request($this->request->post); } else { diff --git a/webui/model/search/search.php b/webui/model/search/search.php index bca9a8d1..f92647cf 100644 --- a/webui/model/search/search.php +++ b/webui/model/search/search.php @@ -205,17 +205,7 @@ class ModelSearchSearch extends Model { } - if(isset($data['aname']) && $data['aname']) { - - $match = $data['aname']; - if($emailfilter) { $match = "( $match ) & $emailfilter"; } - - $query = $this->sphx->query("SELECT id, mid FROM " . SPHINX_ATTACHMENT_INDEX . " WHERE MATCH('" . $match . "') ORDER BY `id` $order LIMIT $offset,$pagelen OPTION max_matches=" . MAX_SEARCH_HITS); - - $total_found = $query->total_found; - $num_rows = $query->num_rows; - } - else if(isset($data['tag']) && $data['tag']) { + if(isset($data['tag']) && $data['tag']) { list ($total_found, $num_rows, $id_list) = $this->get_sphinx_id_list($data['tag'], SPHINX_TAG_INDEX, 'tag', $page); $query = $this->sphx->query("SELECT id FROM " . SPHINX_MAIN_INDEX . " WHERE $folders id IN ($id_list) $sortorder LIMIT 0,$pagelen OPTION max_matches=" . MAX_SEARCH_HITS); } @@ -324,7 +314,6 @@ class ModelSearchSearch extends Model { 'date2' => '', 'direction' => '', 'size' => '', - 'aname' => '', 'attachment_type' => '', 'tag' => '', 'note' => '', @@ -360,7 +349,6 @@ class ModelSearchSearch extends Model { else if($v == 'date1:') { $token = 'date1'; continue; } else if($v == 'date2:') { $token = 'date2'; continue; } else if($v == 'attachment:' || $v == 'a:') { $token = 'match'; $a['match'][] = '@attachment_types'; continue; } - else if($v == 'aname:') { $token = 'aname'; continue; } else if($v == 'size') { $token = 'size'; continue; } else if($v == 'tag:') { $token = 'tag'; continue; } else if($v == 'note:') { $token = 'note'; continue; } @@ -374,13 +362,6 @@ class ModelSearchSearch extends Model { } if($token == 'match') { $a['match'][] = $v; } - else if($token == 'aname') { - if($v != '|') { - $a['aname'] .= '"' . $v . '"'; - } else { - $a['aname'] .= ' | '; - } - } else if($token == 'date1') { $a['date1'] = ' ' . $v; } else if($token == 'date2') { $a['date2'] = ' ' . $v; } else if($token == 'tag') { $a['tag'] .= ' ' . $v; } From 2d084e17d8a564ad4cb05c682b01bd7860bae291 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 24 Oct 2020 18:47:10 +0200 Subject: [PATCH 28/55] Attachment filename is added to the indexed body Signed-off-by: Janos SUTO --- src/parser.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/parser.c b/src/parser.c index 08fb682a..3329effd 100644 --- a/src/parser.c +++ b/src/parser.c @@ -99,12 +99,21 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c for(i=1; i<=state->n_attachments; i++){ + char puf[SMALLBUFSIZE]; + snprintf(puf, sizeof(puf)-1, "%s ", state->attachments[i].filename); + + unsigned int len = strlen(puf); + if(state->bodylen < BIGBUFSIZE-len-1){ + memcpy(&(state->b_body[state->bodylen]), puf, len); + state->bodylen += len; + } + digest_file(state->attachments[i].internalname, &(state->attachments[i].digest[0])); if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest); char *p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type); - unsigned int len = strlen(p); + len = strlen(p); if(strlen(sdata->attachments) < SMALLBUFSIZE-len-1 && !strstr(sdata->attachments, p)) memcpy(&(sdata->attachments[strlen(sdata->attachments)]), p, len); if(state->attachments[i].dumped == 1){ From 98a990ebf1b728fb20b358a4d4b17a14f3c6e942 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 26 Oct 2020 20:24:42 +0100 Subject: [PATCH 29/55] Fixed test case Signed-off-by: Janos SUTO --- tests/cases/05-smtp.inc | 2 ++ tests/setup.inc | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/cases/05-smtp.inc b/tests/cases/05-smtp.inc index 94a46aed..a53ed2f1 100644 --- a/tests/cases/05-smtp.inc +++ b/tests/cases/05-smtp.inc @@ -26,6 +26,8 @@ case1() { run_05_sphinx_tests + docker exec "piler1" su piler -c 'php /usr/libexec/piler/generate_stats.php --webui /var/piler/www --start=2015/01/01 --stop=2020/12/31' + docker exec "piler1" su piler -c 'php /usr/libexec/piler/sign.php --webui /var/piler/www --mode time' } diff --git a/tests/setup.inc b/tests/setup.inc index 2cf689ed..7861fe11 100644 --- a/tests/setup.inc +++ b/tests/setup.inc @@ -33,7 +33,7 @@ launch_containers() { create_rules() { local container="$1" - echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"acts.hu\",\"acts.hu\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh + echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"acts.hu\",\"acts.hu\"),(\"gtsce.com\",\"fictive.com\"),(\"datanet.hu\",\"fictive.com\"),(\"gtsdatanet.hu\",\"fictive.com\"),(\"gts.hu\",\"fictive.com\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh echo 'echo "insert into archiving_rule (subject) values (\"Android táblagép\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh echo 'echo "insert into archiving_rule (\`from\`) values (\"@gmail.com\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh From ffe7616b58b0be659aa715252d8342721fd569c6 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 26 Oct 2020 20:30:38 +0100 Subject: [PATCH 30/55] Fixed test case Signed-off-by: Janos SUTO --- tests/setup.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/setup.inc b/tests/setup.inc index 7861fe11..c267eeca 100644 --- a/tests/setup.inc +++ b/tests/setup.inc @@ -33,7 +33,7 @@ launch_containers() { create_rules() { local container="$1" - echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"acts.hu\",\"acts.hu\"),(\"gtsce.com\",\"fictive.com\"),(\"datanet.hu\",\"fictive.com\"),(\"gtsdatanet.hu\",\"fictive.com\"),(\"gts.hu\",\"fictive.com\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh + echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"acts.hu\",\"acts.hu\"),(\"gtsce.com\",\"gtsce.com\"),(\"datanet.hu\",\"datanet.hu\"),(\"gtsdatanet.hu\",\"gtsdatanet.hu\"),(\"gts.hu\",\"gts.hu\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh echo 'echo "insert into archiving_rule (subject) values (\"Android táblagép\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh echo 'echo "insert into archiving_rule (\`from\`) values (\"@gmail.com\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh From b2358e63078eb2998346c136f7401ed8370d0e57 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 26 Oct 2020 22:32:19 +0100 Subject: [PATCH 31/55] Enable import menu for all deployments Signed-off-by: Janos SUTO --- webui/view/theme/default/templates/common/menu.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui/view/theme/default/templates/common/menu.tpl b/webui/view/theme/default/templates/common/menu.tpl index 4a57d6ce..16d603cc 100644 --- a/webui/view/theme/default/templates/common/menu.tpl +++ b/webui/view/theme/default/templates/common/menu.tpl @@ -42,8 +42,8 @@
  •  
  •  
    • -
  •  
    • +
  •  
  •  
  •  
  •  
    • From 68f72d0784233845bec6dd0b137c6b76aecc53fc Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 26 Oct 2020 23:08:03 +0100 Subject: [PATCH 32/55] imapfetch.py can read imap connection from import table Signed-off-by: Janos SUTO --- util/imapfetch.py | 76 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/util/imapfetch.py b/util/imapfetch.py index f9c98b3d..3616062c 100755 --- a/util/imapfetch.py +++ b/util/imapfetch.py @@ -1,14 +1,32 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- +import MySQLdb as dbapi import argparse +import configparser import imaplib import re +import sys opts = {} INBOX = 'INBOX' +def read_options(filename="", opts={}): + s = "[piler]\n" + open(filename, 'r').read() + config = configparser.ConfigParser() + config.read_string(s) + + if config.has_option('piler', 'mysqlhost'): + opts['dbhost'] = config.get('piler', 'mysqlhost') + else: + opts['dbhost'] = 'localhost' + + opts['username'] = config.get('piler', 'mysqluser') + opts['password'] = config.get('piler', 'mysqlpwd') + opts['database'] = config.get('piler', 'mysqldb') + + def read_folder_list(conn): result = [] @@ -34,11 +52,13 @@ def read_folder_list(conn): def process_folder(conn, folder): - print("Processing {}".format(folder)) + if opts['verbose']: + print("Processing {}".format(folder)) rc, data = conn.select(folder) n = int(data[0]) - print("Folder {} has {} messages".format(folder, n)) + if opts['verbose']: + print("Folder {} has {} messages".format(folder, n)) if n > 0: rc, data = conn.search(None, 'ALL') @@ -46,39 +66,71 @@ def process_folder(conn, folder): rc, data = conn.fetch(num, '(RFC822)') if opts['verbose']: print(rc, num) - opts['counter'] = opts['counter'] + 1 + opts['counter'] += 1 with open("{}.eml".format(opts['counter']), "wb") as f: f.write(data[0][1]) def main(): parser = argparse.ArgumentParser() - parser.add_argument("-s", "--server", type=str, help="imap server", required=True) + parser.add_argument("-c", "--config", type=str, help="piler.conf path", + default="/etc/piler/piler.conf") + parser.add_argument("-s", "--server", type=str, help="imap server") parser.add_argument("-P", "--port", type=int, help="port number", default=143) - parser.add_argument("-u", "--user", type=str, help="imap user", required=True) - parser.add_argument("-p", "--password", type=str, help="imap password", - required=True) + parser.add_argument("-u", "--user", type=str, help="imap user") + parser.add_argument("-p", "--password", type=str, help="imap password") parser.add_argument("-x", "--skip-list", type=str, help="IMAP folders to skip", default="junk,trash,spam,draft") parser.add_argument("-f", "--folders", type=str, help="Comma separated list of IMAP folders to download") + parser.add_argument("-i", "--import-from-table", help="Read imap conn data from import table", action='store_true') parser.add_argument("-v", "--verbose", help="verbose mode", action='store_true') args = parser.parse_args() + print(args) + if not bool(args.import_from_table or args.server): + print("Please specify either --import-from-table or --server ") + sys.exit(1) + opts['skip_folders'] = args.skip_list.split(',') opts['verbose'] = args.verbose opts['counter'] = 0 + opts['db'] = None + + server = '' + user = '' + password = '' + + if args.import_table: + read_options(args.config, opts) + try: + opts['db'] = dbapi.connect(opts['dbhost'], opts['username'], + opts['password'], opts['database']) + + cursor = opts['db'].cursor() + cursor.execute("SELECT server, username, password FROM import WHERE started=0") + + row = cursor.fetchone() + if row: + (server, user, password) = row + + except dbapi.DatabaseError as e: + print("Error %s" % e) + else: + server = args.server + user = args.user + password = args.password if opts['verbose']: print("Skipped folder list: {}".format(opts['skip_folders'])) if args.port == 993: - conn = imaplib.IMAP4_SSL(args.server) + conn = imaplib.IMAP4_SSL(server) else: - conn = imaplib.IMAP4(args.server) + conn = imaplib.IMAP4(server) - conn.login(args.user, args.password) + conn.login(user, password) conn.select() if args.folders: @@ -94,6 +146,10 @@ def main(): conn.close() + if opts['db']: + opts['db'].close() + + print("Processed {} messages".format(opts['counter'])) if __name__ == "__main__": main() From 96dbde7cbae4e9fbc91d4c2cbdcd5479b06d8a93 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 26 Oct 2020 23:10:48 +0100 Subject: [PATCH 33/55] imapfetch.py shall quit if nothing in the import table Signed-off-by: Janos SUTO --- util/imapfetch.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/util/imapfetch.py b/util/imapfetch.py index 3616062c..08b6649f 100755 --- a/util/imapfetch.py +++ b/util/imapfetch.py @@ -114,6 +114,9 @@ def main(): row = cursor.fetchone() if row: (server, user, password) = row + else: + print("Nothing to read from import table") + sys.exit(0) except dbapi.DatabaseError as e: print("Error %s" % e) From 1d615a196792610dc15c63388ffbe041bafb5282 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 27 Oct 2020 21:53:16 +0100 Subject: [PATCH 34/55] Added support for gui import Signed-off-by: Janos SUTO --- src/config.h | 1 + src/defs.h | 1 + src/import.c | 20 ++++++++++++++++++++ src/import.h | 1 + src/import_maildir.c | 4 ++++ src/pilerimport.c | 16 ++++++++++++++-- util/imapfetch.py | 17 +++++++++++++---- 7 files changed, 54 insertions(+), 6 deletions(-) diff --git a/src/config.h b/src/config.h index 117f3f8a..e1542ad5 100644 --- a/src/config.h +++ b/src/config.h @@ -107,6 +107,7 @@ #define SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE "UPDATE " SQL_METADATA_TABLE " SET reference=? WHERE message_id=? AND reference=''" #define SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS "SELECT id, type, username, password, server FROM " SQL_IMPORT_TABLE " WHERE started=0 ORDER BY id LIMIT 0,1" #define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `id`) VALUES(?,?)" +#define SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE "UPDATE " SQL_IMPORT_TABLE " SET status=?, imported=? WHERE id=?" /* Error codes */ diff --git a/src/defs.h b/src/defs.h index 50be06f0..f7477a65 100644 --- a/src/defs.h +++ b/src/defs.h @@ -304,6 +304,7 @@ struct import { int tot_msgs; int port; int seq; + int table_id; char *server; char *username; char *password; diff --git a/src/import.c b/src/import.c index eb16f8f8..601c9b84 100644 --- a/src/import.c +++ b/src/import.c @@ -144,6 +144,26 @@ int import_message(struct session_data *sdata, struct data *data, struct config } +int update_import_table(struct session_data *sdata, struct data *data) { + int ret=ERR, status=2; + struct sql sql; + + if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE) == ERR) return ret; + + p_bind_init(&sql); + + sql.sql[sql.pos] = (char *)&(status); sql.type[sql.pos] = TYPE_LONG; sql.pos++; + sql.sql[sql.pos] = (char *)&(data->import->tot_msgs); sql.type[sql.pos] = TYPE_LONG; sql.pos++; + sql.sql[sql.pos] = (char *)&(data->import->table_id); sql.type[sql.pos] = TYPE_LONG; sql.pos++; + + if(p_exec_stmt(sdata, &sql) == OK) ret = OK; + + close_prepared_statement(&sql); + + return ret; +} + + int get_folder_id(struct session_data *sdata, char *foldername, int parent_id){ int id=ERR_FOLDER; struct sql sql; diff --git a/src/import.h b/src/import.h index 0fa2a211..0979acab 100644 --- a/src/import.h +++ b/src/import.h @@ -7,6 +7,7 @@ int import_message(struct session_data *sdata, struct data *data, struct config *cfg); +int update_import_table(struct session_data *sdata, struct data *data); int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct config *cfg); int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct config *cfg); diff --git a/src/import_maildir.c b/src/import_maildir.c index d5cae635..aac3c6a3 100644 --- a/src/import_maildir.c +++ b/src/import_maildir.c @@ -103,5 +103,9 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir } closedir(dir); + if(data->import->table_id > 0){ + update_import_table(sdata, data); + } + return ret; } diff --git a/src/pilerimport.c b/src/pilerimport.c index 6b34d6f0..0f07d1e9 100644 --- a/src/pilerimport.c +++ b/src/pilerimport.c @@ -52,6 +52,7 @@ void usage(){ printf(" -s Start importing POP3 emails from this position\n"); printf(" -j Move failed to import emails to this folder\n"); printf(" -a Add recipient to the To:/Cc: list\n"); + printf(" -T Update import table at id=\n"); printf(" -D Dry-run, do not import anything\n"); printf(" -o Only download emails for POP3/IMAP import\n"); printf(" -r Remove imported emails\n"); @@ -96,6 +97,7 @@ int main(int argc, char **argv){ memset(import.filename, 0, SMALLBUFSIZE); import.mboxdir = NULL; import.tot_msgs = 0; + import.table_id = 0; import.folder = NULL; data.import = &import; @@ -132,6 +134,7 @@ int main(int argc, char **argv){ {"batch-limit", required_argument, 0, 'b' }, {"timeout", required_argument, 0, 't' }, {"start-position",required_argument, 0, 's' }, + {"table-id", required_argument, 0, 'T' }, {"quiet", no_argument, 0, 'q' }, {"recursive", required_argument, 0, 'R' }, {"remove-after-import",no_argument, 0, 'r' }, @@ -145,9 +148,9 @@ int main(int argc, char **argv){ int option_index = 0; - int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:DRroqh?", long_options, &option_index); + int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:DRroqh?", long_options, &option_index); #else - int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:DRroqh?"); + int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:DRroqh?"); #endif if(c == -1) break; @@ -257,6 +260,15 @@ int main(int argc, char **argv){ data.import->extra_recipient = puf; break; + case 'T' : + if(atoi(optarg) < 1){ + printf("invalid import table id: %s\n", optarg); + return -1; + } + + data.import->table_id = atoi(optarg); + break; + case 'D' : data.import->dryrun = 1; break; diff --git a/util/imapfetch.py b/util/imapfetch.py index 08b6649f..df9d570d 100755 --- a/util/imapfetch.py +++ b/util/imapfetch.py @@ -6,10 +6,12 @@ import argparse import configparser import imaplib import re +import subprocess import sys opts = {} INBOX = 'INBOX' +ST_RUNNING = 1 def read_options(filename="", opts={}): @@ -61,6 +63,12 @@ def process_folder(conn, folder): print("Folder {} has {} messages".format(folder, n)) if n > 0: + if opts['id']: + cursor = opts['db'].cursor() + data = (ST_RUNNING, n, opts['id']) + cursor.execute("UPDATE import SET status=%s, total = total + %s WHERE id=%s", data) + opts['db'].commit() + rc, data = conn.search(None, 'ALL') for num in data[0].split(): rc, data = conn.fetch(num, '(RFC822)') @@ -88,7 +96,6 @@ def main(): args = parser.parse_args() - print(args) if not bool(args.import_from_table or args.server): print("Please specify either --import-from-table or --server ") sys.exit(1) @@ -97,23 +104,24 @@ def main(): opts['verbose'] = args.verbose opts['counter'] = 0 opts['db'] = None + opts['id'] = 0 server = '' user = '' password = '' - if args.import_table: + if args.import_from_table: read_options(args.config, opts) try: opts['db'] = dbapi.connect(opts['dbhost'], opts['username'], opts['password'], opts['database']) cursor = opts['db'].cursor() - cursor.execute("SELECT server, username, password FROM import WHERE started=0") + cursor.execute("SELECT id, server, username, password FROM import WHERE started=0") row = cursor.fetchone() if row: - (server, user, password) = row + (opts['id'], server, user, password) = row else: print("Nothing to read from import table") sys.exit(0) @@ -150,6 +158,7 @@ def main(): conn.close() if opts['db']: + subprocess.call(["pilerimport", "-d", "/var/piler/imap", "-r"]) opts['db'].close() print("Processed {} messages".format(opts['counter'])) From f7f91c0855d911fdb822ad0d7e8c6335035880d7 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 27 Oct 2020 22:11:21 +0100 Subject: [PATCH 35/55] Improved imapfetch.py Signed-off-by: Janos SUTO --- util/imapfetch.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/util/imapfetch.py b/util/imapfetch.py index df9d570d..8898c549 100755 --- a/util/imapfetch.py +++ b/util/imapfetch.py @@ -66,7 +66,7 @@ def process_folder(conn, folder): if opts['id']: cursor = opts['db'].cursor() data = (ST_RUNNING, n, opts['id']) - cursor.execute("UPDATE import SET status=%s, total = total + %s WHERE id=%s", data) + cursor.execute("UPDATE import SET status=%s, total=total+%s WHERE id=%s", data) opts['db'].commit() rc, data = conn.search(None, 'ALL') @@ -91,11 +91,16 @@ def main(): default="junk,trash,spam,draft") parser.add_argument("-f", "--folders", type=str, help="Comma separated list of IMAP folders to download") - parser.add_argument("-i", "--import-from-table", help="Read imap conn data from import table", action='store_true') + parser.add_argument("-d", "--dir", help="directory to chdir", + default="/var/piler/imap") + parser.add_argument("-i", "--import-from-table", action='store_true', + help="Read imap conn data from import table") parser.add_argument("-v", "--verbose", help="verbose mode", action='store_true') args = parser.parse_args() + os.chdir(args.dir) + if not bool(args.import_from_table or args.server): print("Please specify either --import-from-table or --server ") sys.exit(1) @@ -158,10 +163,15 @@ def main(): conn.close() if opts['db']: - subprocess.call(["pilerimport", "-d", "/var/piler/imap", "-r"]) + if opts['id']: + subprocess.call(["pilerimport", + "-d", args.dir, + "-r", + "-T", str(opts['id'])]) opts['db'].close() print("Processed {} messages".format(opts['counter'])) + if __name__ == "__main__": main() From d91e5aadd2f823f05585830a5abc2f7370a3e595 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 27 Oct 2020 22:16:23 +0100 Subject: [PATCH 36/55] import/jobs page refresh should be the same as for the health page Signed-off-by: Janos SUTO --- webui/view/theme/default/templates/common/layout.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui/view/theme/default/templates/common/layout.tpl b/webui/view/theme/default/templates/common/layout.tpl index d3aeff10..c213a172 100644 --- a/webui/view/theme/default/templates/common/layout.tpl +++ b/webui/view/theme/default/templates/common/layout.tpl @@ -38,7 +38,7 @@ if(isset($this->request->get['route'])) { if($this->request->get['route'] == 'health/health') { ?> onload="Piler.load_health(); setInterval('Piler.load_health()', Piler.health_refresh * 1000);"request->get['route'] == 'stat/online') { ?> onload="setInterval('Piler.reload_page()', Piler.health_refresh * 1000);"request->get['route'] == 'import/jobs') { ?> onload="setInterval('Piler.reload_page()', 10 * 1000);"request->get['route'] == 'import/jobs') { ?> onload="setInterval('Piler.reload_page()', Piler.health_refresh * 1000);"> From 27876000a2f6e5e0f3595a486fb479982d51878c Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 27 Oct 2020 22:16:53 +0100 Subject: [PATCH 37/55] Added missing import for imapfetch.py Signed-off-by: Janos SUTO --- util/imapfetch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/util/imapfetch.py b/util/imapfetch.py index 8898c549..c213c323 100755 --- a/util/imapfetch.py +++ b/util/imapfetch.py @@ -5,6 +5,7 @@ import MySQLdb as dbapi import argparse import configparser import imaplib +import os import re import subprocess import sys From 3b3d0157caa5b075255672ef991015b330d690f0 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 27 Oct 2020 22:20:03 +0100 Subject: [PATCH 38/55] Fixed import.sh Signed-off-by: Janos SUTO --- util/import.sh | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/util/import.sh b/util/import.sh index 33528061..19b8960b 100755 --- a/util/import.sh +++ b/util/import.sh @@ -4,19 +4,11 @@ set -o nounset set -o errexit set -o pipefail -export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin -TMPFILE=/var/run/piler/import.tmp +export PATH=$PATH:/usr/libexec/piler:/usr/local/libexec/piler -if [[ -f $TMPFILE ]]; then exit 1; fi +pushd /var/piler/imap -date > $TMPFILE - -function finish { - rm -f $TMPFILE -} - -trap finish EXIT - -cd /var/piler/imap - -pilerimport -G >/dev/null +exec 200 > "/tmp/${0}-lock" || exit 1 +flock 200 || exit 1 +imapfetch.py -i +flock -u 200 From d442fea23a6bf8500a47554adf80d1e8bb5cd276 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 27 Oct 2020 22:23:38 +0100 Subject: [PATCH 39/55] Prepare for gui import Signed-off-by: Janos SUTO --- tests/setup.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/setup.inc b/tests/setup.inc index c267eeca..1a4c03f6 100644 --- a/tests/setup.inc +++ b/tests/setup.inc @@ -39,6 +39,7 @@ create_rules() { echo 'echo "insert into archiving_rule (\`from\`) values (\"@gmail.com\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh echo 'echo "insert into archiving_rule (\`from\`,attachment_type, _attachment_size, attachment_size) values (\"finderis.co.ua\", \"image\", \">\", 100000)"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh echo 'echo "insert into archiving_rule (\`to\`) values (\"undisclosed-recipients\")"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh + echo 'echo "insert into import (\`type\`, username, password, server) values (\"imap\", \"sanyi@aaa.fu\", \"abcde123\", \"imap\")"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh echo 'echo "update user set password=\"\$6\$GKL00T\$8jqoFOe3PyAbOCLwKB7JwndwC.IinHrZRkdoQDZUc8vybZ88sA2qomlz5JceNif8fFpkGzZ03ilvQa7tqQx0v1\""| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh From 3c57d5fec723fd1ba272df325ed9a2146fe01e0b Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 27 Oct 2020 22:39:16 +0100 Subject: [PATCH 40/55] Fixed import.sh locking Signed-off-by: Janos SUTO --- util/import.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/util/import.sh b/util/import.sh index 19b8960b..efff682e 100755 --- a/util/import.sh +++ b/util/import.sh @@ -1,6 +1,5 @@ #!/bin/bash -set -o nounset set -o errexit set -o pipefail @@ -8,7 +7,6 @@ export PATH=$PATH:/usr/libexec/piler:/usr/local/libexec/piler pushd /var/piler/imap -exec 200 > "/tmp/${0}-lock" || exit 1 -flock 200 || exit 1 +[[ "${FLOCKER}" != "$0" ]] && exec env FLOCKER="$0" flock -en "$0" "$0" "$@" + imapfetch.py -i -flock -u 200 From d1fecf3a0ce40ef58865f8ff2b26797c1b58333f Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Wed, 28 Oct 2020 12:38:38 +0100 Subject: [PATCH 41/55] Fixed LDAP() call in model/saas/ldap.php Signed-off-by: Janos SUTO --- webui/model/saas/ldap.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/webui/model/saas/ldap.php b/webui/model/saas/ldap.php index 2fd5768d..cedcca37 100644 --- a/webui/model/saas/ldap.php +++ b/webui/model/saas/ldap.php @@ -82,6 +82,7 @@ class ModelSaasLdap extends Model public function get_accounts_in_domain($domain = '') { $ldap_type = ''; $ldap_host = LDAP_HOST; + $ldap_port = LDAP_PORT; $ldap_base_dn = LDAP_BASE_DN; $ldap_helper_dn = LDAP_HELPER_DN; $ldap_helper_password = LDAP_HELPER_PASSWORD; @@ -102,7 +103,7 @@ class ModelSaasLdap extends Model if($ldap_host == '' || $ldap_helper_password == '') { return array(); } - $ldap = new LDAP($ldap_host, $ldap_helper_dn, $ldap_helper_password); + $ldap = new LDAP($ldap_host, $ldap_port, $ldap_helper_dn, $ldap_helper_password); if($ldap->is_bind_ok()) { From 4e378b09cdfc60d7e4f55920e2123b3a0f19969c Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 29 Oct 2020 09:50:49 +0100 Subject: [PATCH 42/55] Added healthcheck utility Signed-off-by: Janos SUTO --- util/healthcheck.sh | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 util/healthcheck.sh diff --git a/util/healthcheck.sh b/util/healthcheck.sh new file mode 100755 index 00000000..710cc508 --- /dev/null +++ b/util/healthcheck.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +SPHINX_DIR=/var/piler/sphinx +ERROR_FILE=/var/piler/stat/error + +echo "host: $(hostname | sha256sum | awk '{ print $1 }')" +echo "cpus: $(grep -c ^processor /proc/cpuinfo)" +echo "mem: $(grep MemTotal /proc/meminfo | sed 's/MemTotal\s*:\s*//' )" +echo disks: +while read -r p; do df -h "${p##* }" | tail -1; done < <(lsblk | grep part) + +echo +piler -V + +# shellcheck disable=SC2009 +ps uaxw|grep piler + +echo -e "\nCron entries:\n" +crontab -l -u piler + +errors=0 +if [[ -f "$ERROR_FILE" ]]; then + read -r errors < "$ERROR_FILE" +else + errors="$(find /var/piler/error -type f|wc -l)" +fi + +mysqluser="$(pilerconf -q mysqluser|cut -d = -f2)" +mysqlpwd="$(pilerconf -q mysqlpwd|cut -d = -f2)" +mysqldb="$(pilerconf -q mysqldb|cut -d = -f2)" + +echo -e "\nError emails: $errors" +echo -e "Sphinx data: $(du -hs "$SPHINX_DIR")\n" + +mysql -t -u "$mysqluser" -p"$mysqlpwd" "$mysqldb" <<< "select * from counter" + From ffa2b69d4bc973386e213477cf0727e04bc6f9ad Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 29 Oct 2020 10:10:07 +0100 Subject: [PATCH 43/55] Added mysql db size to healthcheck utility Signed-off-by: Janos SUTO --- util/healthcheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/healthcheck.sh b/util/healthcheck.sh index 710cc508..6b0e9122 100755 --- a/util/healthcheck.sh +++ b/util/healthcheck.sh @@ -37,4 +37,4 @@ echo -e "\nError emails: $errors" echo -e "Sphinx data: $(du -hs "$SPHINX_DIR")\n" mysql -t -u "$mysqluser" -p"$mysqlpwd" "$mysqldb" <<< "select * from counter" - +mysql -t -u "$mysqluser" -p"$mysqlpwd" information_schema <<< "select table_schema as db, sum(data_length+index_length) as size from TABLES WHERE table_schema='$mysqldb' GROUP BY table_schema" From 4659af3dd27594a8c2707155c8024839f5739e8d Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 29 Oct 2020 10:12:24 +0100 Subject: [PATCH 44/55] Added healthcheck.sh to make install Signed-off-by: Janos SUTO --- util/Makefile.in | 1 + 1 file changed, 1 insertion(+) diff --git a/util/Makefile.in b/util/Makefile.in index cf61f40b..afe2701d 100644 --- a/util/Makefile.in +++ b/util/Makefile.in @@ -35,6 +35,7 @@ install: $(INSTALL) -m 0755 $(srcdir)/daily-report.php $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/gmail-imap-import.php $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/generate_stats.php $(DESTDIR)$(libexecdir)/piler + $(INSTALL) -m 0755 $(srcdir)/healthcheck.sh $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/mailstat.php $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/sign.php $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/indexer.delta.sh $(DESTDIR)$(libexecdir)/piler From 25c9e116edf2e24ff5616839f693b3bbeb4d81a5 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 29 Oct 2020 10:42:56 +0100 Subject: [PATCH 45/55] added mpstat to healthcheck.sh Signed-off-by: Janos SUTO --- util/healthcheck.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/util/healthcheck.sh b/util/healthcheck.sh index 6b0e9122..113547e0 100755 --- a/util/healthcheck.sh +++ b/util/healthcheck.sh @@ -7,8 +7,15 @@ set -o nounset SPHINX_DIR=/var/piler/sphinx ERROR_FILE=/var/piler/stat/error -echo "host: $(hostname | sha256sum | awk '{ print $1 }')" -echo "cpus: $(grep -c ^processor /proc/cpuinfo)" +if command -v mpstat; then + mpstat + echo +else + echo "host: $(hostname)" + echo "cpus: $(grep -c ^processor /proc/cpuinfo)" +fi + +echo "load: $(cat /proc/loadavg)" echo "mem: $(grep MemTotal /proc/meminfo | sed 's/MemTotal\s*:\s*//' )" echo disks: while read -r p; do df -h "${p##* }" | tail -1; done < <(lsblk | grep part) From 9daad952b492d97326c7f09f820db141dd858be0 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 30 Oct 2020 10:23:13 +0100 Subject: [PATCH 46/55] Fixed branding logo reference Signed-off-by: Janos SUTO --- config.php.in | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config.php.in b/config.php.in index 94aed8ca..b92a117c 100644 --- a/config.php.in +++ b/config.php.in @@ -18,9 +18,13 @@ $config = []; $config['ENABLE_MOBILE_PREVIEW'] = 0; +$config['SITE_LOGO_LG'] = '/view/theme/default/assets/images/archive-logo-lg.png'; +$config['SITE_LOGO_SM'] = '/view/theme/default/assets/images/archive-logo-sm.png'; +$config['COMPATIBILITY'] = 'Which browsers are supported, etc'; + $config['BRANDING_TEXT'] = ''; $config['BRANDING_URL'] = ''; -$config['BRANDING_LOGO'] = ''; +$config['BRANDING_LOGO'] = $config['SITE_LOGO_SM']; $config['BRANDING_BACKGROUND_COLOUR'] = ''; $config['BRANDING_TEXT_COLOUR'] = ''; $config['BRANDING_FAVICON'] = '/view/theme/default/assets/ico/favicon.png'; @@ -156,10 +160,6 @@ $config['GOOGLE_DEVELOPER_KEY'] = 'xxxxxxxxxxxx'; $config['GOOGLE_APPLICATION_NAME'] = 'piler enterprise email archiver'; $config['GOOGLE_ALL_MAIL'] = '[Gmail]/All Mail'; -$config['SITE_LOGO_LG'] = '/view/theme/default/assets/images/archive-logo-lg.png'; -$config['SITE_LOGO_SM'] = '/view/theme/default/assets/images/archive-logo-sm.png'; -$config['COMPATIBILITY'] = 'Which browsers are supported, etc'; - $config['ENABLE_AUDIT'] = 1; $config['MEMCACHED_ENABLED'] = 0; From 4d8be0ac9938c5001df4fb0d6f5a5357bd2e7787 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 30 Oct 2020 20:57:00 +0100 Subject: [PATCH 47/55] Fxed check_your_permission_by_id_list() Signed-off-by: Janos SUTO --- webui/model/search/search.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui/model/search/search.php b/webui/model/search/search.php index f92647cf..06cf54e2 100644 --- a/webui/model/search/search.php +++ b/webui/model/search/search.php @@ -775,7 +775,7 @@ class ModelSearchSearch extends Model { $q = preg_replace("/^\,/", "", $q); - if(Registry::get('auditor_user') == 1 && RESTRICTED_AUDITOR == 0) { + if(Registry::get('auditor_user') == 1 && RESTRICTED_AUDITOR == 0 && ENABLE_FOLDER_RESTRICTIONS == 0) { $query = $this->db->query("SELECT id FROM `" . TABLE_META . "` WHERE `id` IN ($q2)", $arr); } else { From 2b55fe00bc2d19f15d65d6e77201c1acd3a2810b Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 30 Oct 2020 21:19:20 +0100 Subject: [PATCH 48/55] Fixed folder re-read after adding/removing a folder Signed-off-by: Janos SUTO --- webui/model/folder/folder.php | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/webui/model/folder/folder.php b/webui/model/folder/folder.php index a64256fa..4189124e 100644 --- a/webui/model/folder/folder.php +++ b/webui/model/folder/folder.php @@ -236,8 +236,8 @@ class ModelFolderFolder extends Model { $last_id = $this->db->getLastId(); $query = $this->db->query("INSERT INTO " . TABLE_FOLDER_USER . " (id, uid) VALUES(?,?)", array($last_id, $session->get("uid"))); - $folders = $session->get("folders"); - if(!isset($folders[$last_id])) { array_push($folders, $last_id); $session->set("folders", $folders); } + $folders = $this->get_folder_id_array_for_user($session->get("uid"), $session->get("admin_user")); + $session->set("folders", $folders); } return $this->db->countAffected(); @@ -253,8 +253,8 @@ class ModelFolderFolder extends Model { $query = $this->db->query("DELETE FROM " . TABLE_FOLDER . " WHERE id=?", array($id)); $query = $this->db->query("DELETE FROM " . TABLE_FOLDER_USER . " WHERE id=? AND uid=?", array($id, $session->get("uid"))); - $folders = $session->get("folders"); - if(isset($folders[$id])) { unset($folders[$id]); $session->set("folders", $folders); } + $folders = $this->get_folder_id_array_for_user($session->get("uid"), $session->get("admin_user")); + $session->set("folders", $folders); // shall we delete the existing message - folder id assignments from folder_message? } @@ -264,5 +264,3 @@ class ModelFolderFolder extends Model { } - -?> From 5331297c551e3d415c858ee51a637e04074e7ebd Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 30 Oct 2020 23:56:52 +0100 Subject: [PATCH 49/55] Added option to write exported files to a zip file Signed-off-by: Janos SUTO --- src/pilerexport.c | 62 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/src/pilerexport.c b/src/pilerexport.c index 84362de6..0bc1f071 100644 --- a/src/pilerexport.c +++ b/src/pilerexport.c @@ -8,11 +8,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -29,7 +31,7 @@ int verbosity = 0; int max_matches = 1000; char *index_list = "main1,dailydelta1,delta1"; regex_t regexp; - +char *zipfile = NULL; int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg); @@ -49,6 +51,7 @@ void usage(){ printf(" -w Where condition to pass to sphinx, eg. \"match('@subject: piler')\"\n"); printf(" -m Max. matches to apply to sphinx query (default: %d)\n", max_matches); printf(" -i Sphinx indices to use (default: %s)\n", index_list); + printf(" -z Write exported EML files to a zip file\n"); printf(" -A Export all emails from archive\n"); printf(" -d Dry run\n"); @@ -313,6 +316,48 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain } +int write_to_zip_file(char *filename){ + struct zip *z=NULL; + int errorp, ret=ERR; + + z = zip_open(zipfile, ZIP_CREATE, &errorp); + if(!z){ + printf("error: error creating zip file=%s, error code=%d\n", zipfile, errorp); + return ret; + } + + int fd = open(filename, O_RDONLY); + if(fd == -1){ + printf("cannot open: %s\n", filename); + return ret; + } + + struct stat st; + if(fstat(fd, &st)){ + close(fd); + return ret; + } + + char *addr = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + + if(addr == MAP_FAILED) return ret; + + zip_source_t *zs = zip_source_buffer(z, addr, st.st_size, 0); + + if(zip_file_add(z, filename, zs, ZIP_FL_ENC_UTF_8) == -1){ + printf("error adding file %s: %s\n", filename, zip_strerror(z)); + zip_source_free(zs); + } else { + ret = OK; + } + + zip_close(z); + munmap(addr, st.st_size); + + return ret; +} + int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg){ FILE *f; uint64 id, n=0; @@ -364,6 +409,11 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct printf("verification FAILED. %s\n", filename); verification_status = 1; } + + if(zipfile && write_to_zip_file(filename) == OK){ + unlink(filename); + } + } else printf("cannot open: %s\n", filename); } @@ -380,7 +430,6 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct ENDE: close_prepared_statement(&sql); - printf("\n"); return rc; @@ -421,6 +470,7 @@ int main(int argc, char **argv){ {"to-domain", required_argument, 0, 'R' }, {"start-date", required_argument, 0, 'a' }, {"stop-date", required_argument, 0, 'b' }, + {"zip", required_argument, 0, 'z' }, {"where-condition", required_argument, 0, 'w' }, {"max-matches", required_argument, 0, 'm' }, {"index-list", required_argument, 0, 'i' }, @@ -429,9 +479,9 @@ int main(int argc, char **argv){ int option_index = 0; - int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?", long_options, &option_index); + int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Adhv?", long_options, &option_index); #else - int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?"); + int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Adhv?"); #endif if(c == -1) break; @@ -520,6 +570,10 @@ int main(int argc, char **argv){ index_list = optarg; break; + case 'z': zipfile = optarg; + break; + + case 'd' : dryrun = 1; break; From b5a8255be18f465f209140010ac0cd247ab6c085 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 31 Oct 2020 07:58:52 +0100 Subject: [PATCH 50/55] Refactored zip support for pilerexport Signed-off-by: Janos SUTO --- src/pilerexport.c | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/src/pilerexport.c b/src/pilerexport.c index 0bc1f071..1aee477f 100644 --- a/src/pilerexport.c +++ b/src/pilerexport.c @@ -326,34 +326,14 @@ int write_to_zip_file(char *filename){ return ret; } - int fd = open(filename, O_RDONLY); - if(fd == -1){ - printf("cannot open: %s\n", filename); - return ret; - } - - struct stat st; - if(fstat(fd, &st)){ - close(fd); - return ret; - } - - char *addr = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); - close(fd); - - if(addr == MAP_FAILED) return ret; - - zip_source_t *zs = zip_source_buffer(z, addr, st.st_size, 0); - - if(zip_file_add(z, filename, zs, ZIP_FL_ENC_UTF_8) == -1){ - printf("error adding file %s: %s\n", filename, zip_strerror(z)); - zip_source_free(zs); - } else { + zip_source_t *zs = zip_source_file(z, filename, 0, 0); + if(zs && zip_file_add(z, filename, zs, ZIP_FL_ENC_UTF_8) >= 0){ ret = OK; + } else { + printf("error adding file %s: %s\n", filename, zip_strerror(z)); } zip_close(z); - munmap(addr, st.st_size); return ret; } From bfeef2669e07b53cf8878a0a425b4bdd47a25567 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 31 Oct 2020 21:10:41 +0100 Subject: [PATCH 51/55] Added support for wildcard domains Signed-off-by: Janos SUTO --- webui/controller/user/settings.php | 17 +++-- webui/model/search/search.php | 40 ++++++++-- .../theme/default/templates/user/settings.tpl | 75 ++++++++++--------- 3 files changed, 83 insertions(+), 49 deletions(-) diff --git a/webui/controller/user/settings.php b/webui/controller/user/settings.php index e0fe061c..a5ec49cc 100644 --- a/webui/controller/user/settings.php +++ b/webui/controller/user/settings.php @@ -26,14 +26,14 @@ class ControllerUserSettings extends Controller { $this->data['ga'] = $this->model_user_prefs->get_ga_settings($session->get('username')); $this->document->title = $this->data['text_settings']; - + $d = $r = ''; $auditemails = $auditdomains = $auditgroups = $auditfolders = ''; $auditemails = implode(", ", $session->get("emails")); - + $_auditdomains = $session->get("auditdomains"); - + foreach($_auditdomains as $d) { $auditdomains .= ', ' . $d; } @@ -42,17 +42,22 @@ class ControllerUserSettings extends Controller { $auditgroups = preg_replace("/\n/", ", ", $this->model_group_group->get_groups_by_email($session->get("emails"))); $folders = $session->get("folders"); - + foreach ($folders as $r) { $auditfolders .= ', ' . $r; } - $auditfolders = preg_replace("/^,\s/", "", $auditfolders); - + $auditfolders = preg_replace("/^,\s/", "", $auditfolders); + if($auditemails) { $this->data['emails'] = $auditemails; } else { $this->data['emails'] = $this->data['text_none_found']; } if($auditdomains) { $this->data['domains'] = $auditdomains; } else { $this->data['domains'] = $this->data['text_none_found']; } if($auditgroups) { $this->data['groups'] = $auditgroups; } else { $this->data['groups'] = $this->data['text_none_found']; } if($auditfolders) { $this->data['folders'] = $auditfolders; } else { $this->data['folders'] = $this->data['text_none_found']; } + $this->data['wildcard_domains'] = $session->get("wildcard_domains"); + if($this->data['wildcard_domains']) { + $this->data['wildcard_domains'] = implode(", ", $this->data['wildcard_domains']); + } + if(isset($this->request->post['pagelen']) && isset($this->request->post['theme'])) { $this->model_user_prefs->set_user_preferences(Registry::get('username'), $this->request->post); diff --git a/webui/model/search/search.php b/webui/model/search/search.php index 06cf54e2..cc6a5db9 100644 --- a/webui/model/search/search.php +++ b/webui/model/search/search.php @@ -81,8 +81,14 @@ class ModelSearchSearch extends Model { if(ENABLE_FOLDER_RESTRICTIONS == 1) { return ""; } - $all_your_addresses = $this->get_all_your_address(); - return sprintf(" (%s %s | %s %s) ", FROM_TOKEN, $all_your_addresses, TO_TOKEN, $all_your_addresses); + $all_your_addresses = $this->get_all_your_address("emails"); + $all_your_wildcard_domains = $this->get_all_your_address("wildcard_domains"); + + if($all_your_wildcard_domains) { + return sprintf(" ( (%s %s) | (%s %s) | (%s %s) | (%s %s) ) ", FROM_TOKEN, $all_your_addresses, TO_TOKEN, $all_your_addresses, FROMDOMAIN_TOKEN, $all_your_wildcard_domains, TODOMAIN_TOKEN, $all_your_wildcard_domains); + } else { + return sprintf(" ( (%s %s) | (%s %s) ) ", FROM_TOKEN, $all_your_addresses, TO_TOKEN, $all_your_addresses); + } } @@ -645,12 +651,12 @@ class ModelSearchSearch extends Model { } - private function get_all_your_address() { + private function get_all_your_address($session_var) { $s = ''; $session = Registry::get('session'); - $emails = $session->get("emails"); + $emails = $session->get($session_var); while(list($k, $v) = each($emails)) { if($s) { $s .= '| ' . $this->fix_email_address_for_sphinx($v); } @@ -661,6 +667,25 @@ class ModelSearchSearch extends Model { } + private function get_wildcard_domains($arr=[]) { + $query_suffix = ''; + $results = $arr; + + $session = Registry::get('session'); + + $wildcard_domains = $session->get('wildcard_domains'); + + if($wildcard_domains) { + $q = str_repeat('?,', count($wildcard_domains)); + $q = trim($q, ','); + $results = array_merge($results, $wildcard_domains, $wildcard_domains); + $query_suffix = "OR fromdomain IN ($q) OR todomain IN ($q)"; + } + + return [$results, $query_suffix]; + } + + public function check_your_permission_by_id($id = '') { $q = ''; $arr = $a = array(); @@ -717,7 +742,8 @@ class ModelSearchSearch extends Model { if(Registry::get('auditor_user') == 1 && RESTRICTED_AUDITOR == 1) { $query = $this->db->query("SELECT id FROM " . VIEW_MESSAGES . " WHERE id=? AND ( `fromdomain` IN ($q) OR `todomain` IN ($q) )", $arr); } else { - $query = $this->db->query("SELECT id FROM " . VIEW_MESSAGES . " WHERE id=? AND ( `from` IN ($q) OR `to` IN ($q) )", $arr); + [$arr, $query_suffix] = $this->get_wildcard_domains($arr); + $query = $this->db->query("SELECT id FROM " . VIEW_MESSAGES . " WHERE id=? AND ( `from` IN ($q) OR `to` IN ($q) $query_suffix )", $arr); } if(isset($query->row['id'])) { return 1; } @@ -796,7 +822,9 @@ class ModelSearchSearch extends Model { } } - $query = $this->db->query("SELECT id FROM `" . VIEW_MESSAGES . "` WHERE `id` IN ($q2) AND ( `from` IN ($q) OR `to` IN ($q) )", $arr); + [$arr, $query_suffix] = $this->get_wildcard_domains($arr); + + $query = $this->db->query("SELECT id FROM `" . VIEW_MESSAGES . "` WHERE `id` IN ($q2) AND ( `from` IN ($q) OR `to` IN ($q) $query_suffix)", $arr); } } diff --git a/webui/view/theme/default/templates/user/settings.tpl b/webui/view/theme/default/templates/user/settings.tpl index d36c1d80..f90f1a72 100644 --- a/webui/view/theme/default/templates/user/settings.tpl +++ b/webui/view/theme/default/templates/user/settings.tpl @@ -1,55 +1,56 @@ -

    +

    - + - - - - - + + + + + - - - - - + + + + + - + - - - - - + + + + + + + + + + + + + - + - - - - - + + + + + - +
    : - -
    : +
    : - -
    :
    : - -
    :
    Wildcard domains:
    : - -
    :
    - +
    - -

    + +

    @@ -89,7 +90,7 @@
    - +
    From 8877285302f47c09ee963174b87c75a7658cc55d Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 1 Nov 2020 08:23:08 +0100 Subject: [PATCH 52/55] Wildcard domain fixes Signed-off-by: Janos SUTO --- webui/language/cz/messages.php | 1 + webui/language/de/messages.php | 1001 +++++++++-------- webui/language/en/messages.php | 1 + webui/language/es/messages.php | 1 + webui/language/fr/messages.php | 991 ++++++++-------- webui/language/hu/messages.php | 1 + webui/language/pl/messages.php | 1 + webui/language/pt/messages.php | 1 + webui/language/ru/messages.php | 1 + webui/language/tr/messages.php | 1 + .../theme/default/templates/user/settings.tpl | 11 +- 11 files changed, 1010 insertions(+), 1001 deletions(-) diff --git a/webui/language/cz/messages.php b/webui/language/cz/messages.php index 2d15dc50..2d18447c 100644 --- a/webui/language/cz/messages.php +++ b/webui/language/cz/messages.php @@ -491,3 +491,4 @@ $_['text_user_data_officer'] = "Data officer"; $_['text_no_selected_message'] = "no selected message"; $_['text_create_note'] = "Create note"; +$_['text_wildcard_domains'] = "Wildcard domains"; diff --git a/webui/language/de/messages.php b/webui/language/de/messages.php index bd98bc92..33d47636 100644 --- a/webui/language/de/messages.php +++ b/webui/language/de/messages.php @@ -1,500 +1,501 @@ - - - : - - - - Wildcard domains: + : + + : + + From d2512f0cff91676665e0b898347e152d2d6442c6 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 1 Nov 2020 08:28:59 +0100 Subject: [PATCH 53/55] Fixed recursive argument definition in pilerexport.c Signed-off-by: Janos SUTO --- src/pilerimport.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pilerimport.c b/src/pilerimport.c index 0f07d1e9..2291dac5 100644 --- a/src/pilerimport.c +++ b/src/pilerimport.c @@ -136,7 +136,7 @@ int main(int argc, char **argv){ {"start-position",required_argument, 0, 's' }, {"table-id", required_argument, 0, 'T' }, {"quiet", no_argument, 0, 'q' }, - {"recursive", required_argument, 0, 'R' }, + {"recursive", no_argument, 0, 'R' }, {"remove-after-import",no_argument, 0, 'r' }, {"failed-folder", required_argument, 0, 'j' }, {"move-folder", required_argument, 0, 'g' }, From 0d1e34007e777848f68044f1e9ae53bb7322d173 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 1 Nov 2020 08:51:59 +0100 Subject: [PATCH 54/55] Added before/since date support to imapfetch.py Signed-off-by: Janos SUTO --- util/imapfetch.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/util/imapfetch.py b/util/imapfetch.py index c213c323..8e8b82b0 100755 --- a/util/imapfetch.py +++ b/util/imapfetch.py @@ -70,7 +70,7 @@ def process_folder(conn, folder): cursor.execute("UPDATE import SET status=%s, total=total+%s WHERE id=%s", data) opts['db'].commit() - rc, data = conn.search(None, 'ALL') + rc, data = conn.search(None, opts['search']) for num in data[0].split(): rc, data = conn.fetch(num, '(RFC822)') if opts['verbose']: @@ -92,6 +92,8 @@ def main(): default="junk,trash,spam,draft") parser.add_argument("-f", "--folders", type=str, help="Comma separated list of IMAP folders to download") + parser.add_argument("--date", type=str, help="Search before/since a given date," + + "eg. (BEFORE \"01-Jan-2020\") or (SINCE \"01-Jan-2020\")") parser.add_argument("-d", "--dir", help="directory to chdir", default="/var/piler/imap") parser.add_argument("-i", "--import-from-table", action='store_true', @@ -108,10 +110,14 @@ def main(): opts['skip_folders'] = args.skip_list.split(',') opts['verbose'] = args.verbose + opts['search'] = 'ALL' opts['counter'] = 0 opts['db'] = None opts['id'] = 0 + if args.date: + opts['search'] = args.date + server = '' user = '' password = '' From f7cc52433ccda8137ecc6e9068f41a04981c0ac8 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 10 Nov 2020 20:59:11 +0100 Subject: [PATCH 55/55] #1124 Fixed pilerpurge to remove the last referenced attachment Signed-off-by: Janos SUTO --- util/pilerpurge.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/util/pilerpurge.py b/util/pilerpurge.py index 9de546f3..3239cd1a 100755 --- a/util/pilerpurge.py +++ b/util/pilerpurge.py @@ -93,7 +93,12 @@ def purge_attachments_by_attachment_id(opts={}): if rows == (): break else: - remove_attachment_files(rows, opts) + for (id, piler_id, attachment_id, refcount) in rows: + if opts['dry_run'] is False: + unlink(get_attachment_file_path(piler_id, attachment_id, + opts), opts) + else: + print(get_attachment_file_path(piler_id, attachment_id, opts)) def remove_attachment_files(rows=(), opts={}):