From 9225d5b32ef57ed479ad39cb5e83d7bc5e419ecc Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 13 Nov 2020 21:08:11 +0100 Subject: [PATCH 001/106] Obsoleted util/watch_sphinx_main_index.sh by refactoring indexer.main.sh Signed-off-by: Janos SUTO --- etc/cron.jobs.in | 1 - util/Makefile.in | 1 - util/indexer.main.sh.in | 49 +++++++++++++++++++++------------ util/watch_sphinx_main_index.sh | 3 -- 4 files changed, 32 insertions(+), 22 deletions(-) delete mode 100755 util/watch_sphinx_main_index.sh diff --git a/etc/cron.jobs.in b/etc/cron.jobs.in index c849a1e9..80d57523 100644 --- a/etc/cron.jobs.in +++ b/etc/cron.jobs.in @@ -6,7 +6,6 @@ */15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet note1 --rotate */5 * * * * /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \; */5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error -3 * * * * LIBEXECDIR/piler/watch_sphinx_main_index.sh 2 0 * * * LIBEXECDIR/piler/pilerpurge.py ### optional: populate accouting data diff --git a/util/Makefile.in b/util/Makefile.in index afe2701d..77c5f67b 100644 --- a/util/Makefile.in +++ b/util/Makefile.in @@ -46,7 +46,6 @@ install: $(INSTALL) -m 0755 $(srcdir)/purge.sh $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/pilerpurge.py $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/postinstall.sh $(DESTDIR)$(libexecdir)/piler - $(INSTALL) -m 0755 $(srcdir)/watch_sphinx_main_index.sh $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/db-mysql.sql $(DESTDIR)$(datarootdir)/piler $(INSTALL) -m 0755 $(srcdir)/db-mysql-root.sql.in $(DESTDIR)$(datarootdir)/piler diff --git a/util/indexer.main.sh.in b/util/indexer.main.sh.in index 8bf1549a..28d7756e 100755 --- a/util/indexer.main.sh.in +++ b/util/indexer.main.sh.in @@ -1,34 +1,49 @@ #!/bin/bash -export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin -MAINTMPFILE=/var/run/piler/main.indexer.tmp -INDEXER="indexer --config SYSCONFDIR/piler/sphinx.conf" -PRIORITY=mail.error -TOUCHFILE=/var/piler/stat/indexer +set -o nounset +set -o errexit +set -o pipefail -if [ -f $MAINTMPFILE ]; then echo "INDEXER ERROR: indexer merging to main index is already running. It started at "`cat $MAINTMPFILE` | logger -p $PRIORITY ; exit 1; fi +MAINTMPFILE="/var/run/piler/main.indexer.tmp" +SPHINX_CONFIG="SYSCONFDIR/piler/sphinx.conf" +PRIORITY="mail.error" +TOUCHFILE="/var/piler/stat/indexer" +MAIN_INDEX="main1" -date > $MAINTMPFILE +export PATH="/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin" -touch $TOUCHFILE - -function finish { - rm -f $MAINTMPFILE +finish() { + rm -f "$MAINTMPFILE" } + +if [[ -f "$MAINTMPFILE" ]]; then + echo "INDEXER ERROR: indexer merging to main index is already running. It started at $(cat "$MAINTMPFILE")" | logger -p "$PRIORITY" + exit 1 +fi + +date > "$MAINTMPFILE" + +touch "$TOUCHFILE" + trap finish EXIT -echo "INDEXER INFO: merging to main started" | logger -p $PRIORITY +echo "INDEXER INFO: merging to main started" | logger -p "$PRIORITY" -$INDEXER --quiet --merge main1 dailydelta1 --merge-dst-range deleted 0 0 --rotate +indexer --config "$SPHINX_CONFIG" --quiet --merge "$MAIN_INDEX" dailydelta1 --merge-dst-range deleted 0 0 --rotate -echo "INDEXER INFO: merging to main finished" | logger -p $PRIORITY +echo "INDEXER INFO: merging to main finished" | logger -p "$PRIORITY" sleep 5 -echo "INDEXER INFO: resetting daily delta started" | logger -p $PRIORITY +echo "INDEXER INFO: resetting daily delta started" | logger -p "$PRIORITY" -$INDEXER --quiet dailydelta1 --rotate +indexer --config "$SPHINX_CONFIG" --quiet dailydelta1 --rotate -echo "INDEXER INFO: resetting daily delta finished" | logger -p $PRIORITY +echo "INDEXER INFO: resetting daily delta finished" | logger -p "$PRIORITY" +sum=0 +while read -r a b; do + sum=$(( sum + b )) +done < <( find /var/piler/sphinx/ -type f -name main\*.spd -printf "%TY%Tm%Td %s\\n" ) +printf "%d" $sum > /var/piler/stat/main_index_size diff --git a/util/watch_sphinx_main_index.sh b/util/watch_sphinx_main_index.sh deleted file mode 100755 index 3d330f95..00000000 --- a/util/watch_sphinx_main_index.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -find /var/piler/sphinx/ -type f -name main\*.spd -printf "%TY%Tm%Td %s\\n" | sort -r | head -1 | cut -f2 -d ' ' > /var/piler/stat/main_index_size From bb8f6b1e004fac6ebcd2ae60950caf69a4759963 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 24 Nov 2020 21:44:42 +0100 Subject: [PATCH 002/106] Fixed google login Signed-off-by: Janos SUTO --- webui/model/user/google.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/webui/model/user/google.php b/webui/model/user/google.php index 7a375f9b..69a1d431 100644 --- a/webui/model/user/google.php +++ b/webui/model/user/google.php @@ -37,8 +37,8 @@ class ModelUserGoogle extends Model { $session->set("uid", $user['uid']); $session->set("admin_user", 0); $session->set("email", $user['username']); - $session->set("domain", $query->row['domain']); - $session->set("realname", $query->row['realname']); + $session->set("domain", $user['domain']); + $session->set("realname", $user['realname']); $session->set("emails", $this->model_user_user->get_users_all_email_addresses($user['uid'])); $session->set("folders", $this->model_folder_folder->get_folder_id_array_for_user($user['uid'])); From 535a4aa61c7cf2b44645c9cfc8cf32f56fb91d96 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 26 Nov 2020 19:01:45 +0100 Subject: [PATCH 003/106] Added security header feature Signed-off-by: Janos SUTO --- RELEASE_NOTES | 6 +++++ VERSION | 2 +- etc/example.conf | 8 +++++++ src/cfg.c | 1 + src/cfg.h | 2 ++ src/defs.h | 2 ++ src/parser.c | 4 ++++ src/parser_utils.c | 2 ++ src/piler.c | 60 +++++++++++++++++++++++++++------------------- 9 files changed, 61 insertions(+), 26 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 5b84e188..60c346e7 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,3 +1,9 @@ +1.3.10: +------- + +- Added security header feature + + 1.3.9: ------ diff --git a/VERSION b/VERSION index d4c4950a..0c00f610 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.9 +1.3.10 diff --git a/etc/example.conf b/etc/example.conf index c002a15d..70e20231 100644 --- a/etc/example.conf +++ b/etc/example.conf @@ -221,3 +221,11 @@ tweak_sent_time_offset=0 ; whether to enable (1) or not (0) the extra mmap dedup test feature ; if you change it, be sure to stop, then start piler mmap_dedup_test=0 + +; security header that must be present in the first mail header of +; the message. If the security_header value is not empty, then the +; parser checks for this header line. Unless it's found it will discard +; the given email. Note that this feature is supposed to be a security +; mechanism against unwanted email in the archive if limiting smtp +; clients via an IP-address list is not feasible. +security_header= diff --git a/src/cfg.c b/src/cfg.c index 0c40b705..023d04e6 100644 --- a/src/cfg.c +++ b/src/cfg.c @@ -84,6 +84,7 @@ struct _parse_rule config_parse_rules[] = { "piler_header_field", "string", (void*) string_parser, offsetof(struct config, piler_header_field), "X-piler-id:", MAXVAL-1}, { "process_rcpt_to_addresses", "integer", (void*) int_parser, offsetof(struct config, process_rcpt_to_addresses), "0", sizeof(int)}, { "queuedir", "string", (void*) string_parser, offsetof(struct config, queuedir), QUEUE_DIR, MAXVAL-1}, + { "security_header", "string", (void*) string_parser, offsetof(struct config, security_header), "", MAXVAL-1}, { "server_id", "integer", (void*) int_parser, offsetof(struct config, server_id), "0", sizeof(int)}, { "smtp_timeout", "integer", (void*) int_parser, offsetof(struct config, smtp_timeout), "60", sizeof(int)}, { "spam_header_line", "string", (void*) string_parser, offsetof(struct config, spam_header_line), "", MAXVAL-1}, diff --git a/src/cfg.h b/src/cfg.h index 03ea7d12..6932a6c4 100644 --- a/src/cfg.h +++ b/src/cfg.h @@ -64,6 +64,8 @@ struct config { int default_retention_days; + char security_header[MAXVAL]; + // mysql stuff char mysqlcharset[MAXVAL]; diff --git a/src/defs.h b/src/defs.h index f7477a65..7361b453 100644 --- a/src/defs.h +++ b/src/defs.h @@ -209,6 +209,8 @@ struct parser_state { unsigned int bodylen; unsigned int tolen; unsigned int todomainlen; + unsigned int found_security_header; + int journaltolen; int retention; diff --git a/src/parser.c b/src/parser.c index 3329effd..e3c36528 100644 --- a/src/parser.c +++ b/src/parser.c @@ -199,6 +199,10 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata sdata->restored_copy = 1; } + if(cfg->security_header && state->found_security_header == 0 && strstr(buf, cfg->security_header)){ + state->found_security_header = 1; + } + if(*(cfg->piler_header_field) != 0 && strncmp(buf, cfg->piler_header_field, strlen(cfg->piler_header_field)) == 0){ sdata->restored_copy = 1; } diff --git a/src/parser_utils.c b/src/parser_utils.c index 45e03048..5c56a6e3 100644 --- a/src/parser_utils.c +++ b/src/parser_utils.c @@ -102,6 +102,8 @@ void init_state(struct parser_state *state){ state->journaltolen = 0; state->retention = 0; + + state->found_security_header = 0; } diff --git a/src/piler.c b/src/piler.c index 46b99e18..684d90de 100644 --- a/src/piler.c +++ b/src/piler.c @@ -101,11 +101,43 @@ void child_sighup_handler(int sig){ } +int perform_checks(char *filename, struct session_data *sdata, struct data *data, struct parser_state *parser_state, struct config *cfg){ + + if(cfg->security_header && parser_state->found_security_header == 0){ + syslog(LOG_PRIORITY, "%s: discarding: missing security header", filename); + return ERR_DISCARDED; + } + + char *arule = check_against_ruleset(data->archiving_rules, parser_state, sdata->tot_len, sdata->spam_message); + + if(arule){ + syslog(LOG_PRIORITY, "%s: discarding: archiving policy: *%s*", filename, arule); + return ERR_DISCARDED; + } + + + if(cfg->archive_only_mydomains == 1 && sdata->internal_sender == 0 && sdata->internal_recipient == 0){ + syslog(LOG_PRIORITY, "%s: discarding: not on mydomains", filename); + return ERR_DISCARDED; + } + + make_digests(sdata, cfg); + + if(sdata->hdr_len < 10){ + syslog(LOG_PRIORITY, "%s: invalid message, hdr_len: %d", filename, sdata->hdr_len); + return ERR; + } + + int rc = process_message(sdata, parser_state, data, cfg); + unlink(parser_state->message_id_hash); + + return rc; +} + + int process_email(char *filename, struct session_data *sdata, struct data *data, int size, struct config *cfg){ - int rc; char tmpbuf[SMALLBUFSIZE]; char *status=S_STATUS_UNDEF; - char *arule; char *p; struct timezone tz; struct timeval tv1, tv2; @@ -144,29 +176,7 @@ int process_email(char *filename, struct session_data *sdata, struct data *data, } while(rcpt); } - arule = check_against_ruleset(data->archiving_rules, &parser_state, sdata->tot_len, sdata->spam_message); - - if(arule){ - syslog(LOG_PRIORITY, "%s: discarding: archiving policy: *%s*", filename, arule); - rc = ERR_DISCARDED; - } - else if(cfg->archive_only_mydomains == 1 && sdata->internal_sender == 0 && sdata->internal_recipient == 0){ - syslog(LOG_PRIORITY, "%s: discarding: not on mydomains", filename); - rc = ERR_DISCARDED; - } - else { - make_digests(sdata, cfg); - - if(sdata->hdr_len < 10){ - syslog(LOG_PRIORITY, "%s: invalid message, hdr_len: %d", filename, sdata->hdr_len); - rc = ERR; - } - else { - rc = process_message(sdata, &parser_state, data, cfg); - unlink(parser_state.message_id_hash); - } - } - + int rc = perform_checks(filename, sdata, data, &parser_state, cfg); unlink(sdata->tmpframe); remove_stripped_attachments(&parser_state); From 8c6560f302d9dc536975b59d36937f0b6770aa54 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 28 Nov 2020 17:45:37 +0100 Subject: [PATCH 004/106] Disable the security_header check for imported emails Signed-off-by: Janos SUTO --- src/pilerimport.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pilerimport.c b/src/pilerimport.c index 2291dac5..dcf65338 100644 --- a/src/pilerimport.c +++ b/src/pilerimport.c @@ -300,6 +300,8 @@ int main(int argc, char **argv){ cfg = read_config(configfile); + memset(cfg.security_header, 0, MAXVAL); + if((data.recursive_folder_names == 1 || data.import->folder) && cfg.enable_folders == 0){ printf("please set enable_folders=1 in piler.conf to use the folder options\n"); return ERR; From 2ed654c87f48c77052ae7c02c16d13bb5294d437 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 3 Dec 2020 18:12:48 +0100 Subject: [PATCH 005/106] Prepend a random garbage block to the data to be encrypted Signed-off-by: Janos SUTO --- src/archive.c | 15 ++++++++++++++- src/store.c | 28 +++++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/archive.c b/src/archive.c index 9984c26f..f12009e2 100644 --- a/src/archive.c +++ b/src/archive.c @@ -58,6 +58,10 @@ int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){ char *new_ptr; unsigned char out[REALLYBIGBUFSIZE]; + /* expecting deflate with 32k window size (0x78) */ + if(len > 0 && in[0] != 0x78) + return Z_DATA_ERROR; + /* allocate inflate state */ strm.zalloc = Z_NULL; @@ -139,6 +143,7 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de #else EVP_CIPHER_CTX *ctx=NULL; #endif + int blocklen; if(filename == NULL) return 1; @@ -162,15 +167,17 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de #if OPENSSL_VERSION_NUMBER < 0x10100000L EVP_CIPHER_CTX_init(&ctx); EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv); + blocklen = EVP_CIPHER_CTX_block_size(&ctx); #else ctx = EVP_CIPHER_CTX_new(); if(!ctx) goto CLEANUP; EVP_CIPHER_CTX_init(ctx); EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv); + blocklen = EVP_CIPHER_CTX_block_size(ctx); #endif - len = st.st_size+EVP_MAX_BLOCK_LENGTH; + len = st.st_size+blocklen; s = malloc(len); @@ -207,7 +214,13 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de tlen += olen; + + // old fileformat with static IV rc = inf(s, tlen, mode, buffer, dest); + // new fileformat, starting with blocklen bytes of garbage + if(rc != Z_OK && tlen >= blocklen){ + rc = inf(s+blocklen, tlen-blocklen, mode, buffer, dest); + } } else { addr = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); diff --git a/src/store.c b/src/store.c index d903282d..29e11907 100644 --- a/src/store.c +++ b/src/store.c @@ -51,6 +51,8 @@ int store_file(struct session_data *sdata, char *filename, int len, struct confi #else EVP_CIPHER_CTX *ctx; #endif + int blocklen; + unsigned char rnd[EVP_MAX_BLOCK_LENGTH]; unsigned char *outbuf=NULL; int outlen=0, writelen, tmplen; @@ -108,25 +110,45 @@ int store_file(struct session_data *sdata, char *filename, int len, struct confi #if OPENSSL_VERSION_NUMBER < 0x10100000L EVP_CIPHER_CTX_init(&ctx); EVP_EncryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv); + blocklen = EVP_CIPHER_CTX_block_size(&ctx); #else ctx = EVP_CIPHER_CTX_new(); if(!ctx) goto ENDE; EVP_CIPHER_CTX_init(ctx); EVP_EncryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv); + blocklen = EVP_CIPHER_CTX_block_size(ctx); #endif - outbuf = malloc(dstlen + EVP_MAX_BLOCK_LENGTH); + // prepend a block with random data as replacement for dynamic iv + // see e.g. https://crypto.stackexchange.com/questions/5421/using-cbc-with-a-fixed-iv-and-a-random-first-plaintext-block + fd = open(RANDOM_POOL, O_RDONLY); + if(fd == -1) goto ENDE; + tmplen = readFromEntropyPool(fd, rnd, blocklen); + close(fd); + if(tmplen != blocklen) goto ENDE; + // make sure, random data does not start with zlib magic 0x78 + if(rnd[0] == 0x78) rnd[0] =~ rnd[0]; + + outbuf = malloc(dstlen + blocklen * 2); if(outbuf == NULL) goto ENDE; #if OPENSSL_VERSION_NUMBER < 0x10100000L - if(!EVP_EncryptUpdate(&ctx, outbuf, &outlen, z, dstlen)) goto ENDE; + if(!EVP_EncryptUpdate(&ctx, outbuf, &outlen, rnd, blocklen)) goto ENDE; + if(!EVP_EncryptUpdate(&ctx, outbuf + outlen, &tmplen, z, dstlen)) goto ENDE; + #else + if(!EVP_EncryptUpdate(ctx, outbuf, &outlen, rnd, blocklen)) goto ENDE; + if(!EVP_EncryptUpdate(ctx, outbuf + outlen, &tmplen, z, dstlen)) goto ENDE; + #endif + outlen += tmplen; + + #if OPENSSL_VERSION_NUMBER < 0x10100000L if(!EVP_EncryptFinal_ex(&ctx, outbuf + outlen, &tmplen)) goto ENDE; #else - if(!EVP_EncryptUpdate(ctx, outbuf, &outlen, z, dstlen)) goto ENDE; if(!EVP_EncryptFinal_ex(ctx, outbuf + outlen, &tmplen)) goto ENDE; #endif outlen += tmplen; + #if OPENSSL_VERSION_NUMBER < 0x10100000L EVP_CIPHER_CTX_cleanup(&ctx); #else From e8eb74d07aeb5242bdf1c34d3c056336ea8dd81c Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 4 Dec 2020 13:58:47 +0100 Subject: [PATCH 006/106] Fixed gz attachment type search in determine_attachment_type() Signed-off-by: Janos SUTO --- src/parser_utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parser_utils.c b/src/parser_utils.c index 5c56a6e3..87e91cb8 100644 --- a/src/parser_utils.c +++ b/src/parser_utils.c @@ -959,7 +959,7 @@ char *determine_attachment_type(char *filename, char *type){ if(strncasecmp(p, "rar", 3) == 0) return "compressed,"; // tar.gz has the same type - if(strncasecmp(p, "x-gzip", 3) == 0) return "compressed,"; + if(strncasecmp(p, "gz", 2) == 0) return "compressed,"; if(strncasecmp(p, "rtf", 3) == 0) return "word,"; if(strncasecmp(p, "doc", 3) == 0) return "word,"; From a35f8903f7117e04065f2db330e51e86da269c4e Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 4 Dec 2020 13:59:47 +0100 Subject: [PATCH 007/106] Reindex should process the attachments as well Signed-off-by: Janos SUTO --- src/reindex.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/reindex.c b/src/reindex.c index ad04b094..b2b137b4 100644 --- a/src/reindex.c +++ b/src/reindex.c @@ -123,7 +123,7 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename); - state = parse_message(sdata, 0, data, cfg); + state = parse_message(sdata, 1, data, cfg); post_parse(sdata, &state, cfg); rc = store_index_data(sdata, &state, data, stored_id, cfg); From 613f25848d3c0b17d28f2ddd43d660947d1edc09 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 4 Dec 2020 14:07:25 +0100 Subject: [PATCH 008/106] Keep a whitespace character when removing xml tags Signed-off-by: Janos SUTO --- src/extract.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/extract.c b/src/extract.c index 9d391449..488faaf7 100644 --- a/src/extract.c +++ b/src/extract.c @@ -25,12 +25,20 @@ int remove_xml(char *src, char *dest, int destlen, int *html){ memset(dest, 0, destlen); for(; *src; src++){ - if(*src == '<'){ *html = 1; continue; } - if(*src == '>'){ *html = 0; continue; } - - if(*html == 0){ - if(i < destlen) *(dest+i) = *src; - i++; + if(*src == '<'){ + *html = 1; + } + else if(*src == '>'){ + *html = 0; + // make sure there's a whitespace between tag contents + if(i < destlen && i > 0 && !isspace(dest[i-1])) + dest[i++] = ' '; + } + else{ + if(*html == 0){ + if(i < destlen) *(dest+i) = *src; + i++; + } } } From d2f3f018c0147b74383e5a27c40b2caa7e3c58db Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 4 Dec 2020 16:30:59 +0100 Subject: [PATCH 009/106] Introduced new AES-256 encryption method for new emails Signed-off-by: Janos SUTO --- src/archive.c | 15 +++++++++++++-- src/misc.c | 4 +++- src/store.c | 4 ++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/archive.c b/src/archive.c index f12009e2..8ea7b504 100644 --- a/src/archive.c +++ b/src/archive.c @@ -162,18 +162,29 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de return 1; } + // The new encryption scheme uses piler id starting with 5000.... if(cfg->encrypt_messages == 1){ #if OPENSSL_VERSION_NUMBER < 0x10100000L EVP_CIPHER_CTX_init(&ctx); - EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv); + if(strstr(filename, "/5000")){ + EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv); + } else { + EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv); + } + blocklen = EVP_CIPHER_CTX_block_size(&ctx); #else ctx = EVP_CIPHER_CTX_new(); if(!ctx) goto CLEANUP; EVP_CIPHER_CTX_init(ctx); - EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv); + if(strstr(filename, "/5000")){ + EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv); + } else { + EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv); + } + blocklen = EVP_CIPHER_CTX_block_size(ctx); #endif diff --git a/src/misc.c b/src/misc.c index d723f9ec..e5fe4c42 100644 --- a/src/misc.c +++ b/src/misc.c @@ -265,11 +265,13 @@ void create_id(char *id, unsigned char server_id){ get_random_bytes(buf, RND_STR_LEN/2, server_id); + // New encryption scheme using AES-256 + buf[0] = 0x50; + for(i=0; i < RND_STR_LEN/2; i++){ sprintf(id, "%02x", buf[i]); id += 2; } - } diff --git a/src/store.c b/src/store.c index 29e11907..ec58d98f 100644 --- a/src/store.c +++ b/src/store.c @@ -109,14 +109,14 @@ int store_file(struct session_data *sdata, char *filename, int len, struct confi #if OPENSSL_VERSION_NUMBER < 0x10100000L EVP_CIPHER_CTX_init(&ctx); - EVP_EncryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv); + EVP_EncryptInit_ex(&ctx, EVP_aes_256_cbc()(), NULL, cfg->key, cfg->iv); blocklen = EVP_CIPHER_CTX_block_size(&ctx); #else ctx = EVP_CIPHER_CTX_new(); if(!ctx) goto ENDE; EVP_CIPHER_CTX_init(ctx); - EVP_EncryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv); + EVP_EncryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv); blocklen = EVP_CIPHER_CTX_block_size(ctx); #endif From 33ce44dea52b4140d13a17cc913219e03eddacd3 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 4 Dec 2020 16:42:35 +0100 Subject: [PATCH 010/106] Fixed unit test to new piler id Signed-off-by: Janos SUTO --- unit_tests/check_misc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/check_misc.c b/unit_tests/check_misc.c index 923f6a30..2a086e59 100644 --- a/unit_tests/check_misc.c +++ b/unit_tests/check_misc.c @@ -112,7 +112,7 @@ static void test_create_id(){ for(i=0; i<10; i++){ create_id(buf, 0xf); - ASSERT(strncmp(buf, "40000000", strlen("40000000")) == 0, buf); + ASSERT(strncmp(buf, "50000000", strlen("50000000")) == 0, buf); ASSERT(buf[24] == '0' && buf[25] == 'f', buf); } From 82f6d0c0cfb1f07016cd05d4e863b2ccd07d80c7 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 4 Dec 2020 19:38:21 +0100 Subject: [PATCH 011/106] Fixed typo for openssl 1.0.x EVP_bf_cbc() Signed-off-by: Janos SUTO --- src/store.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/store.c b/src/store.c index ec58d98f..31323f53 100644 --- a/src/store.c +++ b/src/store.c @@ -109,7 +109,7 @@ int store_file(struct session_data *sdata, char *filename, int len, struct confi #if OPENSSL_VERSION_NUMBER < 0x10100000L EVP_CIPHER_CTX_init(&ctx); - EVP_EncryptInit_ex(&ctx, EVP_aes_256_cbc()(), NULL, cfg->key, cfg->iv); + EVP_EncryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv); blocklen = EVP_CIPHER_CTX_block_size(&ctx); #else ctx = EVP_CIPHER_CTX_new(); From c08bd7393ff9d76ee88ff7148694199bf92322a0 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 4 Dec 2020 19:43:28 +0100 Subject: [PATCH 012/106] Updated the iv config parameter description Signed-off-by: Janos SUTO --- etc/example.conf | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/etc/example.conf b/etc/example.conf index 70e20231..4816aa9e 100644 --- a/etc/example.conf +++ b/etc/example.conf @@ -22,10 +22,11 @@ username=piler ; The default is 7 years + 2 days (=7*365+2=2557 days) default_retention_days=2557 -; this is a 16 character long vector -; after the installation you must not change it ever -; otherwise you can't access your emails -;iv=**************** +; The initialization vector for encryption. +; By now it has become obsolete. Don't use it for +; new installations. However, if you used it before +; then you must keep it as it is. +;iv= ; whether to encrypt messages (1) or not (0). ; Make sure to set this value to your needs right after installing piler, From d9d5f7db66a58d52362b37186269cbbd2626e8a9 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 12 Dec 2020 21:43:16 +0100 Subject: [PATCH 013/106] Handle the Sender: header line Signed-off-by: Janos SUTO --- src/defs.h | 23 ++++++++++++----------- src/import.c | 2 +- src/message.c | 14 +++++++------- src/parser.c | 23 +++++++++++++++++++++-- src/parser.h | 2 +- src/parser_utils.c | 2 ++ src/piler.c | 2 +- src/reindex.c | 2 +- src/test.c | 5 +++-- src/tokenizer.c | 12 ++++++++++++ unit_tests/check_attachments.c | 2 +- unit_tests/check_rules.c | 6 +++--- unit_tests/common.c | 4 ++-- 13 files changed, 67 insertions(+), 32 deletions(-) diff --git a/src/defs.h b/src/defs.h index 7361b453..d6c653b4 100644 --- a/src/defs.h +++ b/src/defs.h @@ -27,16 +27,17 @@ #define MSG_BODY 0 #define MSG_RECEIVED 1 #define MSG_FROM 2 -#define MSG_TO 3 -#define MSG_CC 4 -#define MSG_SUBJECT 5 -#define MSG_CONTENT_TYPE 6 -#define MSG_CONTENT_TRANSFER_ENCODING 7 -#define MSG_CONTENT_DISPOSITION 8 -#define MSG_MESSAGE_ID 9 -#define MSG_REFERENCES 10 -#define MSG_RECIPIENT 11 -#define MSG_ENVELOPE_TO 12 +#define MSG_SENDER 3 +#define MSG_TO 4 +#define MSG_CC 5 +#define MSG_SUBJECT 6 +#define MSG_CONTENT_TYPE 7 +#define MSG_CONTENT_TRANSFER_ENCODING 8 +#define MSG_CONTENT_DISPOSITION 9 +#define MSG_MESSAGE_ID 10 +#define MSG_REFERENCES 11 +#define MSG_RECIPIENT 12 +#define MSG_ENVELOPE_TO 13 #define MAXHASH 277 @@ -203,7 +204,7 @@ struct parser_state { char reference[SMALLBUFSIZE]; - char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE]; + char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_sender[SMALLBUFSIZE], b_sender_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE]; char b_journal_to[MAXBUFSIZE]; unsigned int bodylen; diff --git a/src/import.c b/src/import.c index 601c9b84..c3decf9f 100644 --- a/src/import.c +++ b/src/import.c @@ -71,7 +71,7 @@ int import_message(struct session_data *sdata, struct data *data, struct config sdata->import = 1; state = parse_message(sdata, 1, data, cfg); - post_parse(sdata, &state, cfg); + post_parse(sdata, data, &state, cfg); rule = check_against_ruleset(data->archiving_rules, &state, sdata->tot_len, sdata->spam_message); if(rule){ diff --git a/src/message.c b/src/message.c index 7700fcdb..5c755839 100644 --- a/src/message.c +++ b/src/message.c @@ -34,18 +34,18 @@ int store_index_data(struct session_data *sdata, struct parser_state *state, str if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc; - fix_email_address_for_sphinx(state->b_from); + fix_email_address_for_sphinx(state->b_sender); fix_email_address_for_sphinx(state->b_to); - fix_email_address_for_sphinx(state->b_from_domain); + fix_email_address_for_sphinx(state->b_sender_domain); fix_email_address_for_sphinx(state->b_to_domain); p_bind_init(&sql); sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++; - sql.sql[sql.pos] = state->b_from; sql.type[sql.pos] = TYPE_STRING; sql.pos++; + sql.sql[sql.pos] = state->b_sender; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = state->b_to; sql.type[sql.pos] = TYPE_STRING; sql.pos++; - sql.sql[sql.pos] = state->b_from_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++; + sql.sql[sql.pos] = state->b_sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = state->b_to_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = state->b_body; sql.type[sql.pos] = TYPE_STRING; sql.pos++; @@ -188,7 +188,7 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru subj = state->b_subject; if(*subj == ' ') subj++; - snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, state->b_from, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest); + snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, state->b_sender, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest); digest_string(s, &vcode[0]); @@ -203,7 +203,7 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru memset(s2, 0, sizeof(s2)); - p = state->b_from; + p = state->b_sender; do { memset(s2, 0, sizeof(s2)); p = split(p, ' ', s2, sizeof(s2)-1, &result); @@ -222,7 +222,7 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru p_bind_init(&sql); sql.sql[sql.pos] = &s2[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++; - sql.sql[sql.pos] = state->b_from_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++; + sql.sql[sql.pos] = state->b_sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = (char *)&sdata->spam_message; sql.type[sql.pos] = TYPE_LONG; sql.pos++; sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++; diff --git a/src/parser.c b/src/parser.c index e3c36528..bcbba16d 100644 --- a/src/parser.c +++ b/src/parser.c @@ -72,7 +72,7 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec } -void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg){ +void post_parse(struct session_data *sdata, struct data *data, struct parser_state *state, struct config *cfg){ int i; clearhash(state->boundaries); @@ -84,6 +84,19 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c if(strlen(state->b_from) > 255) state->b_from[255] = '\0'; if(strlen(state->b_from_domain) > 255) state->b_from_domain[255] = '\0'; + if(strlen(state->b_sender) > 255) state->b_sender[255] = '\0'; + if(strlen(state->b_sender_domain) > 255) state->b_sender_domain[255] = '\0'; + + // If Sender: header doesn't exist, then copy the From: header value to it + // Otherwise append the From: address to the recipients list + + if(state->b_sender[0] == '\0'){ + strcpy(state->b_sender, state->b_from); + strcpy(state->b_sender_domain, state->b_from_domain); + } else { + add_recipient(state->b_from, strlen(state->b_from), sdata, state, data, cfg); + } + // Truncate the message_id if it's >255 characters if(strlen(state->message_id) > 255) state->message_id[255] = '\0'; @@ -358,6 +371,10 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata state->message_state = MSG_FROM; buf += strlen("From:"); } + else if(strncasecmp(buf, "Sender:", strlen("Sender:")) == 0){ + state->message_state = MSG_SENDER; + buf += strlen("Sender:"); + } else if(strncasecmp(buf, "Content-Type:", strlen("Content-Type:")) == 0){ state->message_state = MSG_CONTENT_TYPE; } @@ -545,6 +562,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata memset(state->b_body, 0, BIGBUFSIZE); memset(state->b_from, 0, SMALLBUFSIZE); memset(state->b_from_domain, 0, SMALLBUFSIZE); + memset(state->b_sender, 0, SMALLBUFSIZE); + memset(state->b_sender_domain, 0, SMALLBUFSIZE); memset(state->message_id, 0, SMALLBUFSIZE); sdata->ms_journal = 0; @@ -616,7 +635,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata /* skip irrelevant headers */ - if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT && state->message_state != MSG_ENVELOPE_TO) return 0; + if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_SENDER && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT && state->message_state != MSG_ENVELOPE_TO) return 0; /* don't process body if it's not a text or html part */ diff --git a/src/parser.h b/src/parser.h index 5525af8a..ca81e3bc 100644 --- a/src/parser.h +++ b/src/parser.h @@ -10,7 +10,7 @@ #include "defs.h" struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct data *data, struct config *cfg); -void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg); +void post_parse(struct session_data *sdata, struct data *data, struct parser_state *state, struct config *cfg); int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg); void init_state(struct parser_state *state); diff --git a/src/parser_utils.c b/src/parser_utils.c index 87e91cb8..6cd2bbf9 100644 --- a/src/parser_utils.c +++ b/src/parser_utils.c @@ -90,6 +90,8 @@ void init_state(struct parser_state *state){ memset(state->b_from, 0, SMALLBUFSIZE); memset(state->b_from_domain, 0, SMALLBUFSIZE); + memset(state->b_sender, 0, SMALLBUFSIZE); + memset(state->b_sender_domain, 0, SMALLBUFSIZE); memset(state->b_to, 0, MAXBUFSIZE); memset(state->b_to_domain, 0, SMALLBUFSIZE); memset(state->b_subject, 0, MAXBUFSIZE); diff --git a/src/piler.c b/src/piler.c index 684d90de..fa49a074 100644 --- a/src/piler.c +++ b/src/piler.c @@ -163,7 +163,7 @@ int process_email(char *filename, struct session_data *sdata, struct data *data, parser_state = parse_message(sdata, 1, data, cfg); - post_parse(sdata, &parser_state, cfg); + post_parse(sdata, data, &parser_state, cfg); if(cfg->syslog_recipients == 1){ char *rcpt = parser_state.b_to; diff --git a/src/reindex.c b/src/reindex.c index b2b137b4..812fbe4b 100644 --- a/src/reindex.c +++ b/src/reindex.c @@ -124,7 +124,7 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename); state = parse_message(sdata, 1, data, cfg); - post_parse(sdata, &state, cfg); + post_parse(sdata, data, &state, cfg); rc = store_index_data(sdata, &state, data, stored_id, cfg); diff --git a/src/test.c b/src/test.c index 91e2bcfc..4d7a8a3f 100644 --- a/src/test.c +++ b/src/test.c @@ -134,7 +134,7 @@ int main(int argc, char **argv){ init_session_data(&sdata, &cfg); - + sdata.delivered = 0; sdata.tot_len = st.st_size; sdata.import = 1; @@ -147,10 +147,11 @@ int main(int argc, char **argv){ state = parse_message(&sdata, 1, &data, &cfg); printf("post parsing...\n"); - post_parse(&sdata, &state, &cfg); + post_parse(&sdata, &data, &state, &cfg); printf("message-id: %s / %s\n", state.message_id, state.message_id_hash); printf("from: *%s (%s)*\n", state.b_from, state.b_from_domain); + printf("sender: *%s (%s)*\n", state.b_sender, state.b_sender_domain); printf("to: *%s (%s)*\n", state.b_to, state.b_to_domain); printf("reference: *%s*\n", state.reference); printf("subject: *%s*\n", state.b_subject); diff --git a/src/tokenizer.c b/src/tokenizer.c index 52edbe59..ebadbcff 100644 --- a/src/tokenizer.c +++ b/src/tokenizer.c @@ -75,6 +75,18 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata, } } } + else if(state->message_state == MSG_SENDER && state->is_1st_header == 1 && strlen(state->b_sender) < SMALLBUFSIZE-len-1){ + strtolower(puf); + + q = strchr(puf, '@'); + if(q) fix_plus_sign_in_email_address(puf, &q, &len); + + memcpy(&(state->b_sender[strlen(state->b_sender)]), puf, len); + if(strlen(state->b_sender) < SMALLBUFSIZE-len-1){ + split_email_address(puf); + memcpy(&(state->b_sender[strlen(state->b_sender)]), puf, len); + } + } else if((state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT || state->message_state == MSG_ENVELOPE_TO) && state->is_1st_header == 1 && state->tolen < MAXBUFSIZE-len-1){ strtolower(puf); diff --git a/unit_tests/check_attachments.c b/unit_tests/check_attachments.c index e051ea40..f6ee2b1c 100644 --- a/unit_tests/check_attachments.c +++ b/unit_tests/check_attachments.c @@ -43,7 +43,7 @@ static void test_attachments(struct config *cfg){ snprintf(sdata.tmpframe, SMALLBUFSIZE-1, "%s.m", tests[i].s); state = parse_message(&sdata, 1, &data, cfg); - post_parse(&sdata, &state, cfg); + post_parse(&sdata, &data, &state, cfg); for(j=1; j<=state.n_attachments; j++){ unlink(state.attachments[j].internalname); diff --git a/unit_tests/check_rules.c b/unit_tests/check_rules.c index 596a8c41..47d18ca8 100644 --- a/unit_tests/check_rules.c +++ b/unit_tests/check_rules.c @@ -41,7 +41,7 @@ static void fill_rule_table(struct config *cfg){ for(i=0; idelivered = 0; sdata->tot_len = st.st_size; @@ -30,7 +30,7 @@ int setup_and_parse_message(struct session_data *sdata, struct parser_state *sta snprintf(sdata->tmpframe, SMALLBUFSIZE-1, "%s.m", filename); *state = parse_message(sdata, 1, data, cfg); - post_parse(sdata, state, cfg); + post_parse(sdata, data, state, cfg); return 0; } From 4b37b4fc1aa4b39159d7180ea850be1e4d8ea47c Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 13 Dec 2020 07:50:03 +0100 Subject: [PATCH 014/106] Fixed from/sender processing Signed-off-by: Janos SUTO --- src/message.c | 43 +++++++++++++++++++++------------------ src/parser.c | 14 +++++-------- src/parser.h | 1 + src/parser_utils.c | 21 +++++++++++++++++-- src/tokenizer.c | 13 +++++++++--- unit_tests/check_parser.c | 38 ++++++++++++++++++---------------- 6 files changed, 79 insertions(+), 51 deletions(-) diff --git a/src/message.c b/src/message.c index 5c755839..1b281fab 100644 --- a/src/message.c +++ b/src/message.c @@ -20,7 +20,7 @@ int store_index_data(struct session_data *sdata, struct parser_state *state, struct data *data, uint64 id, struct config *cfg){ int rc=ERR; - char *subj; + char *subj, *sender=state->b_from, *sender_domain=state->b_from_domain; struct sql sql; if(data->folder == 0){ @@ -34,18 +34,24 @@ int store_index_data(struct session_data *sdata, struct parser_state *state, str if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc; + fix_email_address_for_sphinx(state->b_from); fix_email_address_for_sphinx(state->b_sender); fix_email_address_for_sphinx(state->b_to); + fix_email_address_for_sphinx(state->b_from_domain); fix_email_address_for_sphinx(state->b_sender_domain); fix_email_address_for_sphinx(state->b_to_domain); + if(state->b_sender_domain){ + sender = state->b_sender; + sender_domain = state->b_sender_domain; + } p_bind_init(&sql); sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++; - sql.sql[sql.pos] = state->b_sender; sql.type[sql.pos] = TYPE_STRING; sql.pos++; + sql.sql[sql.pos] = sender; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = state->b_to; sql.type[sql.pos] = TYPE_STRING; sql.pos++; - sql.sql[sql.pos] = state->b_sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++; + sql.sql[sql.pos] = sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = state->b_to_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = state->b_body; sql.type[sql.pos] = TYPE_STRING; sql.pos++; @@ -180,15 +186,25 @@ int update_metadata_reference(struct session_data *sdata, struct parser_state *s int store_meta_data(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){ - int rc=ERR, result; - char *subj, *p, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1]; + int rc=ERR; + char *subj, *sender, *sender_domain, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1]; uint64 id=0; struct sql sql; subj = state->b_subject; if(*subj == ' ') subj++; - snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, state->b_sender, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest); + if(state->b_sender_domain){ + sender = state->b_sender; + sender_domain = state->b_sender_domain; + get_first_email_address_from_string(state->b_sender, s2, sizeof(s2)); + } else { + sender = state->b_from; + sender_domain = state->b_from_domain; + get_first_email_address_from_string(state->b_from, s2, sizeof(s2)); + } + + snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, sender, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest); digest_string(s, &vcode[0]); @@ -201,19 +217,6 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR; - memset(s2, 0, sizeof(s2)); - - p = state->b_sender; - do { - memset(s2, 0, sizeof(s2)); - p = split(p, ' ', s2, sizeof(s2)-1, &result); - - if(s2[0] == '\0') continue; - - if(does_it_seem_like_an_email_address(s2) == 1){ break; } - } while(p); - - if(strlen(state->b_to) < 5){ snprintf(state->b_to, SMALLBUFSIZE-1, "undisclosed-recipients@no.domain"); } @@ -222,7 +225,7 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru p_bind_init(&sql); sql.sql[sql.pos] = &s2[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++; - sql.sql[sql.pos] = state->b_sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++; + sql.sql[sql.pos] = sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = (char *)&sdata->spam_message; sql.type[sql.pos] = TYPE_LONG; sql.pos++; sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++; diff --git a/src/parser.c b/src/parser.c index bcbba16d..cd9224b4 100644 --- a/src/parser.c +++ b/src/parser.c @@ -80,22 +80,18 @@ void post_parse(struct session_data *sdata, struct data *data, struct parser_sta clearhash(state->rcpt_domain); clearhash(state->journal_recipient); - // Fix From: line if it's too long + // Fix From: and Sender: lines if they are too long if(strlen(state->b_from) > 255) state->b_from[255] = '\0'; if(strlen(state->b_from_domain) > 255) state->b_from_domain[255] = '\0'; if(strlen(state->b_sender) > 255) state->b_sender[255] = '\0'; if(strlen(state->b_sender_domain) > 255) state->b_sender_domain[255] = '\0'; - // If Sender: header doesn't exist, then copy the From: header value to it - // Otherwise append the From: address to the recipients list + // TODO: If both Sender: and From: headers exist, and they are different, then + // append the From: address to recipients list to give him access to this email + // as well + - if(state->b_sender[0] == '\0'){ - strcpy(state->b_sender, state->b_from); - strcpy(state->b_sender_domain, state->b_from_domain); - } else { - add_recipient(state->b_from, strlen(state->b_from), sdata, state, data, cfg); - } // Truncate the message_id if it's >255 characters if(strlen(state->message_id) > 255) state->message_id[255] = '\0'; diff --git a/src/parser.h b/src/parser.h index ca81e3bc..c18bede2 100644 --- a/src/parser.h +++ b/src/parser.h @@ -38,5 +38,6 @@ void fix_plus_sign_in_email_address(char *puf, char **at_sign, unsigned int *len void tokenize(char *buf, struct parser_state *state, struct session_data *sdata, struct data *data, struct config *cfg); void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned int abuffersize); void fill_attachment_name_buf(struct parser_state *state, char *buf); +int get_first_email_address_from_string(char *str, char *buf, int buflen); #endif /* _PARSER_H */ diff --git a/src/parser_utils.c b/src/parser_utils.c index 6cd2bbf9..abc63ad2 100644 --- a/src/parser_utils.c +++ b/src/parser_utils.c @@ -624,9 +624,9 @@ void translateLine(unsigned char *p, struct parser_state *state){ for(; *p; p++){ - if( (state->message_state == MSG_RECEIVED || state->message_state == MSG_FROM || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && *p == '@'){ continue; } + if( (state->message_state == MSG_RECEIVED || state->message_state == MSG_FROM || state->message_state == MSG_SENDER || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && *p == '@'){ continue; } - if(state->message_state == MSG_FROM || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT){ + if(state->message_state == MSG_FROM || state->message_state == MSG_SENDER || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT){ /* To fix some unusual addresses, eg. * "'user@domain'" -> user@domain @@ -1077,3 +1077,20 @@ void fill_attachment_name_buf(struct parser_state *state, char *buf){ state->anamepos++; } } + + +int get_first_email_address_from_string(char *str, char *buf, int buflen){ + int result; + + char *p = str; + do { + memset(buf, 0, buflen); + p = split(p, ' ', buf, buflen-1, &result); + + if(*buf == '\0') continue; + + if(does_it_seem_like_an_email_address(buf) == 1){ return 1; } + } while(p); + + return 0; +} diff --git a/src/tokenizer.c b/src/tokenizer.c index ebadbcff..f6b32518 100644 --- a/src/tokenizer.c +++ b/src/tokenizer.c @@ -82,9 +82,16 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata, if(q) fix_plus_sign_in_email_address(puf, &q, &len); memcpy(&(state->b_sender[strlen(state->b_sender)]), puf, len); - if(strlen(state->b_sender) < SMALLBUFSIZE-len-1){ - split_email_address(puf); - memcpy(&(state->b_sender[strlen(state->b_sender)]), puf, len); + + if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(puf) == 1 && state->b_sender_domain[0] == '\0'){ + if(q && strlen(q) > 5){ + memcpy(&(state->b_sender_domain), q+1, strlen(q+1)-1); + } + + if(strlen(state->b_sender) < SMALLBUFSIZE-len-1){ + split_email_address(puf); + memcpy(&(state->b_sender[strlen(state->b_sender)]), puf, len); + } } } else if((state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT || state->message_state == MSG_ENVELOPE_TO) && state->is_1st_header == 1 && state->tolen < MAXBUFSIZE-len-1){ diff --git a/unit_tests/check_parser.c b/unit_tests/check_parser.c index edcb743b..9039eb5c 100644 --- a/unit_tests/check_parser.c +++ b/unit_tests/check_parser.c @@ -10,6 +10,8 @@ struct parser_test { char message_id[SMALLBUFSIZE]; char from[SMALLBUFSIZE]; char from_domain[SMALLBUFSIZE]; + char sender[SMALLBUFSIZE]; + char sender_domain[SMALLBUFSIZE]; char to[SMALLBUFSIZE]; char to_domain[SMALLBUFSIZE]; char reference[SMALLBUFSIZE]; @@ -26,22 +28,22 @@ static void test_parser(struct config *cfg){ struct data data; struct parser_test tests[] = { - {"1.eml", "", "játékok birodalma játékbolt hirlevel@jatekokbirodalma.hu hirlevel jatekokbirodalma hu ", "jatekokbirodalma.hu", "architerv m sj@acts.hu sj acts hu ", "acts.hu ", "", "BLACK FRIDAY - Hihetetlen kedvezmények csak 1 napig november 27-én", 2}, - {"2.eml", "<20151101142653.111156815AF6D@acts.hu>", "jml lighting huixinsoft67@foxmail.com huixinsoft67 foxmail com ", "foxmail.com", "sj@acts.hu sj acts hu ", "acts.hu ", "", "New design ultra slim led panel light", 0}, - {"5-ibm-images.eml", "", "ibm rendezveny rendezveny@hu.ibm.com rendezveny hu ibm com ", "hu.ibm.com", "cim1@aaaa.bbb.fu cim1 aaaa bbb fu ajajaj@piler.aaa.fu ajajaj piler aaa fu ibm rendezveny rendezveny@hu.ibm.com rendezveny hu ibm com ", "aaaa.bbb.fu piler.aaa.fu hu.ibm.com ", "", "***Emlékeztető*** - Egészségipar - eEgészségügy (Út a jövőbe, párbeszéd a gazdaságélénkítésről) 2010. november 4.", 5}, - {"9-attached-text.eml", "", "dr lucky amechi clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com", "usuarios-no-listados ", "", "", "Please read my attached letter", 1}, - {"13-xlsx.eml", "", "aaaaa@aaa.fu aaaaa aaa fu ", "aaa.fu", "sj@acts.hu sj acts hu ", "acts.hu ", "", "ez egy teszt", 1}, - {"15-image-only-spam.eml", "", "kriegel paff sketches@pnmarketing.com sketches pnmarketing com ", "pnmarketing.com", "holmon knobel aaaaa@acts.hu aaaaa acts hu ", "acts.hu ", "", "Lack of concentration, backed up by a vocabulary of tremendous scope, a", 1}, - {"16-rfc822-attachment-1.eml", "", "martonagnes martonagnes@lajt.hu martonagnes lajt hu erős istván eistvan@marosheviz.info ", "lajt.hu", "martonagnes@lajt.hu martonagnes lajt hu ", "lajt.hu ", "", "Féláras akció! 31000Ft/2fő/3nap húsvétkor is a Park Inn****-ben!", 2 }, - {"17-attached-text-bogus-mime.eml", "", "dr lucky amechi clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com", "usuarios-no-listados ", "", "", "Please read my attached letter", 1}, - {"18-spam-html-encoding.eml", "", "a1 hitelcentrum kft Üveges szilvia a1hitelcentrum@t-online.hu a1hitelcentrum t online hu ", "t-online.hu", "postmaster@aaa.fu postmaster aaa fu ", "aaa.fu ", "", "TÁJÉKOZTATÁSVargay Péter", 0}, - {"19-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-186.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1}, - {"20-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-187.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1}, - {"21-register-tricky-urls.eml", "", "the register update-49363-08f0f768@list.theregister.co.uk update 49363 08f0f768 list theregister co uk ", "list.theregister.co.uk", "hello@mail.aaa.fu hello mail aaa fu ", "mail.aaa.fu ", "", "[sp@m] Reg Headlines Friday July 20", 0}, - {"30-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "hello@acts.hu hello acts hu ", "acts.hu ", "", "RE: hxx-ajajajaja.com_ Aaagágyi és kia ttt_webstat hiba", 0}, - {"31-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "hello@acts.hu hello acts hu ", "acts.hu ", "", "Re: stanhu \"domain not found\"-dal eldobja a @fohu-ra küldött leveleket...", 0}, - {"32-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "hello@acts.hu hello acts hu ", "acts.hu ", "", " www.ujsag.hu new virtual host reg. --> Aaaaaaaaa", 0}, - {"33-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "hello@acts.hu hello acts hu ", "acts.hu ", "", "[JIRA] Commented: (AAAA-151) A aaa-nek kerek egy XXX-et, ZH74617282, ACC27363484944", 0}, + {"1.eml", "", "játékok birodalma játékbolt hirlevel@jatekokbirodalma.hu hirlevel jatekokbirodalma hu ", "jatekokbirodalma.hu", "", "", "architerv m sj@acts.hu sj acts hu ", "acts.hu ", "", "BLACK FRIDAY - Hihetetlen kedvezmények csak 1 napig november 27-én", 2}, + {"2.eml", "<20151101142653.111156815AF6D@acts.hu>", "jml lighting huixinsoft67@foxmail.com huixinsoft67 foxmail com ", "foxmail.com", "jml lighting hwtwi@mcqw.com hwtwi mcqw com ", "mcqw.com", "sj@acts.hu sj acts hu ", "acts.hu ", "", "New design ultra slim led panel light", 0}, + {"5-ibm-images.eml", "", "ibm rendezveny rendezveny@hu.ibm.com rendezveny hu ibm com ", "hu.ibm.com", "marta riman rimanmarta@hu.ibm.com rimanmarta hu ibm com ", "hu.ibm.com", "cim1@aaaa.bbb.fu cim1 aaaa bbb fu ajajaj@piler.aaa.fu ajajaj piler aaa fu ibm rendezveny rendezveny@hu.ibm.com rendezveny hu ibm com ", "aaaa.bbb.fu piler.aaa.fu hu.ibm.com ", "", "***Emlékeztető*** - Egészségipar - eEgészségügy (Út a jövőbe, párbeszéd a gazdaságélénkítésről) 2010. november 4.", 5}, + {"9-attached-text.eml", "", "dr lucky amechi clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com", "aaa aaa aaa@aaa.fu aaa aaa fu ", "aaa.fu", "usuarios-no-listados ", "", "", "Please read my attached letter", 1}, + {"13-xlsx.eml", "", "aaaaa@aaa.fu aaaaa aaa fu ", "aaa.fu", "", "", "sj@acts.hu sj acts hu ", "acts.hu ", "", "ez egy teszt", 1}, + {"15-image-only-spam.eml", "", "kriegel paff sketches@pnmarketing.com sketches pnmarketing com ", "pnmarketing.com", "", "", "holmon knobel aaaaa@acts.hu aaaaa acts hu ", "acts.hu ", "", "Lack of concentration, backed up by a vocabulary of tremendous scope, a", 1}, + {"16-rfc822-attachment-1.eml", "", "martonagnes martonagnes@lajt.hu martonagnes lajt hu erős istván eistvan@marosheviz.info ", "lajt.hu", "postmaster postmaster@aaa.fu postmaster aaa fu ", "aaa.fu", "martonagnes@lajt.hu martonagnes lajt hu ", "lajt.hu ", "", "Féláras akció! 31000Ft/2fő/3nap húsvétkor is a Park Inn****-ben!", 2 }, + {"17-attached-text-bogus-mime.eml", "", "dr lucky amechi clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com", "postmaster postmaster@aaa.fu postmaster aaa fu ", "aaa.fu", "usuarios-no-listados ", "", "", "Please read my attached letter", 1}, + {"18-spam-html-encoding.eml", "", "a1 hitelcentrum kft Üveges szilvia a1hitelcentrum@t-online.hu a1hitelcentrum t online hu ", "t-online.hu", "postmaster postmaster@aaa.fu postmaster aaa fu ", "aaa.fu", "postmaster@aaa.fu postmaster aaa fu ", "aaa.fu ", "", "TÁJÉKOZTATÁSVargay Péter", 0}, + {"19-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-186.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "", "", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1}, + {"20-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-187.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "", "", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1}, + {"21-register-tricky-urls.eml", "", "the register update-49363-08f0f768@list.theregister.co.uk update 49363 08f0f768 list theregister co uk ", "list.theregister.co.uk", "", "", "hello@mail.aaa.fu hello mail aaa fu ", "mail.aaa.fu ", "", "[sp@m] Reg Headlines Friday July 20", 0}, + {"30-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", "RE: hxx-ajajajaja.com_ Aaagágyi és kia ttt_webstat hiba", 0}, + {"31-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", "Re: stanhu \"domain not found\"-dal eldobja a @fohu-ra küldött leveleket...", 0}, + {"32-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", " www.ujsag.hu new virtual host reg. --> Aaaaaaaaa", 0}, + {"33-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", "[JIRA] Commented: (AAAA-151) A aaa-nek kerek egy XXX-et, ZH74617282, ACC27363484944", 0}, }; @@ -66,7 +68,9 @@ static void test_parser(struct config *cfg){ assert(strcmp(state.message_id, tests[i].message_id) == 0 && "test_parser()1"); assert(strcmp(state.b_from, tests[i].from) == 0 && "test_parser()2a"); - assert(strcmp(state.b_from_domain, tests[i].from_domain) == 0 && "test_parser()2b"); + assert(strcmp(state.b_sender, tests[i].sender) == 0 && "test_parser()2b"); + assert(strcmp(state.b_from_domain, tests[i].from_domain) == 0 && "test_parser()2c"); + assert(strcmp(state.b_sender_domain, tests[i].sender_domain) == 0 && "test_parser()2d"); assert(strcmp(state.b_to, tests[i].to) == 0 && "test_parser()3a"); assert(strcmp(state.b_to_domain, tests[i].to_domain) == 0 && "test_parser()3b"); assert(strcmp(state.b_subject, tests[i].subject) == 0 && "test_parser()4"); From 58a164467e9869baabd413a7c893cd8221b7d571 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 13 Dec 2020 08:36:12 +0100 Subject: [PATCH 015/106] Fixed unit tests for sender Signed-off-by: Janos SUTO --- src/import.c | 2 +- src/parser.c | 18 +++++++++++------- src/parser.h | 2 +- src/piler.c | 2 +- src/reindex.c | 2 +- src/test.c | 2 +- unit_tests/check_attachments.c | 2 +- unit_tests/check_parser.c | 8 ++++---- unit_tests/check_rules.c | 2 +- unit_tests/common.c | 2 +- 10 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/import.c b/src/import.c index c3decf9f..601c9b84 100644 --- a/src/import.c +++ b/src/import.c @@ -71,7 +71,7 @@ int import_message(struct session_data *sdata, struct data *data, struct config sdata->import = 1; state = parse_message(sdata, 1, data, cfg); - post_parse(sdata, data, &state, cfg); + post_parse(sdata, &state, cfg); rule = check_against_ruleset(data->archiving_rules, &state, sdata->tot_len, sdata->spam_message); if(rule){ diff --git a/src/parser.c b/src/parser.c index cd9224b4..dbaa6277 100644 --- a/src/parser.c +++ b/src/parser.c @@ -68,11 +68,21 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec add_recipient(data->import->extra_recipient, strlen(data->import->extra_recipient), sdata, &state, data, cfg); } + // If both Sender: and From: headers exist, and they are different, then append + // the From: address to recipients list to give him access to this email as well + + if(state.b_sender_domain[0] && strcmp(state.b_from, state.b_sender)){ + char tmpbuf[SMALLBUFSIZE]; + get_first_email_address_from_string(state.b_from, tmpbuf, sizeof(tmpbuf)); + tmpbuf[strlen(tmpbuf)] = ' '; + add_recipient(tmpbuf, strlen(tmpbuf), sdata, &state, data, cfg); + } + return state; } -void post_parse(struct session_data *sdata, struct data *data, struct parser_state *state, struct config *cfg){ +void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg){ int i; clearhash(state->boundaries); @@ -87,12 +97,6 @@ void post_parse(struct session_data *sdata, struct data *data, struct parser_sta if(strlen(state->b_sender) > 255) state->b_sender[255] = '\0'; if(strlen(state->b_sender_domain) > 255) state->b_sender_domain[255] = '\0'; - // TODO: If both Sender: and From: headers exist, and they are different, then - // append the From: address to recipients list to give him access to this email - // as well - - - // Truncate the message_id if it's >255 characters if(strlen(state->message_id) > 255) state->message_id[255] = '\0'; diff --git a/src/parser.h b/src/parser.h index c18bede2..5f536fd7 100644 --- a/src/parser.h +++ b/src/parser.h @@ -10,7 +10,7 @@ #include "defs.h" struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct data *data, struct config *cfg); -void post_parse(struct session_data *sdata, struct data *data, struct parser_state *state, struct config *cfg); +void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg); int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg); void init_state(struct parser_state *state); diff --git a/src/piler.c b/src/piler.c index fa49a074..684d90de 100644 --- a/src/piler.c +++ b/src/piler.c @@ -163,7 +163,7 @@ int process_email(char *filename, struct session_data *sdata, struct data *data, parser_state = parse_message(sdata, 1, data, cfg); - post_parse(sdata, data, &parser_state, cfg); + post_parse(sdata, &parser_state, cfg); if(cfg->syslog_recipients == 1){ char *rcpt = parser_state.b_to; diff --git a/src/reindex.c b/src/reindex.c index 812fbe4b..b2b137b4 100644 --- a/src/reindex.c +++ b/src/reindex.c @@ -124,7 +124,7 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename); state = parse_message(sdata, 1, data, cfg); - post_parse(sdata, data, &state, cfg); + post_parse(sdata, &state, cfg); rc = store_index_data(sdata, &state, data, stored_id, cfg); diff --git a/src/test.c b/src/test.c index 4d7a8a3f..bbac1891 100644 --- a/src/test.c +++ b/src/test.c @@ -147,7 +147,7 @@ int main(int argc, char **argv){ state = parse_message(&sdata, 1, &data, &cfg); printf("post parsing...\n"); - post_parse(&sdata, &data, &state, &cfg); + post_parse(&sdata, &state, &cfg); printf("message-id: %s / %s\n", state.message_id, state.message_id_hash); printf("from: *%s (%s)*\n", state.b_from, state.b_from_domain); diff --git a/unit_tests/check_attachments.c b/unit_tests/check_attachments.c index f6ee2b1c..e051ea40 100644 --- a/unit_tests/check_attachments.c +++ b/unit_tests/check_attachments.c @@ -43,7 +43,7 @@ static void test_attachments(struct config *cfg){ snprintf(sdata.tmpframe, SMALLBUFSIZE-1, "%s.m", tests[i].s); state = parse_message(&sdata, 1, &data, cfg); - post_parse(&sdata, &data, &state, cfg); + post_parse(&sdata, &state, cfg); for(j=1; j<=state.n_attachments; j++){ unlink(state.attachments[j].internalname); diff --git a/unit_tests/check_parser.c b/unit_tests/check_parser.c index 9039eb5c..70d53760 100644 --- a/unit_tests/check_parser.c +++ b/unit_tests/check_parser.c @@ -29,14 +29,14 @@ static void test_parser(struct config *cfg){ struct parser_test tests[] = { {"1.eml", "", "játékok birodalma játékbolt hirlevel@jatekokbirodalma.hu hirlevel jatekokbirodalma hu ", "jatekokbirodalma.hu", "", "", "architerv m sj@acts.hu sj acts hu ", "acts.hu ", "", "BLACK FRIDAY - Hihetetlen kedvezmények csak 1 napig november 27-én", 2}, - {"2.eml", "<20151101142653.111156815AF6D@acts.hu>", "jml lighting huixinsoft67@foxmail.com huixinsoft67 foxmail com ", "foxmail.com", "jml lighting hwtwi@mcqw.com hwtwi mcqw com ", "mcqw.com", "sj@acts.hu sj acts hu ", "acts.hu ", "", "New design ultra slim led panel light", 0}, + {"2.eml", "<20151101142653.111156815AF6D@acts.hu>", "jml lighting huixinsoft67@foxmail.com huixinsoft67 foxmail com ", "foxmail.com", "jml lighting hwtwi@mcqw.com hwtwi mcqw com ", "mcqw.com", "sj@acts.hu sj acts hu huixinsoft67@foxmail.com huixinsoft67 foxmail com ", "acts.hu foxmail.com ", "", "New design ultra slim led panel light", 0}, {"5-ibm-images.eml", "", "ibm rendezveny rendezveny@hu.ibm.com rendezveny hu ibm com ", "hu.ibm.com", "marta riman rimanmarta@hu.ibm.com rimanmarta hu ibm com ", "hu.ibm.com", "cim1@aaaa.bbb.fu cim1 aaaa bbb fu ajajaj@piler.aaa.fu ajajaj piler aaa fu ibm rendezveny rendezveny@hu.ibm.com rendezveny hu ibm com ", "aaaa.bbb.fu piler.aaa.fu hu.ibm.com ", "", "***Emlékeztető*** - Egészségipar - eEgészségügy (Út a jövőbe, párbeszéd a gazdaságélénkítésről) 2010. november 4.", 5}, - {"9-attached-text.eml", "", "dr lucky amechi clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com", "aaa aaa aaa@aaa.fu aaa aaa fu ", "aaa.fu", "usuarios-no-listados ", "", "", "Please read my attached letter", 1}, + {"9-attached-text.eml", "", "dr lucky amechi clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com", "aaa aaa aaa@aaa.fu aaa aaa fu ", "aaa.fu", "usuarios-no-listados clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com ", "", "Please read my attached letter", 1}, {"13-xlsx.eml", "", "aaaaa@aaa.fu aaaaa aaa fu ", "aaa.fu", "", "", "sj@acts.hu sj acts hu ", "acts.hu ", "", "ez egy teszt", 1}, {"15-image-only-spam.eml", "", "kriegel paff sketches@pnmarketing.com sketches pnmarketing com ", "pnmarketing.com", "", "", "holmon knobel aaaaa@acts.hu aaaaa acts hu ", "acts.hu ", "", "Lack of concentration, backed up by a vocabulary of tremendous scope, a", 1}, {"16-rfc822-attachment-1.eml", "", "martonagnes martonagnes@lajt.hu martonagnes lajt hu erős istván eistvan@marosheviz.info ", "lajt.hu", "postmaster postmaster@aaa.fu postmaster aaa fu ", "aaa.fu", "martonagnes@lajt.hu martonagnes lajt hu ", "lajt.hu ", "", "Féláras akció! 31000Ft/2fő/3nap húsvétkor is a Park Inn****-ben!", 2 }, - {"17-attached-text-bogus-mime.eml", "", "dr lucky amechi clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com", "postmaster postmaster@aaa.fu postmaster aaa fu ", "aaa.fu", "usuarios-no-listados ", "", "", "Please read my attached letter", 1}, - {"18-spam-html-encoding.eml", "", "a1 hitelcentrum kft Üveges szilvia a1hitelcentrum@t-online.hu a1hitelcentrum t online hu ", "t-online.hu", "postmaster postmaster@aaa.fu postmaster aaa fu ", "aaa.fu", "postmaster@aaa.fu postmaster aaa fu ", "aaa.fu ", "", "TÁJÉKOZTATÁSVargay Péter", 0}, + {"17-attached-text-bogus-mime.eml", "", "dr lucky amechi clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com", "postmaster postmaster@aaa.fu postmaster aaa fu ", "aaa.fu", "usuarios-no-listados clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com ", "", "Please read my attached letter", 1}, + {"18-spam-html-encoding.eml", "", "a1 hitelcentrum kft Üveges szilvia a1hitelcentrum@t-online.hu a1hitelcentrum t online hu ", "t-online.hu", "postmaster postmaster@aaa.fu postmaster aaa fu ", "aaa.fu", "postmaster@aaa.fu postmaster aaa fu a1hitelcentrum@t-online.hu a1hitelcentrum t online hu ", "aaa.fu t-online.hu ", "", "TÁJÉKOZTATÁSVargay Péter", 0}, {"19-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-186.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "", "", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1}, {"20-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-187.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "", "", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1}, {"21-register-tricky-urls.eml", "", "the register update-49363-08f0f768@list.theregister.co.uk update 49363 08f0f768 list theregister co uk ", "list.theregister.co.uk", "", "", "hello@mail.aaa.fu hello mail aaa fu ", "mail.aaa.fu ", "", "[sp@m] Reg Headlines Friday July 20", 0}, diff --git a/unit_tests/check_rules.c b/unit_tests/check_rules.c index 47d18ca8..4dad5631 100644 --- a/unit_tests/check_rules.c +++ b/unit_tests/check_rules.c @@ -116,7 +116,7 @@ static void test_archiving_rule(struct config *cfg){ snprintf(sdata.tmpframe, SMALLBUFSIZE-1, "%s.m", rule_test[i].filename); state = parse_message(&sdata, 1, &data, cfg); - post_parse(&sdata, &data, &state, cfg); + post_parse(&sdata, &state, cfg); rule = check_against_ruleset(data.archiving_rules, &state, st.st_size, sdata.spam_message); diff --git a/unit_tests/common.c b/unit_tests/common.c index 3699db36..c75aecdd 100644 --- a/unit_tests/common.c +++ b/unit_tests/common.c @@ -30,7 +30,7 @@ int setup_and_parse_message(struct session_data *sdata, struct parser_state *sta snprintf(sdata->tmpframe, SMALLBUFSIZE-1, "%s.m", filename); *state = parse_message(sdata, 1, data, cfg); - post_parse(sdata, data, state, cfg); + post_parse(sdata, state, cfg); return 0; } From 61834843f119c426c702ec3ef1fe808db1bd43ed Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 13 Dec 2020 08:52:52 +0100 Subject: [PATCH 016/106] Fixed sender domain check Signed-off-by: Janos SUTO --- src/message.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/message.c b/src/message.c index 1b281fab..9319e4ac 100644 --- a/src/message.c +++ b/src/message.c @@ -41,7 +41,7 @@ int store_index_data(struct session_data *sdata, struct parser_state *state, str fix_email_address_for_sphinx(state->b_sender_domain); fix_email_address_for_sphinx(state->b_to_domain); - if(state->b_sender_domain){ + if(state->b_sender_domain[0]){ sender = state->b_sender; sender_domain = state->b_sender_domain; } @@ -194,7 +194,7 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru subj = state->b_subject; if(*subj == ' ') subj++; - if(state->b_sender_domain){ + if(state->b_sender_domain[0]){ sender = state->b_sender; sender_domain = state->b_sender_domain; get_first_email_address_from_string(state->b_sender, s2, sizeof(s2)); From 9dea29d7836cdf212b35f302e6acb960f5be4719 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 13 Dec 2020 19:49:46 +0100 Subject: [PATCH 017/106] pilertest should calculate retention timestamp starting from the sent timestamp Signed-off-by: Janos SUTO --- src/test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test.c b/src/test.c index bbac1891..956910d0 100644 --- a/src/test.c +++ b/src/test.c @@ -170,7 +170,7 @@ int main(int argc, char **argv){ printf("rules check: %s\n", rule); retention_seconds = query_retain_period(&data, &state, st.st_size, sdata.spam_message, &cfg); - sdata.retained = sdata.now + retention_seconds; + sdata.retained = sdata.sent + retention_seconds; printf("folder: %d\n", get_folder_id_by_rule(&data, &state, st.st_size, sdata.spam_message, &cfg)); From 2b4f27372d550a3ebeafe691f55bab3417fc8556 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 18 Dec 2020 22:42:16 +0100 Subject: [PATCH 018/106] Added -o option to pilerexport to print everything to stdout Signed-off-by: Janos SUTO --- src/pilerexport.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/pilerexport.c b/src/pilerexport.c index 1aee477f..3c65490d 100644 --- a/src/pilerexport.c +++ b/src/pilerexport.c @@ -26,6 +26,7 @@ int dryrun = 0; int exportall = 0; int verification_status = 0; int rc = 0; +int export_to_stdout = 0; char *query=NULL; int verbosity = 0; int max_matches = 1000; @@ -53,6 +54,7 @@ void usage(){ printf(" -i Sphinx indices to use (default: %s)\n", index_list); printf(" -z Write exported EML files to a zip file\n"); printf(" -A Export all emails from archive\n"); + printf(" -o Export emails to stdout\n"); printf(" -d Dry run\n"); regfree(®exp); @@ -369,6 +371,11 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct if(dryrun == 0){ + if(export_to_stdout){ + rc = retrieve_email_from_archive(sdata, stdout, cfg); + continue; + } + snprintf(filename, sizeof(filename)-1, "%llu.eml", id); f = fopen(filename, "w"); @@ -442,6 +449,7 @@ int main(int argc, char **argv){ {"all", no_argument, 0, 'A' }, {"dry-run", no_argument, 0, 'd' }, {"dryrun", no_argument, 0, 'd' }, + {"stdout", no_argument, 0, 'o' }, {"help", no_argument, 0, 'h' }, {"version", no_argument, 0, 'v' }, {"from", required_argument, 0, 'f' }, @@ -459,9 +467,9 @@ int main(int argc, char **argv){ int option_index = 0; - int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Adhv?", long_options, &option_index); + int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:oAdhv?", long_options, &option_index); #else - int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Adhv?"); + int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:oAdhv?"); #endif if(c == -1) break; @@ -554,6 +562,10 @@ int main(int argc, char **argv){ break; + case 'o': + export_to_stdout = 1; + break; + case 'd' : dryrun = 1; break; From 3729ed1ecfe6af15de04a03c58b48990db98e029 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 20 Dec 2020 22:01:34 +0100 Subject: [PATCH 019/106] pilerimport can read data exported by pilerexport from stdin Signed-off-by: Janos SUTO --- configure | 2 +- configure.in | 2 +- src/config.h | 2 + src/defs.h | 1 + src/import.h | 2 + src/import_maildir.c | 2 +- src/import_pilerexport.c | 109 +++++++++++++++++++++++++++++++++++++++ src/pilerexport.c | 1 + src/pilerimport.c | 13 +++-- 9 files changed, 128 insertions(+), 6 deletions(-) create mode 100644 src/import_pilerexport.c diff --git a/configure b/configure index 016ca71b..6423657e 100755 --- a/configure +++ b/configure @@ -4854,7 +4854,7 @@ echo; echo CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wimplicit-fallthrough=2 -Wuninitialized -Wno-format-truncation -g" LIBS="$antispam_libs $sunos_libs " -OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" +OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" ac_config_files="$ac_config_files Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile" diff --git a/configure.in b/configure.in index 554eaee2..0f2a83d5 100644 --- a/configure.in +++ b/configure.in @@ -537,7 +537,7 @@ echo; echo CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wimplicit-fallthrough=2 -Wuninitialized -Wno-format-truncation -g" LIBS="$antispam_libs $sunos_libs " -OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" +OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile]) AC_OUTPUT diff --git a/src/config.h b/src/config.h index e1542ad5..7d110348 100644 --- a/src/config.h +++ b/src/config.h @@ -57,6 +57,8 @@ #define MEMCACHED_MSGS_STORED_SIZE MEMCACHED_CLAPF_PREFIX "stored_size" +#define PILEREXPORT_BEGIN_MARK "x-exported-by-pilerexport: start\n" + #define LOG_PRIORITY LOG_INFO #define _LOG_INFO 3 diff --git a/src/defs.h b/src/defs.h index d6c653b4..6bfece2e 100644 --- a/src/defs.h +++ b/src/defs.h @@ -302,6 +302,7 @@ struct import { int keep_eml; int timeout; int cap_uidplus; + int fd; long total_size; int dryrun; int tot_msgs; diff --git a/src/import.h b/src/import.h index 0979acab..6f0f112d 100644 --- a/src/import.h +++ b/src/import.h @@ -23,4 +23,6 @@ int list_folders(struct data *data); int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct config *cfg); void send_imap_close(struct data *data); +void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg); + #endif /* _IMPORT_H */ diff --git a/src/import_maildir.c b/src/import_maildir.c index aac3c6a3..ced1cafa 100644 --- a/src/import_maildir.c +++ b/src/import_maildir.c @@ -83,7 +83,7 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir printf("ERROR: error importing: '%s'\n", data->import->filename); ret = ERR; } - + if(data->import->remove_after_import == 1 && rc != ERR) unlink(data->import->filename); i++; diff --git a/src/import_pilerexport.c b/src/import_pilerexport.c new file mode 100644 index 00000000..8514a593 --- /dev/null +++ b/src/import_pilerexport.c @@ -0,0 +1,109 @@ +/* + * import_pop3.c + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +void import_the_file(struct session_data *sdata, struct data *data, struct config *cfg){ + close(data->import->fd); + data->import->fd = -1; + + if(import_message(sdata, data, cfg) != ERR){ + unlink(data->import->filename); + } +} + + +void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct config *cfg){ + + if(!strcmp(buf, PILEREXPORT_BEGIN_MARK)){ + if((*count) > 0){ + import_the_file(sdata, data, cfg); + } + + (*count)++; + + snprintf(data->import->filename, SMALLBUFSIZE-1, "import-%llu", *count); + + data->import->fd = open(data->import->filename, O_CREAT|O_WRONLY|O_TRUNC, S_IRUSR|S_IWUSR); + if(data->import->fd == -1){ + // Do some error handling + printf("error: cannot open %s\n", data->import->filename); + } + + } else { + if(write(data->import->fd, buf, buflen) != buflen){ + printf("error: didnt write %d bytes\n", buflen); + } + } +} + + +void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg){ + int n, rc, savedlen=0, puflen; + uint64 count=0; + char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE]; + + memset(savedbuf, 0, sizeof(savedbuf)); + + data->import->fd = -1; + + do { + memset(buf, 0, sizeof(buf)); + n = fread(buf, 1, sizeof(buf)-1, stdin); + + if(savedlen > 0){ + memset(copybuf, 0, sizeof(copybuf)); + + memcpy(copybuf, savedbuf, savedlen); + memcpy(©buf[savedlen], buf, n); + + savedlen = 0; + memset(savedbuf, 0, sizeof(savedbuf)); + + p = ©buf[0]; + } + else { + p = &buf[0]; + } + + do { + puflen = read_one_line(p, '\n', puf, sizeof(puf), &rc); + p += puflen; + + if(puflen > 0){ + if(rc == OK){ + process_buffer(puf, puflen, &count, sdata, data, cfg); + } + else { + snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf); + savedlen = puflen; + } + } + + } while(puflen > 0); + + } while(n > 0); + + if(data->import->fd != -1){ + import_the_file(sdata, data, cfg); + } +} diff --git a/src/pilerexport.c b/src/pilerexport.c index 3c65490d..f15b942b 100644 --- a/src/pilerexport.c +++ b/src/pilerexport.c @@ -372,6 +372,7 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct if(dryrun == 0){ if(export_to_stdout){ + printf("%s", PILEREXPORT_BEGIN_MARK); rc = retrieve_email_from_archive(sdata, stdout, cfg); continue; } diff --git a/src/pilerimport.c b/src/pilerimport.c index dcf65338..51ce727c 100644 --- a/src/pilerimport.c +++ b/src/pilerimport.c @@ -54,6 +54,7 @@ void usage(){ printf(" -a Add recipient to the To:/Cc: list\n"); printf(" -T Update import table at id=\n"); printf(" -D Dry-run, do not import anything\n"); + printf(" -y Read pilerexport data from stdin\n"); printf(" -o Only download emails for POP3/IMAP import\n"); printf(" -r Remove imported emails\n"); printf(" -q Quiet mode\n"); @@ -63,7 +64,7 @@ void usage(){ int main(int argc, char **argv){ - int i, n_mbox=0; + int i, n_mbox=0, read_from_pilerexport=0; char *configfile=CONFIG_FILE, *mbox[MBOX_ARGS], *directory=NULL; char puf[SMALLBUFSIZE], *imapserver=NULL, *pop3server=NULL; struct session_data sdata; @@ -141,6 +142,7 @@ int main(int argc, char **argv){ {"failed-folder", required_argument, 0, 'j' }, {"move-folder", required_argument, 0, 'g' }, {"only-download",no_argument, 0, 'o' }, + {"read-from-export",no_argument, 0, 'y' }, {"dry-run", no_argument, 0, 'D' }, {"help", no_argument, 0, 'h' }, {0,0,0,0} @@ -148,9 +150,9 @@ int main(int argc, char **argv){ int option_index = 0; - int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:DRroqh?", long_options, &option_index); + int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:yDRroqh?", long_options, &option_index); #else - int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:DRroqh?"); + int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:yDRroqh?"); #endif if(c == -1) break; @@ -269,6 +271,10 @@ int main(int argc, char **argv){ data.import->table_id = atoi(optarg); break; + case 'y' : + read_from_pilerexport = 1; + break; + case 'D' : data.import->dryrun = 1; break; @@ -361,6 +367,7 @@ int main(int argc, char **argv){ if(directory) import_from_maildir(&sdata, &data, directory, &cfg); if(imapserver) import_from_imap_server(&sdata, &data, &cfg); if(pop3server) import_from_pop3_server(&sdata, &data, &cfg); + if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &cfg); clearrules(data.archiving_rules); clearrules(data.retention_rules); From fbb5aa84794537debb9ac6c943d2d7d9deadc8df Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 24 Dec 2020 15:57:29 +0100 Subject: [PATCH 020/106] write to zip file requires libzip 1.x Signed-off-by: Janos SUTO --- src/pilerexport.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/pilerexport.c b/src/pilerexport.c index f15b942b..eb9d3a09 100644 --- a/src/pilerexport.c +++ b/src/pilerexport.c @@ -52,7 +52,9 @@ void usage(){ printf(" -w Where condition to pass to sphinx, eg. \"match('@subject: piler')\"\n"); printf(" -m Max. matches to apply to sphinx query (default: %d)\n", max_matches); printf(" -i Sphinx indices to use (default: %s)\n", index_list); +#if LIBZIP_VERSION_MAJOR >= 1 printf(" -z Write exported EML files to a zip file\n"); +#endif printf(" -A Export all emails from archive\n"); printf(" -o Export emails to stdout\n"); printf(" -d Dry run\n"); @@ -317,7 +319,7 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain return rc; } - +#if LIBZIP_VERSION_MAJOR >= 1 int write_to_zip_file(char *filename){ struct zip *z=NULL; int errorp, ret=ERR; @@ -339,6 +341,7 @@ int write_to_zip_file(char *filename){ return ret; } +#endif int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg){ FILE *f; @@ -398,9 +401,11 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct verification_status = 1; } + #if LIBZIP_VERSION_MAJOR >= 1 if(zipfile && write_to_zip_file(filename) == OK){ unlink(filename); } + #endif } else printf("cannot open: %s\n", filename); From 48ec9b5c4a889634932a22a2fde25d3a2c66a263 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 24 Dec 2020 16:23:14 +0100 Subject: [PATCH 021/106] Add -Wimplicit-fallthrough=2 cflag only to gcc version 7+ Signed-off-by: Janos SUTO --- configure | 8 +++++++- configure.in | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/configure b/configure index 6423657e..7eaab2d9 100755 --- a/configure +++ b/configure @@ -4852,7 +4852,13 @@ if test $? -eq 1; then echo "the user \"$RUNNING_USER\" does not exists, please echo; echo -CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wimplicit-fallthrough=2 -Wuninitialized -Wno-format-truncation -g" +gcc_version="$(gcc -dumpversion)" +extra_cflags="" +if [ "${gcc_version:0:1}" -gt 5 ]; then + extra_cflags="-Wimplicit-fallthrough=2" +fi + +CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra $extra_cflags -Wuninitialized -Wno-format-truncation -g" LIBS="$antispam_libs $sunos_libs " OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" diff --git a/configure.in b/configure.in index 0f2a83d5..8c50891d 100644 --- a/configure.in +++ b/configure.in @@ -535,7 +535,13 @@ if test $? -eq 1; then echo "the user \"$RUNNING_USER\" does not exists, please echo; echo -CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wimplicit-fallthrough=2 -Wuninitialized -Wno-format-truncation -g" +gcc_version="$(gcc -dumpversion)" +extra_cflags="" +if [[ "${gcc_version:0:1}" -gt 5 ]]; then + extra_cflags="-Wimplicit-fallthrough=2" +fi + +CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra $extra_cflags -Wuninitialized -Wno-format-truncation -g" LIBS="$antispam_libs $sunos_libs " OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" From f8a32dd025c36cd3bd0c0950a6921e635279df3b Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 27 Dec 2020 23:40:39 +0100 Subject: [PATCH 022/106] Introduced smtp acl Signed-off-by: Janos SUTO --- RELEASE_NOTES | 1 + configure | 19 +--- configure.in | 15 +-- etc/example.conf | 10 ++ piler-config.h.in | 2 - src/Makefile.in | 2 +- src/cfg.c | 1 + src/cfg.h | 2 + src/config.h | 1 + src/defs.h | 12 ++- src/piler-smtp.c | 7 +- src/piler.h | 3 +- src/screen.c | 237 ++++++++++++++++++++++++++++++++++++++++++++++ src/screen.h | 16 ++++ src/session.c | 28 +----- src/smtpcodes.h | 1 + 16 files changed, 294 insertions(+), 63 deletions(-) create mode 100644 src/screen.c create mode 100644 src/screen.h diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 60c346e7..dfc9566a 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -2,6 +2,7 @@ ------- - Added security header feature +- Introduced the smtp acl list, and obsoleted the tcp_wrappers check 1.3.9: diff --git a/configure b/configure index 7eaab2d9..c195b24c 100755 --- a/configure +++ b/configure @@ -4714,23 +4714,6 @@ else echo "zip library: no" fi -if test "$have_tcpwrappers" = "yes"; then - echo "tcpwrappers support: yes" - -cat >>confdefs.h <<_ACEOF -#define HAVE_LIBWRAP 1 -_ACEOF - - - if test "$os" = "FreeBSD"; then - antispam_libs="$antispam_libs -lwrap" - else - antispam_libs="$antispam_libs -lwrap -lnsl" - fi -else - echo "tcpwrappers support: no" -fi - echo @@ -4860,7 +4843,7 @@ fi CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra $extra_cflags -Wuninitialized -Wno-format-truncation -g" LIBS="$antispam_libs $sunos_libs " -OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" +OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o screen.o $objs" ac_config_files="$ac_config_files Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile" diff --git a/configure.in b/configure.in index 8c50891d..2e522ba1 100644 --- a/configure.in +++ b/configure.in @@ -433,19 +433,6 @@ else echo "zip library: no" fi -if test "$have_tcpwrappers" = "yes"; then - echo "tcpwrappers support: yes" - AC_DEFINE_UNQUOTED(HAVE_LIBWRAP, 1, [tcpwrappers support]) - - if test "$os" = "FreeBSD"; then - antispam_libs="$antispam_libs -lwrap" - else - antispam_libs="$antispam_libs -lwrap -lnsl" - fi -else - echo "tcpwrappers support: no" -fi - echo @@ -543,7 +530,7 @@ fi CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra $extra_cflags -Wuninitialized -Wno-format-truncation -g" LIBS="$antispam_libs $sunos_libs " -OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" +OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o screen.o $objs" AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile]) AC_OUTPUT diff --git a/etc/example.conf b/etc/example.conf index 4816aa9e..44e8ec57 100644 --- a/etc/example.conf +++ b/etc/example.conf @@ -230,3 +230,13 @@ mmap_dedup_test=0 ; mechanism against unwanted email in the archive if limiting smtp ; clients via an IP-address list is not feasible. security_header= + +; whether to enable (1) or not (0) an smtp access list similar to +; postfix's postscreen. If so, then create a text file %sysconfdir%/piler/smtp.acl +; An example for /usr/local/etc/piler/smtp.acl: +; +; 1.2.3.4/32 permit +; 10.0.1.0/24 reject +; 172.16.0.0/16 permit +; +smtp_access_list=0 diff --git a/piler-config.h.in b/piler-config.h.in index 416831b5..2352532a 100644 --- a/piler-config.h.in +++ b/piler-config.h.in @@ -26,8 +26,6 @@ #undef HAVE_TNEF #undef HAVE_ZIP -#undef HAVE_LIBWRAP - #undef HAVE_TWEAK_SENT_TIME #undef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT diff --git a/src/Makefile.in b/src/Makefile.in index abc99656..6eed82b8 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -48,7 +48,7 @@ libpiler.a: $(OBJS) $(SQL_OBJS) ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so.$(PILER_VERSION) piler-smtp: piler-smtp.c libpiler.a - $(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< cfg.o misc.o tai.o smtp.o session.o dirs.o sig.o bdat.o $(LIBS) $(LIBDIR) + $(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< cfg.o misc.o tai.o smtp.o session.o dirs.o sig.o bdat.o screen.o $(LIBS) $(LIBDIR) pilerget: pilerget.c libpiler.a $(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR) diff --git a/src/cfg.c b/src/cfg.c index 023d04e6..b348a81a 100644 --- a/src/cfg.c +++ b/src/cfg.c @@ -86,6 +86,7 @@ struct _parse_rule config_parse_rules[] = { "queuedir", "string", (void*) string_parser, offsetof(struct config, queuedir), QUEUE_DIR, MAXVAL-1}, { "security_header", "string", (void*) string_parser, offsetof(struct config, security_header), "", MAXVAL-1}, { "server_id", "integer", (void*) int_parser, offsetof(struct config, server_id), "0", sizeof(int)}, + { "smtp_access_list", "integer", (void*) int_parser, offsetof(struct config, smtp_access_list), "0", sizeof(int)}, { "smtp_timeout", "integer", (void*) int_parser, offsetof(struct config, smtp_timeout), "60", sizeof(int)}, { "spam_header_line", "string", (void*) string_parser, offsetof(struct config, spam_header_line), "", MAXVAL-1}, { "syslog_recipients", "integer", (void*) int_parser, offsetof(struct config, syslog_recipients), "0", sizeof(int)}, diff --git a/src/cfg.h b/src/cfg.h index 6932a6c4..77373869 100644 --- a/src/cfg.h +++ b/src/cfg.h @@ -99,6 +99,8 @@ struct config { int enable_folders; int debug; + + int smtp_access_list; }; diff --git a/src/config.h b/src/config.h index 7d110348..5810c5b0 100644 --- a/src/config.h +++ b/src/config.h @@ -14,6 +14,7 @@ #define HOSTID "mailarchiver" #define CONFIG_FILE CONFDIR "/piler/piler.conf" +#define SMTP_ACL_FILE CONFDIR "/piler/smtp.acl" #define WORK_DIR DATADIR "/piler/tmp" #define QUEUE_DIR DATADIR "/piler/store" #define ERROR_DIR DATADIR "/piler/error" diff --git a/src/defs.h b/src/defs.h index 6bfece2e..ff8be3a4 100644 --- a/src/defs.h +++ b/src/defs.h @@ -13,9 +13,6 @@ #include #include #endif -#ifdef HAVE_LIBWRAP - #include -#endif #include #include @@ -98,6 +95,15 @@ struct node { }; +struct smtp_acl { + char network_str[BUFLEN]; + in_addr_t low, high; + int prefix; + int rejected; + struct smtp_acl *r; +}; + + struct net { int socket; int use_ssl; diff --git a/src/piler-smtp.c b/src/piler-smtp.c index 41cbca46..b5fd2623 100644 --- a/src/piler-smtp.c +++ b/src/piler-smtp.c @@ -38,6 +38,7 @@ char *configfile = CONFIG_FILE; struct config cfg; struct passwd *pwd; struct smtp_session *session, **sessions=NULL; +struct smtp_acl *smtp_acl[MAXHASH]; void usage(){ @@ -66,6 +67,8 @@ void p_clean_exit(int sig){ if(events) free(events); + clear_smtp_acl(smtp_acl); + syslog(LOG_PRIORITY, "%s has been terminated", PROGNAME); ERR_free_strings(); @@ -120,6 +123,8 @@ void initialise_configuration(){ setlocale(LC_MESSAGES, cfg.locale); setlocale(LC_CTYPE, cfg.locale); + load_smtp_acl(smtp_acl); + syslog(LOG_PRIORITY, "reloaded config: %s", configfile); } @@ -270,7 +275,7 @@ int main(int argc, char **argv){ break; } - start_new_session(sessions, client_sockfd, &num_connections, &cfg); + start_new_session(sessions, client_sockfd, &num_connections, smtp_acl, hbuf, &cfg); } continue; diff --git a/src/piler.h b/src/piler.h index e41e2334..a38be51e 100644 --- a/src/piler.h +++ b/src/piler.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -68,7 +69,7 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de void load_mydomains(struct session_data *sdata, struct data *data, struct config *cfg); int is_email_address_on_my_domains(char *email, struct data *data); -int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct config *cfg); +int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg); void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections); struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int max_connections, int socket); void write_envelope_addresses(struct smtp_session *session, struct config *cfg); diff --git a/src/screen.c b/src/screen.c new file mode 100644 index 00000000..0f3fda45 --- /dev/null +++ b/src/screen.c @@ -0,0 +1,237 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +void init_smtp_acl(struct smtp_acl *smtp_acl[]){ + smtp_acl[0] = NULL; +} + + +void clear_smtp_acl(struct smtp_acl *smtp_acl[]){ + struct smtp_acl *q; + + q = smtp_acl[0]; + + while(q){ + struct smtp_acl *p = q; + q = q->r; + + free(p); + } + + smtp_acl[0] = NULL; +} + + +int add_smtp_acl(struct smtp_acl *smtp_acl[], char *network_str, struct smtp_acl *acl){ + struct smtp_acl *q, *p=NULL, *node; + + if((node = malloc(sizeof(struct smtp_acl))) == NULL) return 0; + + memset(node, 0, sizeof(struct smtp_acl)); + + node->low = acl->low; + node->high = acl->high; + node->prefix = acl->prefix; + node->rejected = acl->rejected; + snprintf(node->network_str, sizeof(node->network_str)-1, "%s", network_str); + + node->r = NULL; + + q = smtp_acl[0]; + + while(q){ + p = q; + q = q->r; + } + + if(!p){ + smtp_acl[0] = node; + } else { + p->r = node; + } + + return 1; +} + + +int is_valid_line(char *line){ + // Skip comments + if(line[0] == ';' || line[0] == '#') return 0; + + trimBuffer(line); + + // Skip empty line + if(line[0] == 0) return 0; + + // Currently we support ipv4 stuff only, ie. valid characters are: 0-9./ + // and a line should look like "1.2.3.4/24 permit" or similar (without quotes) + + if(!strchr(line, '.') || !strchr(line, '/') || (!strchr(line, ' ') && !strchr(line, '\t')) ){ + return -1; + } + + // ascii values: + // 46: . + // 47: / + // 48-57: 0-9 + // 65-90: A-Z + // 97-122: a-z + for(; *line; line++){ + if(isalnum(*line) == 0 && isblank(*line) == 0 && *line != 46 && *line != 47) return -1; + } + + return 1; +} + + +int a_to_hl(char *ipstr, in_addr_t *addr){ + struct in_addr in; + + if(inet_aton(ipstr, &in) == 1){ + *addr = ntohl(in.s_addr); + return 1; + } + + return 0; +} + + +in_addr_t netmask(int prefix){ + if(prefix == 0) + return( ~((in_addr_t) -1) ); + else + return ~((1 << (32 - prefix)) - 1); +} + + +in_addr_t network(in_addr_t addr, int prefix){ + return addr & netmask(prefix); +} + + +in_addr_t broadcast(in_addr_t addr, int prefix){ + return addr | ~netmask(prefix); +} + + +int str_to_net_range(char *network_addr_prefix, struct smtp_acl *smtp_acl){ + in_addr_t net = 0; + int prefix = 0; + + smtp_acl->low = 0; + smtp_acl->high = 0; + + // By default we permit unless you specify "reject" (without quotes) + // To be on the safer side we permit even if you misspell the word "reject" + + smtp_acl->rejected = 0; + + if(strcasestr(network_addr_prefix, "reject")){ + smtp_acl->rejected = 1; + } + + char *p = strchr(network_addr_prefix, '/'); + if(!p) return 0; + + if(strlen(network_addr_prefix) > sizeof(smtp_acl->network_str)){ + syslog(LOG_PRIORITY, "line *%s* is longer than %ld bytes, discarded", network_addr_prefix, sizeof(smtp_acl->network_str)); + return 0; + } + + char buf[SMALLBUFSIZE]; + snprintf(buf, sizeof(buf)-1, "%s", network_addr_prefix); + + *p = '\0'; + prefix = atoi(++p); + + if(a_to_hl(network_addr_prefix, &net)){ + smtp_acl->low = network(net, prefix); + smtp_acl->high = broadcast(net, prefix); + smtp_acl->prefix = prefix; + + syslog(LOG_PRIORITY, "info: parsed acl *%s* to low: %u, high: %u, prefix: %d, reject: %d", buf, smtp_acl->low, smtp_acl->high, smtp_acl->prefix, smtp_acl->rejected); + + return 1; + } + + return 0; +} + + +void load_smtp_acl(struct smtp_acl *smtp_acl[]){ + int count=0; + + clear_smtp_acl(smtp_acl); + init_smtp_acl(smtp_acl); + + FILE *f = fopen(SMTP_ACL_FILE, "r"); + if(!f){ + syslog(LOG_PRIORITY, "info: cannot open %s, piler-smtp accepts smtp connections from everywhere", SMTP_ACL_FILE); + return; + } + + char line[SMALLBUFSIZE]; + struct smtp_acl acl; + + while(fgets(line, sizeof(line)-1, f)){ + int rc = is_valid_line(line); + if(rc < 0){ + syslog(LOG_PRIORITY, "warn: invalid network range: *%s*", line); + } + + if(rc == 1 && str_to_net_range(line, &acl) == 1){ + add_smtp_acl(smtp_acl, line, &acl); + count++; + } + } + + fclose(f); + + // If we have entries on the smtp acl list, then add 127.0.0.1/8 + if(count){ + snprintf(line, sizeof(line)-1, "127.0.0.1/8 permit"); + + if(str_to_net_range(line, &acl) == 1){ + add_smtp_acl(smtp_acl, line, &acl); + } + } +} + + +int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr, struct config *cfg){ + struct smtp_acl *q; + in_addr_t addr = 0; + + if(a_to_hl(ipaddr, &addr) == 0){ + syslog(LOG_PRIORITY, "error: invalid smtp client address: *%s*", ipaddr); + return 1; + } + + q = smtp_acl[0]; + + // Empty network ranges list + if(!q){ + syslog(LOG_PRIORITY, "info: empty network ranges list, pass"); + return 0; + } + + while(q){ + if(addr >= q->low && addr <= q->high){ + if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "info: smtp client %s is on %s/%d rejected: %d", ipaddr, q->network_str, q->prefix, q->rejected); + return q->rejected; + } + + q = q->r; + } + + return 1; +} diff --git a/src/screen.h b/src/screen.h new file mode 100644 index 00000000..7574fd61 --- /dev/null +++ b/src/screen.h @@ -0,0 +1,16 @@ +/* + * rules.h, SJ + */ + +#ifndef _NETRANGE_H + #define _NETRANGE_H + +#include "defs.h" + +void init_smtp_acl(struct smtp_acl *smtp_acl[]); +void clear_smtp_acl(struct smtp_acl *smtp_acl[]); +int add_smtp_acl(struct smtp_acl *smtp_acl[], char *network_str, struct smtp_acl *acl); +void load_smtp_acl(struct smtp_acl *smtp_acl[]); +int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr, struct config *cfg); + +#endif /* _NETRANGE_H */ diff --git a/src/session.c b/src/session.c index b1cacb7e..d404e073 100644 --- a/src/session.c +++ b/src/session.c @@ -10,26 +10,7 @@ int get_session_slot(struct smtp_session **sessions, int max_connections); void init_smtp_session(struct smtp_session *session, int slot, int sd, struct config *cfg); -#ifdef HAVE_LIBWRAP -int is_blocked_by_tcp_wrappers(int sd){ - struct request_info req; - - request_init(&req, RQ_DAEMON, "piler", RQ_FILE, sd, 0); - - fromhost(&req); - - if(!hosts_access(&req)){ - send(sd, SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY, strlen(SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY), 0); - syslog(LOG_PRIORITY, "denied connection from %s by tcp_wrappers", eval_client(&req)); - return 1; - } - - return 0; -} -#endif - - -int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct config *cfg){ +int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg){ int slot; /* @@ -43,12 +24,13 @@ int start_new_session(struct smtp_session **sessions, int socket, int *num_conne return -1; } -#ifdef HAVE_LIBWRAP - if(is_blocked_by_tcp_wrappers(socket) == 1){ + // Check remote client against the allowed network ranges + if(cfg->smtp_access_list && is_blocked_by_pilerscreen(smtp_acl, client_addr, cfg)){ + send(socket, SMTP_RESP_550_ERR, strlen(SMTP_RESP_550_ERR), 0); close(socket); + syslog(LOG_PRIORITY, "denied connection from %s by %s", client_addr, SMTP_ACL_FILE); return -1; } -#endif slot = get_session_slot(sessions, cfg->max_connections); diff --git a/src/smtpcodes.h b/src/smtpcodes.h index ee0d2afc..2a9943d5 100644 --- a/src/smtpcodes.h +++ b/src/smtpcodes.h @@ -55,6 +55,7 @@ #define SMTP_RESP_502_ERR "502 Command not implemented\r\n" #define SMTP_RESP_503_ERR "503 Bad command sequence\r\n" #define SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY "550 You are banned by local policy\r\n" +#define SMTP_RESP_550_ERR "550 Service currently unavailable\r\n" // LMTP commands From c1150832dd719c3b2cbaea63f73f8bcdd0c38e4e Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 28 Dec 2020 11:30:11 +0100 Subject: [PATCH 023/106] Fixed header protection text for screen.h Signed-off-by: Janos SUTO --- src/screen.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/screen.h b/src/screen.h index 7574fd61..27276b40 100644 --- a/src/screen.h +++ b/src/screen.h @@ -1,9 +1,9 @@ /* - * rules.h, SJ + * screen.h, SJ */ -#ifndef _NETRANGE_H - #define _NETRANGE_H +#ifndef _SCREEN_H + #define _SCREEN_H #include "defs.h" @@ -13,4 +13,4 @@ int add_smtp_acl(struct smtp_acl *smtp_acl[], char *network_str, struct smtp_acl void load_smtp_acl(struct smtp_acl *smtp_acl[]); int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr, struct config *cfg); -#endif /* _NETRANGE_H */ +#endif /* _SCREEN_H */ From ed8fc2a6e836429d211181f7b01d61f9da7aee44 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 28 Dec 2020 13:11:29 +0100 Subject: [PATCH 024/106] Fixed smtp acl implementation Signed-off-by: Janos SUTO --- etc/example.conf | 11 +++---- etc/smtp.acl.example | 9 ++++++ src/screen.c | 73 +++++++++++++++++++++++++++----------------- src/session.c | 1 - 4 files changed, 59 insertions(+), 35 deletions(-) create mode 100644 etc/smtp.acl.example diff --git a/etc/example.conf b/etc/example.conf index 44e8ec57..f8b65fc7 100644 --- a/etc/example.conf +++ b/etc/example.conf @@ -232,11 +232,10 @@ mmap_dedup_test=0 security_header= ; whether to enable (1) or not (0) an smtp access list similar to -; postfix's postscreen. If so, then create a text file %sysconfdir%/piler/smtp.acl -; An example for /usr/local/etc/piler/smtp.acl: -; -; 1.2.3.4/32 permit -; 10.0.1.0/24 reject -; 172.16.0.0/16 permit +; postfix's postscreen. Valid actions in the acl file are "permit" +; and "reject" (without quotes). See smtp.acl.example for more. ; +; Important! There's an implicit default deny at the end of the +; rules. In other words if you decide to use the acl file, then +; everyone is not explicitly permitted is denied. smtp_access_list=0 diff --git a/etc/smtp.acl.example b/etc/smtp.acl.example new file mode 100644 index 00000000..4019a7e2 --- /dev/null +++ b/etc/smtp.acl.example @@ -0,0 +1,9 @@ +# Allow office365 servers. See the below URI for more +# https://docs.microsoft.com/en-us/microsoft-365/enterprise/urls-and-ip-address-ranges?view=o365-worldwide +# +# Anything is not listed below will be rejected +# +40.92.0.0/15 permit +40.107.0.0/16 permit +52.100.0.0/14 permit +104.47.0.0/17 permit diff --git a/src/screen.c b/src/screen.c index 0f3fda45..d03b2539 100644 --- a/src/screen.c +++ b/src/screen.c @@ -64,29 +64,25 @@ int add_smtp_acl(struct smtp_acl *smtp_acl[], char *network_str, struct smtp_acl int is_valid_line(char *line){ - // Skip comments - if(line[0] == ';' || line[0] == '#') return 0; - - trimBuffer(line); - - // Skip empty line - if(line[0] == 0) return 0; - // Currently we support ipv4 stuff only, ie. valid characters are: 0-9./ // and a line should look like "1.2.3.4/24 permit" or similar (without quotes) if(!strchr(line, '.') || !strchr(line, '/') || (!strchr(line, ' ') && !strchr(line, '\t')) ){ - return -1; + return 0; + } + + if(!strstr(line, "permit") && !strstr(line, "reject")){ + return 0; } // ascii values: // 46: . // 47: / // 48-57: 0-9 - // 65-90: A-Z // 97-122: a-z + // for(; *line; line++){ - if(isalnum(*line) == 0 && isblank(*line) == 0 && *line != 46 && *line != 47) return -1; + if(isalnum(*line) == 0 && isblank(*line) == 0 && *line != 46 && *line != 47) return 0; } return 1; @@ -101,6 +97,8 @@ int a_to_hl(char *ipstr, in_addr_t *addr){ return 1; } + syslog(LOG_PRIORITY, "invalid ipv4 address string: *%s*", ipstr); + return 0; } @@ -151,14 +149,25 @@ int str_to_net_range(char *network_addr_prefix, struct smtp_acl *smtp_acl){ snprintf(buf, sizeof(buf)-1, "%s", network_addr_prefix); *p = '\0'; - prefix = atoi(++p); + p++; + + // Even though the remaining part of the acl line continues with some number + // then whitespace characters and the permit/reject action atoi() can still + // figure out the numeric part properly, that's why I'm lazy here. + prefix = atoi(p); + + // The prefix string (p) must start with a digit and the prefix integer must be in 0..32 range + if(*p < 48 || *p > 57 || prefix < 0 || prefix > 32){ + syslog(LOG_PRIORITY, "error: invalid prefix: %s", p); + return 0; + } if(a_to_hl(network_addr_prefix, &net)){ smtp_acl->low = network(net, prefix); smtp_acl->high = broadcast(net, prefix); smtp_acl->prefix = prefix; - syslog(LOG_PRIORITY, "info: parsed acl *%s* to low: %u, high: %u, prefix: %d, reject: %d", buf, smtp_acl->low, smtp_acl->high, smtp_acl->prefix, smtp_acl->rejected); + syslog(LOG_PRIORITY, "info: parsed acl *%s*", buf); return 1; } @@ -183,20 +192,31 @@ void load_smtp_acl(struct smtp_acl *smtp_acl[]){ struct smtp_acl acl; while(fgets(line, sizeof(line)-1, f)){ - int rc = is_valid_line(line); - if(rc < 0){ - syslog(LOG_PRIORITY, "warn: invalid network range: *%s*", line); - } + // Skip comments + if(line[0] == ';' || line[0] == '#') continue; - if(rc == 1 && str_to_net_range(line, &acl) == 1){ + trimBuffer(line); + + // Skip empty line + if(line[0] == 0) continue; + + char line2[SMALLBUFSIZE]; + int rc = 0; + snprintf(line2, sizeof(line2)-1, "%s", line); + + if(is_valid_line(line) == 1 && str_to_net_range(line, &acl) == 1){ add_smtp_acl(smtp_acl, line, &acl); count++; + rc = 1; } + + if(!rc) syslog(LOG_PRIORITY, "error: failed to parse line: *%s*", line2); } fclose(f); // If we have entries on the smtp acl list, then add 127.0.0.1/8 + // to let the GUI health page connect to the piler-smtp daemon if(count){ snprintf(line, sizeof(line)-1, "127.0.0.1/8 permit"); @@ -208,30 +228,27 @@ void load_smtp_acl(struct smtp_acl *smtp_acl[]){ int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr, struct config *cfg){ - struct smtp_acl *q; + struct smtp_acl *q=smtp_acl[0]; in_addr_t addr = 0; + // Empty acl, let it pass + if(!q) return 0; + if(a_to_hl(ipaddr, &addr) == 0){ syslog(LOG_PRIORITY, "error: invalid smtp client address: *%s*", ipaddr); return 1; } - q = smtp_acl[0]; - - // Empty network ranges list - if(!q){ - syslog(LOG_PRIORITY, "info: empty network ranges list, pass"); - return 0; - } - while(q){ if(addr >= q->low && addr <= q->high){ - if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "info: smtp client %s is on %s/%d rejected: %d", ipaddr, q->network_str, q->prefix, q->rejected); + if(q->rejected) syslog(LOG_PRIORITY, "denied connection from %s, acl: %s/%d reject", ipaddr, q->network_str, q->prefix); return q->rejected; } q = q->r; } + syslog(LOG_PRIORITY, "denied connection from %s by implicit default deny", ipaddr); + return 1; } diff --git a/src/session.c b/src/session.c index d404e073..214267c2 100644 --- a/src/session.c +++ b/src/session.c @@ -28,7 +28,6 @@ int start_new_session(struct smtp_session **sessions, int socket, int *num_conne if(cfg->smtp_access_list && is_blocked_by_pilerscreen(smtp_acl, client_addr, cfg)){ send(socket, SMTP_RESP_550_ERR, strlen(SMTP_RESP_550_ERR), 0); close(socket); - syslog(LOG_PRIORITY, "denied connection from %s by %s", client_addr, SMTP_ACL_FILE); return -1; } From 1e6a4b22c14b2c79b6634b867ae69e49f0e30139 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 28 Dec 2020 20:31:11 +0100 Subject: [PATCH 025/106] smtp acl fixes Signed-off-by: Janos SUTO --- src/defs.h | 2 +- src/screen.c | 2 +- src/screen.h | 2 +- src/session.c | 18 +++++------------- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/defs.h b/src/defs.h index ff8be3a4..4c5a3b49 100644 --- a/src/defs.h +++ b/src/defs.h @@ -398,7 +398,7 @@ struct smtp_session { char mailfrom[SMALLBUFSIZE]; char rcptto[MAX_RCPT_TO][SMALLBUFSIZE]; char buf[MAXBUFSIZE]; - char remote_host[INET6_ADDRSTRLEN]; + char remote_host[INET6_ADDRSTRLEN+1]; time_t lasttime; int protocol_state; int slot; diff --git a/src/screen.c b/src/screen.c index d03b2539..73e780ff 100644 --- a/src/screen.c +++ b/src/screen.c @@ -227,7 +227,7 @@ void load_smtp_acl(struct smtp_acl *smtp_acl[]){ } -int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr, struct config *cfg){ +int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr){ struct smtp_acl *q=smtp_acl[0]; in_addr_t addr = 0; diff --git a/src/screen.h b/src/screen.h index 27276b40..e4e1e9cd 100644 --- a/src/screen.h +++ b/src/screen.h @@ -11,6 +11,6 @@ void init_smtp_acl(struct smtp_acl *smtp_acl[]); void clear_smtp_acl(struct smtp_acl *smtp_acl[]); int add_smtp_acl(struct smtp_acl *smtp_acl[], char *network_str, struct smtp_acl *acl); void load_smtp_acl(struct smtp_acl *smtp_acl[]); -int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr, struct config *cfg); +int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr); #endif /* _SCREEN_H */ diff --git a/src/session.c b/src/session.c index 214267c2..a4d75090 100644 --- a/src/session.c +++ b/src/session.c @@ -7,7 +7,7 @@ int get_session_slot(struct smtp_session **sessions, int max_connections); -void init_smtp_session(struct smtp_session *session, int slot, int sd, struct config *cfg); +void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg); int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg){ @@ -25,7 +25,7 @@ int start_new_session(struct smtp_session **sessions, int socket, int *num_conne } // Check remote client against the allowed network ranges - if(cfg->smtp_access_list && is_blocked_by_pilerscreen(smtp_acl, client_addr, cfg)){ + if(cfg->smtp_access_list && is_blocked_by_pilerscreen(smtp_acl, client_addr)){ send(socket, SMTP_RESP_550_ERR, strlen(SMTP_RESP_550_ERR), 0); close(socket); return -1; @@ -36,7 +36,7 @@ int start_new_session(struct smtp_session **sessions, int socket, int *num_conne if(slot >= 0 && sessions[slot] == NULL){ sessions[slot] = malloc(sizeof(struct smtp_session)); if(sessions[slot]){ - init_smtp_session(sessions[slot], slot, socket, cfg); + init_smtp_session(sessions[slot], slot, socket, client_addr, cfg); char smtp_banner[SMALLBUFSIZE]; snprintf(smtp_banner, sizeof(smtp_banner)-1, SMTP_RESP_220_BANNER, cfg->hostid); @@ -83,10 +83,7 @@ struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int m } -void init_smtp_session(struct smtp_session *session, int slot, int sd, struct config *cfg){ - struct sockaddr_in addr; - socklen_t addr_size = sizeof(struct sockaddr_in); - char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV]; +void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg){ int i; session->slot = slot; @@ -112,16 +109,11 @@ void init_smtp_session(struct smtp_session *session, int slot, int sd, struct co for(i=0; ircptto[i], 0, SMALLBUFSIZE); memset(session->buf, 0, MAXBUFSIZE); - memset(session->remote_host, 0, INET6_ADDRSTRLEN); + snprintf(session->remote_host, sizeof(session->remote_host)-1, "%s", client_addr); reset_bdat_counters(session); time(&(session->lasttime)); - - if(getpeername(session->net.socket, (struct sockaddr *)&addr, &addr_size) == 0 && - getnameinfo((struct sockaddr *)&addr, addr_size, hbuf, sizeof(hbuf), sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV) == 0){ - snprintf(session->remote_host, INET6_ADDRSTRLEN-1, "%s", hbuf); - } } From 1288c9fed340d4b89b73e7a53f22ee27eac6a571 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 28 Dec 2020 20:39:43 +0100 Subject: [PATCH 026/106] Refactored hbuf and sbuf in piler-smtp.c Signed-off-by: Janos SUTO --- src/piler-smtp.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/piler-smtp.c b/src/piler-smtp.c index b5fd2623..f52f1264 100644 --- a/src/piler-smtp.c +++ b/src/piler-smtp.c @@ -135,7 +135,6 @@ int main(int argc, char **argv){ int client_len = sizeof(struct sockaddr_storage); ssize_t readlen; struct sockaddr_storage client_address; - char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV]; char readbuf[BIGBUFSIZE]; int efd; @@ -257,6 +256,10 @@ int main(int argc, char **argv){ } } + char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV]; + memset(hbuf, 0, sizeof(hbuf)); + memset(sbuf, 0, sizeof(sbuf)); + if(getnameinfo((struct sockaddr *)&client_address, client_len, hbuf, sizeof(hbuf), sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV) == 0){ // Strictly speaking it's not correct to log num_connections+1 connections // but it still gives a good clue how many connections we have at the moment From 624cca58e6185520e39d7f13802a4f3b48dced5e Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 3 Jan 2021 18:17:25 +0100 Subject: [PATCH 027/106] Updated php-fpm socket path to match 7.4 Signed-off-by: Janos SUTO --- contrib/webserver/piler-nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/webserver/piler-nginx.conf b/contrib/webserver/piler-nginx.conf index 59a3f41f..c61c8523 100644 --- a/contrib/webserver/piler-nginx.conf +++ b/contrib/webserver/piler-nginx.conf @@ -28,7 +28,7 @@ server { return 404; } - fastcgi_pass unix:/var/run/php/php7.2-fpm.sock; + fastcgi_pass unix:/var/run/php/php7.4-fpm.sock; fastcgi_index index.php; include fastcgi_params; } From b740bd507560a24340957067c98ec8e0f1853c16 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 3 Jan 2021 18:20:32 +0100 Subject: [PATCH 028/106] Fixed docker config Signed-off-by: Janos SUTO --- docker/Dockerfile | 42 +++---- docker/build.sh | 11 ++ docker/docker-compose.yaml | 61 ++++++++++ docker/start.sh | 238 +++++++++++++++++++++++++++---------- 4 files changed, 262 insertions(+), 90 deletions(-) create mode 100755 docker/build.sh create mode 100644 docker/docker-compose.yaml diff --git a/docker/Dockerfile b/docker/Dockerfile index baf3fd80..217161be 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,48 +2,40 @@ FROM ubuntu:20.04 ARG PACKAGE -LABEL description="piler container" \ +LABEL description="piler ubuntu focal image" \ maintainer="Janos SUTO, sj@acts.hu" \ package="${PACKAGE}" ENV DEBIAN_FRONTEND="noninteractive" \ - DISTRO="bionic" \ + DISTRO="focal" \ DOWNLOAD_URL="https://download.mailpiler.com" \ PILER_USER="piler" \ - MYSQL_HOSTNAME="localhost" \ MYSQL_DATABASE="piler" \ - MYSQL_PILER_PASSWORD="piler123" \ - MYSQL_ROOT_PASSWORD="abcde123" \ - SPHINX_BIN_TARGZ="sphinx-3.1.1-bin.tar.gz" + SPHINX_BIN_TARGZ="sphinx-3.3.1-bin.tar.gz" -ADD "https://bitbucket.org/jsuto/piler/downloads/${PACKAGE}" "/${PACKAGE}" -COPY start.sh /start.sh +COPY ${PACKAGE} / RUN apt-get update && \ apt-get -y --no-install-recommends install \ - wget rsyslog openssl sysstat php7.2-cli php7.2-cgi php7.2-mysql php7.2-fpm php7.2-zip php7.2-ldap \ - php7.2-gd php7.2-curl php7.2-xml catdoc unrtf poppler-utils nginx tnef sudo libodbc1 libpq5 libzip4 \ - libtre5 libwrap0 cron libmariadb3 libmysqlclient-dev python python-mysqldb mariadb-server && \ + wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \ + php7.4-gd php7.4-curl php7.4-xml php7.4-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip5 \ + libtre5 cron libmariadb-dev mariadb-client-core-10.3 python3 python3-mysqldb && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ - service mysql start && mysqladmin -u root password ${MYSQL_ROOT_PASSWORD} && \ wget --no-check-certificate -q -O ${SPHINX_BIN_TARGZ} ${DOWNLOAD_URL}/generic-local/${SPHINX_BIN_TARGZ} && \ tar zxvf ${SPHINX_BIN_TARGZ} && \ - rm -f ${SPHINX_BIN_TARGZ} && \ sed -i 's/mail.[iwe].*//' /etc/rsyslog.conf && \ sed -i '/session required pam_loginuid.so/c\#session required pam_loginuid.so' /etc/pam.d/cron && \ - mkdir /etc/piler && \ - printf "[mysql]\nuser = piler\npassword = %s\n" ${MYSQL_PILER_PASSWORD} > /etc/piler/.my.cnf && \ - printf "[mysql]\nuser = root\npassword = %s\n" ${MYSQL_ROOT_PASSWORD} > /root/.my.cnf && \ - echo "alias mysql='mysql --defaults-file=/etc/piler/.my.cnf'" > /root/.bashrc && \ - echo "alias t='tail -f /var/log/syslog'" >> /root/.bashrc && \ - dpkg -i $PACKAGE && \ - crontab -u $PILER_USER /usr/share/piler/piler.cron && \ - touch /var/log/mail.log && \ - rm -f $PACKAGE /etc/nginx/sites-enabled/default && \ - sed -i 's/#ngram/ngram/g' /etc/piler/sphinx.conf.dist && \ - sed -i 's/220/311/g' /etc/piler/sphinx.conf.dist + dpkg -i ${PACKAGE} && \ + ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled && \ + rm -f ${PACKAGE} ${SPHINX_BIN_TARGZ} /etc/nginx/sites-enabled/default /etc/piler/piler.key /etc/piler/piler.pem /etc/piler/config-site.php && \ + crontab -u $PILER_USER /usr/share/piler/piler.cron + +VOLUME ["/etc/piler"] +VOLUME ["/var/piler"] EXPOSE 25 80 443 -VOLUME ["/var/piler"] + +COPY start.sh /start.sh + CMD ["/start.sh"] diff --git a/docker/build.sh b/docker/build.sh new file mode 100755 index 00000000..796063ba --- /dev/null +++ b/docker/build.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +IMAGE_NAME="sutoj/piler:1.3.10" + +if [[ $# -ne 1 ]]; then echo "ERROR: missing package name" 1>&2; exit 1; fi + +docker build --pull --build-arg PACKAGE="$1" -t "$IMAGE_NAME" . diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 00000000..7a42aaa8 --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,61 @@ +version: "3" +services: + + mysql: + image: mariadb:10.4 + restart: unless-stopped + cap_drop: + - ALL + cap_add: + - dac_override + - setuid + - setgid + environment: + - MYSQL_ROOT_PASSWORD=abcde123 + volumes: + - db_data:/var/lib/mysql + + memcached: + image: memcached:latest + restart: unless-stopped + cap_drop: + - ALL + command: -m 64 + + piler: + image: sutoj/piler:1.3.10 + init: true + environment: + - MYSQL_PILER_PASSWORD=piler123 + - MYSQL_ROOT_PASSWORD=abcde123 + - MYSQL_HOSTNAME=mysql + - PILER_HOSTNAME=archive.yourdomain.com + - MEMCACHED_HOST=memcached + ports: + - "25:25" + - "80:80" + - "443:443" + volumes: + - piler_etc:/etc/piler + - piler_var:/var/piler + healthcheck: + test: curl -s smtp://localhost/ + interval: "60s" + timeout: "3s" + start_period: "15s" + retries: 3 + deploy: + resources: + reservations: + memory: 512M + limits: + memory: 512M + + depends_on: + - "memcached" + - "mysql" + +volumes: + db_data: {} + piler_etc: {} + piler_var: {} diff --git a/docker/start.sh b/docker/start.sh index 12b7ee31..dc27e073 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -4,91 +4,199 @@ set -o errexit set -o pipefail set -o nounset -DATAROOTDIR="/usr/share" -SYSCONFDIR="/etc" -SPHINXCFG="/etc/piler/sphinx.conf" -PILER_HOST=${PILER_HOST:-archive.yourdomain.com} -PILER_CONF="/etc/piler/piler.conf" -CONFIG_SITE_PHP="/etc/piler/config-site.php" -CONFIG_PHP="/var/piler/www/config.php" +CONFIG_DIR="/etc/piler" +VOLUME_DIR="/var/piler" +PILER_CONF="${CONFIG_DIR}/piler.conf" +PILER_KEY="${CONFIG_DIR}/piler.key" +PILER_PEM="${CONFIG_DIR}/piler.pem" +PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf" +SPHINX_CONF="${CONFIG_DIR}/sphinx.conf" +CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php" +PILER_MY_CNF="${CONFIG_DIR}/.my.cnf" +ROOT_MY_CNF="/root/.my.cnf" -create_mysql_db() { - echo "Creating mysql database" - sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%g" \ - -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%g" \ - -e "s%MYSQL_USERNAME%${PILER_USER}%g" \ - -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" \ - "${DATAROOTDIR}/piler/db-mysql-root.sql.in" | \ - mysql -h "$MYSQL_HOSTNAME" -u root --password="$MYSQL_ROOT_PASSWORD" - - mysql -h "$MYSQL_HOSTNAME" -u "$PILER_USER" --password="$MYSQL_PILER_PASSWORD" "$MYSQL_DATABASE" < "${DATAROOTDIR}/piler/db-mysql.sql" - - echo "Done." +error() { + echo "ERROR:" "$*" 1>&2 + exit 1 } -pre_seed_sphinx() { - echo "Writing sphinx configuration" - sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \ - -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \ - -e "s%MYSQL_USERNAME%${PILER_USER}%" \ - -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%" \ - -e "s%220%311%" \ - -e "s%type = mysql%type = mysql\n sql_sock = /var/run/mysqld/mysqld.sock%" \ - "${SYSCONFDIR}/piler/sphinx.conf.dist" > "$SPHINXCFG" +log() { + echo "DEBUG:" "$*" +} - echo "Done." - echo "Initializing sphinx indices" - su "$PILER_USER" -c "indexer --all --config ${SYSCONFDIR}/piler/sphinx.conf" - echo "Done." +pre_flight_check() { + [[ -v PILER_HOSTNAME ]] || error "Missing PILER_HOSTNAME env variable" + [[ -v MYSQL_HOSTNAME ]] || error "Missing MYSQL_HOSTNAME env variable" + [[ -v MYSQL_PILER_PASSWORD ]] || error "Missing MYSQL_PILER_PASSWORD env variable" + [[ -v MYSQL_ROOT_PASSWORD ]] || error "Missing MYSQL_ROOT_PASSWORD env variable" +} + + +give_it_to_piler() { + local f="$1" + + [[ -f "$f" ]] || error "${f} does not exist, aborting" + + chown "${PILER_USER}:${PILER_USER}" "$f" + chmod 600 "$f" +} + + +make_certificate() { + local f="$1" + local crt="/tmp/1.cert" + local SSL_CERT_DATA="/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com" + + log "Making an ssl certificate" + + openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 -subj "$SSL_CERT_DATA" -keyout "$f" -out "$crt" -sha1 2>/dev/null + cat "$crt" >> "$f" + rm -f "$crt" + + give_it_to_piler "$f" +} + + +make_piler_key() { + local f="$1" + + log "Generating piler.key" + + dd if=/dev/urandom bs=56 count=1 of="$f" 2>/dev/null + [[ $(stat -c '%s' "$f") -eq 56 ]] || error "could not read 56 bytes from /dev/urandom to ${f}" + + give_it_to_piler "$f" } fix_configs() { - local piler_nginx_conf="/etc/piler/piler-nginx.conf" + [[ -f "$PILER_KEY" ]] || make_piler_key "$PILER_KEY" + [[ -f "$PILER_PEM" ]] || make_certificate "$PILER_PEM" + + if [[ ! -f "$PILER_NGINX_CONF" ]]; then + log "Writing ${PILER_NGINX_CONF}" + + cp "${PILER_NGINX_CONF}.dist" "$PILER_NGINX_CONF" + sed -i "s%PILER_HOST%${PILER_HOSTNAME}%" "$PILER_NGINX_CONF" + fi if [[ ! -f "$PILER_CONF" ]]; then - cp /etc/piler/piler.conf.dist "$PILER_CONF" - chmod 640 "$PILER_CONF" - chown root:piler "$PILER_CONF" - sed -i "s%hostid=.*%hostid=${PILER_HOST%%:*}%" "$PILER_CONF" - sed -i "s%tls_enable=.*%tls_enable=1%" "$PILER_CONF" - sed -i "s%mysqlpwd=.*%mysqlpwd=${MYSQL_PILER_PASSWORD}%" "$PILER_CONF" + log "Writing ${PILER_CONF}" + + sed \ + -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" \ + -e "s/hostid=.*/hostid=${PILER_HOSTNAME}/g" \ + -e "s/tls_enable=.*/tls_enable=1/g" \ + -e "s/mysqlsocket=.*/mysqlsocket=/g" "${PILER_CONF}.dist" > "$PILER_CONF" + + { + echo "mysqlhost=${MYSQL_HOSTNAME}" + } >> "$PILER_CONF" + + give_it_to_piler "$PILER_CONF" fi - if [[ ! -f "$piler_nginx_conf" ]]; then - cp /etc/piler/piler-nginx.conf.dist "$piler_nginx_conf" - sed -i "s%PILER_HOST%${PILER_HOST}%" "$piler_nginx_conf" + if [[ ! -f "$CONFIG_SITE_PHP" ]]; then + log "Writing ${CONFIG_SITE_PHP}" + + cp "${CONFIG_DIR}/config-site.dist.php" "$CONFIG_SITE_PHP" + + sed -i "s%HOSTNAME%${PILER_HOSTNAME}%" "$CONFIG_SITE_PHP" + + { + echo "\$config['DECRYPT_BINARY'] = '/usr/bin/pilerget';" + echo "\$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/bin/pileraget';" + echo "\$config['PILER_BINARY'] = '/usr/sbin/piler';" + echo "\$config['DB_HOSTNAME'] = '$MYSQL_HOSTNAME';" + echo "\$config['DB_PASSWORD'] = '$MYSQL_PILER_PASSWORD';" + echo "\$config['ENABLE_MEMCACHED'] = 1;" + echo "\$memcached_server = ['memcached', 11211];" + } >> "$CONFIG_SITE_PHP" fi - ln -sf "$piler_nginx_conf" /etc/nginx/sites-enabled/piler - - sed -i "s%HOSTNAME%${PILER_HOST}%" "$CONFIG_SITE_PHP" - sed -i "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%" "$CONFIG_SITE_PHP" - - sed -i "s%^\$config\['DECRYPT_BINARY'\].*%\$config\['DECRYPT_BINARY'\] = '/usr/bin/pilerget';%" "$CONFIG_PHP" - sed -i "s%^\$config\['DECRYPT_ATTACHMENT_BINARY'\].*%\$config\['DECRYPT_ATTACHMENT_BINARY'\] = '/usr/bin/pileraget';%" "$CONFIG_PHP" - sed -i "s%^\$config\['PILER_BINARY'\].*%\$config\['PILER_BINARY'\] = '/usr/sbin/piler';%" "$CONFIG_PHP" + sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \ + -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \ + -e "s%MYSQL_USERNAME%${PILER_USER}%" \ + -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%" \ + -i "$SPHINX_CONF" } -service rsyslog start -service mysql start +wait_until_mysql_server_is_ready() { + while true; do if mysql "--defaults-file=${ROOT_MY_CNF}" <<< "show databases"; then break; fi; log "${MYSQL_HOSTNAME} is not ready"; sleep 5; done -create_mysql_db -pre_seed_sphinx + log "${MYSQL_HOSTNAME} is ready" +} + + +init_database() { + local db + local has_piler_db=0 + + wait_until_mysql_server_is_ready + + while read -r db; do + if [[ "$db" == "$MYSQL_DATABASE" ]]; then has_piler_db=1; fi + done < <(mysql "--defaults-file=${ROOT_MY_CNF}" <<< 'show databases') + + if [[ $has_piler_db -eq 0 ]]; then + log "no ${MYSQL_DATABASE} database, creating" + + mysql "--defaults-file=${ROOT_MY_CNF}" <<< "create database ${MYSQL_DATABASE} character set utf8mb4" + mysql "--defaults-file=${ROOT_MY_CNF}" <<< "grant all privileges on ${MYSQL_DATABASE}.* to ${PILER_USER} identified by '${MYSQL_PILER_PASSWORD}'" + mysql "--defaults-file=${ROOT_MY_CNF}" <<< "flush privileges" + + mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" < /usr/share/piler/db-mysql.sql + else + log "${MYSQL_DATABASE} db exists" + fi + + if [[ -v ADMIN_USER_PASSWORD_HASH ]]; then + mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< "update user set password='${ADMIN_USER_PASSWORD_HASH}' where uid=0" + fi +} + + +create_my_cnf_files() { + printf "[client]\nhost = %s\nuser = %s\npassword = %s\n[mysqldump]\nhost = %s\nuser = %s\npassword = %s\n" \ + "$MYSQL_HOSTNAME" "$PILER_USER" "$MYSQL_PILER_PASSWORD" "$MYSQL_HOSTNAME" "$PILER_USER" "$MYSQL_PILER_PASSWORD" \ + > "$PILER_MY_CNF" + printf "[client]\nhost = %s\nuser = root\npassword = %s\n" "$MYSQL_HOSTNAME" "$MYSQL_ROOT_PASSWORD" > "$ROOT_MY_CNF" + + give_it_to_piler "$PILER_MY_CNF" +} + + +start_services() { + service rsyslog start + service cron start + service php7.4-fpm start + service nginx start +} + + +start_piler() { + if [[ ! -f "${VOLUME_DIR}/sphinx/main1.spp" ]]; then + log "main1.spp does not exist, creating index files" + su -c "indexer --all --config ${SPHINX_CONF}" piler + fi + + # No pid file should exist for piler + rm -f /var/run/piler/*pid + + /etc/init.d/rc.searchd start + /etc/init.d/rc.piler start +} + + +pre_flight_check fix_configs +create_my_cnf_files +init_database +start_services +start_piler -service cron start -service php7.2-fpm start -service nginx start -/etc/init.d/rc.searchd start - -# fix for overlay, https://github.com/phusion/baseimage-docker/issues/198 -touch /var/spool/cron/crontabs/piler - -/etc/init.d/rc.piler start - -while true; do sleep 120; done +while true; do sleep 3600; done From 7fc8f74d0f43af30a060ce6ead7d28aad8aef462 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 3 Jan 2021 18:22:42 +0100 Subject: [PATCH 029/106] Improved the nginx config Signed-off-by: Janos SUTO --- contrib/webserver/piler-nginx.conf | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/contrib/webserver/piler-nginx.conf b/contrib/webserver/piler-nginx.conf index c61c8523..737cc4f0 100644 --- a/contrib/webserver/piler-nginx.conf +++ b/contrib/webserver/piler-nginx.conf @@ -4,7 +4,13 @@ server { root /var/piler/www; - gzip on; + server_tokens off; + + add_header X-XSS-Protection "1; mode=block"; + add_header X-Content-Type-Options "nosniff"; + add_header Referrer-Policy "same-origin"; + + gzip on; gzip_types text/plain application/xml text/css; gzip_vary on; From 8678ab7e5659652dc5741765f5e7ed59760fcc4c Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 3 Jan 2021 19:39:25 +0100 Subject: [PATCH 030/106] updated release notes Signed-off-by: Janos SUTO --- RELEASE_NOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index dfc9566a..a5108693 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,7 +3,7 @@ - Added security header feature - Introduced the smtp acl list, and obsoleted the tcp_wrappers check - +- Switched from Blowfish encryption to AES-256 1.3.9: ------ From f8eeb71125af03c7073152ead917f1132d1b4992 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 4 Jan 2021 15:57:19 +0100 Subject: [PATCH 031/106] Updated test cases count number Signed-off-by: Janos SUTO --- tests/cases/05-smtp.inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cases/05-smtp.inc b/tests/cases/05-smtp.inc index a53ed2f1..88914e13 100644 --- a/tests/cases/05-smtp.inc +++ b/tests/cases/05-smtp.inc @@ -17,10 +17,10 @@ case1() { "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus" --socket --no-counter - wait_until_emails_are_processed "piler1" 3000 + wait_until_emails_are_processed "piler1" 3001 docker exec "piler1" su piler -c /usr/libexec/piler/indexer.delta.sh 2>/dev/null - count_status_values 3000 2892 108 0 + count_status_values 3001 2892 109 0 test_retrieved_messages_are_the_same "piler1" "piler" From 99a4bf4819cea7d093b8740aabd3e029cf50c81f Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 4 Jan 2021 15:57:45 +0100 Subject: [PATCH 032/106] Discard header-only message without a Message-ID line Signed-off-by: Janos SUTO --- src/piler.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/piler.c b/src/piler.c index 684d90de..d26016cf 100644 --- a/src/piler.c +++ b/src/piler.c @@ -123,9 +123,13 @@ int perform_checks(char *filename, struct session_data *sdata, struct data *data make_digests(sdata, cfg); + // A normal header is much bigger than 10 bytes. We get here for header-only + // messages without a Message-ID: line. I believe that no such message is valid, and + // it's a reasonable to discard it, and not allowing it to fill up the error directory. + if(sdata->hdr_len < 10){ - syslog(LOG_PRIORITY, "%s: invalid message, hdr_len: %d", filename, sdata->hdr_len); - return ERR; + syslog(LOG_PRIORITY, "%s: discarding: a header-only message without a Message-ID line", filename); + return ERR_DISCARDED; } int rc = process_message(sdata, parser_state, data, cfg); From ca7c65b84a50c45cdf2209466962c1056a74dc02 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 5 Jan 2021 10:05:25 +0100 Subject: [PATCH 033/106] The piler binaries paths in config.php should honor the ./configure options Signed-off-by: Janos SUTO --- Makefile.in | 3 ++- config.php.in | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile.in b/Makefile.in index 491dcb64..0f9c40a5 100644 --- a/Makefile.in +++ b/Makefile.in @@ -64,7 +64,8 @@ $(RECURSIVE_TARGETS): config-php: - sed "s%SYSCONFDIR%$(sysconfdir)%" config.php.in > webui/config.php + sed -e "s%SYSCONFDIR%$(sysconfdir)%" -e "s%SBINDIR%$(sbindir)%g" config.php.in > webui/config.php + sed -e "s%BINDIR%$(bindir)%g" -i webui/config.php installdirs: mkinstalldirs diff --git a/config.php.in b/config.php.in index b92a117c..51ca77b3 100644 --- a/config.php.in +++ b/config.php.in @@ -217,8 +217,8 @@ $config['DIR_STAT'] = '/var/piler/stat'; $config['DIR_IMAP'] = '/var/piler/imap'; $config['DIR_TMP'] = '/var/piler/tmp'; -$config['DECRYPT_BINARY'] = '/usr/local/bin/pilerget'; -$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/local/bin/pileraget'; +$config['DECRYPT_BINARY'] = 'BINDIR/pilerget'; +$config['DECRYPT_ATTACHMENT_BINARY'] = 'BINDIR/pileraget'; $config['DECRYPT_BUFFER_LENGTH'] = 65536; $config['OPENSSL_BINARY'] = '/usr/bin/openssl'; @@ -249,7 +249,7 @@ $config['MAX_EMAIL_LEN'] = 41; $config['RELOAD_COMMAND'] = 'sudo -n /etc/init.d/rc.piler reload'; $config['PILERIMPORT_IMAP_COMMAND'] = '/usr/local/bin/pilerimport -d /var/piler/imap -q -r'; $config['CPU_USAGE_COMMAND'] = "LC_ALL=C mpstat | tail -1 | rev | awk '{ print $1 }' | rev"; -$config['PILER_BINARY'] = "/usr/local/sbin/piler"; +$config['PILER_BINARY'] = "SBINDIR/piler"; $config['LDAP_IMPORT_CONFIG_FILE'] = '/usr/local/etc/ldap-import.cfg'; From cb56eaeaed2b7c44d5fd87bd1c21f1abe813f668 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Wed, 6 Jan 2021 08:36:50 +0100 Subject: [PATCH 034/106] Reset buffer in make_digests() Signed-off-by: Janos SUTO --- src/digest.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/digest.c b/src/digest.c index 5af1fa53..6899dba1 100644 --- a/src/digest.c +++ b/src/digest.c @@ -48,6 +48,7 @@ int make_digests(struct session_data *sdata, struct config *cfg){ fd = open(sdata->filename, O_RDONLY); if(fd == -1) return -1; + memset(buf, 0, sizeof(buf)); while((n = read(fd, buf, sizeof(buf))) > 0){ SHA256_Update(&context2, buf, n); From e6edbc525b590d3575bad359bfce9a80926d03d7 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Wed, 6 Jan 2021 08:38:29 +0100 Subject: [PATCH 035/106] indexer shell scripts should syslog using mail.info Signed-off-by: Janos SUTO --- util/indexer.delta.sh.in | 2 +- util/indexer.main.sh.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/util/indexer.delta.sh.in b/util/indexer.delta.sh.in index 6da9e8bf..6b05fe6f 100755 --- a/util/indexer.delta.sh.in +++ b/util/indexer.delta.sh.in @@ -4,7 +4,7 @@ export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin MAINTMPFILE=/var/run/piler/main.indexer.tmp DELTATMPFILE=/var/run/piler/delta.indexer.tmp INDEXER="indexer --config SYSCONFDIR/piler/sphinx.conf" -PRIORITY=mail.error +PRIORITY=mail.info TOUCHFILE=/var/piler/stat/indexer if [ -f $MAINTMPFILE ]; then echo "INDEXER ERROR: indexer merging to main index is already running. It started at "`cat $MAINTMPFILE` | logger -p $PRIORITY ; exit 1; fi diff --git a/util/indexer.main.sh.in b/util/indexer.main.sh.in index 28d7756e..7332ed48 100755 --- a/util/indexer.main.sh.in +++ b/util/indexer.main.sh.in @@ -6,7 +6,7 @@ set -o pipefail MAINTMPFILE="/var/run/piler/main.indexer.tmp" SPHINX_CONFIG="SYSCONFDIR/piler/sphinx.conf" -PRIORITY="mail.error" +PRIORITY="mail.info" TOUCHFILE="/var/piler/stat/indexer" MAIN_INDEX="main1" From 2c530d52687783a87f307c4377636757d19cbae1 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 16 Jan 2021 15:27:37 +0100 Subject: [PATCH 036/106] Fixed uid collision issue Signed-off-by: Janos SUTO --- webui/model/user/auth.php | 2 +- webui/model/user/user.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/webui/model/user/auth.php b/webui/model/user/auth.php index d9cc383f..9e73f995 100644 --- a/webui/model/user/auth.php +++ b/webui/model/user/auth.php @@ -349,7 +349,7 @@ class ModelUserAuth extends Model { $uid = $this->model_user_user->get_uid_by_email($email); if($uid < 1) { - $uid = $this->model_user_user->get_next_uid(TABLE_EMAIL); + $uid = $this->model_user_user->get_next_uid(); $query = $this->db->query("INSERT INTO " . TABLE_EMAIL . " (uid, email) VALUES(?,?)", array($uid, $email)); } diff --git a/webui/model/user/user.php b/webui/model/user/user.php index fe111f4c..07176efa 100644 --- a/webui/model/user/user.php +++ b/webui/model/user/user.php @@ -306,9 +306,9 @@ class ModelUserUser extends Model { } - public function get_next_uid($table = TABLE_USER) { + public function get_next_uid() { - $query = $this->db->query("SELECT MAX(uid) AS last_id FROM " . $table); + $query = $this->db->query("SELECT MAX(uid) AS last_id FROM " . TABLE_EMAIL); if(isset($query->row['last_id']) && $query->row['last_id'] > 0) { return (int)$query->row['last_id'] + 1; From a07e909c78c4770b27de2e687732fe63225445d0 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 21 Jan 2021 20:41:44 +0100 Subject: [PATCH 037/106] Updated LICENSE file Signed-off-by: Janos SUTO --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 0445531d..63f40739 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ piler, an enterprise level email archiving application -Copyright (C) 2012-2016, Janos SUTO +Copyright (C) 2012-2021, Janos SUTO This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by From c1b717482b1141ec676e39be8bbe44ea3c69241a Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Wed, 27 Jan 2021 09:03:29 +0100 Subject: [PATCH 038/106] Fixed imap server hostname Signed-off-by: Janos SUTO --- tests/setup.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/setup.inc b/tests/setup.inc index 1a4c03f6..7ea3b5da 100644 --- a/tests/setup.inc +++ b/tests/setup.inc @@ -39,7 +39,7 @@ create_rules() { echo 'echo "insert into archiving_rule (\`from\`) values (\"@gmail.com\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh echo 'echo "insert into archiving_rule (\`from\`,attachment_type, _attachment_size, attachment_size) values (\"finderis.co.ua\", \"image\", \">\", 100000)"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh echo 'echo "insert into archiving_rule (\`to\`) values (\"undisclosed-recipients\")"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh - echo 'echo "insert into import (\`type\`, username, password, server) values (\"imap\", \"sanyi@aaa.fu\", \"abcde123\", \"imap\")"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh + echo 'echo "insert into import (\`type\`, username, password, server) values (\"imap\", \"sanyi@aaa.fu\", \"abcde123\", \"imap.aaa.fu\")"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh echo 'echo "update user set password=\"\$6\$GKL00T\$8jqoFOe3PyAbOCLwKB7JwndwC.IinHrZRkdoQDZUc8vybZ88sA2qomlz5JceNif8fFpkGzZ03ilvQa7tqQx0v1\""| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh From 59572444dd5cbceba770c03ab599963becb3625f Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Wed, 27 Jan 2021 09:05:19 +0100 Subject: [PATCH 039/106] imapfetch.py fix Signed-off-by: Janos SUTO --- util/imapfetch.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/util/imapfetch.py b/util/imapfetch.py index 8e8b82b0..45852085 100755 --- a/util/imapfetch.py +++ b/util/imapfetch.py @@ -44,17 +44,20 @@ def read_folder_list(conn): if isinstance(folder, type(b'')): folder = folder.decode('utf-8') elif isinstance(folder, type(())): - folder = re.sub(r'\{\d+\}$', '', folder[0]) + folder[1] + folder = re.sub(r'\{\d+\}$', '', folder[0].decode('utf-8')) + folder[1].decode('utf-8') # The regex should match ' "/" ' and ' "." ' if folder: - f = re.split(r' \"[\/\.]\" ', folder) + f = re.split(r' \"[\/\.\\]+\" ', folder) result.append(f[1]) return [x for x in result if x not in opts['skip_folders']] def process_folder(conn, folder): + # Space in the folder name must be escaped + folder = re.sub(r' ', '\\ ', folder) + if opts['verbose']: print("Processing {}".format(folder)) @@ -75,9 +78,10 @@ def process_folder(conn, folder): rc, data = conn.fetch(num, '(RFC822)') if opts['verbose']: print(rc, num) - opts['counter'] += 1 - with open("{}.eml".format(opts['counter']), "wb") as f: - f.write(data[0][1]) + if isinstance(data[0], tuple): + opts['counter'] += 1 + with open("{}.eml".format(opts['counter']), "wb") as f: + f.write(data[0][1]) def main(): From 3a6d99e7bd97d6cb1df2be30e09853a3f4b56000 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 2 Feb 2021 17:52:58 +0100 Subject: [PATCH 040/106] Fixed dockerfile Signed-off-by: Janos SUTO --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 217161be..7292e731 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,7 +19,7 @@ RUN apt-get update && \ apt-get -y --no-install-recommends install \ wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \ php7.4-gd php7.4-curl php7.4-xml php7.4-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip5 \ - libtre5 cron libmariadb-dev mariadb-client-core-10.3 python3 python3-mysqldb && \ + libtre5 cron libmariadb-dev mariadb-client-core-10.3 python3 python3-mysqldb ca-certificates && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ wget --no-check-certificate -q -O ${SPHINX_BIN_TARGZ} ${DOWNLOAD_URL}/generic-local/${SPHINX_BIN_TARGZ} && \ From 8afbd8f56d5a12e2a49616ace04e4cd35ba83a8e Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 4 Feb 2021 12:34:32 +0100 Subject: [PATCH 041/106] Let the gui return you last 20 messages automatically when you login Signed-off-by: Janos SUTO --- webui/view/javascript/piler-in.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/webui/view/javascript/piler-in.js b/webui/view/javascript/piler-in.js index b5f1d29c..2dc4d39f 100644 --- a/webui/view/javascript/piler-in.js +++ b/webui/view/javascript/piler-in.js @@ -1001,6 +1001,8 @@ var Piler = { Piler.log("[add_shortcuts]"); + $("#button_search").click(); + $(document).keypress(function(e){ if(e.which == 13){ $("#button_search").click(); From 5aa8d62701a01d8d8457667e32553a1bc55160f3 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 5 Feb 2021 17:36:51 +0100 Subject: [PATCH 042/106] Fixed docker setup not using mysql root account Signed-off-by: Janos SUTO --- docker/docker-compose.yaml | 10 +++++++--- docker/start.sh | 32 +++++++++++++++----------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 7a42aaa8..118b803e 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -11,7 +11,10 @@ services: - setuid - setgid environment: - - MYSQL_ROOT_PASSWORD=abcde123 + - MYSQL_DATABASE=piler + - MYSQL_USER=piler + - MYSQL_PASSWORD=piler123 + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci volumes: - db_data:/var/lib/mysql @@ -26,8 +29,9 @@ services: image: sutoj/piler:1.3.10 init: true environment: - - MYSQL_PILER_PASSWORD=piler123 - - MYSQL_ROOT_PASSWORD=abcde123 + - MYSQL_DATABASE=piler + - MYSQL_USER=piler + - MYSQL_PASSWORD=piler123 - MYSQL_HOSTNAME=mysql - PILER_HOSTNAME=archive.yourdomain.com - MEMCACHED_HOST=memcached diff --git a/docker/start.sh b/docker/start.sh index dc27e073..b67d6a92 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -13,7 +13,6 @@ PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf" SPHINX_CONF="${CONFIG_DIR}/sphinx.conf" CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php" PILER_MY_CNF="${CONFIG_DIR}/.my.cnf" -ROOT_MY_CNF="/root/.my.cnf" error() { @@ -30,8 +29,10 @@ log() { pre_flight_check() { [[ -v PILER_HOSTNAME ]] || error "Missing PILER_HOSTNAME env variable" [[ -v MYSQL_HOSTNAME ]] || error "Missing MYSQL_HOSTNAME env variable" - [[ -v MYSQL_PILER_PASSWORD ]] || error "Missing MYSQL_PILER_PASSWORD env variable" - [[ -v MYSQL_ROOT_PASSWORD ]] || error "Missing MYSQL_ROOT_PASSWORD env variable" + [[ -v MYSQL_DATABASE ]] || error "Missing MYSQL_DATABASE env variable" + [[ -v MYSQL_USER ]] || error "Missing MYSQL_USER env variable" + [[ -v MYSQL_PASSWORD ]] || error "Missing MYSQL_PASSWORD env variable" + [[ -v MYSQL_DATABASE ]] || error "Missing MYSQL_DATABASE env variable" } @@ -87,7 +88,9 @@ fix_configs() { log "Writing ${PILER_CONF}" sed \ - -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" \ + -e "s/mysqluser=.*/${MYSQL_USER}/g" \ + -e "s/mysqldb=.*/${MYSQL_DATABASE}/g" \ + -e "s/verystrongpassword/${MYSQL_PASSWORD}/g" \ -e "s/hostid=.*/hostid=${PILER_HOSTNAME}/g" \ -e "s/tls_enable=.*/tls_enable=1/g" \ -e "s/mysqlsocket=.*/mysqlsocket=/g" "${PILER_CONF}.dist" > "$PILER_CONF" @@ -111,7 +114,9 @@ fix_configs() { echo "\$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/bin/pileraget';" echo "\$config['PILER_BINARY'] = '/usr/sbin/piler';" echo "\$config['DB_HOSTNAME'] = '$MYSQL_HOSTNAME';" - echo "\$config['DB_PASSWORD'] = '$MYSQL_PILER_PASSWORD';" + echo "\$config['DB_DATABASE'] = '$MYSQL_DATABASE';" + echo "\$config['DB_USERNAME'] = '$MYSQL_USER';" + echo "\$config['DB_PASSWORD'] = '$MYSQL_PASSWORD';" echo "\$config['ENABLE_MEMCACHED'] = 1;" echo "\$memcached_server = ['memcached', 11211];" } >> "$CONFIG_SITE_PHP" @@ -119,14 +124,14 @@ fix_configs() { sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \ -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \ - -e "s%MYSQL_USERNAME%${PILER_USER}%" \ - -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%" \ + -e "s%MYSQL_USERNAME%${MYSQL_USER}%" \ + -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" \ -i "$SPHINX_CONF" } wait_until_mysql_server_is_ready() { - while true; do if mysql "--defaults-file=${ROOT_MY_CNF}" <<< "show databases"; then break; fi; log "${MYSQL_HOSTNAME} is not ready"; sleep 5; done + while true; do if mysql "--defaults-file=${PILER_MY_CNF}" <<< "show databases"; then break; fi; log "${MYSQL_HOSTNAME} is not ready"; sleep 5; done log "${MYSQL_HOSTNAME} is ready" } @@ -140,15 +145,9 @@ init_database() { while read -r db; do if [[ "$db" == "$MYSQL_DATABASE" ]]; then has_piler_db=1; fi - done < <(mysql "--defaults-file=${ROOT_MY_CNF}" <<< 'show databases') + done < <(mysql "--defaults-file=${PILER_MY_CNF}" <<< 'show databases') if [[ $has_piler_db -eq 0 ]]; then - log "no ${MYSQL_DATABASE} database, creating" - - mysql "--defaults-file=${ROOT_MY_CNF}" <<< "create database ${MYSQL_DATABASE} character set utf8mb4" - mysql "--defaults-file=${ROOT_MY_CNF}" <<< "grant all privileges on ${MYSQL_DATABASE}.* to ${PILER_USER} identified by '${MYSQL_PILER_PASSWORD}'" - mysql "--defaults-file=${ROOT_MY_CNF}" <<< "flush privileges" - mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" < /usr/share/piler/db-mysql.sql else log "${MYSQL_DATABASE} db exists" @@ -162,9 +161,8 @@ init_database() { create_my_cnf_files() { printf "[client]\nhost = %s\nuser = %s\npassword = %s\n[mysqldump]\nhost = %s\nuser = %s\npassword = %s\n" \ - "$MYSQL_HOSTNAME" "$PILER_USER" "$MYSQL_PILER_PASSWORD" "$MYSQL_HOSTNAME" "$PILER_USER" "$MYSQL_PILER_PASSWORD" \ + "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" \ > "$PILER_MY_CNF" - printf "[client]\nhost = %s\nuser = root\npassword = %s\n" "$MYSQL_HOSTNAME" "$MYSQL_ROOT_PASSWORD" > "$ROOT_MY_CNF" give_it_to_piler "$PILER_MY_CNF" } From fd2bcf4845061a259fbc483e56effd4cac711148 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 7 Feb 2021 15:15:21 +0100 Subject: [PATCH 043/106] Renamed message_highlight css field to mssghglght because some it interfered with some words Signed-off-by: Janos SUTO --- webui/model/search/message.php | 4 ++-- webui/view/theme/default/assets/css/metro-bootstrap.css | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/webui/model/search/message.php b/webui/model/search/message.php index 670f2245..5e67bffa 100644 --- a/webui/model/search/message.php +++ b/webui/model/search/message.php @@ -224,7 +224,7 @@ class ModelSearchMessage extends Model { if($html == 0) { while(list($k, $v) = each($terms)) { - $s = preg_replace("/$v/i", "$v", $s); + $s = preg_replace("/$v/i", "$v", $s); } return $s; @@ -246,7 +246,7 @@ class ModelSearchMessage extends Model { reset($terms); while(list($k, $v) = each($terms)) { - $str = preg_replace("/$v/i", "$v", $str); + $str = preg_replace("/$v/i", "$v", $str); } $s .= $str; diff --git a/webui/view/theme/default/assets/css/metro-bootstrap.css b/webui/view/theme/default/assets/css/metro-bootstrap.css index 31d60204..eb86327c 100644 --- a/webui/view/theme/default/assets/css/metro-bootstrap.css +++ b/webui/view/theme/default/assets/css/metro-bootstrap.css @@ -1660,7 +1660,7 @@ html,body{height:auto !important;height:100%;min-height:100%;} #notesbox{position:absolute;top:8px;right:8px;} #folderbox{position:absolute;top:8px;right:380px;} #sspinner{display:none;} -.message_highlight{background:lightblue;} +.mssghglght{background:lightblue;} #searchcontainer{text-align:center;min-width:320px;} #searchcontainer label{display:none;height:0;} #searchcontainer input{height:26px;width:100%;} From ce811951e5e9a4aab053d4d7a420f11cbe7f9d00 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 7 Feb 2021 16:48:03 +0100 Subject: [PATCH 044/106] Fixed docker-compose.yaml Signed-off-by: Janos SUTO --- docker/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 118b803e..889beaa3 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -14,6 +14,7 @@ services: - MYSQL_DATABASE=piler - MYSQL_USER=piler - MYSQL_PASSWORD=piler123 + - MYSQL_RANDOM_ROOT_PASSWORD=yes command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci volumes: - db_data:/var/lib/mysql From 657ceaf11f50065c0e5584cf08da0f5bec9c3963 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 7 Feb 2021 20:33:47 +0100 Subject: [PATCH 045/106] Fixed docker start.sh Signed-off-by: Janos SUTO --- docker/start.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docker/start.sh b/docker/start.sh index b67d6a92..c5d2393c 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -88,8 +88,8 @@ fix_configs() { log "Writing ${PILER_CONF}" sed \ - -e "s/mysqluser=.*/${MYSQL_USER}/g" \ - -e "s/mysqldb=.*/${MYSQL_DATABASE}/g" \ + -e "s/mysqluser=.*/mysqluser=${MYSQL_USER}/g" \ + -e "s/mysqldb=.*/mysqldb=${MYSQL_DATABASE}/g" \ -e "s/verystrongpassword/${MYSQL_PASSWORD}/g" \ -e "s/hostid=.*/hostid=${PILER_HOSTNAME}/g" \ -e "s/tls_enable=.*/tls_enable=1/g" \ @@ -138,19 +138,21 @@ wait_until_mysql_server_is_ready() { init_database() { - local db - local has_piler_db=0 + local table + local has_metadata_table=0 wait_until_mysql_server_is_ready - while read -r db; do - if [[ "$db" == "$MYSQL_DATABASE" ]]; then has_piler_db=1; fi - done < <(mysql "--defaults-file=${PILER_MY_CNF}" <<< 'show databases') + while read -r table; do + if [[ "$table" == metadata ]]; then has_metadata_table=1; fi + done < <(mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< 'show tables') + + if [[ $has_metadata_table -eq 0 ]]; then + log "no metadata table, creating tables" - if [[ $has_piler_db -eq 0 ]]; then mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" < /usr/share/piler/db-mysql.sql else - log "${MYSQL_DATABASE} db exists" + log "metadata table exists" fi if [[ -v ADMIN_USER_PASSWORD_HASH ]]; then From a1d81a0281ea986c690c459cba50ee7b3deef50f Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 8 Feb 2021 20:35:24 +0100 Subject: [PATCH 046/106] Updated docker-compose.yaml to use mariadb:10.5 Signed-off-by: Janos SUTO --- docker/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 889beaa3..c888c56a 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -2,7 +2,7 @@ version: "3" services: mysql: - image: mariadb:10.4 + image: mariadb:10.5 restart: unless-stopped cap_drop: - ALL From 5d70881463f180ecf4946fe015519b86bfda139e Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 8 Feb 2021 20:36:11 +0100 Subject: [PATCH 047/106] Added SMTP HELP command to piler-smtp Signed-off-by: Janos SUTO --- src/smtp.c | 5 +++++ src/smtpcodes.h | 2 ++ 2 files changed, 7 insertions(+) diff --git a/src/smtp.c b/src/smtp.c index bd40af02..479e2ec3 100644 --- a/src/smtp.c +++ b/src/smtp.c @@ -31,6 +31,11 @@ void process_smtp_command(struct smtp_session *session, char *buf, struct config return; } + if(strncasecmp(buf, SMTP_CMD_HELP, strlen(SMTP_CMD_HELP)) == 0){ + send_smtp_response(session, SMTP_RESP_221_PILER_SMTP_OK); + return; + } + if(strncasecmp(buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){ process_command_mail_from(session, buf); return; diff --git a/src/smtpcodes.h b/src/smtpcodes.h index 2a9943d5..0cdcec84 100644 --- a/src/smtpcodes.h +++ b/src/smtpcodes.h @@ -18,6 +18,7 @@ #define SMTP_CMD_HELO "HELO" #define SMTP_CMD_EHLO "EHLO" +#define SMTP_CMD_HELP "HELP" #define SMTP_CMD_MAIL_FROM "MAIL FROM:" #define SMTP_CMD_RCPT_TO "RCPT TO:" #define SMTP_CMD_DATA "DATA" @@ -32,6 +33,7 @@ // SMTP responses +#define SMTP_RESP_221_PILER_SMTP_OK "221 piler-smtp is OK\r\n" #define SMTP_RESP_220_BANNER "220 %s ESMTP\r\n" #define SMTP_RESP_220_READY_TO_START_TLS "220 Ready to start TLS\r\n" #define SMTP_RESP_221_GOODBYE "221 %s Goodbye\r\n" From 0b980f59cec08d38e2a4d57f36dffecd13658eff Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 8 Feb 2021 21:12:40 +0100 Subject: [PATCH 048/106] Added curl to docker image Signed-off-by: Janos SUTO --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7292e731..c6ea12f6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -19,7 +19,7 @@ RUN apt-get update && \ apt-get -y --no-install-recommends install \ wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \ php7.4-gd php7.4-curl php7.4-xml php7.4-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip5 \ - libtre5 cron libmariadb-dev mariadb-client-core-10.3 python3 python3-mysqldb ca-certificates && \ + libtre5 cron libmariadb-dev mariadb-client-core-10.3 python3 python3-mysqldb ca-certificates curl && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ wget --no-check-certificate -q -O ${SPHINX_BIN_TARGZ} ${DOWNLOAD_URL}/generic-local/${SPHINX_BIN_TARGZ} && \ From aa07811d52aba8b9ee295a70061c4e3b96097370 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 8 Feb 2021 21:13:47 +0100 Subject: [PATCH 049/106] Fixed test case Signed-off-by: Janos SUTO --- tests/cases/05-smtp.inc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cases/05-smtp.inc b/tests/cases/05-smtp.inc index 88914e13..3be872bd 100644 --- a/tests/cases/05-smtp.inc +++ b/tests/cases/05-smtp.inc @@ -17,16 +17,16 @@ case1() { "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus" --socket --no-counter - wait_until_emails_are_processed "piler1" 3001 + wait_until_emails_are_processed "piler1" 3002 docker exec "piler1" su piler -c /usr/libexec/piler/indexer.delta.sh 2>/dev/null - count_status_values 3001 2892 109 0 + count_status_values 3002 2893 109 0 test_retrieved_messages_are_the_same "piler1" "piler" run_05_sphinx_tests - docker exec "piler1" su piler -c 'php /usr/libexec/piler/generate_stats.php --webui /var/piler/www --start=2015/01/01 --stop=2020/12/31' + docker exec "piler1" su piler -c 'php /usr/libexec/piler/generate_stats.php --webui /var/piler/www --start=2015/01/01 --stop=2021/12/31' docker exec "piler1" su piler -c 'php /usr/libexec/piler/sign.php --webui /var/piler/www --mode time' } From 9dbfbb74af8224fa9a8c04aff59e7f7444a5aabe Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 9 Feb 2021 07:47:31 +0100 Subject: [PATCH 050/106] Added healthcheck to mysql container Signed-off-by: Janos SUTO --- docker/docker-compose.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index c888c56a..88543b12 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -16,6 +16,12 @@ services: - MYSQL_PASSWORD=piler123 - MYSQL_RANDOM_ROOT_PASSWORD=yes command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + healthcheck: + test: mysql --user=piler --password=piler123 piler --execute "show tables" + interval: "60s" + timeout: "5s" + start_period: "15s" + retries: 3 volumes: - db_data:/var/lib/mysql From 0f2657275c79014b08bfa91dcc6e2abc2bfcaf60 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 9 Feb 2021 08:55:47 +0100 Subject: [PATCH 051/106] Fixed tear_down_session() Signed-off-by: Janos SUTO --- src/session.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/session.c b/src/session.c index a4d75090..b527323c 100644 --- a/src/session.c +++ b/src/session.c @@ -134,7 +134,12 @@ void free_smtp_session(struct smtp_session *session){ void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections){ - syslog(LOG_PRIORITY, "disconnected from %s on fd=%d (%d active connections)", sessions[slot]->remote_host, sessions[slot]->net.socket, (*num_connections)-1); + if(sessions[slot] == NULL){ + syslog(LOG_PRIORITY, "session already torn down, slot=%d (%d active connections)", slot, *num_connections); + return; + } + + syslog(LOG_PRIORITY, "disconnected from %s on fd=%d, slot=%d (%d active connections)", sessions[slot]->remote_host, sessions[slot]->net.socket, slot, (*num_connections)-1); close(sessions[slot]->net.socket); @@ -148,7 +153,7 @@ void tear_down_session(struct smtp_session **sessions, int slot, int *num_connec free_smtp_session(sessions[slot]); sessions[slot] = NULL; - (*num_connections)--; + if(*num_connections > 0) (*num_connections)--; } From ed176bf815342ae74bac8e1d86076936975c63e6 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 14 Feb 2021 16:00:26 +0100 Subject: [PATCH 052/106] Fixed counters for test case Signed-off-by: Janos SUTO --- tests/cases/05-smtp.inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cases/05-smtp.inc b/tests/cases/05-smtp.inc index 3be872bd..ada5b38a 100644 --- a/tests/cases/05-smtp.inc +++ b/tests/cases/05-smtp.inc @@ -17,10 +17,10 @@ case1() { "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus" --socket --no-counter - wait_until_emails_are_processed "piler1" 3002 + wait_until_emails_are_processed "piler1" 3005 docker exec "piler1" su piler -c /usr/libexec/piler/indexer.delta.sh 2>/dev/null - count_status_values 3002 2893 109 0 + count_status_values 3005 2894 111 0 test_retrieved_messages_are_the_same "piler1" "piler" From 2c05ccc5172081ea388bf43702cf402548339f2f Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 14 Feb 2021 16:04:26 +0100 Subject: [PATCH 053/106] Added spam2 folder to sent emails Signed-off-by: Janos SUTO --- tests/cases/05-smtp.inc | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/cases/05-smtp.inc b/tests/cases/05-smtp.inc index ada5b38a..2e4fcc03 100644 --- a/tests/cases/05-smtp.inc +++ b/tests/cases/05-smtp.inc @@ -12,6 +12,7 @@ case1() { "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet2" --socket --no-counter "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam0" --socket --no-counter "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam1" --socket --no-counter + "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam2" --socket --no-counter "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/journal" --socket --no-counter "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/deduptest" --socket --no-counter "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus" --socket --no-counter From e136bbbcb6cac3eecbe6d5b60ccc8519e1053d95 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 14 Feb 2021 16:49:02 +0100 Subject: [PATCH 054/106] Refactored ldap parameters Signed-off-by: Janos SUTO --- RELEASE_NOTES | 6 ++++++ config.php.in | 1 - webui/model/user/auth.php | 6 +++--- webui/system/database/ldap.php | 4 ++-- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index a5108693..c23d2b17 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,3 +1,9 @@ +1.3.11: +------- + +- Obsoleted the LDAP port parameter. Specify the ldap host in the form + of protocol://hostname:port, eg. ldaps://ldap.yourdomain.com:636 + 1.3.10: ------- diff --git a/config.php.in b/config.php.in index 51ca77b3..5c6fd08a 100644 --- a/config.php.in +++ b/config.php.in @@ -82,7 +82,6 @@ $config['WKHTMLTOPDF_COMMAND'] = "wkhtmltopdf"; $config['ENABLE_LDAP_AUTH'] = 0; $config['LDAP_HOST'] = 'ldap.yourdomain.com'; -$config['LDAP_PORT'] = 389; $config['LDAP_HELPER_DN'] = 'cn=....'; $config['LDAP_HELPER_PASSWORD'] = 'xxxxxxx'; $config['LDAP_MAIL_ATTR'] = 'mail'; diff --git a/webui/model/user/auth.php b/webui/model/user/auth.php index 9e73f995..b5728051 100644 --- a/webui/model/user/auth.php +++ b/webui/model/user/auth.php @@ -212,7 +212,7 @@ class ModelUserAuth extends Model { if($ldap_host == '' || $ldap_helper_password == '') { return 0; } - $ldap = new LDAP($ldap_host, $ldap_port, $ldap_helper_dn, $ldap_helper_password); + $ldap = new LDAP($ldap_host, $ldap_helper_dn, $ldap_helper_password); if($ldap->is_bind_ok()) { @@ -221,7 +221,7 @@ class ModelUserAuth extends Model { if(isset($query->row['dn']) && $query->row['dn']) { $a = $query->row; - $ldap_auth = new LDAP($ldap_host, $ldap_port, $a['dn'], $password); + $ldap_auth = new LDAP($ldap_host, $a['dn'], $password); if(LOG_LEVEL >= NORMAL) { syslog(LOG_INFO, "ldap auth against '" . $ldap_host . "', dn: '" . $a['dn'] . "', result: " . $ldap_auth->is_bind_ok()); } @@ -478,7 +478,7 @@ class ModelUserAuth extends Model { if(LOG_LEVEL >= NORMAL) { syslog(LOG_INFO, "sso login: $sso_user"); } - $ldap = new LDAP(LDAP_HOST, LDAP_PORT, LDAP_HELPER_DN, LDAP_HELPER_PASSWORD); + $ldap = new LDAP(LDAP_HOST, LDAP_HELPER_DN, LDAP_HELPER_PASSWORD); if($ldap->is_bind_ok()) { diff --git a/webui/system/database/ldap.php b/webui/system/database/ldap.php index 83ebf50f..49f30ff6 100644 --- a/webui/system/database/ldap.php +++ b/webui/system/database/ldap.php @@ -6,9 +6,9 @@ class LDAP { private $link; private $bind; - public function __construct($ldaphost, $ldapport, $binddn, $bindpw) { + public function __construct($ldaphost, $binddn, $bindpw) { - $this->link = ldap_connect($ldaphost, $ldapport) or exit('Error: ldap_connect()'); + $this->link = ldap_connect($ldaphost) or exit('Error: ldap_connect()'); ldap_set_option($this->link, LDAP_OPT_PROTOCOL_VERSION, 3); ldap_set_option($this->link, LDAP_OPT_REFERRALS, 0); From 890a00f46f832fb359d1939095515f66caf3bdd5 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 15 Feb 2021 07:07:03 +0100 Subject: [PATCH 055/106] Removed dead code: get_message_recipients() Signed-off-by: Janos SUTO --- webui/model/search/search.php | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/webui/model/search/search.php b/webui/model/search/search.php index cc6a5db9..165756ef 100644 --- a/webui/model/search/search.php +++ b/webui/model/search/search.php @@ -586,35 +586,6 @@ class ModelSearchSearch extends Model { } - public function get_message_recipients($id = '') { - $rcpt = array(); - $domains = array(); - - if(Registry::get('auditor_user') == 0) { return $rcpt; } - - $query = $this->db->query("SELECT `domain` FROM " . TABLE_DOMAIN); - foreach($query->rows as $q) { - array_push($domains, $q['domain']); - } - - $query = $this->db->query("SELECT `to` FROM " . VIEW_MESSAGES . " WHERE id=?", array($id)); - - foreach($query->rows as $q) { - $mydomain = 0; - - foreach ($domains as $domain) { - if(preg_match("/\@$domain$/", $q['to'])) { $mydomain = 1; break; } - } - - if($mydomain == 1) { - array_push($rcpt, $q['to']); - } - } - - return $rcpt; - } - - public function get_message_addresses_in_my_domain($id = '') { $addr = array(); $domains = array(); From 6716bec68cb4126786659453548c25bb6cf09abc Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 15 Feb 2021 10:29:44 +0100 Subject: [PATCH 056/106] Improved the ldap group email detection and display Signed-off-by: Janos SUTO --- webui/model/search/search.php | 32 ++++++++++++---- webui/model/user/auth.php | 37 +++++++++++++++++-- .../theme/default/templates/search/helper.tpl | 2 +- 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/webui/model/search/search.php b/webui/model/search/search.php index 165756ef..dbcf5b5b 100644 --- a/webui/model/search/search.php +++ b/webui/model/search/search.php @@ -489,12 +489,10 @@ class ModelSearchSearch extends Model { if(isset($query->rows)) { foreach($query->rows as $r) { - if(!isset($rcpt[$r['id']]) && !in_array($r['to'], $SUPPRESS_RECIPIENTS)) { - $srcpt[$r['id']] = $r['to']; - $rcpt[$r['id']] = $r['to']; - } - else { - if(Registry::get('auditor_user') == 1) { $rcpt[$r['id']] .= ",\n" . $r['to']; } + if(!isset($rcpt[$r['id']])) { $rcpt[$r['id']] = []; } + + if(Registry::get('auditor_user') == 1 || !in_array($r['to'], $SUPPRESS_RECIPIENTS)) { + array_push($rcpt[$r['id']], $r['to']); } } } @@ -542,8 +540,8 @@ class ModelSearchSearch extends Model { $m['shortfrom'] = make_short_string($m['from'], MAX_CGI_FROM_SUBJ_LEN); $m['from'] = escape_gt_lt_quote_symbols($m['from']); - isset($srcpt[$m['id']]) ? $m['shortto'] = $srcpt[$m['id']] : $m['shortto'] = ''; isset($rcpt[$m['id']]) ? $m['to'] = $rcpt[$m['id']] : $m['to'] = ''; + $m['shortto'] = make_short_string($this->get_preferred_recipient($rcpt[$m['id']]), MAX_CGI_FROM_SUBJ_LEN); $m['to'] = escape_gt_lt_quote_symbols($m['to']); @@ -586,6 +584,26 @@ class ModelSearchSearch extends Model { } + private function get_preferred_recipient($arr = []) { + $result = ''; + + $session = Registry::get('session'); + $group_emails = $session->get('group_emails'); + $user_emails = $session->get('user_emails'); + + if(count($arr) < 2 || (!$group_emails && !$user_emails) ) { return $arr[0]; } + + foreach ($arr as $a) { + if($result == '' && in_array($a, $group_emails)) { $result = $a; } + if(in_array($a, $user_emails)) { $result = $a; } + } + + if($result == '') { $result = $arr[0]; } + + return $result; + } + + public function get_message_addresses_in_my_domain($id = '') { $addr = array(); $domains = array(); diff --git a/webui/model/user/auth.php b/webui/model/user/auth.php index b5728051..475634f8 100644 --- a/webui/model/user/auth.php +++ b/webui/model/user/auth.php @@ -234,7 +234,7 @@ class ModelUserAuth extends Model { if($this->check_ldap_membership($ldap_auditor_member_dn, $query->rows) == 1) { $role = 2; } if($this->check_ldap_membership($ldap_admin_member_dn, $query->rows) == 1) { $role = 1; } - $emails = $this->get_email_array_from_ldap_attr($query->rows); + $emails = $this->get_email_array_from_ldap_attr($query->rows, $ldap_distributionlist_objectclass); $extra_emails = $this->model_user_user->get_email_addresses_from_groups($emails); $emails = array_merge($emails, $extra_emails); @@ -292,11 +292,19 @@ class ModelUserAuth extends Model { } - public function get_email_array_from_ldap_attr($e = array()) { + public function get_email_array_from_ldap_attr($e = array(), $group_object_class) { global $mailattrs; $data = []; + $group_emails = []; + $user_emails = []; foreach($e as $a) { + $group_object = 0; + + if($group_object_class && in_array($group_object_class, $a['objectclass'])) { + $group_object = 1; + } + if(LOG_LEVEL >= DEBUG) { syslog(LOG_INFO, "checking ldap entry dn: " . $a['dn'] . ", cn: " . $a['cn']); } foreach ($mailattrs as $mailattr) { @@ -316,7 +324,15 @@ class ModelUserAuth extends Model { } $email = preg_replace("/^([\w]+)\:/i", "", $a[$mailattr][$i]); - if(validemail($email) && !in_array($email, $data)) { array_push($data, $email); } + if(validemail($email)) { + if(!in_array($email, $data)) { array_push($data, $email); } + + if($group_object) { + if(!in_array($email, $group_emails)) { array_push($group_emails, $email); } + } else { + if(!in_array($email, $user_emails)) { array_push($user_emails, $email); } + } + } } } } @@ -324,12 +340,25 @@ class ModelUserAuth extends Model { if(LOG_LEVEL >= DEBUG) { syslog(LOG_INFO, "checking entry #2: " . $a[$mailattr]); } $email = strtolower(preg_replace("/^([\w]+)\:/i", "", $a[$mailattr])); - if(validemail($email) && !in_array($email, $data)) { array_push($data, $email); } + if(validemail($email)) { + if(!in_array($email, $data)) { array_push($data, $email); } + + if($group_object) { + if(!in_array($email, $group_emails)) { array_push($group_emails, $email); } + } else { + if(!in_array($email, $user_emails)) { array_push($user_emails, $email); } + } + } } } } } + $session = Registry::get('session'); + + $session->set("user_emails", $user_emails); + $session->set("group_emails", $group_emails); + return $data; } diff --git a/webui/view/theme/default/templates/search/helper.tpl b/webui/view/theme/default/templates/search/helper.tpl index 92b8a511..0252d44c 100644 --- a/webui/view/theme/default/templates/search/helper.tpl +++ b/webui/view/theme/default/templates/search/helper.tpl @@ -49,7 +49,7 @@ -   + 1) { ?>">  class="xxx"> class="xxx" title="" onclick="$('#ref').val(''); Piler.expert(this);">[+] P From 9856b92c22930151671e3a15615b99b10bc782db Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Wed, 10 Feb 2021 21:10:35 +0100 Subject: [PATCH 057/106] Added debugging to track piler-smtp segfault issue Signed-off-by: Janos SUTO --- src/piler-smtp.c | 6 +++--- src/piler.h | 2 +- src/session.c | 22 +++++++++++++++------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/piler-smtp.c b/src/piler-smtp.c index f52f1264..f44e041a 100644 --- a/src/piler-smtp.c +++ b/src/piler-smtp.c @@ -94,7 +94,7 @@ void check_for_client_timeout(){ for(int i=0; ilasttime >= cfg.smtp_timeout){ syslog(LOG_PRIORITY, "client %s timeout, lasttime: %ld", sessions[i]->remote_host, sessions[i]->lasttime); - tear_down_session(sessions, sessions[i]->slot, &num_connections); + tear_down_session(sessions, sessions[i]->slot, &num_connections, "timeout"); } } } @@ -232,7 +232,7 @@ int main(int argc, char **argv){ if(cfg.verbosity >= _LOG_EXTREME) syslog(LOG_PRIORITY, "ERROR: the remote end hung up without sending QUIT"); session = get_session_by_socket(sessions, cfg.max_connections, events[i].data.fd); if(session) - tear_down_session(sessions, session->slot, &num_connections); + tear_down_session(sessions, session->slot, &num_connections, "hungup"); else close(events[i].data.fd); continue; @@ -333,7 +333,7 @@ int main(int argc, char **argv){ /* Don't wait until the remote client closes the connection after he sent the QUIT command */ if(done || session->protocol_state == SMTP_STATE_FINISHED){ - tear_down_session(sessions, session->slot, &num_connections); + tear_down_session(sessions, session->slot, &num_connections, "done"); } } diff --git a/src/piler.h b/src/piler.h index a38be51e..c33ff60c 100644 --- a/src/piler.h +++ b/src/piler.h @@ -70,7 +70,7 @@ void load_mydomains(struct session_data *sdata, struct data *data, struct config int is_email_address_on_my_domains(char *email, struct data *data); int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg); -void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections); +void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections, char *reason); struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int max_connections, int socket); void write_envelope_addresses(struct smtp_session *session, struct config *cfg); void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg); diff --git a/src/session.c b/src/session.c index b527323c..80fa0a4e 100644 --- a/src/session.c +++ b/src/session.c @@ -118,28 +118,38 @@ void init_smtp_session(struct smtp_session *session, int slot, int sd, char *cli void free_smtp_session(struct smtp_session *session){ - if(session){ + syslog(LOG_PRIORITY, "free_smtp_session()"); if(session->net.use_ssl == 1){ + syslog(LOG_PRIORITY, "SSL_shutdown()"); SSL_shutdown(session->net.ssl); SSL_free(session->net.ssl); + syslog(LOG_PRIORITY, "SSL_free()"); } - if(session->net.ctx) SSL_CTX_free(session->net.ctx); + if(session->net.ctx){ + syslog(LOG_PRIORITY, "SSL_CTX_free"); + SSL_CTX_free(session->net.ctx); + } + syslog(LOG_PRIORITY, "freeing session"); free(session); + syslog(LOG_PRIORITY, "free(session) done"); } } -void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections){ +void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections, char *reason){ if(sessions[slot] == NULL){ - syslog(LOG_PRIORITY, "session already torn down, slot=%d (%d active connections)", slot, *num_connections); + syslog(LOG_PRIORITY, "session already torn down, slot=%d, reason=%s (%d active connections)", slot, reason, *num_connections); return; } - syslog(LOG_PRIORITY, "disconnected from %s on fd=%d, slot=%d (%d active connections)", sessions[slot]->remote_host, sessions[slot]->net.socket, slot, (*num_connections)-1); + if(*num_connections > 0) (*num_connections)--; + + syslog(LOG_PRIORITY, "disconnected from %s on fd=%d, slot=%d, reason=%s (%d active connections)", + sessions[slot]->remote_host, sessions[slot]->net.socket, slot, reason, *num_connections); close(sessions[slot]->net.socket); @@ -152,8 +162,6 @@ void tear_down_session(struct smtp_session **sessions, int slot, int *num_connec free_smtp_session(sessions[slot]); sessions[slot] = NULL; - - if(*num_connections > 0) (*num_connections)--; } From 50a2328f64da2021c007e23e8c5765ba11c083e2 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 15 Feb 2021 21:47:24 +0100 Subject: [PATCH 058/106] Refactored epoll timeout check Signed-off-by: Janos SUTO --- src/piler-smtp.c | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/piler-smtp.c b/src/piler-smtp.c index f44e041a..df8ada84 100644 --- a/src/piler-smtp.c +++ b/src/piler-smtp.c @@ -98,8 +98,6 @@ void check_for_client_timeout(){ } } } - - alarm(cfg.check_for_client_timeout_interval); } @@ -197,7 +195,6 @@ int main(int argc, char **argv){ set_signal_handler(SIGPIPE, SIG_IGN); - set_signal_handler(SIGALRM, check_for_client_timeout); set_signal_handler(SIGHUP, initialise_configuration); // calloc() initialitizes the allocated memory @@ -218,10 +215,8 @@ int main(int argc, char **argv){ if(daemonise == 1 && daemon(1, 0) == -1) fatal(ERR_DAEMON); #endif - alarm(cfg.check_for_client_timeout_interval); - for(;;){ - int n = epoll_wait(efd, events, cfg.max_connections, -1); + int n = epoll_wait(efd, events, cfg.max_connections, 10000); for(i=0; i Date: Wed, 17 Feb 2021 05:42:56 +0100 Subject: [PATCH 059/106] Further improved the session timeout handling Signed-off-by: Janos SUTO --- src/piler-smtp.c | 10 +++++++--- src/session.c | 7 +------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/piler-smtp.c b/src/piler-smtp.c index df8ada84..91481069 100644 --- a/src/piler-smtp.c +++ b/src/piler-smtp.c @@ -40,6 +40,7 @@ struct passwd *pwd; struct smtp_session *session, **sessions=NULL; struct smtp_acl *smtp_acl[MAXHASH]; +time_t prev_timeout_check = 0; void usage(){ printf("\nusage: piler\n\n"); @@ -90,6 +91,8 @@ void check_for_client_timeout(){ if(cfg.verbosity >= LOG_DEBUG) syslog(LOG_PRIORITY, "%s @%ld", __func__, now); + if(now - prev_timeout_check < cfg.smtp_timeout) return; + if(num_connections > 0){ for(int i=0; ilasttime >= cfg.smtp_timeout){ @@ -98,6 +101,8 @@ void check_for_client_timeout(){ } } } + + time(&prev_timeout_check); } @@ -194,6 +199,7 @@ int main(int argc, char **argv){ set_signal_handler(SIGSEGV, p_clean_exit); set_signal_handler(SIGPIPE, SIG_IGN); + set_signal_handler(SIGALRM, SIG_IGN); set_signal_handler(SIGHUP, initialise_configuration); @@ -216,7 +222,7 @@ int main(int argc, char **argv){ #endif for(;;){ - int n = epoll_wait(efd, events, cfg.max_connections, 10000); + int n = epoll_wait(efd, events, cfg.max_connections, 1000); for(i=0; ilasttime)); while(1){ - memset(readbuf, 0, sizeof(readbuf)); - if(session->net.use_ssl == 1) readlen = SSL_read(session->net.ssl, (char*)&readbuf[0], sizeof(readbuf)-1); else diff --git a/src/session.c b/src/session.c index 80fa0a4e..777679a9 100644 --- a/src/session.c +++ b/src/session.c @@ -119,23 +119,17 @@ void init_smtp_session(struct smtp_session *session, int slot, int sd, char *cli void free_smtp_session(struct smtp_session *session){ if(session){ - syslog(LOG_PRIORITY, "free_smtp_session()"); if(session->net.use_ssl == 1){ - syslog(LOG_PRIORITY, "SSL_shutdown()"); SSL_shutdown(session->net.ssl); SSL_free(session->net.ssl); - syslog(LOG_PRIORITY, "SSL_free()"); } if(session->net.ctx){ - syslog(LOG_PRIORITY, "SSL_CTX_free"); SSL_CTX_free(session->net.ctx); } - syslog(LOG_PRIORITY, "freeing session"); free(session); - syslog(LOG_PRIORITY, "free(session) done"); } } @@ -183,6 +177,7 @@ void handle_data(struct smtp_session *session, char *readbuf, int readlen, struc p = ©buf[0]; } else { + readbuf[readlen] = 0; p = readbuf; } From 5c2ceb178b4df0ca4a3ac1b41fb380715af4fb7c Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 23 Feb 2021 07:26:47 +0100 Subject: [PATCH 060/106] Release of 1.3.11 Signed-off-by: Janos SUTO --- RELEASE_NOTES | 1 + VERSION | 2 +- src/config.h | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index c23d2b17..eef68096 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,6 +1,7 @@ 1.3.11: ------- +- [BUGFIX] Refactored the smtp timeout check - Obsoleted the LDAP port parameter. Specify the ldap host in the form of protocol://hostname:port, eg. ldaps://ldap.yourdomain.com:636 diff --git a/VERSION b/VERSION index 0c00f610..17e63e7a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.10 +1.3.11 diff --git a/src/config.h b/src/config.h index 5810c5b0..9856a554 100644 --- a/src/config.h +++ b/src/config.h @@ -9,7 +9,7 @@ #include "piler-config.h" #include "params.h" -#define BUILD 998 +#define BUILD 1001 #define HOSTID "mailarchiver" From ba6c9e56aa15161d944638acc3a7e7d6900b2776 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 23 Feb 2021 11:51:38 +0100 Subject: [PATCH 061/106] Updated link in SECURITY.md Signed-off-by: Janos SUTO --- SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index d926b38f..f185d9de 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -11,7 +11,7 @@ Security update policy If a security vulnerability has found, the details, possible mitigations, workarounds, etc. will be shared on the piler mailing list (piler-user@list.acts.hu) -and on the wiki: http://www.mailpiler.org/ +and on the wiki: https://www.mailpiler.org/ Security configurations From 5886e37f990b107227fa46c7cbffef796bf67944 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 23 Feb 2021 11:52:11 +0100 Subject: [PATCH 062/106] Updated docker image tag Signed-off-by: Janos SUTO --- docker/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build.sh b/docker/build.sh index 796063ba..0aa3b220 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -4,7 +4,7 @@ set -o errexit set -o pipefail set -o nounset -IMAGE_NAME="sutoj/piler:1.3.10" +IMAGE_NAME="sutoj/piler:1.3.11" if [[ $# -ne 1 ]]; then echo "ERROR: missing package name" 1>&2; exit 1; fi From 14f139ce05031fea1189f3f61b025c521da43e02 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 25 Feb 2021 08:18:26 +0100 Subject: [PATCH 063/106] Updated docker readme Signed-off-by: Janos SUTO --- docker/README | 12 ------------ docker/README.md | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 12 deletions(-) delete mode 100644 docker/README create mode 100644 docker/README.md diff --git a/docker/README b/docker/README deleted file mode 100644 index 0065fe4d..00000000 --- a/docker/README +++ /dev/null @@ -1,12 +0,0 @@ -How to build - - Pick the latest deb package from Bitbucket download page (https://bitbucket.org/jsuto/piler/downloads/) - and use it as the PACKAGE build argument, eg. - - docker build --build-arg PACKAGE=piler_1.3.6~bionic-65cc7eb_amd64.deb -t sutoj/piler . - -How to run the image - - Set the PILER_HOST env variable to match your hostname, eg. - - docker run -d --name piler1 -p 25:25 -p 80:80 -e PILER_HOST=archive.yourdomain.com sutoj/piler diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..a0628f0a --- /dev/null +++ b/docker/README.md @@ -0,0 +1,14 @@ +How to run piler + +Edit the variables in docker-compose.yaml, then run + +``` +docker-compose up -d +``` + +How to build the image for yourself + + Pick the latest deb package from Bitbucket download page (https://bitbucket.org/jsuto/piler/downloads/) + and use it as the PACKAGE build argument, eg. + + docker build --build-arg PACKAGE=piler_1.3.11-focal-5c2ceb1_amd64.deb -t piler:1.3.11 . From 1f0dc438e2b4e8c0def7b6cebd472243f731f0ef Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 25 Feb 2021 08:19:43 +0100 Subject: [PATCH 064/106] Updated docker readme #2 Signed-off-by: Janos SUTO --- docker/README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker/README.md b/docker/README.md index a0628f0a..04f28210 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,4 +1,4 @@ -How to run piler +## How to run piler Edit the variables in docker-compose.yaml, then run @@ -6,9 +6,11 @@ Edit the variables in docker-compose.yaml, then run docker-compose up -d ``` -How to build the image for yourself +## How to build the image for yourself - Pick the latest deb package from Bitbucket download page (https://bitbucket.org/jsuto/piler/downloads/) - and use it as the PACKAGE build argument, eg. +Pick the latest deb package from Bitbucket download page (https://bitbucket.org/jsuto/piler/downloads/) +and use it as the PACKAGE build argument, eg. - docker build --build-arg PACKAGE=piler_1.3.11-focal-5c2ceb1_amd64.deb -t piler:1.3.11 . +``` +docker build --build-arg PACKAGE=piler_1.3.11-focal-5c2ceb1_amd64.deb -t piler:1.3.11 . +``` From fc4f7f65d820ae7727e9e0e3b70bb8f76894d0e9 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 27 Feb 2021 07:26:12 +0100 Subject: [PATCH 065/106] Removed duplicate check for MYSQL_DATABASE env var Signed-off-by: Janos SUTO --- docker/start.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/start.sh b/docker/start.sh index c5d2393c..86901238 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -30,9 +30,8 @@ pre_flight_check() { [[ -v PILER_HOSTNAME ]] || error "Missing PILER_HOSTNAME env variable" [[ -v MYSQL_HOSTNAME ]] || error "Missing MYSQL_HOSTNAME env variable" [[ -v MYSQL_DATABASE ]] || error "Missing MYSQL_DATABASE env variable" - [[ -v MYSQL_USER ]] || error "Missing MYSQL_USER env variable" + [[ -v MYSQL_USER ]] || error "Missing MYSQL_USER env variable" [[ -v MYSQL_PASSWORD ]] || error "Missing MYSQL_PASSWORD env variable" - [[ -v MYSQL_DATABASE ]] || error "Missing MYSQL_DATABASE env variable" } From 462e6c46a24030fcc1b5a8b65918324e581932f1 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 2 Mar 2021 06:48:35 +0100 Subject: [PATCH 066/106] Added some proxmox and lxc setup scripts to the contrib dir Signed-off-by: Janos SUTO --- contrib/proxmox-lxc/create_lxc.sh | 195 +++++++++++++++++++++++++++++ contrib/proxmox-lxc/mailpiler.orig | 179 ++++++++++++++++++++++++++ contrib/proxmox-lxc/matrix.orig | 150 ++++++++++++++++++++++ contrib/proxmox-lxc/zmb_mem.orig | 100 +++++++++++++++ 4 files changed, 624 insertions(+) create mode 100644 contrib/proxmox-lxc/create_lxc.sh create mode 100644 contrib/proxmox-lxc/mailpiler.orig create mode 100644 contrib/proxmox-lxc/matrix.orig create mode 100644 contrib/proxmox-lxc/zmb_mem.orig diff --git a/contrib/proxmox-lxc/create_lxc.sh b/contrib/proxmox-lxc/create_lxc.sh new file mode 100644 index 00000000..6c6350bf --- /dev/null +++ b/contrib/proxmox-lxc/create_lxc.sh @@ -0,0 +1,195 @@ +#!/bin/bash + +# This script wil create and fire up a standard debian buster lxc container on your proxmox pve. +# The Script will look for the next free lxc number and take the next free and use it. So take +# care that behind your last number is place for it. + +#### SOME VARIABLES TO ADJUST #### + +# Storage with templates +LXC_TMP="local" + +# Size and pool of rootfs / in GB +SIZ_ROT="100" +S_ROT_P="local-zfs" + +# Size and pool of Filestorage in GB will mounted to /share +SIZ_FIL="100" +S_FIL_P="local-zfs" + +#Weather or not (1 and 0) the container will createt as unpriviliged LXC +LXC_UNP="1" + +# Size of the RAM assigned to the LXC +LXC_MEM="1024" + +# Size of the SWAP assigned to the LXC +LXC_SWA="1024" + +# The hostname (eq. zamba1 or mailpiler1) +LXC_HOST="zamba" + +# The domainname (searchdomain /etc/resolf.conf & hosts) +LXC_SDN="zmb.local" + +# IP-address and subnet +LXC_IP="10.10.80.20/24" + +# Gateway +LXC_GW="10.10.80.10" + +# DNS-server and here shoud be your AD-DC +LXC_DNS="10.10.80.10" + +# Networkbridge for this machine +LXC_BRD="vmbr80" + +# root password - take care to delete from this file +LXC_PWD="MYPASSWD" + +LXC_KEY="ssh-rsa xxxxxxxx" + +############### Zamba-Server-Section ############### + +# Domain Entries to samba/smb.conf. Will be also uses for samba domain-provisioning when zmb-pdc will choosen. +ZMB_REA="ZMB.LOCAL" +ZMB_DOM="ZMB" + +# THE Domain-Admin and passwd for zamba-install +ZMB_ADA="Administrator" +ZMB_APW="MYPASSWORD" + +############### Mailpiler-Section ############### + +# The FQDN vor the Hostname. This must be exactly the same like the LXC_HOST / LXC_SDN at section above. +PILER_DOM="piler.zmb.rocks" +SMARTHOST="10.10.80.20" +PILER_VER="1.3.10" +SPHINX_VER="3.3.1" +PHP_VER="7.4" + +############### Matrix-Section ############### + +# The FQDN vor the Hostname. This should be the same like the LXC_HOST / LXC_SDN at section above. +MRX_DOM="matrix.zmb.rocks" +ELE_DOM="element.zmb.rocks" +ELE_VER="v1.7.21" +JIT_DOM="meet.zmb.rocks" + +################################# + +# CHeck is the newest template available, else download it. + +DEB_LOC=$(pveam list $LXC_TMP | grep debian-10-standard | cut -d'_' -f2) + +DEB_REP=$(pveam available --section system | grep debian-10-standard | cut -d'_' -f2) + +if [[ $DEB_LOC == $DEB_REP ]]; +then + echo "Newest Version of Debian 10 Standard $DEP_REP exists."; +else + echo "Will now download newest Debian 10 Standard $DEP_REP."; + pveam download $LXC_TMP debian-10-standard_$DEB_REP\_amd64.tar.gz +fi + +# Get next free LXC-number +LXC_LST=$( lxc-ls | egrep -o '.{1,5}$' ) +LXC_CHK=$((LXC_LST+1)); + +if [ $LXC_CHK -lt 100 ] || [ -f /etc/pve/qemu-server/$LXC_CHK.conf ]; then + LXC_NBR=$(pvesh get /cluster/nextid); +else + LXC_NBR=$LXC_CHK; +fi + +echo "Will now create LXC Container $LXC_NBR!"; + +# Create the container +pct create $LXC_NBR -unprivileged $LXC_UNP $LXC_TMP:vztmpl/debian-10-standard_$DEB_REP\_amd64.tar.gz -rootfs $S_ROT_P:$SIZ_ROT; +sleep 2; + +pct set $LXC_NBR -memory $LXC_MEM -swap $LXC_SWA -hostname $LXC_HOST \-nameserver $LXC_DNS -searchdomain $LXC_SDN -onboot 1 -timezone Europe/Berlin -net0 name=eth0,bridge=$LXC_BRD,firewall=1,gw=$LXC_GW,ip=$LXC_IP,type=veth; +sleep 2; + +PS3="Select the Server-Function: " + +select opt in just_lxc zmb-standalone zmb-member zmb-pdc mailpiler matrix quit; do + case $opt in + just_lxc) + lxc-start $LXC_NBR; + sleep 5; + # Set the root password and key + echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd; + lxc-attach -n$LXC_NBR mkdir /root/.ssh; + echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys; + lxc-attach -n$LXC_NBR service ssh restart; + echo "Should be ready!" + break + ;; + zmb-standalone) + break + ;; + zmb-member) + echo "Make some additions to LXC for AD-Member-Server!" + pct set $LXC_NBR -mp0 $S_FIL_P:$SIZ_FIL,mp=/tank + sleep 2; + lxc-start $LXC_NBR; + sleep 5; + # Set the root password and key + echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd; + lxc-attach -n$LXC_NBR mkdir /root/.ssh; + echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys; + lxc-attach -n$LXC_NBR service ssh restart; + cp /root/zmb_mem.orig /root/zmb_mem.sh + sed -i "s|#ZMB_VAR|#ZMB_VAR\nZMB_REA='$ZMB_REA'\nZMB_DOM='$ZMB_DOM'\nZMB_ADA='$ZMB_ADA'\nZMB_APW='$ZMB_APW'|" /root/zmb_mem.sh + pct push $LXC_NBR /root/zmb_mem.sh /root/zmb_mem.sh + echo "Install zamba as AD-Member-Server!" + lxc-attach -n$LXC_NBR bash /root/zmb_mem.sh + break + ;; + zmb-pdc) + break + ;; + mailpiler) + echo "Make some additions to LXC for Mailpiler!" + pct set $LXC_NBR -features nesting=1 + sleep 2; + lxc-start $LXC_NBR; + sleep 5; + # Set the root password and key + echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd; + lxc-attach -n$LXC_NBR mkdir /root/.ssh; + echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys; + lxc-attach -n$LXC_NBR service ssh restart; + cp /root/mailpiler.orig /root/mailpiler.sh + sed -i "s|#PILER_VAR|#PILER_VAR\nPILER_DOM='$PILER_DOM'\nSMARTHOST='$SMARTHOST'\nPILER_VER='$PILER_VER'\nSPHINX_VER='$SPHINX_VER'\nPHP_VER='$PHP_VER'|" /root/mailpiler.sh + pct push $LXC_NBR /root/mailpiler.sh /root/mailpiler.sh + echo "Install Mailpiler mailarchiv!" + lxc-attach -n$LXC_NBR bash mailpiler.sh + break + ;; + matrix) + echo "Make some additions to LXC for Matrix!" + lxc-start $LXC_NBR; + sleep 5; + # Set the root password and key + echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd; + lxc-attach -n$LXC_NBR mkdir /root/.ssh; + echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys; + lxc-attach -n$LXC_NBR service ssh restart; + cp /root/matrix.orig /root/matrix.sh + sed -i "s|#MATRIX_VAR|#Matrix_VAR\nMRX_DOM='$MRX_DOM'\nELE_DOM='$ELE_DOM'\nELE_VER='$ELE_VER'\nJIT_DOM='$JIT_DOM'|" /root/matrix.sh + pct push $LXC_NBR /root/matrix.sh /root/matrix.sh + echo "Install Matrix Chatserver!" + lxc-attach -n$LXC_NBR bash matrix.sh + break + ;; + quit) + break + ;; + *) + echo "Invalid option!" + ;; + esac +done + diff --git a/contrib/proxmox-lxc/mailpiler.orig b/contrib/proxmox-lxc/mailpiler.orig new file mode 100644 index 00000000..f20ac8b1 --- /dev/null +++ b/contrib/proxmox-lxc/mailpiler.orig @@ -0,0 +1,179 @@ +#!/bin/bash + +#Variables will be filled in from the mainscript: + +#PILER_VAR + + +HOSTNAME=$(hostname -f) + +echo "Ensure your Hostname is set to your Piler FQDN!" + +echo $HOSTNAME + +if + [ "$HOSTNAME" != "$PILER_DOM" ] +then + echo "Hostname doesn't match Piler_Domain! Check install.sh, /etc/hosts, /etc/hostname." && exit +else + echo "Hostname matches PILER_DOMAIN, so starting installation." +fi + +apt install -y gpg apt-transport-https lsb-release + +wget -q https://packages.sury.org/php/apt.gpg -O- | apt-key add - +echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" | tee /etc/apt/sources.list.d/php.list + +apt update && apt full-upgrade -y + +apt install -y mc sysstat build-essential libwrap0-dev libpst-dev tnef libytnef0-dev unrtf catdoc libtre-dev tre-agrep poppler-utils libzip-dev unixodbc libpq5 software-properties-common libpoppler-dev openssl libssl-dev memcached telnet nginx mariadb-server default-libmysqlclient-dev python-mysqldb gcc libwrap0 libzip4 latex2rtf latex2html catdoc tnef libpq5 zipcmp zipmerge ziptool libsodium23 + +apt update && apt install -y php$PHP_VER-{fpm,common,ldap,mysql,cli,opcache,phpdbg,gd,memcache,json,readline,zip} + +apt purge -y postfix + +cat > /etc/mysql/conf.d/mailpiler.conf <> /usr/local/etc/piler/config-site.php < /etc/nginx/sites-available/$MRX_DOM < /etc/nginx/sites-available/$ELE_DOM <|registration_shared_secret: \"$MRX_PKE\"|" /etc/matrix-synapse/homeserver.yaml +sed -i "s|#public_baseurl: https://example.com/|public_baseurl: https://$MRX_DOM/|" /etc/matrix-synapse/homeserver.yaml +sed -i "s|#enable_registration: false|enable_registration: true|" /etc/matrix-synapse/homeserver.yaml +sed -i "s|name: sqlite3|name: psycopg2|" /etc/matrix-synapse/homeserver.yaml +sed -i "s|database: /var/lib/matrix-synapse/homeserver.db|database: $ELE_DBNAME\n user: $ELE_DBUSER\n password: $ELE_DBPASS\n host: 127.0.0.1\n cp_min: 5\n cp_max: 10|" /etc/matrix-synapse/homeserver.yaml + +systemctl restart matrix-synapse + +register_new_matrix_user -c /etc/matrix-synapse/homeserver.yaml http://127.0.0.1:8008 + +#curl https://download.jitsi.org/jitsi-key.gpg.key | sh -c 'gpg --dearmor > /usr/share/keyrings/jitsi-keyring.gpg' +#echo 'deb [signed-by=/usr/share/keyrings/jitsi-keyring.gpg] https://download.jitsi.org stable/' | tee /etc/apt/sources.list.d/jitsi-stable.list > /dev/null + +#apt update +#apt install -y jitsi-meet + + + diff --git a/contrib/proxmox-lxc/zmb_mem.orig b/contrib/proxmox-lxc/zmb_mem.orig new file mode 100644 index 00000000..92036c6b --- /dev/null +++ b/contrib/proxmox-lxc/zmb_mem.orig @@ -0,0 +1,100 @@ +#!/bin/bash + +#ZMB_VAR + + + +apt update && apt full-upgrade -y +echo -ne '\n' | apt install -y acl dnsutils mc samba winbind libpam-winbind libnss-winbind krb5-user krb5-config samba-dsdb-modules samba-vfs-modules + +mv /etc/krb5.conf /etc/krb5.conf.bak +cat > /etc/krb5.conf < /etc/samba/smb.conf <> /etc/pam.d/common-session + +systemctl restart winbind nmbd +wbinfo -u +wbinfo -g + +mkdir /tank/share +chown 'administrator':'domain users' /tank/share + +setfacl -Rm u:administrator:rwx,g::-,o::- /tank/share +setfacl -Rdm u:administrator:rwx,g::-,o::- /tank/share + +systemctl restart smbd nmbd winbind + From 19ba59b36c76eeb78e1b6dedc6cb71fa8638e894 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 2 Mar 2021 06:51:41 +0100 Subject: [PATCH 067/106] Added readme to contrib/proxmox-lxc Signed-off-by: Janos SUTO --- contrib/proxmox-lxc/README | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 contrib/proxmox-lxc/README diff --git a/contrib/proxmox-lxc/README b/contrib/proxmox-lxc/README new file mode 100644 index 00000000..45b7b5d4 --- /dev/null +++ b/contrib/proxmox-lxc/README @@ -0,0 +1,9 @@ +You nedd proxmox with lxc + +Fill out conf file first and provide public key + +It automatically +- downloads the latest debian template +- provides a container +- installs piler until the post installation step +- gives a self signed certificate From aee8db1b96175088364a46c8f178c7aa4dac4f49 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Wed, 10 Mar 2021 09:32:11 +0100 Subject: [PATCH 068/106] Advanced search should clear the simple search input field Signed-off-by: Janos SUTO --- webui/view/javascript/piler-in.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/webui/view/javascript/piler-in.js b/webui/view/javascript/piler-in.js index 2dc4d39f..5ab6c19d 100644 --- a/webui/view/javascript/piler-in.js +++ b/webui/view/javascript/piler-in.js @@ -727,6 +727,8 @@ var Piler = extra_folders: Piler.extra_folders } + $('input#_search').val(''); + Piler.load_search_results(); $('#searchpopup1').hide(); From f22cd3daaae9c6f1ea6ed10314619a9fb52ba406 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Wed, 10 Mar 2021 18:31:37 +0100 Subject: [PATCH 069/106] Refactored advanced search to modal popup Signed-off-by: Janos SUTO --- .../common/layout-audit-removal.php | 4 - webui/controller/common/layout-audit.php | 1 - webui/controller/common/layout-search.php | 1 - webui/controller/search/popup.php | 17 ---- webui/view/javascript/piler-in.js | 11 ++- .../templates/common/layout-search.tpl | 92 ++++++++++++++++++- .../theme/default/templates/search/popup.tpl | 90 ------------------ 7 files changed, 95 insertions(+), 121 deletions(-) delete mode 100644 webui/controller/search/popup.php delete mode 100644 webui/view/theme/default/templates/search/popup.tpl diff --git a/webui/controller/common/layout-audit-removal.php b/webui/controller/common/layout-audit-removal.php index e9e4105c..fbb781d8 100644 --- a/webui/controller/common/layout-audit-removal.php +++ b/webui/controller/common/layout-audit-removal.php @@ -17,13 +17,9 @@ class ControllerCommonLayoutAuditRemoval extends Controller { $this->children = array( "common/menu", "search/folder", - "search/popup", "common/footer" ); $this->render(); - } - - } diff --git a/webui/controller/common/layout-audit.php b/webui/controller/common/layout-audit.php index 85f9ef4b..18fbbfbb 100644 --- a/webui/controller/common/layout-audit.php +++ b/webui/controller/common/layout-audit.php @@ -17,7 +17,6 @@ class ControllerCommonLayoutAudit extends Controller { $this->children = array( "common/menu", "search/folder", - "search/popup", "common/footer" ); diff --git a/webui/controller/common/layout-search.php b/webui/controller/common/layout-search.php index e0145f2f..1c38b5f4 100644 --- a/webui/controller/common/layout-search.php +++ b/webui/controller/common/layout-search.php @@ -19,7 +19,6 @@ class ControllerCommonLayoutSearch extends Controller { $this->children = array( "common/menu", "search/folder", - "search/popup", "common/footer" ); diff --git a/webui/controller/search/popup.php b/webui/controller/search/popup.php deleted file mode 100644 index 8b750f67..00000000 --- a/webui/controller/search/popup.php +++ /dev/null @@ -1,17 +0,0 @@ -id = "popup"; - $this->template = "search/popup.tpl"; - - $this->render(); - } - - -} - -?> diff --git a/webui/view/javascript/piler-in.js b/webui/view/javascript/piler-in.js index 5ab6c19d..22a94849 100644 --- a/webui/view/javascript/piler-in.js +++ b/webui/view/javascript/piler-in.js @@ -357,6 +357,13 @@ var Piler = }, + show_advanced_search_modal:function() + { + Piler.log("[show_advanced_search_modal]"); + $('#advancedsearch-modal').modal('show'); + }, + + show_bulk_remove_modal:function() { Piler.log("[show_bulk_remove_modal]"); @@ -693,7 +700,7 @@ var Piler = // a = $( a );// a == DOM element // a = Piler.getSource( a );// a == Javascript event - var z = $('div#searchpopup1'); + var z = $('div#advancedsearch-modal'); Piler.search = 'Complex'; @@ -730,8 +737,6 @@ var Piler = $('input#_search').val(''); Piler.load_search_results(); - - $('#searchpopup1').hide(); }, diff --git a/webui/view/theme/default/templates/common/layout-search.tpl b/webui/view/theme/default/templates/common/layout-search.tpl index 25e68b03..027efe40 100644 --- a/webui/view/theme/default/templates/common/layout-search.tpl +++ b/webui/view/theme/default/templates/common/layout-search.tpl @@ -49,6 +49,88 @@ + + - +
@@ -122,8 +204,8 @@
-
- +
+
@@ -135,7 +217,7 @@
- +
diff --git a/webui/view/theme/default/templates/search/popup.tpl b/webui/view/theme/default/templates/search/popup.tpl deleted file mode 100644 index cf1e826f..00000000 --- a/webui/view/theme/default/templates/search/popup.tpl +++ /dev/null @@ -1,90 +0,0 @@ -
- - - - - - - -
- -
- -
- -
- -
-
- -
- -
- -
-
- -
- -
- -
-
- -
- -
- -
-
- -
- -
- -
-
- -
- -
- -
-
- -
- -
- Word checked="checked" onclick="Piler.clear_attachment_any();" /> - Excel checked="checked" onclick="Piler.clear_attachment_any();" /> - PDF checked="checked" onclick="Piler.clear_attachment_any();" /> - image checked="checked" onclick="Piler.clear_attachment_any();" /> - any checked="checked" onclick="Piler.clear_attachment_others();" /> -
-
- - -
- -
- -
-
- -
- -
- -
-
- -
- - -
- - -
- -
- From 2139f8e9e4812a79621319b757e6fa4d40bff8d6 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 23 Mar 2021 15:17:34 +0100 Subject: [PATCH 070/106] -Wimplicit-fallthrough=2 is added for gcc 7+ Signed-off-by: Janos SUTO --- configure | 2 +- configure.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configure b/configure index c195b24c..ae4e20f6 100755 --- a/configure +++ b/configure @@ -4837,7 +4837,7 @@ echo; echo gcc_version="$(gcc -dumpversion)" extra_cflags="" -if [ "${gcc_version:0:1}" -gt 5 ]; then +if [ "${gcc_version:0:1}" -gt 6 ]; then extra_cflags="-Wimplicit-fallthrough=2" fi diff --git a/configure.in b/configure.in index 2e522ba1..b268a207 100644 --- a/configure.in +++ b/configure.in @@ -524,7 +524,7 @@ echo; echo gcc_version="$(gcc -dumpversion)" extra_cflags="" -if [[ "${gcc_version:0:1}" -gt 5 ]]; then +if [[ "${gcc_version:0:1}" -gt 6 ]]; then extra_cflags="-Wimplicit-fallthrough=2" fi From 105ff4110a5b3e44096db9a8e9037d6c70475347 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 26 Mar 2021 05:19:11 +0100 Subject: [PATCH 071/106] Added support to set min. TLS protocol version Signed-off-by: Janos SUTO --- RELEASE_NOTES | 14 ++++++++++++++ VERSION | 2 +- etc/example.conf | 7 +++++++ src/cfg.c | 22 ++++++++++++++++++++++ src/cfg.h | 2 ++ src/defs.h | 5 +++++ src/smtp.c | 13 ++++++++----- 7 files changed, 59 insertions(+), 6 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index eef68096..e392c6bf 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,3 +1,17 @@ +1.3.12: +------- + +- Introduced new piler.conf variable: tls_min_version + + It sets the minimum TLS protocol version the piler-smtp daemon supports. + + Possible values: + - TLSv1 (not recommended) + - TLSv1.1 (not recommended) + - TLSv1.2 (default) + - TLSv1.3 + + 1.3.11: ------- diff --git a/VERSION b/VERSION index 17e63e7a..90a7f602 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.11 +1.3.12 diff --git a/etc/example.conf b/etc/example.conf index f8b65fc7..1c7f7821 100644 --- a/etc/example.conf +++ b/etc/example.conf @@ -107,6 +107,13 @@ pemfile= cipher_list=ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS +; set the minimum TLS protocol version for piler-smtp daemon +; +; Valid values: TLSv1, TLSv1.1, TLSv1.2, TLSv1.3 +; TLSv1 and TLSv1.1 are not recommended for security reasons +tls_min_version=TLSv1.2 + + ; piler's own header to indicate previously archived messages piler_header_field=X-piler-id: diff --git a/src/cfg.c b/src/cfg.c index b348a81a..0b508c45 100644 --- a/src/cfg.c +++ b/src/cfg.c @@ -91,6 +91,7 @@ struct _parse_rule config_parse_rules[] = { "spam_header_line", "string", (void*) string_parser, offsetof(struct config, spam_header_line), "", MAXVAL-1}, { "syslog_recipients", "integer", (void*) int_parser, offsetof(struct config, syslog_recipients), "0", sizeof(int)}, { "tls_enable", "integer", (void*) int_parser, offsetof(struct config, tls_enable), "0", sizeof(int)}, + { "tls_min_version", "string", (void*) string_parser, offsetof(struct config, tls_min_version), "TLSv1.2", MAXVAL-1}, { "tweak_sent_time_offset", "integer", (void*) int_parser, offsetof(struct config, tweak_sent_time_offset), "0", sizeof(int)}, { "update_counters_to_memcached", "integer", (void*) int_parser, offsetof(struct config, update_counters_to_memcached), "0", sizeof(int)}, { "username", "string", (void*) string_parser, offsetof(struct config, username), "piler", MAXVAL-1}, @@ -146,6 +147,24 @@ int parse_config_file(char *configfile, struct config *target_cfg, struct _parse } +int get_tls_protocol_number(char *protocol){ + struct tls_protocol tls_protocols[] = { + { "TLSv1", TLS1_VERSION }, + { "TLSv1.1", TLS1_1_VERSION }, + { "TLSv1.2", TLS1_2_VERSION }, + { "TLSv1.3", TLS1_3_VERSION }, + }; + + for(unsigned int i=0; i 772 + cfg.tls_min_version_number = get_tls_protocol_number(cfg.tls_min_version); + return cfg; } diff --git a/src/cfg.h b/src/cfg.h index 77373869..e374669c 100644 --- a/src/cfg.h +++ b/src/cfg.h @@ -29,6 +29,8 @@ struct config { int tls_enable; char pemfile[MAXVAL]; char cipher_list[MAXVAL]; + char tls_min_version[MAXVAL]; + int tls_min_version_number; int use_antivirus; diff --git a/src/defs.h b/src/defs.h index 4c5a3b49..05b9efbb 100644 --- a/src/defs.h +++ b/src/defs.h @@ -413,4 +413,9 @@ struct smtp_session { struct net net; }; +struct tls_protocol { + char *proto; + int version; +}; + #endif /* _DEFS_H */ diff --git a/src/smtp.c b/src/smtp.c index 479e2ec3..d1eac4c5 100644 --- a/src/smtp.c +++ b/src/smtp.c @@ -171,6 +171,11 @@ int init_ssl(struct smtp_session *session){ return 0; } + if(SSL_CTX_set_min_proto_version(session->net.ctx, session->cfg->tls_min_version_number) == 0){ + syslog(LOG_PRIORITY, "failed SSL_CTX_set_min_proto_version() to %s/%d", session->cfg->tls_min_version, session->cfg->tls_min_version_number); + return 0; + } + if(SSL_CTX_set_cipher_list(session->net.ctx, session->cfg->cipher_list) == 0){ syslog(LOG_PRIORITY, "failed to set cipher list: '%s'", session->cfg->cipher_list); return 0; @@ -198,8 +203,6 @@ void process_command_starttls(struct smtp_session *session){ session->net.ssl = SSL_new(session->net.ctx); if(session->net.ssl){ - SSL_set_options(session->net.ssl, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3); - if(SSL_set_fd(session->net.ssl, session->net.socket) == 1){ session->net.starttls = 1; send_smtp_response(session, SMTP_RESP_220_READY_TO_START_TLS); @@ -209,9 +212,9 @@ void process_command_starttls(struct smtp_session *session){ wait_for_ssl_accept(session); return; - } syslog(LOG_PRIORITY, "%s: SSL_set_fd() failed", session->ttmpfile); - } syslog(LOG_PRIORITY, "%s: SSL_new() failed", session->ttmpfile); - } syslog(LOG_PRIORITY, "SSL ctx is null!"); + } syslog(LOG_PRIORITY, "ERROR: %s: SSL_set_fd() failed", session->ttmpfile); + } syslog(LOG_PRIORITY, "ERROR: %s: SSL_new() failed", session->ttmpfile); + } syslog(LOG_PRIORITY, "ERROR: init_ssl()"); send_smtp_response(session, SMTP_RESP_454_ERR_TLS_TEMP_ERROR); } From 186e56872aac60e4f06678de4a24ce24dea097e6 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Fri, 26 Mar 2021 14:31:48 +0100 Subject: [PATCH 072/106] Added teaser to the configure script Signed-off-by: Janos SUTO --- configure | 6 ++++++ configure.in | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/configure b/configure index ae4e20f6..f877b9d5 100755 --- a/configure +++ b/configure @@ -6173,3 +6173,9 @@ echo echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/wiki/current:upgrade" echo echo + +echo +echo "Did you know that piler has an enterprise edition as well?" +echo "Check out what it can do for you at https://mailpiler.com/piler-enterprise-email-archiver-features/" +echo +echo diff --git a/configure.in b/configure.in index b268a207..8fad88eb 100644 --- a/configure.in +++ b/configure.in @@ -540,3 +540,9 @@ echo echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/wiki/current:upgrade" echo echo + +echo +echo "Did you know that piler has an enterprise edition as well?" +echo "Check out what it can do for you at https://mailpiler.com/piler-enterprise-email-archiver-features/" +echo +echo From d04026d1c3ae5a384e40c07a9d92774f50b89ef9 Mon Sep 17 00:00:00 2001 From: Alexander Noack Date: Wed, 31 Mar 2021 13:29:03 +0200 Subject: [PATCH 073/106] Fix error in SSO against MS AD --- webui/model/user/auth.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui/model/user/auth.php b/webui/model/user/auth.php index 475634f8..aa5db772 100644 --- a/webui/model/user/auth.php +++ b/webui/model/user/auth.php @@ -535,7 +535,7 @@ class ModelUserAuth extends Model { $query = $ldap->query(LDAP_BASE_DN, "(|(&(objectClass=user)(" . $ldap_mail_attr . "$username))(&(objectClass=group)(member=$username))(&(objectClass=group)(member=" . stripslashes($a['dn']) . ")))", array()); - $emails = $this->get_email_array_from_ldap_attr($query->rows); + $emails = $this->get_email_array_from_ldap_attr($query->rows, $ldap_distributionlist_objectclass); $extra_emails = $this->model_user_user->get_email_addresses_from_groups($emails); $emails = array_merge($emails, $extra_emails); From 99a4feb9b1abdc9d5fe9d8c61c4b2168a6d23a44 Mon Sep 17 00:00:00 2001 From: Alexander Noack Date: Wed, 31 Mar 2021 21:11:44 +0200 Subject: [PATCH 074/106] Added auth_gssapi example for Apache --- webui/.htaccess | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/webui/.htaccess b/webui/.htaccess index 849f5498..78befb2b 100644 --- a/webui/.htaccess +++ b/webui/.htaccess @@ -35,3 +35,15 @@ RewriteRule ^view/javascript/piler.js /js.php [QSA,L] + + + RewriteEngine on + RewriteCond %{HTTP:Authorization} !^$ + RewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L] + AuthName "User with domain part (separated by @) in CAPITALS - e.g. 'user@DOMAIN'" + AuthType GSSAPI + GssapiBasicAuth On + GssapiCredStore keytab:/etc/krb5/http.keytab + Require valid-user + + From f71146685d77aeb2aa50668be4e7ab1640b94f03 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 1 Apr 2021 06:29:43 +0200 Subject: [PATCH 075/106] Added readme to gss_api sso auth Signed-off-by: Janos SUTO --- webui/.htaccess | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/webui/.htaccess b/webui/.htaccess index 78befb2b..3eca534e 100644 --- a/webui/.htaccess +++ b/webui/.htaccess @@ -36,6 +36,15 @@ RewriteRule ^view/javascript/piler.js /js.php [QSA,L] + # ktpass -princ HTTP/@ \ + # -mapuser @ \ + # -pass * \ + # -crypto AES256-SHA1 \ + # -ptype KRB5_NT_PRINCIPAL \ + # -out /etc/krb5/http.keytab \ + # + # setspn -s HTTP/ + RewriteEngine on RewriteCond %{HTTP:Authorization} !^$ From d7d0f6cbbc4f760b03ee1adcc3aa813059545137 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 4 Apr 2021 07:40:16 +0200 Subject: [PATCH 076/106] [WIP] Added script to dump all attachments to in/out dirs Signed-off-by: Janos SUTO --- contrib/export-attachments/1.php | 46 ++++++++++++++++++++++++++++++ webui/model/message/attachment.php | 26 ++++++++++++++++- webui/model/search/message.php | 26 ++++++++++++++++- 3 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 contrib/export-attachments/1.php diff --git a/contrib/export-attachments/1.php b/contrib/export-attachments/1.php new file mode 100644 index 00000000..c8902ddd --- /dev/null +++ b/contrib/export-attachments/1.php @@ -0,0 +1,46 @@ +load->model('domain/domain'); +$loader->load->model('search/search'); +$loader->load->model('search/message'); +$loader->load->model('message/attachment'); + + +$db = new DB(DB_DRIVER, DB_HOSTNAME, DB_USERNAME, DB_PASSWORD, DB_DATABASE, DB_PREFIX); +Registry::set('db', $db); + +Registry::set('auditor_user', 1); + +$outdir = "/path/to/attachments"; +$limit = 1000; + +$domain = new ModelDomainDomain(); +$attachment = new ModelMessageAttachment(); +$message = new ModelSearchMessage(); + +$domains = $domain->get_mapped_domains(); + + +for($i=1; $i<$limit; $i++) { + $a = $attachment->get_attachment_by_id($i); + $m = $message->get_message_addresses_by_piler_id($a['piler_id'], $domains); + + $attachment->dump_attachment($outdir, "out", $m['sender'], $i, $a); + + foreach($m['rcpt'] as $rcpt) { + $attachment->dump_attachment($outdir, "in", $rcpt, $i, $a); + } +} diff --git a/webui/model/message/attachment.php b/webui/model/message/attachment.php index e3cacf20..22534284 100644 --- a/webui/model/message/attachment.php +++ b/webui/model/message/attachment.php @@ -6,7 +6,7 @@ class ModelMessageAttachment extends Model { public function get_attachment_by_id($id = 0) { if($id <= 0) { return []; } - $query = $this->db->query("SELECT id, piler_id, attachment_id, name, type FROM " . TABLE_ATTACHMENT . " WHERE id=?", [$id]); + $query = $this->db->query("SELECT id, piler_id, attachment_id, name, type, ptr FROM " . TABLE_ATTACHMENT . " WHERE id=?", [$id]); if(isset($query->row)) { if($query->row['ptr'] > 0) { @@ -77,4 +77,28 @@ class ModelMessageAttachment extends Model { return $images; } + + public function dump_attachment($basedir='', $in_or_out="in", $email='', $id=0, $attachment=[]) { + if($basedir == '' || $email == '') { + return; + } + + $dir = sprintf("%s/%s/%s", $basedir, $email, $in_or_out); + + if(!is_dir($dir)) { + if(!mkdir($dir, 0700, true)) { + die("Failed to create folder $dir"); + } + } + + $fname = sprintf("%s/%d-%s", $dir, $id, $attachment['filename']); + $fp = fopen($fname, "w+"); + if($fp) { + fwrite($fp, $attachment['attachment']); + fclose($fp); + } else { + syslog(LOG_INFO, "ERROR: could not write $fname"); + } + } + } diff --git a/webui/model/search/message.php b/webui/model/search/message.php index 5e67bffa..2351a1de 100644 --- a/webui/model/search/message.php +++ b/webui/model/search/message.php @@ -296,6 +296,30 @@ class ModelSearchMessage extends Model { } + public function get_message_addresses_by_piler_id($piler_id='', $domains=[]) { + $id = 0; + $sender = ''; + $rcpt = []; + + $query = $this->db->query("SELECT id, `from`, `fromdomain` FROM " . TABLE_META . " WHERE piler_id=?", [$piler_id]); + if(isset($query->row)) { + $id = $query->row['id']; + if(in_array($query->row['fromdomain'], $domains)) { + $sender = $query->row['from']; + } + } + + $query = $this->db->query("SELECT `to`, `todomain` FROM " . TABLE_RCPT . " WHERE id=?", [$id]); + foreach($query->rows as $row) { + if(in_array($row['todomain'], $domains)) { + $rcpt[] = $row['to']; + } + } + + return ['sender' => $sender, 'rcpt' => $rcpt]; + } + + public function get_attachment_by_id($id = 0) { if($id <= 0) { return array(); } @@ -459,7 +483,7 @@ class ModelSearchMessage extends Model { foreach ($ids as $id) { $query = $this->db->query("INSERT INTO " . TABLE_TAG . " (id, uid, tag) VALUES(?,?,?)", array($id, $uid, $tag)); } - } + } } From 4b79925c600abf2f22d9ee8355c12406a4b25574 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 4 Apr 2021 08:11:31 +0200 Subject: [PATCH 077/106] Fixed counters in test case Signed-off-by: Janos SUTO --- tests/cases/05-smtp.inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cases/05-smtp.inc b/tests/cases/05-smtp.inc index 2e4fcc03..e05b73cc 100644 --- a/tests/cases/05-smtp.inc +++ b/tests/cases/05-smtp.inc @@ -18,10 +18,10 @@ case1() { "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus" --socket --no-counter - wait_until_emails_are_processed "piler1" 3005 + wait_until_emails_are_processed "piler1" 3007 docker exec "piler1" su piler -c /usr/libexec/piler/indexer.delta.sh 2>/dev/null - count_status_values 3005 2894 111 0 + count_status_values 3007 2896 111 0 test_retrieved_messages_are_the_same "piler1" "piler" From 0d1d6cd012af0ee065c2c612b135e5f7a88dafaa Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 6 Apr 2021 10:24:42 +0200 Subject: [PATCH 078/106] Improved the export-attachments contrib script Signed-off-by: Janos SUTO --- config.php.in | 2 ++ .../{1.php => export-attachments.php} | 15 +++++++++-- webui/model/message/attachment.php | 27 ++++++++++++++++++- 3 files changed, 41 insertions(+), 3 deletions(-) rename contrib/export-attachments/{1.php => export-attachments.php} (75%) diff --git a/config.php.in b/config.php.in index 5c6fd08a..710dda4b 100644 --- a/config.php.in +++ b/config.php.in @@ -472,6 +472,8 @@ define('HEALTH_WORKER_URL', SITE_URL . 'index.php?route=health/worker'); define('LDAP_TYPE_GENERIC', 'generic_ldap'); +define('ATTACHMENT_DUMP_CHECKPOINT', 'attachment_dump_checkpoint'); + define('ACTION_ALL', 0); define('ACTION_UNKNOWN', 1); define('ACTION_LOGIN', 2); diff --git a/contrib/export-attachments/1.php b/contrib/export-attachments/export-attachments.php similarity index 75% rename from contrib/export-attachments/1.php rename to contrib/export-attachments/export-attachments.php index c8902ddd..dc347e0e 100644 --- a/contrib/export-attachments/1.php +++ b/contrib/export-attachments/export-attachments.php @@ -25,7 +25,9 @@ Registry::set('db', $db); Registry::set('auditor_user', 1); $outdir = "/path/to/attachments"; -$limit = 1000; + +openlog("export-attachments", LOG_PID, LOG_MAIL); + $domain = new ModelDomainDomain(); $attachment = new ModelMessageAttachment(); @@ -33,8 +35,13 @@ $message = new ModelSearchMessage(); $domains = $domain->get_mapped_domains(); +$last_id = $attachment->get_last_attachment_id(); +$start_id = $attachment->get_checkpoint(); -for($i=1; $i<$limit; $i++) { +syslog(LOG_INFO, "start: $start, limit: $limit"); + + +for($i=$start_id; $i<$last_id; $i++) { $a = $attachment->get_attachment_by_id($i); $m = $message->get_message_addresses_by_piler_id($a['piler_id'], $domains); @@ -43,4 +50,8 @@ for($i=1; $i<$limit; $i++) { foreach($m['rcpt'] as $rcpt) { $attachment->dump_attachment($outdir, "in", $rcpt, $i, $a); } + + if($i % 100 == 0) { $attachment->update_checkpoint($i); } } + +$attachment->update_checkpoint($i); diff --git a/webui/model/message/attachment.php b/webui/model/message/attachment.php index 22534284..175e951f 100644 --- a/webui/model/message/attachment.php +++ b/webui/model/message/attachment.php @@ -2,7 +2,6 @@ class ModelMessageAttachment extends Model { - public function get_attachment_by_id($id = 0) { if($id <= 0) { return []; } @@ -101,4 +100,30 @@ class ModelMessageAttachment extends Model { } } + + public function get_last_attachment_id() { + $query = $this->db->query("SELECT id FROM " . TABLE_ATTACHMENT . " ORDER BY id DESC LIMIT 1"); + + if(isset($query->row['id'])) { + return $query->row['id']; + } + + return 0; + } + + + public function get_checkpoint() { + $query = $this->db->query("SELECT value FROM `" . TABLE_OPTION . "` WHERE `key`=?", [ATTACHMENT_DUMP_CHECKPOINT]); + if(isset($query->row['value'])) { + return $query->row['value']; + } else { + $this->db->query("INSERT INTO `" . TABLE_OPTION . "` (`key`, value) VALUES(?,0)", [ATTACHMENT_DUMP_CHECKPOINT]); + return 1; + } + } + + + public function update_checkpoint($value=0) { + $this->db->query("UPDATE `" . TABLE_OPTION . "` SET value=? WHERE `key`=?", [$value, ATTACHMENT_DUMP_CHECKPOINT]); + } } From ca6fddedf6f879d5d4a3a23275905c8227649ec2 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 6 Apr 2021 10:34:10 +0200 Subject: [PATCH 079/106] Added support for locking to export-attachments.php Signed-off-by: Janos SUTO --- contrib/export-attachments/export-attachments.php | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/contrib/export-attachments/export-attachments.php b/contrib/export-attachments/export-attachments.php index dc347e0e..11a8a2a1 100644 --- a/contrib/export-attachments/export-attachments.php +++ b/contrib/export-attachments/export-attachments.php @@ -1,5 +1,8 @@ update_checkpoint($i); + +// Release lock +flock($fp, LOCK_UN); +fclose($fp); From edceee12c9064578bf194e70bf6b1209a7067039 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 6 Apr 2021 18:59:16 +0200 Subject: [PATCH 080/106] Added command line options to export-attachments.php Signed-off-by: Janos SUTO --- .../export-attachments/export-attachments.php | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/contrib/export-attachments/export-attachments.php b/contrib/export-attachments/export-attachments.php index 11a8a2a1..74b2fbdb 100644 --- a/contrib/export-attachments/export-attachments.php +++ b/contrib/export-attachments/export-attachments.php @@ -2,8 +2,31 @@ define('LOCK_FILE', '/var/piler/tmp/export-attachments.lock'); +$webuidir = ""; -require_once("config.php"); +$outdir = ''; + + +$opts = 'hw:d:'; +$lopts = [ + 'webui:', + 'dir:', + 'help' +]; + + +if($options = getopt($opts, $lopts)) { + if(isset($options['webui'])) { $webuidir = $options['webui']; } + if(isset($options['dir'])) { $outdir = $options['dir']; } + if(isset($options['help'])) { usage(); } +} + + +if($webuidir == '' || $outdir == '') { usage(); } + + + +require_once("$webuidir/config.php"); require(DIR_SYSTEM . "/startup.php"); @@ -27,8 +50,6 @@ Registry::set('db', $db); Registry::set('auditor_user', 1); -$outdir = "/path/to/attachments"; - openlog("export-attachments", LOG_PID, LOG_MAIL); $fp = fopen(LOCK_FILE, "w"); @@ -68,3 +89,14 @@ $attachment->update_checkpoint($i); // Release lock flock($fp, LOCK_UN); fclose($fp); + + + +function usage() { + print "\nUsage: " . __FILE__ . "\n\n"; + print "\t--webui \n"; + print "\t--dir \n"; + print "\t--help\n\n"; + + exit; +} From e6607b0bf1d44562bcf2a08e3bfed94181b7b95d Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Wed, 7 Apr 2021 20:37:05 +0200 Subject: [PATCH 081/106] pilerexport should syslog the sphinx query Signed-off-by: Janos SUTO --- src/pilerexport.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pilerexport.c b/src/pilerexport.c index eb9d3a09..2d02e2ab 100644 --- a/src/pilerexport.c +++ b/src/pilerexport.c @@ -170,6 +170,8 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char * snprintf(s, sizeof(s)-1, "SELECT id FROM %s WHERE %s AND id > %llu ORDER BY id ASC LIMIT 0,%d", index_list, where_condition, last_id, max_matches); + syslog(LOG_PRIORITY, "sphinx query: %s", s); + if(mysql_real_query(&(sdata2->mysql), s, strlen(s)) == 0){ MYSQL_RES *res = mysql_store_result(&(sdata2->mysql)); if(res != NULL){ From 10309b9c2535155b3731b0c52017b449cfcda378 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 10 Apr 2021 20:53:05 +0200 Subject: [PATCH 082/106] reindex should cleanup its tempfiles Signed-off-by: Janos SUTO --- src/reindex.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/reindex.c b/src/reindex.c index b2b137b4..c3e14a96 100644 --- a/src/reindex.c +++ b/src/reindex.c @@ -128,6 +128,9 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da rc = store_index_data(sdata, &state, data, stored_id, cfg); + unlink(sdata->tmpframe); + remove_stripped_attachments(&state); + if(rc == OK) reindexed++; else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename); From e7d8143186ca8e102fd2c668627ee27b5e7606f6 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 12 Apr 2021 05:13:01 +0200 Subject: [PATCH 083/106] Trim the response from smarthost with the queue id Signed-off-by: Janos SUTO --- webui/model/mail/mail.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui/model/mail/mail.php b/webui/model/mail/mail.php index ca3cc8dc..6cbe4034 100644 --- a/webui/model/mail/mail.php +++ b/webui/model/mail/mail.php @@ -41,7 +41,7 @@ class ModelMailMail extends Model { $l = fgets($r, 4096); - if(preg_match("/^250/", $l)){ $queue_id = $l; $ok = 1; } + if(preg_match("/^250/", $l)){ $queue_id = trim($l); $ok = 1; } fputs($r, "QUIT\r\n"); $l = fgets($r, 4096); From 10544fe02905322e6c7a75caafdaf08bc14b1387 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 12 Apr 2021 05:16:56 +0200 Subject: [PATCH 084/106] Improved zip export performance Signed-off-by: Janos SUTO --- src/pilerexport.c | 91 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 66 insertions(+), 25 deletions(-) diff --git a/src/pilerexport.c b/src/pilerexport.c index 2d02e2ab..bdf7ba57 100644 --- a/src/pilerexport.c +++ b/src/pilerexport.c @@ -33,6 +33,10 @@ int max_matches = 1000; char *index_list = "main1,dailydelta1,delta1"; regex_t regexp; char *zipfile = NULL; +struct zip *z = NULL; +uint64 *zip_ids = NULL; +int zip_counter = 0; +int zip_batch = 2000; int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg); @@ -54,6 +58,7 @@ void usage(){ printf(" -i Sphinx indices to use (default: %s)\n", index_list); #if LIBZIP_VERSION_MAJOR >= 1 printf(" -z Write exported EML files to a zip file\n"); + printf(" -Z Zip batch size. Valid range: 10-10000, default: 2000\n"); #endif printf(" -A Export all emails from archive\n"); printf(" -o Export emails to stdout\n"); @@ -322,26 +327,24 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain } #if LIBZIP_VERSION_MAJOR >= 1 -int write_to_zip_file(char *filename){ - struct zip *z=NULL; - int errorp, ret=ERR; - - z = zip_open(zipfile, ZIP_CREATE, &errorp); - if(!z){ - printf("error: error creating zip file=%s, error code=%d\n", zipfile, errorp); - return ret; - } - - zip_source_t *zs = zip_source_file(z, filename, 0, 0); - if(zs && zip_file_add(z, filename, zs, ZIP_FL_ENC_UTF_8) >= 0){ - ret = OK; - } else { - printf("error adding file %s: %s\n", filename, zip_strerror(z)); - } - +void zip_flush(){ zip_close(z); - return ret; + z = NULL; + zip_counter = 0; + + if(!zip_ids) return; + + for(int i=0; i= 1 - if(zipfile && write_to_zip_file(filename) == OK){ - unlink(filename); - } - #endif + if(zipfile){ + #if LIBZIP_VERSION_MAJOR >= 1 + // Open zip file if handler is NULL + if(!z){ + z = zip_open(zipfile, ZIP_CREATE, &errorp); + if(!z){ + printf("error: error creating zip file=%s, error code=%d\n", zipfile, errorp); + return ERR; + } + } + if(!zip_ids) zip_ids = (uint64*) calloc(sizeof(uint64), zip_batch); + + if(!zip_ids){ + printf("calloc error for zip_ids\n"); + return ERR; + } + + zip_source_t *zs = zip_source_file(z, filename, 0, 0); + if(zs && zip_file_add(z, filename, zs, ZIP_FL_ENC_UTF_8) >= 0){ + *(zip_ids+zip_counter) = id; + zip_counter++; + } else { + printf("error adding file %s: %s\n", filename, zip_strerror(z)); + return ERR; + } + + if(zip_counter == zip_batch){ + zip_flush(); + } + #endif + } } else printf("cannot open: %s\n", filename); } @@ -467,6 +497,7 @@ int main(int argc, char **argv){ {"start-date", required_argument, 0, 'a' }, {"stop-date", required_argument, 0, 'b' }, {"zip", required_argument, 0, 'z' }, + {"zip-batch", required_argument, 0, 'Z' }, {"where-condition", required_argument, 0, 'w' }, {"max-matches", required_argument, 0, 'm' }, {"index-list", required_argument, 0, 'i' }, @@ -475,9 +506,9 @@ int main(int argc, char **argv){ int option_index = 0; - int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:oAdhv?", long_options, &option_index); + int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:oAdhv?", long_options, &option_index); #else - int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:oAdhv?"); + int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:oAdhv?"); #endif if(c == -1) break; @@ -569,6 +600,10 @@ int main(int argc, char **argv){ case 'z': zipfile = optarg; break; + case 'Z': zip_batch = atoi(optarg); + if(zip_batch < 10 || zip_batch > 10000) + zip_batch = 2000; + break; case 'o': export_to_stdout = 1; @@ -634,5 +669,11 @@ int main(int argc, char **argv){ close_database(&sdata); + if(zipfile){ + #if LIBZIP_VERSION_MAJOR >= 1 + zip_flush(); + #endif + } + return verification_status; } From af38efc4d008e800141470d28c32d32845b48035 Mon Sep 17 00:00:00 2001 From: Alexander Noack Date: Mon, 12 Apr 2021 12:14:44 +0200 Subject: [PATCH 085/106] Fixed not to show menu when in Outlook --- webui/view/theme/default/templates/common/layout-search.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui/view/theme/default/templates/common/layout-search.tpl b/webui/view/theme/default/templates/common/layout-search.tpl index 027efe40..6d2cfc0f 100644 --- a/webui/view/theme/default/templates/common/layout-search.tpl +++ b/webui/view/theme/default/templates/common/layout-search.tpl @@ -147,7 +147,7 @@
- + From fbc19e6262786d1af6aece996048fb86a63cae3d Mon Sep 17 00:00:00 2001 From: Alexander Noack Date: Thu, 15 Apr 2021 12:50:41 +0200 Subject: [PATCH 086/106] Fixed get_email_array_from_ldap_attr 2nd parameter error --- webui/model/group/group.php | 2 +- webui/model/user/auth.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/webui/model/group/group.php b/webui/model/group/group.php index 82501b3f..3b3b75b4 100644 --- a/webui/model/group/group.php +++ b/webui/model/group/group.php @@ -194,7 +194,7 @@ class ModelGroupGroup extends Model { $query = $ldap->query(LDAP_BASE_DN, "(&(objectClass=" . LDAP_ACCOUNT_OBJECTCLASS . ")(" . LDAP_MAIL_ATTR . "=" . $username_prefix . $s . "*))", array()); if(isset($query->rows)) { - $emails = $this->model_user_auth->get_email_array_from_ldap_attr($query->rows); + $emails = $this->model_user_auth->get_email_array_from_ldap_attr($query->rows, LDAP_DISTRIBUTIONLIST_OBJECTCLASS); } } } diff --git a/webui/model/user/auth.php b/webui/model/user/auth.php index aa5db772..a51b2929 100644 --- a/webui/model/user/auth.php +++ b/webui/model/user/auth.php @@ -535,7 +535,7 @@ class ModelUserAuth extends Model { $query = $ldap->query(LDAP_BASE_DN, "(|(&(objectClass=user)(" . $ldap_mail_attr . "$username))(&(objectClass=group)(member=$username))(&(objectClass=group)(member=" . stripslashes($a['dn']) . ")))", array()); - $emails = $this->get_email_array_from_ldap_attr($query->rows, $ldap_distributionlist_objectclass); + $emails = $this->get_email_array_from_ldap_attr($query->rows, LDAP_DISTRIBUTIONLIST_OBJECTCLASS); $extra_emails = $this->model_user_user->get_email_addresses_from_groups($emails); $emails = array_merge($emails, $extra_emails); From 5d479b732adf7b975069d352eb912bbd2601b6f6 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 17 Apr 2021 07:40:13 +0200 Subject: [PATCH 087/106] Introduced the archive_address feature Signed-off-by: Janos SUTO --- RELEASE_NOTES | 2 ++ etc/example.conf | 9 +++++++++ src/cfg.c | 2 +- src/cfg.h | 1 + src/smtp.c | 12 ++++++++++-- src/smtp.h | 2 +- src/smtpcodes.h | 1 + 7 files changed, 25 insertions(+), 4 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index e392c6bf..19a18d6b 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,3 +1,5 @@ +- Introduced the archive_address feature, see etc/example.conf for the details + 1.3.12: ------- diff --git a/etc/example.conf b/etc/example.conf index 1c7f7821..54eb264f 100644 --- a/etc/example.conf +++ b/etc/example.conf @@ -238,6 +238,15 @@ mmap_dedup_test=0 ; clients via an IP-address list is not feasible. security_header= +; By default the archive accepts any envelope recipient addresses. +; If your archive's port 25 is wide open to the Internet (which it +; shouldn't be, then spammers may find it, and fill it with spam. +; +; By setting this variable you may restrict the envelope address +; to a single email address, eg. some-random-address-12345@archive.yourdomain.com +; Then the archive will reject any other envelope recipients +archive_address= + ; whether to enable (1) or not (0) an smtp access list similar to ; postfix's postscreen. Valid actions in the acl file are "permit" ; and "reject" (without quotes). See smtp.acl.example for more. diff --git a/src/cfg.c b/src/cfg.c index 0b508c45..563428e0 100644 --- a/src/cfg.c +++ b/src/cfg.c @@ -39,7 +39,7 @@ struct _parse_rule { struct _parse_rule config_parse_rules[] = { - + { "archive_address", "string", (void*) string_parser, offsetof(struct config, archive_address), "", MAXVAL-1}, { "archive_emails_not_having_message_id", "integer", (void*) int_parser, offsetof(struct config, archive_emails_not_having_message_id), "0", sizeof(int)}, { "archive_only_mydomains", "integer", (void*) int_parser, offsetof(struct config, archive_only_mydomains), "0", sizeof(int)}, { "backlog", "integer", (void*) int_parser, offsetof(struct config, backlog), "20", sizeof(int)}, diff --git a/src/cfg.h b/src/cfg.h index e374669c..5b035825 100644 --- a/src/cfg.h +++ b/src/cfg.h @@ -67,6 +67,7 @@ struct config { int default_retention_days; char security_header[MAXVAL]; + char archive_address[MAXVAL]; // mysql stuff diff --git a/src/smtp.c b/src/smtp.c index d1eac4c5..8aa4c3ba 100644 --- a/src/smtp.c +++ b/src/smtp.c @@ -42,7 +42,7 @@ void process_smtp_command(struct smtp_session *session, char *buf, struct config } if(strncasecmp(buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){ - process_command_rcpt_to(session, buf); + process_command_rcpt_to(session, buf, cfg); return; } @@ -239,7 +239,7 @@ void process_command_mail_from(struct smtp_session *session, char *buf){ } -void process_command_rcpt_to(struct smtp_session *session, char *buf){ +void process_command_rcpt_to(struct smtp_session *session, char *buf, struct config *cfg){ if(session->protocol_state == SMTP_STATE_MAIL_FROM || session->protocol_state == SMTP_STATE_RCPT_TO){ @@ -249,6 +249,14 @@ void process_command_rcpt_to(struct smtp_session *session, char *buf){ if(session->num_of_rcpt_to < MAX_RCPT_TO){ extractEmail(buf, session->rcptto[session->num_of_rcpt_to]); + + // Check if we should accept archive_address only + if(cfg->archive_address[0] && !strstr(cfg->archive_address, session->rcptto[session->num_of_rcpt_to])){ + syslog(LOG_PRIORITY, "ERROR: Invalid recipient: *%s*", session->rcptto[session->num_of_rcpt_to]); + send_smtp_response(session, SMTP_RESP_550_ERR_INVALID_RECIPIENT); + return; + } + session->num_of_rcpt_to++; } diff --git a/src/smtp.h b/src/smtp.h index f4b0cb39..a6858d17 100644 --- a/src/smtp.h +++ b/src/smtp.h @@ -16,7 +16,7 @@ void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int bufl void process_command_quit(struct smtp_session *session, char *buf, int buflen); void process_command_reset(struct smtp_session *session); void process_command_mail_from(struct smtp_session *session, char *buf); -void process_command_rcpt_to(struct smtp_session *session, char *buf); +void process_command_rcpt_to(struct smtp_session *session, char *buf, struct config *cfg); void process_command_data(struct smtp_session *session, struct config *cfg); void process_command_period(struct smtp_session *session); void process_command_starttls(struct smtp_session *session); diff --git a/src/smtpcodes.h b/src/smtpcodes.h index 0cdcec84..01e124eb 100644 --- a/src/smtpcodes.h +++ b/src/smtpcodes.h @@ -56,6 +56,7 @@ #define SMTP_RESP_502_ERR "502 Command not implemented\r\n" #define SMTP_RESP_503_ERR "503 Bad command sequence\r\n" +#define SMTP_RESP_550_ERR_INVALID_RECIPIENT "550 Invalid recipient\r\n" #define SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY "550 You are banned by local policy\r\n" #define SMTP_RESP_550_ERR "550 Service currently unavailable\r\n" From be39146fd2b17777d2f8ee55832f77784b6586bb Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 18 Apr 2021 20:19:26 +0200 Subject: [PATCH 088/106] PVS Studio refactoring Signed-off-by: Janos SUTO --- src/archive.c | 2 +- src/digest.c | 4 ++-- src/imap.c | 30 ++++++++++++++++++------------ src/misc.c | 9 +++------ src/parser.c | 4 ++-- src/parser.h | 2 +- src/parser_utils.c | 40 ++++------------------------------------ src/piler.c | 2 +- src/pilerexport.c | 44 +++++++++++++++++++++++++++++--------------- src/pilerimport.c | 2 +- src/rules.c | 17 ++++++----------- src/smtp.c | 2 +- src/store.c | 2 +- 13 files changed, 70 insertions(+), 90 deletions(-) diff --git a/src/archive.c b/src/archive.c index 8ea7b504..97e5a376 100644 --- a/src/archive.c +++ b/src/archive.c @@ -244,7 +244,7 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de CLEANUP: - if(fd != -1) close(fd); + if(fd != -1) close(fd); //-V547 if(s) free(s); if(cfg->encrypt_messages == 1) #if OPENSSL_VERSION_NUMBER < 0x10100000L diff --git a/src/digest.c b/src/digest.c index 6899dba1..20134730 100644 --- a/src/digest.c +++ b/src/digest.c @@ -81,8 +81,8 @@ int make_digests(struct session_data *sdata, struct config *cfg){ SHA256_Final(md2, &context2); for(i=0;ibodydigest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]); - snprintf(sdata->digest + i*2, 2*DIGEST_LENGTH, "%02x", md2[i]); + snprintf(sdata->bodydigest + i*2, 3, "%02x", md[i]); + snprintf(sdata->digest + i*2, 3, "%02x", md2[i]); } return 0; diff --git a/src/imap.c b/src/imap.c index bd5008ac..ade89446 100644 --- a/src/imap.c +++ b/src/imap.c @@ -416,7 +416,7 @@ int list_folders(struct data *data){ } // trim the "A3 OK LIST completed" trailer off - if(p) *p = '\0'; + if(p) *p = '\0'; //-V547 memset(attrs, 0, sizeof(attrs)); @@ -449,21 +449,27 @@ int list_folders(struct data *data){ } else { if(fldrlen) { - ruf = malloc(strlen(q) * 2 + 1); - memset(ruf, 0, strlen(q) * 2 + 1); - memcpy(ruf, q, strlen(q)); - r = ruf; - while(*r != '\0') { - if(*r == '\\') { - memmove(r + 1, r, strlen(r)); + int ruflen = strlen(q) * 2; + ruf = malloc(ruflen + 1); + if(ruf){ + snprintf(ruf, ruflen, "%s", q); + r = ruf; + while(*r != '\0') { + if(*r == '\\') { + memmove(r + 1, r, strlen(r)); + r++; + } r++; } - r++; + + snprintf(folder, sizeof(folder)-1, "%s", ruf); + + free(ruf); + } + else { + printf("error: ruf = malloc()\n"); } - snprintf(folder, sizeof(folder)-1, "%s", ruf); - - free(ruf); fldrlen = 0; } else { snprintf(folder, sizeof(folder)-1, "%s", q); diff --git a/src/misc.c b/src/misc.c index e5fe4c42..703b21d0 100644 --- a/src/misc.c +++ b/src/misc.c @@ -230,11 +230,8 @@ int extractEmail(char *rawmail, char *email){ */ void make_random_string(unsigned char *buf, int buflen){ - int i, len, fd; - int urandom=0; - static char alphanum[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - - len = strlen(alphanum); + const char alphanum[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + int i, fd, urandom=0, len = sizeof(alphanum)-1; fd = open(RANDOM_POOL, O_RDONLY); if(fd != -1){ @@ -289,7 +286,7 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){ taia_now(&now); taia_pack(nowpack, &now); - memcpy(buf, nowpack, 12); + memcpy(buf, nowpack, 12); //-V512 fd = open(RANDOM_POOL, O_RDONLY); if(fd == -1) return ret; diff --git a/src/parser.c b/src/parser.c index dbaa6277..feaae3fa 100644 --- a/src/parser.c +++ b/src/parser.c @@ -45,7 +45,7 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec if(take_into_pieces == 1 && state.writebufpos > 0){ if(write(state.mfd, writebuffer, state.writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: %s: write(), %s, %d, %s", sdata->ttmpfile, __func__, __LINE__, __FILE__); - memset(writebuffer, 0, sizeof(writebuffer)); + memset(writebuffer, 0, sizeof(writebuffer)); //-V597 state.writebufpos = 0; } @@ -212,7 +212,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata sdata->restored_copy = 1; } - if(cfg->security_header && state->found_security_header == 0 && strstr(buf, cfg->security_header)){ + if(cfg->security_header[0] && state->found_security_header == 0 && strstr(buf, cfg->security_header)){ state->found_security_header = 1; } diff --git a/src/parser.h b/src/parser.h index 5f536fd7..c9438491 100644 --- a/src/parser.h +++ b/src/parser.h @@ -20,7 +20,7 @@ void fixupEncodedHeaderLine(char *buf, int buflen); void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state); void fixupBase64EncodedLine(char *buf, struct parser_state *state); void markHTML(char *buf, struct parser_state *state); -int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state); +void setStateHTMLStyle(char *htmlbuf, int pos, struct parser_state *state); void translateLine(unsigned char *p, struct parser_state *state); void fix_email_address_for_sphinx(char *s); void split_email_address(char *s); diff --git a/src/parser_utils.c b/src/parser_utils.c index abc63ad2..d2fa5dce 100644 --- a/src/parser_utils.c +++ b/src/parser_utils.c @@ -538,7 +538,7 @@ void markHTML(char *buf, struct parser_state *state){ if(isspace(*s)){ if(j > 0){ - k += appendHTMLTag(puf, html, pos, state); + setStateHTMLStyle(html, pos, state); memset(html, 0, SMALLBUFSIZE); j=0; } pos++; @@ -563,7 +563,7 @@ void markHTML(char *buf, struct parser_state *state){ if(j > 0){ strncat(html, " ", SMALLBUFSIZE-1); - k += appendHTMLTag(puf, html, pos, state); + setStateHTMLStyle(html, pos, state); memset(html, 0, SMALLBUFSIZE); j=0; } } @@ -571,47 +571,15 @@ void markHTML(char *buf, struct parser_state *state){ } //printf("append last in line:*%s*, html=+%s+, j=%d\n", puf, html, j); - if(j > 0){ appendHTMLTag(puf, html, pos, state); } + if(j > 0){ setStateHTMLStyle(html, pos, state); } strcpy(buf, puf); } -int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state){ - char html[SMALLBUFSIZE]; - int len; - +void setStateHTMLStyle(char *htmlbuf, int pos, struct parser_state *state){ if(pos == 0 && strncmp(htmlbuf, "style ", 6) == 0) state->style = 1; if(pos == 0 && strncmp(htmlbuf, "/style ", 7) == 0) state->style = 0; - - return 0; - - //printf("appendHTML: pos:%d, +%s+\n", pos, htmlbuf); - - if(state->style == 1) return 0; - - if(strlen(htmlbuf) == 0) return 0; - - snprintf(html, SMALLBUFSIZE-1, "HTML*%s", htmlbuf); - len = strlen(html); - - if(len > 8 && strchr(html, '=')){ - char *p = strstr(html, "cid:"); - if(p){ - *(p+3) = '\0'; - strncat(html, " ", SMALLBUFSIZE-1); - } - - strncat(buf, html, MAXBUFSIZE-1); - return len; - } - - if(strstr(html, "http") ){ - strncat(buf, html+5, MAXBUFSIZE-1); - return len-5; - } - - return 0; } diff --git a/src/piler.c b/src/piler.c index d26016cf..cfab8825 100644 --- a/src/piler.c +++ b/src/piler.c @@ -103,7 +103,7 @@ void child_sighup_handler(int sig){ int perform_checks(char *filename, struct session_data *sdata, struct data *data, struct parser_state *parser_state, struct config *cfg){ - if(cfg->security_header && parser_state->found_security_header == 0){ + if(cfg->security_header[0] && parser_state->found_security_header == 0){ syslog(LOG_PRIORITY, "%s: discarding: missing security header", filename); return ERR_DISCARDED; } diff --git a/src/pilerexport.c b/src/pilerexport.c index bdf7ba57..5ee47c86 100644 --- a/src/pilerexport.c +++ b/src/pilerexport.c @@ -25,7 +25,6 @@ extern int optind; int dryrun = 0; int exportall = 0; int verification_status = 0; -int rc = 0; int export_to_stdout = 0; char *query=NULL; int verbosity = 0; @@ -33,7 +32,7 @@ int max_matches = 1000; char *index_list = "main1,dailydelta1,delta1"; regex_t regexp; char *zipfile = NULL; -struct zip *z = NULL; +struct zip *zip = NULL; uint64 *zip_ids = NULL; int zip_counter = 0; int zip_batch = 2000; @@ -165,6 +164,7 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char * MYSQL_ROW row; uint64 id=0; char s[SMALLBUFSIZE]; + int rc=0; *num = 0; @@ -194,6 +194,7 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char * } if(!rc) export_emails_matching_to_query(sdata, query, cfg); + else printf("error: append_string_to_buffer() in run_query()\n"); free(query); query = NULL; @@ -240,6 +241,7 @@ void export_emails_matching_id_list(struct session_data *sdata, struct session_d int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain, int minsize, int maxsize, unsigned long startdate, unsigned long stopdate){ char s[SMALLBUFSIZE]; + int rc=0; if(exportall == 1){ rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM "); @@ -328,9 +330,9 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain #if LIBZIP_VERSION_MAJOR >= 1 void zip_flush(){ - zip_close(z); + zip_close(zip); - z = NULL; + zip = NULL; zip_counter = 0; if(!zip_ids) return; @@ -354,7 +356,7 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct char digest[SMALLBUFSIZE], bodydigest[SMALLBUFSIZE]; char filename[SMALLBUFSIZE]; struct sql sql; - int errorp; + int errorp, rc=0; if(prepare_sql_statement(sdata, &sql, s) == ERR) return ERR; @@ -410,9 +412,9 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct if(zipfile){ #if LIBZIP_VERSION_MAJOR >= 1 // Open zip file if handler is NULL - if(!z){ - z = zip_open(zipfile, ZIP_CREATE, &errorp); - if(!z){ + if(!zip){ + zip = zip_open(zipfile, ZIP_CREATE, &errorp); + if(!zip){ printf("error: error creating zip file=%s, error code=%d\n", zipfile, errorp); return ERR; } @@ -425,12 +427,12 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct return ERR; } - zip_source_t *zs = zip_source_file(z, filename, 0, 0); - if(zs && zip_file_add(z, filename, zs, ZIP_FL_ENC_UTF_8) >= 0){ + zip_source_t *zs = zip_source_file(zip, filename, 0, 0); + if(zs && zip_file_add(zip, filename, zs, ZIP_FL_ENC_UTF_8) >= 0){ *(zip_ids+zip_counter) = id; zip_counter++; } else { - printf("error adding file %s: %s\n", filename, zip_strerror(z)); + printf("error adding file %s: %s\n", filename, zip_strerror(zip)); return ERR; } @@ -539,7 +541,10 @@ int main(int argc, char **argv){ break; } - rc = append_email_to_buffer(&from, optarg); + if(append_email_to_buffer(&from, optarg)){ + printf("error: append_email_to_buffer() for from\n"); + return 1; + } break; @@ -550,7 +555,10 @@ int main(int argc, char **argv){ break; } - rc = append_email_to_buffer(&to, optarg); + if(append_email_to_buffer(&to, optarg)){ + printf("error: append_email_to_buffer() for to\n"); + return 1; + } break; @@ -561,7 +569,10 @@ int main(int argc, char **argv){ break; } - rc = append_email_to_buffer(&fromdomain, optarg); + if(append_email_to_buffer(&fromdomain, optarg)){ + printf("error: append_email_to_buffer() for fromdomain\n"); + return 1; + } break; @@ -572,7 +583,10 @@ int main(int argc, char **argv){ break; } - rc = append_email_to_buffer(&todomain, optarg); + if(append_email_to_buffer(&todomain, optarg)){ + printf("error: append_email_to_buffer() for todomain\n"); + return 1; + } break; diff --git a/src/pilerimport.c b/src/pilerimport.c index 51ce727c..3be6541f 100644 --- a/src/pilerimport.c +++ b/src/pilerimport.c @@ -295,7 +295,7 @@ int main(int argc, char **argv){ } - if(!mbox[0] && !data.import->mboxdir && !data.import->filename && !directory && !imapserver && !pop3server) usage(); + if(!mbox[0] && !data.import->mboxdir && !data.import->filename[0] && !directory && !imapserver && !pop3server) usage(); if(data.import->failed_folder && !can_i_write_directory(data.import->failed_folder)){ printf("cannot write failed directory '%s'\n", data.import->failed_folder); diff --git a/src/rules.c b/src/rules.c index 37c8d3cf..d2cd7c7e 100644 --- a/src/rules.c +++ b/src/rules.c @@ -118,7 +118,6 @@ int append_rule(struct node *xhash[], struct rule_cond *rule_cond){ struct rule *create_rule_item(struct rule_cond *rule_cond){ struct rule *h=NULL; - char empty = '\0'; int len; if(rule_cond == NULL) return NULL; @@ -144,16 +143,16 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){ h->emptyfrom = h->emptyto = h->emptysubject = h->emptyaname = h->emptyatype = 0; - if(rule_cond->from == NULL || strlen(rule_cond->from) < 1){ rule_cond->from[0] = empty; h->emptyfrom = 1; } + if(rule_cond->from[0] == 0){ h->emptyfrom = 1; } if(regcomp(&(h->from), rule_cond->from, REG_ICASE | REG_EXTENDED)) h->compiled = 0; - if(rule_cond->to == NULL || strlen(rule_cond->to) < 1){ rule_cond->to[0] = empty; h->emptyto = 1; } + if(rule_cond->to[0] == 0){ h->emptyto = 1; } if(regcomp(&(h->to), rule_cond->to, REG_ICASE | REG_EXTENDED)) h->compiled = 0; - if(rule_cond->subject == NULL || strlen(rule_cond->subject) < 1){ rule_cond->subject[0] = empty; h->emptysubject = 1; } + if(rule_cond->subject[0] == 0){ h->emptysubject = 1; } if(regcomp(&(h->subject), rule_cond->subject, REG_ICASE | REG_EXTENDED)) h->compiled = 0; - if(rule_cond->body == NULL || strlen(rule_cond->body) < 1){ rule_cond->body[0] = empty; h->emptybody = 1; } + if(rule_cond->body[0] == 0){ h->emptybody = 1; } if(regcomp(&(h->body), rule_cond->body, REG_ICASE | REG_EXTENDED)) h->compiled = 0; h->spam = rule_cond->spam; @@ -161,20 +160,16 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){ h->folder_id = rule_cond->folder_id; h->size = rule_cond->size; - - if(rule_cond->_size == NULL) rule_cond->_size[0] = empty; snprintf(h->_size, 3, "%s", rule_cond->_size); - if(rule_cond->attachment_name == NULL || strlen(rule_cond->attachment_name) < 1){ rule_cond->attachment_name[0] = empty; h->emptyaname = 1; } + if(rule_cond->attachment_name[0] == 0){ h->emptyaname = 1; } if(regcomp(&(h->attachment_name), rule_cond->attachment_name, REG_ICASE | REG_EXTENDED)) h->compiled = 0; - if(rule_cond->attachment_type == NULL || strlen(rule_cond->attachment_type) < 1){ rule_cond->attachment_type[0] = empty; h->emptyatype = 1; } + if(rule_cond->attachment_type[0] == 0){ h->emptyatype = 1; } if(regcomp(&(h->attachment_type), rule_cond->attachment_type, REG_ICASE | REG_EXTENDED)) h->compiled = 0; h->attachment_size = rule_cond->attachment_size; - - if(rule_cond->_attachment_size == NULL) rule_cond->_attachment_size[0] = empty; snprintf(h->_attachment_size, 3, "%s", rule_cond->_attachment_size); len = strlen(rule_cond->domain)+8 + strlen(rule_cond->from)+6 + strlen(rule_cond->to)+4 + strlen(rule_cond->subject)+9 + strlen(rule_cond->body)+6 + strlen(rule_cond->_size)+6 + strlen(rule_cond->attachment_name)+10 + strlen(rule_cond->attachment_type)+10 + strlen(rule_cond->_attachment_size)+10 + 9 + 15 + 15; diff --git a/src/smtp.c b/src/smtp.c index 8aa4c3ba..5cd69fe6 100644 --- a/src/smtp.c +++ b/src/smtp.c @@ -314,7 +314,7 @@ void process_command_period(struct smtp_session *session){ session->buflen = 0; session->last_data_char = 0; - memset(session->buf, 0, SMALLBUFSIZE); + memset(session->buf, 0, sizeof(session->buf)); send_smtp_response(session, buf); } diff --git a/src/store.c b/src/store.c index 31323f53..99abc307 100644 --- a/src/store.c +++ b/src/store.c @@ -228,7 +228,7 @@ int store_file(struct session_data *sdata, char *filename, int len, struct confi ENDE: if(outbuf) free(outbuf); - if(z) free(z); + if(z) free(z); //-V547 return ret; } From 3bb6b401a111667bc22acd8f31221766b07fc024 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 19 Apr 2021 16:00:30 +0200 Subject: [PATCH 089/106] Fixed export-attachments.php typo Signed-off-by: Janos SUTO --- contrib/export-attachments/export-attachments.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) mode change 100644 => 100755 contrib/export-attachments/export-attachments.php diff --git a/contrib/export-attachments/export-attachments.php b/contrib/export-attachments/export-attachments.php old mode 100644 new mode 100755 index 74b2fbdb..0d67144e --- a/contrib/export-attachments/export-attachments.php +++ b/contrib/export-attachments/export-attachments.php @@ -1,6 +1,7 @@ +#!/usr/bin/php get_mapped_domains(); $last_id = $attachment->get_last_attachment_id(); $start_id = $attachment->get_checkpoint(); -syslog(LOG_INFO, "start: $start, limit: $limit"); +syslog(LOG_INFO, "start: $start_id, limit: $last_id"); for($i=$start_id; $i<$last_id; $i++) { From 13b10d5f3d953483ac86d0a6244491970ec13e2a Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Thu, 22 Apr 2021 13:45:05 +0200 Subject: [PATCH 090/106] If the Date: field contains some garbage return the current timestamp Signed-off-by: Janos SUTO --- src/parser_utils.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/parser_utils.c b/src/parser_utils.c index d2fa5dce..e8c1f2c8 100644 --- a/src/parser_utils.c +++ b/src/parser_utils.c @@ -263,6 +263,18 @@ time_t parse_date_header(char *datestr){ ts += get_local_timezone_offset() - offset; + if(ts < 700000000){ + // If the Date: field contains some garbage, eg. + // "Date: [mail_datw]" or similar, and the date + // is before Sat Mar 7 20:26:40 UTC 1992, then + // return the current timestamp + + time_t now; + time(&now); + + return now; + } + return ts; } From b010aaab8973c756f81596687a23bcc62cc66869 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 24 Apr 2021 19:45:45 +0200 Subject: [PATCH 091/106] Added missing private action to audit records Signed-off-by: Janos SUTO --- webui/controller/audit/helper.php | 3 ++- webui/language/cz/messages.php | 1 + webui/language/de/messages.php | 1 + webui/language/en/messages.php | 1 + webui/language/es/messages.php | 1 + webui/language/fr/messages.php | 1 + webui/language/hu/messages.php | 1 + webui/language/pl/messages.php | 1 + webui/language/ru/messages.php | 1 + webui/language/tr/messages.php | 1 + 10 files changed, 11 insertions(+), 1 deletion(-) diff --git a/webui/controller/audit/helper.php b/webui/controller/audit/helper.php index 95c80904..4b53de4f 100644 --- a/webui/controller/audit/helper.php +++ b/webui/controller/audit/helper.php @@ -64,10 +64,11 @@ class ControllerAuditHelper extends Controller { $this->data['actions'][ACTION_DOWNLOAD_ATTACHMENT] = $this->data['text_download_attachment2']; $this->data['actions'][ACTION_UNAUTHORIZED_DOWNLOAD_ATTACHMENT] = $this->data['text_unauthorized_download_attachment']; $this->data['actions'][ACTION_VIEW_JOURNAL] = $this->data['text_view_journal']; + $this->data['actions'][ACTION_MARK_AS_PRIVATE] = $this->data['text_mark_private']; + - /* paging info */ $this->data['prev_page'] = $this->data['page'] - 1; diff --git a/webui/language/cz/messages.php b/webui/language/cz/messages.php index 2d18447c..723e0f5b 100644 --- a/webui/language/cz/messages.php +++ b/webui/language/cz/messages.php @@ -211,6 +211,7 @@ $_['text_logout2'] = "odhlášení"; $_['text_maillog_status'] = "status maillog collectoru"; $_['text_main_title'] = "clapf web UI"; $_['text_mapped_domain'] = "Mapovaná doména"; +$_['text_mark_private'] = "private"; $_['text_marked_for_removal'] = "Zpráva označena k odstranění"; $_['text_memory_usage'] = "Využití paměti"; $_['text_message'] = "zpráva"; diff --git a/webui/language/de/messages.php b/webui/language/de/messages.php index 33d47636..80faf1ce 100644 --- a/webui/language/de/messages.php +++ b/webui/language/de/messages.php @@ -197,6 +197,7 @@ $_['text_logout2'] = "Abmeldung"; $_['text_maillog_status'] = "Sammelstatus Maillogbuch"; $_['text_main_title'] = "Clapf Benutzeroberfläche"; $_['text_mapped_domain'] = "Domänenalias"; +$_['text_mark_private'] = "private"; $_['text_marked_for_removal'] = "Nachricht zum Löschen markiert"; $_['text_memory_usage'] = "Arbeitsspeicher"; $_['text_message'] = "Nachricht"; diff --git a/webui/language/en/messages.php b/webui/language/en/messages.php index 2d6099c1..57817ce4 100644 --- a/webui/language/en/messages.php +++ b/webui/language/en/messages.php @@ -216,6 +216,7 @@ $_['text_logout2'] = "logout"; $_['text_maillog_status'] = "maillog collector status"; $_['text_main_title'] = "clapf web UI"; $_['text_mapped_domain'] = "Mapped domain"; +$_['text_mark_private'] = "private"; $_['text_marked_for_removal'] = "Message marked for removal"; $_['text_memory_usage'] = "Memory usage"; $_['text_message'] = "message"; diff --git a/webui/language/es/messages.php b/webui/language/es/messages.php index 79356d71..fb7b5070 100644 --- a/webui/language/es/messages.php +++ b/webui/language/es/messages.php @@ -197,6 +197,7 @@ $_['text_logout2'] = "cerrar sesión"; $_['text_maillog_status'] = "estado del colector de logs de mails"; $_['text_main_title'] = "IU web clapf"; $_['text_mapped_domain'] = "Dominio mapeado"; +$_['text_mark_private'] = "private"; $_['text_marked_for_removal'] = "Mensaje marcado para remover"; $_['text_memory_usage'] = "Uso de memoria"; $_['text_message'] = "mensaje"; diff --git a/webui/language/fr/messages.php b/webui/language/fr/messages.php index d7347b1e..739480e0 100644 --- a/webui/language/fr/messages.php +++ b/webui/language/fr/messages.php @@ -210,6 +210,7 @@ $_['text_logout2'] = "déconnexion"; $_['text_maillog_status'] = "statut du collecteur de logs mails"; $_['text_main_title'] = "clapf web UI"; $_['text_mapped_domain'] = "Domaine mappé"; +$_['text_mark_private'] = "private"; $_['text_marked_for_removal'] = "Message marqué pour suppression"; $_['text_memory_usage'] = "Utilisation mémoire"; $_['text_message'] = "message"; diff --git a/webui/language/hu/messages.php b/webui/language/hu/messages.php index d71c2d32..48d2888c 100644 --- a/webui/language/hu/messages.php +++ b/webui/language/hu/messages.php @@ -217,6 +217,7 @@ $_['text_logout2'] = "kijelentkezés"; $_['text_maillog_status'] = "maillog gyűjtő státusz"; $_['text_main_title'] = "clapf web UI"; $_['text_mapped_domain'] = "Hozzárendelt domain"; +$_['text_mark_private'] = "privát"; $_['text_marked_for_removal'] = "Levél törlésre jelölve"; $_['text_memory_usage'] = "Memória használat"; $_['text_message'] = "üzenet"; diff --git a/webui/language/pl/messages.php b/webui/language/pl/messages.php index 7ee868b1..47b9a76b 100644 --- a/webui/language/pl/messages.php +++ b/webui/language/pl/messages.php @@ -212,6 +212,7 @@ $_['text_logout2'] = "wylogowany"; $_['text_maillog_status'] = "program od zbierania wiadomości e-mail"; $_['text_main_title'] = "clapf interfejsu użytkownika strony"; $_['text_mapped_domain'] = "Zmapowana domena"; +$_['text_mark_private'] = "private"; $_['text_marked_for_removal'] = "Wiadomość oznaczona do usunięcia"; $_['text_memory_usage'] = "Użycie pamięci"; $_['text_message'] = "wiadomość"; diff --git a/webui/language/ru/messages.php b/webui/language/ru/messages.php index eb7c2bd2..97225783 100644 --- a/webui/language/ru/messages.php +++ b/webui/language/ru/messages.php @@ -210,6 +210,7 @@ $_['text_logout2'] = "выход"; $_['text_maillog_status'] = "статус сборщика maillog"; $_['text_main_title'] = "Основной заголовок"; $_['text_mapped_domain'] = "Отображаемый домен"; +$_['text_mark_private'] = "private"; $_['text_marked_for_removal'] = "Помеченные для удаления сообщения"; $_['text_memory_usage'] = "Использование памяти"; $_['text_message'] = "сообщение"; diff --git a/webui/language/tr/messages.php b/webui/language/tr/messages.php index d0f4a0fd..1ff85585 100644 --- a/webui/language/tr/messages.php +++ b/webui/language/tr/messages.php @@ -209,6 +209,7 @@ $_['text_logout2'] = "çıkış"; $_['text_maillog_status'] = "e-posta logu toplama durumu"; $_['text_main_title'] = "clapf web arabirimi"; $_['text_mapped_domain'] = "Eşleşmiş alan adı"; +$_['text_mark_private'] = "private"; $_['text_marked_for_removal'] = "Mesaj kaldırılmak üzere işaretlenmiş"; $_['text_memory_usage'] = "Hafıza kullanımı"; $_['text_message'] = "mesaj"; From 22076d261ccc3f074acf4a99a57e69eebc5a8d7c Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 26 Apr 2021 06:08:18 +0200 Subject: [PATCH 092/106] Added audit message if the email has been marked for removal Signed-off-by: Janos SUTO --- webui/controller/audit/helper.php | 1 + webui/language/cz/messages.php | 1 + webui/language/de/messages.php | 1 + webui/language/en/messages.php | 1 + webui/language/es/messages.php | 1 + webui/language/fr/messages.php | 1 + webui/language/hu/messages.php | 1 + webui/language/pl/messages.php | 1 + webui/language/pt/messages.php | 2 ++ webui/language/ru/messages.php | 2 ++ webui/language/tr/messages.php | 2 ++ 11 files changed, 14 insertions(+) diff --git a/webui/controller/audit/helper.php b/webui/controller/audit/helper.php index 4b53de4f..af248ce8 100644 --- a/webui/controller/audit/helper.php +++ b/webui/controller/audit/helper.php @@ -64,6 +64,7 @@ class ControllerAuditHelper extends Controller { $this->data['actions'][ACTION_DOWNLOAD_ATTACHMENT] = $this->data['text_download_attachment2']; $this->data['actions'][ACTION_UNAUTHORIZED_DOWNLOAD_ATTACHMENT] = $this->data['text_unauthorized_download_attachment']; $this->data['actions'][ACTION_VIEW_JOURNAL] = $this->data['text_view_journal']; + $this->data['actions'][ACTION_MARK_MESSAGE_FOR_REMOVAL] = $this->data['text_remove_request']; $this->data['actions'][ACTION_MARK_AS_PRIVATE] = $this->data['text_mark_private']; diff --git a/webui/language/cz/messages.php b/webui/language/cz/messages.php index 723e0f5b..86385d15 100644 --- a/webui/language/cz/messages.php +++ b/webui/language/cz/messages.php @@ -493,3 +493,4 @@ $_['text_no_selected_message'] = "no selected message"; $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; +$_['text_remove_request'] = "remove request"; diff --git a/webui/language/de/messages.php b/webui/language/de/messages.php index 80faf1ce..cbed91ad 100644 --- a/webui/language/de/messages.php +++ b/webui/language/de/messages.php @@ -500,3 +500,4 @@ $_['text_no_selected_message'] = "no selected message"; $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; +$_['text_remove_request'] = "remove request"; diff --git a/webui/language/en/messages.php b/webui/language/en/messages.php index 57817ce4..c361427d 100644 --- a/webui/language/en/messages.php +++ b/webui/language/en/messages.php @@ -295,6 +295,7 @@ $_['text_remove'] = "Remove"; $_['text_remove_domain'] = "Remove domain"; $_['text_remove_message'] = "Remove message"; $_['text_remove_message2'] = "remove message"; +$_['text_remove_request'] = "remove request"; $_['text_remove_selected_uids'] = "Remove selected uids"; $_['text_remove_policy'] = "Remove policy"; $_['text_remove_rule'] = "Remove rule"; diff --git a/webui/language/es/messages.php b/webui/language/es/messages.php index fb7b5070..cc230f9c 100644 --- a/webui/language/es/messages.php +++ b/webui/language/es/messages.php @@ -498,3 +498,4 @@ $_['text_no_selected_message'] = "no selected message"; $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; +$_['text_remove_request'] = "remove request"; diff --git a/webui/language/fr/messages.php b/webui/language/fr/messages.php index 739480e0..41b74b4f 100644 --- a/webui/language/fr/messages.php +++ b/webui/language/fr/messages.php @@ -495,3 +495,4 @@ $_['text_no_selected_message'] = "no selected message"; $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; +$_['text_remove_request'] = "remove request"; diff --git a/webui/language/hu/messages.php b/webui/language/hu/messages.php index 48d2888c..d6e70f9c 100644 --- a/webui/language/hu/messages.php +++ b/webui/language/hu/messages.php @@ -296,6 +296,7 @@ $_['text_remove'] = "Törlés"; $_['text_remove_domain'] = "Domain törlése"; $_['text_remove_message'] = "Levél törlése"; $_['text_remove_message2'] = "levél törlése"; +$_['text_remove_request'] = "törlésre jelölve"; $_['text_remove_selected_uids'] = "Kijelölt azonosítók törlése"; $_['text_remove_policy'] = "Házirend törlése"; $_['text_remove_rule'] = "Szabály törlése"; diff --git a/webui/language/pl/messages.php b/webui/language/pl/messages.php index 47b9a76b..515c5880 100644 --- a/webui/language/pl/messages.php +++ b/webui/language/pl/messages.php @@ -494,3 +494,4 @@ $_['text_user_data_officer'] = "Data officer"; $_['text_no_selected_message'] = "no selected message"; $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; +$_['text_remove_request'] = "remove request"; diff --git a/webui/language/pt/messages.php b/webui/language/pt/messages.php index 92a38692..b6f6896d 100644 --- a/webui/language/pt/messages.php +++ b/webui/language/pt/messages.php @@ -486,3 +486,5 @@ $_['text_user_data_officer'] = "Data officer"; $_['text_no_selected_message'] = "no selected message"; $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; +$_['text_remove_request'] = "remove request"; + diff --git a/webui/language/ru/messages.php b/webui/language/ru/messages.php index 97225783..0efcd96e 100644 --- a/webui/language/ru/messages.php +++ b/webui/language/ru/messages.php @@ -495,3 +495,5 @@ $_['text_user_data_officer'] = "Data officer"; $_['text_no_selected_message'] = "no selected message"; $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; +$_['text_remove_request'] = "remove request"; + diff --git a/webui/language/tr/messages.php b/webui/language/tr/messages.php index 1ff85585..83163588 100644 --- a/webui/language/tr/messages.php +++ b/webui/language/tr/messages.php @@ -495,3 +495,5 @@ $_['text_user_data_officer'] = "Data officer"; $_['text_no_selected_message'] = "no selected message"; $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; +$_['text_remove_request'] = "remove request"; + From 391005247c576c04fb04e515918e62a9f5a7426c Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 26 Apr 2021 06:10:08 +0200 Subject: [PATCH 093/106] Added a red "R" sign to the messages been marked for removal Signed-off-by: Janos SUTO --- webui/model/search/search.php | 11 ++++++++--- webui/view/theme/default/templates/search/helper.tpl | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/webui/model/search/search.php b/webui/model/search/search.php index dbcf5b5b..5bb3e03a 100644 --- a/webui/model/search/search.php +++ b/webui/model/search/search.php @@ -473,6 +473,7 @@ class ModelSearchSearch extends Model { $note = array(); $private = []; $deleted = []; + $marked_for_removal = []; $q = ''; global $SUPPRESS_RECIPIENTS; @@ -509,10 +510,13 @@ class ModelSearchSearch extends Model { } if(ENABLE_DELETE) { - $s = $this->db->query("SELECT `id` FROM `" . TABLE_DELETED . "` WHERE deleted=1 AND id IN ($q)", $ids); - + $s = $this->db->query("SELECT `id`, `deleted` FROM `" . TABLE_DELETED . "` WHERE id IN ($q)", $ids); foreach ($s->rows as $p) { - $deleted[$p['id']] = 1; + if($p['id'] == 1) { + $deleted[$p['id']] = 1; + } else { + $marked_for_removal[$p['id']] = 1; + } } } @@ -536,6 +540,7 @@ class ModelSearchSearch extends Model { foreach($query->rows as $m) { // We mark it as deleted even if it's only marked for removal if(ENABLE_DELETE == 1 && ($m['retained'] < NOW || isset($deleted[$m['id']])) ) $m['deleted'] = 1; else $m['deleted'] = 0; + if(ENABLE_DELETE == 1 && isset($marked_for_removal[$m['id']])) $m['marked_for_removal'] = 1; else $m['marked_for_removal'] = 0; $m['shortfrom'] = make_short_string($m['from'], MAX_CGI_FROM_SUBJ_LEN); $m['from'] = escape_gt_lt_quote_symbols($m['from']); diff --git a/webui/view/theme/default/templates/search/helper.tpl b/webui/view/theme/default/templates/search/helper.tpl index 0252d44c..f02d19ed 100644 --- a/webui/view/theme/default/templates/search/helper.tpl +++ b/webui/view/theme/default/templates/search/helper.tpl @@ -51,7 +51,7 @@ 1) { ?>">  - class="xxx"> class="xxx" title="" onclick="$('#ref').val(''); Piler.expert(this);">[+] P + class="xxx"> class="xxx" title="" onclick="$('#ref').val(''); Piler.expert(this);">[+] P R   From 007e244b7bdc979fe8220d031bb9ebdc4d731822 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 26 Apr 2021 06:11:30 +0200 Subject: [PATCH 094/106] Add "R" even if the message is already deleted by data officer Signed-off-by: Janos SUTO --- webui/model/search/search.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/webui/model/search/search.php b/webui/model/search/search.php index 5bb3e03a..ab34226b 100644 --- a/webui/model/search/search.php +++ b/webui/model/search/search.php @@ -514,9 +514,8 @@ class ModelSearchSearch extends Model { foreach ($s->rows as $p) { if($p['id'] == 1) { $deleted[$p['id']] = 1; - } else { - $marked_for_removal[$p['id']] = 1; } + $marked_for_removal[$p['id']] = 1; } } From 6c68a25b63cf583a8cdb3a2b0321e82933d2b8ec Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 26 Apr 2021 06:18:10 +0200 Subject: [PATCH 095/106] Fixed reject button text for data officer Signed-off-by: Janos SUTO --- .../theme/default/templates/common/layout-audit-removal.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui/view/theme/default/templates/common/layout-audit-removal.tpl b/webui/view/theme/default/templates/common/layout-audit-removal.tpl index a1db8a0c..c184e75c 100644 --- a/webui/view/theme/default/templates/common/layout-audit-removal.tpl +++ b/webui/view/theme/default/templates/common/layout-audit-removal.tpl @@ -60,7 +60,7 @@
- +
From 98b4c3c3871d8bf7e84807248532f91dc641b229 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 26 Apr 2021 06:30:14 +0200 Subject: [PATCH 096/106] Create an audit entry in case of rejecting the removal of the message Signed-off-by: Janos SUTO --- config.php.in | 5 ++++- webui/controller/audit/helper.php | 1 + webui/controller/message/rejectremove.php | 2 ++ webui/language/cz/messages.php | 1 + webui/language/de/messages.php | 1 + webui/language/en/messages.php | 1 + webui/language/es/messages.php | 1 + webui/language/fr/messages.php | 1 + webui/language/hu/messages.php | 1 + webui/language/pl/messages.php | 1 + webui/language/pt/messages.php | 1 + webui/language/ru/messages.php | 1 + webui/language/tr/messages.php | 1 + 13 files changed, 17 insertions(+), 1 deletion(-) diff --git a/config.php.in b/config.php.in index 710dda4b..3ce7471e 100644 --- a/config.php.in +++ b/config.php.in @@ -495,6 +495,7 @@ define('ACTION_VIEW_JOURNAL', 17); define('ACTION_NOT_SPAM', 18); define('ACTION_MARK_AS_PRIVATE', 19); define('ACTION_MARK_MESSAGE_FOR_REMOVAL', 20); +define('ACTION_REJECT_REMOVAL', 21); $actions = array( 'unknown' => 1, @@ -509,7 +510,9 @@ $actions = array( 'save_search' => 11, 'download_attachment' => 15, 'journal' => 17, - 'private' => 19 + 'private' => 19, + 'marked_for_removal', 20, + 'reject_removal', 21 ); diff --git a/webui/controller/audit/helper.php b/webui/controller/audit/helper.php index af248ce8..053b5636 100644 --- a/webui/controller/audit/helper.php +++ b/webui/controller/audit/helper.php @@ -66,6 +66,7 @@ class ControllerAuditHelper extends Controller { $this->data['actions'][ACTION_VIEW_JOURNAL] = $this->data['text_view_journal']; $this->data['actions'][ACTION_MARK_MESSAGE_FOR_REMOVAL] = $this->data['text_remove_request']; $this->data['actions'][ACTION_MARK_AS_PRIVATE] = $this->data['text_mark_private']; + $this->data['actions'][ACTION_REJECT_REMOVAL] = $this->data['text_reject_removal']; diff --git a/webui/controller/message/rejectremove.php b/webui/controller/message/rejectremove.php index 1e47b8d2..12208607 100644 --- a/webui/controller/message/rejectremove.php +++ b/webui/controller/message/rejectremove.php @@ -37,6 +37,8 @@ class ControllerMessageRejectRemove extends Controller { // Shouldn't we ask for a token or something as well? + AUDIT(ACTION_REJECT_REMOVAL, '', '', $id, ''); + $db->query("UPDATE " . TABLE_DELETED . " SET deleted=0, approver=?, date2=?, reason2=? WHERE id=?", [$this->data['username'], NOW, $this->request->post['reason2'], $id]); syslog(LOG_INFO, $this->data['username'] . " rejected removing message: $id"); diff --git a/webui/language/cz/messages.php b/webui/language/cz/messages.php index 86385d15..b215459d 100644 --- a/webui/language/cz/messages.php +++ b/webui/language/cz/messages.php @@ -494,3 +494,4 @@ $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; $_['text_remove_request'] = "remove request"; +$_['text_rejected_removal'] = "rejected removal"; diff --git a/webui/language/de/messages.php b/webui/language/de/messages.php index cbed91ad..c18e4738 100644 --- a/webui/language/de/messages.php +++ b/webui/language/de/messages.php @@ -501,3 +501,4 @@ $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; $_['text_remove_request'] = "remove request"; +$_['text_rejected_removal'] = "rejected removal"; diff --git a/webui/language/en/messages.php b/webui/language/en/messages.php index c361427d..e88f048e 100644 --- a/webui/language/en/messages.php +++ b/webui/language/en/messages.php @@ -288,6 +288,7 @@ $_['text_ref'] = "Reference"; $_['text_refresh_period'] = "Refresh period"; $_['text_refresh_qr_code'] = "Refresh QR code"; $_['text_reject'] = "Reject"; +$_['text_rejected_removal'] = "rejected removal"; $_['text_reason_of_rejection'] = "Reason of rejection"; $_['text_relay_details'] = "Relay details"; $_['text_relay_status'] = "Relay status"; diff --git a/webui/language/es/messages.php b/webui/language/es/messages.php index cc230f9c..dba25b13 100644 --- a/webui/language/es/messages.php +++ b/webui/language/es/messages.php @@ -499,3 +499,4 @@ $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; $_['text_remove_request'] = "remove request"; +$_['text_rejected_removal'] = "rejected removal"; diff --git a/webui/language/fr/messages.php b/webui/language/fr/messages.php index 41b74b4f..7f404f30 100644 --- a/webui/language/fr/messages.php +++ b/webui/language/fr/messages.php @@ -496,3 +496,4 @@ $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; $_['text_remove_request'] = "remove request"; +$_['text_rejected_removal'] = "rejected removal"; diff --git a/webui/language/hu/messages.php b/webui/language/hu/messages.php index d6e70f9c..ef3c7474 100644 --- a/webui/language/hu/messages.php +++ b/webui/language/hu/messages.php @@ -290,6 +290,7 @@ $_['text_ref'] = "Hivatkozás"; $_['text_refresh_period'] = "Frissítési periódus"; $_['text_refresh_qr_code'] = "QR kód frissítése"; $_['text_reject'] = "Elutasítás"; +$_['text_rejected_removal'] = "törlés elutasítva"; $_['text_relay_details'] = "Relay részletek"; $_['text_relay_status'] = "Relay státusz"; $_['text_remove'] = "Törlés"; diff --git a/webui/language/pl/messages.php b/webui/language/pl/messages.php index 515c5880..1ab3e5c0 100644 --- a/webui/language/pl/messages.php +++ b/webui/language/pl/messages.php @@ -495,3 +495,4 @@ $_['text_no_selected_message'] = "no selected message"; $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; $_['text_remove_request'] = "remove request"; +$_['text_rejected_removal'] = "rejected removal"; diff --git a/webui/language/pt/messages.php b/webui/language/pt/messages.php index b6f6896d..7aaa6873 100644 --- a/webui/language/pt/messages.php +++ b/webui/language/pt/messages.php @@ -488,3 +488,4 @@ $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; $_['text_remove_request'] = "remove request"; +$_['text_rejected_removal'] = "rejected removal"; diff --git a/webui/language/ru/messages.php b/webui/language/ru/messages.php index 0efcd96e..3ec67ed2 100644 --- a/webui/language/ru/messages.php +++ b/webui/language/ru/messages.php @@ -497,3 +497,4 @@ $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; $_['text_remove_request'] = "remove request"; +$_['text_rejected_removal'] = "rejected removal"; diff --git a/webui/language/tr/messages.php b/webui/language/tr/messages.php index 83163588..fc80d340 100644 --- a/webui/language/tr/messages.php +++ b/webui/language/tr/messages.php @@ -497,3 +497,4 @@ $_['text_create_note'] = "Create note"; $_['text_wildcard_domains'] = "Wildcard domains"; $_['text_remove_request'] = "remove request"; +$_['text_rejected_removal'] = "rejected removal"; From 7b1028d9281002f25a3c704485fdc829a4802268 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 26 Apr 2021 06:44:41 +0200 Subject: [PATCH 097/106] Fixed text for rejected removal Signed-off-by: Janos SUTO --- webui/controller/audit/helper.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui/controller/audit/helper.php b/webui/controller/audit/helper.php index 053b5636..774274ec 100644 --- a/webui/controller/audit/helper.php +++ b/webui/controller/audit/helper.php @@ -66,7 +66,7 @@ class ControllerAuditHelper extends Controller { $this->data['actions'][ACTION_VIEW_JOURNAL] = $this->data['text_view_journal']; $this->data['actions'][ACTION_MARK_MESSAGE_FOR_REMOVAL] = $this->data['text_remove_request']; $this->data['actions'][ACTION_MARK_AS_PRIVATE] = $this->data['text_mark_private']; - $this->data['actions'][ACTION_REJECT_REMOVAL] = $this->data['text_reject_removal']; + $this->data['actions'][ACTION_REJECT_REMOVAL] = $this->data['text_rejected_removal']; From a8f037c3e9ee1619978c6201e243e81e6ef5accd Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sat, 1 May 2021 07:51:46 +0200 Subject: [PATCH 098/106] Introduced convert_date_string_to_ymd_by_template() function Signed-off-by: Janos SUTO --- unit_tests/php/FormatTest.php | 25 +++++++++++++++++++++++++ webui/system/misc.php | 31 ++++++++++++++++++++++--------- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/unit_tests/php/FormatTest.php b/unit_tests/php/FormatTest.php index 3e96c32b..9b2e85fc 100644 --- a/unit_tests/php/FormatTest.php +++ b/unit_tests/php/FormatTest.php @@ -3,6 +3,7 @@ use PHPUnit\Framework\TestCase; include_once("webui/system/model.php"); +include_once("webui/system/misc.php"); include_once("webui/model/health/health.php"); final class FormatTest extends TestCase @@ -25,4 +26,28 @@ final class FormatTest extends TestCase $this->assertEquals($result, $expected_result); } + + public function providerTestConvertDateStringToYmdByTemplateValues() { + return [ + ['2021.05.01', 'y.m.d', ['0','0','0']], + ['2021.05.01', 'Y.m', ['0','0','0']], + ['2021.05.01', 'Y.m.d.e', ['0','0','0']], + ['2021.05.01', 'Y.m.d', ['2021','05','01']], + ['2021.05.01', 'Y.m.d.', ['0','0','0']], + ['2021.05.01', 'Y-m-d', ['2021','05','01']], + ['12/01/2008', 'm/d/Y', ['2008','12','01']], + ['12-01-2008', 'm-d-Y', ['2008','12','01']], + ]; + } + + /** + * @dataProvider providerTestConvertDateStringToYmdByTemplateValues + */ + + public function test_convert_date_string_to_ymd_by_template($date, $date_template, $expected_result) { + $result = convert_date_string_to_ymd_by_template($date, $date_template); + $this->assertEquals($result, $expected_result); + } + + } diff --git a/webui/system/misc.php b/webui/system/misc.php index 0056eaa1..ae356ed2 100644 --- a/webui/system/misc.php +++ b/webui/system/misc.php @@ -436,6 +436,26 @@ function fetch_url($url = '') { } +function convert_date_string_to_ymd_by_template($date_string, $date_template) { + $Y = $m = $d = 0; + + $s = $template_array = preg_split("/(\.|\-|\/)/", $date_template); + sort($s); + + $date_array = preg_split("/(\.|\-|\/)/", $date_string); + + if($s != ['Y','d','m'] || count($template_array) != 3 || count($date_array) != 3) { + return [$Y, $m, $d]; + } + + while(list($k, $v) = each($template_array)) { + $$v = $date_array[$k]; + } + + return [$Y, $m, $d]; +} + + function fixup_date_condition($field = '', $date1 = 0, $date2 = 0) { global $session; @@ -449,12 +469,7 @@ function fixup_date_condition($field = '', $date1 = 0, $date2 = 0) { if($date1) { - list($y,$m,$d) = preg_split("/(\.|\-|\/)/", $date1); - - if(DATE_TEMPLATE == 'd/m/Y') { $a = $y; $y = $d; $d = $a; } - - if($m == '*') { $m = 0; } - if($d == '*') { $d = 0; } + list($y,$m,$d) = convert_date_string_to_ymd_by_template($date1, DATE_TEMPLATE); $date1 = mktime(0, 0, 0, $m, $d, $y); @@ -462,9 +477,7 @@ function fixup_date_condition($field = '', $date1 = 0, $date2 = 0) { } if($date2) { - list($y,$m,$d) = preg_split("/(\.|\-|\/)/", $date2); - - if(DATE_TEMPLATE == 'd/m/Y') { $a = $y; $y = $d; $d = $a; } + list($y,$m,$d) = convert_date_string_to_ymd_by_template($date2, DATE_TEMPLATE); $date2 = mktime(23, 59, 59, $m, $d, $y); From dacb7c85aa7654c70f97c9bd4dc80a52913d842a Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 2 May 2021 08:19:09 +0200 Subject: [PATCH 099/106] Minor gui fixes Signed-off-by: Janos SUTO --- webui/system/misc.php | 31 ++++++++++++++++++- .../theme/default/templates/search/helper.tpl | 8 ++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/webui/system/misc.php b/webui/system/misc.php index ae356ed2..ca9598e5 100644 --- a/webui/system/misc.php +++ b/webui/system/misc.php @@ -1,5 +1,10 @@ $length ? substr($what, 0, $length) . "..." : $what; + if($length < 1) { return ''; } + + if(strlen($what) <= $length) { return $what; } + + $arr = preg_split("/\s/", $what); + $s = ''; + + $i = 0; + foreach($arr as $a) { + if($i == 0) { + if($length > 0 && strlen($a) > $length) { + return substr($a, 0, $length) . '...'; + } + } + + if(strlen($s) + strlen($a) <= $length) { + $s .= $a . ' '; + } else { + break; + } + + $i++; + } + + return $s . '...'; } diff --git a/webui/view/theme/default/templates/search/helper.tpl b/webui/view/theme/default/templates/search/helper.tpl index f02d19ed..dfeaec96 100644 --- a/webui/view/theme/default/templates/search/helper.tpl +++ b/webui/view/theme/default/templates/search/helper.tpl @@ -43,7 +43,7 @@ - + checked="checked" class="restorebox" /> @@ -67,7 +67,7 @@ - + checked="checked" class="restorebox" onclick="Piler.toggle_bulk_check('2');" /> @@ -106,7 +106,7 @@ - + @@ -135,7 +135,7 @@   get("sphx_query")) { ?> - ', 5);">sphinx + ', 5);">sphinx From c0fa39b992dda79f9079a6774fe61d98fc3c6ee4 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 2 May 2021 08:25:40 +0200 Subject: [PATCH 100/106] Fixed typo in misc.php Signed-off-by: Janos SUTO --- webui/system/misc.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webui/system/misc.php b/webui/system/misc.php index ca9598e5..5c35be15 100644 --- a/webui/system/misc.php +++ b/webui/system/misc.php @@ -1,6 +1,6 @@ Date: Sun, 2 May 2021 14:01:37 +0200 Subject: [PATCH 101/106] Introduced the raw: search label for auditors Signed-off-by: Janos SUTO --- RELEASE_NOTES | 1 + webui/controller/search/helper.php | 3 ++- webui/model/search/search.php | 26 +++++++++++--------------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 19a18d6b..438e8a93 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,4 +1,5 @@ - Introduced the archive_address feature, see etc/example.conf for the details +- Introduced the raw: search label 1.3.12: ------- diff --git a/webui/controller/search/helper.php b/webui/controller/search/helper.php index fc5775a7..70498a0c 100644 --- a/webui/controller/search/helper.php +++ b/webui/controller/search/helper.php @@ -15,6 +15,7 @@ class ControllerSearchHelper extends Controller { 'folders' => '', 'extra_folders' => '', 'id' => '', + 'raw' => '', 'match' => array() ); @@ -49,7 +50,7 @@ class ControllerSearchHelper extends Controller { if($this->request->post['searchtype'] == 'expert'){ - if(isset($this->request->post['search']) && preg_match("/(from|to|subject|body|direction|d|size|date1|date2|attachment|a|tag|note|id)\:/", $this->request->post['search'])) { + if(isset($this->request->post['search']) && preg_match("/(from|to|subject|body|direction|d|size|date1|date2|attachment|a|tag|note|id|raw)\:/", $this->request->post['search'])) { $this->a = $this->model_search_search->preprocess_post_expert_request($this->request->post); } else { diff --git a/webui/model/search/search.php b/webui/model/search/search.php index ab34226b..ad4afd8a 100644 --- a/webui/model/search/search.php +++ b/webui/model/search/search.php @@ -112,7 +112,6 @@ class ModelSearchSearch extends Model { $session = Registry::get('session'); - $i = 0; while(list($k, $v) = each($data['match'])) { if($v == "@attachment_types") { @@ -163,6 +162,10 @@ class ModelSearchSearch extends Model { $match = implode(" ", $data['match']); + if(Registry::get('auditor_user') == 1 && RESTRICTED_AUDITOR == 0 && $data['raw']) { + $match .= $data['raw']; + } + if($emailfilter) { if(strlen($match) > 2) { $match = "( $match ) & $emailfilter"; } else { $match = $emailfilter; } @@ -327,6 +330,7 @@ class ModelSearchSearch extends Model { 'folders' => '', 'extra_folders' => '', 'id' => '', + 'raw' => '', 'match' => array() ); @@ -351,15 +355,12 @@ class ModelSearchSearch extends Model { else if($v == 'subject:') { $token = 'match'; $a['match'][] = '@subject'; continue; } else if($v == 'body:') { $token = 'match'; $a['match'][] = '@body'; continue; } else if($v == 'direction:' || $v == 'd:') { $token = 'direction'; continue; } - else if($v == 'size:') { $token = 'size'; continue; } - else if($v == 'date1:') { $token = 'date1'; continue; } - else if($v == 'date2:') { $token = 'date2'; continue; } else if($v == 'attachment:' || $v == 'a:') { $token = 'match'; $a['match'][] = '@attachment_types'; continue; } - else if($v == 'size') { $token = 'size'; continue; } - else if($v == 'tag:') { $token = 'tag'; continue; } - else if($v == 'note:') { $token = 'note'; continue; } - else if($v == 'ref:') { $token = 'ref'; continue; } - else if($v == 'id:') { $token = 'id'; continue; } + + else if(in_array($v, ['size:', 'date1:', 'date2:', 'tag:', 'note:', 'ref:', 'id:', 'raw:'])) { + $token = substr($v, 0, strlen($v)-1); continue; + } + else if($token != 'date1' && $token != 'date2') { if(preg_match("/\d{4}\-\d{1,2}\-\d{1,2}/", $v) || preg_match("/\d{1,2}\/\d{1,2}\/\d{4}/", $v)) { $ndate++; @@ -368,12 +369,7 @@ class ModelSearchSearch extends Model { } if($token == 'match') { $a['match'][] = $v; } - else if($token == 'date1') { $a['date1'] = ' ' . $v; } - else if($token == 'date2') { $a['date2'] = ' ' . $v; } - else if($token == 'tag') { $a['tag'] .= ' ' . $v; } - else if($token == 'note') { $a['note'] .= ' ' . $v; } - else if($token == 'ref') { $a['ref'] = ' ' . $v; } - else if($token == 'id') { $a['id'] .= ' ' . $v; } + else if(in_array($token, ['date1', 'date2', 'ref', 'tag', 'note', 'id', 'raw'])) { $a[$token] .= ' ' . $v; } else if($token == 'direction') { if($v == 'inbound') { $a['direction'] = "0"; } From f2683962333a741c64920d959bc12e23ed797aa4 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 2 May 2021 14:24:45 +0200 Subject: [PATCH 102/106] pilerexport dryrun prints the total number of attachments as well Signed-off-by: Janos SUTO --- src/pilerexport.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/pilerexport.c b/src/pilerexport.c index 5ee47c86..a1f77189 100644 --- a/src/pilerexport.c +++ b/src/pilerexport.c @@ -170,7 +170,7 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char * if(!where_condition) return id; - snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE id IN (", SQL_METADATA_TABLE); + snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE id IN (", SQL_METADATA_TABLE); rc += append_string_to_buffer(&query, s); snprintf(s, sizeof(s)-1, "SELECT id FROM %s WHERE %s AND id > %llu ORDER BY id ASC LIMIT 0,%d", index_list, where_condition, last_id, max_matches); @@ -244,13 +244,13 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain int rc=0; if(exportall == 1){ - rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM "); + rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM "); rc += append_string_to_buffer(&query, SQL_METADATA_TABLE); rc += append_string_to_buffer(&query, " WHERE deleted=0 "); return rc; } - snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE deleted=0 ", SQL_MESSAGES_VIEW); + snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE deleted=0 ", SQL_MESSAGES_VIEW); rc = append_string_to_buffer(&query, s); @@ -356,7 +356,8 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct char digest[SMALLBUFSIZE], bodydigest[SMALLBUFSIZE]; char filename[SMALLBUFSIZE]; struct sql sql; - int errorp, rc=0; + int errorp, rc=0, attachments; + unsigned long total_attachments=0; if(prepare_sql_statement(sdata, &sql, s) == ERR) return ERR; @@ -373,6 +374,7 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++; sql.sql[sql.pos] = &digest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(digest)-2; sql.pos++; sql.sql[sql.pos] = &bodydigest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(bodydigest)-2; sql.pos++; + sql.sql[sql.pos] = (char *)&attachments; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++; p_store_results(&sql); @@ -445,6 +447,7 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct else printf("cannot open: %s\n", filename); } else { + total_attachments += attachments; printf("id:%llu\n", id); } @@ -457,6 +460,10 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct ENDE: close_prepared_statement(&sql); + if(dryrun){ + printf("attachments: %lu\n", total_attachments); + } + printf("\n"); return rc; From 40e4dd48444127579fc287434cec01c44bea48f0 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 2 May 2021 21:11:36 +0200 Subject: [PATCH 103/106] Updated security.md file Signed-off-by: Janos SUTO --- SECURITY.md | 1 + 1 file changed, 1 insertion(+) diff --git a/SECURITY.md b/SECURITY.md index f185d9de..53827cb7 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -17,3 +17,4 @@ Security configurations - Use https for the GUI - Reset the default passwords for admin and auditor + - Use the smtp acl feature to restrict SMTP access to the archive, see https://mailpiler.com/smtp-acl-list/ From 66a8f60d9d1ced70fe46d6e76041989e7b779f0f Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Tue, 4 May 2021 05:37:02 +0200 Subject: [PATCH 104/106] Print the sphinx query to stdout in dry run Signed-off-by: Janos SUTO --- src/pilerexport.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/pilerexport.c b/src/pilerexport.c index a1f77189..8eca2b05 100644 --- a/src/pilerexport.c +++ b/src/pilerexport.c @@ -163,7 +163,7 @@ int append_string_to_buffer(char **buffer, char *str){ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *where_condition, uint64 last_id, int *num, struct config *cfg){ MYSQL_ROW row; uint64 id=0; - char s[SMALLBUFSIZE]; + char s[MAXBUFSIZE]; int rc=0; *num = 0; @@ -175,6 +175,10 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char * snprintf(s, sizeof(s)-1, "SELECT id FROM %s WHERE %s AND id > %llu ORDER BY id ASC LIMIT 0,%d", index_list, where_condition, last_id, max_matches); + if(dryrun){ + printf("sphinx query: %s\n", s); + } + syslog(LOG_PRIORITY, "sphinx query: %s", s); if(mysql_real_query(&(sdata2->mysql), s, strlen(s)) == 0){ From 6a2d41f543192fbdd539de8cf5a4cef533064fdd Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 17 May 2021 06:32:00 +0200 Subject: [PATCH 105/106] Added aaa.fu to test domans Signed-off-by: Janos SUTO --- tests/setup.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/setup.inc b/tests/setup.inc index 7ea3b5da..421b5cc3 100644 --- a/tests/setup.inc +++ b/tests/setup.inc @@ -33,7 +33,7 @@ launch_containers() { create_rules() { local container="$1" - echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"acts.hu\",\"acts.hu\"),(\"gtsce.com\",\"gtsce.com\"),(\"datanet.hu\",\"datanet.hu\"),(\"gtsdatanet.hu\",\"gtsdatanet.hu\"),(\"gts.hu\",\"gts.hu\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh + echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"acts.hu\",\"acts.hu\"),(\"gtsce.com\",\"gtsce.com\"),(\"datanet.hu\",\"datanet.hu\"),(\"gtsdatanet.hu\",\"gtsdatanet.hu\"),(\"gts.hu\",\"gts.hu\"),(\"aaa.fu\",\"aaa.fu\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh echo 'echo "insert into archiving_rule (subject) values (\"Android táblagép\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh echo 'echo "insert into archiving_rule (\`from\`) values (\"@gmail.com\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh From 63b69817e973b478e60c5811f73a9576509ae2aa Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Wed, 26 May 2021 19:58:16 +0200 Subject: [PATCH 106/106] Removed obsoleted ldap_port variable from model/user/auth.php Signed-off-by: Janos SUTO --- webui/model/user/auth.php | 1 - 1 file changed, 1 deletion(-) diff --git a/webui/model/user/auth.php b/webui/model/user/auth.php index a51b2929..26af4a39 100644 --- a/webui/model/user/auth.php +++ b/webui/model/user/auth.php @@ -180,7 +180,6 @@ class ModelUserAuth extends Model { $ldap_type = ''; $ldap_host = LDAP_HOST; - $ldap_port = LDAP_PORT; $ldap_base_dn = LDAP_BASE_DN; $ldap_helper_dn = LDAP_HELPER_DN; $ldap_helper_password = LDAP_HELPER_PASSWORD;