diff --git a/docker/Dockerfile b/docker/Dockerfile index e52adc52..baf3fd80 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 ARG PACKAGE @@ -17,13 +17,15 @@ ENV DEBIAN_FRONTEND="noninteractive" \ SPHINX_BIN_TARGZ="sphinx-3.1.1-bin.tar.gz" ADD "https://bitbucket.org/jsuto/piler/downloads/${PACKAGE}" "/${PACKAGE}" -ADD start.sh /start.sh +COPY start.sh /start.sh RUN apt-get update && \ apt-get -y --no-install-recommends install \ wget rsyslog openssl sysstat php7.2-cli php7.2-cgi php7.2-mysql php7.2-fpm php7.2-zip php7.2-ldap \ php7.2-gd php7.2-curl php7.2-xml catdoc unrtf poppler-utils nginx tnef sudo libodbc1 libpq5 libzip4 \ libtre5 libwrap0 cron libmariadb3 libmysqlclient-dev python python-mysqldb mariadb-server && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ service mysql start && mysqladmin -u root password ${MYSQL_ROOT_PASSWORD} && \ wget --no-check-certificate -q -O ${SPHINX_BIN_TARGZ} ${DOWNLOAD_URL}/generic-local/${SPHINX_BIN_TARGZ} && \ tar zxvf ${SPHINX_BIN_TARGZ} && \ @@ -31,8 +33,8 @@ RUN apt-get update && \ sed -i 's/mail.[iwe].*//' /etc/rsyslog.conf && \ sed -i '/session required pam_loginuid.so/c\#session required pam_loginuid.so' /etc/pam.d/cron && \ mkdir /etc/piler && \ - printf "[mysql]\nuser = piler\npassword = ${MYSQL_PILER_PASSWORD}\n" > /etc/piler/.my.cnf && \ - printf "[mysql]\nuser = root\npassword = ${MYSQL_ROOT_PASSWORD}\n" > /root/.my.cnf && \ + printf "[mysql]\nuser = piler\npassword = %s\n" ${MYSQL_PILER_PASSWORD} > /etc/piler/.my.cnf && \ + printf "[mysql]\nuser = root\npassword = %s\n" ${MYSQL_ROOT_PASSWORD} > /root/.my.cnf && \ echo "alias mysql='mysql --defaults-file=/etc/piler/.my.cnf'" > /root/.bashrc && \ echo "alias t='tail -f /var/log/syslog'" >> /root/.bashrc && \ dpkg -i $PACKAGE && \ diff --git a/src/pilerexport.c b/src/pilerexport.c index 1b8de98a..84362de6 100644 --- a/src/pilerexport.c +++ b/src/pilerexport.c @@ -22,6 +22,7 @@ extern int optind; int dryrun = 0; int exportall = 0; +int verification_status = 0; int rc = 0; char *query=NULL; int verbosity = 0; @@ -224,7 +225,6 @@ void export_emails_matching_id_list(struct session_data *sdata, struct session_d int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain, int minsize, int maxsize, unsigned long startdate, unsigned long stopdate){ - int where_condition=1; char s[SMALLBUFSIZE]; if(exportall == 1){ @@ -239,83 +239,69 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain rc = append_string_to_buffer(&query, s); if(from){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); rc += append_string_to_buffer(&query, "`from` IN ("); rc += append_string_to_buffer(&query, from); rc += append_string_to_buffer(&query, ")"); free(from); - - where_condition++; } if(to){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); rc += append_string_to_buffer(&query, "`to` IN ("); rc += append_string_to_buffer(&query, to); rc += append_string_to_buffer(&query, ")"); free(to); - - where_condition++; } if(fromdomain){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); rc += append_string_to_buffer(&query, "`fromdomain` IN ("); rc += append_string_to_buffer(&query, fromdomain); rc += append_string_to_buffer(&query, ")"); free(fromdomain); - - where_condition++; } if(todomain){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); rc += append_string_to_buffer(&query, "`todomain` IN ("); rc += append_string_to_buffer(&query, todomain); rc += append_string_to_buffer(&query, ")"); free(todomain); - - where_condition++; } if(minsize > 0){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); snprintf(s, sizeof(s)-1, " `size` >= %d", minsize); rc += append_string_to_buffer(&query, s); - - where_condition++; } if(maxsize > 0){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); snprintf(s, sizeof(s)-1, " `size` <= %d", maxsize); rc += append_string_to_buffer(&query, s); - - where_condition++; } if(startdate > 0){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); snprintf(s, sizeof(s)-1, " `sent` >= %lu", startdate); rc += append_string_to_buffer(&query, s); - - where_condition++; } if(stopdate > 0){ - if(where_condition) rc = append_string_to_buffer(&query, " AND "); + rc = append_string_to_buffer(&query, " AND "); snprintf(s, sizeof(s)-1, " `sent` <= %lu", stopdate); rc += append_string_to_buffer(&query, s); } @@ -374,9 +360,10 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0){ printf("exported: %10llu\r", n); fflush(stdout); } - else + else { printf("verification FAILED. %s\n", filename); - + verification_status = 1; + } } else printf("cannot open: %s\n", filename); } @@ -593,5 +580,5 @@ int main(int argc, char **argv){ close_database(&sdata); - return 0; + return verification_status; } diff --git a/src/reindex.c b/src/reindex.c index 2382d9f5..ad04b094 100644 --- a/src/reindex.c +++ b/src/reindex.c @@ -107,45 +107,43 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da while(p_fetch_results(&sql) == OK){ - if(stored_id > 0){ - char filename[SMALLBUFSIZE]; - snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id); + char filename[SMALLBUFSIZE]; + snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id); - FILE *f = fopen(filename, "w"); - if(f){ - int rc = retrieve_email_from_archive(sdata, f, cfg); - fclose(f); - - if(rc){ - printf("cannot retrieve: %s\n", filename); - unlink(filename); - continue; - } - - snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename); - - state = parse_message(sdata, 0, data, cfg); - post_parse(sdata, &state, cfg); - - rc = store_index_data(sdata, &state, data, stored_id, cfg); - - if(rc == OK) reindexed++; - else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename); + FILE *f = fopen(filename, "w"); + if(f){ + int rc = retrieve_email_from_archive(sdata, f, cfg); + fclose(f); + if(rc){ + printf("cannot retrieve: %s\n", filename); unlink(filename); - - if(progressbar){ - printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta)); - fflush(stdout); - } - + continue; + } + + snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename); + + state = parse_message(sdata, 0, data, cfg); + post_parse(sdata, &state, cfg); + + rc = store_index_data(sdata, &state, data, stored_id, cfg); + + if(rc == OK) reindexed++; + else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename); + + unlink(filename); + + if(progressbar){ + printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta)); + fflush(stdout); } - else printf("cannot open: %s\n", filename); } + else printf("cannot open: %s\n", filename); } + p_free_results(&sql); } diff --git a/src/rules.c b/src/rules.c index ec667420..37c8d3cf 100644 --- a/src/rules.c +++ b/src/rules.c @@ -240,7 +240,7 @@ char *check_against_ruleset(struct node *xhash[], struct parser_state *state, in if(q->str){ p = q->str; - if(p && count_match(p, state, size, spam) > 0){ + if(count_match(p, state, size, spam) > 0){ return p->rulestr; } } diff --git a/util/imapfetch.py b/util/imapfetch.py index ed9b7287..f9c98b3d 100755 --- a/util/imapfetch.py +++ b/util/imapfetch.py @@ -3,7 +3,6 @@ import argparse import imaplib -import pprint import re opts = {} diff --git a/util/import.sh b/util/import.sh index 102c0248..33528061 100755 --- a/util/import.sh +++ b/util/import.sh @@ -1,10 +1,13 @@ #!/bin/bash +set -o nounset +set -o errexit +set -o pipefail + export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin -PRIORITY=mail.error TMPFILE=/var/run/piler/import.tmp -if [ -f $TMPFILE ]; then exit 1; fi +if [[ -f $TMPFILE ]]; then exit 1; fi date > $TMPFILE @@ -17,5 +20,3 @@ trap finish EXIT cd /var/piler/imap pilerimport -G >/dev/null - - diff --git a/util/purge.sh b/util/purge.sh index 29ed8bad..7bb2bc20 100644 --- a/util/purge.sh +++ b/util/purge.sh @@ -1,11 +1,14 @@ #!/bin/bash +set -o nounset +set -o errexit +set -o pipefail + export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/libexec/piler:/usr/local/libexec/piler -PRIORITY=mail.error TMPFILE=/var/run/piler/purge.tmp PURGE_BEACON=/var/piler/stat/purge -if [ -f $TMPFILE ]; then exit 1; fi +if [[ -f $TMPFILE ]]; then exit 1; fi date > $TMPFILE