More refactoring

Signed-off-by: Janos SUTO <sj@acts.hu>

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 ARG PACKAGE
 
@@ -17,13 +17,15 @@ ENV DEBIAN_FRONTEND="noninteractive" \
     SPHINX_BIN_TARGZ="sphinx-3.1.1-bin.tar.gz"
 
 ADD "https://bitbucket.org/jsuto/piler/downloads/${PACKAGE}" "/${PACKAGE}"
-ADD start.sh /start.sh
+COPY start.sh /start.sh
 
 RUN apt-get update && \
     apt-get -y --no-install-recommends install \
        wget rsyslog openssl sysstat php7.2-cli php7.2-cgi php7.2-mysql php7.2-fpm php7.2-zip php7.2-ldap \
        php7.2-gd php7.2-curl php7.2-xml catdoc unrtf poppler-utils nginx tnef sudo libodbc1 libpq5 libzip4 \
        libtre5 libwrap0 cron libmariadb3 libmysqlclient-dev python python-mysqldb mariadb-server && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
     service mysql start && mysqladmin -u root password ${MYSQL_ROOT_PASSWORD} && \
     wget --no-check-certificate -q -O ${SPHINX_BIN_TARGZ} ${DOWNLOAD_URL}/generic-local/${SPHINX_BIN_TARGZ} && \
     tar zxvf ${SPHINX_BIN_TARGZ} && \
@@ -31,8 +33,8 @@ RUN apt-get update && \
     sed -i 's/mail.[iwe].*//' /etc/rsyslog.conf && \
     sed -i '/session    required     pam_loginuid.so/c\#session    required     pam_loginuid.so' /etc/pam.d/cron && \
     mkdir /etc/piler && \
-    printf "[mysql]\nuser = piler\npassword = ${MYSQL_PILER_PASSWORD}\n" > /etc/piler/.my.cnf && \
+    printf "[mysql]\nuser = piler\npassword = %s\n" ${MYSQL_PILER_PASSWORD} > /etc/piler/.my.cnf && \
-    printf "[mysql]\nuser = root\npassword = ${MYSQL_ROOT_PASSWORD}\n" > /root/.my.cnf && \
+    printf "[mysql]\nuser = root\npassword = %s\n" ${MYSQL_ROOT_PASSWORD} > /root/.my.cnf && \
     echo "alias mysql='mysql --defaults-file=/etc/piler/.my.cnf'" > /root/.bashrc && \
     echo "alias t='tail -f /var/log/syslog'" >> /root/.bashrc && \
     dpkg -i $PACKAGE && \

src/pilerexport.c

@@ -22,6 +22,7 @@ extern int optind;
 
 int dryrun = 0;
 int exportall = 0;
+int verification_status = 0;
 int rc = 0;
 char *query=NULL;
 int verbosity = 0;
@@ -224,7 +225,6 @@ void export_emails_matching_id_list(struct session_data *sdata, struct session_d
 
 
 int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain, int minsize, int maxsize, unsigned long startdate, unsigned long stopdate){
-   int where_condition=1;
    char s[SMALLBUFSIZE];
 
    if(exportall == 1){
@@ -239,83 +239,69 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain
    rc = append_string_to_buffer(&query, s);
 
    if(from){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`from` IN (");
       rc += append_string_to_buffer(&query, from);
       rc += append_string_to_buffer(&query, ")");
 
       free(from);
-
-      where_condition++;
    }
 
    if(to){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`to` IN (");
       rc += append_string_to_buffer(&query, to);
       rc += append_string_to_buffer(&query, ")");
 
       free(to);
-
-      where_condition++;
    }
 
    if(fromdomain){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`fromdomain` IN (");
       rc += append_string_to_buffer(&query, fromdomain);
       rc += append_string_to_buffer(&query, ")");
 
       free(fromdomain);
-
-      where_condition++;
    }
 
 
    if(todomain){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`todomain` IN (");
       rc += append_string_to_buffer(&query, todomain);
       rc += append_string_to_buffer(&query, ")");
 
       free(todomain);
-
-      where_condition++;
    }
 
    if(minsize > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
       snprintf(s, sizeof(s)-1, " `size` >= %d", minsize);
       rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
    }
 
 
    if(maxsize > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
       snprintf(s, sizeof(s)-1, " `size` <= %d", maxsize);
       rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
    }
 
 
    if(startdate > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
       snprintf(s, sizeof(s)-1, " `sent` >= %lu", startdate);
       rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
    }
 
 
    if(stopdate > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
       snprintf(s, sizeof(s)-1, " `sent` <= %lu", stopdate);
       rc += append_string_to_buffer(&query, s);
    }
@@ -374,9 +360,10 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct
                if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0){
                   printf("exported: %10llu\r", n); fflush(stdout);
                }
-               else
+               else {
                   printf("verification FAILED. %s\n", filename);
-
+                  verification_status = 1;
+               }
             }
             else printf("cannot open: %s\n", filename);
          }
@@ -593,5 +580,5 @@ int main(int argc, char **argv){
 
    close_database(&sdata);
 
-   return 0;
+   return verification_status;
 }

src/reindex.c

@@ -107,45 +107,43 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da
 
       while(p_fetch_results(&sql) == OK){
 
-         if(stored_id > 0){
+         char filename[SMALLBUFSIZE];
-            char filename[SMALLBUFSIZE];
+         snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id);
-            snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id);
 
-            FILE *f = fopen(filename, "w");
+         FILE *f = fopen(filename, "w");
-            if(f){
+         if(f){
-               int rc = retrieve_email_from_archive(sdata, f, cfg);
+            int rc = retrieve_email_from_archive(sdata, f, cfg);
-               fclose(f);
+            fclose(f);
-
-               if(rc){
-                  printf("cannot retrieve: %s\n", filename);
-                  unlink(filename);
-                  continue;
-               }
-
-               snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
-
-               state = parse_message(sdata, 0, data, cfg);
-               post_parse(sdata, &state, cfg);
-
-               rc = store_index_data(sdata, &state, data, stored_id, cfg);
-
-               if(rc == OK) reindexed++;
-               else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename);
 
+            if(rc){
+               printf("cannot retrieve: %s\n", filename);
                unlink(filename);
-
+               continue;
-               if(progressbar){
+            }
-                  printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta));
+
-                  fflush(stdout);
+            snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
-               }
+
-
+            state = parse_message(sdata, 0, data, cfg);
+            post_parse(sdata, &state, cfg);
+
+            rc = store_index_data(sdata, &state, data, stored_id, cfg);
+
+            if(rc == OK) reindexed++;
+            else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename);
+
+            unlink(filename);
+
+            if(progressbar){
+               printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta));
+               fflush(stdout);
             }
-            else printf("cannot open: %s\n", filename);
 
          }
+         else printf("cannot open: %s\n", filename);
 
       }
 
+
       p_free_results(&sql);
    }
 

src/rules.c

@@ -240,7 +240,7 @@ char *check_against_ruleset(struct node *xhash[], struct parser_state *state, in
       if(q->str){
          p = q->str;
 
-         if(p && count_match(p, state, size, spam) > 0){
+         if(count_match(p, state, size, spam) > 0){
             return p->rulestr;
          }
       }

util/imapfetch.py

@@ -3,7 +3,6 @@
 
 import argparse
 import imaplib
-import pprint
 import re
 
 opts = {}

util/import.sh

@@ -1,10 +1,13 @@
 #!/bin/bash
 
+set -o nounset
+set -o errexit
+set -o pipefail
+
 export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
-PRIORITY=mail.error
 TMPFILE=/var/run/piler/import.tmp
 
-if [ -f $TMPFILE ]; then exit 1; fi
+if [[ -f $TMPFILE ]]; then exit 1; fi
 
 date > $TMPFILE
 
@@ -17,5 +20,3 @@ trap finish EXIT
 cd /var/piler/imap
 
 pilerimport -G >/dev/null
-
-

util/purge.sh

@@ -1,11 +1,14 @@
 #!/bin/bash
 
+set -o nounset
+set -o errexit
+set -o pipefail
+
 export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/libexec/piler:/usr/local/libexec/piler
-PRIORITY=mail.error
 TMPFILE=/var/run/piler/purge.tmp
 PURGE_BEACON=/var/piler/stat/purge
 
-if [ -f $TMPFILE ]; then exit 1; fi
+if [[ -f $TMPFILE ]]; then exit 1; fi
 
 date > $TMPFILE