Merge branch 'master' of bitbucket.org:jsuto/piler

RELEASE_NOTES

@@ -1,3 +1,22 @@
+1.3.9:
+------
+
+- Added a separator to searching for attachment names
+- [BUGFIX] Render multiple mail parts in mail view instead of only the last part
+- Use TLS v1.2 with openssl 1.0.x for connecting remote pop3/imap servers
+- Instant search results to the gui when the search page loads
+- Support sphinx-3.3.1, introduced sphinx strict mode variable
+- GUI domain fixes
+- gcc 9 fixes
+- Fix permission on sphinx data dir to 700
+- pilerpurge.py should honor the mysqlhost value
+- Password change enabled by default
+- Health page fixes
+- GUI mime parser fixes
+- Start/stop script fix
+- Optimized search page for mobile devices, set $config['ENABLE_MOBILE_PREVIEW'] = 1; in config-site.php to enable it
+
+
 1.3.0:
 -----
 

VERSION

@@ -1 +1 @@
-1.3.8
+1.3.9

config.php.in

@@ -18,9 +18,13 @@ $config = [];
 
 $config['ENABLE_MOBILE_PREVIEW'] = 0;
 
+$config['SITE_LOGO_LG'] = '/view/theme/default/assets/images/archive-logo-lg.png';
+$config['SITE_LOGO_SM'] = '/view/theme/default/assets/images/archive-logo-sm.png';
+$config['COMPATIBILITY'] = 'Which browsers are supported, etc';
+
 $config['BRANDING_TEXT'] = '';
 $config['BRANDING_URL'] = '';
-$config['BRANDING_LOGO'] = '';
+$config['BRANDING_LOGO'] = $config['SITE_LOGO_SM'];
 $config['BRANDING_BACKGROUND_COLOUR'] = '';
 $config['BRANDING_TEXT_COLOUR'] = '';
 $config['BRANDING_FAVICON'] = '/view/theme/default/assets/ico/favicon.png';
@@ -115,7 +119,11 @@ $config['LDAP_MAIL_ATTR'] = 'proxyAddresses';
 //$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'memberOfGroup';
 //$config['LDAP_MAIL_ATTR'] = 'mail';
 
-
+// Uninvention specific settings
+//$config['LDAP_MAIL_ATTR'] = 'mailPrimaryAddress';
+//$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'person';
+//$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'person';
+//$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mailAlternativeAddress';
 
 
 // enable single sign-on (disabled by default)
@@ -152,10 +160,6 @@ $config['GOOGLE_DEVELOPER_KEY'] = 'xxxxxxxxxxxx';
 $config['GOOGLE_APPLICATION_NAME'] = 'piler enterprise email archiver';
 $config['GOOGLE_ALL_MAIL'] = '[Gmail]/All Mail';
 
-$config['SITE_LOGO_LG'] = '/view/theme/default/assets/images/archive-logo-lg.png';
-$config['SITE_LOGO_SM'] = '/view/theme/default/assets/images/archive-logo-sm.png';
-$config['COMPATIBILITY'] = 'Which browsers are supported, etc';
-
 
 $config['ENABLE_AUDIT'] = 1;
 $config['MEMCACHED_ENABLED'] = 0;
@@ -222,6 +226,7 @@ $config['TSA_URL'] = '';
 $config['TSA_PUBLIC_KEY_FILE'] = '';
 $config['TSA_START_ID'] = 1;
 $config['TSA_STAMP_REQUEST_UNIT_SIZE'] = 10000;
+$config['TSA_VERIFY_CERTIFICATE'] = true;
 
 $config['DB_DRIVER'] = 'mysql';
 $config['DB_PREFIX'] = '';
@@ -238,7 +243,8 @@ $config['SPHINX_MAIN_INDEX'] = 'main1,dailydelta1,delta1';
 $config['SPHINX_ATTACHMENT_INDEX'] = 'att1';
 $config['SPHINX_TAG_INDEX'] = 'tag1';
 $config['SPHINX_NOTE_INDEX'] = 'note1';
-$config['SPHINX_STRICT_SCHEMA'] = 0;
+$config['SPHINX_STRICT_SCHEMA'] = 1;
+$config['MAX_EMAIL_LEN'] = 41;
 
 $config['RELOAD_COMMAND'] = 'sudo -n /etc/init.d/rc.piler reload';
 $config['PILERIMPORT_IMAP_COMMAND'] = '/usr/local/bin/pilerimport -d /var/piler/imap -q -r';
@@ -310,6 +316,8 @@ $config['DELIMITER'] = "\t";
 
 $config['TRACKING_CODE'] = '';
 
+$mailattrs = ["mail", "mailalternateaddress", "proxyaddresses", "zimbraMailForwardingAddress", "member", "memberOfGroup", "othermailbox", "mailprimaryaddress", "mailalternativeaddress"];
+
 $langs = array(
                 'cz',
                 'de',

configure

@@ -4852,9 +4852,9 @@ if test $? -eq 1; then echo "the user \"$RUNNING_USER\" does not exists, please
 
 echo; echo
 
-CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wuninitialized -Wno-format-truncation -g"
+CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wimplicit-fallthrough=2 -Wuninitialized -Wno-format-truncation -g"
 LIBS="$antispam_libs $sunos_libs "
-OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs"
+OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs"
 
 ac_config_files="$ac_config_files Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile"
 

configure.in

@@ -535,9 +535,9 @@ if test $? -eq 1; then echo "the user \"$RUNNING_USER\" does not exists, please
 
 echo; echo
 
-CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wuninitialized -Wno-format-truncation -g"
+CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wimplicit-fallthrough=2 -Wuninitialized -Wno-format-truncation -g"
 LIBS="$antispam_libs $sunos_libs "
-OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs"
+OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs"
 
 AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile])
 AC_OUTPUT

cppcheck.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+cppcheck -DHAVE_PDFTOTEXT -DHAVE_PPTHTML -DHAVE_TNEF -DHAVE_UNRTF -DHAVE_XLS2CSV -DHAVE_CATPPT -DHAVE_CATDOC -DHAVE_ZIP -D_GNU_SOURCE -DHAVE_DAEMON -DHAVE_TRE -DHAVE_CLAMD -DHAVE_LIBCLAMAV -DNEED_MYSQL --error-exitcode=1 --enable=all --suppressions-list=suppressions.txt --force src/ unit_tests/

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 ARG PACKAGE
 
@@ -17,13 +17,15 @@ ENV DEBIAN_FRONTEND="noninteractive" \
     SPHINX_BIN_TARGZ="sphinx-3.1.1-bin.tar.gz"
 
 ADD "https://bitbucket.org/jsuto/piler/downloads/${PACKAGE}" "/${PACKAGE}"
-ADD start.sh /start.sh
+COPY start.sh /start.sh
 
 RUN apt-get update && \
     apt-get -y --no-install-recommends install \
        wget rsyslog openssl sysstat php7.2-cli php7.2-cgi php7.2-mysql php7.2-fpm php7.2-zip php7.2-ldap \
        php7.2-gd php7.2-curl php7.2-xml catdoc unrtf poppler-utils nginx tnef sudo libodbc1 libpq5 libzip4 \
        libtre5 libwrap0 cron libmariadb3 libmysqlclient-dev python python-mysqldb mariadb-server && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
     service mysql start && mysqladmin -u root password ${MYSQL_ROOT_PASSWORD} && \
     wget --no-check-certificate -q -O ${SPHINX_BIN_TARGZ} ${DOWNLOAD_URL}/generic-local/${SPHINX_BIN_TARGZ} && \
     tar zxvf ${SPHINX_BIN_TARGZ} && \
@@ -31,8 +33,8 @@ RUN apt-get update && \
     sed -i 's/mail.[iwe].*//' /etc/rsyslog.conf && \
     sed -i '/session    required     pam_loginuid.so/c\#session    required     pam_loginuid.so' /etc/pam.d/cron && \
     mkdir /etc/piler && \
-    printf "[mysql]\nuser = piler\npassword = ${MYSQL_PILER_PASSWORD}\n" > /etc/piler/.my.cnf && \
+    printf "[mysql]\nuser = piler\npassword = %s\n" ${MYSQL_PILER_PASSWORD} > /etc/piler/.my.cnf && \
-    printf "[mysql]\nuser = root\npassword = ${MYSQL_ROOT_PASSWORD}\n" > /root/.my.cnf && \
+    printf "[mysql]\nuser = root\npassword = %s\n" ${MYSQL_ROOT_PASSWORD} > /root/.my.cnf && \
     echo "alias mysql='mysql --defaults-file=/etc/piler/.my.cnf'" > /root/.bashrc && \
     echo "alias t='tail -f /var/log/syslog'" >> /root/.bashrc && \
     dpkg -i $PACKAGE && \

etc/sphinx.conf.in

@@ -1,7 +1,7 @@
 #!/usr/bin/php
 <?php
 
-define('SPHINX_VERSION', 321); // If you have sphinx-3.2.1, then set SPHINX_VERSION to 321
+define('SPHINX_VERSION', 331); // If you have sphinx-3.3.1, then set SPHINX_VERSION to 331
 define('LOCALSTATEDIR', '@LOCALSTATEDIR@');
 define('NGRAM_CONFIG', "        #ngram_len               = 1\n        #ngram_chars             = U+3000..U+2FA1F\n");
 
@@ -18,7 +18,7 @@ define('SPHINX_CHARSET_TABLE', "0..9, english, _, \
 // Be sure to check out http://www.mailpiler.org/wiki/current:sphinx3 for more
 // NB: The SPHINX_STRICT_SCHEMA in sphinx.conf MUST BE THE SAME as in config.php (or in config-site.php)
 //
-define('SPHINX_STRICT_SCHEMA', 0);
+define('SPHINX_STRICT_SCHEMA', 1);
 
 
 if(SPHINX_STRICT_SCHEMA) {

init.d/rc.piler.in

@@ -57,11 +57,10 @@ case "$1" in
         ;;
 
         status)
-                if check_status;
+                if check_status; then
-                 then
+                   exit 0
-                    exit 0
+                else
-                 else
+                   exit 1
-                    exit 1
                 fi
         ;;
 

src/archive.c

@@ -54,7 +54,6 @@ void zerr(int ret){
 
 int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
    int ret, pos=0;
-   unsigned have;
    z_stream strm;
    char *new_ptr;
    unsigned char out[REALLYBIGBUFSIZE];
@@ -96,7 +95,7 @@ int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
                 return ret;
       }
 
-      have = REALLYBIGBUFSIZE - strm.avail_out;
+      unsigned have = REALLYBIGBUFSIZE - strm.avail_out;
 
       /*
        * write the uncompressed result either to stdout
@@ -235,8 +234,8 @@ CLEANUP:
 
 
 int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct config *cfg){
-   int i, attachments;
+   int attachments;
-   char *buffer=NULL, *saved_buffer, *p, filename[SMALLBUFSIZE], pointer[SMALLBUFSIZE];
+   char *buffer=NULL, *saved_buffer, *p, filename[SMALLBUFSIZE];
    struct ptr_array ptr_arr[MAX_ATTACHMENTS];
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
    struct stat st;
@@ -270,7 +269,8 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct c
       if(buffer){
          saved_buffer = buffer;
 
-         for(i=1; i<=attachments; i++){
+         for(int i=1; i<=attachments; i++){
+            char pointer[SMALLBUFSIZE];
             snprintf(pointer, sizeof(pointer)-1, "ATTACHMENT_POINTER_%s.a%d_XXX_PILER", sdata->ttmpfile, i);
 
             p = strstr(buffer, pointer);

src/attachment.c

@@ -17,7 +17,6 @@
 
 
 int store_attachments(struct session_data *sdata, struct parser_state *state, struct config *cfg){
-   uint64 id=0;
    int i, rc=1, found, affected_rows;
    struct sql sql, sql2;
 
@@ -27,7 +26,7 @@ int store_attachments(struct session_data *sdata, struct parser_state *state, st
 
    for(i=1; i<=state->n_attachments; i++){
       found = 0;
-      id = 0;
+      uint64 id = 0;
 
       if(state->attachments[i].size > 0){
 

src/base64.c

@@ -1,78 +0,0 @@
-/*
- * base64.c, SJ
- */
-
-#include <stdio.h>
-#include <string.h>
-
-
-char base64_value(char c){
-   static const char *base64_table = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-
-   if((int)c > 63) return '=';
-
-   return base64_table[(int)c];
-}
-
-
-void base64_encode_block(unsigned char *in, int inlen, char *out){
-   char a, b, c, d, fragment;
-
-   sprintf(out, "====");
-
-   if(inlen <= 0) return;
-
-   fragment = *in & 0x3;
-
-   a = *in >> 2;
-
-   out[0] = base64_value(a);
-
-   b = fragment << 4;
-
-   if(inlen > 1)
-      b += *(in+1) >> 4;
-
-   out[1] = base64_value(b);
-
-   if(inlen == 1) return;
-
-
-   c = *(in+1) & 0xf;
-   c = c << 2;
-
-   if(inlen > 2){
-      fragment = *(in+2) & 0xfc;
-      c += fragment >> 6;
-
-      d = *(in+2) & 0x3f;
-      out[3] = base64_value(d);
-   }
-
-   out[2] = base64_value(c);
-}
-
-
-void base64_encode(unsigned char *in, int inlen, char *out, int outlen){
-   int i=0, j, pos=0;
-   unsigned char buf[3];
-
-   memset(buf, 0, 3);
-   memset(out, 0, outlen);
-
-   for(j=0; j<inlen; j++){
-
-      if(i == 3){
-         base64_encode_block(buf, 3, &out[pos]); pos += 4;
-
-         memset(buf, 0, 3);
-
-         i = 0;
-      }
-
-      buf[i] = *(in+j);
-      i++;
-   }
-
-   base64_encode_block(buf, i, &out[pos]);
-}

src/bdat.c

@@ -54,8 +54,6 @@ void get_bdat_size_to_read(struct smtp_session *session, char *buf){
 
 
 void process_bdat(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg){
-   char buf[SMALLBUFSIZE];
-
    if(readlen <= 0) return;
 
    if(session->fd == -1){
@@ -106,6 +104,7 @@ void process_bdat(struct smtp_session *session, char *readbuf, int readlen, stru
 
          move_email(session);
 
+         char buf[SMALLBUFSIZE];
          snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
          send_smtp_response(session, buf);
          syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d, client=%s, fd=%d", session->ttmpfile, session->mailfrom, session->tot_len, session->remote_host, session->net.socket);

src/cfg.c

@@ -17,31 +17,12 @@ int string_parser(char *src, char *target, int limit){
    return 0;
 };
 
-int multi_line_string_parser(char *src, char *target, unsigned int limit){
-   if(strlen(src) > 0 && strlen(target) + strlen(src) + 3 < limit){
-      strncat(target, src, limit-strlen(target));
-      strncat(target, "\r\n", limit-strlen(target));
-
-      return 0;
-   }
-
-   return 1;
-};
-
 int int_parser(char *src, int *target){
    *target = strtol(src, (char **) NULL, 10);
 
    return 0;
 };
 
-
-int float_parser(char *src, float *target){
-   *target = strtof(src, (char **) NULL);
-
-   return 0;
-};
-
-
 struct _parse_rule {
    char *name;
    char *type;

src/clamd.c

@@ -20,7 +20,7 @@
 
 int clamd_scan(char *tmpfile, struct config *cfg){
    int s, n;
-   char *p, *q, buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE];
+   char buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE];
    struct sockaddr_un server;
 
    chmod(tmpfile, 0644);
@@ -58,9 +58,9 @@ int clamd_scan(char *tmpfile, struct config *cfg){
    if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: CLAMD DEBUG: %d %s", tmpfile, n, buf);
 
    if(strcasestr(buf, CLAMD_RESP_INFECTED)){
-      p = strchr(buf, ' ');
+      char *p = strchr(buf, ' ');
       if(p){
-         q = strrchr(p, ' ');
+         char *q = strrchr(p, ' ');
          if(q){
             *q = '\0';
             p++;

src/config.h

@@ -107,6 +107,7 @@
 #define SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE  "UPDATE " SQL_METADATA_TABLE " SET reference=? WHERE message_id=? AND reference=''"
 #define SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS        "SELECT id, type, username, password, server FROM " SQL_IMPORT_TABLE " WHERE started=0 ORDER BY id LIMIT 0,1"
 #define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE      "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `id`) VALUES(?,?)"
+#define SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE        "UPDATE " SQL_IMPORT_TABLE " SET status=?, imported=? WHERE id=?"
 
 /* Error codes */
 

src/counters.c

@@ -51,13 +51,12 @@ struct counters load_counters(struct session_data *sdata){
 void update_counters(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    char buf[MAXBUFSIZE];
 #ifdef HAVE_MEMCACHED
-   unsigned long long mc, rcvd;
+   unsigned long long mc;
    struct counters c;
-   char key[MAX_MEMCACHED_KEY_LEN];
    unsigned int flags=0;
 #endif
 
-   if(counters->c_virus + counters->c_duplicate + counters->c_ignore + counters->c_size + counters->c_stored_size <= 0) return;
+   if(counters->c_virus + counters->c_duplicate + counters->c_ignore + counters->c_size + counters->c_stored_size == 0) return;
 
 #ifdef HAVE_MEMCACHED
    if(cfg->update_counters_to_memcached == 1){
@@ -65,7 +64,7 @@ void update_counters(struct session_data *sdata, struct data *data, struct count
       /* increment counters to memcached */
 
       if(memcached_increment(&(data->memc), MEMCACHED_MSGS_RCVD, counters->c_rcvd, &mc) == MEMCACHED_SUCCESS){
-         rcvd = mc;
+         unsigned long long rcvd = mc;
 
          if(counters->c_virus > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_VIRUS, counters->c_virus, &mc);
          if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_DUPLICATE, counters->c_duplicate, &mc);
@@ -79,6 +78,8 @@ void update_counters(struct session_data *sdata, struct data *data, struct count
          snprintf(buf, MAXBUFSIZE-1, "%s %s %s %s %s %s %s", MEMCACHED_MSGS_RCVD, MEMCACHED_MSGS_VIRUS, MEMCACHED_MSGS_DUPLICATE, MEMCACHED_MSGS_IGNORE, MEMCACHED_MSGS_SIZE, MEMCACHED_MSGS_STORED_SIZE, MEMCACHED_COUNTERS_LAST_UPDATE);
 
          if(memcached_mget(&(data->memc), buf) == MEMCACHED_SUCCESS){
+            char key[MAX_MEMCACHED_KEY_LEN];
+
             while((memcached_fetch_result(&(data->memc), &key[0], &buf[0], &flags))){
                if(!strcmp(key, MEMCACHED_MSGS_RCVD)) c.c_rcvd = strtoull(buf, NULL, 10);
                else if(!strcmp(key, MEMCACHED_MSGS_VIRUS)) c.c_virus = strtoull(buf, NULL, 10);

src/decoder.c

@@ -98,20 +98,6 @@ inline void utf8_encode_char(unsigned char c, unsigned char *buf, int buflen, in
 }
 
 
-void sanitiseBase64(char *s){
-   char *p1;
-
-   if(s == NULL) return;
-
-   for(; *s; s++){
-      if(b64[(unsigned int)(*s & 0xFF)] == 255){
-         for(p1 = s; p1[0] != '\0'; p1++)
-            p1[0] = p1[1];
-      }
-   }
-}
-
-
 inline static void pack_4_into_3(char *s, char *s2){
    int j, n[4], k1, k2;
 
@@ -157,7 +143,7 @@ int decodeBase64(char *p){
 
 
 int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
-   int i, len=0, decodedlen;
+   int i, len=0;
    char s[5], s2[3];
 
    if(plen < 4 || plen > blen)
@@ -166,7 +152,7 @@ int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
    for(i=0; i<plen; i+=4){
       memcpy(s, p+i, 4);
       s[4] = '\0';
-      decodedlen = 3;
+      int decodedlen = 3;
 
       /* safety check against abnormally long lines */
 
@@ -191,12 +177,11 @@ int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
 void decodeQP(char *p){
    unsigned int i;
    int k=0, a, b;
-   char c;
 
    if(p == NULL) return;
 
    for(i=0; i<strlen((char*)p); i++){
-      c = p[i];
+      char c = p[i];
 
       if(p[i] == '=' && isxdigit(p[i+1]) && isxdigit(p[i+2])){
          a = p[i+1];

src/decoder.h

@@ -5,9 +5,6 @@
 #ifndef _DECODER_H
  #define _DECODER_H
 
-void base64_encode(unsigned char *in, int inlen, char *out, int outlen);
-
-void sanitiseBase64(char *s);
 int decodeBase64(char *p);
 int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen);
 void decodeQP(char *p);

src/defs.h

@@ -241,10 +241,6 @@ struct session_data {
 #ifdef NEED_MYSQL
    MYSQL mysql;
 #endif
-#ifdef NEED_PSQL
-   PGconn *psql;
-   char conninfo[SMALLBUFSIZE];
-#endif
 };
 
 
@@ -308,6 +304,7 @@ struct import {
    int tot_msgs;
    int port;
    int seq;
+   int table_id;
    char *server;
    char *username;
    char *password;

src/digest.c

@@ -34,7 +34,7 @@ int search_header_end(char *p, int n){
 
 
 int make_digests(struct session_data *sdata, struct config *cfg){
-   int i=0, n, fd, offset=3, hdr_len=0, len=0;
+   int i=0, n, fd, offset=3, hdr_len=0;
    char *body=NULL;
    unsigned char buf[BIGBUFSIZE], md[DIGEST_LENGTH], md2[DIGEST_LENGTH];
    SHA256_CTX context, context2;
@@ -50,8 +50,6 @@ int make_digests(struct session_data *sdata, struct config *cfg){
 
 
    while((n = read(fd, buf, sizeof(buf))) > 0){
-      len += n;
-
       SHA256_Update(&context2, buf, n);
 
       body = (char *)&buf[0];

src/extract.c

@@ -40,7 +40,7 @@ int remove_xml(char *src, char *dest, int destlen, int *html){
 
 #ifdef HAVE_ZIP
 int extract_opendocument(struct session_data *sdata, struct parser_state *state, char *filename, char *prefix){
-   int errorp, i=0, len=0, html=0;
+   int errorp, i=0, html=0;
    unsigned int len2;
    char buf[4*MAXBUFSIZE], puf[4*MAXBUFSIZE];
    struct zip *z;
@@ -60,6 +60,7 @@ int extract_opendocument(struct session_data *sdata, struct parser_state *state,
 
          zf = zip_fopen_index(z, i, 0);
          if(zf){
+            int len;
             while((len = zip_fread(zf, buf, sizeof(buf)-2)) > 0){
 
                len2 = remove_xml(buf, puf, sizeof(puf), &html);
@@ -89,7 +90,7 @@ int extract_opendocument(struct session_data *sdata, struct parser_state *state,
 
 
 int unzip_file(struct session_data *sdata, struct parser_state *state, char *filename, int *rec, struct config *cfg){
-   int errorp, i=0, len=0, fd;
+   int errorp, i=0, fd;
    char *p, extracted_filename[SMALLBUFSIZE], buf[MAXBUFSIZE];
    struct zip *z;
    struct zip_stat sb;
@@ -120,6 +121,7 @@ int unzip_file(struct session_data *sdata, struct parser_state *state, char *fil
             if(fd != -1){
                zf = zip_fopen_index(z, i, 0);
                if(zf){
+                  int len;
                   while((len = zip_fread(zf, buf, sizeof(buf))) > 0){
                      if(write(fd, buf, len) == -1) syslog(LOG_PRIORITY, "ERROR: error writing to fd in %s", __func__);
                   }
@@ -207,9 +209,7 @@ void kill_helper(){
 
 void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct config *cfg){
    int link[2];
-   ssize_t n;
    pid_t pid;
-   char outbuf[MAXBUFSIZE];
 
    if(strcmp(type, "other") == 0 || strcmp(type, "text") == 0) return;
 
@@ -304,6 +304,9 @@ void extract_attachment_content(struct session_data *sdata, struct parser_state
    }
    else {
       close(link[1]);
+      ssize_t n;
+      char outbuf[MAXBUFSIZE];
+
       while((n = read(link[0], outbuf, sizeof(outbuf))) > 0){
          if(state->bodylen < BIGBUFSIZE-n-1){
             memcpy(&(state->b_body[state->bodylen]), outbuf, n);

src/hash.c

@@ -33,12 +33,10 @@ void clearhash(struct node *xhash[]){
          p = q;
 
          q = q->r;
-         if(p){
+         if(p->str){
-            if(p->str){
+            free(p->str);
-               free(p->str);
-            }
-            free(p);
          }
+         free(p);
       }
       xhash[i] = NULL;
    }
@@ -154,12 +152,12 @@ int is_substr_in_hash(struct node *xhash[], char *s){
 
 
 unsigned int DJBHash(char* str, unsigned int len){
-   unsigned int hash = 5381;
+   unsigned int hashval = 5381;
    unsigned int i    = 0;
 
    for(i=0; i < len; str++, i++){
-      hash = ((hash << 5) + hash) + (*str);
+      hashval = ((hashval << 5) + hashval) + (*str);
    }
 
-   return hash;
+   return hashval;
 }

src/imap.c

@@ -314,7 +314,7 @@ void imap_expunge_message(struct data *data){
 
 
 int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct config *cfg){
-   int rc=ERR, i, messages=0;
+   int i, messages=0;
 
    messages = imap_select_cmd_on_folder(folder, data);
 
@@ -330,7 +330,7 @@ int process_imap_folder(char *folder, struct session_data *sdata, struct data *d
          if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/messages); fflush(stdout); }
 
          if(data->import->dryrun == 0){
-            rc = import_message(sdata, data, cfg);
+            int rc = import_message(sdata, data, cfg);
 
             if(data->import->remove_after_import == 1 && rc == OK){
                imap_delete_message(data, i);

src/import.c

@@ -20,7 +20,7 @@
 
 int import_message(struct session_data *sdata, struct data *data, struct config *cfg){
    int rc=ERR;
-   char *p, *rule, newpath[SMALLBUFSIZE];
+   char *rule;
    struct stat st;
    struct parser_state state;
    struct counters counters;
@@ -127,12 +127,13 @@ int import_message(struct session_data *sdata, struct data *data, struct config
    }
 
    if(rc != OK && data->import->failed_folder){
-      p = strrchr(data->import->filename, '/');
+      char *p = strrchr(data->import->filename, '/');
       if(p)
          p++;
       else
          p = data->import->filename;
 
+      char newpath[SMALLBUFSIZE];
       snprintf(newpath, sizeof(newpath)-2, "%s/%s", data->import->failed_folder, p);
 
       if(rename(data->import->filename, newpath))
@@ -143,6 +144,26 @@ int import_message(struct session_data *sdata, struct data *data, struct config
 }
 
 
+int update_import_table(struct session_data *sdata, struct data *data) {
+   int ret=ERR, status=2;
+   struct sql sql;
+
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE) == ERR) return ret;
+
+   p_bind_init(&sql);
+
+   sql.sql[sql.pos] = (char *)&(status); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&(data->import->tot_msgs); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&(data->import->table_id); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+
+   if(p_exec_stmt(sdata, &sql) == OK) ret = OK;
+
+   close_prepared_statement(&sql);
+
+   return ret;
+}
+
+
 int get_folder_id(struct session_data *sdata, char *foldername, int parent_id){
    int id=ERR_FOLDER;
    struct sql sql;

src/import.h

@@ -7,6 +7,7 @@
 
 
 int import_message(struct session_data *sdata, struct data *data, struct config *cfg);
+int update_import_table(struct session_data *sdata, struct data *data);
 
 int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct config *cfg);
 int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct config *cfg);

src/import_imap.c

@@ -73,7 +73,7 @@ int import_from_imap_server(struct session_data *sdata, struct data *data, struc
       q = data->imapfolders[i];
       while(q != NULL){
 
-         if(q && q->str && strlen(q->str) > 1){
+         if(q->str && strlen(q->str) > 1){
 
             skipmatch = 0;
 

src/import_maildir.c

@@ -103,5 +103,9 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir
    }
    closedir(dir);
 
+   if(data->import->table_id > 0){
+      update_import_table(sdata, data);
+   }
+
    return ret;
 }

src/list.c

@@ -1,94 +0,0 @@
-/*
- * list.c, SJ
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include "list.h"
-#include "config.h"
-
-
-int append_list(struct list **list, char *p){
-   struct list *q, *t, *u=NULL;
-
-   q = *list;
-
-   while(q){
-      if(strcmp(q->s, p) == 0)
-         return 0;
-
-      u = q;
-      q = q->r;
-   }
-
-   t = create_list_item(p);
-   if(t){
-      if(*list == NULL)
-         *list = t;
-      else if(u)
-         u->r = t;
-
-      return 1;
-   }
-
-   return -1;
-}
-
-
-struct list *create_list_item(char *s){
-   struct list *h=NULL;
-
-   if((h = malloc(sizeof(struct list))) == NULL)
-      return NULL;
-
-   snprintf(h->s, SMALLBUFSIZE-1, "%s", s);
-   h->r = NULL;
-
-   return h;
-}
-
-
-int is_string_on_list(struct list *list, char *s){
-   struct list *p;
-
-   p = list;
-
-   while(p != NULL){
-      if(strcmp(p->s, s) == 0) return 1;
-      p = p->r;
-   }
-
-   return 0;
-}
-
-
-int is_item_on_string(struct list *list, char *s){
-   struct list *p;
-
-   p = list;
-
-   while(p != NULL){
-      if(strstr(s, p->s)) return 1;
-      p = p->r;
-   }
-
-   return 0;
-}
-
-
-void free_list(struct list *list){
-   struct list *p, *q;
-
-   p = list;
-
-   while(p != NULL){
-      q = p->r;
-
-      if(p)
-         free(p);
-
-      p = q;
-   }
-}

src/list.h

@@ -1,16 +0,0 @@
-/*
- * list.h, SJ
- */
-
-#ifndef _LIST_H
- #define _LIST_H
-
-#include "defs.h"
-
-int append_list(struct list **list, char *p);
-struct list *create_list_item(char *s);
-int is_string_on_list(struct list *list, char *s);
-int is_item_on_string(struct list *list, char *s);
-void free_list(struct list *list);
-
-#endif /* _LIST_H */

src/memc.c

@@ -59,7 +59,7 @@ void memcached_init(struct memcached_server *ptr, char *server_ip, int server_po
 
 
 int set_socket_options(struct memcached_server *ptr){
-   int error, flag=1, flags, rval;
+   int error, flag=1, flags;
    struct timeval waittime;
    struct linger linger;
 
@@ -128,8 +128,7 @@ int set_socket_options(struct memcached_server *ptr){
 
 
    if((flags & O_NONBLOCK) == 0){
-      rval = fcntl(ptr->fd, F_SETFL, flags | O_NONBLOCK);
+      if(fcntl(ptr->fd, F_SETFL, flags | O_NONBLOCK) == -1) return MEMCACHED_FAILURE;
-      if(rval == -1) return MEMCACHED_FAILURE;
    }
 
    return MEMCACHED_SUCCESS;
@@ -223,7 +222,7 @@ int memcached_add(struct memcached_server *ptr, char *cmd, char *key, char *valu
    if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
 
    // cmd could be either 'add' or 'set'
-   snprintf(ptr->buf, MAXBUFSIZE-1, "%s %s %d %ld %d \r\n", cmd, key, flags, expiry, valuelen);
+   snprintf(ptr->buf, MAXBUFSIZE-1, "%s %s %u %lu %u \r\n", cmd, key, flags, expiry, valuelen);
    len = strlen(ptr->buf);
 
    strncat(ptr->buf, value, MAXBUFSIZE-strlen(ptr->buf)-1);
@@ -263,52 +262,6 @@ int memcached_increment(struct memcached_server *ptr, char *key, unsigned long l
 }
 
 
-char *memcached_get(struct memcached_server *ptr, char *key, unsigned int *len, unsigned int *flags){
-   char *p;
-
-   if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return NULL;
-
-   snprintf(ptr->buf, MAXBUFSIZE, "get %s \r\n", key);
-   send(ptr->fd, ptr->buf, strlen(ptr->buf), 0);
-
-   ptr->last_read_bytes = __recvtimeout(ptr->fd, ptr->buf, MAXBUFSIZE, ptr->rcv_timeout);
-
-   if(ptr->last_read_bytes <= 0){
-      memcached_shutdown(ptr);
-      return NULL;
-   }
-
-
-   if(ptr->last_read_bytes < 10) return NULL;
-
-   if(strncmp("VALUE ", ptr->buf, 6)) return NULL;
-
-   p = strchr(ptr->buf, '\r');
-   if(!p) return NULL;
-   *p = '\0';
-   ptr->result = p + 2;
-
-   p = strrchr(ptr->buf + 6, ' ');
-   if(!p) return NULL;
-   *len = atoi(p+1);
-   *p = '\0';
-
-
-   p = strrchr(ptr->buf + 6, ' ');
-   if(!p) return NULL;
-   *flags = atoi(p+1);
-   *p = '\0';
-
-
-   p = strchr(ptr->result, '\r');
-   if(!p) return NULL;
-
-   *p = '\0';
-
-   return ptr->result;
-}
-
-
 int memcached_mget(struct memcached_server *ptr, char *key){
 
    if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;

src/message.c

@@ -136,15 +136,6 @@ int store_recipients(struct session_data *sdata, char *to, uint64 id, struct con
 }
 
 
-void remove_recipients(struct session_data *sdata, uint64 id){
-   char s[SMALLBUFSIZE];
-
-   snprintf(s, sizeof(s)-1, "DELETE FROM " SQL_RECIPIENT_TABLE " WHERE id=%llu", id);
-
-   p_query(sdata, s);
-}
-
-
 int store_folder_id(struct session_data *sdata, struct data *data, uint64 id, struct config *cfg){
    int rc=ERR;
    struct sql sql;

src/misc.c

@@ -88,44 +88,6 @@ long tvdiff(struct timeval a, struct timeval b){
 }
 
 
-/*
- * count a character in buffer
- */
-
-int countCharacterInBuffer(char *p, char c){
-   int i=0;
-
-   for(; *p; p++){
-      if(*p == c)
-         i++;
-   }
-
-   return i;
-}
-
-
-void replaceCharacterInBuffer(char *p, char from, char to){
-   size_t i;
-   int k=0;
-
-   for(i=0; i<strlen(p); i++){
-      if(p[i] == from){
-         if(to > 0){
-            p[k] = to;
-            k++;
-         }
-      }
-      else {
-         p[k] = p[i];
-         k++;
-      }
-
-   }
-
-   p[k] = '\0';
-}
-
-
 /*
  * split a string by a character as delimiter
  */
@@ -181,10 +143,8 @@ char *split_str(char *row, char *what, char *s, int size){
       r += strlen(what);
    }
 
-   if(s != NULL){
+   strncpy(s, row, len);
-      strncpy(s, row, len);
+   s[len] = '\0';
-      s[len] = '\0';
-   }
 
    return r;
 }
@@ -215,8 +175,7 @@ int trimBuffer(char *s){
 
 
 int extract_verp_address(char *email){
-   char *p, *p1, *p2;
+   char *p1;
-   char puf[SMALLBUFSIZE];
 
    // a VERP address is like archive+user=domain.com@myarchive.local
 
@@ -226,13 +185,14 @@ int extract_verp_address(char *email){
 
    p1 = strchr(email, '+');
    if(p1){
-      p2 = strchr(p1, '@');
+      char *p2 = strchr(p1, '@');
       if(p2 && p2 > p1 + 2){
          if(strchr(p1+1, '=')){
+            char puf[SMALLBUFSIZE];
             memset(puf, 0, sizeof(puf));
 
             memcpy(&puf[0], p1+1, p2-p1-1);
-            p = strchr(puf, '=');
+            char *p = strchr(puf, '=');
             if(p) *p = '@';
             strcpy(email, puf);
          }
@@ -349,10 +309,10 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
 
 int readFromEntropyPool(int fd, void *_s, ssize_t n){
    char *s = _s;
-   ssize_t res, pos = 0;
+   ssize_t pos = 0;
 
    while(n > pos){
-      res = read(fd, s + pos, n - pos);
+      ssize_t res = read(fd, s + pos, n - pos);
       switch(res){
          case  -1: continue;
          case   0: return res;
@@ -605,12 +565,6 @@ void strtolower(char *s){
 }
 
 
-void *get_in_addr(struct sockaddr *sa){
-   if(sa->sa_family == AF_INET) return &(((struct sockaddr_in*)sa)->sin_addr);
-   return &(((struct sockaddr_in6*)sa)->sin6_addr);
-}
-
-
 int make_socket_non_blocking(int fd){
    int flags, s;
 

src/misc.h

@@ -19,8 +19,6 @@ int get_build();
 void get_extractor_list();
 void __fatal(char *s);
 long tvdiff(struct timeval a, struct timeval b);
-int countCharacterInBuffer(char *p, char c);
-void replaceCharacterInBuffer(char *p, char from, char to);
 char *split(char *str, int ch, char *buf, int buflen, int *result);
 char *split_str(char *row, char *what, char *s, int size);
 int trimBuffer(char *s);
@@ -42,7 +40,6 @@ void init_session_data(struct session_data *sdata, struct config *cfg);
 int read_from_stdin(struct session_data *sdata);
 void strtolower(char *s);
 
-void *get_in_addr(struct sockaddr *sa);
 int make_socket_non_blocking(int fd);
 int create_and_bind(char *listen_addr, int listen_port);
 

src/mydomains.c

@@ -11,7 +11,6 @@
 
 
 void load_mydomains(struct session_data *sdata, struct data *data, struct config *cfg){
-   int rc;
    char s[SMALLBUFSIZE];
    struct sql sql;
 
@@ -32,9 +31,7 @@ void load_mydomains(struct session_data *sdata, struct data *data, struct config
       p_store_results(&sql);
 
       while(p_fetch_results(&sql) == OK){
-         rc = addnode(data->mydomains, s);
+         if(addnode(data->mydomains, s) == 0) syslog(LOG_PRIORITY, "failed to append mydomain: '%s'", s);
-
-         if(rc == 0) syslog(LOG_PRIORITY, "failed to append mydomain: '%s'", s);
          else if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "added mydomain: '%s'", s);
 
          memset(s, 0, sizeof(s));

src/parser.c

@@ -73,9 +73,7 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec
 
 
 void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg){
-   int i, rec=0;
+   int i;
-   unsigned int len;
-   char *p;
 
    clearhash(state->boundaries);
    clearhash(state->rcpt);
@@ -101,16 +99,25 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c
 
 
    for(i=1; i<=state->n_attachments; i++){
+      char puf[SMALLBUFSIZE];
+      snprintf(puf, sizeof(puf)-1, "%s ", state->attachments[i].filename);
+
+      unsigned int len = strlen(puf);
+      if(state->bodylen < BIGBUFSIZE-len-1){
+         memcpy(&(state->b_body[state->bodylen]), puf, len);
+         state->bodylen += len;
+      }
+
       digest_file(state->attachments[i].internalname, &(state->attachments[i].digest[0]));
 
       if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest);
 
-      p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type);
+      char *p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type);
       len = strlen(p);
       if(strlen(sdata->attachments) < SMALLBUFSIZE-len-1 && !strstr(sdata->attachments, p)) memcpy(&(sdata->attachments[strlen(sdata->attachments)]), p, len);
 
       if(state->attachments[i].dumped == 1){
-         rec = 0;
+         int rec = 0;
          if(cfg->extract_attachments == 1 && state->bodylen < BIGBUFSIZE-1024) extract_attachment_content(sdata, state, state->attachments[i].aname, get_attachment_extractor_by_filename(state->attachments[i].filename), &rec, cfg);
 
          unlink(state->attachments[i].aname);
@@ -153,15 +160,13 @@ void storno_attachment(struct parser_state *state){
 
 
 void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned int abuffersize){
-   int n64;
-   unsigned char b64buffer[MAXBUFSIZE];
-
    if(write(state->fd, abuffer, state->abufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
 
    if(state->b64fd != -1){
       abuffer[state->abufpos] = '\0';
       if(state->base64 == 1){
-         n64 = base64_decode_attachment_buffer(abuffer, &b64buffer[0], sizeof(b64buffer));
+         unsigned char b64buffer[MAXBUFSIZE];
+         int n64 = base64_decode_attachment_buffer(abuffer, &b64buffer[0], sizeof(b64buffer));
          if(write(state->b64fd, b64buffer, n64) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
       }
       else if(write(state->b64fd, abuffer, state->abufpos) == -1){
@@ -175,9 +180,8 @@ void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned
 
 
 int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg){
-   char *p, puf[SMALLBUFSIZE];
+   char *p;
-   char tmpbuf[MAXBUFSIZE];
+   int boundary_line=0;
-   int writelen, boundary_line=0, result;
    unsigned int len;
 
    if(cfg->debug == 1) printf("line: %s", buf);
@@ -284,8 +288,9 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
                syslog(LOG_PRIORITY, "%s: error opening %s", sdata->ttmpfile, state->attachments[state->n_attachments].internalname);
             }
             else {
+               char puf[SMALLBUFSIZE];
                snprintf(puf, sizeof(puf)-1, "ATTACHMENT_POINTER_%s.a%d_XXX_PILER", sdata->ttmpfile, state->n_attachments);
-               writelen = strlen(puf);
+               int writelen = strlen(puf);
                if(writelen + state->writebufpos > writebuffersize-1){
                   if(write(state->mfd, writebuffer, state->writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
                   state->writebufpos = 0;
@@ -637,7 +642,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
 
    /* encode the body if it's not utf-8 encoded */
    if(state->message_state == MSG_BODY && state->utf8 != 1){
-      result = utf8_encode(buf, strlen(buf), &tmpbuf[0], sizeof(tmpbuf), state->charset);
+      char tmpbuf[MAXBUFSIZE];
+      int result = utf8_encode(buf, strlen(buf), &tmpbuf[0], sizeof(tmpbuf), state->charset);
       if(result == OK) snprintf(buf, MAXBUFSIZE-1, "%s", tmpbuf);
    }
 

src/parser_utils.c

@@ -114,7 +114,7 @@ long get_local_timezone_offset(){
 
 
 time_t parse_date_header(char *datestr){
-   int n=0, len;
+   int n=0;
    long offset=0;
    time_t ts=0;
    char *p, *q, *r, *tz, s[SMALLBUFSIZE], tzh[4], tzm[3];
@@ -150,7 +150,7 @@ time_t parse_date_header(char *datestr){
    do {
       p = split_str(p, " ", s, sizeof(s)-1);
 
-      len = strlen(s);
+      int len = strlen(s);
 
       if(len > 0){
          n++;
@@ -264,7 +264,7 @@ time_t parse_date_header(char *datestr){
 
 
 int extract_boundary(char *p, struct parser_state *state){
-   char *q, *q2;
+   char *q;
 
    p += strlen("boundary");
 
@@ -291,7 +291,7 @@ int extract_boundary(char *p, struct parser_state *state){
             break;
       }
 
-      q2 = strchr(p, ';');
+      char *q2 = strchr(p, ';');
       if(q2) *q2 = '\0';
 
       q = strrchr(p, '"');
@@ -315,14 +315,13 @@ int extract_boundary(char *p, struct parser_state *state){
 
 
 void fixupEncodedHeaderLine(char *buf, int buflen){
-   char *p, *q, *r, *s, *e, *end;
+   char *q, *r, *s, *e, *end;
    /*
     * I thought SMALLBUFSIZE would be enough for v, encoding and tmpbuf(2*),
     * but then I saw a 6-7000 byte long subject line, so I've switched to MAXBUFSIZE
     */
    char v[MAXBUFSIZE], u[MAXBUFSIZE], puf[MAXBUFSIZE], encoding[MAXBUFSIZE], tmpbuf[2*MAXBUFSIZE];
    int need_encoding, ret, prev_encoded=0, n_tokens=0;
-   int b64=0, qp=0;
 
    if(buflen < 5) return;
 
@@ -333,7 +332,7 @@ void fixupEncodedHeaderLine(char *buf, int buflen){
    do {
       q = split_str(q, " ", v, sizeof(v)-1);
 
-      p = v;
+      char *p = v;
 
       do {
          memset(u, 0, sizeof(u));
@@ -349,7 +348,7 @@ void fixupEncodedHeaderLine(char *buf, int buflen){
           *    Happy New Year! =?utf-8?q?=F0=9F=8E=86?=
           */
 
-         b64 = qp = 0;
+         int b64=0, qp=0;
          memset(encoding, 0, sizeof(encoding));
 
          r = strstr(p, "=?");
@@ -446,9 +445,9 @@ void fixupEncodedHeaderLine(char *buf, int buflen){
 
 void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state){
    int i=0;
-   char *p, puf[MAXBUFSIZE];
 
    if(strlen(state->qpbuf) > 0){
+      char puf[MAXBUFSIZE];
       memset(puf, 0, sizeof(puf));
       snprintf(puf, sizeof(puf)-1, "%s%s", state->qpbuf, buf);
       snprintf(buf, MAXBUFSIZE-1, "%s", puf);
@@ -461,7 +460,7 @@ void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state){
    }
 
    if(i == 1){
-      p = strrchr(buf, ' ');
+      char *p = strrchr(buf, ' ');
       if(p){
          memset(state->qpbuf, 0, MAX_TOKEN_LEN);
          if(strlen(p) < MAX_TOKEN_LEN-1){
@@ -475,9 +474,8 @@ void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state){
 
 
 void fixupBase64EncodedLine(char *buf, struct parser_state *state){
-   char *p, puf[MAXBUFSIZE];
-
    if(strlen(state->miscbuf) > 0){
+      char puf[MAXBUFSIZE];
       memset(puf, 0, sizeof(puf));
       strncpy(puf, state->miscbuf, sizeof(puf)-strlen(puf)-1);
       strncat(puf, buf, sizeof(puf)-strlen(puf)-1);
@@ -489,7 +487,7 @@ void fixupBase64EncodedLine(char *buf, struct parser_state *state){
    }
 
    if(buf[strlen(buf)-1] != '\n'){
-      p = strrchr(buf, ' ');
+      char *p = strrchr(buf, ' ');
       if(p){
          memcpy(&(state->miscbuf[0]), p+1, MAX_TOKEN_LEN-1);
          *p = '\0';
@@ -576,7 +574,7 @@ void markHTML(char *buf, struct parser_state *state){
 
 
 int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state){
-   char *p, html[SMALLBUFSIZE];
+   char html[SMALLBUFSIZE];
    int len;
 
    if(pos == 0 && strncmp(htmlbuf, "style ", 6) == 0) state->style = 1;
@@ -594,7 +592,7 @@ int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state)
    len = strlen(html);
 
    if(len > 8 && strchr(html, '=')){
-      p = strstr(html, "cid:");
+      char *p = strstr(html, "cid:");
       if(p){
          *(p+3) = '\0';
          strncat(html, " ", SMALLBUFSIZE-1);
@@ -697,11 +695,9 @@ int does_it_seem_like_an_email_address(char *email){
 
 
 void add_recipient(char *email, unsigned int len, struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
-   char *q;
-
    if(findnode(state->rcpt, email) == NULL){
 
-      q = strchr(email, '@');
+      char *q = strchr(email, '@');
 
       /* skip any address matching ...@cfg->hostid, 2013.10.29, SJ */
       if(q && strncmp(q+1, cfg->hostid, cfg->hostid_len) == 0){
@@ -769,7 +765,7 @@ void degenerateToken(unsigned char *p){
    int i=1, d=0, dp=0;
    unsigned char *s;
 
-   /* quit if this the string does not end with a punctuation character */
+   /* quit if the string does not end with a punctuation character */
 
    if(!ispunct(*(p+strlen((char *)p)-1)))
       return;
@@ -825,8 +821,7 @@ void fixURL(char *buf, int buflen){
 
 
 void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultbuflen){
-   int extended=0;
+   char buf[SMALLBUFSIZE], *p, *q;
-   char buf[SMALLBUFSIZE], puf[SMALLBUFSIZE], *p, *q, *encoding;
 
    snprintf(buf, sizeof(buf)-1, "%s", s);
 
@@ -862,6 +857,8 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb
        *
        */
 
+      int extended=0;
+
       p += strlen(name);
       if(*p == '*'){
          extended = 1;
@@ -889,7 +886,7 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb
 
 
          if(extended == 1){
-            encoding = p;
+            char *encoding = p;
             q = strchr(p, '\'');
             if(q){
                *q = '\0';
@@ -906,6 +903,7 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb
                snprintf(resultbuf, resultbuflen-2, "%s", p);
          }
          else {
+            char puf[SMALLBUFSIZE];
             snprintf(puf, sizeof(puf)-1, "%s", p);
             fixupEncodedHeaderLine(puf, sizeof(puf));
 
@@ -919,8 +917,6 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb
 
 
 char *determine_attachment_type(char *filename, char *type){
-   char *p;
-
    if(strncasecmp(type, "text/", strlen("text/")) == 0) return "text,";
    if(strncasecmp(type, "image/", strlen("image/")) == 0) return "image,";
    if(strncasecmp(type, "audio/", strlen("audio/")) == 0) return "audio,";
@@ -951,7 +947,7 @@ char *determine_attachment_type(char *filename, char *type){
 
    if(strncasecmp(type, "application/", 12) == 0){
 
-      p = strrchr(filename, '.');
+      char *p = strrchr(filename, '.');
       if(p){
          p++;
 
@@ -1013,14 +1009,13 @@ char *get_attachment_extractor_by_filename(char *filename){
 
 
 void parse_reference(struct parser_state *state, char *s){
-   int len;
    char puf[SMALLBUFSIZE];
 
    if(strlen(state->reference) > 10) return;
 
    do {
       s = split_str(s, " ", puf, sizeof(puf)-1);
-      len = strlen(puf);
+      int len = strlen(puf);
 
       if(len > 10 && len < SMALLBUFSIZE-1){
          memcpy(&(state->reference[strlen(state->reference)]), puf, len);
@@ -1046,12 +1041,11 @@ int base64_decode_attachment_buffer(char *p, unsigned char *b, int blen){
 
 
 void fix_plus_sign_in_email_address(char *puf, char **at_sign, unsigned int *len){
-   int n;
    char *r;
 
    r = strchr(puf, '+');
    if(r){
-      n = strlen(*at_sign);
+      int n = strlen(*at_sign);
       memmove(r, *at_sign, n);
       *(r+n) = '\0';
       *len = strlen(puf);
@@ -1069,8 +1063,13 @@ void fill_attachment_name_buf(struct parser_state *state, char *buf){
 
    int len = strlen(p);
 
-   if(len + state->anamepos < SMALLBUFSIZE-2){
+   if(len + state->anamepos < SMALLBUFSIZE-3){
       memcpy(&(state->attachment_name_buf[state->anamepos]), p, len);
       state->anamepos += len;
+
+      // add a trailing separator semicolon to make sure there's separation
+      // with the next item
+      state->attachment_name_buf[state->anamepos] = ';';
+      state->anamepos++;
    }
 }

src/piler-smtp.c

@@ -52,14 +52,12 @@ void usage(){
 
 
 void p_clean_exit(int sig){
-   int i;
-
    if(sig > 0) syslog(LOG_PRIORITY, "got signal: %d, %s", sig, strsignal(sig));
 
    if(listenerfd != -1) close(listenerfd);
 
    if(sessions){
-      for(i=0; i<cfg.max_connections; i++){
+      for(int i=0; i<cfg.max_connections; i++){
          if(sessions[i]) free_smtp_session(sessions[i]);
       }
 
@@ -84,14 +82,13 @@ void fatal(char *s){
 
 void check_for_client_timeout(){
    time_t now;
-   int i;
 
    time(&now);
 
    if(cfg.verbosity >= LOG_DEBUG) syslog(LOG_PRIORITY, "%s @%ld", __func__, now);
 
    if(num_connections > 0){
-      for(i=0; i<cfg.max_connections; i++){
+      for(int i=0; i<cfg.max_connections; i++){
          if(sessions[i] && now - sessions[i]->lasttime >= cfg.smtp_timeout){
             syslog(LOG_PRIORITY, "client %s timeout, lasttime: %ld", sessions[i]->remote_host, sessions[i]->lasttime);
             tear_down_session(sessions, sessions[i]->slot, &num_connections);
@@ -128,8 +125,8 @@ void initialise_configuration(){
 
 
 int main(int argc, char **argv){
-   int listenerfd, client_sockfd;
+   int client_sockfd;
-   int i, n, daemonise=0;
+   int i, daemonise=0;
    int client_len = sizeof(struct sockaddr_storage);
    ssize_t readlen;
    struct sockaddr_storage client_address;
@@ -220,7 +217,7 @@ int main(int argc, char **argv){
    alarm(cfg.check_for_client_timeout_interval);
 
    for(;;){
-      n = epoll_wait(efd, events, cfg.max_connections, -1);
+      int n = epoll_wait(efd, events, cfg.max_connections, -1);
       for(i=0; i<n; i++){
 
          // Office365 sometimes behaves oddly: when it receives the 250 OK

src/piler.c

@@ -106,7 +106,6 @@ int process_email(char *filename, struct session_data *sdata, struct data *data,
    char tmpbuf[SMALLBUFSIZE];
    char *status=S_STATUS_UNDEF;
    char *arule;
-   char *rcpt;
    char *p;
    struct timezone tz;
    struct timeval tv1, tv2;
@@ -135,7 +134,7 @@ int process_email(char *filename, struct session_data *sdata, struct data *data,
    post_parse(sdata, &parser_state, cfg);
 
    if(cfg->syslog_recipients == 1){
-      rcpt = parser_state.b_to;
+      char *rcpt = parser_state.b_to;
       do {
          rcpt = split_str(rcpt, " ", tmpbuf, sizeof(tmpbuf)-1);
 
@@ -162,9 +161,10 @@ int process_email(char *filename, struct session_data *sdata, struct data *data,
          syslog(LOG_PRIORITY, "%s: invalid message, hdr_len: %d", filename, sdata->hdr_len);
          rc = ERR;
       }
-
+      else {
-      rc = process_message(sdata, &parser_state, data, cfg);
+         rc = process_message(sdata, &parser_state, data, cfg);
-      unlink(parser_state.message_id_hash);
+         unlink(parser_state.message_id_hash);
+      }
    }
 
    unlink(sdata->tmpframe);
@@ -467,7 +467,7 @@ void initialise_configuration(){
 
 
 int main(int argc, char **argv){
-   int i, daemonise=0, dedupfd;
+   int i, daemonise=0;
    struct stat st;
 
 
@@ -521,7 +521,7 @@ int main(int argc, char **argv){
    if(stat(cfg.pidfile, &st) == 0) fatal(ERR_PID_FILE_EXISTS);
 
    if(cfg.mmap_dedup_test == 1){
-      dedupfd = open(MESSAGE_ID_DEDUP_FILE, O_RDWR);
+      int dedupfd = open(MESSAGE_ID_DEDUP_FILE, O_RDWR);
       if(dedupfd == -1) fatal(ERR_OPEN_DEDUP_FILE);
 
       data.dedup = mmap(NULL, MAXCHILDREN*DIGEST_LENGTH*2, PROT_READ|PROT_WRITE, MAP_SHARED, dedupfd, 0);

src/pilerexport.c

@@ -8,11 +8,13 @@
 #include <sys/time.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/mman.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <time.h>
 #include <locale.h>
 #include <syslog.h>
+#include <zip.h>
 #include <getopt.h>
 #include <piler.h>
 
@@ -22,13 +24,14 @@ extern int optind;
 
 int dryrun = 0;
 int exportall = 0;
+int verification_status = 0;
 int rc = 0;
 char *query=NULL;
 int verbosity = 0;
 int max_matches = 1000;
 char *index_list = "main1,dailydelta1,delta1";
 regex_t regexp;
-
+char *zipfile = NULL;
 
 int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg);
 
@@ -48,6 +51,7 @@ void usage(){
    printf("    -w <where condition>              Where condition to pass to sphinx, eg. \"match('@subject: piler')\"\n");
    printf("    -m <max. matches>                 Max. matches to apply to sphinx query (default: %d)\n", max_matches);
    printf("    -i <index list>                   Sphinx indices to use  (default: %s)\n", index_list);
+   printf("    -z <zip file>                     Write exported EML files to a zip file\n");
    printf("    -A                                Export all emails from archive\n");
    printf("    -d                                Dry run\n");
 
@@ -87,15 +91,12 @@ unsigned long convert_time(char *yyyymmdd, int h, int m, int s){
 
    tm.tm_mday = atoi(yyyymmdd);
 
-
-   tm.tm_isdst = -1;
-
    return mktime(&tm);
 }
 
 
 int append_email_to_buffer(char **buffer, char *email){
-   int len, arglen;
+   int arglen;
    char *s=NULL, emailaddress[SMALLBUFSIZE];
 
    snprintf(emailaddress, sizeof(emailaddress)-1, "'%s'", email);
@@ -107,7 +108,7 @@ int append_email_to_buffer(char **buffer, char *email){
       memcpy(*buffer, emailaddress, arglen);
    }
    else {
-      len = strlen(*buffer);
+      int len = strlen(*buffer);
       s = realloc(*buffer, len + arglen+2);
       if(!s){
          printf("malloc problem!\n");
@@ -126,7 +127,7 @@ int append_email_to_buffer(char **buffer, char *email){
 
 
 int append_string_to_buffer(char **buffer, char *str){
-   int len, arglen;
+   int arglen;
    char *s=NULL;
 
    arglen = strlen(str);
@@ -137,7 +138,7 @@ int append_string_to_buffer(char **buffer, char *str){
       memcpy(*buffer, str, arglen);
    }
    else {
-      len = strlen(*buffer);
+      int len = strlen(*buffer);
       s = realloc(*buffer, len + arglen+1);
       if(!s) return 1;
 
@@ -152,9 +153,7 @@ int append_string_to_buffer(char **buffer, char *str){
 
 
 uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *where_condition, uint64 last_id, int *num, struct config *cfg){
-   MYSQL_RES *res;
    MYSQL_ROW row;
-   int rc=0;
    uint64 id=0;
    char s[SMALLBUFSIZE];
 
@@ -168,7 +167,7 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *
    snprintf(s, sizeof(s)-1, "SELECT id FROM %s WHERE %s AND id > %llu ORDER BY id ASC LIMIT 0,%d", index_list, where_condition, last_id, max_matches);
 
    if(mysql_real_query(&(sdata2->mysql), s, strlen(s)) == 0){
-      res = mysql_store_result(&(sdata2->mysql));
+      MYSQL_RES *res = mysql_store_result(&(sdata2->mysql));
       if(res != NULL){
          while((row = mysql_fetch_row(res))){
             id = strtoull(row[0], NULL, 10);
@@ -194,12 +193,11 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *
 
 
 uint64 get_total_found(struct session_data *sdata){
-   MYSQL_RES *res;
    MYSQL_ROW row;
    uint64 total_found=0;
 
    if(mysql_real_query(&(sdata->mysql), "SHOW META LIKE 'total_found'", 28) == 0){
-      res = mysql_store_result(&(sdata->mysql));
+      MYSQL_RES *res = mysql_store_result(&(sdata->mysql));
       if(res != NULL){
          while((row = mysql_fetch_row(res))){
             total_found = strtoull(row[1], NULL, 10);
@@ -230,7 +228,6 @@ void export_emails_matching_id_list(struct session_data *sdata, struct session_d
 
 
 int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain, int minsize, int maxsize, unsigned long startdate, unsigned long stopdate){
-   int where_condition=1;
    char s[SMALLBUFSIZE];
 
    if(exportall == 1){
@@ -245,87 +242,71 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain
    rc = append_string_to_buffer(&query, s);
 
    if(from){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`from` IN (");
       rc += append_string_to_buffer(&query, from);
       rc += append_string_to_buffer(&query, ")");
 
       free(from);
-
-      where_condition++;
    }
 
    if(to){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`to` IN (");
       rc += append_string_to_buffer(&query, to);
       rc += append_string_to_buffer(&query, ")");
 
       free(to);
-
-      where_condition++;
    }
 
    if(fromdomain){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`fromdomain` IN (");
       rc += append_string_to_buffer(&query, fromdomain);
       rc += append_string_to_buffer(&query, ")");
 
       free(fromdomain);
-
-      where_condition++;
    }
 
 
    if(todomain){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`todomain` IN (");
       rc += append_string_to_buffer(&query, todomain);
       rc += append_string_to_buffer(&query, ")");
 
       free(todomain);
-
-      where_condition++;
    }
 
    if(minsize > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
       snprintf(s, sizeof(s)-1, " `size` >= %d", minsize);
       rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
    }
 
 
    if(maxsize > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
       snprintf(s, sizeof(s)-1, " `size` <= %d", maxsize);
       rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
    }
 
 
    if(startdate > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
-      snprintf(s, sizeof(s)-1, " `sent` >= %ld", startdate);
+      snprintf(s, sizeof(s)-1, " `sent` >= %lu", startdate);
       rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
    }
 
 
    if(stopdate > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
-      snprintf(s, sizeof(s)-1, " `sent` <= %ld", stopdate);
+      snprintf(s, sizeof(s)-1, " `sent` <= %lu", stopdate);
       rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
    }
 
 
@@ -335,12 +316,33 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain
 }
 
 
+int write_to_zip_file(char *filename){
+   struct zip *z=NULL;
+   int errorp, ret=ERR;
+
+   z = zip_open(zipfile, ZIP_CREATE, &errorp);
+   if(!z){
+      printf("error: error creating zip file=%s, error code=%d\n", zipfile, errorp);
+      return ret;
+   }
+
+   zip_source_t *zs = zip_source_file(z, filename, 0, 0);
+   if(zs && zip_file_add(z, filename, zs, ZIP_FL_ENC_UTF_8) >= 0){
+      ret = OK;
+   } else {
+      printf("error adding file %s: %s\n", filename, zip_strerror(z));
+   }
+
+   zip_close(z);
+
+   return ret;
+}
+
 int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg){
    FILE *f;
    uint64 id, n=0;
    char digest[SMALLBUFSIZE], bodydigest[SMALLBUFSIZE];
    char filename[SMALLBUFSIZE];
-   int rc=0;
    struct sql sql;
 
    if(prepare_sql_statement(sdata, &sql, s) == ERR) return ERR;
@@ -383,8 +385,14 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct
                if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0){
                   printf("exported: %10llu\r", n); fflush(stdout);
                }
-               else
+               else {
                   printf("verification FAILED. %s\n", filename);
+                  verification_status = 1;
+               }
+
+               if(zipfile && write_to_zip_file(filename) == OK){
+                  unlink(filename);
+               }
 
             }
             else printf("cannot open: %s\n", filename);
@@ -402,7 +410,6 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct
 ENDE:
    close_prepared_statement(&sql);
 
-
    printf("\n");
 
    return rc;
@@ -410,7 +417,7 @@ ENDE:
 
 
 int main(int argc, char **argv){
-   int c, minsize=0, maxsize=0;
+   int minsize=0, maxsize=0;
    size_t nmatch=0;
    unsigned long startdate=0, stopdate=0;
    char *configfile=CONFIG_FILE;
@@ -443,6 +450,7 @@ int main(int argc, char **argv){
             {"to-domain",    required_argument,  0,  'R' },
             {"start-date",   required_argument,  0,  'a' },
             {"stop-date",    required_argument,  0,  'b' },
+            {"zip",          required_argument,  0,  'z' },
             {"where-condition", required_argument,  0,  'w' },
             {"max-matches",  required_argument,  0,  'm' },
             {"index-list",   required_argument,  0,  'i' },
@@ -451,9 +459,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Adhv?", long_options, &option_index);
 #else
-      c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?");
+      int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Adhv?");
 #endif
 
       if(c == -1) break;
@@ -542,6 +550,10 @@ int main(int argc, char **argv){
                     index_list = optarg;
                     break;
 
+         case 'z':  zipfile = optarg;
+                    break;
+
+
          case 'd' :
                     dryrun = 1;
                     break;
@@ -602,5 +614,5 @@ int main(int argc, char **argv){
 
    close_database(&sdata);
 
-   return 0;
+   return verification_status;
 }

src/pilerimport.c

@@ -52,6 +52,7 @@ void usage(){
    printf("    -s <start position>               Start importing POP3 emails from this position\n");
    printf("    -j <failed folder>                Move failed to import emails to this folder\n");
    printf("    -a <recipient>                    Add recipient to the To:/Cc: list\n");
+   printf("    -T <id>                           Update import table at id=<id>\n");
    printf("    -D                                Dry-run, do not import anything\n");
    printf("    -o                                Only download emails for POP3/IMAP import\n");
    printf("    -r                                Remove imported emails\n");
@@ -62,7 +63,7 @@ void usage(){
 
 
 int main(int argc, char **argv){
-   int i, c, n_mbox=0;
+   int i, n_mbox=0;
    char *configfile=CONFIG_FILE, *mbox[MBOX_ARGS], *directory=NULL;
    char puf[SMALLBUFSIZE], *imapserver=NULL, *pop3server=NULL;
    struct session_data sdata;
@@ -96,6 +97,7 @@ int main(int argc, char **argv){
    memset(import.filename, 0, SMALLBUFSIZE);
    import.mboxdir = NULL;
    import.tot_msgs = 0;
+   import.table_id = 0;
    import.folder = NULL;
 
    data.import = &import;
@@ -132,8 +134,9 @@ int main(int argc, char **argv){
             {"batch-limit",  required_argument,  0,  'b' },
             {"timeout",      required_argument,  0,  't' },
             {"start-position",required_argument,  0,  's' },
+            {"table-id",     required_argument,  0,  'T' },
             {"quiet",        no_argument,        0,  'q' },
-            {"recursive",    required_argument,  0,  'R' },
+            {"recursive",    no_argument,        0,  'R' },
             {"remove-after-import",no_argument,  0,  'r' },
             {"failed-folder",  required_argument,  0,  'j' },
             {"move-folder",  required_argument,  0,  'g' },
@@ -145,9 +148,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:DRroqh?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:DRroqh?", long_options, &option_index);
 #else
-      c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:DRroqh?");
+      int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:DRroqh?");
 #endif
 
       if(c == -1) break;
@@ -257,6 +260,15 @@ int main(int argc, char **argv){
                     data.import->extra_recipient = puf;
                     break;
 
+         case 'T' :
+                    if(atoi(optarg) < 1){
+                       printf("invalid import table id: %s\n", optarg);
+                       return -1;
+                    }
+
+                    data.import->table_id = atoi(optarg);
+                    break;
+
          case 'D' :
                     data.import->dryrun = 1;
                     break;

src/pop3.c

@@ -63,7 +63,7 @@ int connect_to_pop3_server(struct data *data){
 
 
 void get_number_of_total_messages(struct data *data){
-   char *p, buf[MAXBUFSIZE];
+   char buf[MAXBUFSIZE];
 
    data->import->total_messages = 0;
 
@@ -73,7 +73,7 @@ void get_number_of_total_messages(struct data *data){
    recvtimeoutssl(data->net, buf, sizeof(buf));
 
    if(strncmp(buf, "+OK ", 4) == 0){
-      p = strchr(&buf[4], ' ');
+      char *p = strchr(&buf[4], ' ');
       if(p){
          *p = '\0';
          data->import->total_messages = atoi(&buf[4]);
@@ -86,7 +86,7 @@ void get_number_of_total_messages(struct data *data){
 
 
 int pop3_download_email(struct data *data, int i){
-   int n, fd, pos=0, readlen=0, lastpos=0, nreads=0;
+   int n, fd, pos=0, lastpos=0, nreads=0;
    char *p, buf[MAXBUFSIZE];
    char aggrbuf[3*MAXBUFSIZE];
 
@@ -108,7 +108,6 @@ int pop3_download_email(struct data *data, int i){
 
    while((n = recvtimeoutssl(data->net, buf, sizeof(buf))) > 0){
       nreads++;
-      readlen += n;
 
       if(nreads == 1){
 
@@ -167,7 +166,6 @@ void pop3_delete_message(struct data *data, int i){
 
 
 void process_pop3_emails(struct session_data *sdata, struct data *data, struct config *cfg){
-   int i=0, rc=ERR;
    char buf[MAXBUFSIZE];
 
    data->import->processed_messages = 0;
@@ -178,12 +176,12 @@ void process_pop3_emails(struct session_data *sdata, struct data *data, struct c
 
    if(data->import->total_messages <= 0) return;
 
-   for(i=data->import->start_position; i<=data->import->total_messages; i++){
+   for(int i=data->import->start_position; i<=data->import->total_messages; i++){
       if(pop3_download_email(data, i) == OK){
          if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/data->import->total_messages); fflush(stdout); }
 
          if(data->import->dryrun == 0){
-            rc = import_message(sdata, data, cfg);
+            int rc = import_message(sdata, data, cfg);
 
             if(data->import->remove_after_import == 1 && rc == OK){
                pop3_delete_message(data, i);

src/reindex.c

@@ -74,10 +74,7 @@ uint64 get_max_meta_id(struct session_data *sdata){
 
 
 uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *data, uint64 from_id, uint64 to_id, struct config *cfg){
-   FILE *f;
-   char filename[SMALLBUFSIZE];
    char s[SMALLBUFSIZE];
-   int rc=0;
    uint64 stored_id=0, reindexed=0, delta;
    struct parser_state state;
    struct sql sql;
@@ -110,45 +107,43 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da
 
       while(p_fetch_results(&sql) == OK){
 
-         if(stored_id > 0){
+         char filename[SMALLBUFSIZE];
+         snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id);
 
-            snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id);
+         FILE *f = fopen(filename, "w");
-
+         if(f){
-            f = fopen(filename, "w");
+            int rc = retrieve_email_from_archive(sdata, f, cfg);
-            if(f){
+            fclose(f);
-               rc = retrieve_email_from_archive(sdata, f, cfg);
-               fclose(f);
-
-               if(rc){
-                  printf("cannot retrieve: %s\n", filename);
-                  unlink(filename);
-                  continue;
-               }
-
-               snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
-
-               state = parse_message(sdata, 0, data, cfg);
-               post_parse(sdata, &state, cfg);
-
-               rc = store_index_data(sdata, &state, data, stored_id, cfg);
-
-               if(rc == OK) reindexed++;
-               else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename);
 
+            if(rc){
+               printf("cannot retrieve: %s\n", filename);
                unlink(filename);
-
+               continue;
-               if(progressbar){
+            }
-                  printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta));
+
-                  fflush(stdout);
+            snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
-               }
+
-
+            state = parse_message(sdata, 0, data, cfg);
+            post_parse(sdata, &state, cfg);
+
+            rc = store_index_data(sdata, &state, data, stored_id, cfg);
+
+            if(rc == OK) reindexed++;
+            else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename);
+
+            unlink(filename);
+
+            if(progressbar){
+               printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta));
+               fflush(stdout);
             }
-            else printf("cannot open: %s\n", filename);
 
          }
+         else printf("cannot open: %s\n", filename);
 
       }
 
+
       p_free_results(&sql);
    }
 
@@ -162,7 +157,7 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da
 
 
 int main(int argc, char **argv){
-   int c, all=0;
+   int all=0;
    uint64 from_id=0, to_id=0, n=0;
    char *configfile=CONFIG_FILE, *folder=NULL;
    struct session_data sdata;
@@ -171,7 +166,7 @@ int main(int argc, char **argv){
 
 
    while(1){
-      c = getopt(argc, argv, "c:f:t:F:pahv?");
+      int c = getopt(argc, argv, "c:f:t:F:pahv?");
 
       if(c == -1) break;
 
@@ -211,7 +206,7 @@ int main(int argc, char **argv){
    }
 
 
-   if(all == 0 && (from_id <= 0 || to_id <= 0) ) usage();
+   if(all == 0 && (from_id == 0 || to_id == 0) ) usage();
 
    if(!can_i_write_directory(NULL)) __fatal("cannot write current directory!");
 

src/rules.c

@@ -196,13 +196,14 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){
 
 int count_match(struct rule *p, struct parser_state *state, int size, int spam){
    int ismatch=0;
-   size_t nmatch=0;
 
    ismatch += check_spam_rule(spam, p->spam);
    ismatch += check_size_rule(size, p->size, p->_size);
    ismatch += check_attachment_rule(state, p);
 
    if(p->compiled == 1){
+      size_t nmatch=0;
+
       if(p->emptyfrom == 1){
          ismatch += RULE_UNDEF;
       }
@@ -239,7 +240,7 @@ char *check_against_ruleset(struct node *xhash[], struct parser_state *state, in
       if(q->str){
          p = q->str;
 
-         if(p && count_match(p, state, size, spam) > 0){
+         if(count_match(p, state, size, spam) > 0){
             return p->rulestr;
          }
       }
@@ -338,7 +339,6 @@ int check_spam_rule(int is_spam, int spam){
 int check_attachment_rule(struct parser_state *state, struct rule *rule){
    int i;
    size_t nmatch=0;
-   int ismatch = 0;
 
    // If no attachment rule, then return RULE_UNDEF
    if(rule->emptyaname == 1 && rule->emptyatype == 1 && rule->attachment_size == 0) return RULE_UNDEF;
@@ -348,7 +348,7 @@ int check_attachment_rule(struct parser_state *state, struct rule *rule){
 
 
    for(i=1; i<=state->n_attachments; i++){
-      ismatch = 0;
+      int ismatch = 0;
 
       if(rule->emptyaname == 0){
          if(regexec(&(rule->attachment_name), state->attachments[i].filename, nmatch, NULL, 0) == 0)
@@ -379,34 +379,32 @@ void initrules(struct node *xhash[]){
 
 
 void clearrules(struct node *xhash[]){
-   struct node *p, *q;
+   struct node *q;
    struct rule *rule;
 
    q = xhash[0];
 
    while(q != NULL){
-      p = q;
+      struct node *p = q;
       q = q->r;
 
-      if(p){
+      if(p->str){
-         if(p->str){
+         rule = (struct rule*)p->str;
-            rule = (struct rule*)p->str;
 
-            regfree(&(rule->from));
+         regfree(&(rule->from));
-            regfree(&(rule->to));
+         regfree(&(rule->to));
-            regfree(&(rule->subject));
+         regfree(&(rule->subject));
-            regfree(&(rule->body));
+         regfree(&(rule->body));
-            regfree(&(rule->attachment_name));
+         regfree(&(rule->attachment_name));
-            regfree(&(rule->attachment_type));
+         regfree(&(rule->attachment_type));
 
-            free(rule->rulestr);
+         free(rule->rulestr);
 
-            if(rule->domain) free(rule->domain);
+         if(rule->domain) free(rule->domain);
 
-            free(rule);
+         free(rule);
-         }
-         free(p);
       }
+      free(p);
    }
 
    xhash[0] = NULL;

src/session.c

@@ -30,7 +30,6 @@ int is_blocked_by_tcp_wrappers(int sd){
 
 
 int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct config *cfg){
-   char smtp_banner[SMALLBUFSIZE];
    int slot;
 
    /*
@@ -57,6 +56,8 @@ int start_new_session(struct smtp_session **sessions, int socket, int *num_conne
       sessions[slot] = malloc(sizeof(struct smtp_session));
       if(sessions[slot]){
          init_smtp_session(sessions[slot], slot, socket, cfg);
+
+         char smtp_banner[SMALLBUFSIZE];
          snprintf(smtp_banner, sizeof(smtp_banner)-1, SMTP_RESP_220_BANNER, cfg->hostid);
          send(socket, smtp_banner, strlen(smtp_banner), 0);
 
@@ -233,14 +234,12 @@ void handle_data(struct smtp_session *session, char *readbuf, int readlen, struc
 
 
 void write_envelope_addresses(struct smtp_session *session, struct config *cfg){
-   int i;
-   char *p, s[SMALLBUFSIZE];
-
    if(session->fd == -1) return;
 
-   for(i=0; i<session->num_of_rcpt_to; i++){
+   for(int i=0; i<session->num_of_rcpt_to; i++){
-      p = strchr(session->rcptto[i], '@');
+      char *p = strchr(session->rcptto[i], '@');
       if(p && strncmp(p+1, cfg->hostid, cfg->hostid_len)){
+         char s[SMALLBUFSIZE];
          snprintf(s, sizeof(s)-1, "X-Piler-Envelope-To: %s\n", session->rcptto[i]);
          if(write(session->fd, s, strlen(s)) == -1) syslog(LOG_PRIORITY, "ERROR: %s: cannot write envelope to address", session->ttmpfile);
       }

src/smtp.c

@@ -72,15 +72,15 @@ void process_smtp_command(struct smtp_session *session, char *buf, struct config
 
 
 void process_data(struct smtp_session *session, char *buf, int buflen){
-   int len=0, written=0, n_writes=0;
-
    if(session->last_data_char == '\n' && strcmp(buf, ".\r\n") == 0){
       process_command_period(session);
    }
    else {
       // write line to file
+      int written=0, n_writes=0;
+
       while(written < buflen) {
-         len = write(session->fd, buf+written, buflen-written);
+         int len = write(session->fd, buf+written, buflen-written);
          n_writes++;
 
          if(len > 0){
@@ -98,7 +98,6 @@ void process_data(struct smtp_session *session, char *buf, int buflen){
 
 void wait_for_ssl_accept(struct smtp_session *session){
    int rc;
-   char ssl_error[SMALLBUFSIZE];
 
    if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "waiting for ssl handshake");
 
@@ -115,6 +114,8 @@ void wait_for_ssl_accept(struct smtp_session *session){
    }
 
    if(session->cfg->verbosity >= _LOG_DEBUG || session->net.use_ssl == 0){
+      char ssl_error[SMALLBUFSIZE];
+
       ERR_error_string_n(ERR_get_error(), ssl_error, SMALLBUFSIZE);
       syslog(LOG_PRIORITY, "SSL_accept() result, rc=%d, errorcode: %d, error text: %s",
              rc, SSL_get_error(session->net.ssl, rc), ssl_error);

src/stats.c

@@ -134,15 +134,15 @@ void sphinx_queries(struct session_data *sdata, struct stats *stats){
 
 void count_error_emails(struct stats *stats){
    DIR *dir;
-   struct dirent *de;
    struct stat st;
-   char buf[SMALLBUFSIZE];
 
    dir = opendir(ERROR_DIR);
    if(dir){
+      struct dirent *de;
       while((de = readdir(dir))){
          if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
 
+         char buf[SMALLBUFSIZE];
          snprintf(buf, sizeof(buf)-1, "%s/%s", ERROR_DIR, de->d_name);
 
          if(stat(buf, &st) == 0 && S_ISREG(st.st_mode)){

src/store.c

@@ -213,12 +213,11 @@ ENDE:
 
 
 int remove_stored_message_files(struct session_data *sdata, struct parser_state *state, struct config *cfg){
-   int i;
    char s[SMALLBUFSIZE];
 
    if(state->n_attachments > 0){
 
-      for(i=1; i<=state->n_attachments; i++){
+      for(int i=1; i<=state->n_attachments; i++){
          snprintf(s, sizeof(s)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, sdata->ttmpfile[8], sdata->ttmpfile[9], sdata->ttmpfile[10], sdata->ttmpfile[RND_STR_LEN-4], sdata->ttmpfile[RND_STR_LEN-3], sdata->ttmpfile[RND_STR_LEN-2], sdata->ttmpfile[RND_STR_LEN-1], sdata->ttmpfile, i);
 
          if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: unlinking %s", sdata->ttmpfile, s);

src/tai.c

@@ -4,7 +4,6 @@
 #include <sys/time.h>
 #include "tai.h"
 
-static char hex[16] = "0123456789abcdef";
 
 
 void tai_pack(char *s, struct tai *t){
@@ -50,21 +49,3 @@ void taia_now(struct taia *t){
    t->nano = 1000 * now.tv_usec + 500;
    t->atto = 0;
 }
-
-
-void tai_timestamp(char *s){
-   struct tai now;
-   char nowpack[TAI_PACK];
-   int i;
-
-   now.x = 4611686018427387914ULL + (uint64)time((long *) 0);
-
-   tai_pack(nowpack, &now);
-
-   for (i = 0;i < 8;++i) {
-      *(s+i*2) = hex[(nowpack[i] >> 4) & 15];
-      *(s+i*2+1) = hex[nowpack[i] & 15];
-   }
-
-   *(s+2*TAI_PACK) = '\0';
-}

src/test.c

@@ -28,7 +28,7 @@ void usage(){
 
 
 int main(int argc, char **argv){
-   int i, c;
+   int i;
    time_t retention_seconds=0;
    struct stat st;
    struct session_data sdata;
@@ -57,9 +57,9 @@ int main(int argc, char **argv){
       int option_index = 0;
 
 
-      c = getopt_long(argc, argv, "c:m:a:hv?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:m:a:hv?", long_options, &option_index);
 #else
-      c = getopt(argc, argv, "c:m:a:hv?");
+      int c = getopt(argc, argv, "c:m:a:hv?");
 #endif
 
       if(c == -1) break;

suppressions.txt

@@ -0,0 +1,13 @@
+identicalConditionAfterEarlyExit:src/tokenizer.c:40
+invalidPrintfArgType_s:src/extract.c:175
+invalidPrintfArgType_s:src/misc.c:39
+invalidPrintfArgType_s:src/misc.c:43
+invalidPrintfArgType_s:src/misc.c:47
+invalidPrintfArgType_s:src/misc.c:51
+invalidPrintfArgType_s:src/misc.c:55
+invalidPrintfArgType_s:src/misc.c:59
+invalidPrintfArgType_s:src/misc.c:63
+redundantAssignment:src/imap.c:47
+unusedFunction:src/sig.c:33
+unusedFunction:src/sig.c:38
+unusedFunction:src/sig.c:44

tests/cases/05-smtp.inc

@@ -17,14 +17,18 @@ case1() {
    "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus" --socket --no-counter
 
 
-   wait_until_emails_are_processed "piler1" 2999
+   wait_until_emails_are_processed "piler1" 3000
    docker exec "piler1" su piler -c /usr/libexec/piler/indexer.delta.sh 2>/dev/null
 
-   count_status_values 2999 2891 108 0
+   count_status_values 3000 2892 108 0
 
    test_retrieved_messages_are_the_same "piler1" "piler"
 
    run_05_sphinx_tests
+
+   docker exec "piler1" su piler -c 'php /usr/libexec/piler/generate_stats.php --webui /var/piler/www --start=2015/01/01 --stop=2020/12/31'
+
+   docker exec "piler1" su piler -c 'php /usr/libexec/piler/sign.php --webui /var/piler/www --mode time'
 }
 
 

tests/setup.inc

@@ -22,7 +22,7 @@ launch_containers() {
    docker run -d --net=piler "${DOCKER_LIMIT[@]}" --name piler1 \
       -e PACKAGE="$PACKAGE" \
       -e PILER_HOST="cust1.acts.hu" \
-      -p 80:80 -p 25:25 \
+      -p 127.0.0.1:80:80 -p 25:25 \
       -v "${PACKAGE_DIR}:/data:ro" \
       -v "${CONFIG_DIR}/11-aaaa.conf:/etc/rsyslog.d/11-aaaa.conf:ro" \
       "$docker_image"
@@ -33,12 +33,13 @@ launch_containers() {
 create_rules() {
    local container="$1"
 
-   echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"acts.hu\",\"acts.hu\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh
+   echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"acts.hu\",\"acts.hu\"),(\"gtsce.com\",\"gtsce.com\"),(\"datanet.hu\",\"datanet.hu\"),(\"gtsdatanet.hu\",\"gtsdatanet.hu\"),(\"gts.hu\",\"gts.hu\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh
 
    echo 'echo "insert into archiving_rule (subject) values (\"Android táblagép\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
    echo 'echo "insert into archiving_rule (\`from\`) values (\"@gmail.com\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
    echo 'echo "insert into archiving_rule (\`from\`,attachment_type, _attachment_size, attachment_size) values (\"finderis.co.ua\", \"image\", \">\", 100000)"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
    echo 'echo "insert into archiving_rule (\`to\`) values (\"undisclosed-recipients\")"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
+   echo 'echo "insert into import (\`type\`, username, password, server) values (\"imap\", \"sanyi@aaa.fu\", \"abcde123\", \"imap\")"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
 
    echo 'echo "update user set password=\"\$6\$GKL00T\$8jqoFOe3PyAbOCLwKB7JwndwC.IinHrZRkdoQDZUc8vybZ88sA2qomlz5JceNif8fFpkGzZ03ilvQa7tqQx0v1\""| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
 

unit_tests/check_digest.c

@@ -5,7 +5,7 @@
 #include "test.h"
 
 
-struct digest_test tests[] = {
+struct digest_test emls[] = {
    {"1.eml", "7fa4f06f7085986007454b374f6685e4cf838d9e9f8878f3cba89cfe98e29f56", "bbbafb22e2d4c035584a5024e9d4feaf32172559b4d7cacc8b7af4bd548da53e"},
    {"2.eml", "668cb3b91b944af786667323442576b9813d65f3cd3bc33e9d5da303c79de038", "de90475409dd6ab24e80c1b7a987715c40fe8d28d91337b7f063b477159c7b3c"},
    {"3.eml", "0d546d4cb4a8ce74ea5fd4cc51dbb4ebeaa7542f1c691817579da7eeab8d4771", "f585d011340d292ee52ddedb07cda662a8f1e46329d14a2ce92dca0604387bab"},
@@ -41,9 +41,9 @@ static void test_digest_file(){
    unsigned int i;
    char digest[2*DIGEST_LENGTH+1];
 
-   for(i=0; i<sizeof(tests)/sizeof(struct digest_test); i++){
+   for(i=0; i<sizeof(emls)/sizeof(struct digest_test); i++){
-      digest_file(tests[i].s, &digest[0]);
+      digest_file(emls[i].s, &digest[0]);
-      assert(strcmp(digest, tests[i].digest2) == 0 && "test_digest_file()");
+      assert(strcmp(digest, emls[i].digest2) == 0 && "test_digest_file()");
    }
 
    printf("test_digest_file() OK\n");
@@ -64,8 +64,8 @@ static void test_make_digests(struct config *cfg){
    }
 
 
-   for(i=0; i<sizeof(tests)/sizeof(struct digest_test); i++){
+   for(i=0; i<sizeof(emls)/sizeof(struct digest_test); i++){
-      if(setup_and_parse_message(&sdata, &state, &data, tests[i].s, cfg) == 1){
+      if(setup_and_parse_message(&sdata, &state, &data, emls[i].s, cfg) == 1){
          continue;
       }
 
@@ -79,8 +79,8 @@ static void test_make_digests(struct config *cfg){
 
       unlink(sdata.tmpframe);
 
-      assert(strcmp(sdata.bodydigest, tests[i].digest1) == 0 && "test_make_digests()");
+      assert(strcmp(sdata.bodydigest, emls[i].digest1) == 0 && "test_make_digests()");
-      assert(strcmp(sdata.digest, tests[i].digest2) == 0 && "test_make_digests()");
+      assert(strcmp(sdata.digest, emls[i].digest2) == 0 && "test_make_digests()");
 
    }
 

unit_tests/check_misc.c

@@ -123,7 +123,7 @@ static void test_create_id(){
 static void test_split(){
    unsigned int i;
    int result;
-   char *p, buf[SMALLBUFSIZE];
+   char buf[SMALLBUFSIZE];
 
    struct test_data_s_s_i test_data_s_s_i[] = {
       { "hello\nworld\n", "world\n", 1 },
@@ -135,7 +135,7 @@ static void test_split(){
    TEST_HEADER();
 
    for(i=0; i<sizeof(test_data_s_s_i)/sizeof(struct test_data_s_s_i); i++){
-      p = split(test_data_s_s_i[i].s1, '\n', buf, sizeof(buf)-1, &result);
+      char *p = split(test_data_s_s_i[i].s1, '\n', buf, sizeof(buf)-1, &result);
 
       if(p){ ASSERT(strcmp(buf, "hello") == 0 && strcmp(p, test_data_s_s_i[i].s2) == 0 && result == test_data_s_s_i[i].result, test_data_s_s_i[i].s1); }
       else { ASSERT(p == NULL && result == test_data_s_s_i[i].result, test_data_s_s_i[i].s1); }
@@ -147,7 +147,7 @@ static void test_split(){
 
 static void test_split_str(){
    unsigned int i;
-   char *p, buf[SMALLBUFSIZE];
+   char buf[SMALLBUFSIZE];
 
    struct test_data_s_s test_data_s_s[] = {
       { "aaaXXbbbXX", "bbbXX" },
@@ -159,7 +159,7 @@ static void test_split_str(){
    TEST_HEADER();
 
    for(i=0; i<sizeof(test_data_s_s)/sizeof(struct test_data_s_s); i++){
-      p = split_str(test_data_s_s[i].s, "XX", buf, sizeof(buf)-1);
+      char *p = split_str(test_data_s_s[i].s, "XX", buf, sizeof(buf)-1);
 
       if(p){ ASSERT(strcmp(buf, "aaa") == 0 && strcmp(test_data_s_s[i].result, p) == 0, test_data_s_s[i].s); }
       else { ASSERT(strcmp(buf, "aaa") == 0 && p == NULL, test_data_s_s[i].s); }

unit_tests/check_parser_utils.c

@@ -25,7 +25,6 @@ struct str_pair {
 static void test_parse_date_header(){
    unsigned int i;
    struct config cfg;
-   time_t delta;
    struct date_test date_test[] = {
       {"Date: Mon, 02 Nov 2015 09:39:31 -0000", 1446457171},
       {"Date: Mon, 2 Nov 2015 10:39:45 +0100", 1446457185},
@@ -49,7 +48,7 @@ static void test_parse_date_header(){
    TEST_HEADER();
 
    for(i=0; i<sizeof(date_test)/sizeof(struct date_test); i++){
-      delta = parse_date_header(date_test[i].date_str) - date_test[i].timestamp;
+      time_t delta = parse_date_header(date_test[i].date_str) - date_test[i].timestamp;
       ASSERT(delta <= 3600 && delta >= -3600, date_test[i].date_str);
    }
 

unit_tests/smtp.c

@@ -27,10 +27,10 @@ void connect_to_smtp_server(char *server, int port, struct data *data){
    char port_string[8], buf[MAXBUFSIZE];
    struct addrinfo hints, *res;
 
-   data->net->socket = -1;
-
    if(data == NULL) return;
 
+   data->net->socket = -1;
+
    snprintf(port_string, sizeof(port_string)-1, "%d", port);
 
    memset(&hints, 0, sizeof(hints));
@@ -301,7 +301,7 @@ static void test_smtp_commands_period_command_in_its_own_packet(char *server, in
 
 
 int main(int argc, char **argv){
-   int c, port=25;
+   int port=25;
    char *server=NULL;
    struct data data;
    struct net net;
@@ -325,9 +325,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      c = getopt_long(argc, argv, "c:s:p:t:lh?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:s:p:t:lh?", long_options, &option_index);
 #else
-      c = getopt(argc, argv, "c:s:p:t:lh?");
+      int c = getopt(argc, argv, "c:s:p:t:lh?");
 #endif
 
 

util/Makefile.in

@@ -35,6 +35,7 @@ install:
 	$(INSTALL) -m 0755 $(srcdir)/daily-report.php $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/gmail-imap-import.php $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/generate_stats.php $(DESTDIR)$(libexecdir)/piler
+	$(INSTALL) -m 0755 $(srcdir)/healthcheck.sh $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/mailstat.php $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/sign.php $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/indexer.delta.sh $(DESTDIR)$(libexecdir)/piler

util/healthcheck.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+SPHINX_DIR=/var/piler/sphinx
+ERROR_FILE=/var/piler/stat/error
+
+if command -v mpstat; then
+   mpstat
+   echo
+else
+   echo "host: $(hostname)"
+   echo "cpus: $(grep -c ^processor /proc/cpuinfo)"
+fi
+
+echo "load: $(cat /proc/loadavg)"
+echo "mem: $(grep MemTotal /proc/meminfo | sed 's/MemTotal\s*:\s*//' )"
+echo disks:
+while read -r p; do df -h "${p##* }" | tail -1; done < <(lsblk | grep part)
+
+echo
+piler -V
+
+# shellcheck disable=SC2009
+ps uaxw|grep piler
+
+echo -e "\nCron entries:\n"
+crontab -l -u piler
+
+errors=0
+if [[ -f "$ERROR_FILE" ]]; then
+   read -r errors < "$ERROR_FILE"
+else
+   errors="$(find /var/piler/error -type f|wc -l)"
+fi
+
+mysqluser="$(pilerconf -q mysqluser|cut -d = -f2)"
+mysqlpwd="$(pilerconf -q mysqlpwd|cut -d = -f2)"
+mysqldb="$(pilerconf -q mysqldb|cut -d = -f2)"
+
+echo -e "\nError emails: $errors"
+echo -e "Sphinx data: $(du -hs "$SPHINX_DIR")\n"
+
+mysql -t -u "$mysqluser" -p"$mysqlpwd" "$mysqldb" <<< "select * from counter"
+mysql -t -u "$mysqluser" -p"$mysqlpwd" information_schema <<< "select table_schema as db, sum(data_length+index_length) as size from TABLES WHERE table_schema='$mysqldb' GROUP BY table_schema"

util/imapfetch.py

@@ -1,13 +1,33 @@
 #!/usr/bin/python3
 # -*- coding: utf-8 -*-
 
+import MySQLdb as dbapi
 import argparse
+import configparser
 import imaplib
-import pprint
+import os
 import re
+import subprocess
+import sys
 
 opts = {}
 INBOX = 'INBOX'
+ST_RUNNING = 1
+
+
+def read_options(filename="", opts={}):
+    s = "[piler]\n" + open(filename, 'r').read()
+    config = configparser.ConfigParser()
+    config.read_string(s)
+
+    if config.has_option('piler', 'mysqlhost'):
+        opts['dbhost'] = config.get('piler', 'mysqlhost')
+    else:
+        opts['dbhost'] = 'localhost'
+
+    opts['username'] = config.get('piler', 'mysqluser')
+    opts['password'] = config.get('piler', 'mysqlpwd')
+    opts['database'] = config.get('piler', 'mysqldb')
 
 
 def read_folder_list(conn):
@@ -35,51 +55,105 @@ def read_folder_list(conn):
 
 
 def process_folder(conn, folder):
-    print("Processing {}".format(folder))
+    if opts['verbose']:
+        print("Processing {}".format(folder))
 
     rc, data = conn.select(folder)
     n = int(data[0])
-    print("Folder {} has {} messages".format(folder, n))
+    if opts['verbose']:
+        print("Folder {} has {} messages".format(folder, n))
 
     if n > 0:
-        rc, data = conn.search(None, 'ALL')
+        if opts['id']:
+            cursor = opts['db'].cursor()
+            data = (ST_RUNNING, n, opts['id'])
+            cursor.execute("UPDATE import SET status=%s, total=total+%s WHERE id=%s", data)
+            opts['db'].commit()
+
+        rc, data = conn.search(None, opts['search'])
         for num in data[0].split():
             rc, data = conn.fetch(num, '(RFC822)')
             if opts['verbose']:
                 print(rc, num)
-            opts['counter'] = opts['counter'] + 1
+            opts['counter'] += 1
             with open("{}.eml".format(opts['counter']), "wb") as f:
                 f.write(data[0][1])
 
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("-s", "--server", type=str, help="imap server", required=True)
+    parser.add_argument("-c", "--config", type=str, help="piler.conf path",
+                        default="/etc/piler/piler.conf")
+    parser.add_argument("-s", "--server", type=str, help="imap server")
     parser.add_argument("-P", "--port", type=int, help="port number", default=143)
-    parser.add_argument("-u", "--user", type=str, help="imap user", required=True)
+    parser.add_argument("-u", "--user", type=str, help="imap user")
-    parser.add_argument("-p", "--password", type=str, help="imap password",
+    parser.add_argument("-p", "--password", type=str, help="imap password")
-                        required=True)
     parser.add_argument("-x", "--skip-list", type=str, help="IMAP folders to skip",
                         default="junk,trash,spam,draft")
     parser.add_argument("-f", "--folders", type=str,
                         help="Comma separated list of IMAP folders to download")
+    parser.add_argument("--date", type=str, help="Search before/since a given date," +
+                        "eg. (BEFORE \"01-Jan-2020\") or (SINCE \"01-Jan-2020\")")
+    parser.add_argument("-d", "--dir", help="directory to chdir",
+                        default="/var/piler/imap")
+    parser.add_argument("-i", "--import-from-table", action='store_true',
+                        help="Read imap conn data from import table")
     parser.add_argument("-v", "--verbose", help="verbose mode", action='store_true')
 
     args = parser.parse_args()
 
+    os.chdir(args.dir)
+
+    if not bool(args.import_from_table or args.server):
+        print("Please specify either --import-from-table or --server <imap host>")
+        sys.exit(1)
+
     opts['skip_folders'] = args.skip_list.split(',')
     opts['verbose'] = args.verbose
+    opts['search'] = 'ALL'
     opts['counter'] = 0
+    opts['db'] = None
+    opts['id'] = 0
+
+    if args.date:
+        opts['search'] = args.date
+
+    server = ''
+    user = ''
+    password = ''
+
+    if args.import_from_table:
+        read_options(args.config, opts)
+        try:
+            opts['db'] = dbapi.connect(opts['dbhost'], opts['username'],
+                                       opts['password'], opts['database'])
+
+            cursor = opts['db'].cursor()
+            cursor.execute("SELECT id, server, username, password FROM import WHERE started=0")
+
+            row = cursor.fetchone()
+            if row:
+                (opts['id'], server, user, password) = row
+            else:
+                print("Nothing to read from import table")
+                sys.exit(0)
+
+        except dbapi.DatabaseError as e:
+            print("Error %s" % e)
+    else:
+        server = args.server
+        user = args.user
+        password = args.password
 
     if opts['verbose']:
         print("Skipped folder list: {}".format(opts['skip_folders']))
 
     if args.port == 993:
-        conn = imaplib.IMAP4_SSL(args.server)
+        conn = imaplib.IMAP4_SSL(server)
     else:
-        conn = imaplib.IMAP4(args.server)
+        conn = imaplib.IMAP4(server)
 
-    conn.login(args.user, args.password)
+    conn.login(user, password)
     conn.select()
 
     if args.folders:
@@ -95,6 +169,16 @@ def main():
 
     conn.close()
 
+    if opts['db']:
+        if opts['id']:
+            subprocess.call(["pilerimport",
+                             "-d", args.dir,
+                             "-r",
+                             "-T", str(opts['id'])])
+        opts['db'].close()
+
+    print("Processed {} messages".format(opts['counter']))
+
 
 if __name__ == "__main__":
     main()

util/import.sh

@@ -1,21 +1,12 @@
 #!/bin/bash
 
-export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
+set -o errexit
-PRIORITY=mail.error
+set -o pipefail
-TMPFILE=/var/run/piler/import.tmp
 
-if [ -f $TMPFILE ]; then exit 1; fi
+export PATH=$PATH:/usr/libexec/piler:/usr/local/libexec/piler
 
-date > $TMPFILE
+pushd /var/piler/imap
-
-function finish {
-   rm -f $TMPFILE
-}
-
-trap finish EXIT
-
-cd /var/piler/imap
-
-pilerimport -G >/dev/null
 
+[[ "${FLOCKER}" != "$0" ]] && exec env FLOCKER="$0" flock -en "$0" "$0" "$@"
 
+imapfetch.py -i

util/pilerpurge.py

@@ -93,7 +93,12 @@ def purge_attachments_by_attachment_id(opts={}):
         if rows == ():
             break
         else:
-            remove_attachment_files(rows, opts)
+            for (id, piler_id, attachment_id, refcount) in rows:
+                if opts['dry_run'] is False:
+                    unlink(get_attachment_file_path(piler_id, attachment_id,
+                                                    opts), opts)
+                else:
+                    print(get_attachment_file_path(piler_id, attachment_id, opts))
 
 
 def remove_attachment_files(rows=(), opts={}):

util/purge.sh

@@ -1,11 +1,14 @@
 #!/bin/bash
 
+set -o nounset
+set -o errexit
+set -o pipefail
+
 export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/libexec/piler:/usr/local/libexec/piler
-PRIORITY=mail.error
 TMPFILE=/var/run/piler/purge.tmp
 PURGE_BEACON=/var/piler/stat/purge
 
-if [ -f $TMPFILE ]; then exit 1; fi
+if [[ -f $TMPFILE ]]; then exit 1; fi
 
 date > $TMPFILE
 

util/sign.php

@@ -1,11 +1,11 @@
 <?php
 
-define(COUNT, 'count');
+define('COUNT', 'count');
-define(HASH_VALUE, 'hash_value');
+define('HASH_VALUE', 'hash_value');
-define(RESPONSE_STRING, 'response_string');
+define('RESPONSE_STRING', 'response_string');
-define(RESPONSE_TIME, 'response_time');
+define('RESPONSE_TIME', 'response_time');
-define(START_ID, 'start_id');
+define('START_ID', 'start_id');
-define(STOP_ID, 'stop_id');
+define('STOP_ID', 'stop_id');
 
 ini_set("session.save_path", "/tmp");
 
@@ -82,9 +82,12 @@ if(MODE == 'time' && $data[COUNT] < 1) {
    exit;
 }
 
-$requestfile_path = TrustedTimestamps::createRequestfile($data[HASH_VALUE]);
+try {
-
+   $requestfile_path = TrustedTimestamps::createRequestfile($data[HASH_VALUE]);
-$response = TrustedTimestamps::signRequestfile($requestfile_path, TSA_URL);
+   $response = TrustedTimestamps::signRequestfile($requestfile_path, TSA_URL);
+} catch(Exception $e) {
+   die("Error: " . $e->getMessage() . "\n");
+}
 
 $data[RESPONSE_STRING] = $response[RESPONSE_STRING];
 $data[RESPONSE_TIME] = $response[RESPONSE_TIME];
@@ -136,12 +139,12 @@ function get_hash_values() {
 
    if(MODE == 'time') { $stop_id = $start_id + $count - 1; }
 
-   return array(
+   return [
-                START_ID => $start_id,
+      START_ID => $start_id,
-                STOP_ID => $stop_id,
+      STOP_ID => $stop_id,
-                COUNT => $count,
+      COUNT => $count,
-                HASH_VALUE => sha1($s)
+      HASH_VALUE => sha1($s)
-          );
+   ];
 
 }
 

webui/controller/search/helper.php

@@ -8,7 +8,6 @@ class ControllerSearchHelper extends Controller {
                     'date2'           => '',
                     'direction'       => '',
                     'size'            => '',
-                    'aname'           => '',
                     'attachment_type' => '',
                     'tag'             => '',
                     'note'            => '',
@@ -50,7 +49,7 @@ class ControllerSearchHelper extends Controller {
 
       if($this->request->post['searchtype'] == 'expert'){
 
-         if(isset($this->request->post['search']) && preg_match("/(from|to|subject|body|direction|d|size|date1|date2|attachment|a|aname|tag|note|id)\:/", $this->request->post['search'])) {
+         if(isset($this->request->post['search']) && preg_match("/(from|to|subject|body|direction|d|size|date1|date2|attachment|a|tag|note|id)\:/", $this->request->post['search'])) {
             $this->a = $this->model_search_search->preprocess_post_expert_request($this->request->post);
          }
          else {

webui/controller/user/settings.php

@@ -53,6 +53,11 @@ class ControllerUserSettings extends Controller {
       if($auditgroups) { $this->data['groups'] = $auditgroups; } else { $this->data['groups'] = $this->data['text_none_found']; }
       if($auditfolders) { $this->data['folders'] = $auditfolders; } else { $this->data['folders'] = $this->data['text_none_found']; }
 
+      $this->data['wildcard_domains'] = $session->get("wildcard_domains");
+      if($this->data['wildcard_domains']) {
+         $this->data['wildcard_domains'] = implode(", ", $this->data['wildcard_domains']);
+      }
+
       if(isset($this->request->post['pagelen']) && isset($this->request->post['theme'])) {
          $this->model_user_prefs->set_user_preferences(Registry::get('username'), $this->request->post);
 

webui/language/cz/messages.php

@@ -491,3 +491,4 @@ $_['text_user_data_officer'] = "Data officer";
 $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
 
+$_['text_wildcard_domains'] = "Wildcard domains";

webui/language/de/messages.php

@@ -498,3 +498,4 @@ $_['text_user_data_officer'] = "Data officer";
 $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
 
+$_['text_wildcard_domains'] = "Wildcard domains";

webui/language/en/messages.php

@@ -412,6 +412,7 @@ $_['text_view_raw_email'] = "View raw email";
 $_['text_view_user_quarantine'] = "View user's quarantine";
 
 $_['text_warning_about_default_policy'] = "The default policy can be set in clapf.conf";
+$_['text_wildcard_domains'] = "Wildcard domains";
 $_['text_whitelist'] = "Whitelist";
 $_['text_whitelist_settings'] = "Whitelist settings";
 $_['text_with_attachment'] = "with attachment(s)";

webui/language/es/messages.php

@@ -496,3 +496,4 @@ $_['text_user_data_officer'] = "Data officer";
 $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
 
+$_['text_wildcard_domains'] = "Wildcard domains";

webui/language/fr/messages.php

@@ -493,3 +493,4 @@ $_['text_user_data_officer'] = "Data officer";
 $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
 
+$_['text_wildcard_domains'] = "Wildcard domains";

webui/language/hu/messages.php

@@ -414,6 +414,7 @@ $_['text_view_raw_email'] = "Formázatlan levél megtekintése";
 $_['text_view_user_quarantine'] = "Felhasználó karantén megtekintése";
 
 $_['text_warning_about_default_policy'] = "Az alapértelmezett házirend a clapf.conf fájlban van";
+$_['text_wildcard_domains'] = "Wildcard domének";
 $_['text_whitelist'] = "Fehérlista";
 $_['text_whitelist_settings'] = "Fehérlista beállítások";
 $_['text_with_attachment'] = "mellékletekkel";

webui/language/pl/messages.php

@@ -492,3 +492,4 @@ $_['text_user_data_officer'] = "Data officer";
 
 $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
+$_['text_wildcard_domains'] = "Wildcard domains";

webui/language/pt/messages.php

@@ -485,3 +485,4 @@ $_['text_user_data_officer'] = "Data officer";
 
 $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
+$_['text_wildcard_domains'] = "Wildcard domains";

webui/language/ru/messages.php

@@ -493,3 +493,4 @@ $_['text_user_data_officer'] = "Data officer";
 
 $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
+$_['text_wildcard_domains'] = "Wildcard domains";

webui/language/tr/messages.php

@@ -493,3 +493,4 @@ $_['text_user_data_officer'] = "Data officer";
 
 $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
+$_['text_wildcard_domains'] = "Wildcard domains";

webui/model/folder/folder.php

@@ -236,8 +236,8 @@ class ModelFolderFolder extends Model {
          $last_id = $this->db->getLastId();
          $query = $this->db->query("INSERT INTO " . TABLE_FOLDER_USER . " (id, uid) VALUES(?,?)", array($last_id, $session->get("uid")));
 
-         $folders = $session->get("folders");
+         $folders = $this->get_folder_id_array_for_user($session->get("uid"), $session->get("admin_user"));
-         if(!isset($folders[$last_id])) { array_push($folders, $last_id); $session->set("folders", $folders); }
+         $session->set("folders", $folders);
       }
 
       return $this->db->countAffected();
@@ -253,8 +253,8 @@ class ModelFolderFolder extends Model {
          $query = $this->db->query("DELETE FROM " . TABLE_FOLDER . " WHERE id=?", array($id));
          $query = $this->db->query("DELETE FROM " . TABLE_FOLDER_USER . " WHERE id=? AND uid=?", array($id, $session->get("uid")));
 
-         $folders = $session->get("folders");
+         $folders = $this->get_folder_id_array_for_user($session->get("uid"), $session->get("admin_user"));
-         if(isset($folders[$id])) { unset($folders[$id]); $session->set("folders", $folders); }
+         $session->set("folders", $folders);
 
          // shall we delete the existing message - folder id assignments from folder_message?
       }
@@ -264,5 +264,3 @@ class ModelFolderFolder extends Model {
 
 
 }
-
-?>

webui/model/saas/ldap.php

@@ -82,6 +82,7 @@ class ModelSaasLdap extends Model
    public function get_accounts_in_domain($domain = '') {
       $ldap_type = '';
       $ldap_host = LDAP_HOST;
+      $ldap_port = LDAP_PORT;
       $ldap_base_dn = LDAP_BASE_DN;
       $ldap_helper_dn = LDAP_HELPER_DN;
       $ldap_helper_password = LDAP_HELPER_PASSWORD;
@@ -102,7 +103,7 @@ class ModelSaasLdap extends Model
 
       if($ldap_host == '' || $ldap_helper_password == '') { return array(); }
 
-      $ldap = new LDAP($ldap_host, $ldap_helper_dn, $ldap_helper_password);
+      $ldap = new LDAP($ldap_host, $ldap_port, $ldap_helper_dn, $ldap_helper_password);
 
       if($ldap->is_bind_ok()) {
 

webui/model/search/message.php

@@ -185,10 +185,10 @@ class ModelSearchMessage extends Model {
          $body = Piler_Mime_Decode::fixMimeBodyPart($parts[$i]['headers'], $parts[$i]['body']);
 
          if($parts[$i]['headers']['content-type']['type'] == 'text/html') {
-            $this->message['text/html'] = $purifier->purify($body);
+            $this->message['text/html'] .= $purifier->purify($body);
          }
          else {
-            $this->message['text/plain'] = $body;
+            $this->message['text/plain'] .= $body;
          }
 
       }

webui/model/search/search.php

@@ -81,8 +81,14 @@ class ModelSearchSearch extends Model {
 
       if(ENABLE_FOLDER_RESTRICTIONS == 1) { return ""; }
 
-      $all_your_addresses = $this->get_all_your_address();
+      $all_your_addresses = $this->get_all_your_address("emails");
-      return sprintf(" (%s %s | %s %s) ", FROM_TOKEN, $all_your_addresses, TO_TOKEN, $all_your_addresses);
+      $all_your_wildcard_domains = $this->get_all_your_address("wildcard_domains");
+
+      if($all_your_wildcard_domains) {
+         return sprintf(" ( (%s %s) | (%s %s) | (%s %s) | (%s %s) ) ", FROM_TOKEN, $all_your_addresses, TO_TOKEN, $all_your_addresses, FROMDOMAIN_TOKEN, $all_your_wildcard_domains, TODOMAIN_TOKEN, $all_your_wildcard_domains);
+      } else {
+         return sprintf(" ( (%s %s) | (%s %s) ) ", FROM_TOKEN, $all_your_addresses, TO_TOKEN, $all_your_addresses);
+      }
    }
 
 
@@ -205,17 +211,7 @@ class ModelSearchSearch extends Model {
       }
 
 
-      if(isset($data['aname']) && $data['aname']) {
+      if(isset($data['tag']) && $data['tag']) {
-
-         $match = $data['aname'];
-         if($emailfilter) { $match = "( $match ) & $emailfilter"; }
-
-         $query = $this->sphx->query("SELECT id, mid FROM " . SPHINX_ATTACHMENT_INDEX . " WHERE MATCH('" . $match . "') ORDER BY `id` $order LIMIT $offset,$pagelen OPTION max_matches=" . MAX_SEARCH_HITS);
-
-         $total_found = $query->total_found;
-         $num_rows = $query->num_rows;
-      }
-      else if(isset($data['tag']) && $data['tag']) {
          list ($total_found, $num_rows, $id_list) = $this->get_sphinx_id_list($data['tag'], SPHINX_TAG_INDEX, 'tag', $page);
          $query = $this->sphx->query("SELECT id FROM " . SPHINX_MAIN_INDEX . " WHERE $folders id IN ($id_list) $sortorder LIMIT 0,$pagelen OPTION max_matches=" . MAX_SEARCH_HITS);
       }
@@ -324,7 +320,6 @@ class ModelSearchSearch extends Model {
                     'date2'           => '',
                     'direction'       => '',
                     'size'            => '',
-                    'aname'           => '',
                     'attachment_type' => '',
                     'tag'             => '',
                     'note'            => '',
@@ -360,7 +355,6 @@ class ModelSearchSearch extends Model {
          else if($v == 'date1:') { $token = 'date1'; continue; }
          else if($v == 'date2:') { $token = 'date2'; continue; }
          else if($v == 'attachment:' || $v == 'a:') { $token = 'match'; $a['match'][] = '@attachment_types'; continue; }
-         else if($v == 'aname:') { $token = 'aname'; continue; }
          else if($v == 'size') { $token = 'size'; continue; }
          else if($v == 'tag:') { $token = 'tag'; continue; }
          else if($v == 'note:') { $token = 'note'; continue; }
@@ -374,13 +368,6 @@ class ModelSearchSearch extends Model {
          }
 
          if($token == 'match') { $a['match'][] = $v; }
-         else if($token == 'aname') {
-            if($v != '|') {
-               $a['aname'] .= '"' . $v . '"';
-            } else {
-               $a['aname'] .= ' | ';
-            }
-         }
          else if($token == 'date1') { $a['date1'] = ' ' . $v; }
          else if($token == 'date2') { $a['date2'] = ' ' . $v; }
          else if($token == 'tag') { $a['tag'] .= ' ' . $v; }
@@ -664,12 +651,12 @@ class ModelSearchSearch extends Model {
    }
 
 
-   private function get_all_your_address() {
+   private function get_all_your_address($session_var) {
       $s = '';
 
       $session = Registry::get('session');
 
-      $emails = $session->get("emails");
+      $emails = $session->get($session_var);
 
       while(list($k, $v) = each($emails)) {
          if($s) { $s .= '| ' .  $this->fix_email_address_for_sphinx($v); }
@@ -680,6 +667,25 @@ class ModelSearchSearch extends Model {
    }
 
 
+   private function get_wildcard_domains($arr=[]) {
+      $query_suffix = '';
+      $results = $arr;
+
+      $session = Registry::get('session');
+
+      $wildcard_domains = $session->get('wildcard_domains');
+
+      if($wildcard_domains) {
+         $q = str_repeat('?,', count($wildcard_domains));
+         $q = trim($q, ',');
+         $results = array_merge($results, $wildcard_domains, $wildcard_domains);
+         $query_suffix = "OR fromdomain IN ($q) OR todomain IN ($q)";
+      }
+
+      return [$results, $query_suffix];
+   }
+
+
    public function check_your_permission_by_id($id = '') {
       $q = '';
       $arr = $a = array();
@@ -736,7 +742,8 @@ class ModelSearchSearch extends Model {
          if(Registry::get('auditor_user') == 1 && RESTRICTED_AUDITOR == 1) {
             $query = $this->db->query("SELECT id FROM " . VIEW_MESSAGES . " WHERE id=? AND ( `fromdomain` IN ($q) OR `todomain` IN ($q) )", $arr);
          } else {
-            $query = $this->db->query("SELECT id FROM " . VIEW_MESSAGES . " WHERE id=? AND ( `from` IN ($q) OR `to` IN ($q) )", $arr);
+            [$arr, $query_suffix] = $this->get_wildcard_domains($arr);
+            $query = $this->db->query("SELECT id FROM " . VIEW_MESSAGES . " WHERE id=? AND ( `from` IN ($q) OR `to` IN ($q) $query_suffix )", $arr);
          }
 
          if(isset($query->row['id'])) { return 1; }
@@ -794,7 +801,7 @@ class ModelSearchSearch extends Model {
       $q = preg_replace("/^\,/", "", $q);
 
 
-      if(Registry::get('auditor_user') == 1 && RESTRICTED_AUDITOR == 0) {
+      if(Registry::get('auditor_user') == 1 && RESTRICTED_AUDITOR == 0 && ENABLE_FOLDER_RESTRICTIONS == 0) {
          $query = $this->db->query("SELECT id FROM `" . TABLE_META . "` WHERE `id` IN ($q2)", $arr);
       }
       else {
@@ -815,7 +822,9 @@ class ModelSearchSearch extends Model {
                   }
                }
 
-               $query = $this->db->query("SELECT id FROM `" . VIEW_MESSAGES . "` WHERE `id` IN ($q2) AND ( `from` IN ($q) OR `to` IN ($q) )", $arr);
+               [$arr, $query_suffix] = $this->get_wildcard_domains($arr);
+
+               $query = $this->db->query("SELECT id FROM `" . VIEW_MESSAGES . "` WHERE `id` IN ($q2) AND ( `from` IN ($q) OR `to` IN ($q) $query_suffix)", $arr);
             }
 
          }

webui/model/user/auth.php

@@ -293,12 +293,13 @@ class ModelUserAuth extends Model {
 
 
    public function get_email_array_from_ldap_attr($e = array()) {
-      $data = array();
+      global $mailattrs;
+      $data = [];
 
       foreach($e as $a) {
          if(LOG_LEVEL >= DEBUG) { syslog(LOG_INFO, "checking ldap entry dn: " . $a['dn'] . ", cn: " . $a['cn']); }
 
-         foreach (array("mail", "mailalternateaddress", "proxyaddresses", "zimbraMailForwardingAddress", "member", "memberOfGroup", "othermailbox") as $mailattr) {
+         foreach ($mailattrs as $mailattr) {
             if(isset($a[$mailattr])) {
 
                if(is_array($a[$mailattr])) {
@@ -394,6 +395,8 @@ class ModelUserAuth extends Model {
 
             $data = $this->fix_user_data($username, $username, $emails, 0);
 
+            $data['folders'] = $this->model_folder_folder->get_folder_id_array_for_user($data['uid'], 0);
+
             $this->is_ga_code_needed($username);
 
             $session->set("auth_data", $data);
@@ -429,6 +432,8 @@ class ModelUserAuth extends Model {
 
                   $data = $this->fix_user_data($username, $username, $emails, 0);
 
+                  $data['folders'] = $this->model_folder_folder->get_folder_id_array_for_user($data['uid'], 0);
+
                   $this->is_ga_code_needed($username);
 
                   $session = Registry::get('session');

webui/model/user/user.php

@@ -84,7 +84,7 @@ class ModelUserUser extends Model {
 
       if(isset($query->rows)) {
          foreach ($query->rows as $q) {
-            if(!in_array($q['email'], $data)) { array_push($data, $q['email']); }
+            if(validemail($q['email']) && !in_array($q['email'], $data)) { array_push($data, $q['email']); }
          }
       }
 

webui/system/helper/TrustedTimestamps.php

@@ -69,12 +69,17 @@ class TrustedTimestamps
         curl_setopt($ch, CURLOPT_POSTFIELDS, file_get_contents($requestfile_path));
         curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/timestamp-query'));
         curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)");
+        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, TSA_VERIFY_CERTIFICATE);
+
         $binary_response_string = curl_exec($ch);
+
+        $error = curl_error($ch);
         $status = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+
         curl_close($ch);
 
         if ($status != 200 || !strlen($binary_response_string))
-            throw new Exception("The request failed");
+            throw new Exception("The request failed. Status: $status, error: $error");
 
         $base64_response_string = base64_encode($binary_response_string);
 
@@ -167,14 +172,23 @@ class TrustedTimestamps
          *
          * every other case (Certificate not found / invalid / openssl is not installed / ts command not known)
          * are being handled the same way -> retcode 1 + any retarray NOT containing "message imprint mismatch"
+         *
+         * For openssl 1.1.x it's 2 lines actually:
+         *
+         * Using configuration from /usr/lib/ssl/openssl.cnf
+         * Verification: OK
+         *
          */
 
-        if ($retcode === 0 && strtolower(trim($retarray[0])) == "verification: ok")
+        if ($retcode === 0) {
-        {
+           foreach ($retarray as $line) {
-            if (self::getTimestampFromAnswer ($base64_response_string) != $response_time)
+              if(strtolower(trim($line)) == "verification: ok") {
-                throw new Exception("The responsetime of the request was changed");
+                 if (self::getTimestampFromAnswer ($base64_response_string) != $response_time)
+                    throw new Exception("The responsetime of the request was changed");
 
-            return true;
+                 return true;
+              }
+           }
         }
 
         foreach ($retarray as $retline)
@@ -205,5 +219,3 @@ class TrustedTimestamps
         return $tempfilename;
     }
 }
-
-?>

webui/system/misc.php

@@ -142,6 +142,16 @@ function checkemail($email, $domains) {
 function validemail($email = '') {
    if($email == '') { return 0; }
 
+   // sphinxsearch supports tokens up to 41 characters long
+   // If there's a longer token in the query, then sphinx
+   // reports a query error even if the query is itself correct
+   // So the workaround is to get rid of these email addresses
+   if(strlen($email) > MAX_EMAIL_LEN) {
+      $msg = sprintf("discarding email %s: longer than %d", $email, MAX_EMAIL_LEN);
+      syslog(LOG_INFO, $msg);
+      return 0;
+   }
+
    if(preg_match("/@local$/", $email)) { return 1; }
 
    if(preg_match('/^[_a-zA-Z0-9-]+(\.[_a-zA-Z0-9-]+)*@[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)*(\.[a-zA-Z]{2,10})$/', $email)) {

webui/view/theme/default/templates/common/layout.tpl

@@ -38,7 +38,7 @@ if(isset($this->request->get['route'])) {
 
    if($this->request->get['route'] == 'health/health') { ?> onload="Piler.load_health(); setInterval('Piler.load_health()', Piler.health_refresh * 1000);"<?php }
    if($this->request->get['route'] == 'stat/online') { ?> onload="setInterval('Piler.reload_page()', Piler.health_refresh * 1000);"<?php }
-   if($this->request->get['route'] == 'import/jobs') { ?> onload="setInterval('Piler.reload_page()', 10 * 1000);"<?php }
+   if($this->request->get['route'] == 'import/jobs') { ?> onload="setInterval('Piler.reload_page()', Piler.health_refresh * 1000);"<?php }
 
 } ?>>
 

webui/view/theme/default/templates/common/menu.tpl

@@ -42,8 +42,8 @@
         <?php if(ENABLE_SAAS == 1) { ?>
                             <li><a href="index.php?route=ldap/list"><i class="icon-key"></i>&nbsp;<?php print $text_ldap; ?></a></li>
                             <li><a href="index.php?route=customer/list"><i class="icon-wrench"></i>&nbsp;<?php print $text_customers; ?></a></li>
-                            <li><a href="index.php?route=import/list"><i class="icon-lightbulb"></i>&nbsp;<?php print $text_import; ?></a></li>
         <?php } ?>
+                            <li><a href="index.php?route=import/list"><i class="icon-lightbulb"></i>&nbsp;<?php print $text_import; ?></a></li>
                             <li><a href="index.php?route=policy/archiving"><i class="icon-stop"></i>&nbsp;<?php print $text_archiving_rules; ?></a></li>
                             <li><a href="index.php?route=policy/retention"><i class="icon-time"></i>&nbsp;<?php print $text_retention_rules; ?></a></li>
                             <li><a href="index.php?route=policy/folder"><i class="icon-folder-open"></i>&nbsp;<?php print $text_folder_rules; ?></a></li>

webui/view/theme/default/templates/message/view.tpl

@@ -81,7 +81,7 @@
 </div>
 
 <?php foreach($images as $img) { ?>
-   <p><img src="<?php print SITE_URL; ?>/tmp/<?php print $img['name']; ?>" alt="" /></p>
+   <p><img src="/tmp/<?php print $img['name']; ?>" alt="" /></p>
 <?php } ?>
 
 

webui/view/theme/default/templates/user/settings.tpl

@@ -1,47 +1,47 @@
 
 <?php if(!isset($x)){ ?>
 
-	<h4><?php print $text_access_settings; ?></h4>
+    <h4><?php print $text_access_settings; ?></h4>
     <p><em><?php print $text_access_setting_explanation; ?></em></p>
 
     <table class="table table-striped">
-	 <tr>
+        <tr>
-		<td class="span2"><?php print $text_email_addresses; ?>:</td>
+            <td class="span2"><?php print $text_email_addresses; ?>:</td>
-		<td class="span8">
+            <td class="span8"><?php print $emails; ?>
-			<?php print $emails; ?>
+            </td>
-		</td>
+        </tr>
-	 </tr>   
 
 <?php if(Registry::get('auditor_user') == 1 && RESTRICTED_AUDITOR == 1) { ?>
 
-	 <tr>
+    <tr>
-		<td><?php print $text_domains; ?>:</td>
+        <td><?php print $text_domains; ?>:</td>
-		<td>
+        <td><?php print $domains; ?></td>
-			<?php print $domains; ?>
+    </tr>
-		</td>
-	 </tr>
 
 <?php } ?>
 
 <?php if(Registry::get('auditor_user') == 0 || RESTRICTED_AUDITOR == 0) { ?>
 
-	 <tr>
+    <?php if($wildcard_domains) { ?>
-		<td><?php print $text_groups; ?>:</td>
+    <tr>
-		<td>
+        <td><?php print $text_wildcard_domains; ?>:</td>
-			<?php print $groups; ?>
+        <td><?php print $wildcard_domains; ?></td>
-		</td>
+    </tr>
-	 </tr>
+    <?php } ?>
+
+    <tr>
+        <td><?php print $text_groups; ?>:</td>
+        <td><?php print $groups; ?></td>
+    </tr>
 
 <?php } ?>
 
 <?php if(ENABLE_FOLDER_RESTRICTIONS == 1) { ?>
 
-	 <tr>
+    <tr>
-		<td><?php print $text_folders; ?>:</td>
+        <td><?php print $text_folders; ?>:</td>
-		<td>
+        <td><?php print $folders; ?></td>
-			<?php print $folders; ?>
+    </tr>
-		</td>
-	 </tr>
 
 <?php } ?>
 
@@ -49,7 +49,7 @@
 
 <form action="/settings.php" method="post" name="setpagelen" class="form-horizontal">
 
-	<h4><?php print $text_display_settings; ?></h4>
+    <h4><?php print $text_display_settings; ?></h4>
 
     <div class="control-group">
         <label class="control-label" for="pagelen"><?php print $text_page_length; ?></label>