Bumped up version number to 1.4.5

Signed-off-by: Janos SUTO <sj@acts.hu>

Makefile.in

@@ -82,7 +82,7 @@ installdirs: mkinstalldirs
 		$(DESTDIR)$(localstatedir)/piler/stat \
 		$(DESTDIR)$(localstatedir)/piler/tmp \
 		$(DESTDIR)$(localstatedir)/piler/error \
-		$(DESTDIR)$(localstatedir)/piler/sphinx \
+		$(DESTDIR)$(localstatedir)/piler/export \
 		$(DESTDIR)$(localstatedir)/piler/manticore
 
 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/run/piler
@@ -91,7 +91,7 @@ installdirs: mkinstalldirs
 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/stat
 	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/tmp
 	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/error
-	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/sphinx
+	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/export
 	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/manticore
 
 

RELEASE_NOTES

@@ -1,3 +1,46 @@
+1.4.5:
+------
+
+- Introduced 2 new variables in /etc/piler/piler.conf affecting piler-smtp
+
+  ; max message size in bytes
+  ; piler-smtp will reject any message that's bigger than this number
+  max_message_size=50000000
+
+  ; max memory in bytes piler-smtp uses for buffering messages
+  ; when this limit is exceeded, no new emails will be accepted
+  ; until the used memory for all in progress emails decreases
+  ; below this level
+  max_smtp_memory=500000000
+
+  Be sure to adjust these values to your environment!
+
+- Added read-only connection support for manticore
+  If using sphinx, add the following to config-site.php:
+  $config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9306';
+
+- pilerimport supports Zimbra IMAP impersonation
+
+  Generate the following base64 encoded string:
+  (Be sure to use the actual usernames and password):
+
+  pw="$( printf '%s\0%s\0%s' 'username' 'zimbra_admin_username' 'zimbra_admin_password' | base64 )"
+
+  Then specify -u ZIMBRA -p "$pw" for pilerimport, eg.
+
+  pilerimport -i imap.server -u ZIMBRA -p "$pw" ...
+
+  Note that "ZIMBRA" is a special username, it indicates for pilerimport
+  to actually use the imap impersonation for Zimbra.
+
+
+1.4.4:
+------
+
+- Renamed "group" table to "usergroup"
+  Be sure to run util/db-upgrade.sql on the mysql piler database
+
+
 1.4.3:
 ------
 
@@ -53,7 +96,7 @@ Be sure to apply util/db-upgrade.sql
 
 - [BUGFIX] Refactored the smtp timeout check
 - Obsoleted the LDAP port parameter. Specify the ldap host in the form
-  of protocol://hostname:port, eg. ldaps://ldap.yourdomain.com:636
+  of protocol://hostname:port, eg. ldaps://ldap.example.com:636
 
 1.3.10:
 -------

VERSION

@@ -1 +1 @@
-1.4.3
+1.4.5

config.php.in

@@ -34,14 +34,16 @@ $config['TITLE_PREFIX'] = '';
 $config['CUSTOM_PRE_AUTH_FUNCTION'] = '';
 $config['CUSTOM_EMAIL_QUERY_FUNCTION'] = '';
 
+$config['DOMAIN_REGEX'] = '/^[a-zA-Z0-9]+[a-zA-Z0-9-_\.]{0,}\.[a-zA-Z0-9]{2,20}$/';
+
 $config['BOOTSTRAP_THEME'] = '-cosmo';
 
 $config['DEFAULT_LANG'] = 'en';
 
 $config['THEME'] = 'default';
 
-$config['SITE_NAME'] = 'piler.yourdomain.com';
-$config['SITE_URL'] = 'http://piler.yourdomain.com/';
+$config['SITE_NAME'] = 'piler.example.com';
+$config['SITE_URL'] = 'http://piler.example.com/';
 
 $config['EXTERNAL_DASHBOARD_URL'] = '';
 
@@ -81,7 +83,7 @@ $config['WKHTMLTOPDF_COMMAND'] = "wkhtmltopdf";
 // authentication against an ldap directory (disabled by default)
 
 $config['ENABLE_LDAP_AUTH'] = 0;
-$config['LDAP_HOST'] = 'ldap.yourdomain.com';
+$config['LDAP_HOST'] = 'ldap.example.com';
 $config['LDAP_HELPER_DN'] = 'cn=....';
 $config['LDAP_HELPER_PASSWORD'] = 'xxxxxxx';
 $config['LDAP_MAIL_ATTR'] = 'mail';
@@ -112,9 +114,9 @@ $config['LDAP_MAIL_ATTR'] = 'proxyAddresses';
 //$config['LDAP_MAIL_ATTR'] = 'mail';
 
 // iredmail specific settings
-//$config['LDAP_HELPER_DN'] = 'cn=vmailadmin,dc=yourdomain,dc=com';
+//$config['LDAP_HELPER_DN'] = 'cn=vmailadmin,dc=example,dc=com';
 //$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'mailUser';
-//$config['LDAP_BASE_DN'] = 'o=domains,dc=yourdomain,dc=com';
+//$config['LDAP_BASE_DN'] = 'o=domains,dc=example,dc=com';
 //$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'mailList';
 //$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'memberOfGroup';
 //$config['LDAP_MAIL_ATTR'] = 'mail';
@@ -137,14 +139,14 @@ $config['ENABLE_IMAP_AUTH'] = 0;
 $config['RESTORE_OVER_IMAP'] = 0;
 $config['IMAP_RESTORE_FOLDER_INBOX'] = 'INBOX';
 $config['IMAP_RESTORE_FOLDER_SENT'] = 'Sent';
-$config['IMAP_HOST'] = 'mail.yourdomain.com';
+$config['IMAP_HOST'] = 'mail.example.com';
 $config['IMAP_PORT'] =  993;
 $config['IMAP_SSL'] = true;
 
 // enable authentication against a pop3 server (disabled by default)
 
 $config['ENABLE_POP3_AUTH'] = 0;
-$config['POP3_HOST'] = 'mail.yourdomain.com';
+$config['POP3_HOST'] = 'mail.example.com';
 $config['POP3_PORT'] = 995;
 $config['POP3_SSL'] = true;
 
@@ -202,9 +204,11 @@ $config['PILER_HOST'] = '0.0.0.0';
 $config['PILER_PORT'] = 25;
 $config['SMARTHOST'] = '';
 $config['SMARTHOST_PORT'] = 25;
-$config['SMTP_DOMAIN'] = 'yourdomain.com';
-$config['SMTP_FROMADDR'] = 'no-reply@yourdomain.com';
-$config['ADMIN_EMAIL'] = 'admin@yourdomain.com';
+$config['SMARTHOST_USER'] = '';
+$config['SMARTHOST_PASSWORD'] = '';
+$config['SMTP_DOMAIN'] = 'example.com';
+$config['SMTP_FROMADDR'] = 'no-reply@example.com';
+$config['ADMIN_EMAIL'] = 'admin@example.com';
 
 $config['PILER_HEADER_FIELD'] = 'X-piler-id: ';
 
@@ -227,6 +231,12 @@ $config['TSA_PUBLIC_KEY_FILE'] = '';
 $config['TSA_START_ID'] = 1;
 $config['TSA_STAMP_REQUEST_UNIT_SIZE'] = 10000;
 $config['TSA_VERIFY_CERTIFICATE'] = true;
+$config['TSA_RELAXED_CHECK'] = false;
+$config['TSA_AUTH_USER'] = '';
+$config['TSA_AUTH_PASSWORD'] = '';
+$config['TSA_AUTH_CERT_FILE'] = '';
+$config['TSA_AUTH_KEY_FILE'] = '';
+$config['TSA_AUTH_KEY_PASSWORD'] = '';
 
 $config['DB_DRIVER'] = 'mysql';
 $config['DB_PREFIX'] = '';
@@ -239,6 +249,7 @@ $config['DB_CHARSET'] = 'utf8mb4';
 $config['SPHINX_DRIVER'] = 'sphinx';
 $config['SPHINX_DATABASE'] = 'sphinx';
 $config['SPHINX_HOSTNAME'] = '127.0.0.1:9306';
+$config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9307';
 $config['SPHINX_MAIN_INDEX'] = 'main1,dailydelta1,delta1';
 $config['SPHINX_ATTACHMENT_INDEX'] = 'att1';
 $config['SPHINX_TAG_INDEX'] = 'tag1';
@@ -269,7 +280,10 @@ $config['SIZE_Y'] = 250;
 $config['DATE_TEMPLATE'] = 'Y.m.d';
 $config['DATE_FORMAT'] = 'YYYY-MM-DD';
 $config['JQUERY_DATE_FORMAT'] = 'yy-mm-dd';
+$config['DECIMAL_SEPARATOR'] = "."; // See https://www.php.net/manual/en/function.number-format
+$config['THOUSANDS_SEPARATOR'] = ","; // for the format options
 
+$config['ENABLE_GB2312_FIX'] = 1;
 
 $config['FROM_LENGTH_TO_SHOW'] = 28;
 
@@ -387,7 +401,7 @@ foreach ($config as $k => $v) {
 
 
 define('TABLE_USER', 'user');
-define('TABLE_GROUP', 'group');
+define('TABLE_GROUP', 'usergroup');
 define('TABLE_GROUP_USER', 'group_user');
 define('TABLE_GROUP_EMAIL', 'group_email');
 define('TABLE_FOLDER', 'folder');
@@ -427,7 +441,7 @@ define('TABLE_PRIVATE', 'private');
 define('TABLE_DELETED', 'deleted');
 define('VIEW_MESSAGES', 'v_messages');
 
-define('EOL', "\n");
+define('EOL', "\r\n");
 
 define('DIR_SYSTEM', DIR_BASE . 'system/');
 define('DIR_MODEL', DIR_BASE . 'model/');
@@ -460,7 +474,8 @@ define('QSHAPE_DEFERRED_SENDER', DIR_STAT . '/deferred-sender');
 define('CPUSTAT', DIR_STAT . '/cpu.stat');
 define('AD_SYNC_STAT', DIR_STAT . '/adsync.stat');
 define('ARCHIVE_SIZE', DIR_STAT . '/archive.size');
-define('SPHINX_MAIN_INDEX_SIZE', DIR_STAT . '/main_index_size');
+define('SPHINX_CURRENT_MAIN_INDEX_SIZE', DIR_STAT . '/current_main_index_size');
+define('SPHINX_TOTAL_INDEX_SIZE', DIR_STAT . '/total_index_size');
 define('LOCK_FILE', DIR_LOG . 'lock');
 
 define('SEARCH_HELPER_URL', SITE_URL . 'search-helper.php');

configure

@@ -6426,13 +6426,13 @@ fi
 
 echo
 echo
-echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/wiki/current:upgrade"
+echo "IMPORTANT! If you upgrade, be sure to read https://www.mailpiler.org/upgrade-instructions/"
 echo
 echo
 
 echo
 echo "Did you know that piler has an enterprise edition as well?"
-echo "Check out what it can do for you at https://mailpiler.com/piler-enterprise-email-archiver-features/"
+echo "Check out what it can do for you at https://mailpiler.com/#features"
 echo
 echo
 

configure.in

@@ -537,12 +537,12 @@ AC_OUTPUT
 
 echo
 echo
-echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/wiki/current:upgrade"
+echo "IMPORTANT! If you upgrade, be sure to read https://www.mailpiler.org/upgrade-instructions/"
 echo
 echo
 
 echo
 echo "Did you know that piler has an enterprise edition as well?"
-echo "Check out what it can do for you at https://mailpiler.com/piler-enterprise-email-archiver-features/"
+echo "Check out what it can do for you at https://mailpiler.com/#features"
 echo
 echo

contrib/installer/focal.sh

@@ -6,7 +6,7 @@ set -o nounset
 set -x
 
 MY_IP="1.2.3.4"
-PILER_HOSTNAME="${PILER_HOSTNAME:-archive.yourdomain.com}"
+PILER_HOSTNAME="${PILER_HOSTNAME:-archive.example.com}"
 MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
 MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
 SERVER_ID="${SERVER_ID:-0}"
@@ -148,7 +148,7 @@ fix_configs() {
    fi
 
    if [[ ! -f /etc/piler/piler.conf ]]; then
-      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.yourdomain.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > /etc/piler/piler.conf
+      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.example.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > /etc/piler/piler.conf
       chmod 600 /etc/piler/piler.conf
       chown $PILER_USER:$PILER_USER /etc/piler/piler.conf
    fi

contrib/installer/jammy.sh

@@ -0,0 +1,258 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+
+MY_IP="1.2.3.4"
+PILER_HOSTNAME="${PILER_HOSTNAME:-archive.example.com}"
+MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
+MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
+SERVER_ID="${SERVER_ID:-0}"
+USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
+SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
+PHP_FPM_SOCKET="/var/run/php/php8.1-fpm.sock"
+
+MYSQL_HOSTNAME="localhost"
+MYSQL_DATABASE="piler"
+MYSQL_USERNAME="piler"
+
+DOWNLOAD_URL="https://download.mailpiler.com"
+PILER_DEB="piler_1.4.4-jammy-0573c6da_amd64.deb"
+PILER_USER="piler"
+PILER_CONF="/etc/piler/piler.conf"
+CONFIG_SITE_PHP="/etc/piler/config-site.php"
+SEARCHCFG="/etc/piler/manticore.conf"
+
+TRAEFIK_VERSION="v2.11.0"
+ARCH="amd64"
+
+export DEBIAN_FRONTEND=noninteractive
+
+install_prerequisites() {
+   apt-get update
+   apt-get -y --no-install-recommends install \
+      wget rsyslog openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql \
+      php8.1-fpm php8.1-zip php8.1-ldap php8.1-gd php8.1-curl php8.1-xml \
+      php8.1-mbstring ca-certificates zip catdoc unrtf poppler-utils \
+      nginx tnef libzip4 libtre5 libwrap0 cron libmariadb-dev python3 \
+      python3-mysqldb libmariadb-dev mariadb-client-core-10.6 \
+      mariadb-server-10.6
+
+   wget "https://github.com/traefik/traefik/releases/download/${TRAEFIK_VERSION}/traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz"
+   tar zxvf "traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz" traefik
+   chown root:root traefik
+   chmod 755 traefik
+   mv traefik /usr/local/bin
+   setcap cap_net_bind_service+ep /usr/local/bin/traefik
+
+   wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
+   dpkg -i manticore-repo.noarch.deb && \
+   apt-get update && \
+   apt-get install -y manticore manticore-columnar-lib manticore-extra && \
+   rm -f manticore-repo.noarch.deb
+   systemctl stop manticore
+}
+
+
+fix_mysql_settings() {
+   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
+[mysqld]
+
+innodb_buffer_pool_size=512M
+innodb_flush_log_at_trx_commit=1
+innodb_log_buffer_size=64M
+innodb_log_file_size=64M
+innodb_read_io_threads=4
+innodb_write_io_threads=4
+innodb_log_files_in_group=2
+
+innodb_file_per_table
+PILER_CNF
+}
+
+start_mysql() {
+   fix_mysql_settings
+
+   service mysql restart
+   mysqladmin -u root password "${MYSQL_ROOT_PASSWORD}"
+}
+
+
+install_piler() {
+   wget "https://bitbucket.org/jsuto/piler/downloads/${PILER_DEB}"
+   dpkg -i "$PILER_DEB"
+   rm -f "$PILER_DEB"
+
+   sed -i 's/.*indexer.*/###/g' /usr/share/piler/piler.cron
+   crontab -u "$PILER_USER" /usr/share/piler/piler.cron
+
+   rm -f "$PILER_DEB" /etc/nginx/sites-enabled/default
+   ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf
+
+   touch /var/piler/.bash_history
+   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
+}
+
+
+create_my_cnf() {
+   local user=$1
+   local password=$2
+   local my_cnf=$3
+
+   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
+   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
+   chown $PILER_USER:$PILER_USER "$my_cnf"
+   chmod 600 "$my_cnf"
+}
+
+
+fix_config_site_php() {
+   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"
+
+   {
+      echo "\$config['SERVER_ID'] = $SERVER_ID;"
+      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
+   } >> "$CONFIG_SITE_PHP"
+
+   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
+      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
+   fi
+
+   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
+   echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
+}
+
+
+init_db() {
+   sed -e "s%MYSQL_HOSTNAME%$MYSQL_HOSTNAME%g" -e "s%MYSQL_DATABASE%$MYSQL_DATABASE%g" -e "s%MYSQL_USERNAME%$MYSQL_USERNAME%g" -e "s%MYSQL_PASSWORD%$MYSQL_PILER_PASSWORD%g" \
+      /usr/share/piler/db-mysql-root.sql.in | mysql --defaults-file=/etc/piler/.my.cnf-root -h $MYSQL_HOSTNAME
+   mysql --defaults-file=/etc/piler/.my.cnf -h $MYSQL_HOSTNAME $MYSQL_DATABASE < /usr/share/piler/db-mysql.sql
+}
+
+
+add_systemd_services() {
+   pushd /etc/systemd/system
+
+   ln -sf /usr/libexec/piler/piler.service .
+   ln -sf /usr/libexec/piler/piler-smtp.service .
+   ln -sf /usr/libexec/piler/pilersearch.service .
+
+   popd
+
+   systemctl daemon-reload
+
+   systemctl enable piler
+   systemctl enable piler-smtp
+   systemctl enable pilersearch
+   systemctl enable traefik
+}
+
+
+fix_configs() {
+   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
+      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
+      nginx -t
+      nginx -s reload
+
+      sed -i 's/server {/server {\n\tlisten 127.0.0.1:80;/' /etc/piler/piler-nginx.conf
+      systemctl stop nginx
+      sleep 5
+      systemctl start nginx
+   fi
+
+   if [[ ! -f "$PILER_CONF" ]]; then
+      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.example.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > "$PILER_CONF"
+      chmod 600 "$PILER_CONF"
+      chown $PILER_USER:$PILER_USER "$PILER_CONF"
+   fi
+
+   sed -i 's/tls_enable=0/tls_enable=1/g' "$PILER_CONF"
+   sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
+
+   sed -i "s/define('RT.*/define('RT', 1);/" "$SEARCHCFG"
+}
+
+setup_traefik() {
+   wget -O /etc/systemd/system/traefik.service "${DOWNLOAD_URL}/generic-local/traefik.service"
+
+   mkdir -p /usr/local/etc/traefik
+   touch /usr/local/etc/traefik/acme.json
+   chmod 600 /usr/local/etc/traefik/acme.json
+   chown www-data:www-data /usr/local/etc/traefik/acme.json
+
+   cat > /usr/local/etc/traefik/traefik.yaml << TRAEFIK
+log:
+  level: INFO
+
+entryPoints:
+  websecure:
+    address: "$MY_IP:443"
+
+providers:
+  file:
+    filename: "/usr/local/etc/traefik/traefik.yaml"
+
+certificatesResolvers:
+  le:
+    acme:
+      storage: "/usr/local/etc/traefik/acme.json"
+      email: admin@$PILER_HOSTNAME
+      tlsChallenge: {}
+
+tls:
+  options:
+    default:
+      minVersion: VersionTLS13
+
+http:
+  middlewares:
+    piler_headers:
+      headers:
+        customResponseHeaders:
+          Server: ""
+          Strict-Transport-Security: "max-age=31536000"
+          X-Content-Type-Optionsi: "nosniff"
+          Referrer-Policy: "same-origin"
+  routers:
+    master:
+      rule: "Host(\`$PILER_HOSTNAME\`)"
+      service: www
+      middlewares:
+        - "piler_headers"
+      tls:
+        certResolver: le
+  services:
+    www:
+      loadBalancer:
+        servers:
+        - url: "http://127.0.0.1:80/"
+TRAEFIK
+}
+
+
+install_prerequisites
+
+start_mysql
+
+install_piler
+
+create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
+create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf
+
+fix_config_site_php
+
+setup_traefik
+
+add_systemd_services
+
+fix_configs
+init_db
+
+[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler
+
+systemctl start pilersearch
+systemctl start piler
+systemctl start piler-smtp
+systemctl start traefik

docker/Dockerfile

@@ -15,9 +15,9 @@ COPY ${PACKAGE} /
 
 RUN apt-get update && \
     apt-get -y --no-install-recommends install \
-       wget rsyslog openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql php8.1-fpm php8.1-zip php8.1-ldap \
+       wget openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql php8.1-fpm php8.1-zip php8.1-ldap \
        php8.1-gd php8.1-curl php8.1-xml php8.1-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip4 \
-       libtre5 cron libmariadb-dev mariadb-client-core-10.6 python3 python3-mysqldb ca-certificates curl && \
+       libtre5 cron libmariadb-dev mariadb-client-core-10.6 python3 python3-mysqldb ca-certificates curl rsyslog && \
     wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
     dpkg -i manticore-repo.noarch.deb && \
     rm -f manticore-repo.noarch.deb && \
@@ -25,7 +25,6 @@ RUN apt-get update && \
     apt-get install -y manticore manticore-columnar-lib && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \
-    sed -i 's/mail.[iwe].*//' /etc/rsyslog.conf && \
     sed -i '/session    required     pam_loginuid.so/c\#session    required     pam_loginuid.so' /etc/pam.d/cron && \
     dpkg -i ${PACKAGE} && \
     touch /etc/piler/MANTICORE && \

docker/build.sh

@@ -4,7 +4,7 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-IMAGE_NAME="sutoj/piler:1.3.12"
+IMAGE_NAME="sutoj/piler:1.4.4"
 
 if [[ $# -ne 1 ]]; then echo "ERROR: missing package name" 1>&2; exit 1; fi
 

docker/docker-compose.yaml

@@ -1,8 +1,7 @@
 version: "3"
 services:
-
   mysql:
-    image: mariadb:10.6
+    image: mariadb:11.1.2
     container_name: mysql
     restart: unless-stopped
     cap_drop:
@@ -17,42 +16,26 @@ services:
       - MYSQL_PASSWORD=piler123
       - MYSQL_RANDOM_ROOT_PASSWORD=yes
     command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
-    healthcheck:
-      test: mysql --user=piler --password=piler123 piler --execute "show tables"
-      interval: "60s"
-      timeout: "5s"
-      start_period: "15s"
-      retries: 3
     volumes:
       - db_data:/var/lib/mysql
-
-  memcached:
-    container_name: memcached
-    image: memcached:latest
-    restart: unless-stopped
-    cap_drop:
-      - ALL
-    command: -m 64
-
   piler:
-    image: sutoj/piler:1.4.2
+    image: sutoj/piler:1.4.4
     container_name: piler
     init: true
     environment:
+      - MYSQL_HOSTNAME=mysql
       - MYSQL_DATABASE=piler
       - MYSQL_USER=piler
       - MYSQL_PASSWORD=piler123
-      - MYSQL_HOSTNAME=mysql
-      - PILER_HOSTNAME=archive.yourdomain.com
-      - MEMCACHED_HOST=memcached
+      - PILER_HOSTNAME=cust1.acts.hu
+      - RT=1
     ports:
       - "25:25"
       - "80:80"
-      - "443:443"
     volumes:
       - piler_etc:/etc/piler
-      - piler_var_store:/var/piler/store
-      - piler_var_manticore:/var/piler/manticore
+      - piler_manticore:/var/piler/manticore
+      - piler_store:/var/piler/store
     healthcheck:
       test: curl -s smtp://localhost/
       interval: "60s"
@@ -67,11 +50,10 @@ services:
           memory: 512M
 
     depends_on:
-      - "memcached"
       - "mysql"
 
 volumes:
   db_data: {}
   piler_etc: {}
-  piler_var_store: {}
-  piler_var_manticore: {}
+  piler_manticore: {}
+  piler_store: {}

docker/start.sh

@@ -13,7 +13,7 @@ PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf"
 SPHINX_CONF="${CONFIG_DIR}/manticore.conf"
 CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php"
 PILER_MY_CNF="${CONFIG_DIR}/.my.cnf"
-
+RT="${RT:-0}"
 
 error() {
    echo "ERROR:" "$*" 1>&2
@@ -126,6 +126,21 @@ fix_configs() {
        -e "s%MYSQL_USERNAME%${MYSQL_USER}%" \
        -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" \
        -i "$SPHINX_CONF"
+
+   # Fixes for RT index
+
+   if [[ $RT -eq 1 ]]; then
+      sed -i "s/define('RT', 0)/define('RT', 1)/" "$SPHINX_CONF"
+      if ! grep "'RT'" "$CONFIG_SITE_PHP"; then
+         echo "\$config['RT'] = 1;" >> "$CONFIG_SITE_PHP"
+      fi
+
+      if ! grep "'SPHINX_MAIN_INDEX'" "$CONFIG_SITE_PHP"; then
+         echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
+      fi
+
+      sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
+   fi
 }
 
 
@@ -170,17 +185,17 @@ create_my_cnf_files() {
 
 
 start_services() {
-   rsyslogd
    service cron start
    service php8.1-fpm start
    service nginx start
+   rsyslogd
 }
 
 
 start_piler() {
-   if [[ ! -f "${VOLUME_DIR}/manticore/main1.spp" ]]; then
+   if [[ $RT -eq 0 && ! -f "${VOLUME_DIR}/manticore/main1.spp" ]]; then
       log "main1.spp does not exist, creating index files"
-      su -c "indexer --all --config ${SPHINX_CONF}" piler
+      su -c "indexer --all --config ${SPHINX_CONF}" "$PILER_USER"
    fi
 
    # No pid file should exist for piler

etc/example.conf

@@ -50,8 +50,8 @@ max_requests_per_child=10000
 
 ; SMTP HELO identification string
 ; this should be the FQDN part of the email address
-; where you copy emails, eg. archive@piler.yourdomain.com -> piler.yourdomain.com
-hostid=piler.yourdomain.com
+; where you copy emails, eg. archive@piler.example.com -> piler.example.com
+hostid=piler.example.com
 
 ; whether to process rcpt to addresses and add them to rcpt table (1) or not (0)
 process_rcpt_to_addresses=0
@@ -251,7 +251,7 @@ security_header=
 ; shouldn't be, then spammers may find it, and fill it with spam.
 ;
 ; By setting this variable you may restrict the envelope address
-; to a single email address, eg. some-random-address-12345@archive.yourdomain.com
+; to a single email address, eg. some-random-address-12345@archive.example.com
 ; Then the archive will reject any other envelope recipients
 archive_address=
 
@@ -263,3 +263,12 @@ archive_address=
 ; rules. In other words if you decide to use the acl file, then
 ; everyone is not explicitly permitted is denied.
 smtp_access_list=0
+
+; max message size in bytes
+; piler-smtp will reject any message that's bigger than this number
+max_message_size=50000000
+
+; max memory in bytes piler-smtp uses for buffering messages
+; when this limit is exceeded, no new emails will be accepted
+; until the used memory decreases below this level
+max_smtp_memory=500000000

etc/manticore.conf.in

@@ -129,6 +129,7 @@ index dailydelta1
         min_word_len            = 1
         stored_fields           =
         charset_table           = <?php print CHARSET_TABLE; ?>
+        killlist_target         = main1:kl, main2:kl, main3:kl, main4:kl, dailydelta1:kl
 <?php print NGRAM_CONFIG; ?>
 }
 
@@ -247,7 +248,8 @@ index note1
 searchd
 {
         listen                  = 127.0.0.1:9312
-        listen                  = 127.0.0.1:9306:mysql41
+        listen                  = 127.0.0.1:9306:mysql
+        listen                  = 127.0.0.1:9307:mysql_readonly
         log                     = /var/piler/manticore/manticore.log
         binlog_max_log_size     = 256M
         binlog_path             = /var/piler/manticore
@@ -256,7 +258,7 @@ searchd
         network_timeout         = 5
         pid_file                = /var/run/piler/searchd.pid
         seamless_rotate         = 1
-        preopen_indexes         = 1
+        preopen_tables          = 1
         unlink_old              = 1
         thread_stack            = 512k
 <?php if(RT) { ?>

src/archive.c

@@ -16,6 +16,9 @@
 #include <syslog.h>
 #include <openssl/blowfish.h>
 #include <openssl/evp.h>
+#if OPENSSL_VERSION_MAJOR >= 3
+   #include <openssl/provider.h>
+#endif
 #include <zlib.h>
 #include <assert.h>
 #include <piler.h>
@@ -168,9 +171,14 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
    #if OPENSSL_VERSION_NUMBER < 0x10100000L
       EVP_CIPHER_CTX_init(&ctx);
       if(strstr(filename, "/5000")){
-         EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+         rc = EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
       } else {
-         EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+         rc = EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      }
+
+      if(!rc){
+         syslog(LOG_PRIORITY, "ERROR: EVP_DecryptInit_ex()");
+         goto CLEANUP;
       }
 
       blocklen = EVP_CIPHER_CTX_block_size(&ctx);
@@ -180,9 +188,18 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
 
       EVP_CIPHER_CTX_init(ctx);
       if(strstr(filename, "/5000")){
-         EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+         rc = EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
       } else {
-         EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      #if OPENSSL_VERSION_MAJOR >= 3
+         OSSL_PROVIDER_load(NULL, "legacy");
+         OSSL_PROVIDER_load(NULL, "default");
+      #endif
+         rc = EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      }
+
+      if(!rc){
+         syslog(LOG_PRIORITY, "ERROR: EVP_DecryptInit_ex()");
+         goto CLEANUP;
       }
 
       blocklen = EVP_CIPHER_CTX_block_size(ctx);

src/bdat.c

@@ -26,21 +26,21 @@ void reset_bdat_counters(struct smtp_session *session){
 }
 
 
-void get_bdat_size_to_read(struct smtp_session *session, char *buf){
+void get_bdat_size_to_read(struct smtp_session *session){
    char *p;
 
    session->bdat_bytes_to_read = 0;
 
    session->protocol_state = SMTP_STATE_BDAT;
 
-   p = strcasestr(buf, " LAST");
+   p = strcasestr(session->buf, " LAST");
    if(p){
       *p = '\0';
    }
 
    // determine the size to be read
 
-   p = strchr(buf, ' ');
+   p = strchr(session->buf, ' ');
    if(p){
       session->bdat_bytes_to_read = atoi(p);
       if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "fd=%d: BDAT len=%d", session->net.socket, session->bdat_bytes_to_read);

src/cfg.c

@@ -23,6 +23,11 @@ int int_parser(char *src, int *target){
    return 0;
 };
 
+int uint64_parser(char *src, uint64 *target){
+   *target = strtoull(src, (char**)NULL, 10);
+   return 0;
+}
+
 struct _parse_rule {
    char *name;
    char *type;
@@ -63,7 +68,9 @@ struct _parse_rule config_parse_rules[] =
    { "listen_port", "integer", (void*) int_parser, offsetof(struct config, listen_port), "25", sizeof(int)},
    { "locale", "string", (void*) string_parser, offsetof(struct config, locale), "", MAXVAL-1},
    { "max_connections", "integer", (void*) int_parser, offsetof(struct config, max_connections), "64", sizeof(int)},
+   { "max_message_size", "integer", (void*) int_parser, offsetof(struct config, max_message_size), "50000000", sizeof(int)},
    { "max_requests_per_child", "integer", (void*) int_parser, offsetof(struct config, max_requests_per_child), "10000", sizeof(int)},
+   { "max_smtp_memory", "uint64", (void*) uint64_parser, offsetof(struct config, max_smtp_memory), "500000000", sizeof(uint64)},
    { "memcached_servers", "string", (void*) string_parser, offsetof(struct config, memcached_servers), "127.0.0.1", MAXVAL-1},
    { "memcached_to_db_interval", "integer", (void*) int_parser, offsetof(struct config, memcached_to_db_interval), "900", sizeof(int)},
    { "memcached_ttl", "integer", (void*) int_parser, offsetof(struct config, memcached_ttl), "86400", sizeof(int)},
@@ -217,6 +224,7 @@ struct config read_config(char *configfile){
 void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
    int j;
    float f;
+   uint64 u;
    char *p, buf[MAXVAL];
 
    p = (char*)cfg + rules[i].offset;
@@ -225,6 +233,10 @@ void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
       memcpy((char*)&j, p, sizeof(int));
       printf("%s=%d\n", rules[i].name, j);
    }
+   else if(strcmp(rules[i].type, "uint64") == 0){
+      memcpy((char*)&u, p, sizeof(uint64));
+      printf("%s=%llu\n", rules[i].name, u);
+   }
    else if(strcmp(rules[i].type, "float") == 0){
       memcpy((char*)&f, p, sizeof(float));
       printf("%s=%.4f\n", rules[i].name, f);

src/cfg.h

@@ -110,6 +110,9 @@ struct config {
    int debug;
 
    int smtp_access_list;
+
+   int max_message_size;
+   uint64 max_smtp_memory;
 };
 
 

src/config.h

@@ -9,6 +9,8 @@
 #include "piler-config.h"
 #include "params.h"
 
+typedef unsigned long long uint64;
+
 #define BUILD 1001
 
 #define HOSTID "mailarchiver"
@@ -30,6 +32,7 @@
 #define SMALLBUFSIZE 512
 #define BIGBUFSIZE 131072
 #define REALLYBIGBUFSIZE 524288
+#define SMTPBUFSIZE 2048000
 #define TINYBUFSIZE 128
 #define MAXVAL 256
 #define RANDOM_POOL "/dev/urandom"

src/defs.h

@@ -331,7 +331,7 @@ struct import {
    char *mboxdir;
    char *folder;
    char filename[SMALLBUFSIZE];
-   time_t started, updated, finished;
+   time_t started, updated, finished, after, before;
 };
 
 
@@ -403,7 +403,6 @@ struct smtp_session {
    char ttmpfile[SMALLBUFSIZE];
    char mailfrom[SMALLBUFSIZE];
    char rcptto[MAX_RCPT_TO][SMALLBUFSIZE];
-   char buf[MAXBUFSIZE];
    char remote_host[INET6_ADDRSTRLEN+1];
    char nullbyte;
    time_t lasttime;
@@ -411,13 +410,17 @@ struct smtp_session {
    int slot;
    int fd;
    int bad;
-   int buflen;
-   int last_data_char;
    int tot_len;
    int bdat_bytes_to_read;
    int num_of_rcpt_to;
    struct config *cfg;
    struct net net;
+   int max_message_size;
+   char *buf;
+   int buflen;
+   int bufsize;
+   int too_big;
+   int mail_size;
 };
 
 struct tls_protocol {

src/imap.c

@@ -105,7 +105,12 @@ int connect_to_imap_server(struct data *data){
 
    /* imap cmd: LOGIN */
 
-   snprintf(buf, sizeof(buf)-1, "A%d LOGIN %s \"%s\"\r\n", data->import->seq, data->import->username, data->import->password);
+   if(strcmp(data->import->username, "ZIMBRA") == 0){
+      snprintf(buf, sizeof(buf)-1, "A%d AUTHENTICATE PLAIN %s\r\n", data->import->seq, data->import->password);
+   }
+   else {
+      snprintf(buf, sizeof(buf)-1, "A%d LOGIN %s \"%s\"\r\n", data->import->seq, data->import->username, data->import->password);
+   }
 
    write1(data->net, buf, strlen(buf));
    if(read_response(buf, sizeof(buf), data) == 0){
@@ -313,7 +318,7 @@ void imap_expunge_message(struct data *data){
 }
 
 
-int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct config *cfg){
+int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    int i, messages=0;
 
    messages = imap_select_cmd_on_folder(folder, data);
@@ -330,7 +335,7 @@ int process_imap_folder(char *folder, struct session_data *sdata, struct data *d
          if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/messages); fflush(stdout); }
 
          if(data->import->dryrun == 0){
-            int rc = import_message(sdata, data, cfg);
+            int rc = import_message(sdata, data, counters, cfg);
 
             if(data->import->remove_after_import == 1 && rc == OK){
                imap_delete_message(data, i);

src/import.c

@@ -18,12 +18,11 @@
 #include <piler.h>
 
 
-int import_message(struct session_data *sdata, struct data *data, struct config *cfg){
+int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    int rc=ERR;
    char *rule;
    struct stat st;
    struct parser_state state;
-   struct counters counters;
 
    if(data->import->delay > 0){
       struct timespec req;
@@ -93,6 +92,14 @@ int import_message(struct session_data *sdata, struct data *data, struct config
          printf("%s: invalid message, hdr_len: %d\n", data->import->filename, sdata->hdr_len);
          rc = ERR;
       }
+      else if(data->import->after > 0 && sdata->sent < data->import->after){
+         if(cfg->verbosity > 1) printf("discarding older email: %s (%ld/%ld)\n", sdata->filename, sdata->sent, data->import->after);
+         rc = ERR_DISCARDED;
+      }
+      else if(data->import->before > 0 && sdata->sent > data->import->before){
+         if(cfg->verbosity > 1) printf("discarding newer email: %s (%ld/%ld)\n", sdata->filename, sdata->sent, data->import->before);
+         rc = ERR_DISCARDED;
+      }
       else {
          // When importing emails, we should add the retention value (later) to the original sent value
          sdata->retained = sdata->sent;
@@ -114,24 +121,27 @@ int import_message(struct session_data *sdata, struct data *data, struct config
 
    switch(rc) {
       case OK:
-                        bzero(&counters, sizeof(counters));
-                        counters.c_rcvd = 1;
-                        counters.c_size += sdata->tot_len;
-                        counters.c_stored_size = sdata->stored_len;
-                        update_counters(sdata, data, &counters, cfg);
+                        counters->c_rcvd++;
+                        counters->c_size += sdata->tot_len;
+                        counters->c_stored_size += sdata->stored_len;
 
                         break;
 
       case ERR_EXISTS:
                         rc = OK;
 
-                        bzero(&counters, sizeof(counters));
-                        counters.c_duplicate = 1;
-                        update_counters(sdata, data, &counters, cfg);
+                        counters->c_duplicate++;
 
                         if(data->quiet == 0) printf("duplicate: %s (duplicate id: %llu)\n", data->import->filename, sdata->duplicate_id);
                         break;
 
+      case ERR_DISCARDED:
+                        rc = OK;
+
+                        counters->c_ignore++;
+
+                        break;
+
       default:
                         printf("failed to import: %s (id: %s)\n", data->import->filename, sdata->ttmpfile);
                         break;

src/import.h

@@ -6,23 +6,23 @@
    #define _IMPORT_H
 
 
-int import_message(struct session_data *sdata, struct data *data, struct config *cfg);
+int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 int update_import_table(struct session_data *sdata, struct data *data);
 
-int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct config *cfg);
-int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct config *cfg);
-int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct config *cfg);
-void import_from_pop3_server(struct session_data *sdata, struct data *data, struct config *cfg);
-int import_from_imap_server(struct session_data *sdata, struct data *data, struct config *cfg);
+int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg);
+int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 
 int connect_to_pop3_server(struct data *data);
-void process_pop3_emails(struct session_data *sdata, struct data *data, struct config *cfg);
+void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 
 int connect_to_imap_server(struct data *data);
 int list_folders(struct data *data);
-int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct config *cfg);
+int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 void send_imap_close(struct data *data);
 
-void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg);
+void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 
 #endif /* _IMPORT_H */

src/import_imap.c

@@ -22,7 +22,7 @@
 #include <piler.h>
 
 
-int import_from_imap_server(struct session_data *sdata, struct data *data, struct config *cfg){
+int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    int i, rc=ERR, ret=OK, skipmatch;
    char port_string[8], puf[SMALLBUFSIZE];
    struct addrinfo hints, *res;
@@ -88,7 +88,7 @@ int import_from_imap_server(struct session_data *sdata, struct data *data, struc
             else {
                if(data->quiet == 0) printf("processing folder: %s... ", (char *)q->str);
 
-               if(process_imap_folder(q->str, sdata, data, cfg) == ERR) ret = ERR;
+               if(process_imap_folder(q->str, sdata, data, counters, cfg) == ERR) ret = ERR;
             }
 
          }

src/import_mailbox.c

@@ -22,7 +22,7 @@
 #include <piler.h>
 
 
-int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct config *cfg){
+int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    FILE *F, *f=NULL;
    int rc=ERR, tot_msgs=0, ret=OK;
    char buf[MAXBUFSIZE];
@@ -44,7 +44,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *
          if(f){
             fclose(f);
             f = NULL;
-            rc = import_message(sdata, data, cfg);
+            rc = import_message(sdata, data, counters, cfg);
             if(rc == ERR){
                printf("error importing: '%s'\n", data->import->filename);
                ret = ERR;
@@ -64,7 +64,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *
 
    if(f){
       fclose(f);
-      rc = import_message(sdata, data, cfg);
+      rc = import_message(sdata, data, counters, cfg);
       if(rc == ERR){
          printf("ERROR: error importing: '%s'\n", data->import->filename);
          ret = ERR;
@@ -80,7 +80,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *
 }
 
 
-int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct config *cfg){
+int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    DIR *dir;
    struct dirent *de;
    int rc=ERR, ret=OK, i=0;
@@ -103,7 +103,7 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct dat
       if(stat(fname, &st) == 0){
          if(S_ISDIR(st.st_mode)){
             folder = data->folder;
-            rc = import_mbox_from_dir(fname, sdata, data, cfg);
+            rc = import_mbox_from_dir(fname, sdata, data, counters, cfg);
             data->folder = folder;
             if(rc == ERR) ret = ERR;
          }
@@ -126,7 +126,7 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct dat
 
                }
 
-               rc = import_from_mailbox(fname, sdata, data, cfg);
+               rc = import_from_mailbox(fname, sdata, data, counters, cfg);
                if(rc == OK) (data->import->tot_msgs)++;
                else ret = ERR;
 

src/import_maildir.c

@@ -22,7 +22,7 @@
 #include <piler.h>
 
 
-int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct config *cfg){
+int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg){
    DIR *dir;
    struct dirent *de;
    int rc=ERR, ret=OK, i=0;
@@ -46,7 +46,7 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir
          if(S_ISDIR(st.st_mode)){
             folder = data->folder;
             snprintf(subdir, sizeof(subdir)-1, "%s", data->import->filename);
-            rc = import_from_maildir(sdata, data, subdir, cfg);
+            rc = import_from_maildir(sdata, data, subdir, counters, cfg);
             data->folder = folder;
             if(rc == ERR) ret = ERR;
          }
@@ -76,7 +76,7 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir
 
                }
 
-               rc = import_message(sdata, data, cfg);
+               rc = import_message(sdata, data, counters, cfg);
 
                if(rc == OK) (data->import->tot_msgs)++;
                else if(rc == ERR){

src/import_pilerexport.c

@@ -22,21 +22,21 @@
 #include <piler.h>
 
 
-void import_the_file(struct session_data *sdata, struct data *data, struct config *cfg){
+void import_the_file(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    close(data->import->fd);
    data->import->fd = -1;
 
-   if(import_message(sdata, data, cfg) != ERR){
+   if(import_message(sdata, data, counters, cfg) != ERR){
       unlink(data->import->filename);
    }
 }
 
 
-void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct config *cfg){
+void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
 
    if(!strcmp(buf, PILEREXPORT_BEGIN_MARK)){
       if((*count) > 0){
-         import_the_file(sdata, data, cfg);
+         import_the_file(sdata, data, counters, cfg);
       }
 
       (*count)++;
@@ -57,7 +57,7 @@ void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *s
 }
 
 
-void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg){
+void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    int n, rc, nullbyte, savedlen=0, puflen;
    uint64 count=0;
    char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE];
@@ -96,7 +96,7 @@ void import_from_pilerexport(struct session_data *sdata, struct data *data, stru
 
          if(puflen > 0){
             if(rc == OK){
-               process_buffer(puf, puflen, &count, sdata, data, cfg);
+               process_buffer(puf, puflen, &count, sdata, data, counters, cfg);
             }
             else {
                snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
@@ -109,6 +109,6 @@ void import_from_pilerexport(struct session_data *sdata, struct data *data, stru
    } while(n > 0);
 
    if(data->import->fd != -1){
-      import_the_file(sdata, data, cfg);
+      import_the_file(sdata, data, counters, cfg);
    }
 }

src/import_pop3.c

@@ -22,7 +22,7 @@
 #include <piler.h>
 
 
-void import_from_pop3_server(struct session_data *sdata, struct data *data, struct config *cfg){
+void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    int rc;
    char port_string[8];
    struct addrinfo hints, *res;
@@ -58,7 +58,7 @@ void import_from_pop3_server(struct session_data *sdata, struct data *data, stru
       goto ENDE_POP3;
    }
 
-   process_pop3_emails(sdata, data, cfg);
+   process_pop3_emails(sdata, data, counters, cfg);
 
    close_connection(data->net);
 

src/message.c

@@ -50,7 +50,7 @@ int store_index_data(struct session_data *sdata, struct parser_state *state, str
       char a[4*MAXBUFSIZE+4*SMALLBUFSIZE];
       char *query=NULL;
 
-      snprintf(a, sizeof(a)-1, "INSERT INTO %s (id, arrived, sent, size, direction, folder, attachments, attachment_types, sender, rcpt, senderdomain, rcptdomain, subject, body) VALUES (%llu,%ld,%ld,%d,%d,%d,%d,'%s','", cfg->sphxdb, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, data->folder, state->n_attachments, sdata->attachments);
+      snprintf(a, sizeof(a)-1, "REPLACE INTO %s (id, arrived, sent, size, direction, folder, attachments, attachment_types, sender, rcpt, senderdomain, rcptdomain, subject, body) VALUES (%llu,%ld,%ld,%d,%d,%d,%d,'%s','", cfg->sphxdb, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, data->folder, state->n_attachments, sdata->attachments);
 
       int ret = append_string_to_buffer(&query, a);
 

src/misc.c

@@ -781,3 +781,29 @@ int append_string_to_buffer(char **buffer, char *str){
 
    return 0;
 }
+
+
+int get_size_from_smtp_mail_from(char *s){
+   int size=0;
+   char *p;
+
+   p = strcasestr(s, "SIZE=");
+   if(p){
+      p += strlen("SIZE=");
+      char *q = p;
+      for(; *q; q++){
+         if(isspace(*q)) break;
+      }
+
+      // We extract max. 9 characters, which is just under 1GB
+      // and not overflowing an int variable
+      if(q - p <= 9){
+         char c = *q;
+         *q = '\0';
+         size = atoi(p);
+         *q = c;
+      }
+   }
+
+   return size;
+}

src/misc.h

@@ -55,5 +55,6 @@ int init_ssl_to_server(struct data *data);
 #endif
 
 int append_string_to_buffer(char **buffer, char *str);
+int get_size_from_smtp_mail_from(char *s);
 
 #endif /* _MISC_H */

src/piler-smtp.c

@@ -33,6 +33,7 @@ extern int optind;
 struct epoll_event event, *events=NULL;
 int num_connections = 0;
 int listenerfd = -1;
+int loglevel = 1;
 
 char *configfile = CONFIG_FILE;
 struct config cfg;
@@ -48,6 +49,7 @@ void usage(){
    printf("    -d                                Fork to the background\n");
    printf("    -v                                Return the version and build number\n");
    printf("    -V                                Return the version and some build parameters\n");
+   printf("    -L <log level>                    Set the log level: 1-5\n");
 
    exit(0);
 }
@@ -138,16 +140,20 @@ int main(int argc, char **argv){
    int client_len = sizeof(struct sockaddr_storage);
    ssize_t readlen;
    struct sockaddr_storage client_address;
-   char readbuf[BIGBUFSIZE];
+   char readbuf[REALLYBIGBUFSIZE];
    int efd;
 
-   while((i = getopt(argc, argv, "c:dvVh")) > 0){
+   while((i = getopt(argc, argv, "c:L:dvVh")) > 0){
       switch(i){
 
         case 'c' :
                    configfile = optarg;
                    break;
 
+        case 'L':
+                   loglevel = atoi(optarg);
+                   break;
+
         case 'd' :
                    daemonise = 1;
                    break;
@@ -321,6 +327,7 @@ int main(int argc, char **argv){
                   break;
                }
 
+               readbuf[readlen] = '\0';
                handle_data(session, &readbuf[0], readlen, &cfg);
 
                if(session->protocol_state == SMTP_STATE_BDAT && session->bad == 1){

src/pilerimport.c

@@ -59,6 +59,8 @@ void usage(){
    printf("    -o                                Only download emails for POP3/IMAP import\n");
    printf("    -r                                Remove imported emails\n");
    printf("    -q                                Quiet mode\n");
+   printf("    -A <timestamp>                    Import emails sent after this timestamp\n");
+   printf("    -B <timestamp>                    Import emails sent before this timestamp\n");
 
    exit(0);
 }
@@ -73,6 +75,9 @@ int main(int argc, char **argv){
    struct data data;
    struct import import;
    struct net net;
+   struct counters counters;
+
+   bzero(&counters, sizeof(counters));
 
    for(i=0; i<MBOX_ARGS; i++) mbox[i] = NULL;
 
@@ -102,6 +107,8 @@ int main(int argc, char **argv){
    import.table_id = 0;
    import.folder = NULL;
    import.delay = 0;
+   import.after = 0;
+   import.before = 0;
 
    data.import = &import;
 
@@ -144,6 +151,8 @@ int main(int argc, char **argv){
             {"remove-after-import",no_argument,  0,  'r' },
             {"failed-folder",  required_argument,  0,  'j' },
             {"move-folder",  required_argument,  0,  'g' },
+            {"after",        required_argument,  0,  'A' },
+            {"before",       required_argument,  0,  'B' },
             {"only-download",no_argument,        0,  'o' },
             {"read-from-export",no_argument,     0,  'y' },
             {"dry-run",      no_argument,        0,  'D' },
@@ -153,9 +162,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:yDRroqh?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:A:B:yDRroqh?", long_options, &option_index);
 #else
-      int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:yDRroqh?");
+      int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:A:B:yDRroqh?");
 #endif
 
       if(c == -1) break;
@@ -295,6 +304,12 @@ int main(int argc, char **argv){
                     data.quiet = 1;
                     break;
 
+         case 'A' : data.import->after = atol(optarg);
+                    break;
+
+         case 'B' : data.import->before = atol(optarg);
+                    break;
+
          case 'h' :
          case '?' :
                     usage();
@@ -307,7 +322,7 @@ int main(int argc, char **argv){
    }
 
 
-   if(!mbox[0] && !data.import->mboxdir && !data.import->filename[0] && !directory && !imapserver && !pop3server) usage();
+   if(!mbox[0] && !data.import->mboxdir && !data.import->filename[0] && !directory && !imapserver && !pop3server && !read_from_pilerexport) usage();
 
    if(data.import->failed_folder && !can_i_write_directory(data.import->failed_folder)){
       printf("cannot write failed directory '%s'\n", data.import->failed_folder);
@@ -372,18 +387,18 @@ int main(int argc, char **argv){
 
    load_mydomains(&sdata, &data, &cfg);
 
-   if(data.import->filename[0] != '\0') import_message(&sdata, &data, &cfg);
+   if(data.import->filename[0] != '\0') import_message(&sdata, &data, &counters, &cfg);
 
    if(mbox[0]){
       for(i=0; i<n_mbox; i++){
-         import_from_mailbox(mbox[i], &sdata, &data, &cfg);
+         import_from_mailbox(mbox[i], &sdata, &data, &counters, &cfg);
       }
    }
-   if(data.import->mboxdir) import_mbox_from_dir(data.import->mboxdir, &sdata, &data, &cfg);
-   if(directory) import_from_maildir(&sdata, &data, directory, &cfg);
-   if(imapserver) import_from_imap_server(&sdata, &data, &cfg);
-   if(pop3server) import_from_pop3_server(&sdata, &data, &cfg);
-   if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &cfg);
+   if(data.import->mboxdir) import_mbox_from_dir(data.import->mboxdir, &sdata, &data, &counters, &cfg);
+   if(directory) import_from_maildir(&sdata, &data, directory, &counters, &cfg);
+   if(imapserver) import_from_imap_server(&sdata, &data, &counters, &cfg);
+   if(pop3server) import_from_pop3_server(&sdata, &data, &counters, &cfg);
+   if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &counters, &cfg);
 
    clearrules(data.archiving_rules);
    clearrules(data.retention_rules);
@@ -391,6 +406,10 @@ int main(int argc, char **argv){
 
    clearhash(data.mydomains);
 
+   update_counters(&sdata, &data, &counters, &cfg);
+
+   syslog(LOG_PRIORITY, "server=%s, user=%s, directory=%s, imported=%lld, duplicated=%lld, discarded=%lld", data.import->server, data.import->username, directory, counters.c_rcvd, counters.c_duplicate, counters.c_ignore);
+
    close_database(&sdata);
 
    if(cfg.rtindex) close_sphx(&sdata);

src/pop3.c

@@ -175,7 +175,7 @@ void pop3_delete_message(struct data *data, int i){
 }
 
 
-void process_pop3_emails(struct session_data *sdata, struct data *data, struct config *cfg){
+void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    char buf[MAXBUFSIZE];
 
    data->import->processed_messages = 0;
@@ -191,7 +191,7 @@ void process_pop3_emails(struct session_data *sdata, struct data *data, struct c
          if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/data->import->total_messages); fflush(stdout); }
 
          if(data->import->dryrun == 0){
-            int rc = import_message(sdata, data, cfg);
+            int rc = import_message(sdata, data, counters, cfg);
 
             if(data->import->remove_after_import == 1 && rc == OK){
                pop3_delete_message(data, i);

src/session.c

@@ -9,6 +9,36 @@
 int get_session_slot(struct smtp_session **sessions, int max_connections);
 void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg);
 
+#define GOT_CRLF_DOT_CRLF(p) *p == '\r' && *(p+1) == '\n' && *(p+2) == '.' && *(p+3) == '\r' && *(p+4) == '\n' ? 1 : 0
+
+uint64 get_sessions_total_memory(struct smtp_session **sessions, int max_connections){
+   uint64 total = 0;
+
+   for(int i=0; i<max_connections; i++){
+      if(sessions[i]) total += sessions[i]->bufsize;
+   }
+
+   return total;
+}
+
+
+/*
+ * If the sending party sets the email size when it sends the "mail from"
+ * part in the smtp transaction, eg. MAIL FROM:<jajaja@akakak.lo> size=509603
+ * then piler-smtp could know the email size in advance and could do
+ * a better estimate on the allowed number of smtp sessions.
+ */
+
+uint64 get_sessions_total_expected_mail_size(struct smtp_session **sessions, int max_connections){
+   uint64 total = 0;
+
+   for(int i=0; i<max_connections; i++){
+      if(sessions[i]) total += sessions[i]->mail_size;
+   }
+
+   return total;
+}
+
 
 int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg){
    int slot;
@@ -31,6 +61,21 @@ int start_new_session(struct smtp_session **sessions, int socket, int *num_conne
       return -1;
    }
 
+   /*
+    * We are under the max_smtp_memory threshold
+    */
+
+   uint64 expected_total_mail_size = get_sessions_total_expected_mail_size(sessions, cfg->max_connections);
+   uint64 total_memory = get_sessions_total_memory(sessions, cfg->max_connections);
+
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "DEBUG: total smtp memory allocated: %llu, expected total size: %llu", total_memory, expected_total_mail_size);
+
+   if(total_memory > cfg->max_smtp_memory || expected_total_mail_size > cfg->max_smtp_memory){
+      syslog(LOG_PRIORITY, "ERROR: too much memory consumption: %llu", total_memory);
+      send(socket, SMTP_RESP_451_ERR_TOO_MANY_REQUESTS, strlen(SMTP_RESP_451_ERR_TOO_MANY_REQUESTS), 0);
+      return -1;
+   }
+
    slot = get_session_slot(sessions, cfg->max_connections);
 
    if(slot >= 0 && sessions[slot] == NULL){
@@ -84,11 +129,8 @@ struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int m
 
 
 void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg){
-   int i;
-
    session->slot = slot;
 
-   session->buflen = 0;
    session->protocol_state = SMTP_STATE_INIT;
 
    session->cfg = cfg;
@@ -100,27 +142,24 @@ void init_smtp_session(struct smtp_session *session, int slot, int sd, char *cli
    session->net.ssl = NULL;
 
    session->nullbyte = 0;
-   session->last_data_char = 0;
 
-   session->fd = -1;
-
-   memset(session->mailfrom, 0, SMALLBUFSIZE);
-
-   session->num_of_rcpt_to = 0;
-   for(i=0; i<MAX_RCPT_TO; i++) memset(session->rcptto[i], 0, SMALLBUFSIZE);
-
-   memset(session->buf, 0, MAXBUFSIZE);
    snprintf(session->remote_host, sizeof(session->remote_host)-1, "%s", client_addr);
 
-   reset_bdat_counters(session);
+   session->buf = NULL;
+   session->buflen = 0;
+   session->bufsize = 0;
 
-   time(&(session->lasttime));
+   reset_smtp_session(session);
 }
 
 
 void free_smtp_session(struct smtp_session *session){
    if(session){
 
+      if(session->buf != NULL){
+         free(session->buf);
+      }
+
       if(session->net.use_ssl == 1){
          SSL_shutdown(session->net.ssl);
          SSL_free(session->net.ssl);
@@ -160,71 +199,126 @@ void tear_down_session(struct smtp_session **sessions, int slot, int *num_connec
 }
 
 
+inline int get_last_newline_position(char *buf, int buflen){
+   int i;
+
+   for(i=buflen; i>0; i--){
+      if(*(buf+i) == '\n'){
+         i++;
+         break;
+      }
+   }
+
+   return i;
+}
+
+
+void flush_buffer(struct smtp_session *session){
+   // In the DATA phase skip the 1st character if it's a dot (.)
+   // and there are more characters before the trailing CR-LF
+   //
+   // See https://www.ietf.org/rfc/rfc5321.html#section-4.5.2 for more
+   for(int i=0; i<session->buflen; i++){
+      if(*(session->buf+i) == '\n' && *(session->buf+i+1) == '.' && *(session->buf+i+2) == '.'){
+         int dst = i + 2;
+         int src = dst + 1;
+         int l = session->buflen - src;
+         memmove(session->buf + dst, session->buf + src, l);
+         session->buflen -= 1;
+      }
+   }
+
+   // Exclude the trailing \r\n.\r\n sequence
+
+   session->buflen -= 5;
+
+   if(write(session->fd, session->buf, session->buflen) != session->buflen){
+      session->bad = 1;
+      syslog(LOG_PRIORITY, "ERROR (line: %d) %s: failed to write %d bytes", __LINE__, __func__, session->buflen);
+   }
+
+   session->tot_len = session->buflen;
+}
+
+
 void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg){
-   int puflen, rc, nullbyte;
-   char *p, copybuf[BIGBUFSIZE+MAXBUFSIZE], puf[MAXBUFSIZE];
+   // Update lasttime if we have something to process
+   time(&(session->lasttime));
 
-   // if there's something in the saved buffer, then let's merge them
+   if(session->protocol_state == SMTP_STATE_BDAT){
+      process_bdat(session, readbuf, readlen, cfg);
+      return;
+   }
 
-   int remaininglen = readlen + session->buflen;
+   // realloc memory if the new chunk doesn't fit in
 
-   if(session->buflen > 0){
-      memset(copybuf, 0, sizeof(copybuf));
+   if(session->buflen + readlen + 10 > session->bufsize){
+      // Handle if the current memory allocation for this email is above the max_message_size threshold
 
-      memcpy(copybuf, session->buf, session->buflen);
-      memcpy(&copybuf[session->buflen], readbuf, readlen);
+      if(session->buflen > cfg->max_message_size){
+         if(session->too_big == 0) syslog(LOG_PRIORITY, "ERROR: too big email: %d vs %d", session->buflen, cfg->max_message_size);
+         session->bad = 1;
+         session->too_big = 1;
+      }
 
+      if(session->bad == 0){
+         char *q = realloc(session->buf, session->bufsize + SMTPBUFSIZE);
+         if(q){
+            session->buf = q;
+            memset(session->buf+session->bufsize, 0, SMTPBUFSIZE);
+            session->bufsize += SMTPBUFSIZE;
+         } else {
+            syslog(LOG_PRIORITY, "ERROR: realloc %s %s %d", session->ttmpfile, __func__, __LINE__);
+            session->bad = 1;
+         }
+      }
+   }
+
+   // process smtp command
+   if(session->protocol_state != SMTP_STATE_DATA){
+
+      // We got ~2 MB of garbage and no valid smtp command
+      // Terminate the connection
+      if(session->buflen + readlen > SMTPBUFSIZE - 10){
+         session->bad = 1;
+      }
+
+      // We are at the beginning of the smtp transaction
+      if(session->bad == 1){
+         write1(&(session->net), SMTP_RESP_451_ERR, strlen(SMTP_RESP_451_ERR));
+         syslog(LOG_PRIORITY, "ERROR: sent 451 temp error back to client %s", session->ttmpfile);
+         return;
+      }
+
+      //printf("got %d *%s*\n", readlen, readbuf);
+
+      memcpy(session->buf + session->buflen, readbuf, readlen);
+      session->buflen += readlen;
+
+      int pos = get_last_newline_position(session->buf, session->buflen);
+
+      if(pos < readlen) return; // no complete command
+
+      process_smtp_command(session, cfg);
+
+      memset(session->buf, 0, session->bufsize);
       session->buflen = 0;
-      memset(session->buf, 0, MAXBUFSIZE);
 
-      p = &copybuf[0];
-   }
-   else {
-      readbuf[readlen] = 0;
-      p = readbuf;
+      return;
    }
 
+   if(session->bad == 0){
+      memcpy(session->buf + session->buflen, readbuf, readlen);
+      session->buflen += readlen;
 
-   do {
-      puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf)-1, &rc, &nullbyte);
-      p += puflen;
-      remaininglen -= puflen;
-
-      if(nullbyte){
-         session->nullbyte = 1;
+      char *p = session->buf + session->buflen - 5;
+      if(session->buflen >= 5 && GOT_CRLF_DOT_CRLF(p)){
+         flush_buffer(session);
+         process_command_period(session);
       }
-
-      // complete line: rc == OK and puflen > 0
-      // incomplete line with something in the buffer: rc == ERR and puflen > 0
-
-      if(puflen > 0){
-         // Update lasttime if we have a line to process
-         time(&(session->lasttime));
-
-         // Save incomplete line to buffer
-         if(rc == ERR){
-            memcpy(session->buf, puf, puflen);
-            session->buflen = puflen;
-         }
-
-         // We have a complete line to process
-
-         if(rc == OK){
-            if(session->protocol_state == SMTP_STATE_BDAT){
-               process_bdat(session, puf, puflen, cfg);
-            }
-            else if(session->protocol_state == SMTP_STATE_DATA){
-               sig_block(SIGALRM);
-               process_data(session, puf, puflen);
-               sig_unblock(SIGALRM);
-            }
-            else {
-               process_smtp_command(session, puf, cfg);
-            }
-         }
-      }
-
-   } while(puflen > 0);
+   } else if(strstr(readbuf, "\r\n.\r\n")){
+      process_command_period(session);
+   }
 
 }
 

src/smtp.c

@@ -15,59 +15,59 @@
 #include "smtp.h"
 
 
-void process_smtp_command(struct smtp_session *session, char *buf, struct config *cfg){
+void process_smtp_command(struct smtp_session *session, struct config *cfg){
    char response[SMALLBUFSIZE];
 
-   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "got on fd=%d: *%s*", session->net.socket, buf);
+   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "got on fd=%d: *%s*", session->net.socket, session->buf);
 
-   if(strncasecmp(buf, SMTP_CMD_HELO, strlen(SMTP_CMD_HELO)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_HELO, strlen(SMTP_CMD_HELO)) == 0){
       process_command_helo(session, response, sizeof(response));
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_EHLO, strlen(SMTP_CMD_EHLO)) == 0 ||
-         strncasecmp(buf, LMTP_CMD_LHLO, strlen(LMTP_CMD_LHLO)) == 0){
-      process_command_ehlo_lhlo(session, response, sizeof(response));
+   if(strncasecmp(session->buf, SMTP_CMD_EHLO, strlen(SMTP_CMD_EHLO)) == 0 ||
+         strncasecmp(session->buf, LMTP_CMD_LHLO, strlen(LMTP_CMD_LHLO)) == 0){
+      process_command_ehlo_lhlo(session, response, sizeof(response), cfg);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_HELP, strlen(SMTP_CMD_HELP)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_HELP, strlen(SMTP_CMD_HELP)) == 0){
       send_smtp_response(session, SMTP_RESP_221_PILER_SMTP_OK);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){
-      process_command_mail_from(session, buf);
+   if(strncasecmp(session->buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){
+      process_command_mail_from(session);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){
-      process_command_rcpt_to(session, buf, cfg);
+   if(strncasecmp(session->buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){
+      process_command_rcpt_to(session, cfg);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_DATA, strlen(SMTP_CMD_DATA)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_DATA, strlen(SMTP_CMD_DATA)) == 0){
       process_command_data(session, cfg);
       return;
    }
 
    /* Support only BDAT xxxx LAST command */
-   if(session->cfg->enable_chunking == 1 && strncasecmp(buf, SMTP_CMD_BDAT, strlen(SMTP_CMD_BDAT)) == 0 && strcasestr(buf, "LAST")){
-      get_bdat_size_to_read(session, buf);
+   if(session->cfg->enable_chunking == 1 && strncasecmp(session->buf, SMTP_CMD_BDAT, strlen(SMTP_CMD_BDAT)) == 0 && strcasestr(session->buf, "LAST")){
+      get_bdat_size_to_read(session);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_QUIT, strlen(SMTP_CMD_QUIT)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_QUIT, strlen(SMTP_CMD_QUIT)) == 0){
       process_command_quit(session, response, sizeof(response));
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_RESET, strlen(SMTP_CMD_RESET)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_RESET, strlen(SMTP_CMD_RESET)) == 0){
       process_command_reset(session);
       return;
    }
 
-   if(session->cfg->tls_enable == 1 && strncasecmp(buf, SMTP_CMD_STARTTLS, strlen(SMTP_CMD_STARTTLS)) == 0 && session->net.use_ssl == 0){
+   if(session->cfg->tls_enable == 1 && strncasecmp(session->buf, SMTP_CMD_STARTTLS, strlen(SMTP_CMD_STARTTLS)) == 0 && session->net.use_ssl == 0){
       process_command_starttls(session);
       return;
    }
@@ -76,41 +76,6 @@ void process_smtp_command(struct smtp_session *session, char *buf, struct config
 }
 
 
-void process_data(struct smtp_session *session, char *buf, int buflen){
-   if(session->last_data_char == '\n' && strcmp(buf, ".\r\n") == 0){
-      process_command_period(session);
-   }
-   else {
-      // write line to file
-      int written=0, n_writes=0;
-
-      // In the DATA phase skip the 1st character if it's a dot (.)
-      // and there are more characters before the trailing CR-LF
-      //
-      // See https://www.ietf.org/rfc/rfc5321.html#section-4.5.2 for more.
-
-      int dotstuff = 0;
-      if(*buf == '.' && buflen > 1 && *(buf+1) != '\r' && *(buf+1) != '\n') dotstuff = 1;
-
-      while(written < buflen) {
-         int len = write(session->fd, buf+dotstuff+written, buflen-dotstuff-written);
-
-         n_writes++;
-
-         if(len > 0){
-            if(len != buflen-dotstuff) syslog(LOG_PRIORITY, "WARN: partial write: %d/%d bytes (round: %d)", len, buflen-dotstuff, n_writes);
-            written += len + dotstuff;
-            session->tot_len += len;
-            dotstuff = 0;
-         }
-         else syslog(LOG_PRIORITY, "ERROR (line: %d) process_data(): written %d bytes", __LINE__, len);
-      }
-   }
-
-   session->last_data_char = buf[buflen-1];
-}
-
-
 void wait_for_ssl_accept(struct smtp_session *session){
    int rc;
 
@@ -152,7 +117,7 @@ void process_command_helo(struct smtp_session *session, char *buf, int buflen){
 }
 
 
-void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen){
+void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen, struct config *cfg){
    char extensions[SMALLBUFSIZE];
 
    memset(extensions, 0, sizeof(extensions));
@@ -163,7 +128,9 @@ void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int bufl
    if(session->net.use_ssl == 0 && session->cfg->tls_enable == 1) snprintf(extensions, sizeof(extensions)-1, "%s", SMTP_EXTENSION_STARTTLS);
    if(session->cfg->enable_chunking == 1) strncat(extensions, SMTP_EXTENSION_CHUNKING, sizeof(extensions)-strlen(extensions)-2);
 
-   snprintf(buf, buflen-1, SMTP_RESP_250_EXTENSIONS, session->cfg->hostid, extensions);
+   //#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n%s250-SIZE %d\r\n250 8BITMIME\r\n"
+
+   snprintf(buf, buflen-1, SMTP_RESP_250_EXTENSIONS, session->cfg->hostid, cfg->max_message_size, extensions);
 
    send_smtp_response(session, buf);
 }
@@ -234,26 +201,29 @@ void process_command_starttls(struct smtp_session *session){
 }
 
 
-void process_command_mail_from(struct smtp_session *session, char *buf){
+void process_command_mail_from(struct smtp_session *session){
+   memset(session->mailfrom, 0, SMALLBUFSIZE);
+
    if(session->protocol_state != SMTP_STATE_HELO && session->protocol_state != SMTP_STATE_PERIOD && session->protocol_state != SMTP_STATE_BDAT){
       send(session->net.socket, SMTP_RESP_503_ERR, strlen(SMTP_RESP_503_ERR), 0);
    }
    else {
-      memset(&(session->ttmpfile[0]), 0, SMALLBUFSIZE);
-      make_random_string((unsigned char*)&(session->ttmpfile[0]), QUEUE_ID_LEN);
       session->protocol_state = SMTP_STATE_MAIL_FROM;
 
-      extractEmail(buf, session->mailfrom);
+      extractEmail(session->buf, session->mailfrom);
 
-      reset_bdat_counters(session);
-      session->tot_len = 0;
+      int mailsize = get_size_from_smtp_mail_from(session->buf);
+
+      reset_smtp_session(session);
+
+      session->mail_size = mailsize;
 
       send_smtp_response(session, SMTP_RESP_250_OK);
    }
 }
 
 
-void process_command_rcpt_to(struct smtp_session *session, char *buf, struct config *cfg){
+void process_command_rcpt_to(struct smtp_session *session, struct config *cfg){
 
    if(session->protocol_state == SMTP_STATE_MAIL_FROM || session->protocol_state == SMTP_STATE_RCPT_TO){
 
@@ -262,7 +232,7 @@ void process_command_rcpt_to(struct smtp_session *session, char *buf, struct con
       session->protocol_state = SMTP_STATE_RCPT_TO;
 
       if(session->num_of_rcpt_to < MAX_RCPT_TO){
-         extractEmail(buf, session->rcptto[session->num_of_rcpt_to]);
+         extractEmail(session->buf, session->rcptto[session->num_of_rcpt_to]);
 
          // Check if we should accept archive_address only
          if(cfg->archive_address[0] && !strstr(cfg->archive_address, session->rcptto[session->num_of_rcpt_to])){
@@ -322,13 +292,23 @@ void process_command_period(struct smtp_session *session){
 
    syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d, client=%s, fd=%d, fsync=%ld", session->ttmpfile, session->mailfrom, session->tot_len, session->remote_host, session->net.socket, tvdiff(tv2, tv1));
 
-   move_email(session);
+   if(session->bad == 1 || session->too_big == 1){
+      snprintf(buf, sizeof(buf)-1, SMTP_RESP_451_ERR);
+      unlink(session->ttmpfile);
+      syslog(LOG_PRIORITY, "ERROR: problem in processing, removing %s", session->ttmpfile);
+      if(session->too_big)
+         snprintf(buf, sizeof(buf)-1, "%s", SMTP_RESP_552_ERR_TOO_BIG_EMAIL);
+      else
+         snprintf(buf, sizeof(buf)-1, "%s", SMTP_RESP_451_ERR);
+   } else {
+      move_email(session);
+      snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
+   }
 
-   snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
-
-   session->buflen = 0;
-   session->last_data_char = 0;
-   memset(session->buf, 0, sizeof(session->buf));
+   if(session->buf){
+      memset(session->buf, 0, session->bufsize);
+      session->buflen = 0;
+   }
 
    send_smtp_response(session, buf);
 }
@@ -344,15 +324,32 @@ void process_command_quit(struct smtp_session *session, char *buf, int buflen){
 
 
 void process_command_reset(struct smtp_session *session){
+   reset_smtp_session(session);
    send_smtp_response(session, SMTP_RESP_250_OK);
-
-   session->tot_len = 0;
-   session->fd = -1;
    session->protocol_state = SMTP_STATE_HELO;
-   session->last_data_char = 0;
+}
+
+
+void reset_smtp_session(struct smtp_session *session){
+   session->tot_len = 0;
+   session->mail_size = 0;
+   session->too_big = 0;
+   session->bad = 0;
+
+   session->fd = -1;
+
+   session->num_of_rcpt_to = 0;
+   for(int i=0; i<MAX_RCPT_TO; i++) memset(session->rcptto[i], 0, SMALLBUFSIZE);
 
    reset_bdat_counters(session);
 
+   time(&(session->lasttime));
+
    memset(&(session->ttmpfile[0]), 0, SMALLBUFSIZE);
-   make_random_string((unsigned char *)&(session->ttmpfile[0]), QUEUE_ID_LEN);
+   make_random_string((unsigned char*)&(session->ttmpfile[0]), QUEUE_ID_LEN);
+
+   if(session->buf){
+      memset(session->buf, 0, session->bufsize);
+   }
+   session->buflen = 0;
 }

src/smtp.h

@@ -7,22 +7,24 @@
 
 #include <piler.h>
 
-void process_smtp_command(struct smtp_session *session, char *buf, struct config *cfg);
+void process_smtp_command(struct smtp_session *session, struct config *cfg);
 void process_data(struct smtp_session *session, char *buf, int buflen);
 
 void send_smtp_response(struct smtp_session *session, char *buf);
 void process_command_helo(struct smtp_session *session, char *buf, int buflen);
-void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen);
+void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen, struct config *cfg);
 void process_command_quit(struct smtp_session *session, char *buf, int buflen);
 void process_command_reset(struct smtp_session *session);
-void process_command_mail_from(struct smtp_session *session, char *buf);
-void process_command_rcpt_to(struct smtp_session *session, char *buf, struct config *cfg);
+void process_command_mail_from(struct smtp_session *session);
+void process_command_rcpt_to(struct smtp_session *session, struct config *cfg);
 void process_command_data(struct smtp_session *session, struct config *cfg);
 void process_command_period(struct smtp_session *session);
 void process_command_starttls(struct smtp_session *session);
 
 void reset_bdat_counters(struct smtp_session *session);
-void get_bdat_size_to_read(struct smtp_session *session, char *buf);
+void get_bdat_size_to_read(struct smtp_session *session);
 void process_bdat(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg);
 
+void reset_smtp_session(struct smtp_session *session);
+
 #endif

src/smtpcodes.h

@@ -38,7 +38,7 @@
 #define SMTP_RESP_220_READY_TO_START_TLS "220 Ready to start TLS\r\n"
 #define SMTP_RESP_221_GOODBYE "221 %s Goodbye\r\n"
 #define SMTP_RESP_250_OK "250 Ok\r\n"
-#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n250-PIPELINING\r\n%s250-SIZE\r\n250 8BITMIME\r\n"
+#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n250-SIZE %d\r\n%s250 8BITMIME\r\n"
 
 #define SMTP_EXTENSION_STARTTLS "250-STARTTLS\r\n"
 #define SMTP_EXTENSION_CHUNKING "250-CHUNKING\r\n"
@@ -51,6 +51,7 @@
 #define SMTP_RESP_421_ERR_TMP "421 %s service not available\r\n"
 #define SMTP_RESP_421_ERR_WRITE_FAILED "421 writing queue file failed\r\n"
 #define SMTP_RESP_421_ERR_ALL_PORTS_ARE_BUSY "421 All server ports are busy\r\n"
+#define SMTP_RESP_451_ERR_TOO_MANY_REQUESTS "451 Too many requests, try again later\r\n"
 #define SMTP_RESP_451_ERR "451 Error in processing, try again later\r\n"
 #define SMTP_RESP_454_ERR_TLS_TEMP_ERROR "454 TLS not available currently\r\n"
 
@@ -59,6 +60,7 @@
 #define SMTP_RESP_550_ERR_INVALID_RECIPIENT "550 Invalid recipient\r\n"
 #define SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY "550 You are banned by local policy\r\n"
 #define SMTP_RESP_550_ERR "550 Service currently unavailable\r\n"
+#define SMTP_RESP_552_ERR_TOO_BIG_EMAIL "552 Too big email\r\n"
 
 
 // LMTP commands

src/tai.h

@@ -5,12 +5,12 @@
 #ifndef _TAI_H
    #define _TAI_H
 
+#include "config.h"
+
 #define TAI_PACK 8
 #define TAIA_PACK 16
 #define TIMESTAMP 25
 
-typedef unsigned long long uint64;
-
 struct tai {
    uint64 x;
 };

systemd/pilersearch.service

@@ -5,6 +5,7 @@ After=network.target mariadb.service
 [Service]
 ExecStart=/usr/bin/searchd --config /etc/piler/manticore.conf
 ExecStartPre=/bin/bash -c "if [[ ! -d /var/run/piler ]]; then mkdir -p /var/run/piler; chown piler:piler /var/run/piler; fi"
+ExecStop=/usr/bin/searchd --config /etc/piler/manticore.conf --stopwait
 PIDFile=/var/run/piler/searchd.pid
 KillMode=process
 Restart=on-failure

tests/cases/05-smtp.inc

@@ -6,23 +6,23 @@ case1() {
 
    setup
 
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox2" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet2" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam0" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam1" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam2" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/journal" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/deduptest" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/special" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox2"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet2"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam0"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam1"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam2"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/journal"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/deduptest"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/special"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus"  --no-counter
 
 
-   wait_until_emails_are_processed "piler1" 3019
+   wait_until_emails_are_processed "piler1" 3020
    [[ $RT -eq 1 ]] || docker exec "piler1" su piler -c /usr/libexec/piler/indexer.delta.sh 2>/dev/null
 
-   count_status_values 3019 2908 111 0
+   count_status_values 3020 2909 111 0
 
    test_retrieved_messages_are_the_same "piler1" "piler"
 

tests/setup.inc

@@ -2,15 +2,11 @@ NODE_WORKERS=( "worker0" "worker1" )
 SMTP_GW="smtpgw"
 DOCKER_LIMIT=( --pids-limit 256 --memory 512M )
 
-export CONTAINERS=( "$NODE_GUI" "${NODE_WORKERS[@]}" "$SMTP_GW" "$SINGLE_SERVER" "piler1" "syslog.host" )
+export CONTAINERS=( "$NODE_GUI" "${NODE_WORKERS[@]}" "$SMTP_GW" "$SINGLE_SERVER" "piler1" )
 
 setup() {
-   destroy_containers
-   set -x
    launch_containers
-   set +x
-   create_rules "piler1"
-   add_data_officer "piler1"
+   setup_piler "piler1"
 }
 
 cleanup_package() {
@@ -21,43 +17,35 @@ cleanup_package() {
 }
 
 launch_containers() {
+   local composefile="docker-compose-piler.yaml"
 
-   log "starting syslog server"
-   docker run -d --net=piler "${DOCKER_LIMIT[@]}" --name syslog.host sutoj/syslog
+   log "${FUNCNAME[0]}"
 
-   log "starting piler"
-   docker run -d --net=piler "${DOCKER_LIMIT[@]}" --name piler1 \
-      -e PACKAGE="$PACKAGE" \
-      -e PILER_HOST="cust1.acts.hu" \
-      -e RT="$RT" \
-      -p 127.0.0.1:80:80 -p 25:25 \
-      -v "${PACKAGE_DIR}:/data:ro" \
-      -v "${CONFIG_DIR}/11-aaaa.conf:/etc/rsyslog.d/11-aaaa.conf:ro" \
-      "$docker_image"
+   pushd "${WORKSPACE}/ci/test/docker-compose"
 
-   wait_for_sleep_cycle_in_container "piler1"
+   echo -e "PACKAGE=$PACKAGE\nPILER_HOST=cust1.acts.hu\nRT=$RT" > .env
 
-   cleanup_package "$PACKAGE"
+   destroy_containers
+
+   log "running docker compose up"
+
+   docker compose -f "$composefile" up -d
+
+   wait_until_container_is_healthy "piler1"
+
+   docker compose -f "$composefile" ps
+
+   popd
 }
 
-create_rules() {
+setup_piler() {
    local container="$1"
 
-   echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"address.com\",\"address.com\"),(\"acts.hu\",\"acts.hu\"),(\"gtsce.com\",\"gtsce.com\"),(\"datanet.hu\",\"datanet.hu\"),(\"gtsdatanet.hu\",\"gtsdatanet.hu\"),(\"gts.hu\",\"gts.hu\"),(\"aaa.fu\",\"aaa.fu\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh
+   log "${FUNCNAME[0]}"
 
-   echo 'echo "insert into archiving_rule (subject) values (\"Android táblagép\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
-   echo 'echo "insert into archiving_rule (\`from\`) values (\"@gmail.com\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
-   echo 'echo "insert into archiving_rule (\`from\`,attachment_type, _attachment_size, attachment_size) values (\"finderis.co.ua\", \"image\", \">\", 100000)"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
-   echo 'echo "insert into archiving_rule (\`to\`) values (\"undisclosed-recipients\")"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
-   echo 'echo "insert into import (\`type\`, username, password, server) values (\"imap-ssl\", \"sanyi@aaa.fu\", \"abcde123\", \"imap.aaa.fu\")"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
+   pushd "${WORKSPACE}/${PROJECT_ID}/tests"
 
-   echo 'echo "update user set password=\"\$6\$GKL00T\$8jqoFOe3PyAbOCLwKB7JwndwC.IinHrZRkdoQDZUc8vybZ88sA2qomlz5JceNif8fFpkGzZ03ilvQa7tqQx0v1\""| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
+   docker exec "$container" /usr/local/bin/setup-piler.sh
 
-   docker exec "$container" /etc/init.d/rc.piler reload
-}
-
-add_data_officer() {
-   local container="$1"
-   echo 'echo "insert into user (uid, username, realname, password, domain, dn, isadmin) values(2, \"dataofficer\", \"Data officer\", \"\$6\$rX285LfP\$ZxhlacbzKuCcqkaizzBu8SAiYb6.f8K4Us08nUHwSpWMQkNhw4o2rmfKXoTfaM4rnBHUYVK1N4IfBsqN8CAtS/\", \"local\", \"*\", 4)"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh
-   echo 'echo "insert into email (uid, email) values(2, \"do@local\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh
+   popd
 }

tests/setup.sql

@@ -0,0 +1,14 @@
+use piler;
+
+insert into domain (domain, mapped) values("fictive.com","fictive.com"),("address.com","address.com"),("acts.hu","acts.hu"),("gtsce.com","gtsce.com"),("datanet.hu","datanet.hu"),("gtsdatanet.hu","gtsdatanet.hu"),("gts.hu","gts.hu"),("aaa.fu","aaa.fu");
+
+insert into archiving_rule (subject) values ("Android táblagép");
+insert into archiving_rule (`from`) values ("@gmail.com");
+insert into archiving_rule (`from`,attachment_type, _attachment_size, attachment_size) values ("finderis.co.ua", "image", ">", 100000);
+insert into archiving_rule (`to`) values ("undisclosed-recipients");
+insert into import (`type`, username, password, server) values ("imap-ssl", "sanyi@aaa.fu", "abcde123", "imap.aaa.fu");
+
+update user set password="$6$GKL00T$8jqoFOe3PyAbOCLwKB7JwndwC.IinHrZRkdoQDZUc8vybZ88sA2qomlz5JceNif8fFpkGzZ03ilvQa7tqQx0v1";
+
+insert into user (uid, username, realname, password, domain, dn, isadmin) values(2, "dataofficer", "Data officer", "$6$rX285LfP$ZxhlacbzKuCcqkaizzBu8SAiYb6.f8K4Us08nUHwSpWMQkNhw4o2rmfKXoTfaM4rnBHUYVK1N4IfBsqN8CAtS/", "local", "*", 4);
+insert into email (uid, email) values(2, "do@local");

unit_tests/check_attachments.c

@@ -27,7 +27,7 @@ static void test_attachments(struct config *cfg){
 
    TEST_HEADER();
 
-   snprintf(data.licence.hostname, TINYBUFSIZE-1, "yourdomain.com");
+   snprintf(data.licence.hostname, TINYBUFSIZE-1, "example.com");
 
    for(i=0; i<sizeof(tests)/sizeof(struct attachments); i++){
 

unit_tests/run.sh

@@ -24,6 +24,13 @@ setup_mysql() {
    mysql -u piler -ppiler123 piler1 < ../util/db-mysql.sql
 }
 
+run_smtp_tests() {
+   mkdir -p /var/piler/store/00/piler /var/piler/tmp /var/piler/manticore
+   chown -R piler:piler /var/piler/
+   ../src/piler-smtp -L 5 -d
+   ./smtp -s 127.0.0.1
+}
+
 if [[ -v BUILD_NUMBER ]]; then
    setup_mysql
 fi
@@ -37,3 +44,5 @@ fi
 ./check_hash
 ./check_decoder
 ./check_attachments
+
+if [[ -v BUILD_NUMBER ]]; then run_smtp_tests; fi

unit_tests/smtp.c

@@ -4,11 +4,13 @@
 
 #include "test.h"
 
-
 extern char *optarg;
 extern int optind;
 
-char *testmessage = "From: aaa@aaa.fu\nTo: bela@aaa.fu\nMessage-Id: ajajajaja\nSubject: this is a test\n\nAaaaaa.";
+char *testmessage = "From: aaa@aaa.fu\r\nTo: bela@aaa.fu\r\nMessage-Id: ajajajaja\r\nSubject: this is a test\r\n\r\nAaaaaa";
+char *testmessage2 = " and the last line\r\n";
+
+char *recipient = "aaa@worker0";
 
 int helo = 0; // 0=HELO, 1=EHLO
 
@@ -16,12 +18,24 @@ void usage(){
    printf("\nusage: smtp\n\n");
    printf("    -s <smtp server>                  SMTP server\n");
    printf("    -p <smtp port>                    SMTP port (25)\n");
+   printf("    -r <recipient>                    Envelope recipient\n");
    printf("    -t <timeout>                      Timeout in sec (10)\n");
 
    exit(0);
 }
 
 
+int countreplies(char *s){
+   int replies = 0;
+
+   for(; *s; s++){
+      if(*s == '\n') replies++;
+   }
+
+   return replies;
+}
+
+
 void connect_to_smtp_server(char *server, int port, struct data *data){
    int rc;
    char port_string[8], buf[MAXBUFSIZE];
@@ -53,14 +67,15 @@ void connect_to_smtp_server(char *server, int port, struct data *data){
    }
 
    recvtimeoutssl(data->net, buf, sizeof(buf));
-   printf("rcvd: %s", buf);
 
 ENDE:
    freeaddrinfo(res);
 }
 
 
-void send_smtp_command(struct net *net, char *cmd, char *buf, int buflen){
+void send_smtp_command(struct net *net, char *cmd, char *buf, int buflen, int expectedreplies){
+   int tot=0;
+
    if(net == NULL || cmd == NULL) return;
 
    if(net->socket == -1){
@@ -68,25 +83,32 @@ void send_smtp_command(struct net *net, char *cmd, char *buf, int buflen){
       return;
    }
 
-   printf("sent: %s", cmd);
    write1(net, cmd, strlen(cmd));
-   recvtimeoutssl(net, buf, buflen);
-   printf("rcvd: %s", buf);
+
+   while(1){
+      tot += recvtimeoutssl(net, buf+tot, buflen);
+      if(countreplies(buf) == expectedreplies) break;
+   }
 }
 
 
 void send_helo_command(struct net *net){
+   int replies=1;
    char recvbuf[MAXBUFSIZE];
 
    if(helo == 0){
-      send_smtp_command(net, "HELO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1);
-      assert(strncmp(recvbuf, "250 ", 4) == 0 && "HELO");
+      send_smtp_command(net, "HELO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1, replies);
+      ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
    }
    else {
-      send_smtp_command(net, "EHLO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1);
-      assert(strncmp(recvbuf, "250-", 4) == 0 && "EHLO");
-      if(net->use_ssl == 0) assert(strstr(recvbuf, "250-STARTTLS") && "STARTTLS");
-      else assert(strstr(recvbuf, "250-STARTTLS") == NULL && "STARTTLS");
+      //replies = 6;
+      replies = 5;
+      if(net->use_ssl == 1) replies--;
+
+      send_smtp_command(net, "EHLO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1, replies);
+      ASSERT(strncmp(recvbuf, "250-", 4) == 0, recvbuf);
+      if(net->use_ssl == 0){ ASSERT(strstr(recvbuf, "250-STARTTLS"), recvbuf); }
+      else { ASSERT(strstr(recvbuf, "250-STARTTLS") == NULL, recvbuf); }
    }
 }
 
@@ -94,209 +116,392 @@ void send_helo_command(struct net *net){
 static void test_smtp_commands_one_at_a_time(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RCPT");
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "354 ", 4) == 0 && "DATA");
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage);
 
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "PERIOD");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
-static void test_smtp_commands_pipelining(char *server, int port, struct data *data){
+static void test_smtp_commands_one_at_a_time_data_in_2_parts(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+
+   write1(data->net, testmessage, strlen(testmessage));
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage2);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
+
+   close(data->net->socket);
+
+   TEST_FOOTER();
+}
+
+
+/*static void test_smtp_commands_pipelining(char *server, int port, struct data *data){
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+
+   TEST_HEADER();
+
+   connect_to_smtp_server(server, port, data);
+
+   send_helo_command(data->net);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\nQUIT\r\n", testmessage);
 
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 2);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
-}
+
+   TEST_FOOTER();
+}*/
 
 
 static void test_smtp_commands_with_reset_command(char *server, int port, struct data *data){
-   char recvbuf[MAXBUFSIZE];
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+
+   TEST_HEADER();
 
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "RSET\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RSET");
+   send_smtp_command(data->net, "RSET\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "503 ", 4) == 0 && "RCPT");
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "503 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
 static void test_smtp_commands_partial_command(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
    write1(data->net, "M", 1);
-   printf("sent: M\n");
 
-   send_smtp_command(data->net, "AIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, "AIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RCPT");
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "354 ", 4) == 0 && "DATA");
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage);
 
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "PERIOD");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
-static void test_smtp_commands_partial_command_pipelining(char *server, int port, struct data *data){
+/*static void test_smtp_commands_partial_command_pipelining(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
    write1(data->net, "M", 1);
-   printf("sent: M\n");
 
-   send_smtp_command(data->net, "AIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   snprintf(sendbuf, sizeof(sendbuf)-1, "AIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\nQUIT\r\n", testmessage);
 
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 2);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
-}
+
+   TEST_FOOTER();
+}*/
 
 
 static void test_smtp_commands_starttls(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "STARTTLS\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "220 ", 4) == 0 && "STARTTLS");
+   send_smtp_command(data->net, "STARTTLS\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "220 ", 4) == 0, recvbuf);
 
    init_ssl_to_server(data);
    data->net->use_ssl = 1;
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RCPT");
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "354 ", 4) == 0 && "DATA");
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage);
 
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "PERIOD");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
 static void test_smtp_commands_period_command_in_2_parts(char *server, int port, char *part1, char *part2, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   // As long as pipelining support is not reintroduced
+   //
+   /*snprintf(sendbuf, sizeof(sendbuf)-1, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   if(!strstr(recvbuf, "354 ")) recvtimeoutssl(data->net, recvbuf, sizeof(recvbuf)-1);
+   ASSERT(strstr(recvbuf, "354 "), recvbuf);*/
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s%s", testmessage, part1);
    write1(data->net, sendbuf, strlen(sendbuf));
 
    snprintf(sendbuf, sizeof(sendbuf), "%s", part2);
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "QUIT\r\n");
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
 
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
 static void test_smtp_commands_period_command_in_its_own_packet(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   // As long as pipelining support is not reintroduced
+   //
+   /*snprintf(sendbuf, sizeof(sendbuf)-1, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   if(!strstr(recvbuf, "354 ")) recvtimeoutssl(data->net, recvbuf, sizeof(recvbuf)-1);
+   ASSERT(strstr(recvbuf, "354 "), recvbuf);*/
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+
+
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s", testmessage);
    write1(data->net, sendbuf, strlen(sendbuf));
 
    snprintf(sendbuf, sizeof(sendbuf), "\r\n.\r\n");
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "QUIT\r\n");
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
 
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
+}
+
+
+static void test_smtp_commands_with_partial_data_lines(char *server, int port, struct data *data){
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+   int yes=1;
+
+   TEST_HEADER();
+
+   connect_to_smtp_server(server, port, data);
+
+   setsockopt(data->net->socket, IPPROTO_TCP, TCP_NODELAY, &yes, (socklen_t)sizeof(int));
+
+   send_helo_command(data->net);
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "From: aaa@aaa.fu\r\nTo: bela@aaa.fu\r\nMessage-Id: ajajajaja\r\nSubject: this is a test\r\n\r\nAaaaaa");
+   write1(data->net, sendbuf, strlen(sendbuf)); sleep(2);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "jjdkdjkd dkd some garbage.");
+   write1(data->net, sendbuf, strlen(sendbuf)); sleep(2);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "\r\nSome shit again  au aua ua au aua uuu");
+   write1(data->net, sendbuf, strlen(sendbuf)); sleep(2);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, ".\r\nAnd the last line.\r\n.\r\n");
+   write1(data->net, sendbuf, strlen(sendbuf));
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\nQUIT\r\n", testmessage);
+   recvtimeoutssl(data->net, recvbuf, sizeof(recvbuf));
+
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+
+   close(data->net->socket);
+
+   TEST_FOOTER();
+}
+
+
+static void test_smtp_bdat_last_one_at_a_time(char *server, int port, struct data *data){
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+
+   TEST_HEADER();
+
+   connect_to_smtp_server(server, port, data);
+
+   send_helo_command(data->net);
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "BDAT %ld LAST\r\n", strlen(testmessage)+strlen(testmessage2));
+   write1(data->net, sendbuf, strlen(sendbuf));
+
+   write1(data->net, testmessage, strlen(testmessage));
+
+   send_smtp_command(data->net, testmessage2, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
+
+   close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
@@ -317,6 +522,7 @@ int main(int argc, char **argv){
          {
             {"server",       required_argument,  0,  's' },
             {"port",         required_argument,  0,  'p' },
+            {"rcpt",         required_argument,  0,  'r' },
             {"timeout",      required_argument,  0,  't' },
             {"lhlo",         no_argument,        0,  'l' },
             {"help",         no_argument,        0,  'h' },
@@ -325,9 +531,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      int c = getopt_long(argc, argv, "c:s:p:t:lh?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:s:p:t:r:lh?", long_options, &option_index);
 #else
-      int c = getopt(argc, argv, "c:s:p:t:lh?");
+      int c = getopt(argc, argv, "c:s:p:t:r:lh?");
 #endif
 
 
@@ -343,6 +549,10 @@ int main(int argc, char **argv){
                     port = atoi(optarg);
                     break;
 
+         case 'r' :
+                    recipient = optarg;
+                    break;
+
          case 't' :
                     net.timeout = atoi(optarg);
                     break;
@@ -357,7 +567,7 @@ int main(int argc, char **argv){
                     break;
 
 
-         default  : 
+         default  :
                     break;
        }
    }
@@ -367,21 +577,26 @@ int main(int argc, char **argv){
    data.net = &net;
 
    test_smtp_commands_one_at_a_time(server, port, &data);
-   test_smtp_commands_pipelining(server, port, &data);
+   test_smtp_commands_one_at_a_time_data_in_2_parts(server, port, &data);
+   ////test_smtp_commands_pipelining(server, port, &data);
    test_smtp_commands_with_reset_command(server, port, &data);
    test_smtp_commands_partial_command(server, port, &data);
-   test_smtp_commands_partial_command_pipelining(server, port, &data);
+   ////test_smtp_commands_partial_command_pipelining(server, port, &data);
+
    test_smtp_commands_period_command_in_2_parts(server, port, "\r", "\n.\r\n", &data);
    test_smtp_commands_period_command_in_2_parts(server, port, "\r\n", ".\r\n", &data);
    test_smtp_commands_period_command_in_2_parts(server, port, "\r\n.", "\r\n", &data);
    test_smtp_commands_period_command_in_2_parts(server, port, "\r\n.\r", "\n", &data);
    test_smtp_commands_period_command_in_its_own_packet(server, port, &data);
 
-   helo = 1; // we must use EHLO to get the STARTTLS in the response
-   test_smtp_commands_starttls(server, port, &data);
+   test_smtp_commands_with_partial_data_lines(server, port, &data);
 
+   helo = 1; // we must use EHLO to get the STARTTLS in the response
+
+   test_smtp_commands_starttls(server, port, &data);
+   data.net->use_ssl = 0;
+
+   test_smtp_bdat_last_one_at_a_time(server, port, &data);
 
    return 0;
 }
-
-

unit_tests/test.conf

@@ -10,7 +10,7 @@ encrypt_messages=1
 extra_to_field=X-Envelope-To:
 extract_attachments=1
 helper_timeout=20
-hostid=piler.yourdomain.com
+hostid=piler.example.com
 iv=
 listen_addr=0.0.0.0
 listen_port=25

unit_tests/test.h

@@ -8,6 +8,7 @@
 #include <netdb.h>
 #include <arpa/inet.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <fcntl.h>
 #include <locale.h>
 #include <getopt.h>

util/Makefile.in

@@ -36,6 +36,7 @@ install:
 	$(INSTALL) -m 0755 $(srcdir)/gmail-imap-import.php $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/generate_stats.php $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/get-token.py $(DESTDIR)$(libexecdir)/piler
+	$(INSTALL) -m 0755 $(srcdir)/refresh-token.py $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/healthcheck.sh $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/mailstat.php $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/sign.php $(DESTDIR)$(libexecdir)/piler

util/db-mysql.sql

@@ -49,12 +49,12 @@ create table if not exists `metadata` (
 ) Engine=InnoDB;
 
 create index metadata_idx on metadata(`piler_id`);
-create index metadata_idx2 on metadata(`message_id`); 
-create index metadata_idx3 on metadata(`reference`); 
-create index metadata_idx4 on metadata(`bodydigest`); 
-create index metadata_idx5 on metadata(`deleted`); 
-create index metadata_idx6 on metadata(`arrived`); 
-create index metadata_idx7 on metadata(`retained`); 
+create index metadata_idx2 on metadata(`message_id`);
+create index metadata_idx3 on metadata(`reference`);
+create index metadata_idx4 on metadata(`bodydigest`);
+create index metadata_idx5 on metadata(`deleted`);
+create index metadata_idx6 on metadata(`arrived`);
+create index metadata_idx7 on metadata(`retained`);
 create index metadata_idx8 on metadata(`fromdomain`);
 create index metadata_idx9 on metadata(`from`);
 create index metadata_idx10 on metadata(`sent`);
@@ -124,7 +124,7 @@ create table if not exists `archiving_rule` (
    `days` int default 0,
    `folder_id` int default 0,
    primary key (`id`),
-   unique(`from`,`to`,`subject`,`body`,`_size`,`size`,`attachment_name`,`attachment_type`,`_attachment_size`,`attachment_size`,`spam`) 
+   unique(`from`,`to`,`subject`,`body`,`_size`,`size`,`attachment_name`,`attachment_type`,`_attachment_size`,`attachment_size`,`spam`)
 ) ENGINE=InnoDB;
 
 
@@ -242,7 +242,7 @@ create table if not exists `email_groups` (
 ) ENGINE=InnoDB;
 
 
-create table if not exists `group` (
+create table if not exists `usergroup` (
    `id` bigint unsigned not null auto_increment primary key,
    `groupname` char(128) not null unique
 ) ENGINE=InnoDB;
@@ -344,7 +344,7 @@ create table if not exists `audit` (
    `email` varchar(128) not null,
    `domain` varchar(128) not null,
    `action` int not null,
-   `ipaddr` char(15) not null,
+   `ipaddr` varchar(39) not null,
    `meta_id` bigint unsigned not null,
    `description` varchar(255) default null,
    `vcode` char(64) default null,

util/db-upgrade.sql

@@ -6,3 +6,9 @@ alter table sph_index change column todomain todomain blob(512) default null;
 
 
 alter table metadata change column subject `subject` blob(512) default null;
+
+
+SELECT Count(*) INTO @exists FROM information_schema.tables WHERE table_schema = 'piler' AND table_type = 'BASE TABLE' AND table_name = 'group';
+SET @query = If(@exists>0, 'RENAME TABLE `group` TO usergroup', 'SELECT 1 from dual');
+PREPARE stmt FROM @query;
+EXECUTE stmt;

util/get-token.py

@@ -16,6 +16,7 @@ ClientId = "c6843299-05c4-4c2e-9398-64dd42f14b6d"  # Fix this value only
 ClientSecret = ""
 Scopes = ['https://outlook.office.com/IMAP.AccessAsUser.All']
 AccessTokenFileName = "access_token"
+RefreshTokenFileName = "refresh_token"
 
 
 cache = SerializableTokenCache()
@@ -70,7 +71,8 @@ if 'error' in token:
     print(token)
     sys.exit("Failed to get access token")
 
-# print(token['refresh_token'])
-
 with open(AccessTokenFileName, 'w') as f:
     f.write(token['access_token'])
+
+with open(RefreshTokenFileName, 'w') as f:
+    f.write(token['refresh_token'])

util/imapfetch.py

@@ -14,6 +14,7 @@ opts = {}
 INBOX = 'INBOX'
 ST_RUNNING = 1
 
+imaplib._MAXLINE = 10000000
 
 def generate_auth_string(user, token):
     auth_string = f"user={user}\1auth=Bearer {token}\1\1"
@@ -99,6 +100,11 @@ def process_folder(conn, folder):
                 opts['counter'] += 1
                 with open("{}.eml".format(opts['counter']), "wb") as f:
                     f.write(data[0][1])
+                    if opts['remove']:
+                        conn.store(num, '+FLAGS', '\\Deleted')
+
+        if opts['remove']:
+            conn.expunge()
 
 
 def main():
@@ -121,6 +127,7 @@ def main():
                         default="/var/piler/imap")
     parser.add_argument("-i", "--import-from-table", action='store_true',
                         help="Read imap conn data from import table")
+    parser.add_argument("-r", "--remove", help="remove downloaded messages", action='store_true')
     parser.add_argument("-v", "--verbose", help="verbose mode", action='store_true')
 
     args = parser.parse_args()
@@ -139,6 +146,7 @@ def main():
     opts['db'] = None
     opts['id'] = 0
     opts['access_token'] = ''
+    opts['remove'] = args.remove
 
     if args.date:
         opts['search'] = args.date

util/indexer.main.sh.in

@@ -22,6 +22,16 @@ finish() {
    rm -f "$MAINTMPFILE"
 }
 
+get_index_size() {
+   local mainfiles="$1"
+   local sum=0
+
+   # shellcheck disable=SC2034
+   while read -r a b; do
+      sum=$(( sum + b ))
+   done < <( find /var/piler/${INDEXDIR}/ -type f -name "$mainfiles" -printf "%TY%Tm%Td %s\\n" )
+   printf "%d" $sum
+}
 
 if [[ -f "$MAINTMPFILE" ]]; then
    echo "INDEXER ERROR: indexer merging to main index is already running. It started at $(cat "$MAINTMPFILE")" | logger -p "$PRIORITY"
@@ -48,8 +58,5 @@ indexer --config "$CONFIG_FILE" --quiet dailydelta1 --rotate
 
 echo "INDEXER INFO: resetting daily delta finished" | logger -p "$PRIORITY"
 
-sum=0
-while read -r a b; do
-   sum=$(( sum + b ))
-done < <( find /var/piler/${INDEXDIR}/ -type f -name main\*.sp[dp] -printf "%TY%Tm%Td %s\\n" )
-printf "%d" $sum > /var/piler/stat/main_index_size
+get_index_size "main*.sp[dp]" > /var/piler/stat/total_index_size
+get_index_size "${MAIN_INDEX}*.sp[dp]" > /var/piler/stat/current_main_index_size

util/pilerpurge.py

@@ -9,7 +9,7 @@ import sys
 import syslog
 import time
 
-SQL_PURGE_SELECT_QUERY = "SELECT piler_id, size FROM " +\
+SQL_PURGE_SELECT_QUERY = "SELECT id, piler_id, size FROM " +\
     "metadata WHERE deleted=0 AND retained < UNIX_TIMESTAMP(NOW()) " +\
     "AND id NOT IN (SELECT id FROM rcpt WHERE `to` IN " +\
     "(SELECT email FROM legal_hold)) AND id NOT IN (SELECT " +\
@@ -33,7 +33,9 @@ def read_options(filename="", opts={}):
     opts['password'] = config.get('piler', 'mysqlpwd')
     opts['database'] = config.get('piler', 'mysqldb')
     opts['storedir'] = config.get('piler', 'queuedir')
-
+    opts['rtindex'] = config.getint('piler', 'rtindex', fallback=0)
+    opts['sphxhost'] = config.get('piler', 'sphxhost', fallback='127.0.0.1')
+    opts['sphxport'] = config.getint('piler', 'sphxport', fallback=9306)
     opts['server_id'] = "%02x" % config.getint('piler', 'server_id')
 
 
@@ -184,6 +186,17 @@ def get_attachment_file_path(piler_id='', attachment_id=0, opts={}):
                     str(attachment_id)])
 
 
+def purge_index_data(ids=[], opts={}):
+    '''
+    Delete from index data in case of RT index
+    '''
+
+    if opts['rtindex'] == 1 and opts['dry_run'] is False:
+        cursor = opts['sphx'].cursor()
+        a = "," . join([str(x) for x in ids])
+        cursor.execute("DELETE FROM piler WHERE id IN (%s)" % (a))
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("-c", "--config", type=str, help="piler.conf path",
@@ -203,6 +216,7 @@ def main():
     opts['dry_run'] = args.dry_run
     opts['verbose'] = args.verbose
     opts['db'] = None
+    opts['sphx'] = None
     opts['messages'] = 0
     opts['files'] = 0
     opts['size'] = 0
@@ -217,6 +231,8 @@ def main():
         opts['db'] = dbapi.connect(opts['dbhost'], opts['username'],
                                    opts['password'], opts['database'])
 
+        opts['sphx'] = dbapi.connect(host=opts['sphxhost'], port=opts['sphxport'])
+
         if is_purge_enabled(opts) is False:
             syslog.syslog("Purging emails is disabled")
             sys.exit(1)
@@ -229,13 +245,15 @@ def main():
             if rows == ():
                 break
 
-            piler_id = [x[0] for x in rows]
-            size = [x[1] for x in rows]
+            id = [x[0] for x in rows]
+            piler_id = [x[1] for x in rows]
+            size = [x[2] for x in rows]
 
             opts['purged_size'] = opts['purged_size'] + sum(size)
 
             purge_m_files(piler_id, opts)
             purge_attachments_by_piler_id(piler_id, opts)
+            purge_index_data(id, opts)
 
             # It's possible that there's attachment duplication, thus
             # refcount > 0, even though after deleting the duplicates

util/postinstall.sh.in

@@ -406,8 +406,8 @@ SOCKHELPER
       cp webui/.htaccess "$DOCROOT"
    fi
 
-   if [[ -d /var/www/piler.yourdomain.com ]]; then
-      mv /var/www/piler.yourdomain.com "$DOCROOT"
+   if [[ -d /var/www/piler.example.com ]]; then
+      mv /var/www/piler.example.com "$DOCROOT"
    fi
 
 

util/refresh-token.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python3
+
+# Based on the https://github.com/UvA-FNWI/M365-IMAP project
+
+from msal import ConfidentialClientApplication, SerializableTokenCache
+import sys
+
+
+ClientId = "c6843299-05c4-4c2e-9398-64dd42f14b6d"  # Fix this value only
+ClientSecret = ""
+Scopes = ['https://outlook.office.com/IMAP.AccessAsUser.All']
+AccessTokenFileName = "access_token"
+RefreshTokenFileName = "refresh_token"
+
+
+cache = SerializableTokenCache()
+app = ConfidentialClientApplication(ClientId,
+                                    client_credential=ClientSecret,
+                                    token_cache=cache, authority=None)
+
+old_refresh_token = open(RefreshTokenFileName, 'r').read()
+
+token = app.acquire_token_by_refresh_token(old_refresh_token, Scopes)
+
+
+if 'error' in token:
+    print(token)
+    sys.exit("Failed to get access token")
+
+with open(AccessTokenFileName, 'w') as f:
+    f.write(token['access_token'])
+
+with open(RefreshTokenFileName, 'w') as f:
+    f.write(token['refresh_token'])

webui/Zend/Mail/Protocol/Abstract.php

@@ -25,13 +25,13 @@
 /**
  * @see Zend_Validate
  */
-require_once 'Zend/Validate.php';
+//require_once 'Zend/Validate.php';
 
 
 /**
  * @see Zend_Validate_Hostname
  */
-require_once 'Zend/Validate/Hostname.php';
+//require_once 'Zend/Validate/Hostname.php';
 
 
 /**
@@ -134,16 +134,16 @@ abstract class Zend_Mail_Protocol_Abstract
      */
     public function __construct($host = '127.0.0.1', $port = null)
     {
-        $this->_validHost = new Zend_Validate();
-        $this->_validHost->addValidator(new Zend_Validate_Hostname(Zend_Validate_Hostname::ALLOW_ALL));
+        //$this->_validHost = new Zend_Validate();
+        //$this->_validHost->addValidator(new Zend_Validate_Hostname(Zend_Validate_Hostname::ALLOW_ALL));*/
 
-        if (!$this->_validHost->isValid($host)) {
+        //if (!$this->_validHost->isValid($host)) {
             /**
              * @see Zend_Mail_Protocol_Exception
              */
-            require_once 'Zend/Mail/Protocol/Exception.php';
-            throw new Zend_Mail_Protocol_Exception(join(', ', $this->_validHost->getMessages()));
-        }
+            //require_once 'Zend/Mail/Protocol/Exception.php';
+            //throw new Zend_Mail_Protocol_Exception(join(', ', $this->_validHost->getMessages()));
+        //}
 
         $this->_host = $host;
         $this->_port = $port;
@@ -263,8 +263,15 @@ abstract class Zend_Mail_Protocol_Abstract
         $errorNum = 0;
         $errorStr = '';
 
+        $stream_context = stream_context_create(array(
+           'ssl' => array(
+              'verify_peer' => false,
+              'verify_peer_name' => false
+           )
+        ));
+
         // open connection
-        $this->_socket = @stream_socket_client($remote, $errorNum, $errorStr, self::TIMEOUT_CONNECTION);
+        $this->_socket = @stream_socket_client($remote, $errorNum, $errorStr, self::TIMEOUT_CONNECTION, STREAM_CLIENT_CONNECT, $stream_context);
 
         if ($this->_socket === false) {
             if ($errorNum == 0) {

webui/Zend/Mail/Protocol/Smtp.php

@@ -187,13 +187,13 @@ class Zend_Mail_Protocol_Smtp extends Zend_Mail_Protocol_Abstract
         }
 
         // Validate client hostname
-        if (!$this->_validHost->isValid($host)) {
+        //if (!$this->_validHost->isValid($host)) {
             /**
              * @see Zend_Mail_Protocol_Exception
              */
-            require_once 'Zend/Mail/Protocol/Exception.php';
-            throw new Zend_Mail_Protocol_Exception(join(', ', $this->_validHost->getMessages()));
-        }
+            //require_once 'Zend/Mail/Protocol/Exception.php';
+            //throw new Zend_Mail_Protocol_Exception(join(', ', $this->_validHost->getMessages()));
+        //}
 
         // Initiate helo sequence
         $this->_expect(220, 300); // Timeout set for 5 minutes as per RFC 2821 4.5.3.2

webui/controller/message/bulkremove.php

@@ -49,8 +49,8 @@ class ControllerMessageBulkremove extends Controller {
             AUDIT(ACTION_REMOVE_MESSAGE, '', '', $id, '');
             $db->query("UPDATE " . TABLE_META . " SET retained=? WHERE id=?", [NOW, $id]);
             if(RT) {
-               $sphx = Registry::get('sphx');
-               $sphx->query("DELETE FROM " . SPHINX_MAIN_INDEX . " WHERE id=" . (int)$this->data['id']);
+               $sphxrw = Registry::get('sphxrw');
+               $sphxrw->query("DELETE FROM " . SPHINX_MAIN_INDEX . " WHERE id=" . (int)$this->data['id']);
             }
 
             syslog(LOG_INFO, $this->data['username'] . " removed message: $id");

webui/controller/message/remove.php

@@ -41,8 +41,8 @@ class ControllerMessageRemove extends Controller {
       $db->query("UPDATE " . TABLE_META . " SET retained=? WHERE id=?", [NOW, $id]);
       $db->query("UPDATE " . TABLE_DELETED . " SET deleted=1 WHERE id=?", [$id]);
       if(RT) {
-         $sphx = Registry::get('sphx');
-         $sphx->query("DELETE FROM " . SPHINX_MAIN_INDEX . " WHERE id=", (int)$this->data['id']);
+         $sphxrw = Registry::get('sphxrw');
+         $sphxrw->query("DELETE FROM " . SPHINX_MAIN_INDEX . " WHERE id=", (int)$this->data['id']);
       }
 
       syslog(LOG_INFO, $this->data['username'] . " removed message: $id");

webui/index.php

@@ -41,12 +41,14 @@ Registry::set('db', $db);
 
 Registry::set('DB_DRIVER', DB_DRIVER);
 
-$sphx = new DB(SPHINX_DRIVER, SPHINX_HOSTNAME, "", "", SPHINX_DATABASE, "");
+$sphx = new DB(SPHINX_DRIVER, SPHINX_HOSTNAME_READONLY, "", "", SPHINX_DATABASE, "");
+$sphxrw = new DB(SPHINX_DRIVER, SPHINX_HOSTNAME, "", "", SPHINX_DATABASE, "");
 Registry::set('sphx', $sphx);
+Registry::set('sphxrw', $sphxrw);
 
 
 if(MEMCACHED_ENABLED) {
-   $memcache = new Memcache();
+   $memcache = new Memcached();
    foreach ($memcached_servers as $m){
       $memcache->addServer($m[0], $m[1]);
    }
@@ -94,7 +96,7 @@ else if(Registry::get('username')) {
 else {
    if(ENABLE_GOOGLE_LOGIN == 1 && isset($request->get['route']) && $request->get['route'] == 'login/google') {
       $action = new Router('login/google');
-   } 
+   }
    else if(ENABLE_SSO_LOGIN == 1) {
       if(isset($request->get['route']) && $request->get['route'] == 'login/login') {
          $action = new Router('login/login');
@@ -112,6 +114,3 @@ else {
 
 $controller = new Front();
 $controller->dispatch($action, new Router('common/not_found'));
-
-
-?>

webui/model/health/health.php

@@ -31,7 +31,8 @@ class ModelHealthHealth extends Model {
       $this->data['indexer_stat'] = $this->indexer_stat();
       $this->data['purge_stat'] = $this->purge_stat();
 
-      $this->data['sphinx_current_main_size'] = $this->get_current_sphinx_main_index_size();
+      $this->data['sphinx_current_main_size'] = $this->get_index_size(SPHINX_CURRENT_MAIN_INDEX_SIZE);
+      $this->data['sphinx_total_size'] = $this->get_index_size(SPHINX_TOTAL_INDEX_SIZE);
 
       $this->get_average_count_values();
       $this->get_average_size_values($archivesizeraw);
@@ -83,7 +84,7 @@ class ModelHealthHealth extends Model {
          $averagesqlsizeraw = $this->get_database_size() / $this->data['counters']['rcvd'];
 
          //average message sphinx index size, computed for total messages in database
-         $averagesphinxsizeraw = $this->get_sphinx_size() / $this->data['counters']['rcvd'];
+         $averagesphinxsizeraw = $this->data['sphinx_total_size'] / $this->data['counters']['rcvd'];
       }
 
       // average total message size per day, computed over the time period since the first email was archived
@@ -320,31 +321,6 @@ class ModelHealthHealth extends Model {
    }
 
 
-   public function get_sphinx_size($directory = DIR_SPHINX) {
-      $dirSize=0;
-
-      if(!$dh=opendir($directory)) {
-         return false;
-      }
-
-      while($file = readdir($dh)) {
-         if($file == "." || $file == "..") {
-            continue;
-         }
-
-         if(is_file($directory."/".$file)) {
-            $dirSize += filesize($directory."/".$file);
-         }
-         if(is_dir($directory."/".$file)) {
-            $dirSize += $this->get_sphinx_size($directory."/".$file);
-         }
-      }
-
-      closedir($dh);
-      return $dirSize;
-   }
-
-
    public function indexer_stat() {
       $data = array('', '');
 
@@ -377,11 +353,11 @@ class ModelHealthHealth extends Model {
    }
 
 
-   public function get_current_sphinx_main_index_size() {
+   public function get_index_size($statfile = '') {
       $size = 0;
 
-      if(file_exists(SPHINX_MAIN_INDEX_SIZE)) {
-         $size = (int) file_get_contents(SPHINX_MAIN_INDEX_SIZE);
+      if(file_exists($statfile)) {
+         $size = (int) file_get_contents($statfile);
       }
 
       return $size;

webui/model/mail/mail.php

@@ -3,9 +3,15 @@
 class ModelMailMail extends Model {
 
 
-   public function send_smtp_email($smtphost, $smtpport, $yourdomain, $from, $to = array(), $msg){
+   public function send_smtp_email($smtphost, $smtpport, $yourdomain, $from, $to = [], $msg){
+
+      require_once 'Zend/Mail/Protocol/Smtp.php';
+      require_once 'Zend/Mail/Protocol/Smtp/Auth/Login.php';
+
+      // Workaround for the zend framework
+      $msg = str_replace("\r", "", $msg);
+
       $ok = 0;
-      $queue_id = '';
 
       if($to == "" || strlen($msg) < 30){ return $ok; }
 
@@ -13,42 +19,92 @@ class ModelMailMail extends Model {
          $msg = preg_replace("/Message-ID:([^\n]+)\n/i", "Message-ID: <" . generate_random_string(25) . '@' . SITE_NAME . ">\n", $msg);
       }
 
-      $msg = preg_replace("/^\..*$/m", ".$0", $msg);
+      # config array for connection configuration
+      $config = [];
 
-      $r = fsockopen($smtphost, $smtpport);
-      if(!$r){ return -1; }
-
-      $l = fgets($r, 4096);
-
-      fputs($r, "HELO $yourdomain\r\n");
-      $l = fgets($r, 4096);
-
-      fputs($r, "MAIL FROM: <$from>\r\n");
-      $l = fgets($r, 4096);
-
-      foreach($to as $k => $v) {
-         fputs($r, "RCPT TO: <$v>\r\n");
-         $l = fgets($r, 4096);
+      # setting implicit ssl/tls for port 465 and explicit ssl/tls for any other port
+      # in config array zend needs ssl and port 465 for implicit ssl/tls
+      # and tls with any other port for explicit ssl/tls
+      if ($smtpport == '465') {
+         $config['ssl'] = 'ssl';
+      } else {
+         $config['ssl'] = 'tls';
       }
 
-      fputs($r, "DATA\r\n");
-      $l = fgets($r, 4096);
-      if(!preg_match("/^354/", $l)){ $l = fgets($r, 4096); }
+      $config['port'] = $smtpport;
+      $config['name'] = $yourdomain;
 
-      fputs($r, $msg);
+      try {
+         $config['username'] = SMARTHOST_USER;
+         $config['password'] = SMARTHOST_PASSWORD;
 
-      fputs($r, "\r\n.\r\n");
+         if($config['username'] && $config['password']) {
+            $config['auth'] = 'login';
+            $connection = new Zend_Mail_protocol_Smtp_Auth_Login($smtphost, $smtpport, $config);
+         } else {
+            $connection = new Zend_Mail_protocol_Smtp($smtphost, $smtpport, $config);
+         }
 
-      $l = fgets($r, 4096);
+         $connection->connect();
+         $connection->helo($smtphost);
+      } catch (Exception $e) {
+         $connection->__destruct();
 
-      if(preg_match("/^250/", $l)){ $queue_id = trim($l); $ok = 1; }
+         # fallback to unencrypted connection if the port is not 465
+         if ($smtpport != '465') {
+            unset($config['ssl']);
 
-      fputs($r, "QUIT\r\n");
-      $l = fgets($r, 4096);
+            try {
+               if($config['username'] && $config['password']) {
+                  $config['auth'] = 'login';
+                  $connection = new Zend_Mail_protocol_Smtp_Auth_Login($smtphost, $smtpport, $config);
+               } else {
+                  $connection = new Zend_Mail_protocol_Smtp($smtphost, $smtpport, $config);
+               }
 
-      fclose($r);
+               $connection->connect();
+               $connection->helo(SMTP_DOMAIN);
 
-      syslog(LOG_INFO, "sending mail from=$from, rcpt=" . implode(" ", $to) . ", status=$ok, queue_id=$queue_id");
+            } catch (Exception $e) {
+               $connection->__destruct();
+               syslog(LOG_ERR, "sending mail from=$from, rcpt=" . implode(" ", $to) . ", status=$ok (" . $e->getCode() . "), msg=" . rtrim($e->getMessage()) );
+               return -1;
+            }
+         } else {
+            syslog(LOG_ERR, "sending mail from=$from, rcpt=" . implode(" ", $to) . ", status=$ok (" . $e->getCode() . "), msg=" . rtrim($e->getMessage()) );
+            return -1;
+         }
+      }
+
+
+      try {
+         $connection->mail($from);
+
+         foreach ($to as $recipient) {
+            $connection->rcpt($recipient);
+         }
+
+         $connection->data($msg);
+
+         $connectionResponse = $connection->getResponse();
+
+         $connectionResponseArray = explode(" ", $connectionResponse[0]);
+
+         $queue_id = '';
+
+         if($connectionResponseArray[0] == 250) {
+            $ok = 1;
+            $queue_id = trim($connectionResponse[0]);
+         }
+
+         syslog(LOG_INFO, "sending mail from=$from, rcpt=" . implode(" ", $to) . ", status=$ok, queue_id=$queue_id");
+
+      } catch (Exception $e) {
+         syslog(LOG_ERR, "sending mail from=$from, rcpt=" . implode(" ", $to) . ", status=$ok (" . $e->getCode() . "), msg=" . rtrim($e->getMessage()) );
+         $ok = -1;
+      } finally {
+         $connection->__destruct();
+      }
 
       return $ok;
    }
@@ -65,7 +121,7 @@ class ModelMailMail extends Model {
       $s = strstr($hdr, "Subject:");
       if($s) {
          $l1 = strlen($s);
-         $l2 = strlen(strstr($s, "\n"));
+         $l2 = strlen(strstr($s, EOL));
          if($l1 > $l2 + 10) {
             $subject = substr($s, 0, $l1 - $l2) . EOL;
          }
@@ -101,7 +157,7 @@ class ModelMailMail extends Model {
 
       if($this->imap) {
          if($this->imap->login($session->get("username"), $session->get("password"))) { return 1; }
-      }   
+      }
 
       return 0;
    }
@@ -113,5 +169,3 @@ class ModelMailMail extends Model {
 
 
 }
-
-?>

webui/model/search/message.php

@@ -180,17 +180,19 @@ class ModelSearchMessage extends Model {
                              'text/html' => ''
                             );
 
-      for($i=0; $i<count($parts); $i++) {
+      if(is_array($parts)) {
+         for($i=0; $i<count($parts); $i++) {
+            $body = Piler_Mime_Decode::fixMimeBodyPart($parts[$i]['headers'], $parts[$i]['body']);
 
-         $body = Piler_Mime_Decode::fixMimeBodyPart($parts[$i]['headers'], $parts[$i]['body']);
-
-         if($parts[$i]['headers']['content-type']['type'] == 'text/html') {
-            $this->message['text/html'] .= $purifier->purify($body);
+            if($parts[$i]['headers']['content-type']['type'] == 'text/html') {
+               $this->message['text/html'] .= $purifier->purify($body);
+            }
+            else {
+               $this->message['text/plain'] .= $body;
+            }
          }
-         else {
-            $this->message['text/plain'] .= $body;
-         }
-
+      } else {
+         $this->message_array[CONST_TEXTPLAIN] = 'EMPTY_MESSAGE_BODY';
       }
 
       return array('from' => $from,
@@ -198,7 +200,7 @@ class ModelSearchMessage extends Model {
                    'cc' => $cc,
                    'subject' => $this->highlight_search_terms($subject, $terms),
                    'date' => $date,
-                   'message' => $this->message['text/html'] ? $this->message['text/html'] : $this->message['text/plain'],
+                   'message' => $this->message['text/html'] ? $this->highlight_search_terms($this->message['text/html']) : $this->highlight_search_terms($this->message['text/plain']),
                    'has_journal' => $has_journal,
                    'verification' => $this->verification
             );
@@ -220,6 +222,10 @@ class ModelSearchMessage extends Model {
          }
       }
 
+      if($html == 0) {
+         $s = preg_replace("/THE_BREAK_HTML_TAG/", "<br />", $s);
+      }
+
       if(count($terms) <= 0) { return $s; }
 
       if($html == 0) {
@@ -386,8 +392,14 @@ class ModelSearchMessage extends Model {
          }
 
          if($query->row['hash_value'] == $computed_hash) {
-            $validate = TrustedTimestamps::validate($query->row['hash_value'], $query->row['response_string'], $query->row['response_time'], TSA_PUBLIC_KEY_FILE);
-            if($validate == true) { return 1; }
+            try {
+               if(true === TrustedTimestamps::validate($query->row['hash_value'], $query->row['response_string'], $query->row['response_time'], TSA_PUBLIC_KEY_FILE)) {
+                  return 1;
+               }
+            } catch(Exception $e) {
+               syslog(LOG_INFO, "ERROR validating the timestamp: " . $e->getMessage());
+               return 0;
+            }
          }
 
       }
@@ -474,13 +486,13 @@ class ModelSearchMessage extends Model {
 
       if($tag == '') {
          if(RT) {
-            $this->sphx->query("DELETE FROM " . SPHINX_TAG_INDEX . " WHERE uid=$uid AND mid=$id");
+            $this->sphxrw->query("DELETE FROM " . SPHINX_TAG_INDEX . " WHERE uid=$uid AND mid=$id");
          } else {
             $query = $this->db->query("DELETE FROM " . TABLE_TAG . " WHERE uid=? AND id=?", array($uid, $id));
          }
       } else {
          if(RT) {
-            $this->sphx->query("REPLACE INTO " . SPHINX_TAG_INDEX . " (mid, uid, tag) VALUES (?,?,?)",[$id,$uid,$tag]);
+            $this->sphxrw->query("REPLACE INTO " . SPHINX_TAG_INDEX . " (mid, uid, tag) VALUES (?,?,?)",[$id,$uid,$tag]);
          } else {
             $query = $this->db->query("UPDATE " . TABLE_TAG . " SET tag=? WHERE uid=? AND id=?", array($tag, $uid, $id));
             if($this->db->countAffected() == 0) {
@@ -498,7 +510,7 @@ class ModelSearchMessage extends Model {
 
       if(RT) {
          $ids_str = implode(",", $ids);
-         $this->sphx->query("DELETE FROM " . SPHINX_TAG_INDEX . " WHERE uid=$uid AND mid IN ($ids_str)");
+         $this->sphxrw->query("DELETE FROM " . SPHINX_TAG_INDEX . " WHERE uid=$uid AND mid IN ($ids_str)");
       } else {
          $query = $this->db->query("DELETE FROM " . TABLE_TAG . " WHERE uid=? AND id IN ($q)", $arr);
       }
@@ -506,7 +518,7 @@ class ModelSearchMessage extends Model {
       if($tag) {
          foreach ($ids as $id) {
             if(RT) {
-               $this->sphx->query("INSERT INTO " . SPHINX_TAG_INDEX . " (mid, uid, tag) VALUES(?,?,?)", [$id, $uid, $tag]);
+               $this->sphxrw->query("INSERT INTO " . SPHINX_TAG_INDEX . " (mid, uid, tag) VALUES(?,?,?)", [$id, $uid, $tag]);
             } else {
                $query = $this->db->query("INSERT INTO " . TABLE_TAG . " (id, uid, tag) VALUES(?,?,?)", array($id, $uid, $tag));
             }
@@ -549,10 +561,10 @@ class ModelSearchMessage extends Model {
    public function add_message_rt_note($id = '', $uid = 0, $note = '') {
       if($id == '' || $uid <= 0) { return 0; }
 
-      $this->sphx->query("DELETE FROM " . SPHINX_NOTE_INDEX . " WHERE uid=$uid AND mid=$id");
+      $this->sphxrw->query("DELETE FROM " . SPHINX_NOTE_INDEX . " WHERE uid=$uid AND mid=$id");
 
       if($note) {
-         $this->sphx->query("INSERT INTO " . SPHINX_NOTE_INDEX . " (mid, uid, note) VALUES (?,?,?)",[$id,$uid,$note]);
+         $this->sphxrw->query("INSERT INTO " . SPHINX_NOTE_INDEX . " (mid, uid, note) VALUES (?,?,?)",[$id,$uid,$note]);
       }
 
       return 1;

webui/model/search/search.php

@@ -63,12 +63,18 @@ class ModelSearchSearch extends Model {
       if(Registry::get('auditor_user') == 1) {
 
          if(RESTRICTED_AUDITOR == 1) {
-            $session_domains = $this->fix_email_address_for_sphinx($session->get('auditdomains'));
+            $session_domains = $session->get('auditdomains');
 
-            $sd = $this->fix_email_address_for_sphinx($session->get('domain'));
+            $sd = "";
+
+            $domains = $session->get('domain');
+            foreach ($domains as $d) {
+               if($sd) { $sd .= '|'; }
+               $sd .= $this->fix_email_address_for_sphinx($d);
+            }
 
             foreach ($session_domains as $d) {
-               if($d) { $sd .= '|' . $d; }
+               if($d) { $sd .= '|' . $this->fix_email_address_for_sphinx($d); }
             }
 
             $sd = preg_replace("/^\|/", "", $sd);
@@ -554,8 +560,8 @@ class ModelSearchSearch extends Model {
             $m['shortfrom'] = make_short_string($m['from'], MAX_CGI_FROM_SUBJ_LEN);
             $m['from'] = escape_gt_lt_quote_symbols($m['from']);
 
-            isset($rcpt[$m['id']]) ? $m['to'] = $rcpt[$m['id']] : $m['to'] = '';
-            $m['shortto'] = make_short_string($this->get_preferred_recipient($rcpt[$m['id']]), MAX_CGI_FROM_SUBJ_LEN);
+            isset($rcpt[$m['id']]) ? $m['to'] = $rcpt[$m['id']] : $m['to'] = array('');
+            $m['shortto'] = make_short_string($this->get_preferred_recipient($m['to']), MAX_CGI_FROM_SUBJ_LEN);
             $m['to'] = escape_gt_lt_quote_symbols($m['to']);
 
 

webui/securimage/securimage.php

@@ -1438,7 +1438,7 @@ class Securimage
             $length = strlen($code['display']);
 
             for($i = 0; $i < $length; ++$i) {
-                $letter    = $code['display']{$i};
+                $letter    = $code['display'][$i];
                 $letters[] = $letter;
             }
         }

webui/system/database/sphinx.php

@@ -29,6 +29,8 @@ class Sphinx {
       $query->error = 1;
       $query->errmsg = "Error";
       $query->query = $sql;
+      $query->num_rows = 0;
+      $query->rows = [];
       $query->total_found = 0;
 
       $time_start = microtime(true);
@@ -39,8 +41,11 @@ class Sphinx {
       $s = $this->link->prepare($sql);
       if(!$s) { return $query; }
 
-      if(!$s->execute($arr)) {
-         syslog(LOG_INFO, $s->errorInfo()[2]);
+      try {
+         $s->execute($arr);
+      } catch(PDOException $exception) {
+         syslog(LOG_INFO, "ERROR: " . $exception->getMessage());
+         return $query;
       }
 
       $this->affected = $s->rowCount();

webui/system/helper/TrustedTimestamps.php

@@ -71,6 +71,10 @@ class TrustedTimestamps
         if (!file_exists($requestfile_path))
             throw new Exception("The Requestfile was not found");
 
+        $header = array('Content-Type: application/timestamp-query');
+        if(TSA_AUTH_USER)
+            $header[] = "Authorization: Basic " . base64_encode(TSA_AUTH_USER . ':' . TSA_AUTH_PASSWORD);
+
         $ch = curl_init();
         curl_setopt($ch, CURLOPT_URL, $tsa_url);
         curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
@@ -78,10 +82,22 @@ class TrustedTimestamps
         curl_setopt($ch, CURLOPT_POST, 1);
         curl_setopt($ch, CURLOPT_BINARYTRANSFER, 1);
         curl_setopt($ch, CURLOPT_POSTFIELDS, file_get_contents($requestfile_path));
-        curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/timestamp-query'));
+        curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
         curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)");
         curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, TSA_VERIFY_CERTIFICATE);
 
+        if(TSA_AUTH_CERT_FILE && TSA_AUTH_KEY_FILE)
+        {
+            if(!file_exists(TSA_AUTH_CERT_FILE))
+                throw new Exception("Client certificate file " . TSA_AUTH_CERT_FILE . " not found");
+            curl_setopt($ch, CURLOPT_SSLCERT, TSA_AUTH_CERT_FILE);
+            if(!file_exists(TSA_AUTH_KEY_FILE))
+                throw new Exception("Client key file " . TSA_AUTH_KEY_FILE . " not found");
+            curl_setopt($ch, CURLOPT_SSLKEY, TSA_AUTH_KEY_FILE);
+            if(TSA_AUTH_KEY_PASSWORD)
+                curl_setopt($ch, CURLOPT_KEYPASSWD, TSA_AUTH_KEY_PASSWORD);
+        }
+
         $binary_response_string = curl_exec($ch);
 
         $error = curl_error($ch);
@@ -181,7 +197,13 @@ class TrustedTimestamps
         $cmd = OPENSSL_BINARY . " ts -reply -in ".escapeshellarg($responsefile)." -token_out | " . OPENSSL_BINARY . " pkcs7 -inform DER -print_certs -out ".escapeshellarg($untrustedfile);
         shell_exec($cmd);
 
-        $cmd = OPENSSL_BINARY . " ts -verify -digest ".escapeshellarg($hash)." -in ".escapeshellarg($responsefile)." -CAfile ".escapeshellarg($tsa_cert_file)." -untrusted ".escapeshellarg($untrustedfile);
+        if(TSA_RELAXED_CHECK) {
+           $relaxed_check = " -no_check_time ";
+        } else {
+           $relaxed_check = "";
+        }
+
+        $cmd = OPENSSL_BINARY . " ts -verify -digest " . escapeshellarg($hash) . $relaxed_check . " -in ".escapeshellarg($responsefile)." -CAfile ".escapeshellarg($tsa_cert_file)." -untrusted ".escapeshellarg($untrustedfile);
 
         $retarray = array();
         exec($cmd." 2>&1", $retarray, $retcode);

webui/system/helper/mime.php

@@ -5,6 +5,12 @@ class Piler_Mime_Decode {
    const HEADER_FIELDS = ['from', 'to', 'cc', 'subject', 'date'];
 
 
+   public static function normalize_message($message) {
+      $a = preg_split("/\r?\n/", $message);
+      return implode(EOL, $a);
+   }
+
+
    public static function parseMessage($message, &$result) {
 
       self::splitMessage($message, $headers, $body);
@@ -18,10 +24,10 @@ class Piler_Mime_Decode {
             self::parseMessage($body, $result);
          }
          else {
-            $result[] = array(
+            $result[] = [
                'headers' => $headers,
                'body' => $body
-            );
+            ];
          }
 
          return;
@@ -39,7 +45,7 @@ class Piler_Mime_Decode {
          }
          else {
             if(in_array($headers['content-type']['type'], ["text/plain", "text/html"])) {
-               $result[] = array('headers' => $headers, 'body' => $body);
+               $result[] = ['headers' => $headers, 'body' => $body];
             }
             else if($headers['content-type']['type'] == "message/rfc822") {
                self::parseMessage($body, $result);
@@ -51,23 +57,21 @@ class Piler_Mime_Decode {
 
    public static function splitMime($body, $boundary) {
       $start = 0;
-      $res = array();
-
-      $body = self::remove_LF($body);
+      $res = [];
 
       // Extract the mime parts excluding the boundary itself
 
-      $p = strpos($body, '--' . $boundary . "\n", $start);
+      $p = strpos($body, '--' . $boundary . EOL, $start);
       if($p === false) {
          // no parts found!
-         return array();
+         return [];
       }
 
       // Position after first boundary line
 
       $start = $p + 3 + strlen($boundary);
 
-      while(($p = strpos($body, '--' . $boundary . "\n", $start)) !== false) {
+      while(($p = strpos($body, '--' . $boundary . EOL, $start)) !== false) {
          $res[] = substr($body, $start, $p-$start);
          $start = $p + 3 + strlen($boundary);
       }
@@ -76,7 +80,7 @@ class Piler_Mime_Decode {
 
       $p = strpos($body, '--' . $boundary . '--', $start);
       if($p === false) {
-         return array();
+         return [];
       }
 
       // The remaining part also needs to be parsed:
@@ -86,22 +90,22 @@ class Piler_Mime_Decode {
    }
 
 
-   public static function splitMessage($message, &$headers, &$body, $EOL = "\n") {
+   public static function splitMessage($message, &$headers, &$body) {
       self::splitMessageRaw($message, $headers, $journal, $body);
       $headers = self::splitHeaders($headers);
    }
 
 
-   public static function splitMessageRaw($message, &$headers, &$journal, &$body, $EOL = "\n") {
+   public static function splitMessageRaw($message, &$headers, &$journal, &$body) {
       $headers = [];
       $body = '';
 
-      $message = self::remove_LF($message);
+      $message = self::normalize_message($message);
 
       // Find an empty line between headers and body, otherwise we got a header-only message
 
-      if(strpos($message, $EOL . $EOL)) {
-         list($headers, $body) = explode($EOL . $EOL, $message, 2);
+      if(strpos($message, EOL . EOL)) {
+         list($headers, $body) = explode(EOL . EOL, $message, 2);
 
          // Check if the header is actually a journal header
          $headers_array = self::splitHeaders($headers);
@@ -113,7 +117,7 @@ class Piler_Mime_Decode {
             if(count($parts) >= 2) {
                self::splitMessageRaw($parts[0], $s, $j, $journal);
 
-               $i = strpos($parts[1], $EOL . $EOL);
+               $i = strpos($parts[1], EOL . EOL);
                $msg = substr($parts[1], $i);
 
                $i = 0;
@@ -141,26 +145,16 @@ class Piler_Mime_Decode {
    }
 
 
-   public static function removeJournal(&$message, $EOL = "\n") {
+   public static function removeJournal(&$message) {
       $has_journal = 0;
 
-      $crlfs = substr_count($message, "\r\n");
-
       self::splitMessageRaw($message, $headers, $journal, $body);
 
       if($journal) {
          $has_journal = 1;
       }
 
-      // If the message has >10 CRLF sequences, then we assume
-      // that we need to restore the removed LF characters
-      if($crlfs > 10) {
-         $headers = str_replace("\n", "\r\n", $headers);
-         $body = str_replace("\n", "\r\n", $body);
-         $EOL = "\r\n";
-      }
-
-      $message = $headers . $EOL . $EOL . $body;
+      $message = $headers . EOL . EOL . $body;
 
       return $has_journal;
    }
@@ -188,7 +182,7 @@ class Piler_Mime_Decode {
             continue;
          }
 
-         $headers[$lower] = array($headers[$lower], $header);
+         $headers[$lower] = [$headers[$lower], $header];
       }
 
       // Add some default values, if they are missing
@@ -204,9 +198,21 @@ class Piler_Mime_Decode {
       for($i=0; $i<count(self::HEADER_FIELDS); $i++) {
          if(!isset($headers[self::HEADER_FIELDS[$i]])) { $headers[self::HEADER_FIELDS[$i]] = ''; }
 
-         $headers[self::HEADER_FIELDS[$i]] = preg_replace("/gb2312/i", "GBK", $headers[self::HEADER_FIELDS[$i]]);
+         // If the mail header features the same field more than once, eg.
+         // Date: Wed, 23 Mar 2016 21:26:53 +0100
+         // Date: Wed, 23 Mar 2016 21:26:53 +0100
+         // then take the first occurance
 
-         $headers[self::HEADER_FIELDS[$i]] = iconv_mime_decode($headers[self::HEADER_FIELDS[$i]], ICONV_MIME_DECODE_CONTINUE_ON_ERROR);
+         $header = $headers[self::HEADER_FIELDS[$i]];
+         if(is_array($header)) {
+            $header = $header[0];
+         }
+
+         if(ENABLE_GB2312_FIX) {
+            $header = preg_replace("/gb2312/i", "GBK", $header);
+         }
+
+         $headers[self::HEADER_FIELDS[$i]] = iconv_mime_decode($header, ICONV_MIME_DECODE_CONTINUE_ON_ERROR);
       }
 
       $headers['content-type'] = self::splitContentType($headers['content-type']);
@@ -220,9 +226,9 @@ class Piler_Mime_Decode {
    public static function headersToArray($headers = '') {
       $token = '';
       $last_token = '';
-      $result = array();
+      $result = [];
 
-      $headers = explode("\n", $headers);
+      $headers = explode(EOL, $headers);
 
       foreach($headers as $h) {
 
@@ -236,7 +242,7 @@ class Piler_Mime_Decode {
 
          // Skip line if it doesn't have a colon (:) and the 1st character is not a whitespace
 
-         if(!ctype_space($h[0]) && !strchr($h, ':')) { continue; }
+         if($h && !ctype_space($h[0]) && !strchr($h, ':')) { continue; }
 
          if($line) {
             if(substr($line[0], -1) == ':') {
@@ -256,7 +262,7 @@ class Piler_Mime_Decode {
             }
             else {
                if($token) {
-                  $result[$last_token] .= "\n";
+                  $result[$last_token] .= EOL;
                }
 
                $result[$last_token] .= ' ' . $line_str;
@@ -266,9 +272,8 @@ class Piler_Mime_Decode {
       }
 
       foreach($result as $k => $v) {
-
-         if(strchr($v, "\n")) {
-            $result[$k] = explode("\n", $v);
+         if(strchr($v, EOL)) {
+            $result[$k] = explode(EOL, $v);
          }
       }
 
@@ -277,7 +282,7 @@ class Piler_Mime_Decode {
 
 
    public static function splitContentType($field = '') {
-      $split = array();
+      $split = [];
       $what  = 'type';
 
       $field = $what . '=' . $field;
@@ -285,7 +290,7 @@ class Piler_Mime_Decode {
          return $split;
       }
 
-      $split = array();
+      $split = [];
       foreach ($matches[1] as $key => $name) {
          $name = strtolower($name);
          if($matches[2][$key][0] == '"') {
@@ -299,13 +304,7 @@ class Piler_Mime_Decode {
    }
 
 
-   public static function remove_LF($message = '') {
-      return str_replace("\r", "", $message);
-      //return preg_replace("/\r/", "", $message);
-   }
-
-
-   public static function getBoundary($headers = array()) {
+   public static function getBoundary($headers = []) {
       if(isset($headers['content-type']['boundary'])) {
          return $headers['content-type']['boundary'];
       }
@@ -314,7 +313,7 @@ class Piler_Mime_Decode {
    }
 
 
-   public static function fixMimeBodyPart($headers = array(), $body = '') {
+   public static function fixMimeBodyPart($headers = [], $body = '') {
 
       if(isset($headers['content-transfer-encoding'])) {
          if(strtolower($headers['content-transfer-encoding']) == 'quoted-printable') {
@@ -327,16 +326,17 @@ class Piler_Mime_Decode {
       }
 
       if(isset($headers['content-type']['charset'])) {
-         if(strtolower($headers['content-type']['charset']) == 'gb2312') {
+         if(ENABLE_GB2312_FIX && strtolower($headers['content-type']['charset']) == 'gb2312') {
             $headers['content-type']['charset'] = 'GBK';
          }
+
          $body = iconv($headers['content-type']['charset'], 'utf-8' . '//IGNORE', $body);
       }
 
       if(strtolower($headers['content-type']['type']) == 'text/plain') {
          $body = self::escape_lt_gt_symbols($body);
-         $body = preg_replace("/\n/", "<br />\n", $body);
-         $body = "\n" . self::printNicely($body);
+         $body = preg_replace("/\n/", "THE_BREAK_HTML_TAG\n", $body);
+         $body = EOL . self::printNicely($body);
       }
 
       return $body;
@@ -361,9 +361,9 @@ class Piler_Mime_Decode {
          $nice .= $x[$i] . " ";
          $k += strlen($x[$i]);
 
-         if(strstr($x[$i], "\n")){ $k = 0; }
+         if(strstr($x[$i], EOL)){ $k = 0; }
 
-         if($k > 70){ $nice .= "\n"; $k = 0; }
+         if($k > 70){ $nice .= EOL; $k = 0; }
       }
 
       return $nice;

webui/system/misc.php

@@ -145,13 +145,12 @@ function checkemail($email, $domains) {
 
 
 function validemail($email = '') {
-   if($email == '') { return 0; }
+   if($email == '' || !strchr($email, '@')) { return 0; }
 
-   if(preg_match("/@local$/", $email)) { return 1; }
+   $arr = explode("@", $email);
 
-   if(preg_match('/^[_a-zA-Z0-9-]+(\.[_a-zA-Z0-9-]+)*@[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)*(\.[a-zA-Z]{2,10})$/', $email)) {
-      return 1;
-   }
+   // This is a pretty relaxed formula making sure we have something as the local part
+   if(count($arr) == 2 && strlen($arr[0]) >= 1 && validdomain($arr[1])) { return 1; }
 
    return 0;
 }
@@ -173,7 +172,7 @@ function checkdomain($domain, $domains) {
 
 
 function validdomain($domain = '') {
-   if(preg_match("/@?local$/", $domain) || preg_match('/^[a-zA-Z0-9]+[a-zA-Z0-9-_\.]{0,}\.[a-zA-Z0-9]{2,10}$/', $domain)) {
+   if(preg_match("/@?local$/", $domain) || preg_match(DOMAIN_REGEX, $domain)) {
       return 1;
    }
 
@@ -292,6 +291,11 @@ function my_qp_encode($s){
 }
 
 
+function format_number($n) {
+   return number_format($n, 0, DECIMAL_SEPARATOR, THOUSANDS_SEPARATOR);
+}
+
+
 function nice_size($size = 0, $space = '') {
    if($size < 1000) return "1k";
    if($size < 1000000) return round($size/1000) . $space . "k";

webui/tests/MiscTest.php

@@ -0,0 +1,45 @@
+<?php
+
+use PHPUnit\Framework\TestCase;
+
+require_once dirname(dirname(__FILE__)) . '/config.php';
+require_once dirname(dirname(__FILE__)) . '/system/model.php';
+require_once dirname(dirname(__FILE__)) . '/system/loader.php';
+require_once dirname(dirname(__FILE__)) . '/system/language.php';
+require_once dirname(dirname(__FILE__)) . '/system/misc.php';
+
+final class MiscTest extends TestCase
+{
+
+   public function providerTestValiddomain() {
+      return [
+         ['', 0],
+         ['local', 1],
+         ['@local', 1],
+         ['aaa.fu', 1],
+         ['@aaa.fu', 0],
+         ['AAa.fu', 1],
+         ['.aaa.fu', 0],
+         ['-aaa.fu', 0],
+         ['_aaa.fu', 0],
+         ['2aaa.fu', 1],
+         ['aaafu', 0],
+         ['a.com.', 0],
+         ['a.co.uk', 1],
+         ['a.co.u2k', 1],
+         ['a.com', 1],
+         ['ccc.com', 1],
+         ['aaa.bbb.ccc.com', 1],
+         ['aaa.photography', 1],
+      ];
+   }
+
+   /**
+    * @dataProvider providerTestValiddomain
+    */
+
+   public function test_validdomain($data, $expected_result) {
+      $result = validdomain($data);
+      $this->assertEquals($result, $expected_result);
+   }
+}

webui/tests/ParseMessageTest.php

@@ -11,17 +11,17 @@ final class MailParserTest extends TestCase {
 
    public function providerTestParseMessage() {
       return [
-         ["1.eml", 1, ["Liebe Gueste,\n\ndie Einarbeitung der Rechen- und Summenfunktionen ins RK-Formular"]],
-         ["2.eml", 1, ["Hallo!\nDie seltsamen Zeilenumbr=C3=BCche treten tats=C3=A4chlich auf."]],
-         ["3.eml", 1, ["\n\nCan we discuss? Send Reply For more information, THANKS."]],
+         ["1.eml", 1, ["Liebe Gueste,\r\n\r\ndie Einarbeitung der Rechen- und Summenfunktionen ins RK-Formular"]],
+         ["2.eml", 1, ["Hallo!\r\nDie seltsamen Zeilenumbr=C3=BCche treten tats=C3=A4chlich auf."]],
+         ["3.eml", 1, ["\r\n\r\nCan we discuss? Send Reply For more information, THANKS."]],
          ["4.eml", 2, ["=0D=0A=0D=0A=0D=0A=0D=0A", "<HTML><HEAD>=0D=0A<META http-equiv=3D\"Content-Type\" content=3D\"te="]],
-         ["5.eml", 2, ["\nHi ,\n\nIf so, stop by and test out our FREE phishing simulator! Find out how", "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1=2E0 Transitional//EN\" \"http://ww=\nw=2Ew3=2Eorg/TR/xhtml1/DTD/xhtml1-transitional=2Edtd\"><html xmlns=3D\"http:/="]],
+         ["5.eml", 2, ["\r\nHi ,\r\n\r\nIf so, stop by and test out our FREE phishing simulator! Find out how", "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1=2E0 Transitional//EN\" \"http://ww=\r\nw=2Ew3=2Eorg/TR/xhtml1/DTD/xhtml1-transitional=2Edtd\"><html xmlns=3D\"http:/="]],
          ["6.eml", 2, ["RGVhciBTaXJzLA0KDQpHbGFkIHRvIGhlYXIgdGhhdCB5b3UncmUgb24gdGhlIGZpbHRyYXRpb24g", "<html><head><meta http-equiv=3D\"content-type\" content=3D\"text/html; charse="]],
-         ["7.eml", 2, ["Mai ajánlat: \n    \n      Exkluzív!", "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">"]],
-         ["8.eml", 2, ["Hello,\n\nYou have received a newsletter from Chemol Travel.", "<html xmlns=3D\"http://www.w3.org/1999/xhtml\" xmlns:v=3D\"urn:schemas-micro=\nsoft-com:vml\" xmlns:o=3D\"urn:schemas-microsoft-com:office:office\">"]],
+         ["7.eml", 2, ["Mai ajánlat: \r\n    \r\n      Exkluzív!", "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">"]],
+         ["8.eml", 2, ["Hello,\r\n\r\nYou have received a newsletter from Chemol Travel.", "<html xmlns=3D\"http://www.w3.org/1999/xhtml\" xmlns:v=3D\"urn:schemas-micro=\r\nsoft-com:vml\" xmlns:o=3D\"urn:schemas-microsoft-com:office:office\">"]],
       ];
 
-   }   
+   }
 
 
    /**

webui/tests/SplitMessageTest.php

@@ -1,70 +1,65 @@
 <?php
 
-define('CONTENT_TYPE', 'content-type');
-define('TEXT_PLAIN', 'text/plain');
-define('SUBJECT', 'subject');
-define('THIS_IS_A_TEST', 'This is a test');
-
 use PHPUnit\Framework\TestCase;
 
-require_once DIR_BASE . 'system/helper/mime.php';
+require_once dirname(dirname(__FILE__)) . '/system/helper/mime.php';
 
 
 final class SplitMessageTest extends TestCase {
 
    public function providerTestSplitMessage() {
       return [
-         ["From: aaa\r\nTo:bbb\r\nSubject: test\r\n\r\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => TEXT_PLAIN)),
-          THIS_IS_A_TEST],
+         ["From: aaa\r\nTo:bbb\r\nSubject: test\r\n\r\nThis is a test",
+          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', 'subject' => 'test', 'content-type' => array('type' => 'text/plain')),
+          "This is a test"],
 
-         ["From: aaa\r\nTo:bbb\r\nCC ccc\r\nSubject: test\r\n\r\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => TEXT_PLAIN)),
-          THIS_IS_A_TEST],
+         ["From: aaa\r\nSender: alala@aaa\r\nTo:bbb\r\nCC ccc\r\nSubject: test\r\n\r\nThis is a test",
+          array('sender' => 'alala@aaa', 'from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', 'subject' => 'test', 'content-type' => array('type' => 'text/plain')),
+          "This is a test"],
 
-         ["From: aaa\nTo:bbb\nSubject: test\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => TEXT_PLAIN)),
-          THIS_IS_A_TEST],
+         ["From: aaa\nTo:bbb\nSubject: test\n\nThis is a test",
+          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', 'subject' => 'test', 'content-type' => array('type' => 'text/plain')),
+          "This is a test"],
 
-         ["From: aaa\r\nTo:bbb\r\nSubject: test\r\n\r\n\r\n\r\n" . THIS_IS_A_TEST . "\nAaa\n",
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => TEXT_PLAIN)),
-          "\n\n" . THIS_IS_A_TEST . "\nAaa\n"],
+         ["From: aaa\r\nTo:bbb\r\nSubject: test\r\n\r\n\r\n\r\nThis is a test\nAaa\n",
+          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', 'subject' => 'test', 'content-type' => array('type' => 'text/plain')),
+          "\r\n\r\nThis is a test\r\nAaa\r\n"],
 
-         ["From: aaa\r\nTo:bbb\r\nSubject: test\r\nContent-type: text/html\r\n\r\n\r\n" . THIS_IS_A_TEST . "\nAaa\n",
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => 'text/html')),
-          "\n" . THIS_IS_A_TEST . "\nAaa\n"],
+         ["From: aaa\r\nTo:bbb\r\nSubject: test\r\nContent-type: text/html\r\n\r\n\r\nThis is a test\nAaa\n",
+          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', 'subject' => 'test', 'content-type' => array('type' => 'text/html')),
+          "\r\nThis is a test\r\nAaa\r\n"],
 
-         ["From: aaa\nTo:bbb\nSubject: test\nContent-Type: text/plain\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => TEXT_PLAIN)),
-          THIS_IS_A_TEST],
+         ["From: aaa\nTo:bbb\nSubject: test\nContent-Type: text/plain\n\nThis is a test",
+          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', 'subject' => 'test', 'content-type' => array('type' => 'text/plain')),
+          "This is a test"],
 
-         ["From: aaa\nTo:bbb\nSubject: test\nDate: Sun, 17 Apr 2016 22:40:03 +0800\nDKIM-Signature: v=1; a=rsa-sha1; c=relaxed/relaxed; d=chemoltravel.hu; s=ml;\n\tt=1471888357; bh=A/l/HLQe3HM4Xc4jFxAmhaWVCMU=;\n\th=Date:To:From:Subject:Sender:From:To:Subject:Date;\n\tb=JlEqXiAKBOoT/YyXKTMsXnEphh2J6sXxgNmbKbGybjo3cU1rgQEL0m1h26gl5AaBP\nContent-Type: " . TEXT_PLAIN . "\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', SUBJECT => 'test', 'date' => 'Sun, 17 Apr 2016 22:40:03 +0800', 'dkim-signature' => 'v=1; a=rsa-sha1; c=relaxed/relaxed; d=chemoltravel.hu; s=ml;  t=1471888357; bh=A/l/HLQe3HM4Xc4jFxAmhaWVCMU=;  h=Date:To:From:Subject:Sender:From:To:Subject:Date;  b=JlEqXiAKBOoT/YyXKTMsXnEphh2J6sXxgNmbKbGybjo3cU1rgQEL0m1h26gl5AaBP', CONTENT_TYPE => array('type' => TEXT_PLAIN)),
-          THIS_IS_A_TEST],
+         ["From: aaa\nTo:bbb\nSubject: test\nDate: Sun, 17 Apr 2016 22:40:03 +0800\nDKIM-Signature: v=1; a=rsa-sha1; c=relaxed/relaxed; d=chemoltravel.hu; s=ml;\n\tt=1471888357; bh=A/l/HLQe3HM4Xc4jFxAmhaWVCMU=;\n\th=Date:To:From:Subject:Sender:From:To:Subject:Date;\n\tb=JlEqXiAKBOoT/YyXKTMsXnEphh2J6sXxgNmbKbGybjo3cU1rgQEL0m1h26gl5AaBP\nContent-Type: text/plain\n\nThis is a test",
+          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'subject' => 'test', 'date' => 'Sun, 17 Apr 2016 22:40:03 +0800', 'dkim-signature' => 'v=1; a=rsa-sha1; c=relaxed/relaxed; d=chemoltravel.hu; s=ml;  t=1471888357; bh=A/l/HLQe3HM4Xc4jFxAmhaWVCMU=;  h=Date:To:From:Subject:Sender:From:To:Subject:Date;  b=JlEqXiAKBOoT/YyXKTMsXnEphh2J6sXxgNmbKbGybjo3cU1rgQEL0m1h26gl5AaBP', 'content-type' => array('type' => 'text/plain')),
+          "This is a test"],
 
-         ["From: aaa\nTo:bbb\nSubject: test\nContent-Type: text/PLAIN\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => TEXT_PLAIN)),
-          THIS_IS_A_TEST],
+         ["From: aaa\nTo:bbb\nSubject: test\nContent-Type: text/PLAIN\n\nThis is a test",
+          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', 'subject' => 'test', 'content-type' => array('type' => 'text/plain')),
+          "This is a test"],
 
-         ["From: aaa\nTo:bbb\nSubject: test\nContent-Type: text/plain; charset=\"ISO-8859-1\"\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => TEXT_PLAIN, 'charset' => 'ISO-8859-1')),
-          THIS_IS_A_TEST],
+         ["From: aaa\nTo:bbb\nSubject: test\nContent-Type: text/plain; charset=\"ISO-8859-1\"\n\nThis is a test",
+          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', 'subject' => 'test', 'content-type' => array('type' => 'text/plain', 'charset' => 'ISO-8859-1')),
+          "This is a test"],
 
-         ["From: aaa\nTo:bbb\nSubject: test\nMIME-Version: 1.0\nContent-Type: multipart/alternative; boundary=\"_=_SWIFT_v4_1460476188_145aa333fc0127705a7e904aab6d1957_=_\"\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', 'mime-version' => '1.0', CONTENT_TYPE => array('type' => 'multipart/alternative', 'boundary' => '_=_SWIFT_v4_1460476188_145aa333fc0127705a7e904aab6d1957_=_')),
-          THIS_IS_A_TEST],
+         ["From: aaa\nTo:bbb\nSubject: test\nMIME-Version: 1.0\nContent-Type: multipart/alternative; boundary=\"_=_SWIFT_v4_1460476188_145aa333fc0127705a7e904aab6d1957_=_\"\n\nThis is a test",
+          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', 'subject' => 'test', 'mime-version' => '1.0', 'content-type' => array('type' => 'multipart/alternative', 'boundary' => '_=_SWIFT_v4_1460476188_145aa333fc0127705a7e904aab6d1957_=_')),
+          "This is a test"],
 
-         ["From: aaa\nTo:bbb\nSubject: test\nMIME-Version: 1.0\nContent-Type: multipart/alternative;\n boundary=\"_=_SWIFT_v4_1460476188_145aa333fc0127705a7e904aab6d1957_=_\"\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', 'mime-version' => '1.0', CONTENT_TYPE => array('type' => 'multipart/alternative', 'boundary' => '_=_SWIFT_v4_1460476188_145aa333fc0127705a7e904aab6d1957_=_')),
-          THIS_IS_A_TEST],
+         ["From: aaa\nTo:bbb\nSubject: test\nMIME-Version: 1.0\nContent-Type: multipart/alternative;\n boundary=\"_=_SWIFT_v4_1460476188_145aa333fc0127705a7e904aab6d1957_=_\"\n\nThis is a test",
+          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', 'subject' => 'test', 'mime-version' => '1.0', 'content-type' => array('type' => 'multipart/alternative', 'boundary' => '_=_SWIFT_v4_1460476188_145aa333fc0127705a7e904aab6d1957_=_')),
+          "This is a test"],
 
-         ["From: aaa\nTo:bbb\nSubject: test\nMIME-Version: 1.0\nContent-Type: multipart/related;\n\ttype=\"multipart/alternative\";\n\tboundary=\"----=_NextPart_000_0006_01D195BC.69E26510\"\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', 'mime-version' => '1.0', CONTENT_TYPE => array('type' => 'multipart/alternative', 'boundary' => '----=_NextPart_000_0006_01D195BC.69E26510')),
-          THIS_IS_A_TEST],
+         ["From: aaa\nTo:bbb\nSubject: test\nMIME-Version: 1.0\nContent-Type: multipart/related;\n\ttype=\"multipart/alternative\";\n\tboundary=\"----=_NextPart_000_0006_01D195BC.69E26510\"\n\nThis is a test",
+          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', 'subject' => 'test', 'mime-version' => '1.0', 'content-type' => array('type' => 'multipart/alternative', 'boundary' => '----=_NextPart_000_0006_01D195BC.69E26510')),
+          "This is a test"],
 
 
-      ];  
-   }   
+      ];
+   }
 
 
    /**

webui/view/theme/default/templates/health/worker.tpl

@@ -1,9 +1,9 @@
 <div class="container">
-	<div class="alert alert-info"><strong><?php print $text_refresh_period; ?>:</strong> <?php print HEALTH_REFRESH; ?> sec</div>
+        <div class="alert alert-info"><strong><?php print $text_refresh_period; ?>:</strong> <?php print HEALTH_REFRESH; ?> sec</div>
 
-	<div class="row">
-		
-		<div class="span8">
+        <div class="row">
+
+            <div class="span8">
             <h2><i class="icon-dashboard icon-2x pull-left"></i><?php print $health['sysinfo'][0]; ?> <small><?php print $text_status; ?></small></h2>
             <h3><strong>Up For:</strong> <?php print $health['uptime']; ?></h3>
             <table class="table table-striped">
@@ -15,22 +15,22 @@
                 <tr>
                     <th class="span4"><?php print $text_server_operating_system; ?>:</th>
                     <td class="span8"> <?php print $health['sysinfo'][1]; ?></td>
-             	</tr>        
-            
-            	<tr>
-					<th><?php print $text_cpu_usage; ?>:</th>
-					<td><div class="progress <?php if($health['cpuinfo'] < HEALTH_RATIO) { ?>progress-success<?php } else { ?>progress-danger<?php } ?>"><div class="bar" style="width: <?php print $health['cpuinfo']; ?>%"></div>&nbsp;<?php print $health['cpuinfo']; ?>% <?php print $health['cpuload']; ?></div></td>
                 </tr>
 
                 <tr>
-					<th><?php print $text_memory_usage; ?>:</th>
-                    <td><div class="progress <?php if($health['meminfo'] < HEALTH_RATIO) { ?>progress-success<?php } else { ?>progress-danger<?php } ?>"><div class="bar" style="width: <?php print $health['meminfo']; ?>%"></div>&nbsp;<?php print $health['meminfo']; ?>% / <?php print $health['totalmem']; ?> MB</div></td>
-				</tr>
-                
+                    <th><?php print $text_cpu_usage; ?>:</th>
+                    <td><div class="progress <?php if($health['cpuinfo'] < HEALTH_RATIO) { ?>progress-success<?php } else { ?>progress-danger<?php } ?>"><div class="bar" style="width: <?php print $health['cpuinfo']; ?>%"></div>&nbsp;<?php print $health['cpuinfo']; ?>% <?php print $health['cpuload']; ?></div></td>
+                </tr>
+
                 <tr>
-					<th><?php print $text_swap_usage; ?>:</th>
+                    <th><?php print $text_memory_usage; ?>:</th>
+                    <td><div class="progress <?php if($health['meminfo'] < HEALTH_RATIO) { ?>progress-success<?php } else { ?>progress-danger<?php } ?>"><div class="bar" style="width: <?php print $health['meminfo']; ?>%"></div>&nbsp;<?php print $health['meminfo']; ?>% / <?php print $health['totalmem']; ?> MB</div></td>
+                </tr>
+
+                <tr>
+                    <th><?php print $text_swap_usage; ?>:</th>
                     <td><div class="progress <?php if($health['swapinfo'] < HEALTH_RATIO) { ?>progress-success<?php } else { ?>progress-danger<?php } ?>"><div class="bar" style="width: <?php print $health['swapinfo']; ?>%"></div>&nbsp;<?php print $health['swapinfo']; ?>% / <?php print $health['totalswap']; ?> MB</div></td>
-				</tr>
+                </tr>
 
 <?php if(ENABLE_SAAS == 1) { ?>
                 <tr>
@@ -38,18 +38,18 @@
                     <td class="span8"><a href="index.php?route=stat/online"><?php print $health['num_of_online_users']; ?></a></td>
                 </tr>
 <?php } ?>
-                
-            </table>
-            
-            <h2><i class="icon-hdd icon-2x pull-left"></i>&nbsp;<?php print $text_storage; ?></h2>
-			<table class="table table-striped">
-				 <tr>
-					<th><?php print $text_archive_size; ?>:</th>
-					<td><?php print $health['archive_size']; ?>B (<?php print $health['archive_stored_size']; ?>B)</td>
-				 </tr>
 
-				 <tr>
-					<th><?php print $text_disk_usage; ?>:</th>
+            </table>
+
+            <h2><i class="icon-hdd icon-2x pull-left"></i>&nbsp;<?php print $text_storage; ?></h2>
+                        <table class="table table-striped">
+                                 <tr>
+                                        <th><?php print $text_archive_size; ?>:</th>
+                                        <td><?php print $health['archive_size']; ?>B (<?php print $health['archive_stored_size']; ?>B)</td>
+                                 </tr>
+
+                                 <tr>
+                                        <th><?php print $text_disk_usage; ?>:</th>
                                         <td>
                                             <table>
                                                 <?php foreach($health['shortdiskinfo'] as $partition) { ?>
@@ -64,32 +64,31 @@
 
                                             </table>
                                         </td>
-				 </tr>
-		
-				 <tr>
-					<th><?php print $text_smtp_status; ?>:</th>
-					<td>
-					   <?php foreach($health['health'] as $h) {
-								if(preg_match("/^220/", $h[1])) {
-								   $status = 'OK'; $class = 'text-success';
-								} else {
-								   $status = 'ERROR'; $class = 'text-error';
-								}
-					   ?>
-								<div class="<?php print $class; ?>"><span onmouseover="Tip('<?php print preg_replace("/\'/", "\'", $h[1]); ?>, <?php print $h[2]; ?>', BALLOON, true, ABOVE, true)" onmouseout="UnTip()"><?php print $h[3]; ?>: <?php print $status; ?></span></div>
-					   <?php } ?>
-					</td>
-				 </tr>
+                                 </tr>
 
-				 <tr>
-					<th><?php print $text_periodic_purge; ?></th>
-					<td>
+                                 <tr>
+<th><?php print $text_smtp_status; ?>:</th>
+<td>
+                                           <?php foreach($health['health'] as $h) {
+                                                    if(preg_match("/^220/", $h[1])) {
+                                                       $status = 'OK'; $class = 'text-success';
+                                                    } else {
+                                                       $status = 'ERROR'; $class = 'text-error';
+                                                    } ?>
+                                             <div class="<?php print $class; ?>"><span onmouseover="Tip('<?php print preg_replace("/\'/", "\'", $h[1]); ?>, <?php print $h[2]; ?>', BALLOON, true, ABOVE, true)" onmouseout="UnTip()"><?php print $h[3]; ?>: <?php print $status; ?></span></div>
+                                           <?php } ?>
+                                        </td>
+                                 </tr>
+
+                                 <tr>
+                                        <th><?php print $text_periodic_purge; ?></th>
+                                        <td>
                                            <span class="<?php if($health['options']['enable_purge'] == 1) { ?>ok<?php } else { ?>error<?php } ?>"><?php if($health['options']['enable_purge'] == 1) { print $text_enabled; ?>. <a href="<?php print HEALTH_URL; ?>&toggle_enable_purge"><?php print $text_disable; ?></a>
                                               <?php if($health['purge_stat'][0]) { print $text_last; ?>: <?php print $health['purge_stat'][0]; ?>, <?php print $text_next; ?>:  <?php print $health['purge_stat'][1]; } ?>
                                            <?php } else { print $text_disabled; ?>. <a href="<?php print HEALTH_URL; ?>&toggle_enable_purge"><?php print $text_enable; ?></a><?php } ?>
                                            </span>
                                         </td>
-				 </tr>
+                                 </tr>
 
                          <?php if($health['indexer_stat'][0]) { ?>
                                  <tr>
@@ -99,71 +98,70 @@
                                  </tr>
                          <?php } ?>
 
-			 </table>
-		</div>
-				 
-		 <div class="span4">		
-			<table class="table table-striped">
-				<tr>
-					<th colspan="2"><?php print $text_cumulative_counts; ?></th>
-				</tr>
+                         </table>
+                </div>
+
+                <div class="span4">
+                    <table class="table table-striped">
+                                <tr>
+                                        <th colspan="2"><?php print $text_cumulative_counts; ?></th>
+                                </tr>
                                 <tr>
                                         <td><?php print $text_oldest_record; ?></td>
                                         <td><?php print date(DATE_TEMPLATE, $health['oldestmessagets']); ?></td>
-                                </tr>                
-				<tr>
-					<td><?php print $text_processed_emails; ?></td>
+                                </tr>
+                                <tr>
+                                        <td><?php print $text_processed_emails; ?></td>
                                         <td>
-                                           <?php print $health['processed_emails']['last_60_mins_count']; ?> (<?php print $text_60_minutes; ?>)<br />
-                                           <?php print $health['processed_emails']['today_count']; ?> (<?php print $text_24_hours; ?>)<br />
-                                           <?php print $health['processed_emails']['last_7_days_count']; ?> (<?php print $text_1_week; ?>)<br />
-                                           <?php print $health['processed_emails']['last_30_days_count']; ?> (<?php print $text_30_days; ?>)
+                                           <?php print format_number($health['processed_emails']['last_60_mins_count']); ?> (<?php print $text_60_minutes; ?>)<br />
+                                           <?php print format_number($health['processed_emails']['today_count']); ?> (<?php print $text_24_hours; ?>)<br />
+                                           <?php print format_number($health['processed_emails']['last_7_days_count']); ?> (<?php print $text_1_week; ?>)<br />
+                                           <?php print format_number($health['processed_emails']['last_30_days_count']); ?> (<?php print $text_30_days; ?>)
                                         </td>
-				</tr>
-				<tr>
-					<th colspan="2"><?php print $text_message_disposition; ?></th>
-				</tr>
-			   <?php foreach($health['counters'] as $k => $v) {
-						if(!is_numeric($k)) { ?>
-						   <tr>
-							  <td><?php $a = preg_replace("/^_piler\:/", "", $k); if(isset($$a)) { print $$a; } else { print $k; } ?></td>
-							  <td><?php print $v; ?></td>
-						   </tr>
-						<?php } } ?>
-		  
-		  </table>
-          
+                                </tr>
+                                <tr>
+                                        <th colspan="2"><?php print $text_message_disposition; ?></th>
+                                </tr>
+                           <?php foreach($health['counters'] as $k => $v) { if(!is_numeric($k)) { ?>
+                                <tr>
+                                        <td><?php $a = preg_replace("/^_piler\:/", "", $k); if(isset($$a)) { print $$a; } else { print $k; } ?></td>
+                                        <td><?php print format_number($v); ?></td>
+                                </tr>
+                           <?php } } ?>
+
+                    </table>
+
           <h4><?php print $text_space_projection; ?></h4>
           <table class="table table-striped">
-			   <tr>
-				  <td><?php print $text_average_messages_day; ?></td>
-				  <td><?php print $health['averagemessages']; ?></td>
-			   </tr>
-			   <tr>
-				  <td><?php print $text_average_message_size; ?></td>
-				  <td><?php print $health['averagemessagesize']; ?> + <?php print $health['averagesqlsize']; ?> + <?php print $health['averagesphinxsize']; ?></td>
-			   </tr>
-			   <tr>
-				  <td><?php print $text_average_size_day; ?></td>
-				  <td><?php print $health['averagesizeday']; ?></td>
-			   </tr>
-			   <tr>
-				  <td>"<?php print DATA_PARTITION; ?>" <?php print $text_partition_full; ?></td>
-				  <td><?php print $health['daysleftatcurrentrate'][0]; ?> years, <?php print $health['daysleftatcurrentrate'][1]; ?> months, <?php print $health['daysleftatcurrentrate'][2]; ?> days</td>
-			   </tr>
-			   <tr>
-				  <td><?php print $text_usage_trend; ?></td>
-				  <td><?php if ($health['usagetrend'] > 0) { print $text_usage_increasing; } elseif($health['usagetrend'] < 0) { print $text_usage_decreasing; } else { print $text_usage_neutral; } ?></td>
-			   </tr>
                            <tr>
-                                  <td>Sphinx main index</td>
-                                  <td<?php if($health['sphinx_current_main_size'] > SPHINX_MAIN_INDEX_THRESHOLD) { ?> class="text-error"<?php } ?>><?php print nice_size($health['sphinx_current_main_size']); ?></td>
+                                  <td><?php print $text_average_messages_day; ?></td>
+                                  <td><?php print $health['averagemessages']; ?></td>
                            </tr>
-		 </table>
-          
-          
-		 </div>
-	</div>
+                           <tr>
+                                  <td><?php print $text_average_message_size; ?></td>
+                                  <td><?php print $health['averagemessagesize']; ?> + <?php print $health['averagesqlsize']; ?> + <?php print $health['averagesphinxsize']; ?></td>
+                           </tr>
+                           <tr>
+                                  <td><?php print $text_average_size_day; ?></td>
+                                  <td><?php print $health['averagesizeday']; ?></td>
+                           </tr>
+                           <tr>
+                                  <td>"<?php print DATA_PARTITION; ?>" <?php print $text_partition_full; ?></td>
+                                  <td><?php print $health['daysleftatcurrentrate'][0]; ?> years, <?php print $health['daysleftatcurrentrate'][1]; ?> months, <?php print $health['daysleftatcurrentrate'][2]; ?> days</td>
+                           </tr>
+                           <tr>
+                                  <td><?php print $text_usage_trend; ?></td>
+                                  <td><?php if ($health['usagetrend'] > 0) { print $text_usage_increasing; } elseif($health['usagetrend'] < 0) { print $text_usage_decreasing; } else { print $text_usage_neutral; } ?></td>
+                           </tr>
+                           <tr>
+                                  <td>Sphinx main (total) index</td>
+                                  <td<?php if($health['sphinx_current_main_size'] > SPHINX_MAIN_INDEX_THRESHOLD) { ?> class="text-error"<?php } ?>><?php print nice_size($health['sphinx_current_main_size']); ?> (<?php print nice_size($health['sphinx_total_size']); ?>) </td>
+                           </tr>
+                 </table>
+
+
+                </div>
+    </div>
 </div>