Bumped up version number to 1.4.5

Signed-off-by: Janos SUTO <sj@acts.hu>

.gitignore

@@ -0,0 +1,26 @@
+Makefile
+*.o
+*.so
+*.a
+*.deb
+src/libpiler.so.0*
+src/piler
+src/pilerconf
+src/piler-smtp
+src/pileraget
+src/pilerexport
+src/pilerget
+src/pilerimport
+src/pilerstats
+src/pilertest
+src/reindex
+unit_tests/check_attachments
+unit_tests/check_decoder
+unit_tests/check_digest
+unit_tests/check_hash
+unit_tests/check_misc
+unit_tests/check_mydomains
+unit_tests/check_parser
+unit_tests/check_parser_utils
+unit_tests/check_rules
+unit_tests/smtp

G

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+read -r version < VERSION
+
+tarball="piler-${version}.tar.gz"
+
+git archive --prefix="piler-${version}/" HEAD | gzip -c > "$tarball"
+sha256sum "$tarball"

LICENSE

@@ -1,6 +1,6 @@
 piler, an enterprise level email archiving application
 
-Copyright (C) 2012-2016, Janos SUTO <sj@acts.hu>
+Copyright (C) 2012-2023, Janos SUTO <sj@acts.hu>
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

Makefile.in

@@ -64,7 +64,8 @@ $(RECURSIVE_TARGETS):
 
 
 config-php:
-	sed "s%SYSCONFDIR%$(sysconfdir)%" config.php.in > webui/config.php
+	sed -e "s%SYSCONFDIR%$(sysconfdir)%g" -e "s%SBINDIR%$(sbindir)%g" config.php.in > webui/config.php
+	sed -e "s%BINDIR%$(bindir)%g" -i webui/config.php
 
 
 installdirs: mkinstalldirs
@@ -81,7 +82,8 @@ installdirs: mkinstalldirs
 		$(DESTDIR)$(localstatedir)/piler/stat \
 		$(DESTDIR)$(localstatedir)/piler/tmp \
 		$(DESTDIR)$(localstatedir)/piler/error \
-		$(DESTDIR)$(localstatedir)/piler/sphinx
+		$(DESTDIR)$(localstatedir)/piler/export \
+		$(DESTDIR)$(localstatedir)/piler/manticore
 
 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/run/piler
 	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/store
@@ -89,7 +91,8 @@ installdirs: mkinstalldirs
 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/stat
 	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/tmp
 	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/error
-	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/sphinx
+	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/export
+	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/manticore
 
 
 install-am:

RELEASE_NOTES

@@ -1,3 +1,103 @@
+1.4.5:
+------
+
+- Introduced 2 new variables in /etc/piler/piler.conf affecting piler-smtp
+
+  ; max message size in bytes
+  ; piler-smtp will reject any message that's bigger than this number
+  max_message_size=50000000
+
+  ; max memory in bytes piler-smtp uses for buffering messages
+  ; when this limit is exceeded, no new emails will be accepted
+  ; until the used memory for all in progress emails decreases
+  ; below this level
+  max_smtp_memory=500000000
+
+  Be sure to adjust these values to your environment!
+
+- Added read-only connection support for manticore
+  If using sphinx, add the following to config-site.php:
+  $config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9306';
+
+- pilerimport supports Zimbra IMAP impersonation
+
+  Generate the following base64 encoded string:
+  (Be sure to use the actual usernames and password):
+
+  pw="$( printf '%s\0%s\0%s' 'username' 'zimbra_admin_username' 'zimbra_admin_password' | base64 )"
+
+  Then specify -u ZIMBRA -p "$pw" for pilerimport, eg.
+
+  pilerimport -i imap.server -u ZIMBRA -p "$pw" ...
+
+  Note that "ZIMBRA" is a special username, it indicates for pilerimport
+  to actually use the imap impersonation for Zimbra.
+
+
+1.4.4:
+------
+
+- Renamed "group" table to "usergroup"
+  Be sure to run util/db-upgrade.sql on the mysql piler database
+
+
+1.4.3:
+------
+
+- Improved real-time index support (no need to use rtindex.py any more)
+  See https://www.mailpiler.org/wiki/current:manticore for more
+
+
+1.4.2:
+------
+
+- Fixed docker image
+- Fixed php socket path in piler-nginx.conf
+- Fixed manticore index data settings
+
+
+1.4.1:
+------
+
+- Replaced sphinx with manticore 5.0.2, see https://www.mailpiler.org/wiki/current:manticore
+- Replaced obsoleted libchart library with chart.js
+- Added real-time index support
+- Added oauth2 support to imapfetch.py
+- Added support for pilerexport to spread files among several directories. See the -D option
+
+
+1.3.12:
+-------
+
+- Introduced new piler.conf variable: tls_min_version
+
+  It sets the minimum TLS protocol version the piler-smtp daemon supports.
+
+  Possible values:
+  - TLSv1   (not recommended)
+  - TLSv1.1 (not recommended)
+  - TLSv1.2 (default)
+  - TLSv1.3
+
+- Introduced the archive_address feature, see etc/example.conf for the details
+- Introduced the raw: search label
+- Added Italian translation. Credits: Stefano Gatto
+- timestamp signing sorts by 'id' column
+- timestamp hash value defaults to sha256
+- Minor fixes
+- Added support for php 8.1
+- Fixed handling long email addresses
+
+Be sure to apply util/db-upgrade.sql
+
+
+1.3.11:
+-------
+
+- [BUGFIX] Refactored the smtp timeout check
+- Obsoleted the LDAP port parameter. Specify the ldap host in the form
+  of protocol://hostname:port, eg. ldaps://ldap.example.com:636
+
 1.3.10:
 -------
 

SECURITY.md

@@ -11,9 +11,10 @@ Security update policy
 
 If a security vulnerability has found, the details, possible mitigations,
 workarounds, etc. will be shared on the piler mailing list (piler-user@list.acts.hu)
-and on the wiki: http://www.mailpiler.org/
+and on the wiki: https://www.mailpiler.org/
 
 Security configurations
 
  - Use https for the GUI
  - Reset the default passwords for admin and auditor
+ - Use the smtp acl feature to restrict SMTP access to the archive, see https://mailpiler.com/smtp-acl-list/

VERSION

@@ -1 +1 @@
-1.3.10
+1.4.5

bitbucket-pipelines.yml

@@ -18,5 +18,5 @@ pipelines:
           - mysql -u piler -ppiler123 piler1 < /usr/share/piler/db-mysql.sql
           - cd unit_tests
           - ./run.sh
-          - cd ..
+          - cd ../webui
           - phpunit

config.php.in

@@ -34,14 +34,16 @@ $config['TITLE_PREFIX'] = '';
 $config['CUSTOM_PRE_AUTH_FUNCTION'] = '';
 $config['CUSTOM_EMAIL_QUERY_FUNCTION'] = '';
 
+$config['DOMAIN_REGEX'] = '/^[a-zA-Z0-9]+[a-zA-Z0-9-_\.]{0,}\.[a-zA-Z0-9]{2,20}$/';
+
 $config['BOOTSTRAP_THEME'] = '-cosmo';
 
 $config['DEFAULT_LANG'] = 'en';
 
 $config['THEME'] = 'default';
 
-$config['SITE_NAME'] = 'piler.yourdomain.com';
-$config['SITE_URL'] = 'http://piler.yourdomain.com/';
+$config['SITE_NAME'] = 'piler.example.com';
+$config['SITE_URL'] = 'http://piler.example.com/';
 
 $config['EXTERNAL_DASHBOARD_URL'] = '';
 
@@ -81,14 +83,14 @@ $config['WKHTMLTOPDF_COMMAND'] = "wkhtmltopdf";
 // authentication against an ldap directory (disabled by default)
 
 $config['ENABLE_LDAP_AUTH'] = 0;
-$config['LDAP_HOST'] = 'ldap.yourdomain.com';
-$config['LDAP_PORT'] = 389;
+$config['LDAP_HOST'] = 'ldap.example.com';
 $config['LDAP_HELPER_DN'] = 'cn=....';
 $config['LDAP_HELPER_PASSWORD'] = 'xxxxxxx';
 $config['LDAP_MAIL_ATTR'] = 'mail';
 $config['LDAP_AUDITOR_MEMBER_DN'] = '';
 $config['LDAP_ADMIN_MEMBER_DN'] = '';
 $config['LDAP_BASE_DN'] = '';
+$config['LDAP_USE_START_TLS'] = 0;
 
 // AD specific settings
 //
@@ -112,9 +114,9 @@ $config['LDAP_MAIL_ATTR'] = 'proxyAddresses';
 //$config['LDAP_MAIL_ATTR'] = 'mail';
 
 // iredmail specific settings
-//$config['LDAP_HELPER_DN'] = 'cn=vmailadmin,dc=yourdomain,dc=com';
+//$config['LDAP_HELPER_DN'] = 'cn=vmailadmin,dc=example,dc=com';
 //$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'mailUser';
-//$config['LDAP_BASE_DN'] = 'o=domains,dc=yourdomain,dc=com';
+//$config['LDAP_BASE_DN'] = 'o=domains,dc=example,dc=com';
 //$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'mailList';
 //$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'memberOfGroup';
 //$config['LDAP_MAIL_ATTR'] = 'mail';
@@ -137,14 +139,14 @@ $config['ENABLE_IMAP_AUTH'] = 0;
 $config['RESTORE_OVER_IMAP'] = 0;
 $config['IMAP_RESTORE_FOLDER_INBOX'] = 'INBOX';
 $config['IMAP_RESTORE_FOLDER_SENT'] = 'Sent';
-$config['IMAP_HOST'] = 'mail.yourdomain.com';
+$config['IMAP_HOST'] = 'mail.example.com';
 $config['IMAP_PORT'] =  993;
 $config['IMAP_SSL'] = true;
 
 // enable authentication against a pop3 server (disabled by default)
 
 $config['ENABLE_POP3_AUTH'] = 0;
-$config['POP3_HOST'] = 'mail.yourdomain.com';
+$config['POP3_HOST'] = 'mail.example.com';
 $config['POP3_PORT'] = 995;
 $config['POP3_SSL'] = true;
 
@@ -202,23 +204,25 @@ $config['PILER_HOST'] = '0.0.0.0';
 $config['PILER_PORT'] = 25;
 $config['SMARTHOST'] = '';
 $config['SMARTHOST_PORT'] = 25;
-$config['SMTP_DOMAIN'] = 'yourdomain.com';
-$config['SMTP_FROMADDR'] = 'no-reply@yourdomain.com';
-$config['ADMIN_EMAIL'] = 'admin@yourdomain.com';
+$config['SMARTHOST_USER'] = '';
+$config['SMARTHOST_PASSWORD'] = '';
+$config['SMTP_DOMAIN'] = 'example.com';
+$config['SMTP_FROMADDR'] = 'no-reply@example.com';
+$config['ADMIN_EMAIL'] = 'admin@example.com';
 
 $config['PILER_HEADER_FIELD'] = 'X-piler-id: ';
 
 $config['DEFAULT_POLICY'] = 'default_policy';
 
-$config['DIR_BASE'] = '/var/piler/www/';
+$config['DIR_BASE'] = dirname(__FILE__) . '/';
 
-$config['DIR_SPHINX'] = '/var/piler/sphinx/';
+$config['DIR_SPHINX'] = '/var/piler/manticore/';
 $config['DIR_STAT'] = '/var/piler/stat';
 $config['DIR_IMAP'] = '/var/piler/imap';
 $config['DIR_TMP'] = '/var/piler/tmp';
 
-$config['DECRYPT_BINARY'] = '/usr/local/bin/pilerget';
-$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/local/bin/pileraget';
+$config['DECRYPT_BINARY'] = 'BINDIR/pilerget';
+$config['DECRYPT_ATTACHMENT_BINARY'] = 'BINDIR/pileraget';
 $config['DECRYPT_BUFFER_LENGTH'] = 65536;
 
 $config['OPENSSL_BINARY'] = '/usr/bin/openssl';
@@ -227,6 +231,12 @@ $config['TSA_PUBLIC_KEY_FILE'] = '';
 $config['TSA_START_ID'] = 1;
 $config['TSA_STAMP_REQUEST_UNIT_SIZE'] = 10000;
 $config['TSA_VERIFY_CERTIFICATE'] = true;
+$config['TSA_RELAXED_CHECK'] = false;
+$config['TSA_AUTH_USER'] = '';
+$config['TSA_AUTH_PASSWORD'] = '';
+$config['TSA_AUTH_CERT_FILE'] = '';
+$config['TSA_AUTH_KEY_FILE'] = '';
+$config['TSA_AUTH_KEY_PASSWORD'] = '';
 
 $config['DB_DRIVER'] = 'mysql';
 $config['DB_PREFIX'] = '';
@@ -239,17 +249,19 @@ $config['DB_CHARSET'] = 'utf8mb4';
 $config['SPHINX_DRIVER'] = 'sphinx';
 $config['SPHINX_DATABASE'] = 'sphinx';
 $config['SPHINX_HOSTNAME'] = '127.0.0.1:9306';
+$config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9307';
 $config['SPHINX_MAIN_INDEX'] = 'main1,dailydelta1,delta1';
 $config['SPHINX_ATTACHMENT_INDEX'] = 'att1';
 $config['SPHINX_TAG_INDEX'] = 'tag1';
 $config['SPHINX_NOTE_INDEX'] = 'note1';
 $config['SPHINX_STRICT_SCHEMA'] = 1;
+$config['RT'] = 0;
 $config['MAX_EMAIL_LEN'] = 41;
 
 $config['RELOAD_COMMAND'] = 'sudo -n /etc/init.d/rc.piler reload';
 $config['PILERIMPORT_IMAP_COMMAND'] = '/usr/local/bin/pilerimport -d /var/piler/imap -q -r';
 $config['CPU_USAGE_COMMAND'] = "LC_ALL=C mpstat | tail -1 | rev | awk '{ print $1 }' | rev";
-$config['PILER_BINARY'] = "/usr/local/sbin/piler";
+$config['PILER_BINARY'] = "SBINDIR/piler";
 
 $config['LDAP_IMPORT_CONFIG_FILE'] = '/usr/local/etc/ldap-import.cfg';
 
@@ -268,7 +280,10 @@ $config['SIZE_Y'] = 250;
 $config['DATE_TEMPLATE'] = 'Y.m.d';
 $config['DATE_FORMAT'] = 'YYYY-MM-DD';
 $config['JQUERY_DATE_FORMAT'] = 'yy-mm-dd';
+$config['DECIMAL_SEPARATOR'] = "."; // See https://www.php.net/manual/en/function.number-format
+$config['THOUSANDS_SEPARATOR'] = ","; // for the format options
 
+$config['ENABLE_GB2312_FIX'] = 1;
 
 $config['FROM_LENGTH_TO_SHOW'] = 28;
 
@@ -325,10 +340,12 @@ $langs = array(
                 'es',
                 'fr',
                 'hu',
+                'it',
                 'pl',
                 'pt',
                 'ru',
-                'tr'
+                'tr',
+                'tw'
                );
 
 
@@ -357,7 +374,7 @@ define('NOW', time());
  * normally you don't have to change anything below
  */
 
-require_once 'SYSCONFDIR/piler/config-site.php';
+if(file_exists('SYSCONFDIR/piler/config-site.php')) { require_once 'SYSCONFDIR/piler/config-site.php'; }
 
 ini_set('session.cookie_lifetime', $config['SESSION_EXPIRY']);
 
@@ -384,7 +401,7 @@ foreach ($config as $k => $v) {
 
 
 define('TABLE_USER', 'user');
-define('TABLE_GROUP', 'group');
+define('TABLE_GROUP', 'usergroup');
 define('TABLE_GROUP_USER', 'group_user');
 define('TABLE_GROUP_EMAIL', 'group_email');
 define('TABLE_FOLDER', 'folder');
@@ -424,7 +441,7 @@ define('TABLE_PRIVATE', 'private');
 define('TABLE_DELETED', 'deleted');
 define('VIEW_MESSAGES', 'v_messages');
 
-define('EOL', "\n");
+define('EOL', "\r\n");
 
 define('DIR_SYSTEM', DIR_BASE . 'system/');
 define('DIR_MODEL', DIR_BASE . 'model/');
@@ -457,7 +474,8 @@ define('QSHAPE_DEFERRED_SENDER', DIR_STAT . '/deferred-sender');
 define('CPUSTAT', DIR_STAT . '/cpu.stat');
 define('AD_SYNC_STAT', DIR_STAT . '/adsync.stat');
 define('ARCHIVE_SIZE', DIR_STAT . '/archive.size');
-define('SPHINX_MAIN_INDEX_SIZE', DIR_STAT . '/main_index_size');
+define('SPHINX_CURRENT_MAIN_INDEX_SIZE', DIR_STAT . '/current_main_index_size');
+define('SPHINX_TOTAL_INDEX_SIZE', DIR_STAT . '/total_index_size');
 define('LOCK_FILE', DIR_LOG . 'lock');
 
 define('SEARCH_HELPER_URL', SITE_URL . 'search-helper.php');
@@ -473,6 +491,8 @@ define('HEALTH_WORKER_URL', SITE_URL . 'index.php?route=health/worker');
 
 define('LDAP_TYPE_GENERIC', 'generic_ldap');
 
+define('ATTACHMENT_DUMP_CHECKPOINT', 'attachment_dump_checkpoint');
+
 define('ACTION_ALL', 0);
 define('ACTION_UNKNOWN', 1);
 define('ACTION_LOGIN', 2);
@@ -494,6 +514,7 @@ define('ACTION_VIEW_JOURNAL', 17);
 define('ACTION_NOT_SPAM', 18);
 define('ACTION_MARK_AS_PRIVATE', 19);
 define('ACTION_MARK_MESSAGE_FOR_REMOVAL', 20);
+define('ACTION_REJECT_REMOVAL', 21);
 
 $actions = array(
                   'unknown' => 1,
@@ -508,7 +529,9 @@ $actions = array(
                   'save_search' => 11,
                   'download_attachment' => 15,
                   'journal' => 17,
-                  'private' => 19
+                  'private' => 19,
+                  'marked_for_removal', 20,
+                  'reject_removal', 21
                 );
 
 

configure.in

@@ -524,11 +524,11 @@ echo; echo
 
 gcc_version="$(gcc -dumpversion)"
 extra_cflags=""
-if [[ "${gcc_version:0:1}" -gt 5 ]]; then
+if [[ "${gcc_version:0:1}" -gt 6 ]]; then
    extra_cflags="-Wimplicit-fallthrough=2"
 fi
 
-CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra $extra_cflags -Wuninitialized -Wno-format-truncation -g"
+CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra $extra_cflags -Wuninitialized -Wimplicit-fallthrough=2 -Wno-format-truncation -g"
 LIBS="$antispam_libs $sunos_libs "
 OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o screen.o $objs"
 
@@ -537,6 +537,12 @@ AC_OUTPUT
 
 echo
 echo
-echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/wiki/current:upgrade"
+echo "IMPORTANT! If you upgrade, be sure to read https://www.mailpiler.org/upgrade-instructions/"
+echo
+echo
+
+echo
+echo "Did you know that piler has an enterprise edition as well?"
+echo "Check out what it can do for you at https://mailpiler.com/#features"
 echo
 echo

contrib/export-attachments/export-attachments.php

@@ -0,0 +1,103 @@
+#!/usr/bin/php
+<?php
+
+define('EXPORT_LOCK_FILE', '/var/piler/tmp/export-attachments.lock');
+
+$webuidir = "";
+
+$outdir = '';
+
+
+$opts = 'hw:d:';
+$lopts = [
+   'webui:',
+   'dir:',
+   'help'
+];
+
+
+if($options = getopt($opts, $lopts)) {
+   if(isset($options['webui'])) { $webuidir = $options['webui']; }
+   if(isset($options['dir'])) { $outdir = $options['dir']; }
+   if(isset($options['help'])) { usage(); }
+}
+
+
+if($webuidir == '' || $outdir == '') { usage(); }
+
+
+
+require_once("$webuidir/config.php");
+
+require(DIR_SYSTEM . "/startup.php");
+
+$request = new Request();
+Registry::set("request", $request);
+
+
+$start = NULL;
+
+$loader = new Loader();
+Registry::set('load', $loader);
+
+$loader->load->model('domain/domain');
+$loader->load->model('search/search');
+$loader->load->model('search/message');
+$loader->load->model('message/attachment');
+
+
+$db = new DB(DB_DRIVER, DB_HOSTNAME, DB_USERNAME, DB_PASSWORD, DB_DATABASE, DB_PREFIX);
+Registry::set('db', $db);
+
+Registry::set('auditor_user', 1);
+
+openlog("export-attachments", LOG_PID, LOG_MAIL);
+
+$fp = fopen(EXPORT_LOCK_FILE, "w");
+if(!flock($fp, LOCK_EX)) {
+   syslog(LOG_INFO, "WARN: couldn't get a lock on " . EXPORT_LOCK_FILE);
+   exit;
+}
+
+
+$domain = new ModelDomainDomain();
+$attachment = new ModelMessageAttachment();
+$message = new ModelSearchMessage();
+
+$domains = $domain->get_mapped_domains();
+
+$last_id = $attachment->get_last_attachment_id();
+$start_id = $attachment->get_checkpoint();
+
+syslog(LOG_INFO, "start: $start_id, limit: $last_id");
+
+
+for($i=$start_id; $i<$last_id; $i++) {
+   $a = $attachment->get_attachment_by_id($i);
+   $m = $message->get_message_addresses_by_piler_id($a['piler_id'], $domains);
+
+   $attachment->dump_attachment($outdir, "out", $m['sender'], $i, $a);
+
+   foreach($m['rcpt'] as $rcpt) {
+      $attachment->dump_attachment($outdir, "in", $rcpt, $i, $a);
+   }
+
+   if($i % 100 == 0) { $attachment->update_checkpoint($i); }
+}
+
+$attachment->update_checkpoint($i);
+
+// Release lock
+flock($fp, LOCK_UN);
+fclose($fp);
+
+
+
+function usage() {
+   print "\nUsage: " . __FILE__ . "\n\n";
+   print "\t--webui <path to webui directory>\n";
+   print "\t--dir <basedir to write attachments>\n";
+   print "\t--help\n\n";
+
+   exit;
+}

contrib/installer/focal.sh

@@ -0,0 +1,257 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+
+MY_IP="1.2.3.4"
+PILER_HOSTNAME="${PILER_HOSTNAME:-archive.example.com}"
+MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
+MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
+SERVER_ID="${SERVER_ID:-0}"
+USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
+SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
+PHP_FPM_SOCKET="/var/run/php/php7.4-fpm.sock"
+
+MYSQL_HOSTNAME="localhost"
+MYSQL_DATABASE="piler"
+MYSQL_USERNAME="piler"
+
+SPHINX_TARGZ="sphinx-3.3.1-bin.tar.gz"
+DOWNLOAD_URL="https://download.mailpiler.com"
+PILER_DEB="piler_1.3.12-focal-eb2b22b_amd64.deb"
+PILER_USER="piler"
+CONFIG_SITE_PHP="/etc/piler/config-site.php"
+
+export DEBIAN_FRONTEND=noninteractive
+
+install_prerequisites() {
+   apt-get update
+   apt-get -y --no-install-recommends install \
+      wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \
+      php7.4-gd php7.4-curl php7.4-xml ca-certificates zip catdoc unrtf poppler-utils nginx tnef libzip5 \
+      libtre5 libwrap0 cron libmariadb-dev python3 python3-mysqldb libmariadb-dev mariadb-client-core-10.3 \
+      mariadb-server-10.3
+
+   wget -q -O "/tmp/${SPHINX_TARGZ}" "${DOWNLOAD_URL}/generic-local/${SPHINX_TARGZ}"
+   tar -C / -zxvf "/tmp/${SPHINX_TARGZ}"
+   wget -O /usr/local/bin/traefik "${DOWNLOAD_URL}/generic-local/traefik"
+   chmod +x /usr/local/bin/traefik
+   setcap cap_net_bind_service+ep /usr/local/bin/traefik
+}
+
+
+fix_mysql_settings() {
+   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
+[mysqld]
+
+innodb_buffer_pool_size=512M
+innodb_flush_log_at_trx_commit=1
+innodb_log_buffer_size=64M
+innodb_log_file_size=64M
+innodb_read_io_threads=4
+innodb_write_io_threads=4
+innodb_log_files_in_group=2
+
+innodb_file_per_table
+PILER_CNF
+}
+
+start_mysql() {
+   fix_mysql_settings
+
+   service mysql restart
+   mysqladmin -u root password "${MYSQL_ROOT_PASSWORD}"
+}
+
+
+install_piler() {
+   wget "${DOWNLOAD_URL}/piler/${PILER_DEB}"
+   dpkg -i "$PILER_DEB"
+   rm -f "$PILER_DEB"
+
+   crontab -u "$PILER_USER" /usr/share/piler/piler.cron
+
+   rm -f "$PILER_DEB" /etc/nginx/sites-enabled/default
+   ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf
+
+   touch /var/piler/.bash_history
+   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
+}
+
+
+create_my_cnf() {
+   local user=$1
+   local password=$2
+   local my_cnf=$3
+
+   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
+   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
+   chown $PILER_USER:$PILER_USER "$my_cnf"
+   chmod 600 "$my_cnf"
+}
+
+
+fix_config_site_php() {
+   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"
+
+   {
+      echo "\$config['SERVER_ID'] = $SERVER_ID;"
+      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
+      echo "\$config['SPHINX_VERSION'] = 331;"
+   } >> "$CONFIG_SITE_PHP"
+
+   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
+      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
+   fi
+
+   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
+}
+
+
+init_db() {
+   sed -e "s%MYSQL_HOSTNAME%$MYSQL_HOSTNAME%g" -e "s%MYSQL_DATABASE%$MYSQL_DATABASE%g" -e "s%MYSQL_USERNAME%$MYSQL_USERNAME%g" -e "s%MYSQL_PASSWORD%$MYSQL_PILER_PASSWORD%g" \
+      /usr/share/piler/db-mysql-root.sql.in | mysql --defaults-file=/etc/piler/.my.cnf-root -h $MYSQL_HOSTNAME
+   mysql --defaults-file=/etc/piler/.my.cnf -h $MYSQL_HOSTNAME $MYSQL_DATABASE < /usr/share/piler/db-mysql.sql
+}
+
+
+add_systemd_services() {
+   pushd /etc/systemd/system
+
+   ln -sf /usr/libexec/piler/piler.service .
+   ln -sf /usr/libexec/piler/piler-smtp.service .
+   ln -sf /usr/libexec/piler/pilersearch.service .
+
+   popd
+
+   systemctl daemon-reload
+
+   systemctl enable piler
+   systemctl enable piler-smtp
+   systemctl enable pilersearch
+   systemctl enable traefik
+}
+
+
+fix_configs() {
+   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
+      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
+      nginx -t
+      nginx -s reload
+
+      sed -i 's/server {/server {\n\tlisten 127.0.0.1:80;/' /etc/piler/piler-nginx.conf
+      systemctl stop nginx
+      sleep 5
+      systemctl start nginx
+   fi
+
+   if [[ ! -f /etc/piler/piler.conf ]]; then
+      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.example.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > /etc/piler/piler.conf
+      chmod 600 /etc/piler/piler.conf
+      chown $PILER_USER:$PILER_USER /etc/piler/piler.conf
+   fi
+
+   sed -i 's/tls_enable=0/tls_enable=1/g' /etc/piler/piler.conf
+
+   sed -i -e "s/MYSQL_HOSTNAME/${MYSQL_HOSTNAME}/g" \
+          -e "s/MYSQL_DATABASE/${MYSQL_DATABASE}/g" \
+          -e "s/MYSQL_USERNAME/${MYSQL_USERNAME}/g" \
+          -e "s/MYSQL_PASSWORD/${MYSQL_PILER_PASSWORD}/g" \
+          /etc/piler/sphinx.conf
+}
+
+setup_traefik() {
+   wget -O /etc/systemd/system/traefik.service "${DOWNLOAD_URL}/generic-local/traefik.service"
+
+   mkdir -p /usr/local/etc/traefik
+   touch /usr/local/etc/traefik/acme.json
+   chmod 600 /usr/local/etc/traefik/acme.json
+   chown www-data:www-data /usr/local/etc/traefik/acme.json
+
+   cat > /usr/local/etc/traefik/traefik.yaml << TRAEFIK
+log:
+  level: INFO
+
+entryPoints:
+  web:
+    address: "$MY_IP:80"
+    http:
+      redirections:
+        entryPoint:
+          to: websecure
+          scheme: https
+          permanent: true
+  websecure:
+    address: "$MY_IP:443"
+
+providers:
+  file:
+    filename: "/usr/local/etc/traefik/traefik.yaml"
+
+certificatesResolvers:
+  le:
+    acme:
+      storage: "/usr/local/etc/traefik/acme.json"
+      email: admin@$PILER_HOSTNAME
+      httpChallenge:
+        entryPoint: web
+
+tls:
+  options:
+    default:
+      minVersion: VersionTLS13
+
+http:
+  middlewares:
+    piler_headers:
+      headers:
+        customResponseHeaders:
+          Server: ""
+          Strict-Transport-Security: "max-age=31536000"
+          X-Content-Type-Optionsi: "nosniff"
+          Referrer-Policy: "same-origin"
+  routers:
+    master:
+      rule: "Host(\`$PILER_HOSTNAME\`)"
+      service: www
+      middlewares:
+        - "piler_headers"
+      tls:
+        certResolver: le
+  services:
+    www:
+      loadBalancer:
+        servers:
+        - url: "http://127.0.0.1:80/"
+TRAEFIK
+}
+
+
+install_prerequisites
+
+start_mysql
+
+install_piler
+
+create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
+create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf
+
+fix_config_site_php
+
+setup_traefik
+
+add_systemd_services
+
+fix_configs
+init_db
+
+su -c "indexer --all -c /etc/piler/sphinx.conf" $PILER_USER
+
+[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler
+
+systemctl start pilersearch
+systemctl start piler
+systemctl start piler-smtp
+systemctl start traefik

contrib/installer/jammy.sh

@@ -0,0 +1,258 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+
+MY_IP="1.2.3.4"
+PILER_HOSTNAME="${PILER_HOSTNAME:-archive.example.com}"
+MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
+MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
+SERVER_ID="${SERVER_ID:-0}"
+USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
+SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
+PHP_FPM_SOCKET="/var/run/php/php8.1-fpm.sock"
+
+MYSQL_HOSTNAME="localhost"
+MYSQL_DATABASE="piler"
+MYSQL_USERNAME="piler"
+
+DOWNLOAD_URL="https://download.mailpiler.com"
+PILER_DEB="piler_1.4.4-jammy-0573c6da_amd64.deb"
+PILER_USER="piler"
+PILER_CONF="/etc/piler/piler.conf"
+CONFIG_SITE_PHP="/etc/piler/config-site.php"
+SEARCHCFG="/etc/piler/manticore.conf"
+
+TRAEFIK_VERSION="v2.11.0"
+ARCH="amd64"
+
+export DEBIAN_FRONTEND=noninteractive
+
+install_prerequisites() {
+   apt-get update
+   apt-get -y --no-install-recommends install \
+      wget rsyslog openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql \
+      php8.1-fpm php8.1-zip php8.1-ldap php8.1-gd php8.1-curl php8.1-xml \
+      php8.1-mbstring ca-certificates zip catdoc unrtf poppler-utils \
+      nginx tnef libzip4 libtre5 libwrap0 cron libmariadb-dev python3 \
+      python3-mysqldb libmariadb-dev mariadb-client-core-10.6 \
+      mariadb-server-10.6
+
+   wget "https://github.com/traefik/traefik/releases/download/${TRAEFIK_VERSION}/traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz"
+   tar zxvf "traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz" traefik
+   chown root:root traefik
+   chmod 755 traefik
+   mv traefik /usr/local/bin
+   setcap cap_net_bind_service+ep /usr/local/bin/traefik
+
+   wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
+   dpkg -i manticore-repo.noarch.deb && \
+   apt-get update && \
+   apt-get install -y manticore manticore-columnar-lib manticore-extra && \
+   rm -f manticore-repo.noarch.deb
+   systemctl stop manticore
+}
+
+
+fix_mysql_settings() {
+   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
+[mysqld]
+
+innodb_buffer_pool_size=512M
+innodb_flush_log_at_trx_commit=1
+innodb_log_buffer_size=64M
+innodb_log_file_size=64M
+innodb_read_io_threads=4
+innodb_write_io_threads=4
+innodb_log_files_in_group=2
+
+innodb_file_per_table
+PILER_CNF
+}
+
+start_mysql() {
+   fix_mysql_settings
+
+   service mysql restart
+   mysqladmin -u root password "${MYSQL_ROOT_PASSWORD}"
+}
+
+
+install_piler() {
+   wget "https://bitbucket.org/jsuto/piler/downloads/${PILER_DEB}"
+   dpkg -i "$PILER_DEB"
+   rm -f "$PILER_DEB"
+
+   sed -i 's/.*indexer.*/###/g' /usr/share/piler/piler.cron
+   crontab -u "$PILER_USER" /usr/share/piler/piler.cron
+
+   rm -f "$PILER_DEB" /etc/nginx/sites-enabled/default
+   ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf
+
+   touch /var/piler/.bash_history
+   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
+}
+
+
+create_my_cnf() {
+   local user=$1
+   local password=$2
+   local my_cnf=$3
+
+   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
+   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
+   chown $PILER_USER:$PILER_USER "$my_cnf"
+   chmod 600 "$my_cnf"
+}
+
+
+fix_config_site_php() {
+   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"
+
+   {
+      echo "\$config['SERVER_ID'] = $SERVER_ID;"
+      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
+   } >> "$CONFIG_SITE_PHP"
+
+   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
+      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
+   fi
+
+   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
+   echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
+}
+
+
+init_db() {
+   sed -e "s%MYSQL_HOSTNAME%$MYSQL_HOSTNAME%g" -e "s%MYSQL_DATABASE%$MYSQL_DATABASE%g" -e "s%MYSQL_USERNAME%$MYSQL_USERNAME%g" -e "s%MYSQL_PASSWORD%$MYSQL_PILER_PASSWORD%g" \
+      /usr/share/piler/db-mysql-root.sql.in | mysql --defaults-file=/etc/piler/.my.cnf-root -h $MYSQL_HOSTNAME
+   mysql --defaults-file=/etc/piler/.my.cnf -h $MYSQL_HOSTNAME $MYSQL_DATABASE < /usr/share/piler/db-mysql.sql
+}
+
+
+add_systemd_services() {
+   pushd /etc/systemd/system
+
+   ln -sf /usr/libexec/piler/piler.service .
+   ln -sf /usr/libexec/piler/piler-smtp.service .
+   ln -sf /usr/libexec/piler/pilersearch.service .
+
+   popd
+
+   systemctl daemon-reload
+
+   systemctl enable piler
+   systemctl enable piler-smtp
+   systemctl enable pilersearch
+   systemctl enable traefik
+}
+
+
+fix_configs() {
+   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
+      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
+      nginx -t
+      nginx -s reload
+
+      sed -i 's/server {/server {\n\tlisten 127.0.0.1:80;/' /etc/piler/piler-nginx.conf
+      systemctl stop nginx
+      sleep 5
+      systemctl start nginx
+   fi
+
+   if [[ ! -f "$PILER_CONF" ]]; then
+      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.example.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > "$PILER_CONF"
+      chmod 600 "$PILER_CONF"
+      chown $PILER_USER:$PILER_USER "$PILER_CONF"
+   fi
+
+   sed -i 's/tls_enable=0/tls_enable=1/g' "$PILER_CONF"
+   sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
+
+   sed -i "s/define('RT.*/define('RT', 1);/" "$SEARCHCFG"
+}
+
+setup_traefik() {
+   wget -O /etc/systemd/system/traefik.service "${DOWNLOAD_URL}/generic-local/traefik.service"
+
+   mkdir -p /usr/local/etc/traefik
+   touch /usr/local/etc/traefik/acme.json
+   chmod 600 /usr/local/etc/traefik/acme.json
+   chown www-data:www-data /usr/local/etc/traefik/acme.json
+
+   cat > /usr/local/etc/traefik/traefik.yaml << TRAEFIK
+log:
+  level: INFO
+
+entryPoints:
+  websecure:
+    address: "$MY_IP:443"
+
+providers:
+  file:
+    filename: "/usr/local/etc/traefik/traefik.yaml"
+
+certificatesResolvers:
+  le:
+    acme:
+      storage: "/usr/local/etc/traefik/acme.json"
+      email: admin@$PILER_HOSTNAME
+      tlsChallenge: {}
+
+tls:
+  options:
+    default:
+      minVersion: VersionTLS13
+
+http:
+  middlewares:
+    piler_headers:
+      headers:
+        customResponseHeaders:
+          Server: ""
+          Strict-Transport-Security: "max-age=31536000"
+          X-Content-Type-Optionsi: "nosniff"
+          Referrer-Policy: "same-origin"
+  routers:
+    master:
+      rule: "Host(\`$PILER_HOSTNAME\`)"
+      service: www
+      middlewares:
+        - "piler_headers"
+      tls:
+        certResolver: le
+  services:
+    www:
+      loadBalancer:
+        servers:
+        - url: "http://127.0.0.1:80/"
+TRAEFIK
+}
+
+
+install_prerequisites
+
+start_mysql
+
+install_piler
+
+create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
+create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf
+
+fix_config_site_php
+
+setup_traefik
+
+add_systemd_services
+
+fix_configs
+init_db
+
+[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler
+
+systemctl start pilersearch
+systemctl start piler
+systemctl start piler-smtp
+systemctl start traefik

contrib/proxmox-lxc/README

@@ -0,0 +1,9 @@
+You nedd proxmox with lxc
+
+Fill out conf file first and provide public key
+
+It automatically
+- downloads the latest debian template
+- provides a container
+- installs piler until the post installation step
+- gives a self signed certificate

contrib/proxmox-lxc/create_lxc.sh

@@ -0,0 +1,195 @@
+#!/bin/bash
+
+# This script wil create and fire up a standard debian buster lxc container on your proxmox pve.
+# The Script will look for the next free lxc number and take the next free and use it. So take
+# care that behind your last number is place for it. 
+
+#### SOME VARIABLES TO ADJUST ####
+
+# Storage with templates
+LXC_TMP="local"
+
+# Size and pool of rootfs / in GB
+SIZ_ROT="100"
+S_ROT_P="local-zfs"
+
+# Size and pool of Filestorage in GB will mounted to /share
+SIZ_FIL="100"
+S_FIL_P="local-zfs"
+
+#Weather or not (1 and 0) the container will createt as unpriviliged LXC
+LXC_UNP="1"
+
+# Size of the RAM assigned to the LXC
+LXC_MEM="1024"
+
+# Size of the SWAP assigned to the LXC
+LXC_SWA="1024"
+
+# The hostname (eq. zamba1 or mailpiler1)
+LXC_HOST="zamba"
+
+# The domainname (searchdomain /etc/resolf.conf & hosts)
+LXC_SDN="zmb.local"
+
+# IP-address and subnet
+LXC_IP="10.10.80.20/24"
+
+# Gateway
+LXC_GW="10.10.80.10"
+
+# DNS-server and here shoud be your AD-DC
+LXC_DNS="10.10.80.10"
+
+# Networkbridge for this machine
+LXC_BRD="vmbr80"
+
+# root password - take care to delete from this file
+LXC_PWD="MYPASSWD"
+
+LXC_KEY="ssh-rsa xxxxxxxx"
+
+############### Zamba-Server-Section ###############
+
+# Domain Entries to samba/smb.conf. Will be also uses for samba domain-provisioning when zmb-pdc will choosen.
+ZMB_REA="ZMB.LOCAL"
+ZMB_DOM="ZMB"
+
+# THE Domain-Admin and passwd for zamba-install
+ZMB_ADA="Administrator"
+ZMB_APW="MYPASSWORD"
+
+############### Mailpiler-Section ###############
+
+# The FQDN vor the Hostname. This must be exactly the same like the LXC_HOST / LXC_SDN at section above.
+PILER_DOM="piler.zmb.rocks"
+SMARTHOST="10.10.80.20"
+PILER_VER="1.3.10"
+SPHINX_VER="3.3.1"
+PHP_VER="7.4"
+
+############### Matrix-Section ###############
+
+# The FQDN vor the Hostname. This should be the same like the LXC_HOST / LXC_SDN at section above.
+MRX_DOM="matrix.zmb.rocks"
+ELE_DOM="element.zmb.rocks"
+ELE_VER="v1.7.21"
+JIT_DOM="meet.zmb.rocks"
+
+#################################
+
+# CHeck is the newest template available, else download it.
+
+DEB_LOC=$(pveam list $LXC_TMP | grep debian-10-standard | cut -d'_' -f2)
+
+DEB_REP=$(pveam available --section system | grep debian-10-standard | cut -d'_' -f2)
+
+if [[ $DEB_LOC == $DEB_REP ]];
+then
+  echo "Newest Version of Debian 10 Standard $DEP_REP exists.";
+else
+  echo "Will now download newest Debian 10 Standard $DEP_REP.";
+  pveam download $LXC_TMP debian-10-standard_$DEB_REP\_amd64.tar.gz
+fi
+
+# Get next free LXC-number
+LXC_LST=$( lxc-ls | egrep -o '.{1,5}$' )
+LXC_CHK=$((LXC_LST+1));
+
+if  [ $LXC_CHK -lt 100 ] || [ -f /etc/pve/qemu-server/$LXC_CHK.conf ]; then
+  LXC_NBR=$(pvesh get /cluster/nextid);
+else
+  LXC_NBR=$LXC_CHK;
+fi
+
+echo "Will now create LXC Container $LXC_NBR!";
+
+# Create the container
+pct create $LXC_NBR -unprivileged $LXC_UNP $LXC_TMP:vztmpl/debian-10-standard_$DEB_REP\_amd64.tar.gz -rootfs $S_ROT_P:$SIZ_ROT;
+sleep 2;
+
+pct set $LXC_NBR -memory $LXC_MEM -swap $LXC_SWA -hostname $LXC_HOST \-nameserver $LXC_DNS -searchdomain $LXC_SDN -onboot 1 -timezone Europe/Berlin -net0 name=eth0,bridge=$LXC_BRD,firewall=1,gw=$LXC_GW,ip=$LXC_IP,type=veth;
+sleep 2;
+
+PS3="Select the Server-Function: "
+
+select opt in just_lxc zmb-standalone zmb-member zmb-pdc mailpiler matrix quit; do
+  case $opt in
+    just_lxc)
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      echo "Should be ready!"
+      break
+      ;;
+    zmb-standalone)
+      break
+      ;;
+    zmb-member)
+      echo "Make some additions to LXC for AD-Member-Server!"
+      pct set $LXC_NBR -mp0 $S_FIL_P:$SIZ_FIL,mp=/tank
+      sleep 2;
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      cp /root/zmb_mem.orig /root/zmb_mem.sh
+      sed -i "s|#ZMB_VAR|#ZMB_VAR\nZMB_REA='$ZMB_REA'\nZMB_DOM='$ZMB_DOM'\nZMB_ADA='$ZMB_ADA'\nZMB_APW='$ZMB_APW'|" /root/zmb_mem.sh
+      pct push $LXC_NBR /root/zmb_mem.sh /root/zmb_mem.sh
+      echo "Install zamba as AD-Member-Server!"
+      lxc-attach -n$LXC_NBR bash /root/zmb_mem.sh
+      break
+      ;;
+    zmb-pdc)
+      break
+      ;;
+    mailpiler)
+      echo "Make some additions to LXC for Mailpiler!"
+      pct set $LXC_NBR -features nesting=1
+      sleep 2;
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      cp /root/mailpiler.orig /root/mailpiler.sh
+      sed -i "s|#PILER_VAR|#PILER_VAR\nPILER_DOM='$PILER_DOM'\nSMARTHOST='$SMARTHOST'\nPILER_VER='$PILER_VER'\nSPHINX_VER='$SPHINX_VER'\nPHP_VER='$PHP_VER'|" /root/mailpiler.sh
+      pct push $LXC_NBR /root/mailpiler.sh /root/mailpiler.sh
+      echo "Install Mailpiler mailarchiv!"
+      lxc-attach -n$LXC_NBR bash mailpiler.sh
+      break
+      ;;
+    matrix)
+      echo "Make some additions to LXC for Matrix!"
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      cp /root/matrix.orig /root/matrix.sh
+      sed -i "s|#MATRIX_VAR|#Matrix_VAR\nMRX_DOM='$MRX_DOM'\nELE_DOM='$ELE_DOM'\nELE_VER='$ELE_VER'\nJIT_DOM='$JIT_DOM'|" /root/matrix.sh
+      pct push $LXC_NBR /root/matrix.sh /root/matrix.sh
+      echo "Install Matrix Chatserver!"
+      lxc-attach -n$LXC_NBR bash matrix.sh
+      break
+      ;;
+    quit)
+      break
+      ;;
+    *)
+      echo "Invalid option!"
+      ;;
+    esac
+done
+

contrib/proxmox-lxc/mailpiler.orig

@@ -0,0 +1,179 @@
+#!/bin/bash
+
+#Variables will be filled in from the mainscript:
+
+#PILER_VAR
+
+
+HOSTNAME=$(hostname -f)
+
+echo "Ensure your Hostname is set to your Piler FQDN!"
+
+echo $HOSTNAME
+
+if 
+    [ "$HOSTNAME" != "$PILER_DOM" ]
+then
+        echo "Hostname doesn't match Piler_Domain! Check install.sh, /etc/hosts, /etc/hostname." && exit
+else
+        echo "Hostname matches PILER_DOMAIN, so starting installation."
+fi
+
+apt install -y gpg apt-transport-https lsb-release
+
+wget -q https://packages.sury.org/php/apt.gpg -O- | apt-key add -
+echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" | tee /etc/apt/sources.list.d/php.list
+
+apt update && apt full-upgrade -y
+
+apt install -y mc sysstat build-essential libwrap0-dev libpst-dev tnef libytnef0-dev unrtf catdoc libtre-dev tre-agrep poppler-utils libzip-dev unixodbc libpq5 software-properties-common libpoppler-dev openssl libssl-dev memcached telnet nginx mariadb-server default-libmysqlclient-dev python-mysqldb gcc libwrap0 libzip4 latex2rtf latex2html catdoc tnef libpq5 zipcmp zipmerge ziptool libsodium23
+
+apt update && apt install -y php$PHP_VER-{fpm,common,ldap,mysql,cli,opcache,phpdbg,gd,memcache,json,readline,zip}
+
+apt purge -y postfix
+
+cat > /etc/mysql/conf.d/mailpiler.conf <<EOF
+innodb_buffer_pool_size=256M
+innodb_flush_log_at_trx_commit=1
+innodb_log_buffer_size=64M
+innodb_log_file_size=16M
+query_cache_size=0
+query_cache_type=0
+query_cache_limit=2M
+EOF
+
+systemctl restart mariadb
+
+cd /tmp
+wget https://download.mailpiler.com/generic-local/sphinx-$SPHINX_VER-bin.tar.gz
+tar -xvzf sphinx-$SPHINX_VER-bin.tar.gz -C /
+
+groupadd piler
+useradd -g piler -m -s /bin/bash -d /var/piler piler
+usermod -L piler
+chmod 755 /var/piler
+
+wget https://bitbucket.org/jsuto/piler/downloads/piler-$PILER_VER.tar.gz
+tar -xvzf piler-$PILER_VER.tar.gz
+cd piler-$PILER_VER/
+./configure --localstatedir=/var --with-database=mysql --enable-tcpwrappers --enable-memcached
+make
+make install
+ldconfig
+
+cp util/postinstall.sh util/postinstall.sh.bak
+sed -i "s/   SMARTHOST=.*/   SMARTHOST="\"$SMARTHOST\""/" util/postinstall.sh
+sed -i 's/   WWWGROUP=.*/   WWWGROUP="www-data"/' util/postinstall.sh
+
+make postinstall
+
+cp /usr/local/etc/piler/piler.conf /usr/local/etc/piler/piler.conf.bak
+sed -i "s/hostid=.*/hostid=$PILER_DOM/" /usr/local/etc/piler/piler.conf
+sed -i "s/update_counters_to_memcached=.*/update_counters_to_memcached=1/" /usr/local/etc/piler/piler.conf
+
+su piler -c "indexer --all --config /usr/local/etc/piler/sphinx.conf"
+
+/etc/init.d/rc.piler start
+/etc/init.d/rc.searchd start
+
+update-rc.d rc.piler defaults
+update-rc.d rc.searchd defaults
+
+mkdir -p /etc/nginx/ssl
+openssl req -x509 -nodes -days 3650 -newkey rsa:4096 -keyout /etc/nginx/ssl/piler.key -out /etc/nginx/ssl/piler.crt -subj "/CN=$PILER_DOM" -addext "subjectAltName=DNS:$PILER_DOM"
+
+cd /etc/nginx/sites-available
+cp /tmp/piler-$PILER_VER/contrib/webserver/piler-nginx.conf /etc/nginx/sites-available/
+ln -s /etc/nginx/sites-available/piler-nginx.conf /etc/nginx/sites-enabled/piler-nginx.conf
+
+sed -i "s|PILER_HOST|$PILER_DOM|g" /etc/nginx/sites-available/piler-nginx.conf
+sed -i "s|/var/run/php/php7.4-fpm.sock|/var/run/php/php$PHP_VER-fpm.sock|g" /etc/nginx/sites-available/piler-nginx.conf
+
+sed -i "/server_name.*/a \\
+        listen 443 ssl http2;\n\n\
+        ssl_certificate /etc/nginx/ssl/piler.crt;\n\
+        ssl_certificate_key /etc/nginx/ssl/piler.key;\n\n\
+        ssl_session_timeout 1d;\n\
+        ssl_session_cache shared:SSL:15m;\n\
+        ssl_session_tickets off;\n\n\
+        # modern configuration of Mozilla SSL configurator. Tweak to your needs.\n\
+        ssl_protocols TLSv1.2 TLSv1.3;\n\
+        ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;\n\
+        ssl_prefer_server_ciphers off;\n\n\
+        add_header X-Frame-Options SAMEORIGIN;\n\
+        add_header X-Content-Type-Options nosniff;" /etc/nginx/sites-available/piler-nginx.conf
+
+sed -i "/^server {.*/i\
+server {\n\
+        listen 80;\n\
+        server_name $PILER_DOM;\n\
+        server_tokens off;\n\
+        # HTTP to HTTPS redirect.\n\
+        return 301 https://$PILER_DOM;\n\
+}" /etc/nginx/sites-available/piler-nginx.conf
+
+cp /usr/local/etc/piler/config-site.php /usr/local/etc/piler/config-site.php.bak
+sed -i "s|\$config\['SITE_URL'\] = .*|\$config\['SITE_URL'\] = 'https://$PILER_DOM/';|" /usr/local/etc/piler/config-site.php
+cat >> /usr/local/etc/piler/config-site.php <<EOF
+
+// CUSTOM
+\$config['PROVIDED_BY'] = '$PILER_DOM';
+\$config['SUPPORT_LINK'] = 'https://$PILER_DOM';
+\$config['COMPATIBILITY'] = '';
+
+// fancy features.
+\$config['ENABLE_INSTANT_SEARCH'] = 1;
+\$config['ENABLE_TABLE_RESIZE'] = 1;
+
+\$config['ENABLE_DELETE'] = 1;
+\$config['ENABLE_ON_THE_FLY_VERIFICATION'] = 1;
+
+// general settings.
+\$config['TIMEZONE'] = 'Europe/Berlin';
+
+// authentication
+// Enable authentication against an imap server
+//\$config['ENABLE_IMAP_AUTH'] = 1;
+//\$config['RESTORE_OVER_IMAP'] = 1;
+//\$config['IMAP_RESTORE_FOLDER_INBOX'] = 'INBOX';
+//\$config['IMAP_RESTORE_FOLDER_SENT'] = 'Sent';
+//\$config['IMAP_HOST'] = '$SMARTHOST';
+//\$config['IMAP_PORT'] =  993;
+//\$config['IMAP_SSL'] = true;
+
+// authentication against an ldap directory (disabled by default)
+//\$config['ENABLE_LDAP_AUTH'] = 1;
+//\$config['LDAP_HOST'] = '$SMARTHOST';
+//\$config['LDAP_PORT'] = 389;
+//\$config['LDAP_HELPER_DN'] = 'cn=administrator,cn=users,dc=mydomain,dc=local';
+//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
+//\$config['LDAP_MAIL_ATTR'] = 'mail';
+//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
+//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
+//\$config['LDAP_BASE_DN'] = 'ou=Benutzer,dc=krs,dc=local';
+
+// authentication against an Uninvention based ldap directory 
+//\$config['ENABLE_LDAP_AUTH'] = 1;
+//\$config['LDAP_HOST'] = '$SMARTHOST';
+//\$config['LDAP_PORT'] = 7389;
+//\$config['LDAP_HELPER_DN'] = 'uid=ldap-search-user,cn=users,dc=mydomain,dc=local';
+//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
+//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
+//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
+//\$config['LDAP_BASE_DN'] = 'cn=users,dc=mydomain,dc=local';
+//\$config['LDAP_MAIL_ATTR'] = 'mailPrimaryAddress';
+//\$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'person';
+//\$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'person';
+//\$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mailAlternativeAddress';
+
+// special settings.
+\$config['MEMCACHED_ENABLED'] = 1;
+\$config['SPHINX_STRICT_SCHEMA'] = 1; // required for Sphinx $SPHINX_VER, see https://bitbucket.org/jsuto/piler/issues/1085/sphinx-331.
+EOF
+
+nginx -t && systemctl restart nginx
+
+apt autoremove -y
+apt clean -y
+
+

contrib/proxmox-lxc/matrix.orig

@@ -0,0 +1,150 @@
+#!/bin/bash
+
+#MATRIX_VAR
+
+
+MRX_PKE=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+
+ELE_DBNAME="synapse_db"
+ELE_DBUSER="synapse_user"
+ELE_DBPASS=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+
+apt update && apt full-upgrade -y
+
+apt install -y lsb-release apt-transport-https curl gpg software-properties-common net-tools nginx mc postgresql python3-psycopg2
+
+wget wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/matrix-org.list
+apt update && apt install -y matrix-synapse-py3
+systemctl enable matrix-synapse
+
+ss -tulpen
+
+mkdir /etc/nginx/ssl
+openssl req -x509 -nodes -days 3650 -newkey rsa:4096 -keyout /etc/nginx/ssl/matrix.key -out /etc/nginx/ssl/matrix.crt -subj "/CN=$MRX_DOM" -addext "subjectAltName=DNS:$MRX_DOM"
+
+cat > /etc/nginx/sites-available/$MRX_DOM <<EOF
+# Virtual Host configuration for example.com
+#
+# You can move that to a different file under sites-available/ and symlink that
+# to sites-enabled/ to enable it.
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name $MRX_DOM;
+
+    return 301 https://$MRX_DOM;
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    server_name $MRX_DOM;
+
+    ssl on;
+    ssl_certificate /etc/nginx/ssl/matrix.crt;
+    ssl_certificate_key /etc/nginx/ssl/matrix.key;
+
+    location / {
+      proxy_pass http://127.0.0.1:8008;
+      proxy_set_header X-Forwarded-For \$remote_addr;
+    }
+}
+
+server {
+    listen 8448 ssl;
+    listen [::]:8448 ssl;
+    server_name $MRX_DOM;
+
+    ssl on;
+    ssl_certificate /etc/nginx/ssl/matrix.crt;
+    ssl_certificate_key /etc/nginx/ssl/matrix.key;
+
+    # If you don't wanna serve a site, comment this out
+    root /var/www/$MRX_DOM;
+    index index.html index.htm;
+
+    location / {
+        proxy_pass http://127.0.0.1:8008;
+        proxy_set_header X-Forwarded-For \$remote_addr;
+    }
+}
+
+EOF
+ln -s /etc/nginx/sites-available/$MRX_DOM /etc/nginx/sites-enabled/$MRX_DOM
+
+cat > /etc/nginx/sites-available/$ELE_DOM <<EOF
+# Virtual Host configuration for example.com
+#
+# You can move that to a different file under sites-available/ and symlink that
+# to sites-enabled/ to enable it.
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name $ELE_DOM;
+    return 301 https://$ELE_DOM;
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    server_name $ELE_DOM;
+
+    ssl on;
+    ssl_certificate /etc/nginx/ssl/matrix.crt;
+    ssl_certificate_key /etc/nginx/ssl/matrix.key;
+
+    # If you don't wanna serve a site, comment this out
+    root /var/www/$ELE_DOM/element;
+    index index.html index.htm;
+} 
+
+EOF
+
+ln -s /etc/nginx/sites-available/$ELE_DOM /etc/nginx/sites-enabled/$ELE_DOM
+
+systemctl restart nginx
+
+mkdir /var/www/$ELE_DOM
+cd /var/www/$ELE_DOM
+wget https://packages.riot.im/element-release-key.asc
+gpg --import element-release-key.asc
+
+wget https://github.com/vector-im/element-web/releases/download/$ELE_VER/element-$ELE_VER.tar.gz
+wget https://github.com/vector-im/element-web/releases/download/$ELE_VER/element-$ELE_VER.tar.gz.asc
+gpg --verify element-$ELE_VER.tar.gz.asc
+
+tar -xzvf element-$ELE_VER.tar.gz
+ln -s element-$ELE_VER element
+chown www-data:www-data -R element
+cp ./element/config.sample.json ./element/config.json
+sed -i "s|https://matrix-client.matrix.org|https://$MRX_DOM|" ./element/config.json
+sed -i "s|\"server_name\": \"matrix.org\"|\"server_name\": \"$MRX_DOM\"|" ./element/config.json
+
+su postgres <<EOF
+psql -c "CREATE USER $ELE_DBUSER WITH PASSWORD '$ELE_DBPASS';"
+psql -c "CREATE DATABASE $ELE_DBNAME ENCODING 'UTF8' LC_COLLATE='C' LC_CTYPE='C' template=template0 OWNER $ELE_DBUSER;"
+echo "Postgres User '$ELE_DBUSER' and database '$ELE_DBNAME' created."
+EOF
+
+cd /
+sed -i "s|#registration_shared_secret: <PRIVATE STRING>|registration_shared_secret: \"$MRX_PKE\"|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|#public_baseurl: https://example.com/|public_baseurl: https://$MRX_DOM/|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|#enable_registration: false|enable_registration: true|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|name: sqlite3|name: psycopg2|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|database: /var/lib/matrix-synapse/homeserver.db|database: $ELE_DBNAME\n    user: $ELE_DBUSER\n    password: $ELE_DBPASS\n    host: 127.0.0.1\n    cp_min: 5\n    cp_max: 10|" /etc/matrix-synapse/homeserver.yaml
+
+systemctl restart matrix-synapse
+
+register_new_matrix_user -c /etc/matrix-synapse/homeserver.yaml http://127.0.0.1:8008
+
+#curl https://download.jitsi.org/jitsi-key.gpg.key | sh -c 'gpg --dearmor > /usr/share/keyrings/jitsi-keyring.gpg'
+#echo 'deb [signed-by=/usr/share/keyrings/jitsi-keyring.gpg] https://download.jitsi.org stable/' | tee /etc/apt/sources.list.d/jitsi-stable.list > /dev/null
+
+#apt update
+#apt install -y jitsi-meet
+
+
+

contrib/proxmox-lxc/zmb_mem.orig

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+#ZMB_VAR
+
+
+
+apt update && apt full-upgrade -y
+echo -ne '\n' | apt install -y acl dnsutils mc samba winbind libpam-winbind libnss-winbind krb5-user krb5-config samba-dsdb-modules samba-vfs-modules 
+
+mv /etc/krb5.conf /etc/krb5.conf.bak
+cat > /etc/krb5.conf <<EOF
+[libdefaults]
+	default_realm = $ZMB_REA
+    ticket_lifetime = 600
+	dns_lookup_realm = true
+	dns_lookup_kdc = true
+	renew_lifetime = 7d
+EOF
+
+echo -e "$ZMB_APW" | kinit -V $ZMB_ADA
+klist
+
+mv /etc/samba/smb.conf /etc/samba/smb.conf.bak
+cat > /etc/samba/smb.conf <<EOF
+[global]
+	workgroup = $ZMB_DOM
+	security = ADS
+	realm = $ZMB_REA
+	server string = %h server
+
+	vfs objects = acl_xattr shadow_copy2
+    map acl inherit = Yes
+    store dos attributes = Yes
+	idmap config *:backend = tdb
+	idmap config *:range = 3000000-4000000
+	idmap config *:schema_mode = rfc2307
+
+	winbind refresh tickets = Yes
+	winbind use default domain = Yes
+	winbind separator = /
+	winbind nested groups = yes
+	winbind nss info = rfc2307
+
+	pam password change = Yes
+	passwd chat = *Enter\snew\s*\spassword:* %n\n *Retype\snew\s*\spassword:* %n\n *password\supdated\ssuccessfully* .
+	passwd program = /usr/bin/passwd %u
+
+	template homedir = /home/%U
+	template shell = /bin/bash
+	bind interfaces only = Yes
+	interfaces = lo eth0
+	log file = /var/log/samba/log.%m
+	logging = syslog
+	max log size = 1000
+	panic action = /usr/share/samba/panic-action %d
+
+	load printers = No
+	printcap name = /dev/null
+	printing = bsd
+	disable spoolss = Yes
+
+	allow trusted domains = No
+	dns proxy = No
+	shadow: snapdir = .zfs/snapshot
+	shadow: sort = desc
+	shadow: format = -%Y-%m-%d-%H%M
+	shadow: snapprefix = ^zfs-auto-snap_\(frequent\)\{0,1\}\(hourly\)\{0,1\}\(daily\)\{0,1\}\(monthly\)\{0,1\}
+	shadow: delimiter = -20
+
+[share]
+	comment = Main Share
+	path = /tank/share
+	read only = No
+	create mask = 0660
+	directory mask = 0770
+	inherit acls = Yes
+
+
+
+EOF
+
+systemctl restart smbd
+
+echo -e "$ZMB_APW" | net ads join -U $ZMB_ADA createcomputer=Computers
+sed -i "s|files systemd|files systemd winbind|g" /etc/nsswitch.conf
+sed -i "s|#WINBINDD_OPTS=|WINBINDD_OPTS=|" /etc/default/winbind
+echo -e "session optional        pam_mkhomedir.so skel=/etc/skel umask=077" >> /etc/pam.d/common-session
+
+systemctl restart winbind nmbd
+wbinfo -u
+wbinfo -g
+
+mkdir /tank/share
+chown 'administrator':'domain users' /tank/share
+
+setfacl -Rm u:administrator:rwx,g::-,o::- /tank/share
+setfacl -Rdm u:administrator:rwx,g::-,o::- /tank/share
+
+systemctl restart smbd nmbd winbind
+

contrib/webserver/piler-nginx.conf

@@ -34,7 +34,7 @@ server {
                return 404;
             }
 
-            fastcgi_pass unix:/var/run/php/php7.4-fpm.sock;
+            fastcgi_pass unix:/var/run/php/php8.1-fpm.sock;
             fastcgi_index index.php;
             include fastcgi_params;
         }

docker/Dockerfile

@@ -1,38 +1,40 @@
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 ARG PACKAGE
 
-LABEL description="piler ubuntu focal image" \
+LABEL description="piler ubuntu jammy image" \
       maintainer="Janos SUTO, sj@acts.hu" \
       package="${PACKAGE}"
 
 ENV DEBIAN_FRONTEND="noninteractive" \
-    DISTRO="focal" \
-    DOWNLOAD_URL="https://download.mailpiler.com" \
+    DISTRO="jammy" \
     PILER_USER="piler" \
-    MYSQL_DATABASE="piler" \
-    SPHINX_BIN_TARGZ="sphinx-3.3.1-bin.tar.gz"
+    MYSQL_DATABASE="piler"
 
 COPY ${PACKAGE} /
 
 RUN apt-get update && \
     apt-get -y --no-install-recommends install \
-       wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \
-       php7.4-gd php7.4-curl php7.4-xml php7.4-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip5 \
-       libtre5 cron libmariadb-dev mariadb-client-core-10.3 python3 python3-mysqldb && \
+       wget openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql php8.1-fpm php8.1-zip php8.1-ldap \
+       php8.1-gd php8.1-curl php8.1-xml php8.1-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip4 \
+       libtre5 cron libmariadb-dev mariadb-client-core-10.6 python3 python3-mysqldb ca-certificates curl rsyslog && \
+    wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
+    dpkg -i manticore-repo.noarch.deb && \
+    rm -f manticore-repo.noarch.deb && \
+    apt-get update && \
+    apt-get install -y manticore manticore-columnar-lib && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \
-    wget --no-check-certificate -q -O ${SPHINX_BIN_TARGZ} ${DOWNLOAD_URL}/generic-local/${SPHINX_BIN_TARGZ} && \
-    tar zxvf ${SPHINX_BIN_TARGZ} && \
-    sed -i 's/mail.[iwe].*//' /etc/rsyslog.conf && \
     sed -i '/session    required     pam_loginuid.so/c\#session    required     pam_loginuid.so' /etc/pam.d/cron && \
     dpkg -i ${PACKAGE} && \
+    touch /etc/piler/MANTICORE && \
     ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled && \
-    rm -f ${PACKAGE} ${SPHINX_BIN_TARGZ} /etc/nginx/sites-enabled/default /etc/piler/piler.key /etc/piler/piler.pem /etc/piler/config-site.php && \
+    rm -f ${PACKAGE} /etc/nginx/sites-enabled/default /etc/piler/piler.key /etc/piler/piler.pem /etc/piler/config-site.php && \
     crontab -u $PILER_USER /usr/share/piler/piler.cron
 
 VOLUME ["/etc/piler"]
-VOLUME ["/var/piler"]
+VOLUME ["/var/piler/store"]
+VOLUME ["/var/piler/manticore"]
 
 EXPOSE 25 80 443
 

docker/README

@@ -1,12 +0,0 @@
-How to build
-
-  Pick the latest deb package from Bitbucket download page (https://bitbucket.org/jsuto/piler/downloads/)
-  and use it as the PACKAGE build argument, eg.
-
-  docker build --build-arg PACKAGE=piler_1.3.6~bionic-65cc7eb_amd64.deb -t sutoj/piler .
-
-How to run the image
-
-  Set the PILER_HOST env variable to match your hostname, eg.
-
-  docker run -d --name piler1 -p 25:25 -p 80:80 -e PILER_HOST=archive.yourdomain.com sutoj/piler

docker/README.md

@@ -0,0 +1,16 @@
+## How to run piler
+
+Edit the variables in docker-compose.yaml, then run
+
+```
+docker-compose up -d
+```
+
+## How to build the image for yourself
+
+Pick the latest deb package from Bitbucket download page (https://bitbucket.org/jsuto/piler/downloads/)
+and use it as the PACKAGE build argument, eg.
+
+```
+docker build --build-arg PACKAGE=piler_1.4.1-jammy-c860ca67_amd64.deb -t piler:1.4.1 .
+```

docker/build.sh

@@ -4,7 +4,7 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-IMAGE_NAME="sutoj/piler:1.3.10"
+IMAGE_NAME="sutoj/piler:1.4.4"
 
 if [[ $# -ne 1 ]]; then echo "ERROR: missing package name" 1>&2; exit 1; fi
 

docker/docker-compose.yaml

@@ -1,8 +1,8 @@
 version: "3"
 services:
-
   mysql:
-    image: mariadb:10.4
+    image: mariadb:11.1.2
+    container_name: mysql
     restart: unless-stopped
     cap_drop:
       - ALL
@@ -11,33 +11,31 @@ services:
       - setuid
       - setgid
     environment:
-      - MYSQL_ROOT_PASSWORD=abcde123
+      - MYSQL_DATABASE=piler
+      - MYSQL_USER=piler
+      - MYSQL_PASSWORD=piler123
+      - MYSQL_RANDOM_ROOT_PASSWORD=yes
+    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
     volumes:
       - db_data:/var/lib/mysql
-
-  memcached:
-    image: memcached:latest
-    restart: unless-stopped
-    cap_drop:
-      - ALL
-    command: -m 64
-
   piler:
-    image: sutoj/piler:1.3.10
+    image: sutoj/piler:1.4.4
+    container_name: piler
     init: true
     environment:
-      - MYSQL_PILER_PASSWORD=piler123
-      - MYSQL_ROOT_PASSWORD=abcde123
       - MYSQL_HOSTNAME=mysql
-      - PILER_HOSTNAME=archive.yourdomain.com
-      - MEMCACHED_HOST=memcached
+      - MYSQL_DATABASE=piler
+      - MYSQL_USER=piler
+      - MYSQL_PASSWORD=piler123
+      - PILER_HOSTNAME=cust1.acts.hu
+      - RT=1
     ports:
       - "25:25"
       - "80:80"
-      - "443:443"
     volumes:
       - piler_etc:/etc/piler
-      - piler_var:/var/piler
+      - piler_manticore:/var/piler/manticore
+      - piler_store:/var/piler/store
     healthcheck:
       test: curl -s smtp://localhost/
       interval: "60s"
@@ -52,10 +50,10 @@ services:
           memory: 512M
 
     depends_on:
-      - "memcached"
       - "mysql"
 
 volumes:
   db_data: {}
   piler_etc: {}
-  piler_var: {}
+  piler_manticore: {}
+  piler_store: {}

docker/start.sh

@@ -10,11 +10,10 @@ PILER_CONF="${CONFIG_DIR}/piler.conf"
 PILER_KEY="${CONFIG_DIR}/piler.key"
 PILER_PEM="${CONFIG_DIR}/piler.pem"
 PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf"
-SPHINX_CONF="${CONFIG_DIR}/sphinx.conf"
+SPHINX_CONF="${CONFIG_DIR}/manticore.conf"
 CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php"
 PILER_MY_CNF="${CONFIG_DIR}/.my.cnf"
-ROOT_MY_CNF="/root/.my.cnf"
-
+RT="${RT:-0}"
 
 error() {
    echo "ERROR:" "$*" 1>&2
@@ -30,8 +29,9 @@ log() {
 pre_flight_check() {
    [[ -v PILER_HOSTNAME ]] || error "Missing PILER_HOSTNAME env variable"
    [[ -v MYSQL_HOSTNAME ]] || error "Missing MYSQL_HOSTNAME env variable"
-   [[ -v MYSQL_PILER_PASSWORD ]] || error "Missing MYSQL_PILER_PASSWORD env variable"
-   [[ -v MYSQL_ROOT_PASSWORD ]] || error "Missing MYSQL_ROOT_PASSWORD env variable"
+   [[ -v MYSQL_DATABASE ]] || error "Missing MYSQL_DATABASE env variable"
+   [[ -v MYSQL_USER ]]     || error "Missing MYSQL_USER env variable"
+   [[ -v MYSQL_PASSWORD ]] || error "Missing MYSQL_PASSWORD env variable"
 }
 
 
@@ -87,7 +87,9 @@ fix_configs() {
       log "Writing ${PILER_CONF}"
 
       sed \
-         -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" \
+         -e "s/mysqluser=.*/mysqluser=${MYSQL_USER}/g" \
+         -e "s/mysqldb=.*/mysqldb=${MYSQL_DATABASE}/g" \
+         -e "s/verystrongpassword/${MYSQL_PASSWORD}/g" \
          -e "s/hostid=.*/hostid=${PILER_HOSTNAME}/g" \
          -e "s/tls_enable=.*/tls_enable=1/g" \
          -e "s/mysqlsocket=.*/mysqlsocket=/g" "${PILER_CONF}.dist" > "$PILER_CONF"
@@ -111,7 +113,9 @@ fix_configs() {
          echo "\$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/bin/pileraget';"
          echo "\$config['PILER_BINARY'] = '/usr/sbin/piler';"
          echo "\$config['DB_HOSTNAME'] = '$MYSQL_HOSTNAME';"
-         echo "\$config['DB_PASSWORD'] = '$MYSQL_PILER_PASSWORD';"
+         echo "\$config['DB_DATABASE'] = '$MYSQL_DATABASE';"
+         echo "\$config['DB_USERNAME'] = '$MYSQL_USER';"
+         echo "\$config['DB_PASSWORD'] = '$MYSQL_PASSWORD';"
          echo "\$config['ENABLE_MEMCACHED'] = 1;"
          echo "\$memcached_server = ['memcached', 11211];"
       } >> "$CONFIG_SITE_PHP"
@@ -119,39 +123,50 @@ fix_configs() {
 
    sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \
        -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \
-       -e "s%MYSQL_USERNAME%${PILER_USER}%" \
-       -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%" \
+       -e "s%MYSQL_USERNAME%${MYSQL_USER}%" \
+       -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" \
        -i "$SPHINX_CONF"
+
+   # Fixes for RT index
+
+   if [[ $RT -eq 1 ]]; then
+      sed -i "s/define('RT', 0)/define('RT', 1)/" "$SPHINX_CONF"
+      if ! grep "'RT'" "$CONFIG_SITE_PHP"; then
+         echo "\$config['RT'] = 1;" >> "$CONFIG_SITE_PHP"
+      fi
+
+      if ! grep "'SPHINX_MAIN_INDEX'" "$CONFIG_SITE_PHP"; then
+         echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
+      fi
+
+      sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
+   fi
 }
 
 
 wait_until_mysql_server_is_ready() {
-   while true; do if mysql "--defaults-file=${ROOT_MY_CNF}" <<< "show databases"; then break; fi; log "${MYSQL_HOSTNAME} is not ready"; sleep 5; done
+   while true; do if mysql "--defaults-file=${PILER_MY_CNF}" <<< "show databases"; then break; fi; log "${MYSQL_HOSTNAME} is not ready"; sleep 5; done
 
    log "${MYSQL_HOSTNAME} is ready"
 }
 
 
 init_database() {
-   local db
-   local has_piler_db=0
+   local table
+   local has_metadata_table=0
 
    wait_until_mysql_server_is_ready
 
-   while read -r db; do
-      if [[ "$db" == "$MYSQL_DATABASE" ]]; then has_piler_db=1; fi
-   done < <(mysql "--defaults-file=${ROOT_MY_CNF}" <<< 'show databases')
+   while read -r table; do
+      if [[ "$table" == metadata ]]; then has_metadata_table=1; fi
+   done < <(mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< 'show tables')
 
-   if [[ $has_piler_db -eq 0 ]]; then
-      log "no ${MYSQL_DATABASE} database, creating"
-
-      mysql "--defaults-file=${ROOT_MY_CNF}" <<< "create database ${MYSQL_DATABASE} character set utf8mb4"
-      mysql "--defaults-file=${ROOT_MY_CNF}" <<< "grant all privileges on ${MYSQL_DATABASE}.* to ${PILER_USER} identified by '${MYSQL_PILER_PASSWORD}'"
-      mysql "--defaults-file=${ROOT_MY_CNF}" <<< "flush privileges"
+   if [[ $has_metadata_table -eq 0 ]]; then
+      log "no metadata table, creating tables"
 
       mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" < /usr/share/piler/db-mysql.sql
    else
-      log "${MYSQL_DATABASE} db exists"
+      log "metadata table exists"
    fi
 
    if [[ -v ADMIN_USER_PASSWORD_HASH ]]; then
@@ -162,26 +177,25 @@ init_database() {
 
 create_my_cnf_files() {
    printf "[client]\nhost = %s\nuser = %s\npassword = %s\n[mysqldump]\nhost = %s\nuser = %s\npassword = %s\n" \
-      "$MYSQL_HOSTNAME" "$PILER_USER" "$MYSQL_PILER_PASSWORD" "$MYSQL_HOSTNAME" "$PILER_USER" "$MYSQL_PILER_PASSWORD" \
+      "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" \
       > "$PILER_MY_CNF"
-   printf "[client]\nhost = %s\nuser = root\npassword = %s\n" "$MYSQL_HOSTNAME" "$MYSQL_ROOT_PASSWORD" > "$ROOT_MY_CNF"
 
    give_it_to_piler "$PILER_MY_CNF"
 }
 
 
 start_services() {
-   service rsyslog start
    service cron start
-   service php7.4-fpm start
+   service php8.1-fpm start
    service nginx start
+   rsyslogd
 }
 
 
 start_piler() {
-   if [[ ! -f "${VOLUME_DIR}/sphinx/main1.spp" ]]; then
+   if [[ $RT -eq 0 && ! -f "${VOLUME_DIR}/manticore/main1.spp" ]]; then
       log "main1.spp does not exist, creating index files"
-      su -c "indexer --all --config ${SPHINX_CONF}" piler
+      su -c "indexer --all --config ${SPHINX_CONF}" "$PILER_USER"
    fi
 
    # No pid file should exist for piler
@@ -199,4 +213,4 @@ init_database
 start_services
 start_piler
 
-while true; do sleep 3600; done
+sleep infinity

etc/Makefile.in

@@ -32,7 +32,9 @@ all:
 install:
 	$(INSTALL) -m 0640 -g $(RUNNING_GROUP) $(srcdir)/piler.conf $(DESTDIR)$(sysconfdir)/piler/piler.conf.dist
 	sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/sphinx.conf.in > sphinx.conf.dist
+	sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/manticore.conf.in > manticore.conf.dist
 	$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/sphinx.conf.dist $(DESTDIR)$(sysconfdir)/piler/sphinx.conf.dist
+	$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/manticore.conf.dist $(DESTDIR)$(sysconfdir)/piler/manticore.conf.dist
 	$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/config-site.dist.php $(DESTDIR)$(sysconfdir)/piler/config-site.dist.php
 	$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/cron.jobs.in $(DESTDIR)$(datarootdir)/piler/piler.cron
 	sed -i -e 's%LOCALSTATEDIR%$(localstatedir)%g' $(DESTDIR)$(datarootdir)/piler/piler.cron

etc/cron.jobs.in

@@ -2,13 +2,13 @@
 5,35 * * * * LIBEXECDIR/piler/indexer.delta.sh
 30   2 * * * LIBEXECDIR/piler/indexer.main.sh
 40 3 * * * LIBEXECDIR/piler/purge.sh
-*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet tag1 --rotate
-*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet note1 --rotate
+*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet tag1 --rotate
+*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet note1 --rotate
 */5 * * * *  /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \;
 */5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error
-2 0 * * * LIBEXECDIR/piler/pilerpurge.py
 
-### optional: populate accouting data
+### optional
 ###30 6 * * * /usr/bin/php LIBEXECDIR/piler/generate_stats.php --webui LOCALSTATEDIR/piler/www >/dev/null
+###*/5 * * * * LIBEXECDIR/piler/import.sh
 
 ### PILEREND

etc/example.conf

@@ -50,8 +50,8 @@ max_requests_per_child=10000
 
 ; SMTP HELO identification string
 ; this should be the FQDN part of the email address
-; where you copy emails, eg. archive@piler.yourdomain.com -> piler.yourdomain.com
-hostid=piler.yourdomain.com
+; where you copy emails, eg. archive@piler.example.com -> piler.example.com
+hostid=piler.example.com
 
 ; whether to process rcpt to addresses and add them to rcpt table (1) or not (0)
 process_rcpt_to_addresses=0
@@ -107,6 +107,13 @@ pemfile=
 cipher_list=ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS
 
 
+; set the minimum TLS protocol version for piler-smtp daemon
+;
+; Valid values: TLSv1, TLSv1.1, TLSv1.2, TLSv1.3
+; TLSv1 and TLSv1.1 are not recommended for security reasons
+tls_min_version=TLSv1.2
+
+
 ; piler's own header to indicate previously archived messages
 piler_header_field=X-piler-id:
 
@@ -212,6 +219,14 @@ mysqldb=piler
 mysql_connect_timeout=2
 
 
+;
+; manticore stuff
+;
+sphxhost=127.0.0.1
+sphxport=9306
+; whether to enable (1) or not (0) the rt index. If so, then piler
+; writes the index data directly to manticore's sql API
+rtindex=0
 
 ; if you want to change the 'sent' time as you archive the message
 ; set this in seconds. This can be a postive or negative value.
@@ -231,6 +246,15 @@ mmap_dedup_test=0
 ; clients via an IP-address list is not feasible.
 security_header=
 
+; By default the archive accepts any envelope recipient addresses.
+; If your archive's port 25 is wide open to the Internet (which it
+; shouldn't be, then spammers may find it, and fill it with spam.
+;
+; By setting this variable you may restrict the envelope address
+; to a single email address, eg. some-random-address-12345@archive.example.com
+; Then the archive will reject any other envelope recipients
+archive_address=
+
 ; whether to enable (1) or not (0) an smtp access list similar to
 ; postfix's postscreen. Valid actions in the acl file are "permit"
 ; and "reject" (without quotes). See smtp.acl.example for more.
@@ -239,3 +263,12 @@ security_header=
 ; rules. In other words if you decide to use the acl file, then
 ; everyone is not explicitly permitted is denied.
 smtp_access_list=0
+
+; max message size in bytes
+; piler-smtp will reject any message that's bigger than this number
+max_message_size=50000000
+
+; max memory in bytes piler-smtp uses for buffering messages
+; when this limit is exceeded, no new emails will be accepted
+; until the used memory decreases below this level
+max_smtp_memory=500000000

etc/manticore.conf.in

@@ -0,0 +1,268 @@
+#!/usr/bin/php
+<?php
+
+define('LOCALSTATEDIR', '/var');
+define('NGRAM_CONFIG', "        #ngram_len               = 1\n        #ngram_chars             = U+3000..U+2FA1F\n");
+
+# See http://sphinxsearch.com/wiki/doku.php?id=charset_tables for more on the charset_table settings
+# The following settings contains English and some Latin extras
+define('CHARSET_TABLE', "0..9, english, _, \
+                                  U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
+                                  U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
+                                  U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
+                                  U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171\n");
+
+define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
+
+define('RT', 0);
+
+?>
+
+#
+# minimal manticore configuration suited to piler
+#
+
+<?php if(RT == 0) { ?>
+
+source base
+{
+   type = mysql
+   sql_host = MYSQL_HOSTNAME
+   sql_db = MYSQL_DATABASE
+   sql_user = MYSQL_USERNAME
+   sql_pass = MYSQL_PASSWORD
+
+   sql_attr_uint = size
+   sql_attr_uint = sent
+   sql_attr_uint = attachments
+}
+
+source delta : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query_pre  = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index
+   sql_query_post_index  = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
+
+   sql_query_killlist = SELECT `id` FROM `metadata` WHERE `deleted`=1
+}
+
+source main1 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source main2 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source main3 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source main4 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source dailydelta : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+index main1
+{
+        source                  = main1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index main2
+{
+        source                  = main2
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main2
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index main3
+{
+        source                  = main3
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main3
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index main4
+{
+        source                  = main4
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main4
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index dailydelta1
+{
+        source                  = dailydelta
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/dailydelta1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+        killlist_target         = main1:kl, main2:kl, main3:kl, main4:kl, dailydelta1:kl
+<?php print NGRAM_CONFIG; ?>
+}
+
+index delta1
+{
+        source                  = delta
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/delta1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+source tag : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query  = SELECT `_id`, `id` AS iid, `uid`, `tag` FROM `tag`
+
+   sql_attr_uint = iid
+   sql_attr_uint = uid
+}
+
+source note : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query  = SELECT `_id`, `id` AS iid, `uid`, `note` FROM `note`
+
+   sql_attr_uint = iid
+   sql_attr_uint = uid
+}
+
+index tag1
+{
+        source                  = tag
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/tag1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+        stored_fields           =
+<?php print NGRAM_CONFIG; ?>
+}
+
+
+index note1
+{
+        source                  = note
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/note1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+        stored_fields           =
+<?php print NGRAM_CONFIG; ?>
+}
+
+<?php } else { ?>
+
+index piler1
+{
+    type = rt
+    path = /var/piler/manticore/piler1
+    rt_mem_limit = 512M
+    stored_fields =
+    min_word_len = 1
+    min_prefix_len = 5
+    charset_table  = <?php print CHARSET_TABLE; ?>
+    # See https://manual.manticoresearch.com/Creating_an_index/Data_types#Row-wise-and-columnar-attribute-storages
+    # if you want to enable columnar storage
+    # columnar_attrs = *
+    rt_field = sender
+    rt_field = rcpt
+    rt_field = senderdomain
+    rt_field = rcptdomain
+    rt_field = subject
+    rt_field = body
+    rt_field = attachment_types
+    rt_attr_bigint = arrived
+    rt_attr_bigint = sent
+    rt_attr_uint = size
+    rt_attr_uint = direction
+    rt_attr_uint = folder
+    rt_attr_uint = attachments
+}
+
+index tag1
+{
+    type = rt
+    path = /var/piler/manticore/tag1
+    rt_mem_limit = 16M
+    stored_fields = tag
+    min_word_len = 2
+    min_prefix_len = 5
+    charset_table  = <?php print CHARSET_TABLE; ?>
+    rt_field = tag
+    rt_attr_bigint = mid
+    rt_attr_uint = uid
+}
+
+index note1
+{
+    type = rt
+    path = /var/piler/manticore/note1
+    rt_mem_limit = 16M
+    stored_fields = note
+    min_word_len = 2
+    min_prefix_len = 5
+    charset_table  = <?php print CHARSET_TABLE; ?>
+    rt_field = note
+    rt_attr_bigint = mid
+    rt_attr_uint = uid
+}
+
+<?php } ?>
+
+
+searchd
+{
+        listen                  = 127.0.0.1:9312
+        listen                  = 127.0.0.1:9306:mysql
+        listen                  = 127.0.0.1:9307:mysql_readonly
+        log                     = /var/piler/manticore/manticore.log
+        binlog_max_log_size     = 256M
+        binlog_path             = /var/piler/manticore
+        binlog_flush            = 2
+        query_log               = /var/piler/manticore/query.log
+        network_timeout         = 5
+        pid_file                = /var/run/piler/searchd.pid
+        seamless_rotate         = 1
+        preopen_tables          = 1
+        unlink_old              = 1
+        thread_stack            = 512k
+<?php if(RT) { ?>
+        rt_flush_period         = 300
+<?php } ?>
+
+}

etc/sphinx.conf.in

@@ -19,7 +19,7 @@ define('SPHINX_CHARSET_TABLE', "0..9, english, _, \
 // NB: The SPHINX_STRICT_SCHEMA in sphinx.conf MUST BE THE SAME as in config.php (or in config-site.php)
 //
 define('SPHINX_STRICT_SCHEMA', 1);
-
+define('RT', 0);
 
 if(SPHINX_STRICT_SCHEMA) {
    define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
@@ -46,6 +46,8 @@ source base
    sql_attr_uint = attachments
 }
 
+<?php if(!RT) { ?>
+
 source delta : base
 {
    sql_query_pre = SET NAMES utf8mb4
@@ -89,6 +91,7 @@ source dailydelta : base
    sql_query_pre = SET NAMES utf8mb4
    sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
 }
+<?php } ?>
 
 source tag : base
 {
@@ -110,7 +113,7 @@ source note : base
 
 }
 
-
+<?php if(!RT) { ?>
 index main1
 {
         source                  = main1
@@ -198,6 +201,7 @@ index delta1
 <?php } ?>
 }
 
+<?php } ?>
 
 index tag1
 {
@@ -228,6 +232,27 @@ index note1
 <?php print NGRAM_CONFIG; ?>
 }
 
+<?php if(RT) { ?>
+index piler1
+{
+        type = rt
+        path = /var/piler/sphinx/piler1
+        rt_mem_limit = 128M
+        rt_field = sender
+        rt_field = rcpt
+        rt_field = senderdomain
+        rt_field = rcptdomain
+        rt_field = subject
+        rt_field = body
+        rt_field = attachment_types
+        rt_attr_bigint = arrived
+        rt_attr_bigint = sent
+        rt_attr_uint = size
+        rt_attr_uint = direction
+        rt_attr_uint = folder
+        rt_attr_uint = attachments
+}
+<?php } ?>
 
 indexer
 {
@@ -240,7 +265,12 @@ searchd
         listen                  = 127.0.0.1:9312
         listen                  = 127.0.0.1:9306:mysql41
         log                     = /dev/null
-	binlog_path             = 
+        binlog_path             = /var/piler/sphinx
+        binlog_max_log_size     = 16M
+        binlog_flush            = 2
+<?php if(RT) { ?>
+        rt_flush_period         = 300
+<?php } ?>
         ##query_log             =
         read_timeout            = 5
         max_children            = 30
@@ -249,5 +279,5 @@ searchd
         preopen_indexes         = 1
         unlink_old              = 1
         thread_stack            = 512k
-	workers			= threads # for RT to work
+        workers                 = thread_pool
 }

init.d/rc.searchd.in

@@ -16,7 +16,13 @@ export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
 
 NAME=searchd
 PID_FILE=/var/run/piler/searchd.pid
-PID_NUMBER=`test -f ${PID_FILE} && cat ${PID_FILE}`
+PID_NUMBER=$(test -f ${PID_FILE} && cat ${PID_FILE})
+CONFIG_FILE=sphinx.conf
+
+if [ -f SYSCONFDIR/piler/MANTICORE ]; then
+   CONFIG_FILE=manticore.conf
+fi
+
 
 start() {
 	echo "starting searchd . . ."
@@ -26,8 +32,8 @@ start() {
 		chown piler:piler /var/run/piler
 	fi
 
-        if [ `id -u` -eq 0 ]; then
-		su piler -c "searchd --config SYSCONFDIR/piler/sphinx.conf"
+        if [ $(id -u) -eq 0 ]; then
+		su piler -c "searchd --config SYSCONFDIR/piler/${CONFIG_FILE}"
 	else
 		searchd
 	fi
@@ -35,7 +41,7 @@ start() {
 
 stop() {
 	echo "stopping searchd"
-	kill ${PID_NUMBER}
+	kill "$PID_NUMBER"
 }
 
 check_status(){

src/archive.c

@@ -16,6 +16,9 @@
 #include <syslog.h>
 #include <openssl/blowfish.h>
 #include <openssl/evp.h>
+#if OPENSSL_VERSION_MAJOR >= 3
+   #include <openssl/provider.h>
+#endif
 #include <zlib.h>
 #include <assert.h>
 #include <piler.h>
@@ -168,9 +171,14 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
    #if OPENSSL_VERSION_NUMBER < 0x10100000L
       EVP_CIPHER_CTX_init(&ctx);
       if(strstr(filename, "/5000")){
-         EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+         rc = EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
       } else {
-         EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+         rc = EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      }
+
+      if(!rc){
+         syslog(LOG_PRIORITY, "ERROR: EVP_DecryptInit_ex()");
+         goto CLEANUP;
       }
 
       blocklen = EVP_CIPHER_CTX_block_size(&ctx);
@@ -180,9 +188,18 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
 
       EVP_CIPHER_CTX_init(ctx);
       if(strstr(filename, "/5000")){
-         EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+         rc = EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
       } else {
-         EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      #if OPENSSL_VERSION_MAJOR >= 3
+         OSSL_PROVIDER_load(NULL, "legacy");
+         OSSL_PROVIDER_load(NULL, "default");
+      #endif
+         rc = EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      }
+
+      if(!rc){
+         syslog(LOG_PRIORITY, "ERROR: EVP_DecryptInit_ex()");
+         goto CLEANUP;
       }
 
       blocklen = EVP_CIPHER_CTX_block_size(ctx);
@@ -244,7 +261,7 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
 
 
 CLEANUP:
-   if(fd != -1) close(fd);
+   if(fd != -1) close(fd); //-V547
    if(s) free(s);
    if(cfg->encrypt_messages == 1)
    #if OPENSSL_VERSION_NUMBER < 0x10100000L
@@ -277,7 +294,7 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct c
       return 1;
    }
 
-   snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
+   snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, *(sdata->ttmpfile+24), *(sdata->ttmpfile+25), *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
    if(stat(filename, &st)){
       snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+RND_STR_LEN-6), *(sdata->ttmpfile+RND_STR_LEN-5), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
@@ -304,7 +321,7 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct c
                buffer = p + strlen(pointer);
 
                if(strlen(ptr_arr[i].piler_id) == RND_STR_LEN){
-                  snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);
+                  snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, ptr_arr[i].piler_id[24], ptr_arr[i].piler_id[25], ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);
 
                #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
                   if(stat(filename, &st)){

src/bdat.c

@@ -26,21 +26,21 @@ void reset_bdat_counters(struct smtp_session *session){
 }
 
 
-void get_bdat_size_to_read(struct smtp_session *session, char *buf){
+void get_bdat_size_to_read(struct smtp_session *session){
    char *p;
 
    session->bdat_bytes_to_read = 0;
 
    session->protocol_state = SMTP_STATE_BDAT;
 
-   p = strcasestr(buf, " LAST");
+   p = strcasestr(session->buf, " LAST");
    if(p){
       *p = '\0';
    }
 
    // determine the size to be read
 
-   p = strchr(buf, ' ');
+   p = strchr(session->buf, ' ');
    if(p){
       session->bdat_bytes_to_read = atoi(p);
       if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "fd=%d: BDAT len=%d", session->net.socket, session->bdat_bytes_to_read);

src/cfg.c

@@ -23,6 +23,11 @@ int int_parser(char *src, int *target){
    return 0;
 };
 
+int uint64_parser(char *src, uint64 *target){
+   *target = strtoull(src, (char**)NULL, 10);
+   return 0;
+}
+
 struct _parse_rule {
    char *name;
    char *type;
@@ -39,7 +44,7 @@ struct _parse_rule {
 
 struct _parse_rule config_parse_rules[] =
 {
-
+   { "archive_address", "string", (void*) string_parser, offsetof(struct config, archive_address), "", MAXVAL-1},
    { "archive_emails_not_having_message_id", "integer", (void*) int_parser, offsetof(struct config, archive_emails_not_having_message_id), "0", sizeof(int)},
    { "archive_only_mydomains", "integer", (void*) int_parser, offsetof(struct config, archive_only_mydomains), "0", sizeof(int)},
    { "backlog", "integer", (void*) int_parser, offsetof(struct config, backlog), "20", sizeof(int)},
@@ -63,7 +68,9 @@ struct _parse_rule config_parse_rules[] =
    { "listen_port", "integer", (void*) int_parser, offsetof(struct config, listen_port), "25", sizeof(int)},
    { "locale", "string", (void*) string_parser, offsetof(struct config, locale), "", MAXVAL-1},
    { "max_connections", "integer", (void*) int_parser, offsetof(struct config, max_connections), "64", sizeof(int)},
+   { "max_message_size", "integer", (void*) int_parser, offsetof(struct config, max_message_size), "50000000", sizeof(int)},
    { "max_requests_per_child", "integer", (void*) int_parser, offsetof(struct config, max_requests_per_child), "10000", sizeof(int)},
+   { "max_smtp_memory", "uint64", (void*) uint64_parser, offsetof(struct config, max_smtp_memory), "500000000", sizeof(uint64)},
    { "memcached_servers", "string", (void*) string_parser, offsetof(struct config, memcached_servers), "127.0.0.1", MAXVAL-1},
    { "memcached_to_db_interval", "integer", (void*) int_parser, offsetof(struct config, memcached_to_db_interval), "900", sizeof(int)},
    { "memcached_ttl", "integer", (void*) int_parser, offsetof(struct config, memcached_ttl), "86400", sizeof(int)},
@@ -84,13 +91,18 @@ struct _parse_rule config_parse_rules[] =
    { "piler_header_field", "string", (void*) string_parser, offsetof(struct config, piler_header_field), "X-piler-id:", MAXVAL-1},
    { "process_rcpt_to_addresses", "integer", (void*) int_parser, offsetof(struct config, process_rcpt_to_addresses), "0", sizeof(int)},
    { "queuedir", "string", (void*) string_parser, offsetof(struct config, queuedir), QUEUE_DIR, MAXVAL-1},
+   { "rtindex", "integer", (void*) int_parser, offsetof(struct config, rtindex), "0", sizeof(int)},
    { "security_header", "string", (void*) string_parser, offsetof(struct config, security_header), "", MAXVAL-1},
    { "server_id", "integer", (void*) int_parser, offsetof(struct config, server_id), "0", sizeof(int)},
+   { "sphxdb", "string", (void*) string_parser, offsetof(struct config, sphxdb), "piler1", MAXVAL-1},
+   { "sphxhost", "string", (void*) string_parser, offsetof(struct config, sphxhost), "127.0.0.1", MAXVAL-1},
+   { "sphxport", "integer", (void*) int_parser, offsetof(struct config, sphxport), "9306", sizeof(int)},
    { "smtp_access_list", "integer", (void*) int_parser, offsetof(struct config, smtp_access_list), "0", sizeof(int)},
    { "smtp_timeout", "integer", (void*) int_parser, offsetof(struct config, smtp_timeout), "60", sizeof(int)},
    { "spam_header_line", "string", (void*) string_parser, offsetof(struct config, spam_header_line), "", MAXVAL-1},
    { "syslog_recipients", "integer", (void*) int_parser, offsetof(struct config, syslog_recipients), "0", sizeof(int)},
    { "tls_enable", "integer", (void*) int_parser, offsetof(struct config, tls_enable), "0", sizeof(int)},
+   { "tls_min_version", "string", (void*) string_parser, offsetof(struct config, tls_min_version), "TLSv1.2", MAXVAL-1},
    { "tweak_sent_time_offset", "integer", (void*) int_parser, offsetof(struct config, tweak_sent_time_offset), "0", sizeof(int)},
    { "update_counters_to_memcached", "integer", (void*) int_parser, offsetof(struct config, update_counters_to_memcached), "0", sizeof(int)},
    { "username", "string", (void*) string_parser, offsetof(struct config, username), "piler", MAXVAL-1},
@@ -146,6 +158,26 @@ int parse_config_file(char *configfile, struct config *target_cfg, struct _parse
 }
 
 
+int get_tls_protocol_number(char *protocol){
+   struct tls_protocol tls_protocols[] = {
+      { "TLSv1", TLS1_VERSION },
+      { "TLSv1.1", TLS1_1_VERSION },
+      { "TLSv1.2", TLS1_2_VERSION },
+   #ifdef TLS1_3_VERSION
+      { "TLSv1.3", TLS1_3_VERSION },
+   #endif
+   };
+
+   for(unsigned int i=0; i<sizeof(tls_protocols)/sizeof(struct tls_protocol); i++){
+      if(!strcmp(protocol, tls_protocols[i].proto)) {
+         return tls_protocols[i].version;
+      }
+   }
+
+   return 0;
+}
+
+
 int load_default_config(struct config *cfg, struct _parse_rule *rules){
    int i=0;
 
@@ -178,6 +210,9 @@ struct config read_config(char *configfile){
 
    cfg.hostid_len = strlen(cfg.hostid);
 
+   // Get the TLS protocol constant from string, ie. TLSv1.3 -> 772
+   cfg.tls_min_version_number = get_tls_protocol_number(cfg.tls_min_version);
+
    return cfg;
 }
 
@@ -189,6 +224,7 @@ struct config read_config(char *configfile){
 void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
    int j;
    float f;
+   uint64 u;
    char *p, buf[MAXVAL];
 
    p = (char*)cfg + rules[i].offset;
@@ -197,6 +233,10 @@ void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
       memcpy((char*)&j, p, sizeof(int));
       printf("%s=%d\n", rules[i].name, j);
    }
+   else if(strcmp(rules[i].type, "uint64") == 0){
+      memcpy((char*)&u, p, sizeof(uint64));
+      printf("%s=%llu\n", rules[i].name, u);
+   }
    else if(strcmp(rules[i].type, "float") == 0){
       memcpy((char*)&f, p, sizeof(float));
       printf("%s=%.4f\n", rules[i].name, f);

src/cfg.h

@@ -29,6 +29,8 @@ struct config {
    int tls_enable;
    char pemfile[MAXVAL];
    char cipher_list[MAXVAL];
+   char tls_min_version[MAXVAL];
+   int tls_min_version_number;
 
    int use_antivirus;
 
@@ -65,6 +67,7 @@ struct config {
    int default_retention_days;
 
    char security_header[MAXVAL];
+   char archive_address[MAXVAL];
 
    // mysql stuff
 
@@ -77,6 +80,12 @@ struct config {
    char mysqldb[MAXVAL];
    int mysql_connect_timeout;
 
+   // manticore stuff
+   char sphxhost[MAXVAL];
+   int sphxport;
+   char sphxdb[MAXVAL];
+   int rtindex;
+
    int update_counters_to_memcached;
    int memcached_to_db_interval;
 
@@ -101,6 +110,9 @@ struct config {
    int debug;
 
    int smtp_access_list;
+
+   int max_message_size;
+   uint64 max_smtp_memory;
 };
 
 

src/config.h

@@ -9,7 +9,9 @@
 #include "piler-config.h"
 #include "params.h"
 
-#define BUILD 998
+typedef unsigned long long uint64;
+
+#define BUILD 1001
 
 #define HOSTID "mailarchiver"
 
@@ -30,6 +32,7 @@
 #define SMALLBUFSIZE 512
 #define BIGBUFSIZE 131072
 #define REALLYBIGBUFSIZE 524288
+#define SMTPBUFSIZE 2048000
 #define TINYBUFSIZE 128
 #define MAXVAL 256
 #define RANDOM_POOL "/dev/urandom"
@@ -39,6 +42,20 @@
 #define IPLEN 16+1
 #define KEYLEN 56
 #define MIN_EMAIL_ADDRESS_LEN 9
+// Sphinx 3.x has an issue with tokens longer than 41 characters.
+//
+// When a regular user executes a query, then his default email address filter
+// causes the query to fail with the below error message, even when the query
+// itself is correct:
+//
+// SELECT id FROM main1,dailydelta1,delta1 WHERE MATCH(' ( (@sender thisisanextremelylongemailaddressyesareallylongoneyeahitolyouXaddressXcom ) | (@rcpt thisisanextremelylongemailaddressyesareallylongoneyeahitolyouXaddressXcom) ) ') ORDER BY `sent` DESC LIMIT 0,20 OPTION max_matches=1000'
+//
+// ERROR 1064 (42000): index dailydelta1,delta1,main1: syntax error, unexpected $end near ' '
+//
+// Note that we use 42, because the parser adds a trailing space to the tokens
+// See https://www.mailpiler.org/wiki/current:sphinx3 and
+// https://bitbucket.org/jsuto/piler/issues/1082/no-sphinx-results-with-long-email for more
+#define MAX_EMAIL_ADDRESS_SPHINX_LEN 42
 
 #define CRLF "\n"
 
@@ -108,9 +125,10 @@
 #define SQL_PREPARED_STMT_GET_FOLDER_ID              "SELECT `id` FROM " SQL_FOLDER_TABLE " WHERE `name`=? AND `parent_id`=?"
 #define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE   "INSERT INTO `" SQL_FOLDER_TABLE "` (`name`, `parent_id`) VALUES(?,?)"
 #define SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE  "UPDATE " SQL_METADATA_TABLE " SET reference=? WHERE message_id=? AND reference=''"
+#define SQL_PREPARED_STMT_GET_METADATA_REFERENCE     "SELECT COUNT(*) AS count FROM " SQL_METADATA_TABLE " WHERE reference=?"
 #define SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS        "SELECT id, type, username, password, server FROM " SQL_IMPORT_TABLE " WHERE started=0 ORDER BY id LIMIT 0,1"
 #define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE      "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `id`) VALUES(?,?)"
-#define SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE        "UPDATE " SQL_IMPORT_TABLE " SET status=?, imported=? WHERE id=?"
+#define SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE        "UPDATE " SQL_IMPORT_TABLE " SET started=?, status=?, imported=? WHERE id=?"
 
 /* Error codes */
 

src/defs.h

@@ -14,6 +14,7 @@
    #include <tre/regex.h>
 #endif
 
+#include <openssl/md5.h>
 #include <openssl/sha.h>
 #include <openssl/ssl.h>
 #include <netinet/in.h>
@@ -40,7 +41,8 @@
 
 #define BASE64_RATIO 1.33333333
 
-#define DIGEST_LENGTH SHA256_DIGEST_LENGTH
+#define DIGEST_LENGTH EVP_MAX_MD_SIZE
+#define DIGEST_HEX_LENGTH 2*DIGEST_LENGTH+1
 
 #define UNDEF 0
 #define READY 1
@@ -171,6 +173,7 @@ struct parser_state {
    int qp;
    int htmltag;
    int style;
+   int meta_content_type;
    int skip_html;
    int has_to_dump;
    int has_to_dump_whole_body;
@@ -184,11 +187,13 @@ struct parser_state {
    int saved_size;
    unsigned int writebufpos;
    unsigned int abufpos;
+   unsigned int received_header;
    char attachedfile[RND_STR_LEN+SMALLBUFSIZE];
    char message_id[SMALLBUFSIZE];
    char message_id_hash[2*DIGEST_LENGTH+1];
    char miscbuf[MAX_TOKEN_LEN];
    char qpbuf[MAX_TOKEN_LEN];
+   char receivedbuf[SMALLBUFSIZE];
    unsigned long n_token;
    unsigned long n_subject_token;
    unsigned long n_body_token;
@@ -218,7 +223,7 @@ struct parser_state {
    unsigned int todomainlen;
    unsigned int found_security_header;
 
-   int journaltolen;
+   long unsigned int journaltolen;
 
    int retention;
 };
@@ -248,7 +253,7 @@ struct session_data {
    int journal_envelope_length, journal_bottom_length;
    unsigned int sql_errno;
 #ifdef NEED_MYSQL
-   MYSQL mysql;
+   MYSQL mysql, sphx;
 #endif
 };
 
@@ -315,6 +320,7 @@ struct import {
    int port;
    int seq;
    int table_id;
+   int delay;
    char *server;
    char *username;
    char *password;
@@ -325,7 +331,7 @@ struct import {
    char *mboxdir;
    char *folder;
    char filename[SMALLBUFSIZE];
-   time_t started, updated, finished;
+   time_t started, updated, finished, after, before;
 };
 
 
@@ -397,20 +403,29 @@ struct smtp_session {
    char ttmpfile[SMALLBUFSIZE];
    char mailfrom[SMALLBUFSIZE];
    char rcptto[MAX_RCPT_TO][SMALLBUFSIZE];
-   char buf[MAXBUFSIZE];
    char remote_host[INET6_ADDRSTRLEN+1];
+   char nullbyte;
    time_t lasttime;
    int protocol_state;
    int slot;
    int fd;
    int bad;
-   int buflen;
-   int last_data_char;
    int tot_len;
    int bdat_bytes_to_read;
    int num_of_rcpt_to;
    struct config *cfg;
    struct net net;
+   int max_message_size;
+   char *buf;
+   int buflen;
+   int bufsize;
+   int too_big;
+   int mail_size;
+};
+
+struct tls_protocol {
+   char *proto;
+   int version;
 };
 
 #endif /* _DEFS_H */

src/digest.c

@@ -34,23 +34,38 @@ int search_header_end(char *p, int n){
 
 
 int make_digests(struct session_data *sdata, struct config *cfg){
-   int i=0, n, fd, offset=3, hdr_len=0;
+   int n, fd, offset=3, hdr_len=0;
    char *body=NULL;
-   unsigned char buf[BIGBUFSIZE], md[DIGEST_LENGTH], md2[DIGEST_LENGTH];
-   SHA256_CTX context, context2;
+   unsigned char buf[BIGBUFSIZE];
+
+   EVP_MD_CTX *ctx, *ctx2;
+   const EVP_MD *md, *md2;
+   unsigned int i=0, md_len, md_len2;
+   unsigned char md_value[EVP_MAX_MD_SIZE], md_value2[EVP_MAX_MD_SIZE];
 
    memset(sdata->bodydigest, 0, 2*DIGEST_LENGTH+1);
    memset(sdata->digest, 0, 2*DIGEST_LENGTH+1);
-   SHA256_Init(&context);
-   SHA256_Init(&context2);
 
+   md = EVP_get_digestbyname("sha256");
+   md2 = EVP_get_digestbyname("sha256");
+   if(md == NULL || md2 == NULL){
+      syslog(LOG_PRIORITY, "ERROR: unknown message digest: sha256 in %s:%d", __func__, __LINE__);
+      return 1;
+   }
+
+   ctx = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx, md, NULL);
+
+   ctx2 = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx2, md2, NULL);
 
    fd = open(sdata->filename, O_RDONLY);
    if(fd == -1) return -1;
 
+   memset(buf, 0, sizeof(buf));
 
    while((n = read(fd, buf, sizeof(buf))) > 0){
-      SHA256_Update(&context2, buf, n);
+      EVP_DigestUpdate(ctx2, buf, n);
 
       body = (char *)&buf[0];
 
@@ -67,7 +82,7 @@ int make_digests(struct session_data *sdata, struct config *cfg){
       }
 
 
-      SHA256_Update(&context, body, n);
+      EVP_DigestUpdate(ctx, body, n);
 
       i++;
    }
@@ -76,58 +91,93 @@ int make_digests(struct session_data *sdata, struct config *cfg){
 
    sdata->hdr_len = hdr_len;
 
-   SHA256_Final(md, &context);
-   SHA256_Final(md2, &context2);
+   EVP_DigestFinal_ex(ctx, md_value, &md_len);
+   EVP_MD_CTX_free(ctx);
+   EVP_DigestFinal_ex(ctx2, md_value2, &md_len2);
+   EVP_MD_CTX_free(ctx2);
 
-   for(i=0;i<DIGEST_LENGTH;i++){
-      snprintf(sdata->bodydigest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
-      snprintf(sdata->digest + i*2, 2*DIGEST_LENGTH, "%02x", md2[i]);
+   for(i=0;i<md_len;i++){
+      snprintf(sdata->bodydigest + i*2, 3, "%02x", md_value[i]);
+   }
+
+   for(i=0;i<md_len2;i++){
+      snprintf(sdata->digest + i*2, 3, "%02x", md_value2[i]);
    }
 
    return 0;
 }
 
 
-void digest_file(char *filename, char *digest){
-   int fd, i, n;
-   unsigned char buf[MAXBUFSIZE], md[DIGEST_LENGTH];
-   SHA256_CTX context;
+void raw_digest_file(char *digestname, char *filename, unsigned char *md_value){
+   int fd, n;
+   unsigned char buf[MAXBUFSIZE];
+   EVP_MD_CTX *ctx;
+   const EVP_MD *md;
+   unsigned int md_len;
 
-   memset(digest, 0, 2*DIGEST_LENGTH+1);
+   md = EVP_get_digestbyname(digestname);
+   if(md == NULL){
+      syslog(LOG_PRIORITY, "ERROR: unknown message digest: '%s' in %s:%d", digestname, __func__, __LINE__);
+      return;
+   }
 
    fd = open(filename, O_RDONLY);
    if(fd == -1) return;
 
-   SHA256_Init(&context);
+   ctx = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx, md, NULL);
 
    while((n = read(fd, buf, sizeof(buf))) > 0){
-      SHA256_Update(&context, buf, n);
+      EVP_DigestUpdate(ctx, buf, n);
    }
 
    close(fd);
 
-   SHA256_Final(md, &context);
-
-   for(i=0;i<DIGEST_LENGTH;i++)
-      snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
-
+   EVP_DigestFinal_ex(ctx, md_value, &md_len);
+   EVP_MD_CTX_free(ctx);
 }
 
 
-void digest_string(char *s, char *digest){
-   int i;
+void digest_file(char *filename, char *digest){
    unsigned char md[DIGEST_LENGTH];
-   SHA256_CTX context;
+
+   raw_digest_file("sha256", filename, &md[0]);
 
    memset(digest, 0, 2*DIGEST_LENGTH+1);
 
-   SHA256_Init(&context);
-
-   SHA256_Update(&context, s, strlen(s));
-
-   SHA256_Final(md, &context);
-
-   for(i=0;i<DIGEST_LENGTH;i++)
+   for(int i=0;i<SHA256_DIGEST_LENGTH;i++){
       snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
-
+   }
+}
+
+
+void digest_string(char *digestname, char *s, char *digest){
+   EVP_MD_CTX *ctx;
+   const EVP_MD *md;
+   unsigned int i, md_len;
+   unsigned char md_value[DIGEST_LENGTH];
+
+   memset(digest, 0, 2*DIGEST_LENGTH+2);
+
+   md = EVP_get_digestbyname(digestname);
+   if(md == NULL){
+      syslog(LOG_PRIORITY, "ERROR: unknown message digest: '%s' in %s:%d", digestname, __func__, __LINE__);
+      return;
+   }
+
+   ctx = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx, md, NULL);
+   EVP_DigestUpdate(ctx, s, strlen(s));
+   EVP_DigestFinal_ex(ctx, md_value, &md_len);
+   EVP_MD_CTX_free(ctx);
+
+   for(i=0;i<md_len;i++){
+      snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md_value[i]);
+   }
+}
+
+
+void create_md5_from_email_address(char *puf, char *md5buf){
+   digest_string("md5", puf, md5buf);
+   md5buf[2*MD5_DIGEST_LENGTH] = ' ';
 }

src/imap.c

@@ -105,7 +105,12 @@ int connect_to_imap_server(struct data *data){
 
    /* imap cmd: LOGIN */
 
+   if(strcmp(data->import->username, "ZIMBRA") == 0){
+      snprintf(buf, sizeof(buf)-1, "A%d AUTHENTICATE PLAIN %s\r\n", data->import->seq, data->import->password);
+   }
+   else {
       snprintf(buf, sizeof(buf)-1, "A%d LOGIN %s \"%s\"\r\n", data->import->seq, data->import->username, data->import->password);
+   }
 
    write1(data->net, buf, strlen(buf));
    if(read_response(buf, sizeof(buf), data) == 0){
@@ -159,7 +164,7 @@ int imap_select_cmd_on_folder(char *folder, struct data *data){
       }
    }
 
-   printf("found %d messages\n", messages);
+   if(data->quiet == 0){printf("found %d messages\n", messages); }
 
    data->import->total_messages += messages;
 
@@ -313,7 +318,7 @@ void imap_expunge_message(struct data *data){
 }
 
 
-int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct config *cfg){
+int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    int i, messages=0;
 
    messages = imap_select_cmd_on_folder(folder, data);
@@ -330,7 +335,7 @@ int process_imap_folder(char *folder, struct session_data *sdata, struct data *d
          if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/messages); fflush(stdout); }
 
          if(data->import->dryrun == 0){
-            int rc = import_message(sdata, data, cfg);
+            int rc = import_message(sdata, data, counters, cfg);
 
             if(data->import->remove_after_import == 1 && rc == OK){
                imap_delete_message(data, i);
@@ -356,7 +361,7 @@ int process_imap_folder(char *folder, struct session_data *sdata, struct data *d
    }
 
 
-   printf("\n");
+   if(data->quiet == 0){printf("\n"); }
 
    return OK;
 }
@@ -375,7 +380,7 @@ int list_folders(struct data *data){
    char attrs[SMALLBUFSIZE], folder[SMALLBUFSIZE];
    int len=MAXBUFSIZE+3, pos=0, n, rc=ERR, fldrlen=0, result;
 
-   printf("List of IMAP folders:\n");
+   if(data->quiet == 0){printf("List of IMAP folders:\n"); }
 
    buf = malloc(len);
    if(!buf) return rc;
@@ -416,7 +421,7 @@ int list_folders(struct data *data){
    }
 
    // trim the "A3 OK LIST completed" trailer off
-   if(p) *p = '\0';
+   if(p) *p = '\0'; //-V547
 
    memset(attrs, 0, sizeof(attrs));
 
@@ -449,9 +454,10 @@ int list_folders(struct data *data){
             } else {
 
                if(fldrlen) {
-                  ruf = malloc(strlen(q) * 2 + 1);
-                  memset(ruf, 0, strlen(q) * 2 + 1);
-                  memcpy(ruf, q, strlen(q));
+                  int ruflen = strlen(q) * 2;
+                  ruf = malloc(ruflen + 1);
+                  if(ruf){
+                     snprintf(ruf, ruflen, "%s", q);
                      r = ruf;
                      while(*r != '\0') {
                         if(*r == '\\') {
@@ -464,6 +470,11 @@ int list_folders(struct data *data){
                      snprintf(folder, sizeof(folder)-1, "%s", ruf);
 
                      free(ruf);
+                  }
+                  else {
+                     printf("error: ruf = malloc()\n");
+                  }
+
                   fldrlen = 0;
                } else {
                   snprintf(folder, sizeof(folder)-1, "%s", q);
@@ -472,9 +483,9 @@ int list_folders(struct data *data){
                if(!strstr(attrs, "\\Noselect")){
                   addnode(data->imapfolders, folder);
                }
-               else printf("skipping ");
+               else if(data->quiet == 0){printf("skipping "); }
 
-               printf("=> '%s [%s]'\n", folder, attrs);
+               if(data->quiet == 0){printf("=> '%s [%s]'\n", folder, attrs); }
 
                memset(attrs, 0, sizeof(attrs));
             }

src/import.c

@@ -18,13 +18,20 @@
 #include <piler.h>
 
 
-int import_message(struct session_data *sdata, struct data *data, struct config *cfg){
+int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    int rc=ERR;
    char *rule;
    struct stat st;
    struct parser_state state;
-   struct counters counters;
 
+   if(data->import->delay > 0){
+      struct timespec req;
+
+      req.tv_sec = 0;
+      req.tv_nsec = 1000000 * data->import->delay;
+
+      nanosleep(&req, NULL);
+   }
 
    init_session_data(sdata, cfg);
 
@@ -85,11 +92,22 @@ int import_message(struct session_data *sdata, struct data *data, struct config
          printf("%s: invalid message, hdr_len: %d\n", data->import->filename, sdata->hdr_len);
          rc = ERR;
       }
+      else if(data->import->after > 0 && sdata->sent < data->import->after){
+         if(cfg->verbosity > 1) printf("discarding older email: %s (%ld/%ld)\n", sdata->filename, sdata->sent, data->import->after);
+         rc = ERR_DISCARDED;
+      }
+      else if(data->import->before > 0 && sdata->sent > data->import->before){
+         if(cfg->verbosity > 1) printf("discarding newer email: %s (%ld/%ld)\n", sdata->filename, sdata->sent, data->import->before);
+         rc = ERR_DISCARDED;
+      }
       else {
          // When importing emails, we should add the retention value (later) to the original sent value
          sdata->retained = sdata->sent;
 
+         // backup original value of data->folder
+         int folder = data->folder;
          rc = process_message(sdata, &state, data, cfg);
+         data->folder = folder;
          unlink(state.message_id_hash);
       }
    }
@@ -103,24 +121,27 @@ int import_message(struct session_data *sdata, struct data *data, struct config
 
    switch(rc) {
       case OK:
-                        bzero(&counters, sizeof(counters));
-                        counters.c_rcvd = 1;
-                        counters.c_size += sdata->tot_len;
-                        counters.c_stored_size = sdata->stored_len;
-                        update_counters(sdata, data, &counters, cfg);
+                        counters->c_rcvd++;
+                        counters->c_size += sdata->tot_len;
+                        counters->c_stored_size += sdata->stored_len;
 
                         break;
 
       case ERR_EXISTS:
                         rc = OK;
 
-                        bzero(&counters, sizeof(counters));
-                        counters.c_duplicate = 1;
-                        update_counters(sdata, data, &counters, cfg);
+                        counters->c_duplicate++;
 
                         if(data->quiet == 0) printf("duplicate: %s (duplicate id: %llu)\n", data->import->filename, sdata->duplicate_id);
                         break;
 
+      case ERR_DISCARDED:
+                        rc = OK;
+
+                        counters->c_ignore++;
+
+                        break;
+
       default:
                         printf("failed to import: %s (id: %s)\n", data->import->filename, sdata->ttmpfile);
                         break;
@@ -145,13 +166,14 @@ int import_message(struct session_data *sdata, struct data *data, struct config
 
 
 int update_import_table(struct session_data *sdata, struct data *data) {
-   int ret=ERR, status=2;
+   int ret=ERR, status=2, started=1;
    struct sql sql;
 
    if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE) == ERR) return ret;
 
    p_bind_init(&sql);
 
+   sql.sql[sql.pos] = (char *)&(started); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
    sql.sql[sql.pos] = (char *)&(status); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
    sql.sql[sql.pos] = (char *)&(data->import->tot_msgs); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
    sql.sql[sql.pos] = (char *)&(data->import->table_id); sql.type[sql.pos] = TYPE_LONG; sql.pos++;

src/import.h

@@ -6,23 +6,23 @@
    #define _IMPORT_H
 
 
-int import_message(struct session_data *sdata, struct data *data, struct config *cfg);
+int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 int update_import_table(struct session_data *sdata, struct data *data);
 
-int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct config *cfg);
-int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct config *cfg);
-int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct config *cfg);
-void import_from_pop3_server(struct session_data *sdata, struct data *data, struct config *cfg);
-int import_from_imap_server(struct session_data *sdata, struct data *data, struct config *cfg);
+int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg);
+int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 
 int connect_to_pop3_server(struct data *data);
-void process_pop3_emails(struct session_data *sdata, struct data *data, struct config *cfg);
+void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 
 int connect_to_imap_server(struct data *data);
 int list_folders(struct data *data);
-int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct config *cfg);
+int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 void send_imap_close(struct data *data);
 
-void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg);
+void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 
 #endif /* _IMPORT_H */

src/import_imap.c

@@ -22,7 +22,7 @@
 #include <piler.h>
 
 
-int import_from_imap_server(struct session_data *sdata, struct data *data, struct config *cfg){
+int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    int i, rc=ERR, ret=OK, skipmatch;
    char port_string[8], puf[SMALLBUFSIZE];
    struct addrinfo hints, *res;
@@ -88,7 +88,7 @@ int import_from_imap_server(struct session_data *sdata, struct data *data, struc
             else {
                if(data->quiet == 0) printf("processing folder: %s... ", (char *)q->str);
 
-               if(process_imap_folder(q->str, sdata, data, cfg) == ERR) ret = ERR;
+               if(process_imap_folder(q->str, sdata, data, counters, cfg) == ERR) ret = ERR;
             }
 
          }

src/import_mailbox.c

@@ -22,7 +22,7 @@
 #include <piler.h>
 
 
-int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct config *cfg){
+int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    FILE *F, *f=NULL;
    int rc=ERR, tot_msgs=0, ret=OK;
    char buf[MAXBUFSIZE];
@@ -44,7 +44,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *
          if(f){
             fclose(f);
             f = NULL;
-            rc = import_message(sdata, data, cfg);
+            rc = import_message(sdata, data, counters, cfg);
             if(rc == ERR){
                printf("error importing: '%s'\n", data->import->filename);
                ret = ERR;
@@ -64,7 +64,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *
 
    if(f){
       fclose(f);
-      rc = import_message(sdata, data, cfg);
+      rc = import_message(sdata, data, counters, cfg);
       if(rc == ERR){
          printf("ERROR: error importing: '%s'\n", data->import->filename);
          ret = ERR;
@@ -80,7 +80,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *
 }
 
 
-int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct config *cfg){
+int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    DIR *dir;
    struct dirent *de;
    int rc=ERR, ret=OK, i=0;
@@ -103,7 +103,7 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct dat
       if(stat(fname, &st) == 0){
          if(S_ISDIR(st.st_mode)){
             folder = data->folder;
-            rc = import_mbox_from_dir(fname, sdata, data, cfg);
+            rc = import_mbox_from_dir(fname, sdata, data, counters, cfg);
             data->folder = folder;
             if(rc == ERR) ret = ERR;
          }
@@ -126,7 +126,7 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct dat
 
                }
 
-               rc = import_from_mailbox(fname, sdata, data, cfg);
+               rc = import_from_mailbox(fname, sdata, data, counters, cfg);
                if(rc == OK) (data->import->tot_msgs)++;
                else ret = ERR;
 

src/import_maildir.c

@@ -22,7 +22,7 @@
 #include <piler.h>
 
 
-int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct config *cfg){
+int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg){
    DIR *dir;
    struct dirent *de;
    int rc=ERR, ret=OK, i=0;
@@ -46,7 +46,7 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir
          if(S_ISDIR(st.st_mode)){
             folder = data->folder;
             snprintf(subdir, sizeof(subdir)-1, "%s", data->import->filename);
-            rc = import_from_maildir(sdata, data, subdir, cfg);
+            rc = import_from_maildir(sdata, data, subdir, counters, cfg);
             data->folder = folder;
             if(rc == ERR) ret = ERR;
          }
@@ -76,7 +76,7 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir
 
                }
 
-               rc = import_message(sdata, data, cfg);
+               rc = import_message(sdata, data, counters, cfg);
 
                if(rc == OK) (data->import->tot_msgs)++;
                else if(rc == ERR){

src/import_pilerexport.c

@@ -22,21 +22,21 @@
 #include <piler.h>
 
 
-void import_the_file(struct session_data *sdata, struct data *data, struct config *cfg){
+void import_the_file(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    close(data->import->fd);
    data->import->fd = -1;
 
-   if(import_message(sdata, data, cfg) != ERR){
+   if(import_message(sdata, data, counters, cfg) != ERR){
       unlink(data->import->filename);
    }
 }
 
 
-void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct config *cfg){
+void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
 
    if(!strcmp(buf, PILEREXPORT_BEGIN_MARK)){
       if((*count) > 0){
-         import_the_file(sdata, data, cfg);
+         import_the_file(sdata, data, counters, cfg);
       }
 
       (*count)++;
@@ -57,8 +57,8 @@ void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *s
 }
 
 
-void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg){
-   int n, rc, savedlen=0, puflen;
+void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   int n, rc, nullbyte, savedlen=0, puflen;
    uint64 count=0;
    char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE];
 
@@ -70,12 +70,16 @@ void import_from_pilerexport(struct session_data *sdata, struct data *data, stru
       memset(buf, 0, sizeof(buf));
       n = fread(buf, 1, sizeof(buf)-1, stdin);
 
+      int remaininglen = n;
+
       if(savedlen > 0){
          memset(copybuf, 0, sizeof(copybuf));
 
          memcpy(copybuf, savedbuf, savedlen);
          memcpy(&copybuf[savedlen], buf, n);
 
+         remaininglen += savedlen;
+
          savedlen = 0;
          memset(savedbuf, 0, sizeof(savedbuf));
 
@@ -86,12 +90,13 @@ void import_from_pilerexport(struct session_data *sdata, struct data *data, stru
       }
 
       do {
-         puflen = read_one_line(p, '\n', puf, sizeof(puf), &rc);
+         puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf), &rc, &nullbyte);
          p += puflen;
+         remaininglen -= puflen;
 
          if(puflen > 0){
             if(rc == OK){
-               process_buffer(puf, puflen, &count, sdata, data, cfg);
+               process_buffer(puf, puflen, &count, sdata, data, counters, cfg);
             }
             else {
                snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
@@ -104,6 +109,6 @@ void import_from_pilerexport(struct session_data *sdata, struct data *data, stru
    } while(n > 0);
 
    if(data->import->fd != -1){
-      import_the_file(sdata, data, cfg);
+      import_the_file(sdata, data, counters, cfg);
    }
 }

src/import_pop3.c

@@ -22,7 +22,7 @@
 #include <piler.h>
 
 
-void import_from_pop3_server(struct session_data *sdata, struct data *data, struct config *cfg){
+void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    int rc;
    char port_string[8];
    struct addrinfo hints, *res;
@@ -58,7 +58,7 @@ void import_from_pop3_server(struct session_data *sdata, struct data *data, stru
       goto ENDE_POP3;
    }
 
-   process_pop3_emails(sdata, data, cfg);
+   process_pop3_emails(sdata, data, counters, cfg);
 
    close_connection(data->net);
 

src/message.c

@@ -30,10 +30,6 @@ int store_index_data(struct session_data *sdata, struct parser_state *state, str
    subj = state->b_subject;
    if(*subj == ' ') subj++;
 
-
-   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc;
-
-
    fix_email_address_for_sphinx(state->b_from);
    fix_email_address_for_sphinx(state->b_sender);
    fix_email_address_for_sphinx(state->b_to);
@@ -46,6 +42,68 @@ int store_index_data(struct session_data *sdata, struct parser_state *state, str
       sender_domain = state->b_sender_domain;
    }
 
+   if(cfg->rtindex){
+      // Manticore doesn't support prepared statements using sphinxQL
+      // so we have to go through a painful query assembly escaping
+      // the untrusted input
+      //
+      char a[4*MAXBUFSIZE+4*SMALLBUFSIZE];
+      char *query=NULL;
+
+      snprintf(a, sizeof(a)-1, "REPLACE INTO %s (id, arrived, sent, size, direction, folder, attachments, attachment_types, sender, rcpt, senderdomain, rcptdomain, subject, body) VALUES (%llu,%ld,%ld,%d,%d,%d,%d,'%s','", cfg->sphxdb, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, data->folder, state->n_attachments, sdata->attachments);
+
+      int ret = append_string_to_buffer(&query, a);
+
+      unsigned long len = strlen(sender);
+      char *s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, sender, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(state->b_to);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, state->b_to, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(sender_domain);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, sender_domain, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(state->b_to_domain);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, state->b_to_domain, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(subj);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, subj, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(state->b_body);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, state->b_body, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "')");
+
+      if(ret == 0 && mysql_real_query(&(sdata->sphx), query, strlen(query)) == OK) rc = OK;
+      else syslog(LOG_PRIORITY, "ERROR: %s failed to store index data for id=%llu, errno=%d, append ret=%d", sdata->ttmpfile, id, mysql_errno(&(sdata->sphx)), ret);
+
+      free(query);
+   }
+   else {
+      if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc;
+
       p_bind_init(&sql);
 
       sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
@@ -67,6 +125,7 @@ int store_index_data(struct session_data *sdata, struct parser_state *state, str
       else syslog(LOG_PRIORITY, "ERROR: %s failed to store index data for id=%llu, sql_errno=%d", sdata->ttmpfile, id, sdata->sql_errno);
 
       close_prepared_statement(&sql);
+   }
 
    return rc;
 }
@@ -206,13 +265,40 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru
 
    snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, sender, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
 
-   digest_string(s, &vcode[0]);
+   digest_string("sha256", s, &vcode[0]);
 
    memset(ref, 0, sizeof(ref));
    if(strlen(state->reference) > 10){
-      digest_string(state->reference, &ref[0]);
+      digest_string("sha256", state->reference, &ref[0]);
       update_metadata_reference(sdata, state, &ref[0], cfg);
    }
+   else if(state->reference[0] == 0){
+      // during import, the order of messages is often random
+      // check if this is a message which is already referenced
+      uint64 count=0;
+
+      digest_string("sha256", state->message_id, &ref[0]);
+      if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_METADATA_REFERENCE) != ERR){
+         p_bind_init(&sql);
+
+         sql.sql[sql.pos] = &ref[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+
+         if(p_exec_stmt(sdata, &sql) == OK){
+            p_bind_init(&sql);
+
+            sql.sql[sql.pos] = (char *)&count; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+            p_store_results(&sql);
+            p_fetch_results(&sql);
+            p_free_results(&sql);
+         }
+      }
+
+      close_prepared_statement(&sql);
+
+      // no reference yet
+      if(count <= 0)
+         ref[0] = 0;
+   }
 
 
    if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR;
@@ -308,7 +394,7 @@ void remove_stripped_attachments(struct parser_state *state){
 }
 
 
-int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
+int is_duplicated_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
    int fd;
    char piler_id[SMALLBUFSIZE];
 
@@ -319,7 +405,7 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru
    if(sdata->duplicate_id > 0){
       remove_stripped_attachments(state);
 
-      if(strlen(state->b_journal_to) > 0){
+      if(sdata->restored_copy == 0 && strlen(state->b_journal_to) > 0){
          if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to add journal rcpt (%s) to id=%llu for message-id: '%s'", sdata->ttmpfile, state->b_journal_to, sdata->duplicate_id, state->message_id);
          store_recipients(sdata, state->b_journal_to, sdata->duplicate_id, cfg);
       }
@@ -338,7 +424,6 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru
    if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: touch %s OK (%s)", sdata->ttmpfile, state->message_id_hash, state->message_id);
 
 
-
    if(cfg->mmap_dedup_test == 1 && data->dedup != MAP_FAILED && data->child_serial >= 0 && data->child_serial < MAXCHILDREN){
 
       if(strstr(data->dedup, state->message_id_hash)){
@@ -354,6 +439,14 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru
       memcpy(data->dedup + data->child_serial*DIGEST_LENGTH*2, state->message_id_hash, DIGEST_LENGTH*2);
    }
 
+   return OK;
+}
+
+
+int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
+
+   if(is_duplicated_message(sdata, state, data, cfg) == ERR_EXISTS)
+      return ERR_EXISTS;
 
    sdata->retained += query_retain_period(data, state, sdata->tot_len, sdata->spam_message, cfg);
 

src/misc.c

@@ -230,11 +230,8 @@ int extractEmail(char *rawmail, char *email){
  */
 
 void make_random_string(unsigned char *buf, int buflen){
-   int i, len, fd;
-   int urandom=0;
-   static char alphanum[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
-
-   len = strlen(alphanum);
+   const char alphanum[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+   int i, fd, urandom=0, len = sizeof(alphanum)-1;
 
    fd = open(RANDOM_POOL, O_RDONLY);
    if(fd != -1){
@@ -289,7 +286,7 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
    taia_now(&now);
    taia_pack(nowpack, &now);
 
-   memcpy(buf, nowpack, 12);
+   memcpy(buf, nowpack, 12); //-V512
 
    fd = open(RANDOM_POOL, O_RDONLY);
    if(fd == -1) return ret;
@@ -655,34 +652,42 @@ void move_email(struct smtp_session *session){
 
    snprintf(buf, sizeof(buf)-1, "%d/%s", session->ttmpfile[0] % session->cfg->number_of_worker_processes, session->ttmpfile);
 
+   if(session->nullbyte){
+      snprintf(buf, sizeof(buf)-1, "%s/%s", ERROR_DIR, session->ttmpfile);
+      syslog(LOG_PRIORITY, "ERROR: %s contains an invalid NUL-byte, moving it to %s", session->ttmpfile, ERROR_DIR);
+   }
+
    if(rename(session->ttmpfile, buf)){
       syslog(LOG_PRIORITY, "ERROR: couldn't rename %s to %s (reason: %s)", session->ttmpfile, buf, strerror(errno));
    }
 }
 
 
-int read_one_line(char *s, int c, char *buf, int buflen, int *rc){
+int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte){
    int i=0;
 
    *rc = ERR;
+   *nullbyte = 0;
 
    memset(buf, 0, buflen);
 
-   if(s == NULL){
-      return i;
+   for(int j=0; j<slen; j++){
+      if(i<buflen-2){
+         if(*(s+j) == 0){
+            *nullbyte = 1;
          }
 
-   for(; *s; s++){
-      if(i<buflen-2){
-         buf[i] = *s;
+         buf[i] = *(s+j);
          i++;
 
-         if(*s == c){
+         if(*(s+j) == c){
             *rc = OK;
             break;
          }
       }
-      else break;
+      else {
+         break;
+      }
    }
 
    return i;
@@ -711,7 +716,7 @@ int init_ssl_to_server(struct data *data){
    n = SSL_connect(data->net->ssl);
    CHK_SSL(n, "internal ssl error");
 
-   printf("Cipher: %s\n", SSL_get_cipher(data->net->ssl));
+   //printf("Cipher: %s\n", SSL_get_cipher(data->net->ssl));
 
    server_cert = SSL_get_peer_certificate(data->net->ssl);
    CHK_NULL(server_cert, "server cert error");
@@ -750,3 +755,55 @@ char *strcasestr(const char *s, const char *find){
    return((char*)s);
 }
 #endif
+
+
+int append_string_to_buffer(char **buffer, char *str){
+   int arglen;
+   char *s=NULL;
+
+   arglen = strlen(str);
+
+   if(!*buffer){
+      *buffer = malloc(arglen+1);
+      memset(*buffer, 0, arglen+1);
+      memcpy(*buffer, str, arglen);
+   }
+   else {
+      int len = strlen(*buffer);
+      s = realloc(*buffer, len + arglen+1);
+      if(!s) return 1;
+
+      *buffer = s;
+
+      memset(*buffer+len, 0, arglen+1);
+      memcpy(*buffer+len, str, arglen);
+   }
+
+   return 0;
+}
+
+
+int get_size_from_smtp_mail_from(char *s){
+   int size=0;
+   char *p;
+
+   p = strcasestr(s, "SIZE=");
+   if(p){
+      p += strlen("SIZE=");
+      char *q = p;
+      for(; *q; q++){
+         if(isspace(*q)) break;
+      }
+
+      // We extract max. 9 characters, which is just under 1GB
+      // and not overflowing an int variable
+      if(q - p <= 9){
+         char c = *q;
+         *q = '\0';
+         size = atoi(p);
+         *q = c;
+      }
+   }
+
+   return size;
+}

src/misc.h

@@ -46,7 +46,7 @@ int create_and_bind(char *listen_addr, int listen_port);
 int can_i_write_directory(char *dir);
 
 void move_email(struct smtp_session *session);
-int read_one_line(char *s, int c, char *buf, int buflen, int *rc);
+int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte);
 
 int init_ssl_to_server(struct data *data);
 
@@ -54,4 +54,7 @@ int init_ssl_to_server(struct data *data);
    char *strcasestr(const char *s, const char *find);
 #endif
 
+int append_string_to_buffer(char **buffer, char *str);
+int get_size_from_smtp_mail_from(char *s);
+
 #endif /* _MISC_H */

src/mysql.c

@@ -33,11 +33,40 @@ int open_database(struct session_data *sdata, struct config *cfg){
 }
 
 
+int open_sphx(struct session_data *sdata, struct config *cfg){
+   int rc=1;
+   char buf[BUFLEN];
+
+   mysql_init(&(sdata->sphx));
+
+   mysql_options(&(sdata->sphx), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg->mysql_connect_timeout);
+   mysql_options(&(sdata->sphx), MYSQL_OPT_RECONNECT, (const char*)&rc);
+
+   if(mysql_real_connect(&(sdata->sphx), cfg->sphxhost, "", "", cfg->sphxdb, cfg->sphxport, "", 0) == 0){
+      syslog(LOG_PRIORITY, "cant connect to %s:%d", cfg->sphxhost, cfg->sphxport);
+      return ERR;
+   }
+
+   snprintf(buf, sizeof(buf)-2, "SET NAMES %s", cfg->mysqlcharset);
+   mysql_real_query(&(sdata->sphx), buf, strlen(buf));
+
+   snprintf(buf, sizeof(buf)-2, "SET CHARACTER SET %s", cfg->mysqlcharset);
+   mysql_real_query(&(sdata->sphx), buf, strlen(buf));
+
+   return OK;
+}
+
+
 void close_database(struct session_data *sdata){
    mysql_close(&(sdata->mysql));
 }
 
 
+void close_sphx(struct session_data *sdata){
+   mysql_close(&(sdata->sphx));
+}
+
+
 void p_bind_init(struct sql *sql){
    int i;
 

src/parser.c

@@ -45,7 +45,7 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec
 
    if(take_into_pieces == 1 && state.writebufpos > 0){
       if(write(state.mfd, writebuffer, state.writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: %s: write(), %s, %d, %s", sdata->ttmpfile, __func__, __LINE__, __FILE__);
-      memset(writebuffer, 0, sizeof(writebuffer));
+      memset(writebuffer, 0, sizeof(writebuffer)); //-V597
       state.writebufpos = 0;
    }
 
@@ -65,17 +65,23 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec
    fclose(f);
 
    if(data->import && data->import->extra_recipient){
-      add_recipient(data->import->extra_recipient, strlen(data->import->extra_recipient), sdata, &state, data, cfg);
+      char tmpbuf[SMALLBUFSIZE];
+      snprintf(tmpbuf, sizeof(tmpbuf)-1, "%s", data->import->extra_recipient);
+      add_recipient(tmpbuf, strlen(tmpbuf), sdata, &state, data, cfg);
    }
 
    // If both Sender: and From: headers exist, and they are different, then append
    // the From: address to recipients list to give him access to this email as well
 
-   if(state.b_sender_domain[0] && strcmp(state.b_from, state.b_sender)){
-      char tmpbuf[SMALLBUFSIZE];
-      get_first_email_address_from_string(state.b_from, tmpbuf, sizeof(tmpbuf));
-      tmpbuf[strlen(tmpbuf)] = ' ';
-      add_recipient(tmpbuf, strlen(tmpbuf), sdata, &state, data, cfg);
+   if(state.b_sender_domain[0]){
+      char frombuf[SMALLBUFSIZE];
+      char senderbuf[SMALLBUFSIZE];
+      get_first_email_address_from_string(state.b_from, frombuf, sizeof(frombuf));
+      get_first_email_address_from_string(state.b_sender, senderbuf, sizeof(senderbuf));
+      if(strcmp(frombuf, senderbuf)){
+         frombuf[strlen(frombuf)] = ' ';
+         add_recipient(frombuf, strlen(frombuf), sdata, &state, data, cfg);
+      }
    }
 
    return state;
@@ -110,6 +116,27 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c
       if(sdata->internal_recipient == 1 && sdata->external_recipient == 1) sdata->direction = DIRECTION_INTERNAL_AND_OUTGOING;
    }
 
+   char *q = strrchr(state->receivedbuf, ';');
+   // I've seen some odd 1st Received: lines where the date
+   // wasn't the last item after a semicolon, eg.
+   //
+   // Received: from some.server.local [172.16.102.18]
+   //   by some.server.com with POP3 (Message Archiver)
+   //   for <nobody@127.0.0.1> (single-drop); Fri, 22 Nov 2019 11:24:49 -0500 (EST)
+   //   smtp.auth=3Daaaaaaaa@yourdomain.com|Received:=20by=20yourdomain
+   //   .com=20with=20ESMTPA=20id=20md50124862414.msg=3B=20Fri,=2022=20
+   //   ...
+   //
+   // Such line messes with the parse_date_header() function
+   // so the workaround is to discard such Received: line
+   if(q && strlen(q+1) < 45){
+      time_t received_timestamp = parse_date_header(q+1);
+      if(received_timestamp > 10000000){
+         // If the calculated date based on Date: header line differs more than 1 week
+         // then we'll override it with the data parsed from the first Received: line
+         if(labs(received_timestamp - sdata->sent) > 604800) sdata->sent = received_timestamp;
+      }
+   }
 
    for(i=1; i<=state->n_attachments; i++){
       char puf[SMALLBUFSIZE];
@@ -123,7 +150,7 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c
 
       digest_file(state->attachments[i].internalname, &(state->attachments[i].digest[0]));
 
-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest);
+      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s, dumped: %d", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest, state->attachments[i].dumped);
 
       char *p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type);
       len = strlen(p);
@@ -146,7 +173,7 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c
    }
 
 
-   digest_string(state->message_id, &(state->message_id_hash[0]));
+   digest_string("sha256", state->message_id, &(state->message_id_hash[0]));
 
    if(sdata->sent == 0) sdata->sent = sdata->now;
 }
@@ -212,7 +239,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
          sdata->restored_copy = 1;
       }
 
-      if(cfg->security_header && state->found_security_header == 0 && strstr(buf, cfg->security_header)){
+      if(cfg->security_header[0] && state->found_security_header == 0 && strstr(buf, cfg->security_header)){
          state->found_security_header = 1;
       }
 
@@ -406,7 +433,10 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
          state->message_state = MSG_CC;
          buf += strlen("Bcc:");
       }
-      else if(strncasecmp(buf, "Message-Id:", 11) == 0) state->message_state = MSG_MESSAGE_ID;
+      else if(strncasecmp(buf, "Message-Id:", 11) == 0){
+         state->message_state = MSG_MESSAGE_ID;
+         buf += strlen("Message-Id:");
+      }
       else if(strncasecmp(buf, "References:", 11) == 0) state->message_state = MSG_REFERENCES;
       else if(strncasecmp(buf, "Subject:", strlen("Subject:")) == 0){
          state->message_state = MSG_SUBJECT;
@@ -430,32 +460,39 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
 
          if(strstr(buf, "=?") && strstr(buf, "?=")) fixupEncodedHeaderLine(buf, MAXBUFSIZE);
 
-         sdata->sent = parse_date_header(buf);
+         sdata->sent = parse_date_header(buf+5);
 
          /* allow +2 days drift in the parsed Date: value */
 
          if(sdata->sent - sdata->now > 2*86400) sdata->sent = sdata->now;
       }
 
-      else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf);
-      else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0) state->message_state = MSG_RECEIVED;
+      else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf+14);
+      else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0){
+         state->message_state = MSG_RECEIVED;
+         state->received_header++;
+      }
       else if(cfg->extra_to_field[0] != '\0' && strncasecmp(buf, cfg->extra_to_field, strlen(cfg->extra_to_field)) == 0){
-         state->message_state = MSG_TO;
+         state->message_state = MSG_RECIPIENT;
          buf += strlen(cfg->extra_to_field);
       }
 
       if(state->message_state == MSG_MESSAGE_ID && state->message_id[0] == 0){
-         p = strchr(buf+11, ' ');
-         if(p) p = buf + 12;
-         else p = buf + 11;
+         while(isspace(*buf)){
+            buf++;
+         }
 
-         snprintf(state->message_id, SMALLBUFSIZE-1, "%s", p);
+         snprintf(state->message_id, SMALLBUFSIZE-1, "%s", buf);
       }
 
       if(state->message_state == MSG_CONTENT_TYPE || state->message_state == MSG_CONTENT_DISPOSITION){
          fill_attachment_name_buf(state, buf);
       }
 
+      if(state->received_header == 1 && state->message_state == MSG_RECEIVED && strlen(state->receivedbuf) + len < sizeof(state->receivedbuf)){
+         memcpy(&(state->receivedbuf[strlen(state->receivedbuf)]), buf, len);
+      }
+
       /* we are interested in only From:, To:, Subject:, Received:, Content-*: header lines */
       if(state->message_state <= 0) return 0;
    }
@@ -553,6 +590,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
          state->message_rfc822 = 1;
          state->is_header = 1;
 
+         state->has_to_dump = 0;
+
          if(sdata->ms_journal == 1){
             state->is_1st_header = 1;
 
@@ -618,7 +657,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
       state->pushed_pointer = 0;
 
       memset(state->type, 0, TINYBUFSIZE);
-      snprintf(state->charset, TINYBUFSIZE-1, "unknown");
+      memset(state->charset, 0, TINYBUFSIZE);
 
       memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
       state->anamepos = 0;
@@ -661,7 +700,18 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
    if(state->texthtml == 1 && state->message_state == MSG_BODY) markHTML(buf, state);
 
 
-   if(state->texthtml == 1) decodeHTML(buf, state->utf8);
+   if(state->texthtml == 1){
+      size_t buflen = strlen(buf);
+      decodeHTML(buf, state->utf8);
+      /* decodeHTML converted some entities to iso-8859-1 */
+      if(state->utf8 != 1 && strlen(buf) != buflen){
+        /* no charset or us-ascii: switch to iso-8859-1 */
+        if (state->charset[0] == 0 || strcasecmp(state->charset, "us-ascii") == 0){
+          syslog(LOG_PRIORITY, "%s: assuming iso-8859-1 encoding for HTML (was '%s')", sdata->ttmpfile, state->charset);
+          snprintf(state->charset, TINYBUFSIZE-1, "ISO8859-1");
+        }
+      }
+   }
 
    /* encode the body if it's not utf-8 encoded */
    if(state->message_state == MSG_BODY && state->utf8 != 1){

src/parser.h

@@ -20,7 +20,7 @@ void fixupEncodedHeaderLine(char *buf, int buflen);
 void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state);
 void fixupBase64EncodedLine(char *buf, struct parser_state *state);
 void markHTML(char *buf, struct parser_state *state);
-int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state);
+void setStateHTML(char *htmlbuf, int pos, struct parser_state *state);
 void translateLine(unsigned char *p, struct parser_state *state);
 void fix_email_address_for_sphinx(char *s);
 void split_email_address(char *s);

src/parser_utils.c

@@ -40,6 +40,7 @@ void init_state(struct parser_state *state){
 
    state->htmltag = 0;
    state->style = 0;
+   state->meta_content_type = 0;
 
    state->skip_html = 0;
 
@@ -49,8 +50,10 @@ void init_state(struct parser_state *state){
    memset(state->message_id_hash, 0, 2*DIGEST_LENGTH+1);
    memset(state->miscbuf, 0, MAX_TOKEN_LEN);
    memset(state->qpbuf, 0, MAX_TOKEN_LEN);
+   memset(state->receivedbuf, 0, sizeof(state->receivedbuf));
 
    memset(state->type, 0, TINYBUFSIZE);
+   memset(state->charset, 0, TINYBUFSIZE);
 
    memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
    state->anamepos = 0;
@@ -67,6 +70,7 @@ void init_state(struct parser_state *state){
 
    state->writebufpos = 0;
    state->abufpos = 0;
+   state->received_header = 0;
 
    inithash(state->boundaries);
    inithash(state->rcpt);
@@ -124,7 +128,6 @@ time_t parse_date_header(char *datestr){
    char *p, *q, *r, *tz, s[SMALLBUFSIZE], tzh[4], tzm[3];
    struct tm tm;
 
-   datestr += 5;
    p = datestr;
 
    for(; *datestr; datestr++){
@@ -263,6 +266,18 @@ time_t parse_date_header(char *datestr){
 
    ts += get_local_timezone_offset() - offset;
 
+   if(ts < 700000000){
+      // If the Date: field contains some garbage, eg.
+      // "Date: [mail_datw]" or similar, and the date
+      // is before Sat Mar  7 20:26:40 UTC 1992, then
+      // return the current timestamp
+
+      time_t now;
+      time(&now);
+
+      return now;
+   }
+
    return ts;
 }
 
@@ -538,7 +553,7 @@ void markHTML(char *buf, struct parser_state *state){
 
             if(isspace(*s)){
                if(j > 0){
-                  k += appendHTMLTag(puf, html, pos, state);
+                  setStateHTML(html, pos, state);
                   memset(html, 0, SMALLBUFSIZE); j=0;
                }
                pos++;
@@ -563,55 +578,51 @@ void markHTML(char *buf, struct parser_state *state){
 
          if(j > 0){
             strncat(html, " ", SMALLBUFSIZE-1);
-            k += appendHTMLTag(puf, html, pos, state);
+            setStateHTML(html, pos, state);
             memset(html, 0, SMALLBUFSIZE); j=0;
          }
+         state->meta_content_type = 0;
       }
 
    }
 
    //printf("append last in line:*%s*, html=+%s+, j=%d\n", puf, html, j);
-   if(j > 0){ appendHTMLTag(puf, html, pos, state); }
+   if(j > 0){ setStateHTML(html, pos, state); }
 
    strcpy(buf, puf);
 }
 
 
-int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state){
-   char html[SMALLBUFSIZE];
-   int len;
-
+void setStateHTML(char *htmlbuf, int pos, struct parser_state *state){
    if(pos == 0 && strncmp(htmlbuf, "style ", 6) == 0) state->style = 1;
    if(pos == 0 && strncmp(htmlbuf, "/style ", 7) == 0) state->style = 0;
 
-   return 0;
+   if(pos == 0 && state->charset[0] == 0 && strncmp(htmlbuf, "meta ", 5) == 0) state->meta_content_type = 0x1;
+   if(state->meta_content_type){
+     if((state->meta_content_type & 0x2) == 0 && strstr(htmlbuf, "http-equiv=content-type "))
+       state->meta_content_type |= 0x2;
 
-   //printf("appendHTML: pos:%d, +%s+\n", pos, htmlbuf);
+     if((state->meta_content_type & 0x4) == 0 && strstr(htmlbuf, "content=text/html;"))
+       state->meta_content_type |= 0x4;
 
-   if(state->style == 1) return 0;
+     if(state->meta_content_type == 0x7){
+       char *p, *q;
 
-   if(strlen(htmlbuf) == 0) return 0;
-
-   snprintf(html, SMALLBUFSIZE-1, "HTML*%s", htmlbuf);
-   len = strlen(html);
-
-   if(len > 8 && strchr(html, '=')){
-      char *p = strstr(html, "cid:");
+       p = strstr(htmlbuf, "charset=");
        if(p){
-         *(p+3) = '\0';
-         strncat(html, " ", SMALLBUFSIZE-1);
-      }
+         p += 8;
+         for(q = p; isalnum(*q) || index("-_", *q); q++)
+           ;
 
-      strncat(buf, html, MAXBUFSIZE-1);
-      return len;
+         if(q > p && q-p+1 < (int) sizeof(state->charset)){
+           syslog(LOG_PRIORITY, "Changing HTML charset from '%s' to '%*s' due to meta tag", state->charset, (int)(q-p), p);
+           strncpy(state->charset, p, q-p);
+           state->charset[q-p+1] = '\0';
+           state->meta_content_type = 0;
+         }
+       }
      }
-
-   if(strstr(html, "http") ){
-      strncat(buf, html+5, MAXBUFSIZE-1);
-      return len-5;
    }
-
-   return 0;
 }
 
 

src/piler-smtp.c

@@ -33,6 +33,7 @@ extern int optind;
 struct epoll_event event, *events=NULL;
 int num_connections = 0;
 int listenerfd = -1;
+int loglevel = 1;
 
 char *configfile = CONFIG_FILE;
 struct config cfg;
@@ -40,6 +41,7 @@ struct passwd *pwd;
 struct smtp_session *session, **sessions=NULL;
 struct smtp_acl *smtp_acl[MAXHASH];
 
+time_t prev_timeout_check = 0;
 
 void usage(){
    printf("\nusage: piler\n\n");
@@ -47,6 +49,7 @@ void usage(){
    printf("    -d                                Fork to the background\n");
    printf("    -v                                Return the version and build number\n");
    printf("    -V                                Return the version and some build parameters\n");
+   printf("    -L <log level>                    Set the log level: 1-5\n");
 
    exit(0);
 }
@@ -90,16 +93,18 @@ void check_for_client_timeout(){
 
    if(cfg.verbosity >= LOG_DEBUG) syslog(LOG_PRIORITY, "%s @%ld", __func__, now);
 
+   if(now - prev_timeout_check < cfg.smtp_timeout) return;
+
    if(num_connections > 0){
       for(int i=0; i<cfg.max_connections; i++){
          if(sessions[i] && now - sessions[i]->lasttime >= cfg.smtp_timeout){
             syslog(LOG_PRIORITY, "client %s timeout, lasttime: %ld", sessions[i]->remote_host, sessions[i]->lasttime);
-            tear_down_session(sessions, sessions[i]->slot, &num_connections);
+            tear_down_session(sessions, sessions[i]->slot, &num_connections, "timeout");
          }
       }
    }
 
-   alarm(cfg.check_for_client_timeout_interval);
+   time(&prev_timeout_check);
 }
 
 
@@ -135,16 +140,20 @@ int main(int argc, char **argv){
    int client_len = sizeof(struct sockaddr_storage);
    ssize_t readlen;
    struct sockaddr_storage client_address;
-   char readbuf[BIGBUFSIZE];
+   char readbuf[REALLYBIGBUFSIZE];
    int efd;
 
-   while((i = getopt(argc, argv, "c:dvVh")) > 0){
+   while((i = getopt(argc, argv, "c:L:dvVh")) > 0){
       switch(i){
 
         case 'c' :
                    configfile = optarg;
                    break;
 
+        case 'L':
+                   loglevel = atoi(optarg);
+                   break;
+
         case 'd' :
                    daemonise = 1;
                    break;
@@ -196,8 +205,8 @@ int main(int argc, char **argv){
    set_signal_handler(SIGSEGV, p_clean_exit);
 
    set_signal_handler(SIGPIPE, SIG_IGN);
+   set_signal_handler(SIGALRM, SIG_IGN);
 
-   set_signal_handler(SIGALRM, check_for_client_timeout);
    set_signal_handler(SIGHUP, initialise_configuration);
 
    // calloc() initialitizes the allocated memory
@@ -218,10 +227,8 @@ int main(int argc, char **argv){
    if(daemonise == 1 && daemon(1, 0) == -1) fatal(ERR_DAEMON);
 #endif
 
-   alarm(cfg.check_for_client_timeout_interval);
-
    for(;;){
-      int n = epoll_wait(efd, events, cfg.max_connections, -1);
+      int n = epoll_wait(efd, events, cfg.max_connections, 1000);
       for(i=0; i<n; i++){
 
          // Office365 sometimes behaves oddly: when it receives the 250 OK
@@ -232,7 +239,7 @@ int main(int argc, char **argv){
             if(cfg.verbosity >= _LOG_EXTREME) syslog(LOG_PRIORITY, "ERROR: the remote end hung up without sending QUIT");
             session = get_session_by_socket(sessions, cfg.max_connections, events[i].data.fd);
             if(session)
-               tear_down_session(sessions, session->slot, &num_connections);
+               tear_down_session(sessions, session->slot, &num_connections, "hungup");
             else
                close(events[i].data.fd);
             continue;
@@ -300,8 +307,6 @@ int main(int argc, char **argv){
             time(&(session->lasttime));
 
             while(1){
-               memset(readbuf, 0, sizeof(readbuf));
-
                if(session->net.use_ssl == 1)
                   readlen = SSL_read(session->net.ssl, (char*)&readbuf[0], sizeof(readbuf)-1);
                else
@@ -322,6 +327,7 @@ int main(int argc, char **argv){
                   break;
                }
 
+               readbuf[readlen] = '\0';
                handle_data(session, &readbuf[0], readlen, &cfg);
 
                if(session->protocol_state == SMTP_STATE_BDAT && session->bad == 1){
@@ -333,12 +339,14 @@ int main(int argc, char **argv){
             /* Don't wait until the remote client closes the connection after he sent the QUIT command */
 
             if(done || session->protocol_state == SMTP_STATE_FINISHED){
-               tear_down_session(sessions, session->slot, &num_connections);
+               tear_down_session(sessions, session->slot, &num_connections, "done");
             }
          }
 
 
       }
+
+      check_for_client_timeout();
    }
 
    return 0;

src/piler.c

@@ -103,7 +103,7 @@ void child_sighup_handler(int sig){
 
 int perform_checks(char *filename, struct session_data *sdata, struct data *data, struct parser_state *parser_state, struct config *cfg){
 
-   if(cfg->security_header && parser_state->found_security_header == 0){
+   if(cfg->security_header[0] && parser_state->found_security_header == 0){
       syslog(LOG_PRIORITY, "%s: discarding: missing security header", filename);
       return ERR_DISCARDED;
    }
@@ -123,9 +123,13 @@ int perform_checks(char *filename, struct session_data *sdata, struct data *data
 
    make_digests(sdata, cfg);
 
+   // A normal header is much bigger than 10 bytes. We get here for header-only
+   // messages without a Message-ID: line. I believe that no such message is valid, and
+   // it's a reasonable to discard it, and not allowing it to fill up the error directory.
+
    if(sdata->hdr_len < 10){
-      syslog(LOG_PRIORITY, "%s: invalid message, hdr_len: %d", filename, sdata->hdr_len);
-      return ERR;
+      syslog(LOG_PRIORITY, "%s: discarding: a header-only message without a Message-ID line", filename);
+      return ERR_DISCARDED;
    }
 
    int rc = process_message(sdata, parser_state, data, cfg);
@@ -283,10 +287,19 @@ void child_main(struct child *ptr){
 
       sig_block(SIGHUP);
 
-      if(open_database(&sdata, &cfg) == OK){
+      int sphxopen = 0;
+      if(cfg.rtindex && open_sphx(&sdata, &cfg) == OK){
+         sphxopen = 1;
+      }
+
+      if((cfg.rtindex == 0 || sphxopen == 1) && open_database(&sdata, &cfg) == OK){
          ptr->messages += process_dir(dir, &sdata, &data, &cfg);
          close_database(&sdata);
 
+         if(cfg.rtindex){
+            close_sphx(&sdata);
+         }
+
          sleep(1);
       }
       else {

src/piler.h

@@ -37,7 +37,8 @@ int do_av_check(char *filename, struct config *cfg);
 
 int make_digests(struct session_data *sdata, struct config *cfg);
 void digest_file(char *filename, char *digest);
-void digest_string(char *s, char *digest);
+void digest_string(char *digestname, char *s, char *digest);
+void create_md5_from_email_address(char *puf, char *md5buf);
 
 void remove_stripped_attachments(struct parser_state *state);
 int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
@@ -50,6 +51,7 @@ int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr);
 
 struct config read_config(char *configfile);
 
+void createdir(char *path, uid_t uid, gid_t gid, mode_t mode);
 void check_and_create_directories(struct config *cfg, uid_t uid, gid_t gid);
 
 void update_counters(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
@@ -70,7 +72,7 @@ void load_mydomains(struct session_data *sdata, struct data *data, struct config
 int is_email_address_on_my_domains(char *email, struct data *data);
 
 int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg);
-void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections);
+void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections, char *reason);
 struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int max_connections, int socket);
 void write_envelope_addresses(struct smtp_session *session, struct config *cfg);
 void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg);

src/pileraget.c

@@ -46,7 +46,7 @@ int main(int argc, char **argv){
       return 1;
    }
 
-   snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
+   snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, argv[1][24], argv[1][25], argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
    if(stat(filename, &st)){
       snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][RND_STR_LEN-6], argv[1][RND_STR_LEN-5], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));

src/pilerexport.c

@@ -25,14 +25,19 @@ extern int optind;
 int dryrun = 0;
 int exportall = 0;
 int verification_status = 0;
-int rc = 0;
 int export_to_stdout = 0;
 char *query=NULL;
 int verbosity = 0;
 int max_matches = 1000;
 char *index_list = "main1,dailydelta1,delta1";
+struct passwd *pwd;
 regex_t regexp;
 char *zipfile = NULL;
+struct zip *zip = NULL;
+uint64 *zip_ids = NULL;
+int zip_counter = 0;
+int zip_batch = 2000;
+int max_files_in_export_dir = 0;
 
 int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg);
 
@@ -54,8 +59,10 @@ void usage(){
    printf("    -i <index list>                   Sphinx indices to use  (default: %s)\n", index_list);
 #if LIBZIP_VERSION_MAJOR >= 1
    printf("    -z <zip file>                     Write exported EML files to a zip file\n");
+   printf("    -Z <batch size>                   Zip batch size. Valid range: 10-10000, default: 2000\n");
 #endif
    printf("    -A                                Export all emails from archive\n");
+   printf("    -D <max files>                    Max. number of files to put in a single directory, default: 2000\n");
    printf("    -o                                Export emails to stdout\n");
    printf("    -d                                Dry run\n");
 
@@ -130,46 +137,27 @@ int append_email_to_buffer(char **buffer, char *email){
 }
 
 
-int append_string_to_buffer(char **buffer, char *str){
-   int arglen;
-   char *s=NULL;
-
-   arglen = strlen(str);
-
-   if(!*buffer){
-      *buffer = malloc(arglen+1);
-      memset(*buffer, 0, arglen+1);
-      memcpy(*buffer, str, arglen);
-   }
-   else {
-      int len = strlen(*buffer);
-      s = realloc(*buffer, len + arglen+1);
-      if(!s) return 1;
-
-      *buffer = s;
-
-      memset(*buffer+len, 0, arglen+1);
-      memcpy(*buffer+len, str, arglen);
-   }
-
-   return 0;
-}
-
-
 uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *where_condition, uint64 last_id, int *num, struct config *cfg){
    MYSQL_ROW row;
    uint64 id=0;
-   char s[SMALLBUFSIZE];
+   char s[MAXBUFSIZE];
+   int rc=0;
 
    *num = 0;
 
    if(!where_condition) return id;
 
-   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE id IN (", SQL_METADATA_TABLE);
+   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE id IN (", SQL_METADATA_TABLE);
    rc += append_string_to_buffer(&query, s);
 
    snprintf(s, sizeof(s)-1, "SELECT id FROM %s WHERE %s AND id > %llu ORDER BY id ASC LIMIT 0,%d", index_list, where_condition, last_id, max_matches);
 
+   if(dryrun){
+      printf("sphinx query: %s\n", s);
+   }
+
+   syslog(LOG_PRIORITY, "sphinx query: %s", s);
+
    if(mysql_real_query(&(sdata2->mysql), s, strlen(s)) == 0){
       MYSQL_RES *res = mysql_store_result(&(sdata2->mysql));
       if(res != NULL){
@@ -187,6 +175,7 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *
    }
 
    if(!rc) export_emails_matching_to_query(sdata, query, cfg);
+   else printf("error: append_string_to_buffer() in run_query()\n");
 
    free(query);
    query = NULL;
@@ -233,15 +222,16 @@ void export_emails_matching_id_list(struct session_data *sdata, struct session_d
 
 int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain, int minsize, int maxsize, unsigned long startdate, unsigned long stopdate){
    char s[SMALLBUFSIZE];
+   int rc=0;
 
    if(exportall == 1){
-      rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM ");
+      rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM ");
       rc += append_string_to_buffer(&query, SQL_METADATA_TABLE);
       rc += append_string_to_buffer(&query, " WHERE deleted=0 ");
       return rc;
    }
 
-   snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE deleted=0 ", SQL_MESSAGES_VIEW);
+   snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE deleted=0 ", SQL_MESSAGES_VIEW);
 
    rc = append_string_to_buffer(&query, s);
 
@@ -320,35 +310,36 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain
 }
 
 #if LIBZIP_VERSION_MAJOR >= 1
-int write_to_zip_file(char *filename){
-   struct zip *z=NULL;
-   int errorp, ret=ERR;
+void zip_flush(){
+   zip_close(zip);
 
-   z = zip_open(zipfile, ZIP_CREATE, &errorp);
-   if(!z){
-      printf("error: error creating zip file=%s, error code=%d\n", zipfile, errorp);
-      return ret;
+   zip = NULL;
+   zip_counter = 0;
+
+   if(!zip_ids) return;
+
+   for(int i=0; i<zip_batch; i++){
+      if(*(zip_ids+i)){
+         char filename[SMALLBUFSIZE];
+         snprintf(filename, sizeof(filename)-1, "%llu.eml", *(zip_ids+i));
+         unlink(filename);
+      }
    }
 
-   zip_source_t *zs = zip_source_file(z, filename, 0, 0);
-   if(zs && zip_file_add(z, filename, zs, ZIP_FL_ENC_UTF_8) >= 0){
-      ret = OK;
-   } else {
-      printf("error adding file %s: %s\n", filename, zip_strerror(z));
-   }
-
-   zip_close(z);
-
-   return ret;
+   free(zip_ids);
+   zip_ids = NULL;
 }
 #endif
 
 int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg){
    FILE *f;
-   uint64 id, n=0;
+   uint64 id, n=0, dir_counter=0;
    char digest[SMALLBUFSIZE], bodydigest[SMALLBUFSIZE];
    char filename[SMALLBUFSIZE];
+   char export_subdir[SMALLBUFSIZE];
    struct sql sql;
+   int errorp, rc=0, attachments;
+   unsigned long total_attachments=0;
 
    if(prepare_sql_statement(sdata, &sql, s) == ERR) return ERR;
 
@@ -365,6 +356,7 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct
    sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++;
    sql.sql[sql.pos] = &digest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(digest)-2; sql.pos++;
    sql.sql[sql.pos] = &bodydigest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(bodydigest)-2; sql.pos++;
+   sql.sql[sql.pos] = (char *)&attachments; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++;
 
    p_store_results(&sql);
 
@@ -380,6 +372,19 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct
                continue;
             }
 
+            if(max_files_in_export_dir > 0 && n % max_files_in_export_dir == 0){
+               dir_counter++;
+               snprintf(export_subdir, sizeof(export_subdir)-1, "export-%llu", dir_counter);
+               if(n > 0 && chdir("..")){
+                  p_clean_exit("error chdir(\"..\")", 1);
+               }
+
+               createdir(export_subdir, pwd->pw_uid, pwd->pw_gid, 0700);
+               if(chdir(export_subdir)){
+                  p_clean_exit("error chdir to export-* dir", 1);
+               }
+            }
+
             snprintf(filename, sizeof(filename)-1, "%llu.eml", id);
 
             f = fopen(filename, "w");
@@ -401,16 +406,43 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct
                   verification_status = 1;
                }
 
+               if(zipfile){
                #if LIBZIP_VERSION_MAJOR >= 1
-               if(zipfile && write_to_zip_file(filename) == OK){
-                  unlink(filename);
+                  // Open zip file if handler is NULL
+                  if(!zip){
+                     zip = zip_open(zipfile, ZIP_CREATE, &errorp);
+                     if(!zip){
+                        printf("error: error creating zip file=%s, error code=%d\n", zipfile, errorp);
+                        return ERR;
+                     }
+                  }
+
+                  if(!zip_ids) zip_ids = (uint64*) calloc(sizeof(uint64), zip_batch);
+
+                  if(!zip_ids){
+                     printf("calloc error for zip_ids\n");
+                     return ERR;
+                  }
+
+                  zip_source_t *zs = zip_source_file(zip, filename, 0, 0);
+                  if(zs && zip_file_add(zip, filename, zs, ZIP_FL_ENC_UTF_8) >= 0){
+                     *(zip_ids+zip_counter) = id;
+                     zip_counter++;
+                  } else {
+                     printf("error adding file %s: %s\n", filename, zip_strerror(zip));
+                     return ERR;
+                  }
+
+                  if(zip_counter == zip_batch){
+                     zip_flush();
                   }
                #endif
-
+               }
             }
             else printf("cannot open: %s\n", filename);
          }
          else {
+            total_attachments += attachments;
             printf("id:%llu\n", id);
          }
 
@@ -423,6 +455,10 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct
 ENDE:
    close_prepared_statement(&sql);
 
+   if(dryrun){
+      printf("attachments: %lu\n", total_attachments);
+   }
+
    printf("\n");
 
    return rc;
@@ -465,7 +501,9 @@ int main(int argc, char **argv){
             {"start-date",   required_argument,  0,  'a' },
             {"stop-date",    required_argument,  0,  'b' },
             {"zip",          required_argument,  0,  'z' },
+            {"zip-batch",    required_argument,  0,  'Z' },
             {"where-condition", required_argument,  0,  'w' },
+            {"max-files",    required_argument,  0,  'D' },
             {"max-matches",  required_argument,  0,  'm' },
             {"index-list",   required_argument,  0,  'i' },
             {0,0,0,0}
@@ -473,9 +511,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:oAdhv?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:D:oAdhv?", long_options, &option_index);
 #else
-      int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:oAdhv?");
+      int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:D:oAdhv?");
 #endif
 
       if(c == -1) break;
@@ -506,7 +544,10 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&from, optarg);
+                    if(append_email_to_buffer(&from, optarg)){
+                       printf("error: append_email_to_buffer() for from\n");
+                       return 1;
+                    }
 
                     break;
 
@@ -517,7 +558,10 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&to, optarg);
+                    if(append_email_to_buffer(&to, optarg)){
+                       printf("error: append_email_to_buffer() for to\n");
+                       return 1;
+                    }
 
                     break;
 
@@ -528,7 +572,10 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&fromdomain, optarg);
+                    if(append_email_to_buffer(&fromdomain, optarg)){
+                       printf("error: append_email_to_buffer() for fromdomain\n");
+                       return 1;
+                    }
 
                     break;
 
@@ -539,7 +586,10 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&todomain, optarg);
+                    if(append_email_to_buffer(&todomain, optarg)){
+                       printf("error: append_email_to_buffer() for todomain\n");
+                       return 1;
+                    }
 
                     break;
 
@@ -567,6 +617,15 @@ int main(int argc, char **argv){
          case 'z':  zipfile = optarg;
                     break;
 
+         case 'Z':  zip_batch = atoi(optarg);
+                    if(zip_batch < 10 || zip_batch > 10000)
+                       zip_batch = 2000;
+                    break;
+
+         case 'D':  max_files_in_export_dir = atoi(optarg);
+                    if(max_files_in_export_dir < 10 || max_files_in_export_dir > 100000)
+                       max_files_in_export_dir = 2000;
+                    break;
 
          case 'o':
                     export_to_stdout = 1;
@@ -599,6 +658,11 @@ int main(int argc, char **argv){
 
    if(read_key(&cfg)) p_clean_exit(ERR_READING_KEY, 1);
 
+   if(strlen(cfg.username) > 1){
+      pwd = getpwnam(cfg.username);
+      if(!pwd) __fatal(ERR_NON_EXISTENT_USER);
+   }
+
 
    init_session_data(&sdata, &cfg);
 
@@ -632,5 +696,11 @@ int main(int argc, char **argv){
 
    close_database(&sdata);
 
+   if(zipfile){
+   #if LIBZIP_VERSION_MAJOR >= 1
+      zip_flush();
+   #endif
+   }
+
    return verification_status;
 }

src/pilerimport.c

@@ -53,11 +53,14 @@ void usage(){
    printf("    -j <failed folder>                Move failed to import emails to this folder\n");
    printf("    -a <recipient>                    Add recipient to the To:/Cc: list\n");
    printf("    -T <id>                           Update import table at id=<id>\n");
+   printf("    -Z <ms>                           Delay Z milliseconds in between emails being imported\n");
    printf("    -D                                Dry-run, do not import anything\n");
    printf("    -y                                Read pilerexport data from stdin\n");
    printf("    -o                                Only download emails for POP3/IMAP import\n");
    printf("    -r                                Remove imported emails\n");
    printf("    -q                                Quiet mode\n");
+   printf("    -A <timestamp>                    Import emails sent after this timestamp\n");
+   printf("    -B <timestamp>                    Import emails sent before this timestamp\n");
 
    exit(0);
 }
@@ -72,6 +75,9 @@ int main(int argc, char **argv){
    struct data data;
    struct import import;
    struct net net;
+   struct counters counters;
+
+   bzero(&counters, sizeof(counters));
 
    for(i=0; i<MBOX_ARGS; i++) mbox[i] = NULL;
 
@@ -100,6 +106,9 @@ int main(int argc, char **argv){
    import.tot_msgs = 0;
    import.table_id = 0;
    import.folder = NULL;
+   import.delay = 0;
+   import.after = 0;
+   import.before = 0;
 
    data.import = &import;
 
@@ -136,11 +145,14 @@ int main(int argc, char **argv){
             {"timeout",      required_argument,  0,  't' },
             {"start-position",required_argument,  0,  's' },
             {"table-id",     required_argument,  0,  'T' },
+            {"delay",        required_argument,  0,  'Z' },
             {"quiet",        no_argument,        0,  'q' },
             {"recursive",    no_argument,        0,  'R' },
             {"remove-after-import",no_argument,  0,  'r' },
             {"failed-folder",  required_argument,  0,  'j' },
             {"move-folder",  required_argument,  0,  'g' },
+            {"after",        required_argument,  0,  'A' },
+            {"before",       required_argument,  0,  'B' },
             {"only-download",no_argument,        0,  'o' },
             {"read-from-export",no_argument,     0,  'y' },
             {"dry-run",      no_argument,        0,  'D' },
@@ -150,9 +162,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:yDRroqh?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:A:B:yDRroqh?", long_options, &option_index);
 #else
-      int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:yDRroqh?");
+      int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:A:B:yDRroqh?");
 #endif
 
       if(c == -1) break;
@@ -271,6 +283,15 @@ int main(int argc, char **argv){
                     data.import->table_id = atoi(optarg);
                     break;
 
+         case 'Z' :
+                    if(atoi(optarg) < 1){
+                       printf("invalid delay value: %s\n", optarg);
+                       return -1;
+                    }
+
+                    data.import->delay = atoi(optarg);
+                    break;
+
          case 'y' :
                     read_from_pilerexport = 1;
                     break;
@@ -283,6 +304,12 @@ int main(int argc, char **argv){
                     data.quiet = 1;
                     break;
 
+         case 'A' : data.import->after = atol(optarg);
+                    break;
+
+         case 'B' : data.import->before = atol(optarg);
+                    break;
+
          case 'h' :
          case '?' :
                     usage();
@@ -295,7 +322,7 @@ int main(int argc, char **argv){
    }
 
 
-   if(!mbox[0] && !data.import->mboxdir && !data.import->filename && !directory && !imapserver && !pop3server) usage();
+   if(!mbox[0] && !data.import->mboxdir && !data.import->filename[0] && !directory && !imapserver && !pop3server && !read_from_pilerexport) usage();
 
    if(data.import->failed_folder && !can_i_write_directory(data.import->failed_folder)){
       printf("cannot write failed directory '%s'\n", data.import->failed_folder);
@@ -316,6 +343,9 @@ int main(int argc, char **argv){
    /* make sure we don't discard messages without a valid Message-Id when importing manually */
    cfg.archive_emails_not_having_message_id = 1;
 
+   /* The mmap_dedup_test feature is expected to work with the piler daemon only */
+   cfg.mmap_dedup_test = 0;
+
    if(read_key(&cfg)){
       printf("%s\n", ERR_READING_KEY);
       return ERR;
@@ -326,6 +356,7 @@ int main(int argc, char **argv){
 
    if(open_database(&sdata, &cfg) == ERR) return 0;
 
+   if(cfg.rtindex && open_sphx(&sdata, &cfg) == ERR) return 0;
 
    setlocale(LC_CTYPE, cfg.locale);
 
@@ -356,18 +387,18 @@ int main(int argc, char **argv){
 
    load_mydomains(&sdata, &data, &cfg);
 
-   if(data.import->filename[0] != '\0') import_message(&sdata, &data, &cfg);
+   if(data.import->filename[0] != '\0') import_message(&sdata, &data, &counters, &cfg);
 
    if(mbox[0]){
       for(i=0; i<n_mbox; i++){
-         import_from_mailbox(mbox[i], &sdata, &data, &cfg);
+         import_from_mailbox(mbox[i], &sdata, &data, &counters, &cfg);
       }
    }
-   if(data.import->mboxdir) import_mbox_from_dir(data.import->mboxdir, &sdata, &data, &cfg);
-   if(directory) import_from_maildir(&sdata, &data, directory, &cfg);
-   if(imapserver) import_from_imap_server(&sdata, &data, &cfg);
-   if(pop3server) import_from_pop3_server(&sdata, &data, &cfg);
-   if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &cfg);
+   if(data.import->mboxdir) import_mbox_from_dir(data.import->mboxdir, &sdata, &data, &counters, &cfg);
+   if(directory) import_from_maildir(&sdata, &data, directory, &counters, &cfg);
+   if(imapserver) import_from_imap_server(&sdata, &data, &counters, &cfg);
+   if(pop3server) import_from_pop3_server(&sdata, &data, &counters, &cfg);
+   if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &counters, &cfg);
 
    clearrules(data.archiving_rules);
    clearrules(data.retention_rules);
@@ -375,8 +406,14 @@ int main(int argc, char **argv){
 
    clearhash(data.mydomains);
 
+   update_counters(&sdata, &data, &counters, &cfg);
+
+   syslog(LOG_PRIORITY, "server=%s, user=%s, directory=%s, imported=%lld, duplicated=%lld, discarded=%lld", data.import->server, data.import->username, directory, counters.c_rcvd, counters.c_duplicate, counters.c_ignore);
+
    close_database(&sdata);
 
+   if(cfg.rtindex) close_sphx(&sdata);
+
    if(data.quiet == 0) printf("\n");
 
    return 0;

src/pop3.c

@@ -23,16 +23,6 @@
 #include <piler.h>
 
 
-int is_last_complete_pop3_packet(char *s, int len){
-
-   if(*(s+len-5) == '\r' && *(s+len-4) == '\n' && *(s+len-3) == '.' && *(s+len-2) == '\r' && *(s+len-1) == '\n'){
-      return 1;
-   }
-
-   return 0;
-}
-
-
 int connect_to_pop3_server(struct data *data){
    char buf[MAXBUFSIZE];
 
@@ -86,66 +76,86 @@ void get_number_of_total_messages(struct data *data){
 
 
 int pop3_download_email(struct data *data, int i){
-   int n, fd, pos=0, lastpos=0, nreads=0;
-   char *p, buf[MAXBUFSIZE];
-   char aggrbuf[3*MAXBUFSIZE];
+   char *p, buf[MAXBUFSIZE], savedbuf[MAXBUFSIZE], copybuf[2*MAXBUFSIZE];
 
    data->import->processed_messages++;
 
    snprintf(data->import->filename, SMALLBUFSIZE-1, "pop3-tmp-%d-%d.txt", getpid(), i);
    unlink(data->import->filename);
 
-   fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
+   int fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
    if(fd == -1){
       printf("cannot open: %s\n", data->import->filename);
       return ERR;
    }
 
+   memset(savedbuf, 0, sizeof(savedbuf));
+
    snprintf(buf, sizeof(buf)-1, "RETR %d\r\n", i);
    write1(data->net, buf, strlen(buf));
 
-   memset(aggrbuf, 0, sizeof(aggrbuf));
+   int nlines = 0;
+   int endofmessage = 0;
+   int savedlen = 0;
+   int n = 0;
 
    while((n = recvtimeoutssl(data->net, buf, sizeof(buf))) > 0){
-      nreads++;
+      int remaininglen = n;
 
-      if(nreads == 1){
+      if(savedlen){
+         memset(copybuf, 0, sizeof(copybuf));
+         memcpy(copybuf, savedbuf, savedlen);
+         memcpy(&copybuf[savedlen], buf, n);
 
-         if(strncmp(buf, "+OK", 3) == 0){
-            p = strchr(&buf[3], '\n');
-            if(p){
-               *p = '\0';
-               pos = strlen(buf)+1;
-               *p = '\n';
-            }
-         }
-         else { printf("error: %s", buf); return ERR; }
+         remaininglen += savedlen;
 
+         savedlen = 0;
+         memset(savedbuf, 0, sizeof(savedbuf));
+
+         p = &copybuf[0];
+      } else {
+         p = &buf[0];
       }
 
-      if((uint)(lastpos + 1 + n) < sizeof(aggrbuf)){
+      int puflen=0;
+      int rc=OK;
+      int nullbyte=0;
 
-         if(nreads == 1){
-            memcpy(aggrbuf+lastpos, buf+pos, n-pos);
-            lastpos += n-pos;
-         }
-         else {
-            memcpy(aggrbuf+lastpos, buf, n);
-            lastpos += n;
-         }
-      }
-      else {
-         if(write(fd, aggrbuf, sizeof(buf)) == -1) printf("ERROR: writing to fd\n");
+      do {
+         char puf[MAXBUFSIZE];
 
-         memmove(aggrbuf, aggrbuf+sizeof(buf), lastpos-sizeof(buf));
-         lastpos -= sizeof(buf);
+         puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf)-1, &rc, &nullbyte);
+         remaininglen -= puflen;
+         nlines++;
 
-         memcpy(aggrbuf+lastpos, buf, n);
-         lastpos += n;
+         if(nlines == 1){
+            if(strncmp(puf, "+OK", 3)){
+               printf("error: %s", puf);
+               return ERR;
+            }
+         } else {
+            if(puf[puflen-1] == '\n'){
+               if(puflen == 3 && puf[0] == '.' && puf[1] == '\r' && puf[2] == '\n'){
+                  endofmessage = 1;
+                  break;
                }
 
-      if(is_last_complete_pop3_packet(aggrbuf, lastpos) == 1){
-         if(write(fd, aggrbuf, lastpos-3) == -1) printf("ERROR: writing to fd\n");
+               int dotstuff = 0;
+               if(puf[0] == '.' && puf[1] != '\r' && puf[1] != '\n') dotstuff = 1;
+
+               if(write(fd, &puf[dotstuff], puflen-dotstuff) == -1) printf("ERROR: writing to fd\n");
+
+            } else if(puflen > 0) {
+               savedlen = puflen;
+               snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
+            }
+         }
+
+         p += puflen;
+
+      } while(puflen > 0);
+
+      if(endofmessage){
          break;
       }
    }
@@ -165,7 +175,7 @@ void pop3_delete_message(struct data *data, int i){
 }
 
 
-void process_pop3_emails(struct session_data *sdata, struct data *data, struct config *cfg){
+void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    char buf[MAXBUFSIZE];
 
    data->import->processed_messages = 0;
@@ -181,7 +191,7 @@ void process_pop3_emails(struct session_data *sdata, struct data *data, struct c
          if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/data->import->total_messages); fflush(stdout); }
 
          if(data->import->dryrun == 0){
-            int rc = import_message(sdata, data, cfg);
+            int rc = import_message(sdata, data, counters, cfg);
 
             if(data->import->remove_after_import == 1 && rc == OK){
                pop3_delete_message(data, i);

src/reindex.c

@@ -75,12 +75,10 @@ uint64 get_max_meta_id(struct session_data *sdata){
 
 uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *data, uint64 from_id, uint64 to_id, struct config *cfg){
    char s[SMALLBUFSIZE];
-   uint64 stored_id=0, reindexed=0, delta;
+   uint64 stored_id=0, reindexed=0;
    struct parser_state state;
    struct sql sql;
 
-   delta = to_id - from_id;
-
    if(cfg->enable_folders == 1)
       snprintf(s, sizeof(s)-1, "SELECT m.`id`, `piler_id`, `arrived`, `sent`, f.folder_id FROM %s m, %s f WHERE m.id=f.id AND (m.id BETWEEN %llu AND %llu) AND `deleted`=0", SQL_METADATA_TABLE, SQL_FOLDER_MESSAGE_TABLE, from_id, to_id);
    else
@@ -123,17 +121,28 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da
 
             snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
 
+            struct stat st;
+            sdata->tot_len = stat(filename, &st) == 0 ? st.st_size : 0;
+
+            sdata->internal_sender = sdata->internal_recipient = sdata->external_recipient = sdata->direction = 0;
+            memset(sdata->attachments, 0, SMALLBUFSIZE);
+
             state = parse_message(sdata, 1, data, cfg);
             post_parse(sdata, &state, cfg);
 
             rc = store_index_data(sdata, &state, data, stored_id, cfg);
 
+            unlink(sdata->tmpframe);
+            remove_stripped_attachments(&state);
+
             if(rc == OK) reindexed++;
             else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename);
 
             unlink(filename);
 
             if(progressbar){
+               uint64 delta = to_id - from_id + 1;
+
                printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta));
                fflush(stdout);
             }
@@ -236,6 +245,11 @@ int main(int argc, char **argv){
       p_clean_exit("cannot connect to mysql server", 1);
    }
 
+   if(cfg.rtindex && open_sphx(&sdata, &cfg) == ERR){
+      p_clean_exit("cannot connect to 127.0.0.1:9306", 1);
+   }
+
+
    load_rules(&sdata, data.folder_rules, SQL_FOLDER_RULE_TABLE);
 
    if(folder){
@@ -263,6 +277,7 @@ int main(int argc, char **argv){
    clearhash(data.mydomains);
 
    close_database(&sdata);
+   if(cfg.rtindex) close_sphx(&sdata);
 
    return 0;
 }

src/rules.c

@@ -118,7 +118,6 @@ int append_rule(struct node *xhash[], struct rule_cond *rule_cond){
 
 struct rule *create_rule_item(struct rule_cond *rule_cond){
    struct rule *h=NULL;
-   char empty = '\0';
    int len;
 
    if(rule_cond == NULL) return NULL;
@@ -144,16 +143,16 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){
    h->emptyfrom = h->emptyto = h->emptysubject = h->emptyaname = h->emptyatype = 0;
 
 
-   if(rule_cond->from == NULL || strlen(rule_cond->from) < 1){ rule_cond->from[0] = empty; h->emptyfrom = 1; }
+   if(rule_cond->from[0] == 0){ h->emptyfrom = 1; }
    if(regcomp(&(h->from), rule_cond->from, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->to == NULL || strlen(rule_cond->to) < 1){ rule_cond->to[0] = empty; h->emptyto = 1; }
+   if(rule_cond->to[0] == 0){ h->emptyto = 1; }
    if(regcomp(&(h->to), rule_cond->to, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->subject == NULL || strlen(rule_cond->subject) < 1){ rule_cond->subject[0] = empty; h->emptysubject = 1; }
+   if(rule_cond->subject[0] == 0){ h->emptysubject = 1; }
    if(regcomp(&(h->subject), rule_cond->subject, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->body == NULL || strlen(rule_cond->body) < 1){ rule_cond->body[0] = empty; h->emptybody = 1; }
+   if(rule_cond->body[0] == 0){ h->emptybody = 1; }
    if(regcomp(&(h->body), rule_cond->body, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
    h->spam = rule_cond->spam;
@@ -161,20 +160,16 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){
    h->folder_id = rule_cond->folder_id;
 
    h->size = rule_cond->size;
-
-   if(rule_cond->_size == NULL) rule_cond->_size[0] = empty;
    snprintf(h->_size, 3, "%s", rule_cond->_size);
 
-   if(rule_cond->attachment_name == NULL || strlen(rule_cond->attachment_name) < 1){ rule_cond->attachment_name[0] = empty; h->emptyaname = 1; }
+   if(rule_cond->attachment_name[0] == 0){ h->emptyaname = 1; }
    if(regcomp(&(h->attachment_name), rule_cond->attachment_name, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->attachment_type == NULL || strlen(rule_cond->attachment_type) < 1){ rule_cond->attachment_type[0] = empty; h->emptyatype = 1; }
+   if(rule_cond->attachment_type[0] == 0){ h->emptyatype = 1; }
    if(regcomp(&(h->attachment_type), rule_cond->attachment_type, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
 
    h->attachment_size = rule_cond->attachment_size;
-
-   if(rule_cond->_attachment_size == NULL) rule_cond->_attachment_size[0] = empty;
    snprintf(h->_attachment_size, 3, "%s", rule_cond->_attachment_size);
 
    len = strlen(rule_cond->domain)+8 + strlen(rule_cond->from)+6 + strlen(rule_cond->to)+4 + strlen(rule_cond->subject)+9 + strlen(rule_cond->body)+6 + strlen(rule_cond->_size)+6 + strlen(rule_cond->attachment_name)+10 + strlen(rule_cond->attachment_type)+10 + strlen(rule_cond->_attachment_size)+10 + 9 + 15 + 15;

src/session.c

@@ -9,6 +9,36 @@
 int get_session_slot(struct smtp_session **sessions, int max_connections);
 void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg);
 
+#define GOT_CRLF_DOT_CRLF(p) *p == '\r' && *(p+1) == '\n' && *(p+2) == '.' && *(p+3) == '\r' && *(p+4) == '\n' ? 1 : 0
+
+uint64 get_sessions_total_memory(struct smtp_session **sessions, int max_connections){
+   uint64 total = 0;
+
+   for(int i=0; i<max_connections; i++){
+      if(sessions[i]) total += sessions[i]->bufsize;
+   }
+
+   return total;
+}
+
+
+/*
+ * If the sending party sets the email size when it sends the "mail from"
+ * part in the smtp transaction, eg. MAIL FROM:<jajaja@akakak.lo> size=509603
+ * then piler-smtp could know the email size in advance and could do
+ * a better estimate on the allowed number of smtp sessions.
+ */
+
+uint64 get_sessions_total_expected_mail_size(struct smtp_session **sessions, int max_connections){
+   uint64 total = 0;
+
+   for(int i=0; i<max_connections; i++){
+      if(sessions[i]) total += sessions[i]->mail_size;
+   }
+
+   return total;
+}
+
 
 int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg){
    int slot;
@@ -31,6 +61,21 @@ int start_new_session(struct smtp_session **sessions, int socket, int *num_conne
       return -1;
    }
 
+   /*
+    * We are under the max_smtp_memory threshold
+    */
+
+   uint64 expected_total_mail_size = get_sessions_total_expected_mail_size(sessions, cfg->max_connections);
+   uint64 total_memory = get_sessions_total_memory(sessions, cfg->max_connections);
+
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "DEBUG: total smtp memory allocated: %llu, expected total size: %llu", total_memory, expected_total_mail_size);
+
+   if(total_memory > cfg->max_smtp_memory || expected_total_mail_size > cfg->max_smtp_memory){
+      syslog(LOG_PRIORITY, "ERROR: too much memory consumption: %llu", total_memory);
+      send(socket, SMTP_RESP_451_ERR_TOO_MANY_REQUESTS, strlen(SMTP_RESP_451_ERR_TOO_MANY_REQUESTS), 0);
+      return -1;
+   }
+
    slot = get_session_slot(sessions, cfg->max_connections);
 
    if(slot >= 0 && sessions[slot] == NULL){
@@ -84,11 +129,8 @@ struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int m
 
 
 void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg){
-   int i;
-
    session->slot = slot;
 
-   session->buflen = 0;
    session->protocol_state = SMTP_STATE_INIT;
 
    session->cfg = cfg;
@@ -99,42 +141,49 @@ void init_smtp_session(struct smtp_session *session, int slot, int sd, char *cli
    session->net.ctx = NULL;
    session->net.ssl = NULL;
 
-   session->last_data_char = 0;
+   session->nullbyte = 0;
 
-   session->fd = -1;
-
-   memset(session->mailfrom, 0, SMALLBUFSIZE);
-
-   session->num_of_rcpt_to = 0;
-   for(i=0; i<MAX_RCPT_TO; i++) memset(session->rcptto[i], 0, SMALLBUFSIZE);
-
-   memset(session->buf, 0, MAXBUFSIZE);
    snprintf(session->remote_host, sizeof(session->remote_host)-1, "%s", client_addr);
 
-   reset_bdat_counters(session);
+   session->buf = NULL;
+   session->buflen = 0;
+   session->bufsize = 0;
 
-   time(&(session->lasttime));
+   reset_smtp_session(session);
 }
 
 
 void free_smtp_session(struct smtp_session *session){
-
    if(session){
 
+      if(session->buf != NULL){
+         free(session->buf);
+      }
+
       if(session->net.use_ssl == 1){
          SSL_shutdown(session->net.ssl);
          SSL_free(session->net.ssl);
       }
 
-      if(session->net.ctx) SSL_CTX_free(session->net.ctx);
+      if(session->net.ctx){
+         SSL_CTX_free(session->net.ctx);
+      }
 
       free(session);
    }
 }
 
 
-void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections){
-   syslog(LOG_PRIORITY, "disconnected from %s on fd=%d (%d active connections)", sessions[slot]->remote_host, sessions[slot]->net.socket, (*num_connections)-1);
+void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections, char *reason){
+   if(sessions[slot] == NULL){
+      syslog(LOG_PRIORITY, "session already torn down, slot=%d, reason=%s (%d active connections)", slot, reason, *num_connections);
+      return;
+   }
+
+   if(*num_connections > 0) (*num_connections)--;
+
+   syslog(LOG_PRIORITY, "disconnected from %s on fd=%d, slot=%d, reason=%s (%d active connections)",
+          sessions[slot]->remote_host, sessions[slot]->net.socket, slot, reason, *num_connections);
 
    close(sessions[slot]->net.socket);
 
@@ -147,61 +196,129 @@ void tear_down_session(struct smtp_session **sessions, int slot, int *num_connec
 
    free_smtp_session(sessions[slot]);
    sessions[slot] = NULL;
+}
 
-   (*num_connections)--;
+
+inline int get_last_newline_position(char *buf, int buflen){
+   int i;
+
+   for(i=buflen; i>0; i--){
+      if(*(buf+i) == '\n'){
+         i++;
+         break;
+      }
+   }
+
+   return i;
+}
+
+
+void flush_buffer(struct smtp_session *session){
+   // In the DATA phase skip the 1st character if it's a dot (.)
+   // and there are more characters before the trailing CR-LF
+   //
+   // See https://www.ietf.org/rfc/rfc5321.html#section-4.5.2 for more
+   for(int i=0; i<session->buflen; i++){
+      if(*(session->buf+i) == '\n' && *(session->buf+i+1) == '.' && *(session->buf+i+2) == '.'){
+         int dst = i + 2;
+         int src = dst + 1;
+         int l = session->buflen - src;
+         memmove(session->buf + dst, session->buf + src, l);
+         session->buflen -= 1;
+      }
+   }
+
+   // Exclude the trailing \r\n.\r\n sequence
+
+   session->buflen -= 5;
+
+   if(write(session->fd, session->buf, session->buflen) != session->buflen){
+      session->bad = 1;
+      syslog(LOG_PRIORITY, "ERROR (line: %d) %s: failed to write %d bytes", __LINE__, __func__, session->buflen);
+   }
+
+   session->tot_len = session->buflen;
 }
 
 
 void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg){
-   int puflen, rc;
-   char *p, copybuf[BIGBUFSIZE+MAXBUFSIZE], puf[MAXBUFSIZE];
-
-   // if there's something in the saved buffer, then let's merge them
-
-   if(session->buflen > 0){
-      memset(copybuf, 0, sizeof(copybuf));
-
-      memcpy(copybuf, session->buf, session->buflen);
-      memcpy(&copybuf[session->buflen], readbuf, readlen);
-
-      session->buflen = 0;
-      memset(session->buf, 0, MAXBUFSIZE);
-
-      p = &copybuf[0];
-   }
-   else {
-      p = readbuf;
-   }
-
-
-   do {
-      puflen = read_one_line(p, '\n', puf, sizeof(puf)-1, &rc);
-      p += puflen;
-
-      if(puflen > 0){
-         // Update lasttime if we have a line to process
+   // Update lasttime if we have something to process
    time(&(session->lasttime));
 
-         // pass the puffer to process_data() only if there was an '\n'
-         // on the line or the puffer does not start with a period
-         if(session->protocol_state == SMTP_STATE_DATA && (rc == OK || puf[0] != '.')){
-            sig_block(SIGALRM);
-            process_data(session, puf, puflen);
-            sig_unblock(SIGALRM);
+   if(session->protocol_state == SMTP_STATE_BDAT){
+      process_bdat(session, readbuf, readlen, cfg);
+      return;
    }
-         else if(session->protocol_state == SMTP_STATE_BDAT){
-            process_bdat(session, puf, puflen, cfg);
+
+   // realloc memory if the new chunk doesn't fit in
+
+   if(session->buflen + readlen + 10 > session->bufsize){
+      // Handle if the current memory allocation for this email is above the max_message_size threshold
+
+      if(session->buflen > cfg->max_message_size){
+         if(session->too_big == 0) syslog(LOG_PRIORITY, "ERROR: too big email: %d vs %d", session->buflen, cfg->max_message_size);
+         session->bad = 1;
+         session->too_big = 1;
       }
-         else if(rc == OK){
-            process_smtp_command(session, puf, cfg);
+
+      if(session->bad == 0){
+         char *q = realloc(session->buf, session->bufsize + SMTPBUFSIZE);
+         if(q){
+            session->buf = q;
+            memset(session->buf+session->bufsize, 0, SMTPBUFSIZE);
+            session->bufsize += SMTPBUFSIZE;
+         } else {
+            syslog(LOG_PRIORITY, "ERROR: realloc %s %s %d", session->ttmpfile, __func__, __LINE__);
+            session->bad = 1;
          }
-         else {
-            snprintf(session->buf, MAXBUFSIZE-1, "%s", puf);
-            session->buflen = puflen;
       }
    }
 
-   } while(puflen > 0);
+   // process smtp command
+   if(session->protocol_state != SMTP_STATE_DATA){
+
+      // We got ~2 MB of garbage and no valid smtp command
+      // Terminate the connection
+      if(session->buflen + readlen > SMTPBUFSIZE - 10){
+         session->bad = 1;
+      }
+
+      // We are at the beginning of the smtp transaction
+      if(session->bad == 1){
+         write1(&(session->net), SMTP_RESP_451_ERR, strlen(SMTP_RESP_451_ERR));
+         syslog(LOG_PRIORITY, "ERROR: sent 451 temp error back to client %s", session->ttmpfile);
+         return;
+      }
+
+      //printf("got %d *%s*\n", readlen, readbuf);
+
+      memcpy(session->buf + session->buflen, readbuf, readlen);
+      session->buflen += readlen;
+
+      int pos = get_last_newline_position(session->buf, session->buflen);
+
+      if(pos < readlen) return; // no complete command
+
+      process_smtp_command(session, cfg);
+
+      memset(session->buf, 0, session->bufsize);
+      session->buflen = 0;
+
+      return;
+   }
+
+   if(session->bad == 0){
+      memcpy(session->buf + session->buflen, readbuf, readlen);
+      session->buflen += readlen;
+
+      char *p = session->buf + session->buflen - 5;
+      if(session->buflen >= 5 && GOT_CRLF_DOT_CRLF(p)){
+         flush_buffer(session);
+         process_command_period(session);
+      }
+   } else if(strstr(readbuf, "\r\n.\r\n")){
+      process_command_period(session);
+   }
 
 }
 

src/smtp.c

@@ -15,54 +15,59 @@
 #include "smtp.h"
 
 
-void process_smtp_command(struct smtp_session *session, char *buf, struct config *cfg){
+void process_smtp_command(struct smtp_session *session, struct config *cfg){
    char response[SMALLBUFSIZE];
 
-   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "got on fd=%d: *%s*", session->net.socket, buf);
+   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "got on fd=%d: *%s*", session->net.socket, session->buf);
 
-   if(strncasecmp(buf, SMTP_CMD_HELO, strlen(SMTP_CMD_HELO)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_HELO, strlen(SMTP_CMD_HELO)) == 0){
       process_command_helo(session, response, sizeof(response));
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_EHLO, strlen(SMTP_CMD_EHLO)) == 0 ||
-         strncasecmp(buf, LMTP_CMD_LHLO, strlen(LMTP_CMD_LHLO)) == 0){
-      process_command_ehlo_lhlo(session, response, sizeof(response));
+   if(strncasecmp(session->buf, SMTP_CMD_EHLO, strlen(SMTP_CMD_EHLO)) == 0 ||
+         strncasecmp(session->buf, LMTP_CMD_LHLO, strlen(LMTP_CMD_LHLO)) == 0){
+      process_command_ehlo_lhlo(session, response, sizeof(response), cfg);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){
-      process_command_mail_from(session, buf);
+   if(strncasecmp(session->buf, SMTP_CMD_HELP, strlen(SMTP_CMD_HELP)) == 0){
+      send_smtp_response(session, SMTP_RESP_221_PILER_SMTP_OK);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){
-      process_command_rcpt_to(session, buf);
+   if(strncasecmp(session->buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){
+      process_command_mail_from(session);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_DATA, strlen(SMTP_CMD_DATA)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){
+      process_command_rcpt_to(session, cfg);
+      return;
+   }
+
+   if(strncasecmp(session->buf, SMTP_CMD_DATA, strlen(SMTP_CMD_DATA)) == 0){
       process_command_data(session, cfg);
       return;
    }
 
    /* Support only BDAT xxxx LAST command */
-   if(session->cfg->enable_chunking == 1 && strncasecmp(buf, SMTP_CMD_BDAT, strlen(SMTP_CMD_BDAT)) == 0 && strcasestr(buf, "LAST")){
-      get_bdat_size_to_read(session, buf);
+   if(session->cfg->enable_chunking == 1 && strncasecmp(session->buf, SMTP_CMD_BDAT, strlen(SMTP_CMD_BDAT)) == 0 && strcasestr(session->buf, "LAST")){
+      get_bdat_size_to_read(session);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_QUIT, strlen(SMTP_CMD_QUIT)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_QUIT, strlen(SMTP_CMD_QUIT)) == 0){
       process_command_quit(session, response, sizeof(response));
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_RESET, strlen(SMTP_CMD_RESET)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_RESET, strlen(SMTP_CMD_RESET)) == 0){
       process_command_reset(session);
       return;
    }
 
-   if(session->cfg->tls_enable == 1 && strncasecmp(buf, SMTP_CMD_STARTTLS, strlen(SMTP_CMD_STARTTLS)) == 0 && session->net.use_ssl == 0){
+   if(session->cfg->tls_enable == 1 && strncasecmp(session->buf, SMTP_CMD_STARTTLS, strlen(SMTP_CMD_STARTTLS)) == 0 && session->net.use_ssl == 0){
       process_command_starttls(session);
       return;
    }
@@ -71,31 +76,6 @@ void process_smtp_command(struct smtp_session *session, char *buf, struct config
 }
 
 
-void process_data(struct smtp_session *session, char *buf, int buflen){
-   if(session->last_data_char == '\n' && strcmp(buf, ".\r\n") == 0){
-      process_command_period(session);
-   }
-   else {
-      // write line to file
-      int written=0, n_writes=0;
-
-      while(written < buflen) {
-         int len = write(session->fd, buf+written, buflen-written);
-         n_writes++;
-
-         if(len > 0){
-            if(len != buflen) syslog(LOG_PRIORITY, "WARN: partial write: %d/%d bytes (round: %d)", len, buflen, n_writes);
-            written += len;
-            session->tot_len += len;
-         }
-         else syslog(LOG_PRIORITY, "ERROR (line: %d) process_data(): written %d bytes", __LINE__, len);
-      }
-   }
-
-   session->last_data_char = buf[buflen-1];
-}
-
-
 void wait_for_ssl_accept(struct smtp_session *session){
    int rc;
 
@@ -137,7 +117,7 @@ void process_command_helo(struct smtp_session *session, char *buf, int buflen){
 }
 
 
-void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen){
+void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen, struct config *cfg){
    char extensions[SMALLBUFSIZE];
 
    memset(extensions, 0, sizeof(extensions));
@@ -148,7 +128,9 @@ void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int bufl
    if(session->net.use_ssl == 0 && session->cfg->tls_enable == 1) snprintf(extensions, sizeof(extensions)-1, "%s", SMTP_EXTENSION_STARTTLS);
    if(session->cfg->enable_chunking == 1) strncat(extensions, SMTP_EXTENSION_CHUNKING, sizeof(extensions)-strlen(extensions)-2);
 
-   snprintf(buf, buflen-1, SMTP_RESP_250_EXTENSIONS, session->cfg->hostid, extensions);
+   //#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n%s250-SIZE %d\r\n250 8BITMIME\r\n"
+
+   snprintf(buf, buflen-1, SMTP_RESP_250_EXTENSIONS, session->cfg->hostid, cfg->max_message_size, extensions);
 
    send_smtp_response(session, buf);
 }
@@ -166,6 +148,15 @@ int init_ssl(struct smtp_session *session){
       return 0;
    }
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+   SSL_CTX_set_options(session->net.ctx, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3|SSL_OP_NO_TLSv1|SSL_OP_NO_TLSv1_1);
+#else
+   if(SSL_CTX_set_min_proto_version(session->net.ctx, session->cfg->tls_min_version_number) == 0){
+      syslog(LOG_PRIORITY, "failed SSL_CTX_set_min_proto_version() to %s/%d", session->cfg->tls_min_version, session->cfg->tls_min_version_number);
+      return 0;
+   }
+#endif
+
    if(SSL_CTX_set_cipher_list(session->net.ctx, session->cfg->cipher_list) == 0){
       syslog(LOG_PRIORITY, "failed to set cipher list: '%s'", session->cfg->cipher_list);
       return 0;
@@ -193,8 +184,6 @@ void process_command_starttls(struct smtp_session *session){
       session->net.ssl = SSL_new(session->net.ctx);
       if(session->net.ssl){
 
-         SSL_set_options(session->net.ssl, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3);
-
          if(SSL_set_fd(session->net.ssl, session->net.socket) == 1){
             session->net.starttls = 1;
             send_smtp_response(session, SMTP_RESP_220_READY_TO_START_TLS);
@@ -204,34 +193,37 @@ void process_command_starttls(struct smtp_session *session){
                wait_for_ssl_accept(session);
 
             return;
-         } syslog(LOG_PRIORITY, "%s: SSL_set_fd() failed", session->ttmpfile);
-      } syslog(LOG_PRIORITY, "%s: SSL_new() failed", session->ttmpfile);
-   } syslog(LOG_PRIORITY, "SSL ctx is null!");
+         } syslog(LOG_PRIORITY, "ERROR: %s: SSL_set_fd() failed", session->ttmpfile);
+      } syslog(LOG_PRIORITY, "ERROR: %s: SSL_new() failed", session->ttmpfile);
+   } syslog(LOG_PRIORITY, "ERROR: init_ssl()");
 
    send_smtp_response(session, SMTP_RESP_454_ERR_TLS_TEMP_ERROR);
 }
 
 
-void process_command_mail_from(struct smtp_session *session, char *buf){
+void process_command_mail_from(struct smtp_session *session){
+   memset(session->mailfrom, 0, SMALLBUFSIZE);
+
    if(session->protocol_state != SMTP_STATE_HELO && session->protocol_state != SMTP_STATE_PERIOD && session->protocol_state != SMTP_STATE_BDAT){
       send(session->net.socket, SMTP_RESP_503_ERR, strlen(SMTP_RESP_503_ERR), 0);
    }
    else {
-      memset(&(session->ttmpfile[0]), 0, SMALLBUFSIZE);
-      make_random_string((unsigned char*)&(session->ttmpfile[0]), QUEUE_ID_LEN);
       session->protocol_state = SMTP_STATE_MAIL_FROM;
 
-      extractEmail(buf, session->mailfrom);
+      extractEmail(session->buf, session->mailfrom);
 
-      reset_bdat_counters(session);
-      session->tot_len = 0;
+      int mailsize = get_size_from_smtp_mail_from(session->buf);
+
+      reset_smtp_session(session);
+
+      session->mail_size = mailsize;
 
       send_smtp_response(session, SMTP_RESP_250_OK);
    }
 }
 
 
-void process_command_rcpt_to(struct smtp_session *session, char *buf){
+void process_command_rcpt_to(struct smtp_session *session, struct config *cfg){
 
    if(session->protocol_state == SMTP_STATE_MAIL_FROM || session->protocol_state == SMTP_STATE_RCPT_TO){
 
@@ -240,7 +232,15 @@ void process_command_rcpt_to(struct smtp_session *session, char *buf){
       session->protocol_state = SMTP_STATE_RCPT_TO;
 
       if(session->num_of_rcpt_to < MAX_RCPT_TO){
-         extractEmail(buf, session->rcptto[session->num_of_rcpt_to]);
+         extractEmail(session->buf, session->rcptto[session->num_of_rcpt_to]);
+
+         // Check if we should accept archive_address only
+         if(cfg->archive_address[0] && !strstr(cfg->archive_address, session->rcptto[session->num_of_rcpt_to])){
+            syslog(LOG_PRIORITY, "ERROR: Invalid recipient: *%s*", session->rcptto[session->num_of_rcpt_to]);
+            send_smtp_response(session, SMTP_RESP_550_ERR_INVALID_RECIPIENT);
+            return;
+         }
+
          session->num_of_rcpt_to++;
       }
 
@@ -292,13 +292,23 @@ void process_command_period(struct smtp_session *session){
 
    syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d, client=%s, fd=%d, fsync=%ld", session->ttmpfile, session->mailfrom, session->tot_len, session->remote_host, session->net.socket, tvdiff(tv2, tv1));
 
+   if(session->bad == 1 || session->too_big == 1){
+      snprintf(buf, sizeof(buf)-1, SMTP_RESP_451_ERR);
+      unlink(session->ttmpfile);
+      syslog(LOG_PRIORITY, "ERROR: problem in processing, removing %s", session->ttmpfile);
+      if(session->too_big)
+         snprintf(buf, sizeof(buf)-1, "%s", SMTP_RESP_552_ERR_TOO_BIG_EMAIL);
+      else
+         snprintf(buf, sizeof(buf)-1, "%s", SMTP_RESP_451_ERR);
+   } else {
       move_email(session);
-
       snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
+   }
 
+   if(session->buf){
+      memset(session->buf, 0, session->bufsize);
       session->buflen = 0;
-   session->last_data_char = 0;
-   memset(session->buf, 0, SMALLBUFSIZE);
+   }
 
    send_smtp_response(session, buf);
 }
@@ -314,15 +324,32 @@ void process_command_quit(struct smtp_session *session, char *buf, int buflen){
 
 
 void process_command_reset(struct smtp_session *session){
+   reset_smtp_session(session);
    send_smtp_response(session, SMTP_RESP_250_OK);
-
-   session->tot_len = 0;
-   session->fd = -1;
    session->protocol_state = SMTP_STATE_HELO;
-   session->last_data_char = 0;
+}
+
+
+void reset_smtp_session(struct smtp_session *session){
+   session->tot_len = 0;
+   session->mail_size = 0;
+   session->too_big = 0;
+   session->bad = 0;
+
+   session->fd = -1;
+
+   session->num_of_rcpt_to = 0;
+   for(int i=0; i<MAX_RCPT_TO; i++) memset(session->rcptto[i], 0, SMALLBUFSIZE);
 
    reset_bdat_counters(session);
 
+   time(&(session->lasttime));
+
    memset(&(session->ttmpfile[0]), 0, SMALLBUFSIZE);
    make_random_string((unsigned char*)&(session->ttmpfile[0]), QUEUE_ID_LEN);
+
+   if(session->buf){
+      memset(session->buf, 0, session->bufsize);
+   }
+   session->buflen = 0;
 }

src/smtp.h

@@ -7,22 +7,24 @@
 
 #include <piler.h>
 
-void process_smtp_command(struct smtp_session *session, char *buf, struct config *cfg);
+void process_smtp_command(struct smtp_session *session, struct config *cfg);
 void process_data(struct smtp_session *session, char *buf, int buflen);
 
 void send_smtp_response(struct smtp_session *session, char *buf);
 void process_command_helo(struct smtp_session *session, char *buf, int buflen);
-void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen);
+void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen, struct config *cfg);
 void process_command_quit(struct smtp_session *session, char *buf, int buflen);
 void process_command_reset(struct smtp_session *session);
-void process_command_mail_from(struct smtp_session *session, char *buf);
-void process_command_rcpt_to(struct smtp_session *session, char *buf);
+void process_command_mail_from(struct smtp_session *session);
+void process_command_rcpt_to(struct smtp_session *session, struct config *cfg);
 void process_command_data(struct smtp_session *session, struct config *cfg);
 void process_command_period(struct smtp_session *session);
 void process_command_starttls(struct smtp_session *session);
 
 void reset_bdat_counters(struct smtp_session *session);
-void get_bdat_size_to_read(struct smtp_session *session, char *buf);
+void get_bdat_size_to_read(struct smtp_session *session);
 void process_bdat(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg);
 
+void reset_smtp_session(struct smtp_session *session);
+
 #endif

src/smtpcodes.h

@@ -18,6 +18,7 @@
 
 #define SMTP_CMD_HELO "HELO"
 #define SMTP_CMD_EHLO "EHLO"
+#define SMTP_CMD_HELP "HELP"
 #define SMTP_CMD_MAIL_FROM "MAIL FROM:"
 #define SMTP_CMD_RCPT_TO "RCPT TO:"
 #define SMTP_CMD_DATA "DATA"
@@ -32,11 +33,12 @@
 
 // SMTP responses
 
+#define SMTP_RESP_221_PILER_SMTP_OK "221 piler-smtp is OK\r\n"
 #define SMTP_RESP_220_BANNER "220 %s ESMTP\r\n"
 #define SMTP_RESP_220_READY_TO_START_TLS "220 Ready to start TLS\r\n"
 #define SMTP_RESP_221_GOODBYE "221 %s Goodbye\r\n"
 #define SMTP_RESP_250_OK "250 Ok\r\n"
-#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n250-PIPELINING\r\n%s250-SIZE\r\n250 8BITMIME\r\n"
+#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n250-SIZE %d\r\n%s250 8BITMIME\r\n"
 
 #define SMTP_EXTENSION_STARTTLS "250-STARTTLS\r\n"
 #define SMTP_EXTENSION_CHUNKING "250-CHUNKING\r\n"
@@ -49,13 +51,16 @@
 #define SMTP_RESP_421_ERR_TMP "421 %s service not available\r\n"
 #define SMTP_RESP_421_ERR_WRITE_FAILED "421 writing queue file failed\r\n"
 #define SMTP_RESP_421_ERR_ALL_PORTS_ARE_BUSY "421 All server ports are busy\r\n"
+#define SMTP_RESP_451_ERR_TOO_MANY_REQUESTS "451 Too many requests, try again later\r\n"
 #define SMTP_RESP_451_ERR "451 Error in processing, try again later\r\n"
 #define SMTP_RESP_454_ERR_TLS_TEMP_ERROR "454 TLS not available currently\r\n"
 
 #define SMTP_RESP_502_ERR "502 Command not implemented\r\n"
 #define SMTP_RESP_503_ERR "503 Bad command sequence\r\n"
+#define SMTP_RESP_550_ERR_INVALID_RECIPIENT "550 Invalid recipient\r\n"
 #define SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY "550 You are banned by local policy\r\n"
 #define SMTP_RESP_550_ERR "550 Service currently unavailable\r\n"
+#define SMTP_RESP_552_ERR_TOO_BIG_EMAIL "552 Too big email\r\n"
 
 
 // LMTP commands

src/sql.h

@@ -7,7 +7,9 @@
 
 
 int open_database(struct session_data *sdata, struct config *cfg);
+int open_sphx(struct session_data *sdata, struct config *cfg);
 void close_database(struct session_data *sdata);
+void close_sphx(struct session_data *sdata);
 int prepare_sql_statement(struct session_data *sdata, struct sql *sql, char *s);
 void p_query(struct session_data *sdata, char *s);
 int p_exec_stmt(struct session_data *sdata, struct sql *sql);

src/store.c

@@ -228,7 +228,7 @@ int store_file(struct session_data *sdata, char *filename, int len, struct confi
 
 ENDE:
    if(outbuf) free(outbuf);
-   if(z) free(z);
+   if(z) free(z); //-V547
 
    return ret;
 }

src/tai.h

@@ -5,12 +5,12 @@
 #ifndef _TAI_H
    #define _TAI_H
 
+#include "config.h"
+
 #define TAI_PACK 8
 #define TAIA_PACK 16
 #define TIMESTAMP 25
 
-typedef unsigned long long uint64;
-
 struct tai {
    uint64 x;
 };

src/test.c

@@ -153,6 +153,7 @@ int main(int argc, char **argv){
    printf("from: *%s (%s)*\n", state.b_from, state.b_from_domain);
    printf("sender: *%s (%s)*\n", state.b_sender, state.b_sender_domain);
    printf("to: *%s (%s)*\n", state.b_to, state.b_to_domain);
+   printf("journal recipients: *%s*\n", state.b_journal_to);
    printf("reference: *%s*\n", state.reference);
    printf("subject: *%s*\n", state.b_subject);
    printf("body: *%s*\n", state.b_body);
@@ -183,7 +184,7 @@ int main(int argc, char **argv){
    clearhash(data.mydomains);
 
    for(i=1; i<=state.n_attachments; i++){
-      printf("i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s\n", i, state.attachments[i].filename, state.attachments[i].type, state.attachments[i].size, state.attachments[i].internalname, state.attachments[i].digest);
+      printf("i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, dumped: %d, digest: %s\n", i, state.attachments[i].filename, state.attachments[i].type, state.attachments[i].size, state.attachments[i].internalname, state.attachments[i].dumped, state.attachments[i].digest);
       unlink(state.attachments[i].internalname);
    }
 
@@ -195,6 +196,8 @@ int main(int argc, char **argv){
 
    printf("spam: %d\n", sdata.spam_message);
 
+   printf("1st received line: %s\n", state.receivedbuf);
+
    if(sdata.internal_sender == 0 && sdata.internal_recipient == 0) printf("NOT IN mydomains\n");
 
    printf("\n\n");

src/tokenizer.c

@@ -50,6 +50,8 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata,
          continue;
       }
 
+      char md5buf[2*DIGEST_LENGTH+2];
+
       if(state->message_state == MSG_FROM && state->is_1st_header == 1 && strlen(state->b_from) < SMALLBUFSIZE-len-1){
          strtolower(puf);
 
@@ -69,6 +71,11 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata,
 
             if(is_email_address_on_my_domains(puf, data) == 1) sdata->internal_sender = 1;
 
+            if(len >= MAX_EMAIL_ADDRESS_SPHINX_LEN && strlen(state->b_from) < SMALLBUFSIZE-len-1){
+               create_md5_from_email_address(puf, md5buf);
+               memcpy(&(state->b_from[strlen(state->b_from)]), md5buf, strlen(md5buf));
+            }
+
             if(strlen(state->b_from) < SMALLBUFSIZE-len-1){
                split_email_address(puf);
                memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
@@ -88,6 +95,11 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata,
                memcpy(&(state->b_sender_domain), q+1, strlen(q+1)-1);
             }
 
+            if(len >= MAX_EMAIL_ADDRESS_SPHINX_LEN && strlen(state->b_sender) < SMALLBUFSIZE-len-1){
+               create_md5_from_email_address(puf, md5buf);
+               memcpy(&(state->b_sender[strlen(state->b_sender)]), md5buf, strlen(md5buf));
+            }
+
             if(strlen(state->b_sender) < SMALLBUFSIZE-len-1){
                split_email_address(puf);
                memcpy(&(state->b_sender[strlen(state->b_sender)]), puf, len);
@@ -101,12 +113,18 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata,
          q = strchr(puf, '@');
          if(q) fix_plus_sign_in_email_address(puf, &q, &len);
 
-         if((state->message_state == MSG_RECIPIENT || state->message_state == MSG_ENVELOPE_TO) && findnode(state->journal_recipient, puf) == NULL){
+         if((state->message_state == MSG_RECIPIENT || state->message_state == MSG_ENVELOPE_TO) && findnode(state->journal_recipient, puf) == NULL && state->journaltolen < sizeof(state->b_journal_to)-len-1){
             addnode(state->journal_recipient, puf);
             memcpy(&(state->b_journal_to[state->journaltolen]), puf, len);
+            state->journaltolen += len;
             if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: journal rcpt: '%s'", sdata->ttmpfile, puf);
          }
 
+         if(len >= MAX_EMAIL_ADDRESS_SPHINX_LEN){
+            create_md5_from_email_address(puf, md5buf);
+            add_recipient(md5buf, strlen(md5buf), sdata, state, data, cfg);
+         }
+
          add_recipient(puf, len, sdata, state, data, cfg);
       }
       else if(state->message_state == MSG_BODY && len >= (unsigned int)(cfg->min_word_len) && state->bodylen < BIGBUFSIZE-len-1){

systemd/pilersearch.service

@@ -3,8 +3,9 @@ Description=Searchd for piler daemon
 After=network.target mariadb.service
 
 [Service]
-ExecStart=/usr/bin/searchd --config /etc/piler/sphinx.conf
+ExecStart=/usr/bin/searchd --config /etc/piler/manticore.conf
 ExecStartPre=/bin/bash -c "if [[ ! -d /var/run/piler ]]; then mkdir -p /var/run/piler; chown piler:piler /var/run/piler; fi"
+ExecStop=/usr/bin/searchd --config /etc/piler/manticore.conf --stopwait
 PIDFile=/var/run/piler/searchd.pid
 KillMode=process
 Restart=on-failure

tests/cases/05-smtp.inc

@@ -6,29 +6,33 @@ case1() {
 
    setup
 
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox2" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet2" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam0" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam1" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/journal" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/deduptest" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox2"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet2"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam0"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam1"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam2"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/journal"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/deduptest"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/special"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus"  --no-counter
 
 
-   wait_until_emails_are_processed "piler1" 3000
-   docker exec "piler1" su piler -c /usr/libexec/piler/indexer.delta.sh 2>/dev/null
+   wait_until_emails_are_processed "piler1" 3020
+   [[ $RT -eq 1 ]] || docker exec "piler1" su piler -c /usr/libexec/piler/indexer.delta.sh 2>/dev/null
 
-   count_status_values 3000 2892 108 0
+   count_status_values 3020 2909 111 0
 
    test_retrieved_messages_are_the_same "piler1" "piler"
 
    run_05_sphinx_tests
 
-   docker exec "piler1" su piler -c 'php /usr/libexec/piler/generate_stats.php --webui /var/piler/www --start=2015/01/01 --stop=2020/12/31'
+   docker exec "piler1" su piler -c 'php /usr/libexec/piler/generate_stats.php --webui /var/piler/www --start=2015/01/01 --stop=2021/12/31'
 
    docker exec "piler1" su piler -c 'php /usr/libexec/piler/sign.php --webui /var/piler/www --mode time'
+
+   run_import_job "piler1"
 }
 
 
@@ -65,10 +69,23 @@ run_05_sphinx_tests() {
 
 
 append_queries() {
-   append_query "select * from main1,dailydelta1,delta1 WHERE MATCH('@subject budaörsi dc felmérés')" 9
-   append_query "select * from main1,dailydelta1,delta1 WHERE MATCH('@sender gruppi.hu')" 4
-   append_query "select * from main1,dailydelta1,delta1 WHERE attachments > 0 AND MATCH('@subject spam  ')" 85
-   append_query "select * from main1,dailydelta1,delta1 WHERE attachments > 0 AND MATCH('@subject spam @attachment_types image')" 15
-   append_query "select * from main1,dailydelta1,delta1 WHERE MATCH('@(subject,body)  New Task TSK002*')" 29
-   append_query "select * from main1,dailydelta1,delta1 WHERE MATCH('@subject \"virtualfax daily summary\" | \"adsl hibajegy\"')" 11
+   local index="main1,dailydelta1,delta1"
+
+   if [[ $RT -eq 1 ]]; then index="piler1"; fi
+
+   append_query "select * from ${index} WHERE MATCH('@subject budaörsi dc felmérés')" 9
+   append_query "select * from ${index} WHERE MATCH('@sender gruppi.hu')" 4
+   append_query "select * from ${index} WHERE attachments > 0 AND MATCH('@subject spam  ')" 85
+   append_query "select * from ${index} WHERE attachments > 0 AND MATCH('@subject spam @attachment_types image')" 15
+   append_query "select * from ${index} WHERE MATCH('@(subject,body)  New Task TSK002*')" 29
+   append_query "select * from ${index} WHERE MATCH('@subject \"virtualfax daily summary\" | \"adsl hibajegy\"')" 11
+}
+
+
+run_import_job() {
+   local container="$1"
+
+   log "${FUNCNAME[0]}"
+
+   docker exec "$container" su piler -c /usr/libexec/piler/import.sh
 }

tests/setup.inc

@@ -2,52 +2,50 @@ NODE_WORKERS=( "worker0" "worker1" )
 SMTP_GW="smtpgw"
 DOCKER_LIMIT=( --pids-limit 256 --memory 512M )
 
-export CONTAINERS=( "$NODE_GUI" "${NODE_WORKERS[@]}" "$SMTP_GW" "$SINGLE_SERVER" "piler1" "syslog.host" )
+export CONTAINERS=( "$NODE_GUI" "${NODE_WORKERS[@]}" "$SMTP_GW" "$SINGLE_SERVER" "piler1" )
 
 setup() {
-   destroy_containers
-   set -x
    launch_containers
-   set +x
-   create_rules "piler1"
-   add_data_officer "piler1"
+   setup_piler "piler1"
+}
+
+cleanup_package() {
+   local pkg="$1"
+
+   log "Removing ${PACKAGE_DIR}/${pkg}"
+   rm -f "${PACKAGE_DIR}/${pkg}"
 }
 
 launch_containers() {
+   local composefile="docker-compose-piler.yaml"
 
-   log "starting syslog server"
-   docker run -d --net=piler "${DOCKER_LIMIT[@]}" --name syslog.host sutoj/syslog
+   log "${FUNCNAME[0]}"
 
-   log "starting piler"
-   docker run -d --net=piler "${DOCKER_LIMIT[@]}" --name piler1 \
-      -e PACKAGE="$PACKAGE" \
-      -e PILER_HOST="cust1.acts.hu" \
-      -p 127.0.0.1:80:80 -p 25:25 \
-      -v "${PACKAGE_DIR}:/data:ro" \
-      -v "${CONFIG_DIR}/11-aaaa.conf:/etc/rsyslog.d/11-aaaa.conf:ro" \
-      "$docker_image"
+   pushd "${WORKSPACE}/ci/test/docker-compose"
 
-   wait_for_sleep_cycle_in_container "piler1"
+   echo -e "PACKAGE=$PACKAGE\nPILER_HOST=cust1.acts.hu\nRT=$RT" > .env
+
+   destroy_containers
+
+   log "running docker compose up"
+
+   docker compose -f "$composefile" up -d
+
+   wait_until_container_is_healthy "piler1"
+
+   docker compose -f "$composefile" ps
+
+   popd
 }
 
-create_rules() {
+setup_piler() {
    local container="$1"
 
-   echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"acts.hu\",\"acts.hu\"),(\"gtsce.com\",\"gtsce.com\"),(\"datanet.hu\",\"datanet.hu\"),(\"gtsdatanet.hu\",\"gtsdatanet.hu\"),(\"gts.hu\",\"gts.hu\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh
+   log "${FUNCNAME[0]}"
 
-   echo 'echo "insert into archiving_rule (subject) values (\"Android táblagép\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
-   echo 'echo "insert into archiving_rule (\`from\`) values (\"@gmail.com\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
-   echo 'echo "insert into archiving_rule (\`from\`,attachment_type, _attachment_size, attachment_size) values (\"finderis.co.ua\", \"image\", \">\", 100000)"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
-   echo 'echo "insert into archiving_rule (\`to\`) values (\"undisclosed-recipients\")"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
-   echo 'echo "insert into import (\`type\`, username, password, server) values (\"imap\", \"sanyi@aaa.fu\", \"abcde123\", \"imap\")"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
+   pushd "${WORKSPACE}/${PROJECT_ID}/tests"
 
-   echo 'echo "update user set password=\"\$6\$GKL00T\$8jqoFOe3PyAbOCLwKB7JwndwC.IinHrZRkdoQDZUc8vybZ88sA2qomlz5JceNif8fFpkGzZ03ilvQa7tqQx0v1\""| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
+   docker exec "$container" /usr/local/bin/setup-piler.sh
 
-   docker exec "$container" /etc/init.d/rc.piler reload
-}
-
-add_data_officer() {
-   local container="$1"
-   echo 'echo "insert into user (uid, username, realname, password, domain, dn, isadmin) values(2, \"dataofficer\", \"Data officer\", \"\$6\$rX285LfP\$ZxhlacbzKuCcqkaizzBu8SAiYb6.f8K4Us08nUHwSpWMQkNhw4o2rmfKXoTfaM4rnBHUYVK1N4IfBsqN8CAtS/\", \"local\", \"*\", 4)"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh
-   echo 'echo "insert into email (uid, email) values(2, \"do@local\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh
+   popd
 }

tests/setup.sql

@@ -0,0 +1,14 @@
+use piler;
+
+insert into domain (domain, mapped) values("fictive.com","fictive.com"),("address.com","address.com"),("acts.hu","acts.hu"),("gtsce.com","gtsce.com"),("datanet.hu","datanet.hu"),("gtsdatanet.hu","gtsdatanet.hu"),("gts.hu","gts.hu"),("aaa.fu","aaa.fu");
+
+insert into archiving_rule (subject) values ("Android táblagép");
+insert into archiving_rule (`from`) values ("@gmail.com");
+insert into archiving_rule (`from`,attachment_type, _attachment_size, attachment_size) values ("finderis.co.ua", "image", ">", 100000);
+insert into archiving_rule (`to`) values ("undisclosed-recipients");
+insert into import (`type`, username, password, server) values ("imap-ssl", "sanyi@aaa.fu", "abcde123", "imap.aaa.fu");
+
+update user set password="$6$GKL00T$8jqoFOe3PyAbOCLwKB7JwndwC.IinHrZRkdoQDZUc8vybZ88sA2qomlz5JceNif8fFpkGzZ03ilvQa7tqQx0v1";
+
+insert into user (uid, username, realname, password, domain, dn, isadmin) values(2, "dataofficer", "Data officer", "$6$rX285LfP$ZxhlacbzKuCcqkaizzBu8SAiYb6.f8K4Us08nUHwSpWMQkNhw4o2rmfKXoTfaM4rnBHUYVK1N4IfBsqN8CAtS/", "local", "*", 4);
+insert into email (uid, email) values(2, "do@local");

unit_tests/check_attachments.c

@@ -27,7 +27,7 @@ static void test_attachments(struct config *cfg){
 
    TEST_HEADER();
 
-   snprintf(data.licence.hostname, TINYBUFSIZE-1, "yourdomain.com");
+   snprintf(data.licence.hostname, TINYBUFSIZE-1, "example.com");
 
    for(i=0; i<sizeof(tests)/sizeof(struct attachments); i++){
 

unit_tests/check_digest.c

@@ -29,7 +29,7 @@ static void test_digest_string(){
    };
 
    for(i=0; i<sizeof(tests)/sizeof(struct digest_test); i++){
-      digest_string(tests[i].s, &digest[0]);
+      digest_string("sha256", tests[i].s, &digest[0]);
       assert(strcmp(digest, tests[i].digest1) == 0 && "test_digest_string()");
    }
 

unit_tests/check_parser.c

@@ -39,7 +39,7 @@ static void test_parser(struct config *cfg){
       {"18-spam-html-encoding.eml", "<list-435458392@mail.aaa.fu>", "a1 hitelcentrum kft Üveges szilvia a1hitelcentrum@t-online.hu a1hitelcentrum t online hu ", "t-online.hu", "postmaster postmaster@aaa.fu postmaster aaa fu ", "aaa.fu", "postmaster@aaa.fu postmaster aaa fu a1hitelcentrum@t-online.hu a1hitelcentrum t online hu ", "aaa.fu t-online.hu ", "", "TÁJÉKOZTATÁSVargay Péter", 0},
       {"19-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-186.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "", "", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1},
       {"20-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-187.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "", "", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1},
-      {"21-register-tricky-urls.eml", "<E1IBifn-0001un-MD@admin4.theregister.co.uk>", "the register update-49363-08f0f768@list.theregister.co.uk update 49363 08f0f768 list theregister co uk ", "list.theregister.co.uk", "", "", "hello@mail.aaa.fu hello mail aaa fu ", "mail.aaa.fu ", "", "[sp@m] Reg Headlines Friday July 20", 0},
+      {"21-register-tricky-urls.eml", "<E1IBifn-0001un-MD@admin4.theregister.co.uk>", "the register update-49363-08f0f768@list.theregister.co.uk 30cbee0b0f411fcf170416fb9f996c6f update 49363 08f0f768 list theregister co uk ", "list.theregister.co.uk", "", "", "hello@mail.aaa.fu hello mail aaa fu ", "mail.aaa.fu ", "", "[sp@m] Reg Headlines Friday July 20", 0},
       {"30-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", "RE: hxx-ajajajaja.com_ Aaagágyi és kia ttt_webstat hiba", 0},
       {"31-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", "Re: stanhu \"domain not found\"-dal eldobja a @fohu-ra küldött leveleket...", 0},
       {"32-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", "<GD-XXXX/1-2015> www.ujsag.hu new virtual host reg. --> Aaaaaaaaa", 0},

unit_tests/php/FormatTest.php

@@ -1,28 +0,0 @@
-<?php
-
-use PHPUnit\Framework\TestCase;
-
-include_once("webui/system/model.php");
-include_once("webui/model/health/health.php");
-
-final class FormatTest extends TestCase
-{
-
-   public function providerTestTimeFormatValues(){
-      return [
-         ['0', '0.00 ms'],
-         ['15', '15.00 sec'],
-         ['0.87', '870.00 ms']
-      ];
-   }
-
-   /**
-    * @dataProvider providerTestTimeFormatValues
-    */
-
-   public function test_format_time_1($timeval, $expected_result) {
-      $result = ModelHealthHealth::format_time($timeval);
-      $this->assertEquals($result, $expected_result);
-   }
-
-}

unit_tests/php/SplitMessageTest.php

@@ -1,81 +0,0 @@
-<?php
-
-define('CONTENT_TYPE', 'content-type');
-define('TEXT_PLAIN', 'text/plain');
-define('SUBJECT', 'subject');
-define('THIS_IS_A_TEST', 'This is a test');
-
-use PHPUnit\Framework\TestCase;
-
-require_once DIR_BASE . 'system/helper/mime.php';
-
-
-final class SplitMessageTest extends TestCase {
-
-   public function providerTestSplitMessage() {
-      return [
-         ["From: aaa\r\nTo:bbb\r\nSubject: test\r\n\r\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => TEXT_PLAIN)),
-          THIS_IS_A_TEST],
-
-         ["From: aaa\r\nTo:bbb\r\nCC ccc\r\nSubject: test\r\n\r\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => TEXT_PLAIN)),
-          THIS_IS_A_TEST],
-
-         ["From: aaa\nTo:bbb\nSubject: test\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => TEXT_PLAIN)),
-          THIS_IS_A_TEST],
-
-         ["From: aaa\r\nTo:bbb\r\nSubject: test\r\n\r\n\r\n\r\n" . THIS_IS_A_TEST . "\nAaa\n",
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => TEXT_PLAIN)),
-          "\n\n" . THIS_IS_A_TEST . "\nAaa\n"],
-
-         ["From: aaa\r\nTo:bbb\r\nSubject: test\r\nContent-type: text/html\r\n\r\n\r\n" . THIS_IS_A_TEST . "\nAaa\n",
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => 'text/html')),
-          "\n" . THIS_IS_A_TEST . "\nAaa\n"],
-
-         ["From: aaa\nTo:bbb\nSubject: test\nContent-Type: text/plain\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => TEXT_PLAIN)),
-          THIS_IS_A_TEST],
-
-         ["From: aaa\nTo:bbb\nSubject: test\nDate: Sun, 17 Apr 2016 22:40:03 +0800\nDKIM-Signature: v=1; a=rsa-sha1; c=relaxed/relaxed; d=chemoltravel.hu; s=ml;\n\tt=1471888357; bh=A/l/HLQe3HM4Xc4jFxAmhaWVCMU=;\n\th=Date:To:From:Subject:Sender:From:To:Subject:Date;\n\tb=JlEqXiAKBOoT/YyXKTMsXnEphh2J6sXxgNmbKbGybjo3cU1rgQEL0m1h26gl5AaBP\nContent-Type: " . TEXT_PLAIN . "\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', SUBJECT => 'test', 'date' => 'Sun, 17 Apr 2016 22:40:03 +0800', 'dkim-signature' => 'v=1; a=rsa-sha1; c=relaxed/relaxed; d=chemoltravel.hu; s=ml;  t=1471888357; bh=A/l/HLQe3HM4Xc4jFxAmhaWVCMU=;  h=Date:To:From:Subject:Sender:From:To:Subject:Date;  b=JlEqXiAKBOoT/YyXKTMsXnEphh2J6sXxgNmbKbGybjo3cU1rgQEL0m1h26gl5AaBP', CONTENT_TYPE => array('type' => TEXT_PLAIN)),
-          THIS_IS_A_TEST],
-
-         ["From: aaa\nTo:bbb\nSubject: test\nContent-Type: text/PLAIN\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => TEXT_PLAIN)),
-          THIS_IS_A_TEST],
-
-         ["From: aaa\nTo:bbb\nSubject: test\nContent-Type: text/plain; charset=\"ISO-8859-1\"\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', CONTENT_TYPE => array('type' => TEXT_PLAIN, 'charset' => 'ISO-8859-1')),
-          THIS_IS_A_TEST],
-
-         ["From: aaa\nTo:bbb\nSubject: test\nMIME-Version: 1.0\nContent-Type: multipart/alternative; boundary=\"_=_SWIFT_v4_1460476188_145aa333fc0127705a7e904aab6d1957_=_\"\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', 'mime-version' => '1.0', CONTENT_TYPE => array('type' => 'multipart/alternative', 'boundary' => '_=_SWIFT_v4_1460476188_145aa333fc0127705a7e904aab6d1957_=_')),
-          THIS_IS_A_TEST],
-
-         ["From: aaa\nTo:bbb\nSubject: test\nMIME-Version: 1.0\nContent-Type: multipart/alternative;\n boundary=\"_=_SWIFT_v4_1460476188_145aa333fc0127705a7e904aab6d1957_=_\"\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', 'mime-version' => '1.0', CONTENT_TYPE => array('type' => 'multipart/alternative', 'boundary' => '_=_SWIFT_v4_1460476188_145aa333fc0127705a7e904aab6d1957_=_')),
-          THIS_IS_A_TEST],
-
-         ["From: aaa\nTo:bbb\nSubject: test\nMIME-Version: 1.0\nContent-Type: multipart/related;\n\ttype=\"multipart/alternative\";\n\tboundary=\"----=_NextPart_000_0006_01D195BC.69E26510\"\n\n" . THIS_IS_A_TEST,
-          array('from' => 'aaa', 'to' => 'bbb', 'cc' => '', 'date' => '', SUBJECT => 'test', 'mime-version' => '1.0', CONTENT_TYPE => array('type' => 'multipart/alternative', 'boundary' => '----=_NextPart_000_0006_01D195BC.69E26510')),
-          THIS_IS_A_TEST],
-
-
-      ];  
-   }   
-
-
-   /**
-    * @dataProvider providerTestSplitMessage
-    */
-   public function test_split_message($input, $expected_headers, $expected_body) {
-      Piler_Mime_Decode::splitMessage($input, $headers, $body);
-
-      $this->assertEquals($headers, $expected_headers);
-      $this->assertEquals($body, $expected_body);
-   }
-
-
-}

unit_tests/run.sh

@@ -15,10 +15,22 @@ export LD_LIBRARY_PATH=../src
 pushd "$SCRIPT_DIR"
 
 setup_mysql() {
+   if [[ "$DISTRO" == "jammy" ]]; then
+      service mariadb start
+   else
       service mysql start
+   fi
+
    mysql -u piler -ppiler123 piler1 < ../util/db-mysql.sql
 }
 
+run_smtp_tests() {
+   mkdir -p /var/piler/store/00/piler /var/piler/tmp /var/piler/manticore
+   chown -R piler:piler /var/piler/
+   ../src/piler-smtp -L 5 -d
+   ./smtp -s 127.0.0.1
+}
+
 if [[ -v BUILD_NUMBER ]]; then
    setup_mysql
 fi
@@ -32,3 +44,5 @@ fi
 ./check_hash
 ./check_decoder
 ./check_attachments
+
+if [[ -v BUILD_NUMBER ]]; then run_smtp_tests; fi

unit_tests/smtp.c

@@ -4,11 +4,13 @@
 
 #include "test.h"
 
-
 extern char *optarg;
 extern int optind;
 
-char *testmessage = "From: aaa@aaa.fu\nTo: bela@aaa.fu\nMessage-Id: ajajajaja\nSubject: this is a test\n\nAaaaaa.";
+char *testmessage = "From: aaa@aaa.fu\r\nTo: bela@aaa.fu\r\nMessage-Id: ajajajaja\r\nSubject: this is a test\r\n\r\nAaaaaa";
+char *testmessage2 = " and the last line\r\n";
+
+char *recipient = "aaa@worker0";
 
 int helo = 0; // 0=HELO, 1=EHLO
 
@@ -16,12 +18,24 @@ void usage(){
    printf("\nusage: smtp\n\n");
    printf("    -s <smtp server>                  SMTP server\n");
    printf("    -p <smtp port>                    SMTP port (25)\n");
+   printf("    -r <recipient>                    Envelope recipient\n");
    printf("    -t <timeout>                      Timeout in sec (10)\n");
 
    exit(0);
 }
 
 
+int countreplies(char *s){
+   int replies = 0;
+
+   for(; *s; s++){
+      if(*s == '\n') replies++;
+   }
+
+   return replies;
+}
+
+
 void connect_to_smtp_server(char *server, int port, struct data *data){
    int rc;
    char port_string[8], buf[MAXBUFSIZE];
@@ -53,14 +67,15 @@ void connect_to_smtp_server(char *server, int port, struct data *data){
    }
 
    recvtimeoutssl(data->net, buf, sizeof(buf));
-   printf("rcvd: %s", buf);
 
 ENDE:
    freeaddrinfo(res);
 }
 
 
-void send_smtp_command(struct net *net, char *cmd, char *buf, int buflen){
+void send_smtp_command(struct net *net, char *cmd, char *buf, int buflen, int expectedreplies){
+   int tot=0;
+
    if(net == NULL || cmd == NULL) return;
 
    if(net->socket == -1){
@@ -68,25 +83,32 @@ void send_smtp_command(struct net *net, char *cmd, char *buf, int buflen){
       return;
    }
 
-   printf("sent: %s", cmd);
    write1(net, cmd, strlen(cmd));
-   recvtimeoutssl(net, buf, buflen);
-   printf("rcvd: %s", buf);
+
+   while(1){
+      tot += recvtimeoutssl(net, buf+tot, buflen);
+      if(countreplies(buf) == expectedreplies) break;
+   }
 }
 
 
 void send_helo_command(struct net *net){
+   int replies=1;
    char recvbuf[MAXBUFSIZE];
 
    if(helo == 0){
-      send_smtp_command(net, "HELO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1);
-      assert(strncmp(recvbuf, "250 ", 4) == 0 && "HELO");
+      send_smtp_command(net, "HELO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1, replies);
+      ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
    }
    else {
-      send_smtp_command(net, "EHLO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1);
-      assert(strncmp(recvbuf, "250-", 4) == 0 && "EHLO");
-      if(net->use_ssl == 0) assert(strstr(recvbuf, "250-STARTTLS") && "STARTTLS");
-      else assert(strstr(recvbuf, "250-STARTTLS") == NULL && "STARTTLS");
+      //replies = 6;
+      replies = 5;
+      if(net->use_ssl == 1) replies--;
+
+      send_smtp_command(net, "EHLO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1, replies);
+      ASSERT(strncmp(recvbuf, "250-", 4) == 0, recvbuf);
+      if(net->use_ssl == 0){ ASSERT(strstr(recvbuf, "250-STARTTLS"), recvbuf); }
+      else { ASSERT(strstr(recvbuf, "250-STARTTLS") == NULL, recvbuf); }
    }
 }
 
@@ -94,209 +116,392 @@ void send_helo_command(struct net *net){
 static void test_smtp_commands_one_at_a_time(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RCPT");
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "354 ", 4) == 0 && "DATA");
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage);
 
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "PERIOD");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
-static void test_smtp_commands_pipelining(char *server, int port, struct data *data){
+static void test_smtp_commands_one_at_a_time_data_in_2_parts(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+
+   write1(data->net, testmessage, strlen(testmessage));
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage2);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
+
+   close(data->net->socket);
+
+   TEST_FOOTER();
+}
+
+
+/*static void test_smtp_commands_pipelining(char *server, int port, struct data *data){
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+
+   TEST_HEADER();
+
+   connect_to_smtp_server(server, port, data);
+
+   send_helo_command(data->net);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\nQUIT\r\n", testmessage);
 
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 2);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
-}
+
+   TEST_FOOTER();
+}*/
 
 
 static void test_smtp_commands_with_reset_command(char *server, int port, struct data *data){
-   char recvbuf[MAXBUFSIZE];
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+
+   TEST_HEADER();
 
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "RSET\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RSET");
+   send_smtp_command(data->net, "RSET\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "503 ", 4) == 0 && "RCPT");
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "503 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
 static void test_smtp_commands_partial_command(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
    write1(data->net, "M", 1);
-   printf("sent: M\n");
 
-   send_smtp_command(data->net, "AIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, "AIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RCPT");
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "354 ", 4) == 0 && "DATA");
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage);
 
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "PERIOD");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
-static void test_smtp_commands_partial_command_pipelining(char *server, int port, struct data *data){
+/*static void test_smtp_commands_partial_command_pipelining(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
    write1(data->net, "M", 1);
-   printf("sent: M\n");
 
-   send_smtp_command(data->net, "AIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   snprintf(sendbuf, sizeof(sendbuf)-1, "AIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\nQUIT\r\n", testmessage);
 
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 2);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
-}
+
+   TEST_FOOTER();
+}*/
 
 
 static void test_smtp_commands_starttls(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "STARTTLS\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "220 ", 4) == 0 && "STARTTLS");
+   send_smtp_command(data->net, "STARTTLS\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "220 ", 4) == 0, recvbuf);
 
    init_ssl_to_server(data);
    data->net->use_ssl = 1;
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RCPT");
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "354 ", 4) == 0 && "DATA");
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage);
 
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "PERIOD");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
 static void test_smtp_commands_period_command_in_2_parts(char *server, int port, char *part1, char *part2, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   // As long as pipelining support is not reintroduced
+   //
+   /*snprintf(sendbuf, sizeof(sendbuf)-1, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   if(!strstr(recvbuf, "354 ")) recvtimeoutssl(data->net, recvbuf, sizeof(recvbuf)-1);
+   ASSERT(strstr(recvbuf, "354 "), recvbuf);*/
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s%s", testmessage, part1);
    write1(data->net, sendbuf, strlen(sendbuf));
 
    snprintf(sendbuf, sizeof(sendbuf), "%s", part2);
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "QUIT\r\n");
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
 
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
 static void test_smtp_commands_period_command_in_its_own_packet(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   // As long as pipelining support is not reintroduced
+   //
+   /*snprintf(sendbuf, sizeof(sendbuf)-1, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   if(!strstr(recvbuf, "354 ")) recvtimeoutssl(data->net, recvbuf, sizeof(recvbuf)-1);
+   ASSERT(strstr(recvbuf, "354 "), recvbuf);*/
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+
+
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s", testmessage);
    write1(data->net, sendbuf, strlen(sendbuf));
 
    snprintf(sendbuf, sizeof(sendbuf), "\r\n.\r\n");
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "QUIT\r\n");
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
 
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
+}
+
+
+static void test_smtp_commands_with_partial_data_lines(char *server, int port, struct data *data){
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+   int yes=1;
+
+   TEST_HEADER();
+
+   connect_to_smtp_server(server, port, data);
+
+   setsockopt(data->net->socket, IPPROTO_TCP, TCP_NODELAY, &yes, (socklen_t)sizeof(int));
+
+   send_helo_command(data->net);
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "From: aaa@aaa.fu\r\nTo: bela@aaa.fu\r\nMessage-Id: ajajajaja\r\nSubject: this is a test\r\n\r\nAaaaaa");
+   write1(data->net, sendbuf, strlen(sendbuf)); sleep(2);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "jjdkdjkd dkd some garbage.");
+   write1(data->net, sendbuf, strlen(sendbuf)); sleep(2);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "\r\nSome shit again  au aua ua au aua uuu");
+   write1(data->net, sendbuf, strlen(sendbuf)); sleep(2);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, ".\r\nAnd the last line.\r\n.\r\n");
+   write1(data->net, sendbuf, strlen(sendbuf));
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\nQUIT\r\n", testmessage);
+   recvtimeoutssl(data->net, recvbuf, sizeof(recvbuf));
+
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+
+   close(data->net->socket);
+
+   TEST_FOOTER();
+}
+
+
+static void test_smtp_bdat_last_one_at_a_time(char *server, int port, struct data *data){
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+
+   TEST_HEADER();
+
+   connect_to_smtp_server(server, port, data);
+
+   send_helo_command(data->net);
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "BDAT %ld LAST\r\n", strlen(testmessage)+strlen(testmessage2));
+   write1(data->net, sendbuf, strlen(sendbuf));
+
+   write1(data->net, testmessage, strlen(testmessage));
+
+   send_smtp_command(data->net, testmessage2, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
+
+   close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
@@ -317,6 +522,7 @@ int main(int argc, char **argv){
          {
             {"server",       required_argument,  0,  's' },
             {"port",         required_argument,  0,  'p' },
+            {"rcpt",         required_argument,  0,  'r' },
             {"timeout",      required_argument,  0,  't' },
             {"lhlo",         no_argument,        0,  'l' },
             {"help",         no_argument,        0,  'h' },
@@ -325,9 +531,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      int c = getopt_long(argc, argv, "c:s:p:t:lh?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:s:p:t:r:lh?", long_options, &option_index);
 #else
-      int c = getopt(argc, argv, "c:s:p:t:lh?");
+      int c = getopt(argc, argv, "c:s:p:t:r:lh?");
 #endif
 
 
@@ -343,6 +549,10 @@ int main(int argc, char **argv){
                     port = atoi(optarg);
                     break;
 
+         case 'r' :
+                    recipient = optarg;
+                    break;
+
          case 't' :
                     net.timeout = atoi(optarg);
                     break;
@@ -367,21 +577,26 @@ int main(int argc, char **argv){
    data.net = &net;
 
    test_smtp_commands_one_at_a_time(server, port, &data);
-   test_smtp_commands_pipelining(server, port, &data);
+   test_smtp_commands_one_at_a_time_data_in_2_parts(server, port, &data);
+   ////test_smtp_commands_pipelining(server, port, &data);
    test_smtp_commands_with_reset_command(server, port, &data);
    test_smtp_commands_partial_command(server, port, &data);
-   test_smtp_commands_partial_command_pipelining(server, port, &data);
+   ////test_smtp_commands_partial_command_pipelining(server, port, &data);
+
    test_smtp_commands_period_command_in_2_parts(server, port, "\r", "\n.\r\n", &data);
    test_smtp_commands_period_command_in_2_parts(server, port, "\r\n", ".\r\n", &data);
    test_smtp_commands_period_command_in_2_parts(server, port, "\r\n.", "\r\n", &data);
    test_smtp_commands_period_command_in_2_parts(server, port, "\r\n.\r", "\n", &data);
    test_smtp_commands_period_command_in_its_own_packet(server, port, &data);
 
-   helo = 1; // we must use EHLO to get the STARTTLS in the response
-   test_smtp_commands_starttls(server, port, &data);
+   test_smtp_commands_with_partial_data_lines(server, port, &data);
 
+   helo = 1; // we must use EHLO to get the STARTTLS in the response
+
+   test_smtp_commands_starttls(server, port, &data);
+   data.net->use_ssl = 0;
+
+   test_smtp_bdat_last_one_at_a_time(server, port, &data);
 
    return 0;
 }
-
-

unit_tests/test.conf

@@ -10,7 +10,7 @@ encrypt_messages=1
 extra_to_field=X-Envelope-To:
 extract_attachments=1
 helper_timeout=20
-hostid=piler.yourdomain.com
+hostid=piler.example.com
 iv=
 listen_addr=0.0.0.0
 listen_port=25

unit_tests/test.h

@@ -8,6 +8,7 @@
 #include <netdb.h>
 #include <arpa/inet.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <fcntl.h>
 #include <locale.h>
 #include <getopt.h>

util/Makefile.in

@@ -35,6 +35,8 @@ install:
 	$(INSTALL) -m 0755 $(srcdir)/daily-report.php $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/gmail-imap-import.php $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/generate_stats.php $(DESTDIR)$(libexecdir)/piler
+	$(INSTALL) -m 0755 $(srcdir)/get-token.py $(DESTDIR)$(libexecdir)/piler
+	$(INSTALL) -m 0755 $(srcdir)/refresh-token.py $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/healthcheck.sh $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/mailstat.php $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/sign.php $(DESTDIR)$(libexecdir)/piler

util/db-mysql.sql

@@ -9,9 +9,9 @@ create table if not exists `sph_counter` (
 create table if not exists `sph_index` (
   `id` bigint not null,
   `from` tinyblob default null,
-  `to` text(8192) default null,
-  `fromdomain` char(255) default null,
-  `todomain` text(512) default null,
+  `to` blob(8192) default null,
+  `fromdomain` tinyblob default null,
+  `todomain` blob(512) default null,
   `subject` blob(512) default null,
   `arrived` int unsigned not null,
   `sent` int unsigned not null,
@@ -73,7 +73,7 @@ create index `rcpt_idx3` on `rcpt`(`todomain`);
 
 
 drop view if exists `v_messages`;
-create view `v_messages` AS select `metadata`.`id` AS `id`,`metadata`.`piler_id` AS `piler_id`,`metadata`.`from` AS `from`,`metadata`.`fromdomain` AS `fromdomain`,`rcpt`.`to` AS `to`,`rcpt`.`todomain` AS `todomain`,`metadata`.`subject` AS `subject`, `metadata`.`size` AS `size`, `metadata`.`direction` AS `direction`, `metadata`.`sent` AS `sent`, `metadata`.`retained` AS `retained`, `metadata`.`arrived` AS `arrived`, `metadata`.`digest` AS `digest`, `metadata`.`bodydigest` AS `bodydigest`, `metadata`.`deleted` AS `deleted` from (`metadata` join `rcpt`) where (`metadata`.`id` = `rcpt`.`id`);
+create view `v_messages` AS select `metadata`.`id` AS `id`,`metadata`.`piler_id` AS `piler_id`,`metadata`.`from` AS `from`,`metadata`.`fromdomain` AS `fromdomain`,`rcpt`.`to` AS `to`,`rcpt`.`todomain` AS `todomain`,`metadata`.`subject` AS `subject`, `metadata`.`size` AS `size`, `metadata`.`direction` AS `direction`, `metadata`.`sent` AS `sent`, `metadata`.`retained` AS `retained`, `metadata`.`arrived` AS `arrived`, `metadata`.`digest` AS `digest`, `metadata`.`bodydigest` AS `bodydigest`, `metadata`.`deleted` AS `deleted`, `metadata`.`attachments` AS `attachments` from (`metadata` join `rcpt`) where (`metadata`.`id` = `rcpt`.`id`);
 
 
 create table if not exists `attachment` (
@@ -242,7 +242,7 @@ create table if not exists `email_groups` (
 ) ENGINE=InnoDB;
 
 
-create table if not exists `group` (
+create table if not exists `usergroup` (
    `id` bigint unsigned not null auto_increment primary key,
    `groupname` char(128) not null unique
 ) ENGINE=InnoDB;
@@ -344,7 +344,7 @@ create table if not exists `audit` (
    `email` varchar(128) not null,
    `domain` varchar(128) not null,
    `action` int not null,
-   `ipaddr` char(15) not null,
+   `ipaddr` varchar(39) not null,
    `meta_id` bigint unsigned not null,
    `description` varchar(255) default null,
    `vcode` char(64) default null,
@@ -463,7 +463,7 @@ create table if not exists `timestamp` (
   `id` bigint unsigned not null auto_increment,
   `start_id` bigint default 0,
   `stop_id` bigint default 0,
-  `hash_value` char(40),
+  `hash_value` varchar(128),
   `count` int default 0,
   `response_time` bigint default 0,
   `response_string` blob not null,

util/db-upgrade.sql

@@ -0,0 +1,14 @@
+alter table timestamp change column hash_value hash_value varchar(128) default null;
+
+alter table sph_index change column `to` `to` blob(8192) default null;
+alter table sph_index change column fromdomain fromdomain tinyblob default null;
+alter table sph_index change column todomain todomain blob(512) default null;
+
+
+alter table metadata change column subject `subject` blob(512) default null;
+
+
+SELECT Count(*) INTO @exists FROM information_schema.tables WHERE table_schema = 'piler' AND table_type = 'BASE TABLE' AND table_name = 'group';
+SET @query = If(@exists>0, 'RENAME TABLE `group` TO usergroup', 'SELECT 1 from dual');
+PREPARE stmt FROM @query;
+EXECUTE stmt;

util/download-imap.php

@@ -103,7 +103,7 @@ function saveMessages($storage, $folder = '', $num = 0) {
 
    $messages = $storage->piler_batch_fetch(1, $num);
 
-   while(list($k, $v) = each($messages)) {
+   foreach($messages as $k => $v) {
       $uuid = $storage->getUniqueId($k);
 
       $tmpname = "piler-" . $username . "-" . $folder . "-" . $k . "-" . $uuid . ".eml";

util/get-token.py

@@ -0,0 +1,78 @@
+#!/usr/bin/python3
+
+# Based on the https://github.com/UvA-FNWI/M365-IMAP project
+
+from msal import ConfidentialClientApplication, SerializableTokenCache
+import http.server
+import os
+import sys
+import threading
+import urllib.parse
+
+
+redirect_uri = "http://localhost:8745"
+
+ClientId = "c6843299-05c4-4c2e-9398-64dd42f14b6d"  # Fix this value only
+ClientSecret = ""
+Scopes = ['https://outlook.office.com/IMAP.AccessAsUser.All']
+AccessTokenFileName = "access_token"
+RefreshTokenFileName = "refresh_token"
+
+
+cache = SerializableTokenCache()
+app = ConfidentialClientApplication(ClientId,
+                                    client_credential=ClientSecret,
+                                    token_cache=cache, authority=None)
+
+url = app.get_authorization_request_url(Scopes, redirect_uri=redirect_uri)
+
+print("Navigate to the following url in a web browser:\n", url)
+
+
+class Handler(http.server.BaseHTTPRequestHandler):
+    def do_GET(self):
+        parsed_url = urllib.parse.urlparse(self.path)
+        parsed_query = urllib.parse.parse_qs(parsed_url.query)
+        global code
+        code = next(iter(parsed_query['code']), '')
+
+        response_body = b'Success. Look back at your terminal.\r\n'
+        self.send_response(200)
+        self.send_header('Content-Type', 'text/plain')
+        self.send_header('Content-Length', len(response_body))
+        self.end_headers()
+        self.wfile.write(response_body)
+
+        global httpd
+        t = threading.Thread(target=lambda: httpd.shutdown())
+        t.start()
+
+
+code = ''
+
+server_address = ('127.0.0.1', 8745)
+httpd = http.server.HTTPServer(server_address, Handler)
+
+if not os.getenv('SSH_CONNECTION'):
+    httpd.serve_forever()
+
+if code == '':
+    print('After login, you will be redirected to a blank (or error) page ' +
+          'with a url containing an access code. Paste the url below.')
+    resp = input('Response url: ')
+
+    i = resp.find('code') + 5
+    code = resp[i: resp.find('&', i)] if i > 4 else resp
+
+token = app.acquire_token_by_authorization_code(code, Scopes,
+                                                redirect_uri=redirect_uri)
+
+if 'error' in token:
+    print(token)
+    sys.exit("Failed to get access token")
+
+with open(AccessTokenFileName, 'w') as f:
+    f.write(token['access_token'])
+
+with open(RefreshTokenFileName, 'w') as f:
+    f.write(token['refresh_token'])

util/imapfetch.py

@@ -14,6 +14,12 @@ opts = {}
 INBOX = 'INBOX'
 ST_RUNNING = 1
 
+imaplib._MAXLINE = 10000000
+
+def generate_auth_string(user, token):
+    auth_string = f"user={user}\1auth=Bearer {token}\1\1"
+    return auth_string
+
 
 def read_options(filename="", opts={}):
     s = "[piler]\n" + open(filename, 'r').read()
@@ -44,21 +50,35 @@ def read_folder_list(conn):
         if isinstance(folder, type(b'')):
             folder = folder.decode('utf-8')
         elif isinstance(folder, type(())):
-            folder = re.sub(r'\{\d+\}$', '', folder[0]) + folder[1]
+            folder = re.sub(r'\{\d+\}$', '',
+                            folder[0].decode('utf-8')) + folder[1].decode('utf-8')
 
         # The regex should match ' "/" ' and ' "." '
         if folder:
-            f = re.split(r' \"[\/\.]\" ', folder)
+            f = re.split(r' \"[\/\.\\]+\" ', folder)
             result.append(f[1])
 
     return [x for x in result if x not in opts['skip_folders']]
 
 
 def process_folder(conn, folder):
+    # Space in the folder name must be escaped
+    folder = re.sub(r' ', '\\ ', folder)
+
     if opts['verbose']:
         print("Processing {}".format(folder))
 
+    try:
         rc, data = conn.select(folder)
+    except:
+        print("Error processing folder {}".format(folder))
+        return
+
+    if rc != "OK":
+        print("Error processing folder {}, rc={}, response={}".format(folder,
+                                                                      rc, data))
+        return
+
     n = int(data[0])
     if opts['verbose']:
         print("Folder {} has {} messages".format(folder, n))
@@ -67,7 +87,8 @@ def process_folder(conn, folder):
         if opts['id']:
             cursor = opts['db'].cursor()
             data = (ST_RUNNING, n, opts['id'])
-            cursor.execute("UPDATE import SET status=%s, total=total+%s WHERE id=%s", data)
+            cursor.execute("UPDATE import SET status=%s, total=total+%s WHERE id=%s",
+                           data)
             opts['db'].commit()
 
         rc, data = conn.search(None, opts['search'])
@@ -75,9 +96,15 @@ def process_folder(conn, folder):
             rc, data = conn.fetch(num, '(RFC822)')
             if opts['verbose']:
                 print(rc, num)
+            if isinstance(data[0], tuple):
                 opts['counter'] += 1
                 with open("{}.eml".format(opts['counter']), "wb") as f:
                     f.write(data[0][1])
+                    if opts['remove']:
+                        conn.store(num, '+FLAGS', '\\Deleted')
+
+        if opts['remove']:
+            conn.expunge()
 
 
 def main():
@@ -86,10 +113,12 @@ def main():
                         default="/etc/piler/piler.conf")
     parser.add_argument("-s", "--server", type=str, help="imap server")
     parser.add_argument("-P", "--port", type=int, help="port number", default=143)
+    parser.add_argument("--no_ssl", help="Do not use ssl/tls", action='store_true')
     parser.add_argument("-u", "--user", type=str, help="imap user")
     parser.add_argument("-p", "--password", type=str, help="imap password")
+    parser.add_argument("--oauth2-token", type=str, help="oauth2 access token file")
     parser.add_argument("-x", "--skip-list", type=str, help="IMAP folders to skip",
-                        default="junk,trash,spam,draft")
+                        default="junk,trash,spam,draft,\"[Gmail]\"")
     parser.add_argument("-f", "--folders", type=str,
                         help="Comma separated list of IMAP folders to download")
     parser.add_argument("--date", type=str, help="Search before/since a given date," +
@@ -98,6 +127,7 @@ def main():
                         default="/var/piler/imap")
     parser.add_argument("-i", "--import-from-table", action='store_true',
                         help="Read imap conn data from import table")
+    parser.add_argument("-r", "--remove", help="remove downloaded messages", action='store_true')
     parser.add_argument("-v", "--verbose", help="verbose mode", action='store_true')
 
     args = parser.parse_args()
@@ -112,12 +142,22 @@ def main():
     opts['verbose'] = args.verbose
     opts['search'] = 'ALL'
     opts['counter'] = 0
+    opts['use_ssl'] = True
     opts['db'] = None
     opts['id'] = 0
+    opts['access_token'] = ''
+    opts['remove'] = args.remove
 
     if args.date:
         opts['search'] = args.date
 
+    if args.no_ssl:
+        opts['use_ssl'] = False
+
+    if args.oauth2_token:
+        with open(args.oauth2_token, 'r') as f:
+            opts['access_token'] = f.read()
+
     server = ''
     user = ''
     password = ''
@@ -129,7 +169,8 @@ def main():
                                        opts['password'], opts['database'])
 
             cursor = opts['db'].cursor()
-            cursor.execute("SELECT id, server, username, password FROM import WHERE started=0")
+            cursor.execute("SELECT id, server, username, password " +
+                           "FROM import WHERE started=0")
 
             row = cursor.fetchone()
             if row:
@@ -148,12 +189,17 @@ def main():
     if opts['verbose']:
         print("Skipped folder list: {}".format(opts['skip_folders']))
 
-    if args.port == 993:
+    if opts['use_ssl']:
         conn = imaplib.IMAP4_SSL(server)
     else:
         conn = imaplib.IMAP4(server)
 
+    if opts['access_token']:
+        conn.authenticate("XOAUTH2", lambda x: generate_auth_string(
+            user, opts['access_token']))
+    else:
         conn.login(user, password)
+
     conn.select()
 
     if args.folders:

util/indexer.delta.sh.in

@@ -4,12 +4,19 @@ export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
 MAINTMPFILE=/var/run/piler/main.indexer.tmp
 DELTATMPFILE=/var/run/piler/delta.indexer.tmp
 INDEXER="indexer --config SYSCONFDIR/piler/sphinx.conf"
-PRIORITY=mail.error
+PRIORITY=mail.info
 TOUCHFILE=/var/piler/stat/indexer
+CONFIG_FILE=sphinx.conf
 
-if [ -f $MAINTMPFILE ]; then echo "INDEXER ERROR: indexer merging to main index is already running. It started at "`cat $MAINTMPFILE` | logger -p $PRIORITY ; exit 1; fi
+if [[ -f SYSCONFDIR/piler/MANTICORE ]]; then
+   CONFIG_FILE=manticore.conf
+fi
 
-if [ -f $DELTATMPFILE ]; then echo "INDEXER ERROR: delta indexing and merging is already running. It started at "`cat $DELTATMPFILE` | logger -p $PRIORITY ; exit 1; fi
+INDEXER="indexer --config SYSCONFDIR/piler/${CONFIG_FILE}"
+
+if [[ -f "$MAINTMPFILE" ]]; then echo "INDEXER ERROR: indexer merging to main index is already running. It started at $(cat "$MAINTMPFILE")" | logger -p "$PRIORITY" ; exit 1; fi
+
+if [[ -f "$DELTATMPFILE" ]]; then echo "INDEXER ERROR: delta indexing and merging is already running. It started at $(cat "$DELTATMPFILE")" | logger -p "$PRIORITY" ; exit 1; fi
 
 date > $DELTATMPFILE
 
@@ -34,4 +41,3 @@ echo "INDEXER INFO: merging delta to dailydelta started" | logger -p $PRIORITY
 $INDEXER --quiet --merge dailydelta1 delta1 --merge-dst-range deleted 0 0 --rotate
 
 echo "INDEXER INFO: merging delta to dailydelta finished" | logger -p $PRIORITY
-

util/indexer.main.sh.in

@@ -5,17 +5,33 @@ set -o errexit
 set -o pipefail
 
 MAINTMPFILE="/var/run/piler/main.indexer.tmp"
-SPHINX_CONFIG="SYSCONFDIR/piler/sphinx.conf"
-PRIORITY="mail.error"
+CONFIG_FILE="SYSCONFDIR/piler/sphinx.conf"
+INDEXDIR=sphinx
+PRIORITY="mail.info"
 TOUCHFILE="/var/piler/stat/indexer"
 MAIN_INDEX="main1"
 
+if [[ -f SYSCONFDIR/piler/MANTICORE ]]; then
+   CONFIG_FILE=SYSCONFDIR/piler/manticore.conf
+   INDEXDIR=manticore
+fi
+
 export PATH="/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
 
 finish() {
    rm -f "$MAINTMPFILE"
 }
 
+get_index_size() {
+   local mainfiles="$1"
+   local sum=0
+
+   # shellcheck disable=SC2034
+   while read -r a b; do
+      sum=$(( sum + b ))
+   done < <( find /var/piler/${INDEXDIR}/ -type f -name "$mainfiles" -printf "%TY%Tm%Td %s\\n" )
+   printf "%d" $sum
+}
 
 if [[ -f "$MAINTMPFILE" ]]; then
    echo "INDEXER ERROR: indexer merging to main index is already running. It started at $(cat "$MAINTMPFILE")" | logger -p "$PRIORITY"
@@ -30,7 +46,7 @@ trap finish EXIT
 
 echo "INDEXER INFO: merging to main started" | logger -p "$PRIORITY"
 
-indexer --config "$SPHINX_CONFIG" --quiet --merge "$MAIN_INDEX" dailydelta1 --merge-dst-range deleted 0 0 --rotate
+indexer --config "$CONFIG_FILE" --quiet --merge "$MAIN_INDEX" dailydelta1 --merge-dst-range deleted 0 0 --rotate
 
 echo "INDEXER INFO: merging to main finished" | logger -p "$PRIORITY"
 
@@ -38,12 +54,9 @@ sleep 5
 
 echo "INDEXER INFO: resetting daily delta started" | logger -p "$PRIORITY"
 
-indexer --config "$SPHINX_CONFIG" --quiet dailydelta1 --rotate
+indexer --config "$CONFIG_FILE" --quiet dailydelta1 --rotate
 
 echo "INDEXER INFO: resetting daily delta finished" | logger -p "$PRIORITY"
 
-sum=0
-while read -r a b; do
-   sum=$(( sum + b ))
-done < <( find /var/piler/sphinx/ -type f -name main\*.spd -printf "%TY%Tm%Td %s\\n" )
-printf "%d" $sum > /var/piler/stat/main_index_size
+get_index_size "main*.sp[dp]" > /var/piler/stat/total_index_size
+get_index_size "${MAIN_INDEX}*.sp[dp]" > /var/piler/stat/current_main_index_size

util/pilerpurge.py

@@ -9,7 +9,7 @@ import sys
 import syslog
 import time
 
-SQL_PURGE_SELECT_QUERY = "SELECT piler_id, size FROM " +\
+SQL_PURGE_SELECT_QUERY = "SELECT id, piler_id, size FROM " +\
     "metadata WHERE deleted=0 AND retained < UNIX_TIMESTAMP(NOW()) " +\
     "AND id NOT IN (SELECT id FROM rcpt WHERE `to` IN " +\
     "(SELECT email FROM legal_hold)) AND id NOT IN (SELECT " +\
@@ -33,7 +33,9 @@ def read_options(filename="", opts={}):
     opts['password'] = config.get('piler', 'mysqlpwd')
     opts['database'] = config.get('piler', 'mysqldb')
     opts['storedir'] = config.get('piler', 'queuedir')
-
+    opts['rtindex'] = config.getint('piler', 'rtindex', fallback=0)
+    opts['sphxhost'] = config.get('piler', 'sphxhost', fallback='127.0.0.1')
+    opts['sphxport'] = config.getint('piler', 'sphxport', fallback=9306)
     opts['server_id'] = "%02x" % config.getint('piler', 'server_id')
 
 
@@ -54,12 +56,26 @@ def purge_m_files(ids=[], opts={}):
         remove_m_files(ids, opts)
 
         # Set deleted=1 for aged metadata entries
+        # as well as clean other tables
 
         if opts['dry_run'] is False:
             cursor = opts['db'].cursor()
             format = ", ".join(['%s'] * len(ids))
-            cursor.execute("UPDATE metadata SET deleted=1 WHERE piler_id IN " +
+            cursor.execute("UPDATE metadata SET deleted=1, subject=NULL, `from`=''," +
+                           "fromdomain='', message_id='' WHERE piler_id IN " +
                            "(%s)" % (format), ids)
+
+            cursor.execute("DELETE FROM rcpt WHERE id IN (SELECT id FROM metadata " +
+                           "WHERE piler_id IN (%s))" % (format), ids)
+            cursor.execute("DELETE FROM note WHERE id IN (SELECT id FROM metadata " +
+                           "WHERE piler_id IN (%s))" % (format), ids)
+            cursor.execute("DELETE FROM tag WHERE id IN (SELECT id FROM metadata " +
+                           "WHERE piler_id IN (%s))" % (format), ids)
+            cursor.execute("DELETE FROM private WHERE id IN (SELECT id FROM metadata " +
+                           "WHERE piler_id IN (%s))" % (format), ids)
+            cursor.execute("DELETE FROM folder_message WHERE id IN (SELECT id FROM " +
+                           "metadata WHERE piler_id IN (%s))" % (format), ids)
+
             opts['db'].commit()
 
 
@@ -85,7 +101,7 @@ def purge_attachments_by_attachment_id(opts={}):
     cursor = opts['db'].cursor()
 
     cursor.execute("SELECT i, piler_id, attachment_id, refcount FROM " +
-                   "v_attachment WHERE i IN (%s)" %
+                   "v_attachment WHERE refcount=0 AND i IN (%s)" %
                    (format), opts['referenced_attachments'])
 
     while True:
@@ -160,16 +176,27 @@ def unlink(filename="", opts={}):
 
 
 def get_m_file_path(id='', opts={}):
-    return "/".join([opts['storedir'], opts['server_id'], id[8:11], id[32:34],
+    return "/".join([opts['storedir'], id[24:26], id[8:11], id[32:34],
                     id[34:36], id + ".m"])
 
 
 def get_attachment_file_path(piler_id='', attachment_id=0, opts={}):
-    return "/".join([opts['storedir'], opts['server_id'], piler_id[8:11],
+    return "/".join([opts['storedir'], piler_id[24:26], piler_id[8:11],
                     piler_id[32:34], piler_id[34:36], piler_id + ".a" +
                     str(attachment_id)])
 
 
+def purge_index_data(ids=[], opts={}):
+    '''
+    Delete from index data in case of RT index
+    '''
+
+    if opts['rtindex'] == 1 and opts['dry_run'] is False:
+        cursor = opts['sphx'].cursor()
+        a = "," . join([str(x) for x in ids])
+        cursor.execute("DELETE FROM piler WHERE id IN (%s)" % (a))
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("-c", "--config", type=str, help="piler.conf path",
@@ -189,6 +216,7 @@ def main():
     opts['dry_run'] = args.dry_run
     opts['verbose'] = args.verbose
     opts['db'] = None
+    opts['sphx'] = None
     opts['messages'] = 0
     opts['files'] = 0
     opts['size'] = 0
@@ -203,6 +231,8 @@ def main():
         opts['db'] = dbapi.connect(opts['dbhost'], opts['username'],
                                    opts['password'], opts['database'])
 
+        opts['sphx'] = dbapi.connect(host=opts['sphxhost'], port=opts['sphxport'])
+
         if is_purge_enabled(opts) is False:
             syslog.syslog("Purging emails is disabled")
             sys.exit(1)
@@ -215,13 +245,15 @@ def main():
             if rows == ():
                 break
 
-            piler_id = [x[0] for x in rows]
-            size = [x[1] for x in rows]
+            id = [x[0] for x in rows]
+            piler_id = [x[1] for x in rows]
+            size = [x[2] for x in rows]
 
             opts['purged_size'] = opts['purged_size'] + sum(size)
 
             purge_m_files(piler_id, opts)
             purge_attachments_by_piler_id(piler_id, opts)
+            purge_index_data(id, opts)
 
             # It's possible that there's attachment duplication, thus
             # refcount > 0, even though after deleting the duplicates

util/postinstall.sh.in

@@ -36,7 +36,7 @@ load_default_values() {
    MYSQL_ROOT_PASSWORD=""
    MYSQL_SOCKET=""
 
-   SPHINXCFG="${PILER_CONFIG_DIR}/sphinx.conf"
+   SEARCHCFG="${PILER_CONFIG_DIR}/manticore.conf"
 
    WWWGROUP="apache"
    DOCROOT="/var/piler/www"
@@ -196,9 +196,9 @@ askYN() {
 preinstall_check() {
    check_user root
 
-   if [[ "$INDEXER" == "" ]]; then "ERROR: cannot find sphinx indexer"; echo ""; exit ; fi
+   if [[ "$INDEXER" == "" ]]; then "ERROR: cannot find indexer binary"; echo ""; exit ; fi
 
-   if [[ "$SEARCHD" == "" ]]; then "ERROR: cannot find sphinx searchd"; echo ""; exit 0; fi
+   if [[ "$SEARCHD" == "" ]]; then "ERROR: cannot find searchd binary"; echo ""; exit 0; fi
 
    if [[ -f "$KEYFILE" ]]; then echo "ERROR: found existing keyfile (${KEYFILE}), aborting install"; echo ""; exit 0; fi
 }
@@ -281,11 +281,12 @@ make_cron_entries() {
       echo "30   2 * * * ${LIBEXECDIR}/piler/indexer.main.sh";
       echo "40 3 * * * ${LIBEXECDIR}/piler/purge.sh";
       echo "3 * * * * ${LIBEXECDIR}/piler/watch_sphinx_main_index.sh";
-      echo "*/15 * * * * ${INDEXER} --quiet tag1 --rotate --config ${PILER_CONFIG_DIR}/sphinx.conf";
-      echo "*/15 * * * * ${INDEXER} --quiet note1 --rotate --config ${PILER_CONFIG_DIR}/sphinx.conf";
+      echo "*/15 * * * * ${INDEXER} --quiet tag1 --rotate --config ${PILER_CONFIG_DIR}/manticore.conf";
+      echo "*/15 * * * * ${INDEXER} --quiet note1 --rotate --config ${PILER_CONFIG_DIR}/manticore.conf";
       echo "30   6 * * * /usr/bin/php ${LIBEXECDIR}/piler/generate_stats.php --webui ${DOCROOT} >/dev/null";
       echo "*/5 * * * * /usr/bin/find ${LOCALSTATEDIR}/piler/error -type f|wc -l > ${LOCALSTATEDIR}/piler/stat/error";
       echo "*/5 * * * * /usr/bin/find ${DOCROOT}/tmp -type f -name i.\* -exec rm -f {} \;";
+      echo "#*/5 * * * * ${LIBEXECDIR}/piler/import.sh";
       echo "### PILEREND";
    } >> "$CRON_TMP"
 }
@@ -315,8 +316,8 @@ show_summary() {
    echo "mysql password: *******"
    echo
 
-   echo "sphinx indexer: ${INDEXER}"
-   echo "sphinx config file: ${SPHINXCFG}"
+   echo "indexer: ${INDEXER}"
+   echo "config file: ${SEARCHCFG}"
    echo
 
    echo "vhost docroot: ${DOCROOT}"
@@ -357,14 +358,12 @@ execute_post_install_tasks() {
    echo "Done."
 
    echo -n "Writing sphinx configuration... ";
-   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" -e "s%MYSQL_USERNAME%${MYSQL_USERNAME}%" -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" "${SYSCONFDIR}/piler/sphinx.conf.dist" > "$SPHINXCFG"
-   chmod +x "$SPHINXCFG"
+   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" -e "s%MYSQL_USERNAME%${MYSQL_USERNAME}%" -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" "${SYSCONFDIR}/piler/manticore.conf.dist" > "$SEARCHCFG"
+   chmod +x "$SEARCHCFG"
    echo "Done."
 
-   if indexer --version | grep ^Sphinx\ 3; then sed -i 's/220/311/' "$SPHINXCFG"; fi
-
-   echo -n "Initializing sphinx indices... ";
-   su "$PILERUSER" -c "indexer --all --config ${SPHINXCFG}"
+   echo -n "Initializing manticore indices... ";
+   su "$PILERUSER" -c "indexer --all --config ${SEARCHCFG}"
    echo "Done."
 
 
@@ -407,8 +406,8 @@ SOCKHELPER
       cp webui/.htaccess "$DOCROOT"
    fi
 
-   if [[ -d /var/www/piler.yourdomain.com ]]; then
-      mv /var/www/piler.yourdomain.com "$DOCROOT"
+   if [[ -d /var/www/piler.example.com ]]; then
+      mv /var/www/piler.example.com "$DOCROOT"
    fi
 
 

util/refresh-token.py

@@ -0,0 +1,34 @@
+#!/usr/bin/python3
+
+# Based on the https://github.com/UvA-FNWI/M365-IMAP project
+
+from msal import ConfidentialClientApplication, SerializableTokenCache
+import sys
+
+
+ClientId = "c6843299-05c4-4c2e-9398-64dd42f14b6d"  # Fix this value only
+ClientSecret = ""
+Scopes = ['https://outlook.office.com/IMAP.AccessAsUser.All']
+AccessTokenFileName = "access_token"
+RefreshTokenFileName = "refresh_token"
+
+
+cache = SerializableTokenCache()
+app = ConfidentialClientApplication(ClientId,
+                                    client_credential=ClientSecret,
+                                    token_cache=cache, authority=None)
+
+old_refresh_token = open(RefreshTokenFileName, 'r').read()
+
+token = app.acquire_token_by_refresh_token(old_refresh_token, Scopes)
+
+
+if 'error' in token:
+    print(token)
+    sys.exit("Failed to get access token")
+
+with open(AccessTokenFileName, 'w') as f:
+    f.write(token['access_token'])
+
+with open(RefreshTokenFileName, 'w') as f:
+    f.write(token['refresh_token'])

util/reindex.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+INSTALL_PREFIX=/usr/local
+THRESHOLD=100000000
+
+start_id=1
+stop_id=5000000
+
+while [[ $start_id -lt $stop_id ]]; do
+   y=$(( start_id + 9999 ))
+   echo $start_id $y
+   "${INSTALL_PREFIX}/bin/reindex" -f $start_id -t $y -p
+   "${INSTALL_PREFIX}/libexec/piler/indexer.delta.sh"
+   start_id=$(( start_id + 10000 ))
+
+   if [[ "$(stat -c %s /var/piler/sphinx/dailydelta1.spp)" -gt "$THRESHOLD" ]]; then
+      "${INSTALL_PREFIX}/libexec/piler/indexer.main.sh"
+   fi
+done

util/rtindex.py

@@ -0,0 +1,115 @@
+#!/usr/bin/python3
+
+import configparser
+import MySQLdb as dbapi
+import argparse
+import getpass
+import os
+import sys
+import syslog
+import time
+
+SQL_SELECT_QUERY = "SELECT id, `from`, `to`, fromdomain, todomain, subject, " + \
+                   "arrived, sent, body, size, direction, folder, attachments, " + \
+                   "attachment_types FROM sph_index"
+SQL_INSERT_QUERY = "INSERT INTO piler1 (id, sender, rcpt, senderdomain, " + \
+                   "rcptdomain, subject, arrived, sent, body, size, direction, " + \
+                   "folder, attachments, attachment_types) VALUES (%s, %s, %s, " + \
+                   "%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
+SQL_DELETE_QUERY = "DELETE FROM sph_index WHERE id IN (%s)"
+SLEEP_DELAY = 5
+
+opts = {}
+
+
+def read_options(filename="", opts={}):
+    s = "[piler]\n" + open(filename, 'r').read()
+    config = configparser.ConfigParser()
+    config.read_string(s)
+
+    if config.has_option('piler', 'mysqlhost'):
+        opts['dbhost'] = config.get('piler', 'mysqlhost')
+    else:
+        opts['dbhost'] = 'localhost'
+
+    opts['username'] = config.get('piler', 'mysqluser')
+    opts['password'] = config.get('piler', 'mysqlpwd')
+    opts['database'] = config.get('piler', 'mysqldb')
+
+
+def process_batch(opts):
+    try:
+        opts['db'] = dbapi.connect(opts['dbhost'], opts['username'],
+                                   opts['password'], opts['database'])
+
+        cursor = opts['db'].cursor()
+
+        while True:
+            cursor.execute(SQL_SELECT_QUERY)
+            rows = cursor.fetchmany(opts['batch_size'])
+            if rows == ():
+                time.sleep(SLEEP_DELAY)
+                break
+
+            ids = [x[0] for x in rows]
+
+            opts['sphx'] = dbapi.connect(host=opts['sphinx_host'],
+                                         port=opts['sphinx_port'])
+            sphx_cursor = opts['sphx'].cursor()
+
+            sphx_cursor.executemany(SQL_INSERT_QUERY, rows)
+            opts['sphx'].commit()
+            opts['sphx'].close()
+
+            syslog.syslog("%d records inserted" % (sphx_cursor.rowcount))
+
+            format = ", ".join(['%s'] * len(ids))
+            cursor.execute(SQL_DELETE_QUERY % (format), ids)
+            opts['db'].commit()
+
+    except dbapi.DatabaseError as e:
+        syslog.syslog("Error %s" % e)
+        time.sleep(SLEEP_DELAY)
+
+    if opts['db']:
+        opts['db'].close()
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-c", "--config", type=str, help="piler.conf path",
+                        default="/etc/piler/piler.conf")
+    parser.add_argument("-b", "--batch-size", type=int, help="batch size " +
+                        "to process", default=1000)
+    parser.add_argument("-s", "--sphinx", type=str, help="sphinx server",
+                        default="127.0.0.1")
+    parser.add_argument("-p", "--port", type=int, help="sphinx sql port",
+                        default=9306)
+    parser.add_argument("-d", "--dry-run", help="dry run", action='store_true')
+    parser.add_argument("-v", "--verbose", help="verbose mode",
+                        action='store_true')
+
+    args = parser.parse_args()
+
+    if getpass.getuser() not in ['root', 'piler']:
+        print("Please run me as user 'piler'")
+        sys.exit(1)
+
+    opts['dry_run'] = args.dry_run
+    opts['verbose'] = args.verbose
+    opts['sphinx_host'] = args.sphinx
+    opts['sphinx_port'] = args.port
+    opts['batch_size'] = args.batch_size
+    opts['db'] = None
+    opts['sphx'] = None
+
+    syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_MAIL)
+
+    read_options(args.config, opts)
+
+    while True:
+        process_batch(opts)
+
+
+if __name__ == "__main__":
+    main()

util/sign.php

@@ -12,11 +12,13 @@ ini_set("session.save_path", "/tmp");
 $webuidir = "";
 $verbose = 0;
 $mode = "unit";
+$algo = "sha256";
 
 $opts = 'h::v';
 $lopts = array(
     'webui:',
     'mode:',
+    'algo:',
     'verbose'
     );
 
@@ -37,6 +39,10 @@ if ( $options = getopt( $opts, $lopts ) )
        $mode = $options['mode'];
     }
 
+    if ( isset($options['algo']) ) {
+       $algo = $options['algo'];
+    }
+
     if ( isset($options['h']) )
     {
         display_help();
@@ -66,6 +72,7 @@ Registry::set('db', $db);
 Registry::set('DB_DRIVER', DB_DRIVER);
 
 define('MODE', $mode);
+define('ALGO', $algo);
 
 
 $data = get_hash_values();
@@ -121,29 +128,29 @@ function get_hash_values() {
 
    if($last_id == 0) {
       $start_id = TSA_START_ID;
-      if(MODE == 'unit') { $stop_id = $start_id + TSA_STAMP_REQUEST_UNIT_SIZE - 1; }
-      else { $stop_id = 1000000000; }
-   }
-   else {
+   } else {
       $start_id = $last_id + 1;
-      if(MODE == 'unit') { $stop_id = $start_id + TSA_STAMP_REQUEST_UNIT_SIZE - 1; }
-      else { $stop_id = 1000000000; }
    }
 
-   $query = $db->query("SELECT id, digest FROM " . TABLE_META . " WHERE id >= ? AND id <= ?", array($start_id, $stop_id));
+   if(MODE == 'unit') {
+      $limit = TSA_STAMP_REQUEST_UNIT_SIZE;
+   } else {
+      $limit = 100000; // stay well below default PHP memory_limit
+   }
+
+   $query = $db->query("SELECT id, digest FROM " . TABLE_META . " WHERE id >= ? ORDER BY id LIMIT $limit", array($start_id));
 
    foreach($query->rows as $q) {
       $count++;
+      $last_id = $q['id'];
       $s .= $q['digest'];
    }
 
-   if(MODE == 'time') { $stop_id = $start_id + $count - 1; }
-
    return [
       START_ID => $start_id,
-      STOP_ID => $stop_id,
+      STOP_ID => $last_id,
       COUNT => $count,
-      HASH_VALUE => sha1($s)
+      HASH_VALUE => hash(ALGO, $s)
    ];
 
 }

webui/.htaccess

@@ -35,3 +35,24 @@ RewriteRule ^view/javascript/piler.js /js.php [QSA,L]
    </FilesMatch>
 </IfModule>
 
+<IfModule auth_gssapi_module>
+   # ktpass -princ HTTP/<webserver-fqdn>@<WINDOWS AD DOMAIN IN CAPITALS> \
+   #        -mapuser <ldap helper user>@<WINDOWS AD DOMAIN IN CAPITALS> \
+   #        -pass * \
+   #        -crypto AES256-SHA1 \
+   #        -ptype KRB5_NT_PRINCIPAL \
+   #        -out /etc/krb5/http.keytab \
+   #
+   # setspn -s HTTP/<webserver-fqdn> <ldap helper user>
+
+   <FilesMatch "sso\.php$">
+      RewriteEngine on
+      RewriteCond %{HTTP:Authorization}  !^$
+      RewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
+      AuthName "User with domain part (separated by @) in CAPITALS - e.g. 'user@DOMAIN'"
+      AuthType GSSAPI
+      GssapiBasicAuth On
+      GssapiCredStore keytab:/etc/krb5/http.keytab
+      Require valid-user
+   </FilesMatch>
+</IfModule>