Bumped up version number to 1.4.5

Signed-off-by: Janos SUTO <sj@acts.hu>

.gitignore

@@ -0,0 +1,26 @@
+Makefile
+*.o
+*.so
+*.a
+*.deb
+src/libpiler.so.0*
+src/piler
+src/pilerconf
+src/piler-smtp
+src/pileraget
+src/pilerexport
+src/pilerget
+src/pilerimport
+src/pilerstats
+src/pilertest
+src/reindex
+unit_tests/check_attachments
+unit_tests/check_decoder
+unit_tests/check_digest
+unit_tests/check_hash
+unit_tests/check_misc
+unit_tests/check_mydomains
+unit_tests/check_parser
+unit_tests/check_parser_utils
+unit_tests/check_rules
+unit_tests/smtp

G

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+read -r version < VERSION
+
+tarball="piler-${version}.tar.gz"
+
+git archive --prefix="piler-${version}/" HEAD | gzip -c > "$tarball"
+sha256sum "$tarball"

LICENSE

@@ -1,6 +1,6 @@
 piler, an enterprise level email archiving application
 
-Copyright (C) 2012-2016, Janos SUTO <sj@acts.hu>
+Copyright (C) 2012-2023, Janos SUTO <sj@acts.hu>
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

Makefile.in

@@ -33,7 +33,7 @@ INSTALL = @INSTALL@
 am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \
 	piler-config.h stamp.h stamp-h1 params.h
 
-all: all-recursive
+all: all-recursive config-php
 install: installdirs install-recursive
 
 
@@ -63,6 +63,10 @@ $(RECURSIVE_TARGETS):
 	fi; test -z "$$fail"
 
 
+config-php:
+	sed -e "s%SYSCONFDIR%$(sysconfdir)%g" -e "s%SBINDIR%$(sbindir)%g" config.php.in > webui/config.php
+	sed -e "s%BINDIR%$(bindir)%g" -i webui/config.php
+
 
 installdirs: mkinstalldirs
 	$(srcdir)/mkinstalldirs \
@@ -78,7 +82,8 @@ installdirs: mkinstalldirs
 		$(DESTDIR)$(localstatedir)/piler/stat \
 		$(DESTDIR)$(localstatedir)/piler/tmp \
 		$(DESTDIR)$(localstatedir)/piler/error \
-		$(DESTDIR)$(localstatedir)/piler/sphinx
+		$(DESTDIR)$(localstatedir)/piler/export \
+		$(DESTDIR)$(localstatedir)/piler/manticore
 
 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/run/piler
 	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/store
@@ -86,7 +91,8 @@ installdirs: mkinstalldirs
 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/stat
 	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/tmp
 	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/error
-	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/sphinx
+	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/export
+	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/manticore
 
 
 install-am:
@@ -120,7 +126,7 @@ key:
 	dd if=/dev/urandom bs=56 count=1 of=piler.key
 
 postinstall:
-	@sh util/postinstall.sh $(RUNNING_USER) $(RUNNING_GROUP) $(sysconfdir) $(localstatedir) $(libexecdir)
+	@bash util/postinstall.sh $(RUNNING_USER) $(RUNNING_GROUP) $(sysconfdir) $(localstatedir) $(libexecdir)
 
 
 

RELEASE_NOTES

@@ -1,3 +1,129 @@
+1.4.5:
+------
+
+- Introduced 2 new variables in /etc/piler/piler.conf affecting piler-smtp
+
+  ; max message size in bytes
+  ; piler-smtp will reject any message that's bigger than this number
+  max_message_size=50000000
+
+  ; max memory in bytes piler-smtp uses for buffering messages
+  ; when this limit is exceeded, no new emails will be accepted
+  ; until the used memory for all in progress emails decreases
+  ; below this level
+  max_smtp_memory=500000000
+
+  Be sure to adjust these values to your environment!
+
+- Added read-only connection support for manticore
+  If using sphinx, add the following to config-site.php:
+  $config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9306';
+
+- pilerimport supports Zimbra IMAP impersonation
+
+  Generate the following base64 encoded string:
+  (Be sure to use the actual usernames and password):
+
+  pw="$( printf '%s\0%s\0%s' 'username' 'zimbra_admin_username' 'zimbra_admin_password' | base64 )"
+
+  Then specify -u ZIMBRA -p "$pw" for pilerimport, eg.
+
+  pilerimport -i imap.server -u ZIMBRA -p "$pw" ...
+
+  Note that "ZIMBRA" is a special username, it indicates for pilerimport
+  to actually use the imap impersonation for Zimbra.
+
+
+1.4.4:
+------
+
+- Renamed "group" table to "usergroup"
+  Be sure to run util/db-upgrade.sql on the mysql piler database
+
+
+1.4.3:
+------
+
+- Improved real-time index support (no need to use rtindex.py any more)
+  See https://www.mailpiler.org/wiki/current:manticore for more
+
+
+1.4.2:
+------
+
+- Fixed docker image
+- Fixed php socket path in piler-nginx.conf
+- Fixed manticore index data settings
+
+
+1.4.1:
+------
+
+- Replaced sphinx with manticore 5.0.2, see https://www.mailpiler.org/wiki/current:manticore
+- Replaced obsoleted libchart library with chart.js
+- Added real-time index support
+- Added oauth2 support to imapfetch.py
+- Added support for pilerexport to spread files among several directories. See the -D option
+
+
+1.3.12:
+-------
+
+- Introduced new piler.conf variable: tls_min_version
+
+  It sets the minimum TLS protocol version the piler-smtp daemon supports.
+
+  Possible values:
+  - TLSv1   (not recommended)
+  - TLSv1.1 (not recommended)
+  - TLSv1.2 (default)
+  - TLSv1.3
+
+- Introduced the archive_address feature, see etc/example.conf for the details
+- Introduced the raw: search label
+- Added Italian translation. Credits: Stefano Gatto
+- timestamp signing sorts by 'id' column
+- timestamp hash value defaults to sha256
+- Minor fixes
+- Added support for php 8.1
+- Fixed handling long email addresses
+
+Be sure to apply util/db-upgrade.sql
+
+
+1.3.11:
+-------
+
+- [BUGFIX] Refactored the smtp timeout check
+- Obsoleted the LDAP port parameter. Specify the ldap host in the form
+  of protocol://hostname:port, eg. ldaps://ldap.example.com:636
+
+1.3.10:
+-------
+
+- Added security header feature
+- Introduced the smtp acl list, and obsoleted the tcp_wrappers check
+- Switched from Blowfish encryption to AES-256
+
+1.3.9:
+------
+
+- Added a separator to searching for attachment names
+- [BUGFIX] Render multiple mail parts in mail view instead of only the last part
+- Use TLS v1.2 with openssl 1.0.x for connecting remote pop3/imap servers
+- Instant search results to the gui when the search page loads
+- Support sphinx-3.3.1, introduced sphinx strict mode variable
+- GUI domain fixes
+- gcc 9 fixes
+- Fix permission on sphinx data dir to 700
+- pilerpurge.py should honor the mysqlhost value
+- Password change enabled by default
+- Health page fixes
+- GUI mime parser fixes
+- Start/stop script fix
+- Optimized search page for mobile devices, set $config['ENABLE_MOBILE_PREVIEW'] = 1; in config-site.php to enable it
+
+
 1.3.0:
 -----
 

SECURITY.md

@@ -0,0 +1,20 @@
+Disclosure policy
+
+If you find a security issue, please contact the project owner at sj@acts.hu
+with the details (ie. piler version, details of the setup, how to exploit the
+vulnerability, etc).
+
+Please provide 30 days for verifying the vulnerability, fixing the issue, and
+notifying the piler users.
+
+Security update policy
+
+If a security vulnerability has found, the details, possible mitigations,
+workarounds, etc. will be shared on the piler mailing list (piler-user@list.acts.hu)
+and on the wiki: https://www.mailpiler.org/
+
+Security configurations
+
+ - Use https for the GUI
+ - Reset the default passwords for admin and auditor
+ - Use the smtp acl feature to restrict SMTP access to the archive, see https://mailpiler.com/smtp-acl-list/

VERSION

@@ -1 +1 @@
-1.3.3
+1.4.5

bitbucket-pipelines.yml

@@ -4,7 +4,7 @@
 # -----
 # You can specify a custom docker image from Docker Hub as your build environment.
 
-image: sutoj/builder:xenial
+image: sutoj/builder:bionic
 clone:
   depth: 5
 
@@ -12,19 +12,11 @@ pipelines:
   default:
     - step:
         script:
-          - echo "Everything is awesome!"
-
-  branches:
-    epoll:
-      - step:
-          script:
-            - service mysql start
-            - ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --with-database=mariadb --enable-tcpwrappers
-            - make clean all install
-            - echo "create database piler character set 'utf8mb4'" | mysql
-            - echo "grant all privileges on piler.* to piler@localhost identified by 'piler123'" | mysql
-            - echo "flush privileges" | mysql
-            - mysql -u piler -ppiler123 piler < /usr/share/piler/db-mysql.sql
-            - cd unit_tests
-            - ./run.sh
-
+          - service mysql start
+          - ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --with-database=mariadb
+          - make clean all install
+          - mysql -u piler -ppiler123 piler1 < /usr/share/piler/db-mysql.sql
+          - cd unit_tests
+          - ./run.sh
+          - cd ../webui
+          - phpunit

config.php.in

@@ -8,7 +8,7 @@ ini_set('session.use_only_cookies', 1);
 define('NORMAL', 1);
 define('DEBUG', 5);
 
-$config = array();
+$config = [];
 
 
 /*
@@ -16,9 +16,15 @@ $config = array();
  * variable to be overridden in config-site.php
  */
 
+$config['ENABLE_MOBILE_PREVIEW'] = 0;
+
+$config['SITE_LOGO_LG'] = '/view/theme/default/assets/images/archive-logo-lg.png';
+$config['SITE_LOGO_SM'] = '/view/theme/default/assets/images/archive-logo-sm.png';
+$config['COMPATIBILITY'] = 'Which browsers are supported, etc';
+
 $config['BRANDING_TEXT'] = '';
 $config['BRANDING_URL'] = '';
-$config['BRANDING_LOGO'] = '';
+$config['BRANDING_LOGO'] = $config['SITE_LOGO_SM'];
 $config['BRANDING_BACKGROUND_COLOUR'] = '';
 $config['BRANDING_TEXT_COLOUR'] = '';
 $config['BRANDING_FAVICON'] = '/view/theme/default/assets/ico/favicon.png';
@@ -28,16 +34,21 @@ $config['TITLE_PREFIX'] = '';
 $config['CUSTOM_PRE_AUTH_FUNCTION'] = '';
 $config['CUSTOM_EMAIL_QUERY_FUNCTION'] = '';
 
+$config['DOMAIN_REGEX'] = '/^[a-zA-Z0-9]+[a-zA-Z0-9-_\.]{0,}\.[a-zA-Z0-9]{2,20}$/';
+
 $config['BOOTSTRAP_THEME'] = '-cosmo';
 
 $config['DEFAULT_LANG'] = 'en';
 
 $config['THEME'] = 'default';
 
-$config['SITE_NAME'] = 'piler.yourdomain.com';
-$config['SITE_URL'] = 'http://piler.yourdomain.com/';
+$config['SITE_NAME'] = 'piler.example.com';
+$config['SITE_URL'] = 'http://piler.example.com/';
+
+$config['EXTERNAL_DASHBOARD_URL'] = '';
 
 $config['SESSION_EXPIRY'] = 3600;
+$config['DELTA_INDEXER_PERIOD'] = 1800;
 
 $config['ENABLE_SAAS'] = 0;
 $config['CAPTCHA_FAILED_LOGIN_COUNT'] = 0;
@@ -53,6 +64,8 @@ $config['SHOW_MENU_FOR_OUTLOOK'] = 0;
 
 $config['SEARCH_QUERY_QUOTING'] = 0;
 
+$config['LOCALIZE_MESSAGE_HEADERS_IN_PREVIEW'] = 1;
+
 $config['TIMEZONE'] = 'Europe/Budapest';
 
 $config['PROVIDED_BY'] = 'www.mailpiler.org';
@@ -62,16 +75,22 @@ $config['SITE_DESCRIPTION'] = 'piler email archiver';
 $config['INDEXER_BEACON'] = '/var/piler/stat/indexer';
 $config['PURGE_BEACON'] = '/var/piler/stat/purge';
 
+$config['ENABLE_PDF_DOWNLOAD'] = 0;
+// You may need to run "Xvfb :1 -screen 0 1024x768x16"
+// In this case specify WKHTMLTOPDF_COMMAND = "DISPLAY=:1.0 wkhtmltopdf";
+$config['WKHTMLTOPDF_COMMAND'] = "wkhtmltopdf";
+
 // authentication against an ldap directory (disabled by default)
 
 $config['ENABLE_LDAP_AUTH'] = 0;
-$config['LDAP_HOST'] = 'ldap.yourdomain.com';
+$config['LDAP_HOST'] = 'ldap.example.com';
 $config['LDAP_HELPER_DN'] = 'cn=....';
 $config['LDAP_HELPER_PASSWORD'] = 'xxxxxxx';
 $config['LDAP_MAIL_ATTR'] = 'mail';
 $config['LDAP_AUDITOR_MEMBER_DN'] = '';
 $config['LDAP_ADMIN_MEMBER_DN'] = '';
 $config['LDAP_BASE_DN'] = '';
+$config['LDAP_USE_START_TLS'] = 0;
 
 // AD specific settings
 //
@@ -95,14 +114,18 @@ $config['LDAP_MAIL_ATTR'] = 'proxyAddresses';
 //$config['LDAP_MAIL_ATTR'] = 'mail';
 
 // iredmail specific settings
-//$config['LDAP_HELPER_DN'] = 'cn=vmailadmin,dc=yourdomain,dc=com';
+//$config['LDAP_HELPER_DN'] = 'cn=vmailadmin,dc=example,dc=com';
 //$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'mailUser';
-//$config['LDAP_BASE_DN'] = 'o=domains,dc=yourdomain,dc=com';
+//$config['LDAP_BASE_DN'] = 'o=domains,dc=example,dc=com';
 //$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'mailList';
 //$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'memberOfGroup';
 //$config['LDAP_MAIL_ATTR'] = 'mail';
 
-
+// Uninvention specific settings
+//$config['LDAP_MAIL_ATTR'] = 'mailPrimaryAddress';
+//$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'person';
+//$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'person';
+//$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mailAlternativeAddress';
 
 
 // enable single sign-on (disabled by default)
@@ -116,14 +139,14 @@ $config['ENABLE_IMAP_AUTH'] = 0;
 $config['RESTORE_OVER_IMAP'] = 0;
 $config['IMAP_RESTORE_FOLDER_INBOX'] = 'INBOX';
 $config['IMAP_RESTORE_FOLDER_SENT'] = 'Sent';
-$config['IMAP_HOST'] = 'mail.yourdomain.com';
+$config['IMAP_HOST'] = 'mail.example.com';
 $config['IMAP_PORT'] =  993;
 $config['IMAP_SSL'] = true;
 
 // enable authentication against a pop3 server (disabled by default)
 
 $config['ENABLE_POP3_AUTH'] = 0;
-$config['POP3_HOST'] = 'mail.yourdomain.com';
+$config['POP3_HOST'] = 'mail.example.com';
 $config['POP3_PORT'] = 995;
 $config['POP3_SSL'] = true;
 
@@ -139,21 +162,22 @@ $config['GOOGLE_DEVELOPER_KEY'] = 'xxxxxxxxxxxx';
 $config['GOOGLE_APPLICATION_NAME'] = 'piler enterprise email archiver';
 $config['GOOGLE_ALL_MAIL'] = '[Gmail]/All Mail';
 
-$config['SITE_LOGO_LG'] = 'view/theme/default/assets/images/archive-logo-lg.png';
-$config['SITE_LOGO_SM'] = 'view/theme/default/assets/images/archive-logo-sm.png';
-$config['COMPATIBILITY'] = 'Which browsers are supported, etc';
-
 
 $config['ENABLE_AUDIT'] = 1;
 $config['MEMCACHED_ENABLED'] = 0;
-$config['PASSWORD_CHANGE_ENABLED'] = 0;
+$config['PASSWORD_CHANGE_ENABLED'] = 1;
 $config['ENABLE_STATISTICS'] = 1;
 $config['ENABLE_HISTORY'] = 1;
 $config['ENABLE_DELETE'] = 0;
+$config['NEED_TO_APPROVE_DELETE'] = 0;
 $config['ENABLE_REMOTE_IMAGES'] = '0';
 $config['ENABLE_ON_THE_FLY_VERIFICATION'] = 0;
 $config['ENABLE_LDAP_IMPORT_FEATURE'] = 0;
 $config['ENABLE_FOLDER_RESTRICTIONS'] = 0;
+
+// When enabled the search page auto loads the search results
+// ie. as if the user had clicked on the Search button
+$config['ENABLE_INSTANT_SEARCH'] = 0;
 $config['SEARCH_RESULT_CHECKBOX_CHECKED'] = 0;
 $config['HELPER_URL'] = '';
 $config['LOG_LEVEL'] = NORMAL;
@@ -166,11 +190,13 @@ $config['BULK_DOWNLOAD_FOR_USERS'] = 1;
 $config['MAX_DOWNLOAD_PER_HOUR'] = 0;
 $config['MAX_RESTORE_PER_HOUR'] = 0;
 
-$config['MAX_CGI_FROM_SUBJ_LEN'] = 34;
+$config['MAX_CGI_FROM_SUBJ_LEN'] = 48;
 $config['PAGE_LEN'] = 20;
 $config['MAX_NUMBER_OF_FROM_ITEMS'] = 5;
 $config['MAX_SEARCH_HITS'] = 1000;
 
+$config['SPHINX_MAIN_INDEX_THRESHOLD'] = 2000000000;
+
 $config['DEFAULT_RETENTION'] = 0;
 
 $config['LOCALHOST'] = '127.0.0.1';
@@ -178,23 +204,25 @@ $config['PILER_HOST'] = '0.0.0.0';
 $config['PILER_PORT'] = 25;
 $config['SMARTHOST'] = '';
 $config['SMARTHOST_PORT'] = 25;
-$config['SMTP_DOMAIN'] = 'yourdomain.com';
-$config['SMTP_FROMADDR'] = 'no-reply@yourdomain.com';
-$config['ADMIN_EMAIL'] = 'admin@yourdomain.com';
+$config['SMARTHOST_USER'] = '';
+$config['SMARTHOST_PASSWORD'] = '';
+$config['SMTP_DOMAIN'] = 'example.com';
+$config['SMTP_FROMADDR'] = 'no-reply@example.com';
+$config['ADMIN_EMAIL'] = 'admin@example.com';
 
 $config['PILER_HEADER_FIELD'] = 'X-piler-id: ';
 
 $config['DEFAULT_POLICY'] = 'default_policy';
 
-$config['DIR_BASE'] = '/var/piler/www/';
+$config['DIR_BASE'] = dirname(__FILE__) . '/';
 
-$config['DIR_SPHINX'] = '/var/piler/sphinx/';
+$config['DIR_SPHINX'] = '/var/piler/manticore/';
 $config['DIR_STAT'] = '/var/piler/stat';
 $config['DIR_IMAP'] = '/var/piler/imap';
 $config['DIR_TMP'] = '/var/piler/tmp';
 
-$config['DECRYPT_BINARY'] = '/usr/local/bin/pilerget';
-$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/local/bin/pileraget';
+$config['DECRYPT_BINARY'] = 'BINDIR/pilerget';
+$config['DECRYPT_ATTACHMENT_BINARY'] = 'BINDIR/pileraget';
 $config['DECRYPT_BUFFER_LENGTH'] = 65536;
 
 $config['OPENSSL_BINARY'] = '/usr/bin/openssl';
@@ -202,6 +230,13 @@ $config['TSA_URL'] = '';
 $config['TSA_PUBLIC_KEY_FILE'] = '';
 $config['TSA_START_ID'] = 1;
 $config['TSA_STAMP_REQUEST_UNIT_SIZE'] = 10000;
+$config['TSA_VERIFY_CERTIFICATE'] = true;
+$config['TSA_RELAXED_CHECK'] = false;
+$config['TSA_AUTH_USER'] = '';
+$config['TSA_AUTH_PASSWORD'] = '';
+$config['TSA_AUTH_CERT_FILE'] = '';
+$config['TSA_AUTH_KEY_FILE'] = '';
+$config['TSA_AUTH_KEY_PASSWORD'] = '';
 
 $config['DB_DRIVER'] = 'mysql';
 $config['DB_PREFIX'] = '';
@@ -214,15 +249,19 @@ $config['DB_CHARSET'] = 'utf8mb4';
 $config['SPHINX_DRIVER'] = 'sphinx';
 $config['SPHINX_DATABASE'] = 'sphinx';
 $config['SPHINX_HOSTNAME'] = '127.0.0.1:9306';
+$config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9307';
 $config['SPHINX_MAIN_INDEX'] = 'main1,dailydelta1,delta1';
 $config['SPHINX_ATTACHMENT_INDEX'] = 'att1';
 $config['SPHINX_TAG_INDEX'] = 'tag1';
 $config['SPHINX_NOTE_INDEX'] = 'note1';
+$config['SPHINX_STRICT_SCHEMA'] = 1;
+$config['RT'] = 0;
+$config['MAX_EMAIL_LEN'] = 41;
 
 $config['RELOAD_COMMAND'] = 'sudo -n /etc/init.d/rc.piler reload';
 $config['PILERIMPORT_IMAP_COMMAND'] = '/usr/local/bin/pilerimport -d /var/piler/imap -q -r';
 $config['CPU_USAGE_COMMAND'] = "LC_ALL=C mpstat | tail -1 | rev | awk '{ print $1 }' | rev";
-$config['PILER_BINARY'] = "/usr/local/sbin/piler";
+$config['PILER_BINARY'] = "SBINDIR/piler";
 
 $config['LDAP_IMPORT_CONFIG_FILE'] = '/usr/local/etc/ldap-import.cfg';
 
@@ -241,7 +280,10 @@ $config['SIZE_Y'] = 250;
 $config['DATE_TEMPLATE'] = 'Y.m.d';
 $config['DATE_FORMAT'] = 'YYYY-MM-DD';
 $config['JQUERY_DATE_FORMAT'] = 'yy-mm-dd';
+$config['DECIMAL_SEPARATOR'] = "."; // See https://www.php.net/manual/en/function.number-format
+$config['THOUSANDS_SEPARATOR'] = ","; // for the format options
 
+$config['ENABLE_GB2312_FIX'] = 1;
 
 $config['FROM_LENGTH_TO_SHOW'] = 28;
 
@@ -255,15 +297,26 @@ $config['MIN_PASSWORD_LENGTH'] = 6;
 
 $config['MIN_PREFIX_LEN'] = 5;
 
-$config['CGI_INPUT_FIELD_WIDTH'] = 50;
-$config['CGI_INPUT_FIELD_HEIGHT'] = 7;
-
 $config['ADMIN_CAN_POWER_SEARCH'] = 0;
 $config['FOUR_EYES_LOGIN_FOR_AUDITOR'] = 0;
 
 $config['MEMCACHED_PREFIX'] = '_piler:';
 $config['MEMCACHED_TTL'] = 900;
 
+$config['JS_CODE'] = '
+    <script src="//code.jquery.com/jquery-2.2.4.min.js" integrity="sha256-BbhdlvQf/xTY9gja0Dq3HiwQF8LaCRTXxZKRutelT44=" crossorigin="anonymous"></script>
+    <script src="//code.jquery.com/ui/1.12.1/jquery-ui.min.js" integrity="sha256-VazP97ZCwtekAsvgPBSUwPFKdrwD3unUfSGVYrahUqU=" crossorigin="anonymous"></script>
+    <script src="//stackpath.bootstrapcdn.com/bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha384-aJ21OjlMXNL5UyIl/XNwTMqvzeRMZH2w8c5cRVpzpU8Y5bApTppSuUkhZXN0VxHd" crossorigin="anonymous"></script>
+
+    <script type="text/javascript" src="/view/javascript/rc-splitter.js"></script>
+    <script type="text/javascript" src="/view/javascript/piler.js"></script>
+';
+
+$config['CSS_CODE'] = '
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css" integrity="sha384-HSMxcRTRxnN+Bdg0JdbxYKrThecOKuH5zCYotlSAcp1+c8xmyTe9GYg1l9a69psu" crossorigin="anonymous">
+   <link rel="stylesheet" href="/view/theme/default/assets/css/piler.css" />
+';
+
 $SUPPRESS_RECIPIENTS = array();
 
 
@@ -278,6 +331,8 @@ $config['DELIMITER'] = "\t";
 
 $config['TRACKING_CODE'] = '';
 
+$mailattrs = ["mail", "mailalternateaddress", "proxyaddresses", "zimbraMailForwardingAddress", "member", "memberOfGroup", "othermailbox", "mailprimaryaddress", "mailalternativeaddress"];
+
 $langs = array(
                 'cz',
                 'de',
@@ -285,16 +340,17 @@ $langs = array(
                 'es',
                 'fr',
                 'hu',
+                'it',
                 'pl',
                 'pt',
                 'ru',
-                'tr'
+                'tr',
+                'tw'
                );
 
 
 $themes = array(
                 'default',
-                'mobile'
                );
 
 
@@ -318,7 +374,7 @@ define('NOW', time());
  * normally you don't have to change anything below
  */
 
-require_once '@sysconfdir@/piler/config-site.php';
+if(file_exists('SYSCONFDIR/piler/config-site.php')) { require_once 'SYSCONFDIR/piler/config-site.php'; }
 
 ini_set('session.cookie_lifetime', $config['SESSION_EXPIRY']);
 
@@ -333,8 +389,6 @@ if($session->get("theme") && preg_match("/^([a-zA-Z0-9\-\_]+)$/", $session->get(
 
 include("system/helper/detectmobilebrowser.php");
 
-if(MOBILE_DEVICE == 1) { $config['THEME'] = 'mobile'; }
-
 // make sure auditors are restricted in a saas environment
 if($config['ENABLE_SAAS'] == 1) { $config['RESTRICTED_AUDITOR'] = 1; }
 if($session->get("username") == 'auditor@local' || isset($_SERVER['argv'][2]) ) { $config['RESTRICTED_AUDITOR'] = 0; }
@@ -347,7 +401,7 @@ foreach ($config as $k => $v) {
 
 
 define('TABLE_USER', 'user');
-define('TABLE_GROUP', 'group');
+define('TABLE_GROUP', 'usergroup');
 define('TABLE_GROUP_USER', 'group_user');
 define('TABLE_GROUP_EMAIL', 'group_email');
 define('TABLE_FOLDER', 'folder');
@@ -384,9 +438,10 @@ define('TABLE_AUTOSEARCH', 'autosearch');
 define('TABLE_LEGAL_HOLD', 'legal_hold');
 define('TABLE_TIMESTAMP', 'timestamp');
 define('TABLE_PRIVATE', 'private');
+define('TABLE_DELETED', 'deleted');
 define('VIEW_MESSAGES', 'v_messages');
 
-define('EOL', "\n");
+define('EOL', "\r\n");
 
 define('DIR_SYSTEM', DIR_BASE . 'system/');
 define('DIR_MODEL', DIR_BASE . 'model/');
@@ -419,6 +474,8 @@ define('QSHAPE_DEFERRED_SENDER', DIR_STAT . '/deferred-sender');
 define('CPUSTAT', DIR_STAT . '/cpu.stat');
 define('AD_SYNC_STAT', DIR_STAT . '/adsync.stat');
 define('ARCHIVE_SIZE', DIR_STAT . '/archive.size');
+define('SPHINX_CURRENT_MAIN_INDEX_SIZE', DIR_STAT . '/current_main_index_size');
+define('SPHINX_TOTAL_INDEX_SIZE', DIR_STAT . '/total_index_size');
 define('LOCK_FILE', DIR_LOG . 'lock');
 
 define('SEARCH_HELPER_URL', SITE_URL . 'search-helper.php');
@@ -434,6 +491,8 @@ define('HEALTH_WORKER_URL', SITE_URL . 'index.php?route=health/worker');
 
 define('LDAP_TYPE_GENERIC', 'generic_ldap');
 
+define('ATTACHMENT_DUMP_CHECKPOINT', 'attachment_dump_checkpoint');
+
 define('ACTION_ALL', 0);
 define('ACTION_UNKNOWN', 1);
 define('ACTION_LOGIN', 2);
@@ -454,6 +513,8 @@ define('ACTION_UNAUTHORIZED_DOWNLOAD_ATTACHMENT', 16);
 define('ACTION_VIEW_JOURNAL', 17);
 define('ACTION_NOT_SPAM', 18);
 define('ACTION_MARK_AS_PRIVATE', 19);
+define('ACTION_MARK_MESSAGE_FOR_REMOVAL', 20);
+define('ACTION_REJECT_REMOVAL', 21);
 
 $actions = array(
                   'unknown' => 1,
@@ -468,7 +529,9 @@ $actions = array(
                   'save_search' => 11,
                   'download_attachment' => 15,
                   'journal' => 17,
-                  'private' => 19
+                  'private' => 19,
+                  'marked_for_removal', 20,
+                  'reject_removal', 21
                 );
 
 
@@ -486,5 +549,14 @@ if(!isset($health_smtp_servers)) {
    $health_smtp_servers = array( array(PILER_HOST, PILER_PORT, "piler"), array(SMARTHOST, SMARTHOST_PORT, "smarthost") );
 }
 
-
-?>
+if(SPHINX_STRICT_SCHEMA) {
+   define('FROM_TOKEN', '@sender');
+   define('FROMDOMAIN_TOKEN', '@senderdomain');
+   define('TO_TOKEN', '@rcpt');
+   define('TODOMAIN_TOKEN', '@rcptdomain');
+} else {
+   define('FROM_TOKEN', '@from');
+   define('FROMDOMAIN_TOKEN', '@fromdomain');
+   define('TO_TOKEN', '@to');
+   define('TODOMAIN_TOKEN', '@todomain');
+}

configure.in

@@ -85,7 +85,7 @@ fi
 echo "\"Configure command: ./configure $PARAMS\"" >> $CONFIGURE_PARAMS_FILE
 
 
-SUBDIRS="src etc util init.d unit_tests"
+SUBDIRS="src etc util init.d systemd unit_tests webui"
 
 
 dnl static build
@@ -115,18 +115,13 @@ AC_ARG_ENABLE(memcached,
         [  --enable-memcached      build memcached support], want_memcached=$enableval, want_memcached="no")
 
 
-AC_ARG_ENABLE(tcpwrappers,
-	[  --enable-tcpwrappers    build tcpwrappers support], want_tcpwrappers=$enableval, want_tcpwrappers="no")
+dnl tcpwrappers library
 
+AC_CHECK_HEADERS(tcpd.h, have_tcpwrappers=yes, echo "tcpd.h is not found")
+AC_CHECK_LIB([wrap],[hosts_access],[AC_CHECK_LIB(wrap, hosts_access, have_tcpwrappers=yes, echo "libwrap is not found"; have_tcpwrappers=no)],[],[])ac_cv_lib_wrap=ac_cv_lib_wrap_main
 
-if test "$want_tcpwrappers" = "yes"; then
-   AC_CHECK_HEADERS(tcpd.h, have_tcpwrappers=yes, have_tcpwrappers=no)
-   AC_CHECK_LIB([wrap],[hosts_access],[AC_CHECK_LIB(wrap, hosts_access, have_tcpwrappers=yes, echo "libwrap is not found"; have_tcpwrappers=no)],[],[])ac_cv_lib_wrap=ac_cv_lib_wrap_main
-
-   if test "$have_tcpwrappers" = "no"; then
-      echo "can't find either tcpd.h or libwrap";
-      exit 1;
-   fi
+if test "$have_tcpwrappers" = "no"; then
+   echo "can't find either tcpd.h or libwrap. Tcpwrappers support is disabled";
 fi
 
 
@@ -263,6 +258,10 @@ if test z`which timeout 2>/dev/null` != "z"; then
 fi
 
 
+read -r version < VERSION
+AC_DEFINE_UNQUOTED(VERSION, "$version")
+
+
 dnl user running piler
 
 AC_ARG_WITH(piler-user,
@@ -430,19 +429,8 @@ if test "$have_zip" = "yes"; then
    echo "zip library: yes"
    AC_DEFINE_UNQUOTED(HAVE_ZIP, 1, [libzip support])
    antispam_libs="$antispam_libs -lzip"
-fi
-
-if test "$have_tcpwrappers" = "yes"; then
-   echo "tcpwrappers support: yes"
-   AC_DEFINE_UNQUOTED(HAVE_LIBWRAP, 1, [tcpwrappers support])
-
-   if test "$os" = "FreeBSD"; then
-      antispam_libs="$antispam_libs -lwrap"
-   else
-      antispam_libs="$antispam_libs -lwrap -lnsl"
-   fi
 else
-   echo "tcpwrappers support: no"
+   echo "zip library: no"
 fi
 
 
@@ -534,17 +522,27 @@ if test $? -eq 1; then echo "the user \"$RUNNING_USER\" does not exists, please
 
 echo; echo
 
-CFLAGS="$static -O2 -Wall -g"
-LIBS="$antispam_libs $sunos_libs "
-OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o $objs"
+gcc_version="$(gcc -dumpversion)"
+extra_cflags=""
+if [[ "${gcc_version:0:1}" -gt 6 ]]; then
+   extra_cflags="-Wimplicit-fallthrough=2"
+fi
 
-AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile unit_tests/Makefile contrib/imap/Makefile config.php])
+CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra $extra_cflags -Wuninitialized -Wimplicit-fallthrough=2 -Wno-format-truncation -g"
+LIBS="$antispam_libs $sunos_libs "
+OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o screen.o $objs"
+
+AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile])
 AC_OUTPUT
 
-mv config.php webui/config.php
+echo
+echo
+echo "IMPORTANT! If you upgrade, be sure to read https://www.mailpiler.org/upgrade-instructions/"
+echo
+echo
 
 echo
-echo
-echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/wiki/current:upgrade"
+echo "Did you know that piler has an enterprise edition as well?"
+echo "Check out what it can do for you at https://mailpiler.com/#features"
 echo
 echo

contrib/export-attachments/export-attachments.php

@@ -0,0 +1,103 @@
+#!/usr/bin/php
+<?php
+
+define('EXPORT_LOCK_FILE', '/var/piler/tmp/export-attachments.lock');
+
+$webuidir = "";
+
+$outdir = '';
+
+
+$opts = 'hw:d:';
+$lopts = [
+   'webui:',
+   'dir:',
+   'help'
+];
+
+
+if($options = getopt($opts, $lopts)) {
+   if(isset($options['webui'])) { $webuidir = $options['webui']; }
+   if(isset($options['dir'])) { $outdir = $options['dir']; }
+   if(isset($options['help'])) { usage(); }
+}
+
+
+if($webuidir == '' || $outdir == '') { usage(); }
+
+
+
+require_once("$webuidir/config.php");
+
+require(DIR_SYSTEM . "/startup.php");
+
+$request = new Request();
+Registry::set("request", $request);
+
+
+$start = NULL;
+
+$loader = new Loader();
+Registry::set('load', $loader);
+
+$loader->load->model('domain/domain');
+$loader->load->model('search/search');
+$loader->load->model('search/message');
+$loader->load->model('message/attachment');
+
+
+$db = new DB(DB_DRIVER, DB_HOSTNAME, DB_USERNAME, DB_PASSWORD, DB_DATABASE, DB_PREFIX);
+Registry::set('db', $db);
+
+Registry::set('auditor_user', 1);
+
+openlog("export-attachments", LOG_PID, LOG_MAIL);
+
+$fp = fopen(EXPORT_LOCK_FILE, "w");
+if(!flock($fp, LOCK_EX)) {
+   syslog(LOG_INFO, "WARN: couldn't get a lock on " . EXPORT_LOCK_FILE);
+   exit;
+}
+
+
+$domain = new ModelDomainDomain();
+$attachment = new ModelMessageAttachment();
+$message = new ModelSearchMessage();
+
+$domains = $domain->get_mapped_domains();
+
+$last_id = $attachment->get_last_attachment_id();
+$start_id = $attachment->get_checkpoint();
+
+syslog(LOG_INFO, "start: $start_id, limit: $last_id");
+
+
+for($i=$start_id; $i<$last_id; $i++) {
+   $a = $attachment->get_attachment_by_id($i);
+   $m = $message->get_message_addresses_by_piler_id($a['piler_id'], $domains);
+
+   $attachment->dump_attachment($outdir, "out", $m['sender'], $i, $a);
+
+   foreach($m['rcpt'] as $rcpt) {
+      $attachment->dump_attachment($outdir, "in", $rcpt, $i, $a);
+   }
+
+   if($i % 100 == 0) { $attachment->update_checkpoint($i); }
+}
+
+$attachment->update_checkpoint($i);
+
+// Release lock
+flock($fp, LOCK_UN);
+fclose($fp);
+
+
+
+function usage() {
+   print "\nUsage: " . __FILE__ . "\n\n";
+   print "\t--webui <path to webui directory>\n";
+   print "\t--dir <basedir to write attachments>\n";
+   print "\t--help\n\n";
+
+   exit;
+}

contrib/installer/focal.sh

@@ -0,0 +1,257 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+
+MY_IP="1.2.3.4"
+PILER_HOSTNAME="${PILER_HOSTNAME:-archive.example.com}"
+MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
+MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
+SERVER_ID="${SERVER_ID:-0}"
+USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
+SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
+PHP_FPM_SOCKET="/var/run/php/php7.4-fpm.sock"
+
+MYSQL_HOSTNAME="localhost"
+MYSQL_DATABASE="piler"
+MYSQL_USERNAME="piler"
+
+SPHINX_TARGZ="sphinx-3.3.1-bin.tar.gz"
+DOWNLOAD_URL="https://download.mailpiler.com"
+PILER_DEB="piler_1.3.12-focal-eb2b22b_amd64.deb"
+PILER_USER="piler"
+CONFIG_SITE_PHP="/etc/piler/config-site.php"
+
+export DEBIAN_FRONTEND=noninteractive
+
+install_prerequisites() {
+   apt-get update
+   apt-get -y --no-install-recommends install \
+      wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \
+      php7.4-gd php7.4-curl php7.4-xml ca-certificates zip catdoc unrtf poppler-utils nginx tnef libzip5 \
+      libtre5 libwrap0 cron libmariadb-dev python3 python3-mysqldb libmariadb-dev mariadb-client-core-10.3 \
+      mariadb-server-10.3
+
+   wget -q -O "/tmp/${SPHINX_TARGZ}" "${DOWNLOAD_URL}/generic-local/${SPHINX_TARGZ}"
+   tar -C / -zxvf "/tmp/${SPHINX_TARGZ}"
+   wget -O /usr/local/bin/traefik "${DOWNLOAD_URL}/generic-local/traefik"
+   chmod +x /usr/local/bin/traefik
+   setcap cap_net_bind_service+ep /usr/local/bin/traefik
+}
+
+
+fix_mysql_settings() {
+   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
+[mysqld]
+
+innodb_buffer_pool_size=512M
+innodb_flush_log_at_trx_commit=1
+innodb_log_buffer_size=64M
+innodb_log_file_size=64M
+innodb_read_io_threads=4
+innodb_write_io_threads=4
+innodb_log_files_in_group=2
+
+innodb_file_per_table
+PILER_CNF
+}
+
+start_mysql() {
+   fix_mysql_settings
+
+   service mysql restart
+   mysqladmin -u root password "${MYSQL_ROOT_PASSWORD}"
+}
+
+
+install_piler() {
+   wget "${DOWNLOAD_URL}/piler/${PILER_DEB}"
+   dpkg -i "$PILER_DEB"
+   rm -f "$PILER_DEB"
+
+   crontab -u "$PILER_USER" /usr/share/piler/piler.cron
+
+   rm -f "$PILER_DEB" /etc/nginx/sites-enabled/default
+   ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf
+
+   touch /var/piler/.bash_history
+   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
+}
+
+
+create_my_cnf() {
+   local user=$1
+   local password=$2
+   local my_cnf=$3
+
+   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
+   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
+   chown $PILER_USER:$PILER_USER "$my_cnf"
+   chmod 600 "$my_cnf"
+}
+
+
+fix_config_site_php() {
+   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"
+
+   {
+      echo "\$config['SERVER_ID'] = $SERVER_ID;"
+      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
+      echo "\$config['SPHINX_VERSION'] = 331;"
+   } >> "$CONFIG_SITE_PHP"
+
+   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
+      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
+   fi
+
+   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
+}
+
+
+init_db() {
+   sed -e "s%MYSQL_HOSTNAME%$MYSQL_HOSTNAME%g" -e "s%MYSQL_DATABASE%$MYSQL_DATABASE%g" -e "s%MYSQL_USERNAME%$MYSQL_USERNAME%g" -e "s%MYSQL_PASSWORD%$MYSQL_PILER_PASSWORD%g" \
+      /usr/share/piler/db-mysql-root.sql.in | mysql --defaults-file=/etc/piler/.my.cnf-root -h $MYSQL_HOSTNAME
+   mysql --defaults-file=/etc/piler/.my.cnf -h $MYSQL_HOSTNAME $MYSQL_DATABASE < /usr/share/piler/db-mysql.sql
+}
+
+
+add_systemd_services() {
+   pushd /etc/systemd/system
+
+   ln -sf /usr/libexec/piler/piler.service .
+   ln -sf /usr/libexec/piler/piler-smtp.service .
+   ln -sf /usr/libexec/piler/pilersearch.service .
+
+   popd
+
+   systemctl daemon-reload
+
+   systemctl enable piler
+   systemctl enable piler-smtp
+   systemctl enable pilersearch
+   systemctl enable traefik
+}
+
+
+fix_configs() {
+   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
+      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
+      nginx -t
+      nginx -s reload
+
+      sed -i 's/server {/server {\n\tlisten 127.0.0.1:80;/' /etc/piler/piler-nginx.conf
+      systemctl stop nginx
+      sleep 5
+      systemctl start nginx
+   fi
+
+   if [[ ! -f /etc/piler/piler.conf ]]; then
+      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.example.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > /etc/piler/piler.conf
+      chmod 600 /etc/piler/piler.conf
+      chown $PILER_USER:$PILER_USER /etc/piler/piler.conf
+   fi
+
+   sed -i 's/tls_enable=0/tls_enable=1/g' /etc/piler/piler.conf
+
+   sed -i -e "s/MYSQL_HOSTNAME/${MYSQL_HOSTNAME}/g" \
+          -e "s/MYSQL_DATABASE/${MYSQL_DATABASE}/g" \
+          -e "s/MYSQL_USERNAME/${MYSQL_USERNAME}/g" \
+          -e "s/MYSQL_PASSWORD/${MYSQL_PILER_PASSWORD}/g" \
+          /etc/piler/sphinx.conf
+}
+
+setup_traefik() {
+   wget -O /etc/systemd/system/traefik.service "${DOWNLOAD_URL}/generic-local/traefik.service"
+
+   mkdir -p /usr/local/etc/traefik
+   touch /usr/local/etc/traefik/acme.json
+   chmod 600 /usr/local/etc/traefik/acme.json
+   chown www-data:www-data /usr/local/etc/traefik/acme.json
+
+   cat > /usr/local/etc/traefik/traefik.yaml << TRAEFIK
+log:
+  level: INFO
+
+entryPoints:
+  web:
+    address: "$MY_IP:80"
+    http:
+      redirections:
+        entryPoint:
+          to: websecure
+          scheme: https
+          permanent: true
+  websecure:
+    address: "$MY_IP:443"
+
+providers:
+  file:
+    filename: "/usr/local/etc/traefik/traefik.yaml"
+
+certificatesResolvers:
+  le:
+    acme:
+      storage: "/usr/local/etc/traefik/acme.json"
+      email: admin@$PILER_HOSTNAME
+      httpChallenge:
+        entryPoint: web
+
+tls:
+  options:
+    default:
+      minVersion: VersionTLS13
+
+http:
+  middlewares:
+    piler_headers:
+      headers:
+        customResponseHeaders:
+          Server: ""
+          Strict-Transport-Security: "max-age=31536000"
+          X-Content-Type-Optionsi: "nosniff"
+          Referrer-Policy: "same-origin"
+  routers:
+    master:
+      rule: "Host(\`$PILER_HOSTNAME\`)"
+      service: www
+      middlewares:
+        - "piler_headers"
+      tls:
+        certResolver: le
+  services:
+    www:
+      loadBalancer:
+        servers:
+        - url: "http://127.0.0.1:80/"
+TRAEFIK
+}
+
+
+install_prerequisites
+
+start_mysql
+
+install_piler
+
+create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
+create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf
+
+fix_config_site_php
+
+setup_traefik
+
+add_systemd_services
+
+fix_configs
+init_db
+
+su -c "indexer --all -c /etc/piler/sphinx.conf" $PILER_USER
+
+[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler
+
+systemctl start pilersearch
+systemctl start piler
+systemctl start piler-smtp
+systemctl start traefik

contrib/installer/jammy.sh

@@ -0,0 +1,258 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+
+MY_IP="1.2.3.4"
+PILER_HOSTNAME="${PILER_HOSTNAME:-archive.example.com}"
+MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
+MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
+SERVER_ID="${SERVER_ID:-0}"
+USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
+SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
+PHP_FPM_SOCKET="/var/run/php/php8.1-fpm.sock"
+
+MYSQL_HOSTNAME="localhost"
+MYSQL_DATABASE="piler"
+MYSQL_USERNAME="piler"
+
+DOWNLOAD_URL="https://download.mailpiler.com"
+PILER_DEB="piler_1.4.4-jammy-0573c6da_amd64.deb"
+PILER_USER="piler"
+PILER_CONF="/etc/piler/piler.conf"
+CONFIG_SITE_PHP="/etc/piler/config-site.php"
+SEARCHCFG="/etc/piler/manticore.conf"
+
+TRAEFIK_VERSION="v2.11.0"
+ARCH="amd64"
+
+export DEBIAN_FRONTEND=noninteractive
+
+install_prerequisites() {
+   apt-get update
+   apt-get -y --no-install-recommends install \
+      wget rsyslog openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql \
+      php8.1-fpm php8.1-zip php8.1-ldap php8.1-gd php8.1-curl php8.1-xml \
+      php8.1-mbstring ca-certificates zip catdoc unrtf poppler-utils \
+      nginx tnef libzip4 libtre5 libwrap0 cron libmariadb-dev python3 \
+      python3-mysqldb libmariadb-dev mariadb-client-core-10.6 \
+      mariadb-server-10.6
+
+   wget "https://github.com/traefik/traefik/releases/download/${TRAEFIK_VERSION}/traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz"
+   tar zxvf "traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz" traefik
+   chown root:root traefik
+   chmod 755 traefik
+   mv traefik /usr/local/bin
+   setcap cap_net_bind_service+ep /usr/local/bin/traefik
+
+   wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
+   dpkg -i manticore-repo.noarch.deb && \
+   apt-get update && \
+   apt-get install -y manticore manticore-columnar-lib manticore-extra && \
+   rm -f manticore-repo.noarch.deb
+   systemctl stop manticore
+}
+
+
+fix_mysql_settings() {
+   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
+[mysqld]
+
+innodb_buffer_pool_size=512M
+innodb_flush_log_at_trx_commit=1
+innodb_log_buffer_size=64M
+innodb_log_file_size=64M
+innodb_read_io_threads=4
+innodb_write_io_threads=4
+innodb_log_files_in_group=2
+
+innodb_file_per_table
+PILER_CNF
+}
+
+start_mysql() {
+   fix_mysql_settings
+
+   service mysql restart
+   mysqladmin -u root password "${MYSQL_ROOT_PASSWORD}"
+}
+
+
+install_piler() {
+   wget "https://bitbucket.org/jsuto/piler/downloads/${PILER_DEB}"
+   dpkg -i "$PILER_DEB"
+   rm -f "$PILER_DEB"
+
+   sed -i 's/.*indexer.*/###/g' /usr/share/piler/piler.cron
+   crontab -u "$PILER_USER" /usr/share/piler/piler.cron
+
+   rm -f "$PILER_DEB" /etc/nginx/sites-enabled/default
+   ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf
+
+   touch /var/piler/.bash_history
+   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
+}
+
+
+create_my_cnf() {
+   local user=$1
+   local password=$2
+   local my_cnf=$3
+
+   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
+   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
+   chown $PILER_USER:$PILER_USER "$my_cnf"
+   chmod 600 "$my_cnf"
+}
+
+
+fix_config_site_php() {
+   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"
+
+   {
+      echo "\$config['SERVER_ID'] = $SERVER_ID;"
+      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
+   } >> "$CONFIG_SITE_PHP"
+
+   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
+      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
+   fi
+
+   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
+   echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
+}
+
+
+init_db() {
+   sed -e "s%MYSQL_HOSTNAME%$MYSQL_HOSTNAME%g" -e "s%MYSQL_DATABASE%$MYSQL_DATABASE%g" -e "s%MYSQL_USERNAME%$MYSQL_USERNAME%g" -e "s%MYSQL_PASSWORD%$MYSQL_PILER_PASSWORD%g" \
+      /usr/share/piler/db-mysql-root.sql.in | mysql --defaults-file=/etc/piler/.my.cnf-root -h $MYSQL_HOSTNAME
+   mysql --defaults-file=/etc/piler/.my.cnf -h $MYSQL_HOSTNAME $MYSQL_DATABASE < /usr/share/piler/db-mysql.sql
+}
+
+
+add_systemd_services() {
+   pushd /etc/systemd/system
+
+   ln -sf /usr/libexec/piler/piler.service .
+   ln -sf /usr/libexec/piler/piler-smtp.service .
+   ln -sf /usr/libexec/piler/pilersearch.service .
+
+   popd
+
+   systemctl daemon-reload
+
+   systemctl enable piler
+   systemctl enable piler-smtp
+   systemctl enable pilersearch
+   systemctl enable traefik
+}
+
+
+fix_configs() {
+   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
+      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
+      nginx -t
+      nginx -s reload
+
+      sed -i 's/server {/server {\n\tlisten 127.0.0.1:80;/' /etc/piler/piler-nginx.conf
+      systemctl stop nginx
+      sleep 5
+      systemctl start nginx
+   fi
+
+   if [[ ! -f "$PILER_CONF" ]]; then
+      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.example.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > "$PILER_CONF"
+      chmod 600 "$PILER_CONF"
+      chown $PILER_USER:$PILER_USER "$PILER_CONF"
+   fi
+
+   sed -i 's/tls_enable=0/tls_enable=1/g' "$PILER_CONF"
+   sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
+
+   sed -i "s/define('RT.*/define('RT', 1);/" "$SEARCHCFG"
+}
+
+setup_traefik() {
+   wget -O /etc/systemd/system/traefik.service "${DOWNLOAD_URL}/generic-local/traefik.service"
+
+   mkdir -p /usr/local/etc/traefik
+   touch /usr/local/etc/traefik/acme.json
+   chmod 600 /usr/local/etc/traefik/acme.json
+   chown www-data:www-data /usr/local/etc/traefik/acme.json
+
+   cat > /usr/local/etc/traefik/traefik.yaml << TRAEFIK
+log:
+  level: INFO
+
+entryPoints:
+  websecure:
+    address: "$MY_IP:443"
+
+providers:
+  file:
+    filename: "/usr/local/etc/traefik/traefik.yaml"
+
+certificatesResolvers:
+  le:
+    acme:
+      storage: "/usr/local/etc/traefik/acme.json"
+      email: admin@$PILER_HOSTNAME
+      tlsChallenge: {}
+
+tls:
+  options:
+    default:
+      minVersion: VersionTLS13
+
+http:
+  middlewares:
+    piler_headers:
+      headers:
+        customResponseHeaders:
+          Server: ""
+          Strict-Transport-Security: "max-age=31536000"
+          X-Content-Type-Optionsi: "nosniff"
+          Referrer-Policy: "same-origin"
+  routers:
+    master:
+      rule: "Host(\`$PILER_HOSTNAME\`)"
+      service: www
+      middlewares:
+        - "piler_headers"
+      tls:
+        certResolver: le
+  services:
+    www:
+      loadBalancer:
+        servers:
+        - url: "http://127.0.0.1:80/"
+TRAEFIK
+}
+
+
+install_prerequisites
+
+start_mysql
+
+install_piler
+
+create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
+create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf
+
+fix_config_site_php
+
+setup_traefik
+
+add_systemd_services
+
+fix_configs
+init_db
+
+[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler
+
+systemctl start pilersearch
+systemctl start piler
+systemctl start piler-smtp
+systemctl start traefik

contrib/proxmox-lxc/README

@@ -0,0 +1,9 @@
+You nedd proxmox with lxc
+
+Fill out conf file first and provide public key
+
+It automatically
+- downloads the latest debian template
+- provides a container
+- installs piler until the post installation step
+- gives a self signed certificate

contrib/proxmox-lxc/create_lxc.sh

@@ -0,0 +1,195 @@
+#!/bin/bash
+
+# This script wil create and fire up a standard debian buster lxc container on your proxmox pve.
+# The Script will look for the next free lxc number and take the next free and use it. So take
+# care that behind your last number is place for it. 
+
+#### SOME VARIABLES TO ADJUST ####
+
+# Storage with templates
+LXC_TMP="local"
+
+# Size and pool of rootfs / in GB
+SIZ_ROT="100"
+S_ROT_P="local-zfs"
+
+# Size and pool of Filestorage in GB will mounted to /share
+SIZ_FIL="100"
+S_FIL_P="local-zfs"
+
+#Weather or not (1 and 0) the container will createt as unpriviliged LXC
+LXC_UNP="1"
+
+# Size of the RAM assigned to the LXC
+LXC_MEM="1024"
+
+# Size of the SWAP assigned to the LXC
+LXC_SWA="1024"
+
+# The hostname (eq. zamba1 or mailpiler1)
+LXC_HOST="zamba"
+
+# The domainname (searchdomain /etc/resolf.conf & hosts)
+LXC_SDN="zmb.local"
+
+# IP-address and subnet
+LXC_IP="10.10.80.20/24"
+
+# Gateway
+LXC_GW="10.10.80.10"
+
+# DNS-server and here shoud be your AD-DC
+LXC_DNS="10.10.80.10"
+
+# Networkbridge for this machine
+LXC_BRD="vmbr80"
+
+# root password - take care to delete from this file
+LXC_PWD="MYPASSWD"
+
+LXC_KEY="ssh-rsa xxxxxxxx"
+
+############### Zamba-Server-Section ###############
+
+# Domain Entries to samba/smb.conf. Will be also uses for samba domain-provisioning when zmb-pdc will choosen.
+ZMB_REA="ZMB.LOCAL"
+ZMB_DOM="ZMB"
+
+# THE Domain-Admin and passwd for zamba-install
+ZMB_ADA="Administrator"
+ZMB_APW="MYPASSWORD"
+
+############### Mailpiler-Section ###############
+
+# The FQDN vor the Hostname. This must be exactly the same like the LXC_HOST / LXC_SDN at section above.
+PILER_DOM="piler.zmb.rocks"
+SMARTHOST="10.10.80.20"
+PILER_VER="1.3.10"
+SPHINX_VER="3.3.1"
+PHP_VER="7.4"
+
+############### Matrix-Section ###############
+
+# The FQDN vor the Hostname. This should be the same like the LXC_HOST / LXC_SDN at section above.
+MRX_DOM="matrix.zmb.rocks"
+ELE_DOM="element.zmb.rocks"
+ELE_VER="v1.7.21"
+JIT_DOM="meet.zmb.rocks"
+
+#################################
+
+# CHeck is the newest template available, else download it.
+
+DEB_LOC=$(pveam list $LXC_TMP | grep debian-10-standard | cut -d'_' -f2)
+
+DEB_REP=$(pveam available --section system | grep debian-10-standard | cut -d'_' -f2)
+
+if [[ $DEB_LOC == $DEB_REP ]];
+then
+  echo "Newest Version of Debian 10 Standard $DEP_REP exists.";
+else
+  echo "Will now download newest Debian 10 Standard $DEP_REP.";
+  pveam download $LXC_TMP debian-10-standard_$DEB_REP\_amd64.tar.gz
+fi
+
+# Get next free LXC-number
+LXC_LST=$( lxc-ls | egrep -o '.{1,5}$' )
+LXC_CHK=$((LXC_LST+1));
+
+if  [ $LXC_CHK -lt 100 ] || [ -f /etc/pve/qemu-server/$LXC_CHK.conf ]; then
+  LXC_NBR=$(pvesh get /cluster/nextid);
+else
+  LXC_NBR=$LXC_CHK;
+fi
+
+echo "Will now create LXC Container $LXC_NBR!";
+
+# Create the container
+pct create $LXC_NBR -unprivileged $LXC_UNP $LXC_TMP:vztmpl/debian-10-standard_$DEB_REP\_amd64.tar.gz -rootfs $S_ROT_P:$SIZ_ROT;
+sleep 2;
+
+pct set $LXC_NBR -memory $LXC_MEM -swap $LXC_SWA -hostname $LXC_HOST \-nameserver $LXC_DNS -searchdomain $LXC_SDN -onboot 1 -timezone Europe/Berlin -net0 name=eth0,bridge=$LXC_BRD,firewall=1,gw=$LXC_GW,ip=$LXC_IP,type=veth;
+sleep 2;
+
+PS3="Select the Server-Function: "
+
+select opt in just_lxc zmb-standalone zmb-member zmb-pdc mailpiler matrix quit; do
+  case $opt in
+    just_lxc)
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      echo "Should be ready!"
+      break
+      ;;
+    zmb-standalone)
+      break
+      ;;
+    zmb-member)
+      echo "Make some additions to LXC for AD-Member-Server!"
+      pct set $LXC_NBR -mp0 $S_FIL_P:$SIZ_FIL,mp=/tank
+      sleep 2;
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      cp /root/zmb_mem.orig /root/zmb_mem.sh
+      sed -i "s|#ZMB_VAR|#ZMB_VAR\nZMB_REA='$ZMB_REA'\nZMB_DOM='$ZMB_DOM'\nZMB_ADA='$ZMB_ADA'\nZMB_APW='$ZMB_APW'|" /root/zmb_mem.sh
+      pct push $LXC_NBR /root/zmb_mem.sh /root/zmb_mem.sh
+      echo "Install zamba as AD-Member-Server!"
+      lxc-attach -n$LXC_NBR bash /root/zmb_mem.sh
+      break
+      ;;
+    zmb-pdc)
+      break
+      ;;
+    mailpiler)
+      echo "Make some additions to LXC for Mailpiler!"
+      pct set $LXC_NBR -features nesting=1
+      sleep 2;
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      cp /root/mailpiler.orig /root/mailpiler.sh
+      sed -i "s|#PILER_VAR|#PILER_VAR\nPILER_DOM='$PILER_DOM'\nSMARTHOST='$SMARTHOST'\nPILER_VER='$PILER_VER'\nSPHINX_VER='$SPHINX_VER'\nPHP_VER='$PHP_VER'|" /root/mailpiler.sh
+      pct push $LXC_NBR /root/mailpiler.sh /root/mailpiler.sh
+      echo "Install Mailpiler mailarchiv!"
+      lxc-attach -n$LXC_NBR bash mailpiler.sh
+      break
+      ;;
+    matrix)
+      echo "Make some additions to LXC for Matrix!"
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      cp /root/matrix.orig /root/matrix.sh
+      sed -i "s|#MATRIX_VAR|#Matrix_VAR\nMRX_DOM='$MRX_DOM'\nELE_DOM='$ELE_DOM'\nELE_VER='$ELE_VER'\nJIT_DOM='$JIT_DOM'|" /root/matrix.sh
+      pct push $LXC_NBR /root/matrix.sh /root/matrix.sh
+      echo "Install Matrix Chatserver!"
+      lxc-attach -n$LXC_NBR bash matrix.sh
+      break
+      ;;
+    quit)
+      break
+      ;;
+    *)
+      echo "Invalid option!"
+      ;;
+    esac
+done
+

contrib/proxmox-lxc/mailpiler.orig

@@ -0,0 +1,179 @@
+#!/bin/bash
+
+#Variables will be filled in from the mainscript:
+
+#PILER_VAR
+
+
+HOSTNAME=$(hostname -f)
+
+echo "Ensure your Hostname is set to your Piler FQDN!"
+
+echo $HOSTNAME
+
+if 
+    [ "$HOSTNAME" != "$PILER_DOM" ]
+then
+        echo "Hostname doesn't match Piler_Domain! Check install.sh, /etc/hosts, /etc/hostname." && exit
+else
+        echo "Hostname matches PILER_DOMAIN, so starting installation."
+fi
+
+apt install -y gpg apt-transport-https lsb-release
+
+wget -q https://packages.sury.org/php/apt.gpg -O- | apt-key add -
+echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" | tee /etc/apt/sources.list.d/php.list
+
+apt update && apt full-upgrade -y
+
+apt install -y mc sysstat build-essential libwrap0-dev libpst-dev tnef libytnef0-dev unrtf catdoc libtre-dev tre-agrep poppler-utils libzip-dev unixodbc libpq5 software-properties-common libpoppler-dev openssl libssl-dev memcached telnet nginx mariadb-server default-libmysqlclient-dev python-mysqldb gcc libwrap0 libzip4 latex2rtf latex2html catdoc tnef libpq5 zipcmp zipmerge ziptool libsodium23
+
+apt update && apt install -y php$PHP_VER-{fpm,common,ldap,mysql,cli,opcache,phpdbg,gd,memcache,json,readline,zip}
+
+apt purge -y postfix
+
+cat > /etc/mysql/conf.d/mailpiler.conf <<EOF
+innodb_buffer_pool_size=256M
+innodb_flush_log_at_trx_commit=1
+innodb_log_buffer_size=64M
+innodb_log_file_size=16M
+query_cache_size=0
+query_cache_type=0
+query_cache_limit=2M
+EOF
+
+systemctl restart mariadb
+
+cd /tmp
+wget https://download.mailpiler.com/generic-local/sphinx-$SPHINX_VER-bin.tar.gz
+tar -xvzf sphinx-$SPHINX_VER-bin.tar.gz -C /
+
+groupadd piler
+useradd -g piler -m -s /bin/bash -d /var/piler piler
+usermod -L piler
+chmod 755 /var/piler
+
+wget https://bitbucket.org/jsuto/piler/downloads/piler-$PILER_VER.tar.gz
+tar -xvzf piler-$PILER_VER.tar.gz
+cd piler-$PILER_VER/
+./configure --localstatedir=/var --with-database=mysql --enable-tcpwrappers --enable-memcached
+make
+make install
+ldconfig
+
+cp util/postinstall.sh util/postinstall.sh.bak
+sed -i "s/   SMARTHOST=.*/   SMARTHOST="\"$SMARTHOST\""/" util/postinstall.sh
+sed -i 's/   WWWGROUP=.*/   WWWGROUP="www-data"/' util/postinstall.sh
+
+make postinstall
+
+cp /usr/local/etc/piler/piler.conf /usr/local/etc/piler/piler.conf.bak
+sed -i "s/hostid=.*/hostid=$PILER_DOM/" /usr/local/etc/piler/piler.conf
+sed -i "s/update_counters_to_memcached=.*/update_counters_to_memcached=1/" /usr/local/etc/piler/piler.conf
+
+su piler -c "indexer --all --config /usr/local/etc/piler/sphinx.conf"
+
+/etc/init.d/rc.piler start
+/etc/init.d/rc.searchd start
+
+update-rc.d rc.piler defaults
+update-rc.d rc.searchd defaults
+
+mkdir -p /etc/nginx/ssl
+openssl req -x509 -nodes -days 3650 -newkey rsa:4096 -keyout /etc/nginx/ssl/piler.key -out /etc/nginx/ssl/piler.crt -subj "/CN=$PILER_DOM" -addext "subjectAltName=DNS:$PILER_DOM"
+
+cd /etc/nginx/sites-available
+cp /tmp/piler-$PILER_VER/contrib/webserver/piler-nginx.conf /etc/nginx/sites-available/
+ln -s /etc/nginx/sites-available/piler-nginx.conf /etc/nginx/sites-enabled/piler-nginx.conf
+
+sed -i "s|PILER_HOST|$PILER_DOM|g" /etc/nginx/sites-available/piler-nginx.conf
+sed -i "s|/var/run/php/php7.4-fpm.sock|/var/run/php/php$PHP_VER-fpm.sock|g" /etc/nginx/sites-available/piler-nginx.conf
+
+sed -i "/server_name.*/a \\
+        listen 443 ssl http2;\n\n\
+        ssl_certificate /etc/nginx/ssl/piler.crt;\n\
+        ssl_certificate_key /etc/nginx/ssl/piler.key;\n\n\
+        ssl_session_timeout 1d;\n\
+        ssl_session_cache shared:SSL:15m;\n\
+        ssl_session_tickets off;\n\n\
+        # modern configuration of Mozilla SSL configurator. Tweak to your needs.\n\
+        ssl_protocols TLSv1.2 TLSv1.3;\n\
+        ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;\n\
+        ssl_prefer_server_ciphers off;\n\n\
+        add_header X-Frame-Options SAMEORIGIN;\n\
+        add_header X-Content-Type-Options nosniff;" /etc/nginx/sites-available/piler-nginx.conf
+
+sed -i "/^server {.*/i\
+server {\n\
+        listen 80;\n\
+        server_name $PILER_DOM;\n\
+        server_tokens off;\n\
+        # HTTP to HTTPS redirect.\n\
+        return 301 https://$PILER_DOM;\n\
+}" /etc/nginx/sites-available/piler-nginx.conf
+
+cp /usr/local/etc/piler/config-site.php /usr/local/etc/piler/config-site.php.bak
+sed -i "s|\$config\['SITE_URL'\] = .*|\$config\['SITE_URL'\] = 'https://$PILER_DOM/';|" /usr/local/etc/piler/config-site.php
+cat >> /usr/local/etc/piler/config-site.php <<EOF
+
+// CUSTOM
+\$config['PROVIDED_BY'] = '$PILER_DOM';
+\$config['SUPPORT_LINK'] = 'https://$PILER_DOM';
+\$config['COMPATIBILITY'] = '';
+
+// fancy features.
+\$config['ENABLE_INSTANT_SEARCH'] = 1;
+\$config['ENABLE_TABLE_RESIZE'] = 1;
+
+\$config['ENABLE_DELETE'] = 1;
+\$config['ENABLE_ON_THE_FLY_VERIFICATION'] = 1;
+
+// general settings.
+\$config['TIMEZONE'] = 'Europe/Berlin';
+
+// authentication
+// Enable authentication against an imap server
+//\$config['ENABLE_IMAP_AUTH'] = 1;
+//\$config['RESTORE_OVER_IMAP'] = 1;
+//\$config['IMAP_RESTORE_FOLDER_INBOX'] = 'INBOX';
+//\$config['IMAP_RESTORE_FOLDER_SENT'] = 'Sent';
+//\$config['IMAP_HOST'] = '$SMARTHOST';
+//\$config['IMAP_PORT'] =  993;
+//\$config['IMAP_SSL'] = true;
+
+// authentication against an ldap directory (disabled by default)
+//\$config['ENABLE_LDAP_AUTH'] = 1;
+//\$config['LDAP_HOST'] = '$SMARTHOST';
+//\$config['LDAP_PORT'] = 389;
+//\$config['LDAP_HELPER_DN'] = 'cn=administrator,cn=users,dc=mydomain,dc=local';
+//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
+//\$config['LDAP_MAIL_ATTR'] = 'mail';
+//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
+//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
+//\$config['LDAP_BASE_DN'] = 'ou=Benutzer,dc=krs,dc=local';
+
+// authentication against an Uninvention based ldap directory 
+//\$config['ENABLE_LDAP_AUTH'] = 1;
+//\$config['LDAP_HOST'] = '$SMARTHOST';
+//\$config['LDAP_PORT'] = 7389;
+//\$config['LDAP_HELPER_DN'] = 'uid=ldap-search-user,cn=users,dc=mydomain,dc=local';
+//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
+//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
+//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
+//\$config['LDAP_BASE_DN'] = 'cn=users,dc=mydomain,dc=local';
+//\$config['LDAP_MAIL_ATTR'] = 'mailPrimaryAddress';
+//\$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'person';
+//\$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'person';
+//\$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mailAlternativeAddress';
+
+// special settings.
+\$config['MEMCACHED_ENABLED'] = 1;
+\$config['SPHINX_STRICT_SCHEMA'] = 1; // required for Sphinx $SPHINX_VER, see https://bitbucket.org/jsuto/piler/issues/1085/sphinx-331.
+EOF
+
+nginx -t && systemctl restart nginx
+
+apt autoremove -y
+apt clean -y
+
+

contrib/proxmox-lxc/matrix.orig

@@ -0,0 +1,150 @@
+#!/bin/bash
+
+#MATRIX_VAR
+
+
+MRX_PKE=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+
+ELE_DBNAME="synapse_db"
+ELE_DBUSER="synapse_user"
+ELE_DBPASS=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+
+apt update && apt full-upgrade -y
+
+apt install -y lsb-release apt-transport-https curl gpg software-properties-common net-tools nginx mc postgresql python3-psycopg2
+
+wget wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/matrix-org.list
+apt update && apt install -y matrix-synapse-py3
+systemctl enable matrix-synapse
+
+ss -tulpen
+
+mkdir /etc/nginx/ssl
+openssl req -x509 -nodes -days 3650 -newkey rsa:4096 -keyout /etc/nginx/ssl/matrix.key -out /etc/nginx/ssl/matrix.crt -subj "/CN=$MRX_DOM" -addext "subjectAltName=DNS:$MRX_DOM"
+
+cat > /etc/nginx/sites-available/$MRX_DOM <<EOF
+# Virtual Host configuration for example.com
+#
+# You can move that to a different file under sites-available/ and symlink that
+# to sites-enabled/ to enable it.
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name $MRX_DOM;
+
+    return 301 https://$MRX_DOM;
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    server_name $MRX_DOM;
+
+    ssl on;
+    ssl_certificate /etc/nginx/ssl/matrix.crt;
+    ssl_certificate_key /etc/nginx/ssl/matrix.key;
+
+    location / {
+      proxy_pass http://127.0.0.1:8008;
+      proxy_set_header X-Forwarded-For \$remote_addr;
+    }
+}
+
+server {
+    listen 8448 ssl;
+    listen [::]:8448 ssl;
+    server_name $MRX_DOM;
+
+    ssl on;
+    ssl_certificate /etc/nginx/ssl/matrix.crt;
+    ssl_certificate_key /etc/nginx/ssl/matrix.key;
+
+    # If you don't wanna serve a site, comment this out
+    root /var/www/$MRX_DOM;
+    index index.html index.htm;
+
+    location / {
+        proxy_pass http://127.0.0.1:8008;
+        proxy_set_header X-Forwarded-For \$remote_addr;
+    }
+}
+
+EOF
+ln -s /etc/nginx/sites-available/$MRX_DOM /etc/nginx/sites-enabled/$MRX_DOM
+
+cat > /etc/nginx/sites-available/$ELE_DOM <<EOF
+# Virtual Host configuration for example.com
+#
+# You can move that to a different file under sites-available/ and symlink that
+# to sites-enabled/ to enable it.
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name $ELE_DOM;
+    return 301 https://$ELE_DOM;
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    server_name $ELE_DOM;
+
+    ssl on;
+    ssl_certificate /etc/nginx/ssl/matrix.crt;
+    ssl_certificate_key /etc/nginx/ssl/matrix.key;
+
+    # If you don't wanna serve a site, comment this out
+    root /var/www/$ELE_DOM/element;
+    index index.html index.htm;
+} 
+
+EOF
+
+ln -s /etc/nginx/sites-available/$ELE_DOM /etc/nginx/sites-enabled/$ELE_DOM
+
+systemctl restart nginx
+
+mkdir /var/www/$ELE_DOM
+cd /var/www/$ELE_DOM
+wget https://packages.riot.im/element-release-key.asc
+gpg --import element-release-key.asc
+
+wget https://github.com/vector-im/element-web/releases/download/$ELE_VER/element-$ELE_VER.tar.gz
+wget https://github.com/vector-im/element-web/releases/download/$ELE_VER/element-$ELE_VER.tar.gz.asc
+gpg --verify element-$ELE_VER.tar.gz.asc
+
+tar -xzvf element-$ELE_VER.tar.gz
+ln -s element-$ELE_VER element
+chown www-data:www-data -R element
+cp ./element/config.sample.json ./element/config.json
+sed -i "s|https://matrix-client.matrix.org|https://$MRX_DOM|" ./element/config.json
+sed -i "s|\"server_name\": \"matrix.org\"|\"server_name\": \"$MRX_DOM\"|" ./element/config.json
+
+su postgres <<EOF
+psql -c "CREATE USER $ELE_DBUSER WITH PASSWORD '$ELE_DBPASS';"
+psql -c "CREATE DATABASE $ELE_DBNAME ENCODING 'UTF8' LC_COLLATE='C' LC_CTYPE='C' template=template0 OWNER $ELE_DBUSER;"
+echo "Postgres User '$ELE_DBUSER' and database '$ELE_DBNAME' created."
+EOF
+
+cd /
+sed -i "s|#registration_shared_secret: <PRIVATE STRING>|registration_shared_secret: \"$MRX_PKE\"|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|#public_baseurl: https://example.com/|public_baseurl: https://$MRX_DOM/|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|#enable_registration: false|enable_registration: true|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|name: sqlite3|name: psycopg2|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|database: /var/lib/matrix-synapse/homeserver.db|database: $ELE_DBNAME\n    user: $ELE_DBUSER\n    password: $ELE_DBPASS\n    host: 127.0.0.1\n    cp_min: 5\n    cp_max: 10|" /etc/matrix-synapse/homeserver.yaml
+
+systemctl restart matrix-synapse
+
+register_new_matrix_user -c /etc/matrix-synapse/homeserver.yaml http://127.0.0.1:8008
+
+#curl https://download.jitsi.org/jitsi-key.gpg.key | sh -c 'gpg --dearmor > /usr/share/keyrings/jitsi-keyring.gpg'
+#echo 'deb [signed-by=/usr/share/keyrings/jitsi-keyring.gpg] https://download.jitsi.org stable/' | tee /etc/apt/sources.list.d/jitsi-stable.list > /dev/null
+
+#apt update
+#apt install -y jitsi-meet
+
+
+

contrib/proxmox-lxc/zmb_mem.orig

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+#ZMB_VAR
+
+
+
+apt update && apt full-upgrade -y
+echo -ne '\n' | apt install -y acl dnsutils mc samba winbind libpam-winbind libnss-winbind krb5-user krb5-config samba-dsdb-modules samba-vfs-modules 
+
+mv /etc/krb5.conf /etc/krb5.conf.bak
+cat > /etc/krb5.conf <<EOF
+[libdefaults]
+	default_realm = $ZMB_REA
+    ticket_lifetime = 600
+	dns_lookup_realm = true
+	dns_lookup_kdc = true
+	renew_lifetime = 7d
+EOF
+
+echo -e "$ZMB_APW" | kinit -V $ZMB_ADA
+klist
+
+mv /etc/samba/smb.conf /etc/samba/smb.conf.bak
+cat > /etc/samba/smb.conf <<EOF
+[global]
+	workgroup = $ZMB_DOM
+	security = ADS
+	realm = $ZMB_REA
+	server string = %h server
+
+	vfs objects = acl_xattr shadow_copy2
+    map acl inherit = Yes
+    store dos attributes = Yes
+	idmap config *:backend = tdb
+	idmap config *:range = 3000000-4000000
+	idmap config *:schema_mode = rfc2307
+
+	winbind refresh tickets = Yes
+	winbind use default domain = Yes
+	winbind separator = /
+	winbind nested groups = yes
+	winbind nss info = rfc2307
+
+	pam password change = Yes
+	passwd chat = *Enter\snew\s*\spassword:* %n\n *Retype\snew\s*\spassword:* %n\n *password\supdated\ssuccessfully* .
+	passwd program = /usr/bin/passwd %u
+
+	template homedir = /home/%U
+	template shell = /bin/bash
+	bind interfaces only = Yes
+	interfaces = lo eth0
+	log file = /var/log/samba/log.%m
+	logging = syslog
+	max log size = 1000
+	panic action = /usr/share/samba/panic-action %d
+
+	load printers = No
+	printcap name = /dev/null
+	printing = bsd
+	disable spoolss = Yes
+
+	allow trusted domains = No
+	dns proxy = No
+	shadow: snapdir = .zfs/snapshot
+	shadow: sort = desc
+	shadow: format = -%Y-%m-%d-%H%M
+	shadow: snapprefix = ^zfs-auto-snap_\(frequent\)\{0,1\}\(hourly\)\{0,1\}\(daily\)\{0,1\}\(monthly\)\{0,1\}
+	shadow: delimiter = -20
+
+[share]
+	comment = Main Share
+	path = /tank/share
+	read only = No
+	create mask = 0660
+	directory mask = 0770
+	inherit acls = Yes
+
+
+
+EOF
+
+systemctl restart smbd
+
+echo -e "$ZMB_APW" | net ads join -U $ZMB_ADA createcomputer=Computers
+sed -i "s|files systemd|files systemd winbind|g" /etc/nsswitch.conf
+sed -i "s|#WINBINDD_OPTS=|WINBINDD_OPTS=|" /etc/default/winbind
+echo -e "session optional        pam_mkhomedir.so skel=/etc/skel umask=077" >> /etc/pam.d/common-session
+
+systemctl restart winbind nmbd
+wbinfo -u
+wbinfo -g
+
+mkdir /tank/share
+chown 'administrator':'domain users' /tank/share
+
+setfacl -Rm u:administrator:rwx,g::-,o::- /tank/share
+setfacl -Rdm u:administrator:rwx,g::-,o::- /tank/share
+
+systemctl restart smbd nmbd winbind
+

contrib/smtp/smtp-source.py

@@ -168,4 +168,4 @@ while i < args.count:
 
 
 if args.debug == 0:
-    print
+    print('')

contrib/webserver/piler-nginx.conf

@@ -4,10 +4,13 @@ server {
 
         root /var/piler/www;
 
-        access_log  /var/log/nginx/access.log;
-        error_log   /var/log/nginx/error.log;
+        server_tokens off;
 
-        gzip  on;
+        add_header X-XSS-Protection "1; mode=block";
+        add_header X-Content-Type-Options "nosniff";
+        add_header Referrer-Policy "same-origin";
+
+        gzip on;
         gzip_types text/plain application/xml text/css;
         gzip_vary on;
 
@@ -31,8 +34,7 @@ server {
                return 404;
             }
 
-            #fastcgi_pass unix:/var/run/php5-fpm.sock;
-            fastcgi_pass unix:/var/run/php/php7.0-fpm.sock;
+            fastcgi_pass unix:/var/run/php/php8.1-fpm.sock;
             fastcgi_index index.php;
             include fastcgi_params;
         }
@@ -49,6 +51,7 @@ server {
         rewrite /message.php /index.php?route=message/view;
         rewrite /bulkrestore.php /index.php?route=message/bulkrestore;
         rewrite /bulkremove.php /index.php?route=message/bulkremove;
+        rewrite /rejectremove.php /index.php?route=message/rejectremove;
         rewrite /bulkpdf.php /index.php?route=message/bulkpdf;
         rewrite /folders.php /index.php?route=folder/list&;
         rewrite /settings.php /index.php?route=user/settings;

cppcheck.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+cppcheck -DHAVE_PDFTOTEXT -DHAVE_PPTHTML -DHAVE_TNEF -DHAVE_UNRTF -DHAVE_XLS2CSV -DHAVE_CATPPT -DHAVE_CATDOC -DHAVE_ZIP -D_GNU_SOURCE -DHAVE_DAEMON -DHAVE_TRE -DHAVE_CLAMD -DHAVE_LIBCLAMAV -DNEED_MYSQL --error-exitcode=1 --enable=all --suppressions-list=suppressions.txt --force src/ unit_tests/

docker/Dockerfile

@@ -0,0 +1,43 @@
+FROM ubuntu:22.04
+
+ARG PACKAGE
+
+LABEL description="piler ubuntu jammy image" \
+      maintainer="Janos SUTO, sj@acts.hu" \
+      package="${PACKAGE}"
+
+ENV DEBIAN_FRONTEND="noninteractive" \
+    DISTRO="jammy" \
+    PILER_USER="piler" \
+    MYSQL_DATABASE="piler"
+
+COPY ${PACKAGE} /
+
+RUN apt-get update && \
+    apt-get -y --no-install-recommends install \
+       wget openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql php8.1-fpm php8.1-zip php8.1-ldap \
+       php8.1-gd php8.1-curl php8.1-xml php8.1-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip4 \
+       libtre5 cron libmariadb-dev mariadb-client-core-10.6 python3 python3-mysqldb ca-certificates curl rsyslog && \
+    wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
+    dpkg -i manticore-repo.noarch.deb && \
+    rm -f manticore-repo.noarch.deb && \
+    apt-get update && \
+    apt-get install -y manticore manticore-columnar-lib && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    sed -i '/session    required     pam_loginuid.so/c\#session    required     pam_loginuid.so' /etc/pam.d/cron && \
+    dpkg -i ${PACKAGE} && \
+    touch /etc/piler/MANTICORE && \
+    ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled && \
+    rm -f ${PACKAGE} /etc/nginx/sites-enabled/default /etc/piler/piler.key /etc/piler/piler.pem /etc/piler/config-site.php && \
+    crontab -u $PILER_USER /usr/share/piler/piler.cron
+
+VOLUME ["/etc/piler"]
+VOLUME ["/var/piler/store"]
+VOLUME ["/var/piler/manticore"]
+
+EXPOSE 25 80 443
+
+COPY start.sh /start.sh
+
+CMD ["/start.sh"]

docker/README.md

@@ -0,0 +1,16 @@
+## How to run piler
+
+Edit the variables in docker-compose.yaml, then run
+
+```
+docker-compose up -d
+```
+
+## How to build the image for yourself
+
+Pick the latest deb package from Bitbucket download page (https://bitbucket.org/jsuto/piler/downloads/)
+and use it as the PACKAGE build argument, eg.
+
+```
+docker build --build-arg PACKAGE=piler_1.4.1-jammy-c860ca67_amd64.deb -t piler:1.4.1 .
+```

docker/build.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+IMAGE_NAME="sutoj/piler:1.4.4"
+
+if [[ $# -ne 1 ]]; then echo "ERROR: missing package name" 1>&2; exit 1; fi
+
+docker build --pull --build-arg PACKAGE="$1" -t "$IMAGE_NAME" .

docker/docker-compose.yaml

@@ -0,0 +1,59 @@
+version: "3"
+services:
+  mysql:
+    image: mariadb:11.1.2
+    container_name: mysql
+    restart: unless-stopped
+    cap_drop:
+      - ALL
+    cap_add:
+      - dac_override
+      - setuid
+      - setgid
+    environment:
+      - MYSQL_DATABASE=piler
+      - MYSQL_USER=piler
+      - MYSQL_PASSWORD=piler123
+      - MYSQL_RANDOM_ROOT_PASSWORD=yes
+    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
+    volumes:
+      - db_data:/var/lib/mysql
+  piler:
+    image: sutoj/piler:1.4.4
+    container_name: piler
+    init: true
+    environment:
+      - MYSQL_HOSTNAME=mysql
+      - MYSQL_DATABASE=piler
+      - MYSQL_USER=piler
+      - MYSQL_PASSWORD=piler123
+      - PILER_HOSTNAME=cust1.acts.hu
+      - RT=1
+    ports:
+      - "25:25"
+      - "80:80"
+    volumes:
+      - piler_etc:/etc/piler
+      - piler_manticore:/var/piler/manticore
+      - piler_store:/var/piler/store
+    healthcheck:
+      test: curl -s smtp://localhost/
+      interval: "60s"
+      timeout: "3s"
+      start_period: "15s"
+      retries: 3
+    deploy:
+      resources:
+        reservations:
+          memory: 512M
+        limits:
+          memory: 512M
+
+    depends_on:
+      - "mysql"
+
+volumes:
+  db_data: {}
+  piler_etc: {}
+  piler_manticore: {}
+  piler_store: {}

docker/start.sh

@@ -0,0 +1,216 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+CONFIG_DIR="/etc/piler"
+VOLUME_DIR="/var/piler"
+PILER_CONF="${CONFIG_DIR}/piler.conf"
+PILER_KEY="${CONFIG_DIR}/piler.key"
+PILER_PEM="${CONFIG_DIR}/piler.pem"
+PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf"
+SPHINX_CONF="${CONFIG_DIR}/manticore.conf"
+CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php"
+PILER_MY_CNF="${CONFIG_DIR}/.my.cnf"
+RT="${RT:-0}"
+
+error() {
+   echo "ERROR:" "$*" 1>&2
+   exit 1
+}
+
+
+log() {
+   echo "DEBUG:" "$*"
+}
+
+
+pre_flight_check() {
+   [[ -v PILER_HOSTNAME ]] || error "Missing PILER_HOSTNAME env variable"
+   [[ -v MYSQL_HOSTNAME ]] || error "Missing MYSQL_HOSTNAME env variable"
+   [[ -v MYSQL_DATABASE ]] || error "Missing MYSQL_DATABASE env variable"
+   [[ -v MYSQL_USER ]]     || error "Missing MYSQL_USER env variable"
+   [[ -v MYSQL_PASSWORD ]] || error "Missing MYSQL_PASSWORD env variable"
+}
+
+
+give_it_to_piler() {
+   local f="$1"
+
+   [[ -f "$f" ]] || error "${f} does not exist, aborting"
+
+   chown "${PILER_USER}:${PILER_USER}" "$f"
+   chmod 600 "$f"
+}
+
+
+make_certificate() {
+   local f="$1"
+   local crt="/tmp/1.cert"
+   local SSL_CERT_DATA="/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com"
+
+   log "Making an ssl certificate"
+
+   openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 -subj "$SSL_CERT_DATA" -keyout "$f" -out "$crt" -sha1 2>/dev/null
+   cat "$crt" >> "$f"
+   rm -f "$crt"
+
+   give_it_to_piler "$f"
+}
+
+
+make_piler_key() {
+   local f="$1"
+
+   log "Generating piler.key"
+
+   dd if=/dev/urandom bs=56 count=1 of="$f" 2>/dev/null
+   [[ $(stat -c '%s' "$f") -eq 56 ]] || error "could not read 56 bytes from /dev/urandom to ${f}"
+
+   give_it_to_piler "$f"
+}
+
+
+fix_configs() {
+   [[ -f "$PILER_KEY" ]] || make_piler_key "$PILER_KEY"
+   [[ -f "$PILER_PEM" ]] || make_certificate "$PILER_PEM"
+
+   if [[ ! -f "$PILER_NGINX_CONF" ]]; then
+      log "Writing ${PILER_NGINX_CONF}"
+
+      cp "${PILER_NGINX_CONF}.dist" "$PILER_NGINX_CONF"
+      sed -i "s%PILER_HOST%${PILER_HOSTNAME}%" "$PILER_NGINX_CONF"
+   fi
+
+   if [[ ! -f "$PILER_CONF" ]]; then
+      log "Writing ${PILER_CONF}"
+
+      sed \
+         -e "s/mysqluser=.*/mysqluser=${MYSQL_USER}/g" \
+         -e "s/mysqldb=.*/mysqldb=${MYSQL_DATABASE}/g" \
+         -e "s/verystrongpassword/${MYSQL_PASSWORD}/g" \
+         -e "s/hostid=.*/hostid=${PILER_HOSTNAME}/g" \
+         -e "s/tls_enable=.*/tls_enable=1/g" \
+         -e "s/mysqlsocket=.*/mysqlsocket=/g" "${PILER_CONF}.dist" > "$PILER_CONF"
+
+      {
+         echo "mysqlhost=${MYSQL_HOSTNAME}"
+      } >> "$PILER_CONF"
+
+      give_it_to_piler "$PILER_CONF"
+   fi
+
+   if [[ ! -f "$CONFIG_SITE_PHP" ]]; then
+      log "Writing ${CONFIG_SITE_PHP}"
+
+      cp "${CONFIG_DIR}/config-site.dist.php" "$CONFIG_SITE_PHP"
+
+      sed -i "s%HOSTNAME%${PILER_HOSTNAME}%" "$CONFIG_SITE_PHP"
+
+      {
+         echo "\$config['DECRYPT_BINARY'] = '/usr/bin/pilerget';"
+         echo "\$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/bin/pileraget';"
+         echo "\$config['PILER_BINARY'] = '/usr/sbin/piler';"
+         echo "\$config['DB_HOSTNAME'] = '$MYSQL_HOSTNAME';"
+         echo "\$config['DB_DATABASE'] = '$MYSQL_DATABASE';"
+         echo "\$config['DB_USERNAME'] = '$MYSQL_USER';"
+         echo "\$config['DB_PASSWORD'] = '$MYSQL_PASSWORD';"
+         echo "\$config['ENABLE_MEMCACHED'] = 1;"
+         echo "\$memcached_server = ['memcached', 11211];"
+      } >> "$CONFIG_SITE_PHP"
+   fi
+
+   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \
+       -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \
+       -e "s%MYSQL_USERNAME%${MYSQL_USER}%" \
+       -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" \
+       -i "$SPHINX_CONF"
+
+   # Fixes for RT index
+
+   if [[ $RT -eq 1 ]]; then
+      sed -i "s/define('RT', 0)/define('RT', 1)/" "$SPHINX_CONF"
+      if ! grep "'RT'" "$CONFIG_SITE_PHP"; then
+         echo "\$config['RT'] = 1;" >> "$CONFIG_SITE_PHP"
+      fi
+
+      if ! grep "'SPHINX_MAIN_INDEX'" "$CONFIG_SITE_PHP"; then
+         echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
+      fi
+
+      sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
+   fi
+}
+
+
+wait_until_mysql_server_is_ready() {
+   while true; do if mysql "--defaults-file=${PILER_MY_CNF}" <<< "show databases"; then break; fi; log "${MYSQL_HOSTNAME} is not ready"; sleep 5; done
+
+   log "${MYSQL_HOSTNAME} is ready"
+}
+
+
+init_database() {
+   local table
+   local has_metadata_table=0
+
+   wait_until_mysql_server_is_ready
+
+   while read -r table; do
+      if [[ "$table" == metadata ]]; then has_metadata_table=1; fi
+   done < <(mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< 'show tables')
+
+   if [[ $has_metadata_table -eq 0 ]]; then
+      log "no metadata table, creating tables"
+
+      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" < /usr/share/piler/db-mysql.sql
+   else
+      log "metadata table exists"
+   fi
+
+   if [[ -v ADMIN_USER_PASSWORD_HASH ]]; then
+      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< "update user set password='${ADMIN_USER_PASSWORD_HASH}' where uid=0"
+   fi
+}
+
+
+create_my_cnf_files() {
+   printf "[client]\nhost = %s\nuser = %s\npassword = %s\n[mysqldump]\nhost = %s\nuser = %s\npassword = %s\n" \
+      "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" \
+      > "$PILER_MY_CNF"
+
+   give_it_to_piler "$PILER_MY_CNF"
+}
+
+
+start_services() {
+   service cron start
+   service php8.1-fpm start
+   service nginx start
+   rsyslogd
+}
+
+
+start_piler() {
+   if [[ $RT -eq 0 && ! -f "${VOLUME_DIR}/manticore/main1.spp" ]]; then
+      log "main1.spp does not exist, creating index files"
+      su -c "indexer --all --config ${SPHINX_CONF}" "$PILER_USER"
+   fi
+
+   # No pid file should exist for piler
+   rm -f /var/run/piler/*pid
+
+   /etc/init.d/rc.searchd start
+   /etc/init.d/rc.piler start
+}
+
+
+pre_flight_check
+fix_configs
+create_my_cnf_files
+init_database
+start_services
+start_piler
+
+sleep infinity

etc/Makefile.in

@@ -31,8 +31,10 @@ all:
 
 install:
 	$(INSTALL) -m 0640 -g $(RUNNING_GROUP) $(srcdir)/piler.conf $(DESTDIR)$(sysconfdir)/piler/piler.conf.dist
-	sed -e 's%LOCALSTATEDIR%$(localstatedir)%g' $(srcdir)/sphinx.conf.in > sphinx.conf.dist
-	$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/sphinx.conf.dist $(DESTDIR)$(sysconfdir)/piler/sphinx.conf.dist
+	sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/sphinx.conf.in > sphinx.conf.dist
+	sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/manticore.conf.in > manticore.conf.dist
+	$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/sphinx.conf.dist $(DESTDIR)$(sysconfdir)/piler/sphinx.conf.dist
+	$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/manticore.conf.dist $(DESTDIR)$(sysconfdir)/piler/manticore.conf.dist
 	$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/config-site.dist.php $(DESTDIR)$(sysconfdir)/piler/config-site.dist.php
 	$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/cron.jobs.in $(DESTDIR)$(datarootdir)/piler/piler.cron
 	sed -i -e 's%LOCALSTATEDIR%$(localstatedir)%g' $(DESTDIR)$(datarootdir)/piler/piler.cron

etc/config-site.dist.php

@@ -1,11 +1,13 @@
 <?php
 
-$config['SITE_NAME'] = 'HOSTNAME';
-$config['SITE_URL'] = 'http://' . $config['SITE_NAME'] . '/';
+define('SITE_NAME_CONST', 'SITE_NAME');
 
-$config['SMTP_DOMAIN'] = $config['SITE_NAME'];
-$config['SMTP_FROMADDR'] = 'no-reply@' . $config['SITE_NAME'];
-$config['ADMIN_EMAIL'] = 'admin@' . $config['SITE_NAME'];
+$config[SITE_NAME_CONST] = 'HOSTNAME';
+$config['SITE_URL'] = 'http://' . $config[SITE_NAME_CONST] . '/';
+
+$config['SMTP_DOMAIN'] = $config[SITE_NAME_CONST];
+$config['SMTP_FROMADDR'] = 'no-reply@' . $config[SITE_NAME_CONST];
+$config['ADMIN_EMAIL'] = 'admin@' . $config[SITE_NAME_CONST];
 
 $config['DB_PASSWORD'] = 'MYSQL_PASSWORD';
 

etc/cron.jobs.in

@@ -1,20 +1,14 @@
 ### PILERSTART
 5,35 * * * * LIBEXECDIR/piler/indexer.delta.sh
 30   2 * * * LIBEXECDIR/piler/indexer.main.sh
-15,45 * * * * LIBEXECDIR/piler/indexer.attachment.sh
-*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet tag1 --rotate
-*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet note1 --rotate
+40 3 * * * LIBEXECDIR/piler/purge.sh
+*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet tag1 --rotate
+*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet note1 --rotate
 */5 * * * *  /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \;
 */5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error
 
-### optional: the same report you can see on the health page
-###30 7 * * * /usr/bin/php LIBEXECDIR/piler/daily-report.php --webui LOCALSTATEDIR/piler/www
-
-### optional: populate accouting data
+### optional
 ###30 6 * * * /usr/bin/php LIBEXECDIR/piler/generate_stats.php --webui LOCALSTATEDIR/piler/www >/dev/null
-
-### optional: purge aged emails
-###2 0 * * * BINDIR/pilerpurge
-
+###*/5 * * * * LIBEXECDIR/piler/import.sh
 
 ### PILEREND

etc/example.conf

@@ -22,10 +22,11 @@ username=piler
 ; The default is 7 years + 2 days (=7*365+2=2557 days)
 default_retention_days=2557
 
-; this is a 16 character long vector
-; after the installation you must not change it ever
-; otherwise you can't access your emails
-;iv=****************
+; The initialization vector for encryption.
+; By now it has become obsolete. Don't use it for
+; new installations. However, if you used it before
+; then you must keep it as it is.
+;iv=
 
 ; whether to encrypt messages (1) or not (0).
 ; Make sure to set this value to your needs right after installing piler,
@@ -45,12 +46,12 @@ number_of_worker_processes=2
 max_connections=64
 
 ; number of processed emails per each piler process
-max_requests_per_child=1000
+max_requests_per_child=10000
 
 ; SMTP HELO identification string
 ; this should be the FQDN part of the email address
-; where you copy emails, eg. archive@piler.yourdomain.com -> piler.yourdomain.com
-hostid=piler.yourdomain.com
+; where you copy emails, eg. archive@piler.example.com -> piler.example.com
+hostid=piler.example.com
 
 ; whether to process rcpt to addresses and add them to rcpt table (1) or not (0)
 process_rcpt_to_addresses=0
@@ -66,6 +67,9 @@ listen_port=25
 
 clamd_socket=/tmp/clamd
 
+; check for client timeout interval. Default: 20 sec
+check_for_client_timeout_interval=20
+
 ; smtp timeout. Default: 60 sec
 smtp_timeout=60
 
@@ -103,6 +107,13 @@ pemfile=
 cipher_list=ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS
 
 
+; set the minimum TLS protocol version for piler-smtp daemon
+;
+; Valid values: TLSv1, TLSv1.1, TLSv1.2, TLSv1.3
+; TLSv1 and TLSv1.1 are not recommended for security reasons
+tls_min_version=TLSv1.2
+
+
 ; piler's own header to indicate previously archived messages
 piler_header_field=X-piler-id:
 
@@ -186,7 +197,7 @@ update_counters_to_memcached=0
 ; hint: if you are using a mysql replicated environment and you do _not_ want
 ; piler to write to the replicated database (because you do sync it some other
 ; way to the master database or you are not interested in keeping the counters
-; persistantly at all), then specify a big number here, that fits to the 
+; persistantly at all), then specify a big number here, that fits to the
 ; "long int" size, eg. 2147483647
 memcached_to_db_interval=900
 
@@ -208,6 +219,14 @@ mysqldb=piler
 mysql_connect_timeout=2
 
 
+;
+; manticore stuff
+;
+sphxhost=127.0.0.1
+sphxport=9306
+; whether to enable (1) or not (0) the rt index. If so, then piler
+; writes the index data directly to manticore's sql API
+rtindex=0
 
 ; if you want to change the 'sent' time as you archive the message
 ; set this in seconds. This can be a postive or negative value.
@@ -218,3 +237,38 @@ tweak_sent_time_offset=0
 ; whether to enable (1) or not (0) the extra mmap dedup test feature
 ; if you change it, be sure to stop, then start piler
 mmap_dedup_test=0
+
+; security header that must be present in the first mail header of
+; the message. If the security_header value is not empty, then the
+; parser checks for this header line. Unless it's found it will discard
+; the given email. Note that this feature is supposed to be a security
+; mechanism against unwanted email in the archive if limiting smtp
+; clients via an IP-address list is not feasible.
+security_header=
+
+; By default the archive accepts any envelope recipient addresses.
+; If your archive's port 25 is wide open to the Internet (which it
+; shouldn't be, then spammers may find it, and fill it with spam.
+;
+; By setting this variable you may restrict the envelope address
+; to a single email address, eg. some-random-address-12345@archive.example.com
+; Then the archive will reject any other envelope recipients
+archive_address=
+
+; whether to enable (1) or not (0) an smtp access list similar to
+; postfix's postscreen. Valid actions in the acl file are "permit"
+; and "reject" (without quotes). See smtp.acl.example for more.
+;
+; Important! There's an implicit default deny at the end of the
+; rules. In other words if you decide to use the acl file, then
+; everyone is not explicitly permitted is denied.
+smtp_access_list=0
+
+; max message size in bytes
+; piler-smtp will reject any message that's bigger than this number
+max_message_size=50000000
+
+; max memory in bytes piler-smtp uses for buffering messages
+; when this limit is exceeded, no new emails will be accepted
+; until the used memory decreases below this level
+max_smtp_memory=500000000

etc/manticore.conf.in

@@ -0,0 +1,268 @@
+#!/usr/bin/php
+<?php
+
+define('LOCALSTATEDIR', '/var');
+define('NGRAM_CONFIG', "        #ngram_len               = 1\n        #ngram_chars             = U+3000..U+2FA1F\n");
+
+# See http://sphinxsearch.com/wiki/doku.php?id=charset_tables for more on the charset_table settings
+# The following settings contains English and some Latin extras
+define('CHARSET_TABLE', "0..9, english, _, \
+                                  U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
+                                  U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
+                                  U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
+                                  U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171\n");
+
+define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
+
+define('RT', 0);
+
+?>
+
+#
+# minimal manticore configuration suited to piler
+#
+
+<?php if(RT == 0) { ?>
+
+source base
+{
+   type = mysql
+   sql_host = MYSQL_HOSTNAME
+   sql_db = MYSQL_DATABASE
+   sql_user = MYSQL_USERNAME
+   sql_pass = MYSQL_PASSWORD
+
+   sql_attr_uint = size
+   sql_attr_uint = sent
+   sql_attr_uint = attachments
+}
+
+source delta : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query_pre  = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index
+   sql_query_post_index  = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
+
+   sql_query_killlist = SELECT `id` FROM `metadata` WHERE `deleted`=1
+}
+
+source main1 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source main2 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source main3 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source main4 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source dailydelta : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+index main1
+{
+        source                  = main1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index main2
+{
+        source                  = main2
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main2
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index main3
+{
+        source                  = main3
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main3
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index main4
+{
+        source                  = main4
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main4
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index dailydelta1
+{
+        source                  = dailydelta
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/dailydelta1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+        killlist_target         = main1:kl, main2:kl, main3:kl, main4:kl, dailydelta1:kl
+<?php print NGRAM_CONFIG; ?>
+}
+
+index delta1
+{
+        source                  = delta
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/delta1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+source tag : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query  = SELECT `_id`, `id` AS iid, `uid`, `tag` FROM `tag`
+
+   sql_attr_uint = iid
+   sql_attr_uint = uid
+}
+
+source note : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query  = SELECT `_id`, `id` AS iid, `uid`, `note` FROM `note`
+
+   sql_attr_uint = iid
+   sql_attr_uint = uid
+}
+
+index tag1
+{
+        source                  = tag
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/tag1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+        stored_fields           =
+<?php print NGRAM_CONFIG; ?>
+}
+
+
+index note1
+{
+        source                  = note
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/note1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+        stored_fields           =
+<?php print NGRAM_CONFIG; ?>
+}
+
+<?php } else { ?>
+
+index piler1
+{
+    type = rt
+    path = /var/piler/manticore/piler1
+    rt_mem_limit = 512M
+    stored_fields =
+    min_word_len = 1
+    min_prefix_len = 5
+    charset_table  = <?php print CHARSET_TABLE; ?>
+    # See https://manual.manticoresearch.com/Creating_an_index/Data_types#Row-wise-and-columnar-attribute-storages
+    # if you want to enable columnar storage
+    # columnar_attrs = *
+    rt_field = sender
+    rt_field = rcpt
+    rt_field = senderdomain
+    rt_field = rcptdomain
+    rt_field = subject
+    rt_field = body
+    rt_field = attachment_types
+    rt_attr_bigint = arrived
+    rt_attr_bigint = sent
+    rt_attr_uint = size
+    rt_attr_uint = direction
+    rt_attr_uint = folder
+    rt_attr_uint = attachments
+}
+
+index tag1
+{
+    type = rt
+    path = /var/piler/manticore/tag1
+    rt_mem_limit = 16M
+    stored_fields = tag
+    min_word_len = 2
+    min_prefix_len = 5
+    charset_table  = <?php print CHARSET_TABLE; ?>
+    rt_field = tag
+    rt_attr_bigint = mid
+    rt_attr_uint = uid
+}
+
+index note1
+{
+    type = rt
+    path = /var/piler/manticore/note1
+    rt_mem_limit = 16M
+    stored_fields = note
+    min_word_len = 2
+    min_prefix_len = 5
+    charset_table  = <?php print CHARSET_TABLE; ?>
+    rt_field = note
+    rt_attr_bigint = mid
+    rt_attr_uint = uid
+}
+
+<?php } ?>
+
+
+searchd
+{
+        listen                  = 127.0.0.1:9312
+        listen                  = 127.0.0.1:9306:mysql
+        listen                  = 127.0.0.1:9307:mysql_readonly
+        log                     = /var/piler/manticore/manticore.log
+        binlog_max_log_size     = 256M
+        binlog_path             = /var/piler/manticore
+        binlog_flush            = 2
+        query_log               = /var/piler/manticore/query.log
+        network_timeout         = 5
+        pid_file                = /var/run/piler/searchd.pid
+        seamless_rotate         = 1
+        preopen_tables          = 1
+        unlink_old              = 1
+        thread_stack            = 512k
+<?php if(RT) { ?>
+        rt_flush_period         = 300
+<?php } ?>
+
+}

etc/smtp.acl.example

@@ -0,0 +1,9 @@
+# Allow office365 servers. See the below URI for more
+# https://docs.microsoft.com/en-us/microsoft-365/enterprise/urls-and-ip-address-ranges?view=o365-worldwide
+#
+# Anything is not listed below will be rejected
+#
+40.92.0.0/15 permit
+40.107.0.0/16 permit
+52.100.0.0/14 permit
+104.47.0.0/17 permit

etc/sphinx.conf.in

@@ -1,3 +1,34 @@
+#!/usr/bin/php
+<?php
+
+define('SPHINX_VERSION', 331); // If you have sphinx-3.3.1, then set SPHINX_VERSION to 331
+define('LOCALSTATEDIR', '@LOCALSTATEDIR@');
+define('NGRAM_CONFIG', "        #ngram_len               = 1\n        #ngram_chars             = U+3000..U+2FA1F\n");
+
+# See http://sphinxsearch.com/wiki/doku.php?id=charset_tables for more on the charset_table settings
+# The following settings contains English and some Latin extras
+define('SPHINX_CHARSET_TABLE', "0..9, english, _, \
+                                  U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
+                                  U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
+                                  U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
+                                  U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171\n");
+
+// Sphinx 3.3.1 introduced some strict rules for fulltext search column names
+// In order to comply with it you must enable SPHINX_STRICT_SCHEMA variable
+// Be sure to check out http://www.mailpiler.org/wiki/current:sphinx3 for more
+// NB: The SPHINX_STRICT_SCHEMA in sphinx.conf MUST BE THE SAME as in config.php (or in config-site.php)
+//
+define('SPHINX_STRICT_SCHEMA', 1);
+define('RT', 0);
+
+if(SPHINX_STRICT_SCHEMA) {
+   define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
+} else {
+   define('SELECT_FIELDS', 'id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
+}
+
+?>
+
 #
 # minimal sphinx configuration suited to piler
 #
@@ -15,46 +46,52 @@ source base
    sql_attr_uint = attachments
 }
 
+<?php if(!RT) { ?>
+
 source delta : base
 {
    sql_query_pre = SET NAMES utf8mb4
    sql_query_pre  = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index
    sql_query_post_index  = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index \
-        WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
 
+<?php if(SPHINX_VERSION < 300) { ?>
    sql_query_killlist = SELECT `id` FROM `metadata` WHERE `deleted`=1
+<?php } else { ?>
+   sql_query_kbatch = SELECT `id` FROM `metadata` WHERE `deleted`=1
+<?php } ?>
 }
 
 source main1 : base
 {
    sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
 }
 
 source main2 : base
 {
    sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
 }
 
 source main3 : base
 {
    sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
 }
 
 source main4 : base
 {
    sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
 }
 
 source dailydelta : base
 {
    sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
 }
+<?php } ?>
 
 source tag : base
 {
@@ -76,148 +113,171 @@ source note : base
 
 }
 
-source att : base
-{
-
-   sql_query_pre = SET NAMES utf8mb4
-   sql_query  =  select a.id as aid, m.id as mid, a.name AS aname, a.size, REPLACE(REPLACE(m.`from`, '@', 'X'), '.', 'X') as `from`, REPLACE(REPLACE((select group_concat(rcpt.`to` ORDER BY `to` ASC SEPARATOR ' ') from rcpt where rcpt.id=mid group by rcpt.id), '@', 'X'), '.', 'X') as `to` from attachment a, metadata m where m.piler_id=a.piler_id
-
-   sql_attr_uint = size
-   sql_attr_uint = mid
-}
-
-
+<?php if(!RT) { ?>
 index main1
 {
-	source			= main1
-	path			= LOCALSTATEDIR/piler/sphinx/main1
-	docinfo			= extern
-	dict			= keywords
+        source                  = main1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main1
+<?php if(SPHINX_VERSION < 300) { ?>
+        docinfo                 = extern
+        dict                    = keywords
+<?php } ?>
         min_prefix_len          = 5
         min_word_len            = 1
-        #ngram_len               = 1
-        #ngram_chars             = U+3000..U+2FA1F
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
 }
 
 index main2
 {
         source                  = main2
-        path                    = LOCALSTATEDIR/piler/sphinx/main2
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main2
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-	dict			= keywords
+        dict                    = keywords
+<?php } ?>
         min_prefix_len          = 5
         min_word_len            = 1
-        #ngram_len               = 1
-        #ngram_chars             = U+3000..U+2FA1F
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
 }
 
 index main3
 {
         source                  = main3
-        path                    = LOCALSTATEDIR/piler/sphinx/main3
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main3
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-	dict			= keywords
+        dict                    = keywords
+<?php } ?>
         min_prefix_len          = 5
         min_word_len            = 1
-        #ngram_len               = 1
-        #ngram_chars             = U+3000..U+2FA1F
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
 }
 
 index main4
 {
         source                  = main4
-        path                    = LOCALSTATEDIR/piler/sphinx/main4
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main4
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-	dict			= keywords
+        dict                    = keywords
+<?php } ?>
         min_prefix_len          = 5
         min_word_len            = 1
-        #ngram_len               = 1
-        #ngram_chars             = U+3000..U+2FA1F
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
 }
 
 index dailydelta1
 {
         source                  = dailydelta
-        path                    = LOCALSTATEDIR/piler/sphinx/dailydelta1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/dailydelta1
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-	dict			= keywords
+        dict                    = keywords
+<?php } ?>
         min_prefix_len          = 5
         min_word_len            = 1
-        #ngram_len               = 1
-        #ngram_chars             = U+3000..U+2FA1F
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
 }
 
 index delta1
 {
         source                  = delta
-        path                    = LOCALSTATEDIR/piler/sphinx/delta1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/delta1
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-	dict			= keywords
+        dict                    = keywords
+<?php } ?>
         min_prefix_len          = 5
         min_word_len            = 1
-        #ngram_len               = 1
-        #ngram_chars             = U+3000..U+2FA1F
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+<?php if(SPHINX_VERSION >= 310) { ?>
+        kbatch = main1, main2, main3, main4, dailydelta1
+<?php } ?>
 }
 
+<?php } ?>
 
 index tag1
 {
         source                  = tag
-        path                    = LOCALSTATEDIR/piler/sphinx/tag1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/tag1
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-	dict			= keywords
+        dict                    = keywords
+<?php } ?>
         min_prefix_len          = 5
         min_word_len            = 1
-        #ngram_len               = 1
-        #ngram_chars             = U+3000..U+2FA1F
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
 }
 
 
 index note1
 {
         source                  = note
-        path                    = LOCALSTATEDIR/piler/sphinx/note1
-        docinfo                 = extern
-	dict			= keywords
-        min_prefix_len          = 5
-        min_word_len            = 1
-        #ngram_len               = 1
-        #ngram_chars             = U+3000..U+2FA1F
-}
-
-
-index att1
-{
-        source                  = att
-        path                    = /var/piler/sphinx/att1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/note1
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
         dict                    = keywords
-        min_prefix_len          = 6
+<?php } ?>
+        min_prefix_len          = 5
         min_word_len            = 1
-        ngram_len               = 1
-        ngram_chars             = U+1100..U+2FA1F
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
 }
 
+<?php if(RT) { ?>
+index piler1
+{
+        type = rt
+        path = /var/piler/sphinx/piler1
+        rt_mem_limit = 128M
+        rt_field = sender
+        rt_field = rcpt
+        rt_field = senderdomain
+        rt_field = rcptdomain
+        rt_field = subject
+        rt_field = body
+        rt_field = attachment_types
+        rt_attr_bigint = arrived
+        rt_attr_bigint = sent
+        rt_attr_uint = size
+        rt_attr_uint = direction
+        rt_attr_uint = folder
+        rt_attr_uint = attachments
+}
+<?php } ?>
 
 indexer
 {
-	mem_limit		= 256M
+        mem_limit               = 256M
 }
 
 
 searchd
 {
-	listen			= 127.0.0.1:9312
-	listen			= 127.0.0.1:9306:mysql41
-	log			= /dev/null
-	binlog_path             = 
-	##query_log		=
-	read_timeout		= 5
-	max_children		= 30
-	pid_file		= /var/run/piler/searchd.pid
-	seamless_rotate		= 1
-	preopen_indexes		= 1
-	unlink_old		= 1
-	thread_stack		= 512k
-	workers			= threads # for RT to work
+        listen                  = 127.0.0.1:9312
+        listen                  = 127.0.0.1:9306:mysql41
+        log                     = /dev/null
+        binlog_path             = /var/piler/sphinx
+        binlog_max_log_size     = 16M
+        binlog_flush            = 2
+<?php if(RT) { ?>
+        rt_flush_period         = 300
+<?php } ?>
+        ##query_log             =
+        read_timeout            = 5
+        max_children            = 30
+        pid_file                = /var/run/piler/searchd.pid
+        seamless_rotate         = 1
+        preopen_indexes         = 1
+        unlink_old              = 1
+        thread_stack            = 512k
+        workers                 = thread_pool
 }

init.d/rc.piler.in

@@ -15,27 +15,30 @@
 
 NAME=piler
 OPTIONS=""
-PID_FILE=`SBINDIR/pilerconf $OPTIONS -q pidfile | cut -f2 -d=`
-PID_NUMBER=`test -f ${PID_FILE} && cat ${PID_FILE}`
-PILER_SMTP_PID=$(ps uaxw | grep -w piler-smtp | grep -v grep | awk '{print $2}')
+PID_FILE="$(SBINDIR/pilerconf $OPTIONS -q pidfile | cut -f2 -d=)"
+PID_NUMBER="$(test -f "$PID_FILE" && cat "$PID_FILE")"
+PILER_SMTP_PID="$(pgrep piler-smtp)"
 
 start() {
-	echo "starting piler-smtp . . . "
-	SBINDIR/piler-smtp -d
+   echo "starting piler-smtp . . . "
+   SBINDIR/piler-smtp -d
 
-	echo "starting $NAME . . ."
-	SBINDIR/piler -d $OPTIONS
+   echo "starting $NAME . . ."
+   SBINDIR/piler -d $OPTIONS
 }
 
 stop() {
-	echo "stopping $NAME"
-	kill ${PID_NUMBER}
+   if [ "$PID_NUMBER" != "" ]; then echo "stopping piler"; kill "$PID_NUMBER"; fi
 
-	if [ $PILER_SMTP_PID != '' ]; then echo "stopping piler-smtp"; kill $PILER_SMTP_PID; fi
+   if [ "$PILER_SMTP_PID" != "" ]; then echo "stopping piler-smtp"; kill "$PILER_SMTP_PID"; fi
 }
 
 check_status(){
-        test -f /proc/${PID_NUMBER}/status
+        if [ -f "/proc/${PID_NUMBER}/status" ]; then
+           echo "piler is running, pid: ${PID_NUMBER}";
+        else
+           echo "piler is NOT running";
+        fi
 
         if [ "${PILER_SMTP_PID}" != '' ]; then
            echo "piler-smtp is running, pid: ${PILER_SMTP_PID}";
@@ -45,37 +48,34 @@ check_status(){
 }
 
 case "$1" in
-	start)
-		start;
-   	;;
+        start)
+                start;
+        ;;
 
-	stop)
-		stop;
-   	;;
+        stop)
+                stop;
+        ;;
 
         status)
-                if check_status;
-                 then
-                    echo "${NAME} is running."
-                    exit 0
-                 else
-                    echo "${NAME} is not running."
-                    exit 1
+                if check_status; then
+                   exit 0
+                else
+                   exit 1
                 fi
         ;;
 
-	restart)
-		stop;
-		sleep 1;
-		start;
-   	;;
+        restart)
+                stop;
+                sleep 1;
+                start;
+        ;;
 
-	reload)
-		kill -HUP $PID_NUMBER
-		echo "reloaded"
-	;;
+        reload)
+                kill -HUP "$PID_NUMBER"
+                echo "reloaded"
+        ;;
 
-	*)
-		echo "Usage: $0 start|stop|restart|reload|status"
+        *)
+                echo "Usage: $0 start|stop|restart|reload|status"
+        ;;
 esac
-

init.d/rc.searchd.in

@@ -16,7 +16,13 @@ export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
 
 NAME=searchd
 PID_FILE=/var/run/piler/searchd.pid
-PID_NUMBER=`test -f ${PID_FILE} && cat ${PID_FILE}`
+PID_NUMBER=$(test -f ${PID_FILE} && cat ${PID_FILE})
+CONFIG_FILE=sphinx.conf
+
+if [ -f SYSCONFDIR/piler/MANTICORE ]; then
+   CONFIG_FILE=manticore.conf
+fi
+
 
 start() {
 	echo "starting searchd . . ."
@@ -26,8 +32,8 @@ start() {
 		chown piler:piler /var/run/piler
 	fi
 
-        if [ `id -u` -eq 0 ]; then
-		su piler -c "searchd --config SYSCONFDIR/piler/sphinx.conf"
+        if [ $(id -u) -eq 0 ]; then
+		su piler -c "searchd --config SYSCONFDIR/piler/${CONFIG_FILE}"
 	else
 		searchd
 	fi
@@ -35,7 +41,7 @@ start() {
 
 stop() {
 	echo "stopping searchd"
-	kill ${PID_NUMBER}
+	kill "$PID_NUMBER"
 }
 
 check_status(){

piler-config.h.in

@@ -2,6 +2,8 @@
  * piler-config.h.in, SJ
  */
 
+#define VERSION "x.x.x"
+
 #define CONFDIR "/usr/local/etc"
 #define DATADIR "/usr/local/var"
 #define DATAROOTDIR "/usr/local/share"
@@ -24,9 +26,6 @@
 #undef HAVE_TNEF
 #undef HAVE_ZIP
 
-#undef HAVE_LIBWRAP
-
 #undef HAVE_TWEAK_SENT_TIME
 
 #undef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
-

src/Makefile.in

@@ -33,7 +33,7 @@ MAKE = `which make`
 
 INSTALL = @INSTALL@
 
-all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport reindex test piler-smtp
+all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport reindex test stats piler-smtp
 install: install-piler
 
 
@@ -43,12 +43,12 @@ piler: piler.c libpiler.a
 libpiler.a: $(OBJS) $(SQL_OBJS)
 	ar cr libpiler.a $(OBJS) $(SQL_OBJS)
 	ranlib libpiler.a
-	$(CC) -shared -o libpiler.so.$(LIBPILER_VERSION) $(OBJS) $(SQL_OBJS)
+	$(CC) -shared -Wl,-soname,libpiler.so.$(LIBPILER_VERSION) -o libpiler.so.$(LIBPILER_VERSION) $(OBJS) $(SQL_OBJS)
 	ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so
 	ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so.$(PILER_VERSION)
 
 piler-smtp: piler-smtp.c libpiler.a
-	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< cfg.o misc.o tai.o smtp.o session.o dirs.o sig.o bdat.o $(LIBS) $(LIBDIR)
+	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< cfg.o misc.o tai.o smtp.o session.o dirs.o sig.o bdat.o screen.o $(LIBS) $(LIBDIR)
 
 pilerget: pilerget.c libpiler.a
 	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
@@ -71,8 +71,11 @@ reindex: reindex.c libpiler.a
 test: test.c libpiler.a
 	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o pilertest $< -lpiler $(LIBS) $(LIBDIR) @LDFLAGS@
 
+stats: stats.c libpiler.a
+	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o pilerstats $< -lpiler $(LIBS) $(LIBDIR) @LDFLAGS@
+
 %.o: $(srcdir)/%.c
-	$(CC) $(CFLAGS) -fPIC $(INCDIR) $(DEFS) -c $< -o $@
+	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -c $< -o $@
 
 
 install-piler:
@@ -90,9 +93,10 @@ install-piler:
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerexport $(DESTDIR)$(bindir)
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) reindex $(DESTDIR)$(bindir)
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilertest $(DESTDIR)$(bindir)
+	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerstats $(DESTDIR)$(bindir)
 
 clean:
-	rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilertest reindex piler-smtp
+	rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilertest pilerstats reindex piler-smtp
 
 distclean: clean
 	rm -f Makefile

src/archive.c

@@ -16,6 +16,9 @@
 #include <syslog.h>
 #include <openssl/blowfish.h>
 #include <openssl/evp.h>
+#if OPENSSL_VERSION_MAJOR >= 3
+   #include <openssl/provider.h>
+#endif
 #include <zlib.h>
 #include <assert.h>
 #include <piler.h>
@@ -54,11 +57,14 @@ void zerr(int ret){
 
 int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
    int ret, pos=0;
-   unsigned have;
    z_stream strm;
    char *new_ptr;
    unsigned char out[REALLYBIGBUFSIZE];
 
+   /* expecting deflate with 32k window size (0x78) */
+   if(len > 0 && in[0] != 0x78)
+      return Z_DATA_ERROR;
+
    /* allocate inflate state */
 
    strm.zalloc = Z_NULL;
@@ -96,7 +102,7 @@ int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
                 return ret;
       }
 
-      have = REALLYBIGBUFSIZE - strm.avail_out;
+      unsigned have = REALLYBIGBUFSIZE - strm.avail_out;
 
       /*
        * write the uncompressed result either to stdout
@@ -138,8 +144,9 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
 #if OPENSSL_VERSION_NUMBER < 0x10100000L
    EVP_CIPHER_CTX ctx;
 #else
-   EVP_CIPHER_CTX *ctx;
+   EVP_CIPHER_CTX *ctx=NULL;
 #endif
+   int blocklen;
 
 
    if(filename == NULL) return 1;
@@ -158,20 +165,47 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
       return 1;
    }
 
+   // The new encryption scheme uses piler id starting with 5000....
 
    if(cfg->encrypt_messages == 1){
    #if OPENSSL_VERSION_NUMBER < 0x10100000L
       EVP_CIPHER_CTX_init(&ctx);
-      EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      if(strstr(filename, "/5000")){
+         rc = EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+      } else {
+         rc = EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      }
+
+      if(!rc){
+         syslog(LOG_PRIORITY, "ERROR: EVP_DecryptInit_ex()");
+         goto CLEANUP;
+      }
+
+      blocklen = EVP_CIPHER_CTX_block_size(&ctx);
    #else
       ctx = EVP_CIPHER_CTX_new();
       if(!ctx) goto CLEANUP;
 
       EVP_CIPHER_CTX_init(ctx);
-      EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      if(strstr(filename, "/5000")){
+         rc = EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+      } else {
+      #if OPENSSL_VERSION_MAJOR >= 3
+         OSSL_PROVIDER_load(NULL, "legacy");
+         OSSL_PROVIDER_load(NULL, "default");
+      #endif
+         rc = EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      }
+
+      if(!rc){
+         syslog(LOG_PRIORITY, "ERROR: EVP_DecryptInit_ex()");
+         goto CLEANUP;
+      }
+
+      blocklen = EVP_CIPHER_CTX_block_size(ctx);
    #endif
 
-      len = st.st_size+EVP_MAX_BLOCK_LENGTH;
+      len = st.st_size+blocklen;
 
       s = malloc(len);
 
@@ -208,7 +242,13 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
 
 
       tlen += olen;
+
+      // old fileformat with static IV
       rc = inf(s, tlen, mode, buffer, dest);
+      // new fileformat, starting with blocklen bytes of garbage
+      if(rc != Z_OK && tlen >= blocklen){
+         rc = inf(s+blocklen, tlen-blocklen, mode, buffer, dest);
+      }
    }
    else {
       addr = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
@@ -221,22 +261,22 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
 
 
 CLEANUP:
-   if(fd != -1) close(fd);
+   if(fd != -1) close(fd); //-V547
    if(s) free(s);
    if(cfg->encrypt_messages == 1)
    #if OPENSSL_VERSION_NUMBER < 0x10100000L
       EVP_CIPHER_CTX_cleanup(&ctx);
    #else
-      EVP_CIPHER_CTX_free(ctx);
+      if(ctx) EVP_CIPHER_CTX_free(ctx);
    #endif
 
    return 0;
 }
 
 
-int retrieve_email_from_archive(struct session_data *sdata, struct data *data, FILE *dest, struct config *cfg){
-   int i, attachments;
-   char *buffer=NULL, *saved_buffer, *p, filename[SMALLBUFSIZE], pointer[SMALLBUFSIZE];
+int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct config *cfg){
+   int attachments;
+   char *buffer=NULL, *saved_buffer, *p, filename[SMALLBUFSIZE];
    struct ptr_array ptr_arr[MAX_ATTACHMENTS];
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
    struct stat st;
@@ -247,14 +287,14 @@ int retrieve_email_from_archive(struct session_data *sdata, struct data *data, F
       return 1;
    }
 
-   attachments = query_attachments(sdata, data, &ptr_arr[0]);
+   attachments = query_attachments(sdata, &ptr_arr[0]);
 
    if(attachments == -1){
       printf("problem querying the attachment of %s\n", sdata->ttmpfile);
       return 1;
    }
 
-   snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
+   snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, *(sdata->ttmpfile+24), *(sdata->ttmpfile+25), *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
    if(stat(filename, &st)){
       snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+RND_STR_LEN-6), *(sdata->ttmpfile+RND_STR_LEN-5), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
@@ -270,7 +310,8 @@ int retrieve_email_from_archive(struct session_data *sdata, struct data *data, F
       if(buffer){
          saved_buffer = buffer;
 
-         for(i=1; i<=attachments; i++){
+         for(int i=1; i<=attachments; i++){
+            char pointer[SMALLBUFSIZE];
             snprintf(pointer, sizeof(pointer)-1, "ATTACHMENT_POINTER_%s.a%d_XXX_PILER", sdata->ttmpfile, i);
 
             p = strstr(buffer, pointer);
@@ -280,7 +321,7 @@ int retrieve_email_from_archive(struct session_data *sdata, struct data *data, F
                buffer = p + strlen(pointer);
 
                if(strlen(ptr_arr[i].piler_id) == RND_STR_LEN){
-                  snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);
+                  snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, ptr_arr[i].piler_id[24], ptr_arr[i].piler_id[25], ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);
 
                #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
                   if(stat(filename, &st)){
@@ -305,5 +346,3 @@ int retrieve_email_from_archive(struct session_data *sdata, struct data *data, F
 
    return 0;
 }
-
-

src/attachment.c

@@ -16,8 +16,7 @@
 #include <piler.h>
 
 
-int store_attachments(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
-   uint64 id=0;
+int store_attachments(struct session_data *sdata, struct parser_state *state, struct config *cfg){
    int i, rc=1, found, affected_rows;
    struct sql sql, sql2;
 
@@ -27,7 +26,7 @@ int store_attachments(struct session_data *sdata, struct parser_state *state, st
 
    for(i=1; i<=state->n_attachments; i++){
       found = 0;
-      id = 0;
+      uint64 id = 0;
 
       if(state->attachments[i].size > 0){
 
@@ -91,7 +90,7 @@ CLOSE:
 }
 
 
-int query_attachment_pointers(struct session_data *sdata, struct data *data, uint64 ptr, char *piler_id, int *id){
+int query_attachment_pointers(struct session_data *sdata, uint64 ptr, char *piler_id, int *id){
    int rc=0;
    struct sql sql;
 
@@ -120,7 +119,7 @@ int query_attachment_pointers(struct session_data *sdata, struct data *data, uin
 }
 
 
-int query_attachments(struct session_data *sdata, struct data *data, struct ptr_array *ptr_arr){
+int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr){
    int i, rc, id, attachments=0;
    uint64 ptr;
    struct sql sql;
@@ -147,7 +146,7 @@ int query_attachments(struct session_data *sdata, struct data *data, struct ptr_
       if(id > 0 && id < MAX_ATTACHMENTS){
          if(ptr > 0){
             ptr_arr[id].ptr = ptr;
-            rc = query_attachment_pointers(sdata, data, ptr, &(ptr_arr[id].piler_id[0]), &(ptr_arr[id].attachment_id));
+            rc = query_attachment_pointers(sdata, ptr, &(ptr_arr[id].piler_id[0]), &(ptr_arr[id].attachment_id));
             if(!rc){
                attachments = -1;
                goto CLOSE;
@@ -169,5 +168,3 @@ CLOSE:
 
    return attachments;
 }
-
-

src/base64.c

@@ -1,79 +0,0 @@
-/*
- * base64.c, SJ
- */
-
-#include <stdio.h>
-#include <string.h>
-
-
-char base64_value(char c){
-   static const char *base64_table = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-
-   if((int)c > 63) return '=';
-
-   return base64_table[(int)c];
-}
-
-
-void base64_encode_block(unsigned char *in, int inlen, char *out){
-   char a, b, c, d, fragment;
-
-   sprintf(out, "====");
-
-   if(inlen <= 0) return;
-
-   fragment = *in & 0x3;
-
-   a = *in >> 2;
-
-   out[0] = base64_value(a);
-
-   b = fragment << 4;
-
-   if(inlen > 1)
-      b += *(in+1) >> 4;
-
-   out[1] = base64_value(b);
-
-   if(inlen == 1) return;
-
-
-   c = *(in+1) & 0xf;
-   c = c << 2;
-
-   if(inlen > 2){
-      fragment = *(in+2) & 0xfc;
-      c += fragment >> 6;
-
-      d = *(in+2) & 0x3f;
-      out[3] = base64_value(d);
-   }
-
-   out[2] = base64_value(c);
-}
-
-
-void base64_encode(unsigned char *in, int inlen, char *out, int outlen){
-   int i=0, j, pos=0;
-   unsigned char buf[3];
-
-   memset(buf, 0, 3);
-   memset(out, 0, outlen);
-
-   for(j=0; j<inlen; j++){
-
-      if(i == 3){
-         base64_encode_block(buf, 3, &out[pos]); pos += 4;
-
-         memset(buf, 0, 3);
-
-         i = 0;
-      }
-
-      buf[i] = *(in+j);
-      i++;       
-   }
-
-   base64_encode_block(buf, i, &out[pos]);
-}
-

src/bdat.c

@@ -21,36 +21,29 @@
 
 
 void reset_bdat_counters(struct smtp_session *session){
-   session->bdat_rounds = 0;
-   session->bdat_last_round = 0;
    session->bdat_bytes_to_read = 0;
    session->bad = 0;
 }
 
 
-void get_bdat_size_to_read(struct smtp_session *session, char *buf){
+void get_bdat_size_to_read(struct smtp_session *session){
    char *p;
 
-   session->bdat_rounds++;
    session->bdat_bytes_to_read = 0;
 
    session->protocol_state = SMTP_STATE_BDAT;
 
-   // determine if this is the last BDAT command
-
-   p = strcasestr(buf, " LAST");
+   p = strcasestr(session->buf, " LAST");
    if(p){
-      session->bdat_last_round = 1;
-      syslog(LOG_INFO, "%s: BDAT LAST", session->ttmpfile);
       *p = '\0';
    }
 
    // determine the size to be read
 
-   p = strchr(buf, ' ');
+   p = strchr(session->buf, ' ');
    if(p){
       session->bdat_bytes_to_read = atoi(p);
-      if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: BDAT len=%d", session->ttmpfile, session->bdat_bytes_to_read);
+      if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "fd=%d: BDAT len=%d", session->net.socket, session->bdat_bytes_to_read);
    }
 
    if(!p || session->bdat_bytes_to_read <= 0){
@@ -60,17 +53,16 @@ void get_bdat_size_to_read(struct smtp_session *session, char *buf){
 }
 
 
-void process_bdat(struct smtp_session *session, char *readbuf, int readlen){
-   int i;
-   char buf[SMALLBUFSIZE];
-
+void process_bdat(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg){
    if(readlen <= 0) return;
 
-   if(session->bdat_rounds == 1 && session->fd == -1){
+   if(session->fd == -1){
       session->fd = open(session->ttmpfile, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR|S_IRGRP);
       if(session->fd == -1){
          syslog(LOG_PRIORITY, "%s: %s", ERR_OPEN_TMP_FILE, session->ttmpfile);
       }
+
+      if(cfg->process_rcpt_to_addresses == 1) write_envelope_addresses(session, cfg);
    }
 
    session->bdat_bytes_to_read -= readlen;
@@ -79,12 +71,12 @@ void process_bdat(struct smtp_session *session, char *readbuf, int readlen){
       if(write(session->fd, readbuf, readlen) != -1){
          session->tot_len += readlen;
 
-         if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: wrote %d bytes, %d bytes to go", session->ttmpfile, readlen, session->bdat_bytes_to_read);
+         if(session->cfg->verbosity >= _LOG_EXTREME) syslog(LOG_INFO, "%s: wrote %d bytes, %d bytes to go", session->ttmpfile, readlen, session->bdat_bytes_to_read);
       }
       else syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
    }
 
-   
+
    if(session->bdat_bytes_to_read < 0){
       // malformed data from client: we got more data then had been told in BDAT argument
       syslog(LOG_PRIORITY, "ERROR: invalid BDAT data. Expected %d, got %d bytes", session->bdat_bytes_to_read + readlen, readlen);
@@ -92,45 +84,34 @@ void process_bdat(struct smtp_session *session, char *readbuf, int readlen){
 
       close(session->fd);
       unlink(session->ttmpfile);
+
+      session->fd = -1;
    }
 
-   if(session->bdat_bytes_to_read == 0){
 
-      if(session->bdat_last_round == 1){
-         if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: read all bdat data in %d rounds", session->ttmpfile, session->bdat_rounds);
+   // If there's nothing more to read, then send response to smtp client
 
-         // send back the smtp answers
-         for(i=0; i<session->bdat_rounds; i++){
-            if(session->fd == -1){
-               send_smtp_response(session, SMTP_RESP_421_ERR_WRITE_FAILED);
-            }
-            else {
-               if(i == 0){
-                  fsync(session->fd);
-                  close(session->fd);
+   if(session->bdat_bytes_to_read <= 0){
 
-                  move_email(session);
-
-                  snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
-                  send_smtp_response(session, buf);
-                  syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d", session->ttmpfile, session->mailfrom, session->tot_len);
-               }
-               else send_smtp_response(session, SMTP_RESP_250_BDAT);
-            }
-         }
-
-         // technically we are not in the PERIOD state, but it's good enough
-         // to quit the BDAT processing state
-         session->protocol_state = SMTP_STATE_PERIOD;
+      if(session->fd == -1){
+         send_smtp_response(session, SMTP_RESP_421_ERR_WRITE_FAILED);
+      }
+      else {
+         fsync(session->fd);
+         close(session->fd);
 
          session->fd = -1;
+
+         move_email(session);
+
+         char buf[SMALLBUFSIZE];
+         snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
+         send_smtp_response(session, buf);
+         syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d, client=%s, fd=%d", session->ttmpfile, session->mailfrom, session->tot_len, session->remote_host, session->net.socket);
       }
 
-      else {
-         // this is not the last BDAT round, let's go back
-         // after the rcpt state, and wait for the next
-         // BDAT command
-         session->protocol_state = SMTP_STATE_RCPT_TO;
-      }
+      // technically we are not in the PERIOD state, but it's good enough
+      // to quit the BDAT processing state
+      session->protocol_state = SMTP_STATE_PERIOD;
    }
 }

src/cfg.c

@@ -17,30 +17,16 @@ int string_parser(char *src, char *target, int limit){
    return 0;
 };
 
-int multi_line_string_parser(char *src, char *target, unsigned int limit){
-   if(strlen(src) > 0 && strlen(target) + strlen(src) + 3 < limit){
-      strncat(target, src, limit-strlen(target));
-      strncat(target, "\r\n", limit-strlen(target));
-
-      return 0;
-   }
-
-   return 1;
-};
-
 int int_parser(char *src, int *target){
    *target = strtol(src, (char **) NULL, 10);
 
    return 0;
 };
 
-
-int float_parser(char *src, float *target){
-   *target = strtof(src, (char **) NULL);
-
+int uint64_parser(char *src, uint64 *target){
+   *target = strtoull(src, (char**)NULL, 10);
    return 0;
-};
-
+}
 
 struct _parse_rule {
    char *name;
@@ -58,10 +44,11 @@ struct _parse_rule {
 
 struct _parse_rule config_parse_rules[] =
 {
-
+   { "archive_address", "string", (void*) string_parser, offsetof(struct config, archive_address), "", MAXVAL-1},
    { "archive_emails_not_having_message_id", "integer", (void*) int_parser, offsetof(struct config, archive_emails_not_having_message_id), "0", sizeof(int)},
    { "archive_only_mydomains", "integer", (void*) int_parser, offsetof(struct config, archive_only_mydomains), "0", sizeof(int)},
    { "backlog", "integer", (void*) int_parser, offsetof(struct config, backlog), "20", sizeof(int)},
+   { "check_for_client_timeout_interval", "integer", (void*) int_parser, offsetof(struct config, check_for_client_timeout_interval), "20", sizeof(int)},
    { "cipher_list", "string", (void*) string_parser, offsetof(struct config, cipher_list), "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS", MAXVAL-1},
    { "clamd_addr", "string", (void*) string_parser, offsetof(struct config, clamd_addr), "", MAXVAL-1},
    { "clamd_port", "integer", (void*) int_parser, offsetof(struct config, clamd_port), "0", sizeof(int)},
@@ -81,7 +68,9 @@ struct _parse_rule config_parse_rules[] =
    { "listen_port", "integer", (void*) int_parser, offsetof(struct config, listen_port), "25", sizeof(int)},
    { "locale", "string", (void*) string_parser, offsetof(struct config, locale), "", MAXVAL-1},
    { "max_connections", "integer", (void*) int_parser, offsetof(struct config, max_connections), "64", sizeof(int)},
-   { "max_requests_per_child", "integer", (void*) int_parser, offsetof(struct config, max_requests_per_child), "1000", sizeof(int)},
+   { "max_message_size", "integer", (void*) int_parser, offsetof(struct config, max_message_size), "50000000", sizeof(int)},
+   { "max_requests_per_child", "integer", (void*) int_parser, offsetof(struct config, max_requests_per_child), "10000", sizeof(int)},
+   { "max_smtp_memory", "uint64", (void*) uint64_parser, offsetof(struct config, max_smtp_memory), "500000000", sizeof(uint64)},
    { "memcached_servers", "string", (void*) string_parser, offsetof(struct config, memcached_servers), "127.0.0.1", MAXVAL-1},
    { "memcached_to_db_interval", "integer", (void*) int_parser, offsetof(struct config, memcached_to_db_interval), "900", sizeof(int)},
    { "memcached_ttl", "integer", (void*) int_parser, offsetof(struct config, memcached_ttl), "86400", sizeof(int)},
@@ -102,11 +91,18 @@ struct _parse_rule config_parse_rules[] =
    { "piler_header_field", "string", (void*) string_parser, offsetof(struct config, piler_header_field), "X-piler-id:", MAXVAL-1},
    { "process_rcpt_to_addresses", "integer", (void*) int_parser, offsetof(struct config, process_rcpt_to_addresses), "0", sizeof(int)},
    { "queuedir", "string", (void*) string_parser, offsetof(struct config, queuedir), QUEUE_DIR, MAXVAL-1},
+   { "rtindex", "integer", (void*) int_parser, offsetof(struct config, rtindex), "0", sizeof(int)},
+   { "security_header", "string", (void*) string_parser, offsetof(struct config, security_header), "", MAXVAL-1},
    { "server_id", "integer", (void*) int_parser, offsetof(struct config, server_id), "0", sizeof(int)},
+   { "sphxdb", "string", (void*) string_parser, offsetof(struct config, sphxdb), "piler1", MAXVAL-1},
+   { "sphxhost", "string", (void*) string_parser, offsetof(struct config, sphxhost), "127.0.0.1", MAXVAL-1},
+   { "sphxport", "integer", (void*) int_parser, offsetof(struct config, sphxport), "9306", sizeof(int)},
+   { "smtp_access_list", "integer", (void*) int_parser, offsetof(struct config, smtp_access_list), "0", sizeof(int)},
    { "smtp_timeout", "integer", (void*) int_parser, offsetof(struct config, smtp_timeout), "60", sizeof(int)},
    { "spam_header_line", "string", (void*) string_parser, offsetof(struct config, spam_header_line), "", MAXVAL-1},
    { "syslog_recipients", "integer", (void*) int_parser, offsetof(struct config, syslog_recipients), "0", sizeof(int)},
    { "tls_enable", "integer", (void*) int_parser, offsetof(struct config, tls_enable), "0", sizeof(int)},
+   { "tls_min_version", "string", (void*) string_parser, offsetof(struct config, tls_min_version), "TLSv1.2", MAXVAL-1},
    { "tweak_sent_time_offset", "integer", (void*) int_parser, offsetof(struct config, tweak_sent_time_offset), "0", sizeof(int)},
    { "update_counters_to_memcached", "integer", (void*) int_parser, offsetof(struct config, update_counters_to_memcached), "0", sizeof(int)},
    { "username", "string", (void*) string_parser, offsetof(struct config, username), "piler", MAXVAL-1},
@@ -147,7 +143,7 @@ int parse_config_file(char *configfile, struct config *target_cfg, struct _parse
                   printf("failed to parse %s: \"%s\"\n", line, chpos+1);
                }
                break;
-            }				
+            }
 
             i++;
          }
@@ -162,6 +158,26 @@ int parse_config_file(char *configfile, struct config *target_cfg, struct _parse
 }
 
 
+int get_tls_protocol_number(char *protocol){
+   struct tls_protocol tls_protocols[] = {
+      { "TLSv1", TLS1_VERSION },
+      { "TLSv1.1", TLS1_1_VERSION },
+      { "TLSv1.2", TLS1_2_VERSION },
+   #ifdef TLS1_3_VERSION
+      { "TLSv1.3", TLS1_3_VERSION },
+   #endif
+   };
+
+   for(unsigned int i=0; i<sizeof(tls_protocols)/sizeof(struct tls_protocol); i++){
+      if(!strcmp(protocol, tls_protocols[i].proto)) {
+         return tls_protocols[i].version;
+      }
+   }
+
+   return 0;
+}
+
+
 int load_default_config(struct config *cfg, struct _parse_rule *rules){
    int i=0;
 
@@ -194,6 +210,9 @@ struct config read_config(char *configfile){
 
    cfg.hostid_len = strlen(cfg.hostid);
 
+   // Get the TLS protocol constant from string, ie. TLSv1.3 -> 772
+   cfg.tls_min_version_number = get_tls_protocol_number(cfg.tls_min_version);
+
    return cfg;
 }
 
@@ -205,6 +224,7 @@ struct config read_config(char *configfile){
 void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
    int j;
    float f;
+   uint64 u;
    char *p, buf[MAXVAL];
 
    p = (char*)cfg + rules[i].offset;
@@ -213,6 +233,10 @@ void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
       memcpy((char*)&j, p, sizeof(int));
       printf("%s=%d\n", rules[i].name, j);
    }
+   else if(strcmp(rules[i].type, "uint64") == 0){
+      memcpy((char*)&u, p, sizeof(uint64));
+      printf("%s=%llu\n", rules[i].name, u);
+   }
    else if(strcmp(rules[i].type, "float") == 0){
       memcpy((char*)&f, p, sizeof(float));
       printf("%s=%.4f\n", rules[i].name, f);
@@ -229,7 +253,7 @@ void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
       trimBuffer(p);
       printf("%s=%s\n", rules[i].name, p);
    }
- 
+
 }
 
 
@@ -296,12 +320,10 @@ void print_config(char *configfile, struct config *cfg){
 
             i++;
          }
- 
+
          if(!rules[i].name) printf("unknown key: \"%s\" \n", line);
       }
    }
 
    fclose(f);
 }
-
-

src/cfg.h

@@ -29,6 +29,8 @@ struct config {
    int tls_enable;
    char pemfile[MAXVAL];
    char cipher_list[MAXVAL];
+   char tls_min_version[MAXVAL];
+   int tls_min_version_number;
 
    int use_antivirus;
 
@@ -49,6 +51,7 @@ struct config {
    int verbosity;
    char locale[MAXVAL];
 
+   int check_for_client_timeout_interval;
    int smtp_timeout;
    int helper_timeout;
    int extract_attachments;
@@ -63,6 +66,9 @@ struct config {
 
    int default_retention_days;
 
+   char security_header[MAXVAL];
+   char archive_address[MAXVAL];
+
    // mysql stuff
 
    char mysqlcharset[MAXVAL];
@@ -74,6 +80,12 @@ struct config {
    char mysqldb[MAXVAL];
    int mysql_connect_timeout;
 
+   // manticore stuff
+   char sphxhost[MAXVAL];
+   int sphxport;
+   char sphxdb[MAXVAL];
+   int rtindex;
+
    int update_counters_to_memcached;
    int memcached_to_db_interval;
 
@@ -96,6 +108,11 @@ struct config {
    int enable_folders;
 
    int debug;
+
+   int smtp_access_list;
+
+   int max_message_size;
+   uint64 max_smtp_memory;
 };
 
 

src/clamd.c

@@ -20,7 +20,7 @@
 
 int clamd_scan(char *tmpfile, struct config *cfg){
    int s, n;
-   char *p, *q, buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE];
+   char buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE];
    struct sockaddr_un server;
 
    chmod(tmpfile, 0644);
@@ -41,7 +41,7 @@ int clamd_scan(char *tmpfile, struct config *cfg){
 
    /* issue the SCAN command with full path to the temporary directory */
 
-   
+
    memset(scan_cmd, 0, SMALLBUFSIZE);
    snprintf(scan_cmd, SMALLBUFSIZE-1, "SCAN %s/%s\r\n", cfg->workdir, tmpfile);
 
@@ -58,9 +58,9 @@ int clamd_scan(char *tmpfile, struct config *cfg){
    if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: CLAMD DEBUG: %d %s", tmpfile, n, buf);
 
    if(strcasestr(buf, CLAMD_RESP_INFECTED)){
-      p = strchr(buf, ' ');
+      char *p = strchr(buf, ' ');
       if(p){
-         q = strrchr(p, ' ');
+         char *q = strrchr(p, ' ');
          if(q){
             *q = '\0';
             p++;

src/config.h

@@ -9,13 +9,14 @@
 #include "piler-config.h"
 #include "params.h"
 
-#define VERSION "1.3.3"
+typedef unsigned long long uint64;
 
-#define BUILD 986
+#define BUILD 1001
 
 #define HOSTID "mailarchiver"
 
 #define CONFIG_FILE CONFDIR "/piler/piler.conf"
+#define SMTP_ACL_FILE CONFDIR "/piler/smtp.acl"
 #define WORK_DIR DATADIR "/piler/tmp"
 #define QUEUE_DIR DATADIR "/piler/store"
 #define ERROR_DIR DATADIR "/piler/error"
@@ -31,6 +32,7 @@
 #define SMALLBUFSIZE 512
 #define BIGBUFSIZE 131072
 #define REALLYBIGBUFSIZE 524288
+#define SMTPBUFSIZE 2048000
 #define TINYBUFSIZE 128
 #define MAXVAL 256
 #define RANDOM_POOL "/dev/urandom"
@@ -40,6 +42,20 @@
 #define IPLEN 16+1
 #define KEYLEN 56
 #define MIN_EMAIL_ADDRESS_LEN 9
+// Sphinx 3.x has an issue with tokens longer than 41 characters.
+//
+// When a regular user executes a query, then his default email address filter
+// causes the query to fail with the below error message, even when the query
+// itself is correct:
+//
+// SELECT id FROM main1,dailydelta1,delta1 WHERE MATCH(' ( (@sender thisisanextremelylongemailaddressyesareallylongoneyeahitolyouXaddressXcom ) | (@rcpt thisisanextremelylongemailaddressyesareallylongoneyeahitolyouXaddressXcom) ) ') ORDER BY `sent` DESC LIMIT 0,20 OPTION max_matches=1000'
+//
+// ERROR 1064 (42000): index dailydelta1,delta1,main1: syntax error, unexpected $end near ' '
+//
+// Note that we use 42, because the parser adds a trailing space to the tokens
+// See https://www.mailpiler.org/wiki/current:sphinx3 and
+// https://bitbucket.org/jsuto/piler/issues/1082/no-sphinx-results-with-long-email for more
+#define MAX_EMAIL_ADDRESS_SPHINX_LEN 42
 
 #define CRLF "\n"
 
@@ -59,6 +75,8 @@
 #define MEMCACHED_MSGS_STORED_SIZE MEMCACHED_CLAPF_PREFIX "stored_size"
 
 
+#define PILEREXPORT_BEGIN_MARK "x-exported-by-pilerexport: start\n"
+
 #define LOG_PRIORITY LOG_INFO
 
 #define _LOG_INFO 3
@@ -107,8 +125,10 @@
 #define SQL_PREPARED_STMT_GET_FOLDER_ID              "SELECT `id` FROM " SQL_FOLDER_TABLE " WHERE `name`=? AND `parent_id`=?"
 #define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE   "INSERT INTO `" SQL_FOLDER_TABLE "` (`name`, `parent_id`) VALUES(?,?)"
 #define SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE  "UPDATE " SQL_METADATA_TABLE " SET reference=? WHERE message_id=? AND reference=''"
+#define SQL_PREPARED_STMT_GET_METADATA_REFERENCE     "SELECT COUNT(*) AS count FROM " SQL_METADATA_TABLE " WHERE reference=?"
 #define SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS        "SELECT id, type, username, password, server FROM " SQL_IMPORT_TABLE " WHERE started=0 ORDER BY id LIMIT 0,1"
 #define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE      "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `id`) VALUES(?,?)"
+#define SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE        "UPDATE " SQL_IMPORT_TABLE " SET started=?, status=?, imported=? WHERE id=?"
 
 /* Error codes */
 
@@ -138,4 +158,3 @@
 #define S_STATUS_ERROR "error"
 
 #endif /* _CONFIG_H */
-

src/counters.c

@@ -11,7 +11,7 @@
 #include <piler.h>
 
 
-struct counters load_counters(struct session_data *sdata, struct data *data){
+struct counters load_counters(struct session_data *sdata){
    char buf[SMALLBUFSIZE];
    struct counters counters;
    struct sql sql;
@@ -51,32 +51,35 @@ struct counters load_counters(struct session_data *sdata, struct data *data){
 void update_counters(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    char buf[MAXBUFSIZE];
 #ifdef HAVE_MEMCACHED
-   unsigned long long mc, rcvd;
+   unsigned long long mc;
    struct counters c;
-   char key[MAX_MEMCACHED_KEY_LEN];
    unsigned int flags=0;
+#endif
 
-   if(counters->c_virus + counters->c_duplicate + counters->c_ignore + counters->c_size + counters->c_stored_size <= 0) return;
+   if(counters->c_virus + counters->c_duplicate + counters->c_ignore + counters->c_size + counters->c_stored_size == 0) return;
 
+#ifdef HAVE_MEMCACHED
    if(cfg->update_counters_to_memcached == 1){
 
       /* increment counters to memcached */
 
-      if(memcached_increment(&(data->memc), MEMCACHED_MSGS_RCVD, strlen(MEMCACHED_MSGS_RCVD), counters->c_rcvd, &mc) == MEMCACHED_SUCCESS){
-         rcvd = mc;
+      if(memcached_increment(&(data->memc), MEMCACHED_MSGS_RCVD, counters->c_rcvd, &mc) == MEMCACHED_SUCCESS){
+         unsigned long long rcvd = mc;
 
-         if(counters->c_virus > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), counters->c_virus, &mc);
-         if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), counters->c_duplicate, &mc);
-         if(counters->c_ignore > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), counters->c_ignore, &mc);
-         if(counters->c_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_SIZE, strlen(MEMCACHED_MSGS_SIZE), counters->c_size, &mc);
-         if(counters->c_stored_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_STORED_SIZE, strlen(MEMCACHED_MSGS_STORED_SIZE), counters->c_stored_size, &mc);
+         if(counters->c_virus > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_VIRUS, counters->c_virus, &mc);
+         if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_DUPLICATE, counters->c_duplicate, &mc);
+         if(counters->c_ignore > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_IGNORE, counters->c_ignore, &mc);
+         if(counters->c_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_SIZE, counters->c_size, &mc);
+         if(counters->c_stored_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_STORED_SIZE, counters->c_stored_size, &mc);
 
 
-         bzero(&c, sizeof(c)); 
+         bzero(&c, sizeof(c));
 
          snprintf(buf, MAXBUFSIZE-1, "%s %s %s %s %s %s %s", MEMCACHED_MSGS_RCVD, MEMCACHED_MSGS_VIRUS, MEMCACHED_MSGS_DUPLICATE, MEMCACHED_MSGS_IGNORE, MEMCACHED_MSGS_SIZE, MEMCACHED_MSGS_STORED_SIZE, MEMCACHED_COUNTERS_LAST_UPDATE);
 
          if(memcached_mget(&(data->memc), buf) == MEMCACHED_SUCCESS){
+            char key[MAX_MEMCACHED_KEY_LEN];
+
             while((memcached_fetch_result(&(data->memc), &key[0], &buf[0], &flags))){
                if(!strcmp(key, MEMCACHED_MSGS_RCVD)) c.c_rcvd = strtoull(buf, NULL, 10);
                else if(!strcmp(key, MEMCACHED_MSGS_VIRUS)) c.c_virus = strtoull(buf, NULL, 10);
@@ -88,11 +91,11 @@ void update_counters(struct session_data *sdata, struct data *data, struct count
             }
 
 
-            if(sdata->now - mc > cfg->memcached_to_db_interval && c.c_rcvd > 0 && c.c_rcvd >= rcvd){
-               snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_set(&(data->memc), MEMCACHED_COUNTERS_LAST_UPDATE, strlen(MEMCACHED_COUNTERS_LAST_UPDATE), buf, strlen(buf), 0, 0);
+            if(sdata->now - mc > (unsigned long long)cfg->memcached_to_db_interval && c.c_rcvd > 0 && c.c_rcvd >= rcvd){
+               snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), "set", MEMCACHED_COUNTERS_LAST_UPDATE, buf, strlen(buf), 0, 0);
 
                snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET `rcvd`=%llu, `virus`=%llu, `duplicate`=%llu, `ignore`=%llu, `size`=%llu, `stored_size`=%llu", SQL_COUNTER_TABLE, c.c_rcvd, c.c_virus, c.c_duplicate, c.c_ignore, c.c_size, c.c_stored_size);
- 
+
                //if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: update counters: %s", sdata->ttmpfile, buf);
 
                p_query(sdata, buf);
@@ -102,23 +105,21 @@ void update_counters(struct session_data *sdata, struct data *data, struct count
       }
       else {
 
-         c = load_counters(sdata, data);
+         c = load_counters(sdata);
 
-         snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), MEMCACHED_COUNTERS_LAST_UPDATE, strlen(MEMCACHED_COUNTERS_LAST_UPDATE), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), "add", MEMCACHED_COUNTERS_LAST_UPDATE, buf, strlen(buf), 0, 0);
 
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_virus + counters->c_virus); memcached_add(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_rcvd + counters->c_rcvd); memcached_add(&(data->memc), MEMCACHED_MSGS_RCVD, strlen(MEMCACHED_MSGS_RCVD), buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_duplicate + counters->c_duplicate); memcached_add(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_ignore + counters->c_ignore); memcached_add(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_size + counters->c_size); memcached_add(&(data->memc), MEMCACHED_MSGS_SIZE, strlen(MEMCACHED_MSGS_SIZE), buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_stored_size + counters->c_stored_size); memcached_add(&(data->memc), MEMCACHED_MSGS_STORED_SIZE, strlen(MEMCACHED_MSGS_STORED_SIZE), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_virus + counters->c_virus); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_VIRUS, buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_rcvd + counters->c_rcvd); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_RCVD, buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_duplicate + counters->c_duplicate); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_DUPLICATE, buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_ignore + counters->c_ignore); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_IGNORE, buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_size + counters->c_size); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_SIZE, buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_stored_size + counters->c_stored_size); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_STORED_SIZE, buf, strlen(buf), 0, 0);
       }
 
    }
    else {
 #endif
-      if(counters->c_virus + counters->c_duplicate + counters->c_ignore + counters->c_size + counters->c_stored_size <= 0) return;
-
       snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET `rcvd`=`rcvd`+%llu, `virus`=`virus`+%llu, `duplicate`=`duplicate`+%llu, `ignore`=`ignore`+%llu, `size`=`size`+%llu, `stored_size`=`stored_size`+%llu", SQL_COUNTER_TABLE, counters->c_rcvd, counters->c_virus, counters->c_duplicate, counters->c_ignore, counters->c_size, counters->c_stored_size);
       p_query(sdata, buf);
 
@@ -127,5 +128,3 @@ void update_counters(struct session_data *sdata, struct data *data, struct count
 #endif
 
 }
-
-

src/decoder.c

@@ -98,20 +98,6 @@ inline void utf8_encode_char(unsigned char c, unsigned char *buf, int buflen, in
 }
 
 
-void sanitiseBase64(char *s){
-   char *p1;
-
-   if(s == NULL) return;
-
-   for(; *s; s++){
-      if(b64[(unsigned int)(*s & 0xFF)] == 255){
-         for(p1 = s; p1[0] != '\0'; p1++)
-            p1[0] = p1[1];
-      }
-   }
-}
-
-
 inline static void pack_4_into_3(char *s, char *s2){
    int j, n[4], k1, k2;
 
@@ -157,7 +143,7 @@ int decodeBase64(char *p){
 
 
 int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
-   int i, len=0, decodedlen;
+   int i, len=0;
    char s[5], s2[3];
 
    if(plen < 4 || plen > blen)
@@ -166,7 +152,7 @@ int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
    for(i=0; i<plen; i+=4){
       memcpy(s, p+i, 4);
       s[4] = '\0';
-      decodedlen = 3;
+      int decodedlen = 3;
 
       /* safety check against abnormally long lines */
 
@@ -191,12 +177,11 @@ int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
 void decodeQP(char *p){
    unsigned int i;
    int k=0, a, b;
-   char c;
 
    if(p == NULL) return;
 
    for(i=0; i<strlen((char*)p); i++){
-      c = p[i];
+      char c = p[i];
 
       if(p[i] == '=' && isxdigit(p[i+1]) && isxdigit(p[i+2])){
          a = p[i+1];

src/decoder.h

@@ -5,9 +5,6 @@
 #ifndef _DECODER_H
  #define _DECODER_H
 
-void base64_encode(unsigned char *in, int inlen, char *out, int outlen);
-
-void sanitiseBase64(char *s);
 int decodeBase64(char *p);
 int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen);
 void decodeQP(char *p);

src/defs.h

@@ -13,10 +13,8 @@
    #include <tre/tre.h>
    #include <tre/regex.h>
 #endif
-#ifdef HAVE_LIBWRAP
-   #include <tcpd.h>
-#endif
 
+#include <openssl/md5.h>
 #include <openssl/sha.h>
 #include <openssl/ssl.h>
 #include <netinet/in.h>
@@ -27,21 +25,24 @@
 #define MSG_BODY 0
 #define MSG_RECEIVED 1
 #define MSG_FROM 2
-#define MSG_TO 3
-#define MSG_CC 4
-#define MSG_SUBJECT 5
-#define MSG_CONTENT_TYPE 6
-#define MSG_CONTENT_TRANSFER_ENCODING 7
-#define MSG_CONTENT_DISPOSITION 8
-#define MSG_MESSAGE_ID 9
-#define MSG_REFERENCES 10
-#define MSG_RECIPIENT 11
+#define MSG_SENDER 3
+#define MSG_TO 4
+#define MSG_CC 5
+#define MSG_SUBJECT 6
+#define MSG_CONTENT_TYPE 7
+#define MSG_CONTENT_TRANSFER_ENCODING 8
+#define MSG_CONTENT_DISPOSITION 9
+#define MSG_MESSAGE_ID 10
+#define MSG_REFERENCES 11
+#define MSG_RECIPIENT 12
+#define MSG_ENVELOPE_TO 13
 
 #define MAXHASH 277
 
 #define BASE64_RATIO 1.33333333
 
-#define DIGEST_LENGTH SHA256_DIGEST_LENGTH
+#define DIGEST_LENGTH EVP_MAX_MD_SIZE
+#define DIGEST_HEX_LENGTH 2*DIGEST_LENGTH+1
 
 #define UNDEF 0
 #define READY 1
@@ -96,6 +97,15 @@ struct node {
 };
 
 
+struct smtp_acl {
+   char network_str[BUFLEN];
+   in_addr_t low, high;
+   int prefix;
+   int rejected;
+   struct smtp_acl *r;
+};
+
+
 struct net {
    int socket;
    int use_ssl;
@@ -163,8 +173,10 @@ struct parser_state {
    int qp;
    int htmltag;
    int style;
+   int meta_content_type;
    int skip_html;
    int has_to_dump;
+   int has_to_dump_whole_body;
    int fd;
    int b64fd;
    int mfd;
@@ -173,13 +185,15 @@ struct parser_state {
    int content_type_is_set;
    int pushed_pointer;
    int saved_size;
-   int writebufpos;
-   int abufpos;
+   unsigned int writebufpos;
+   unsigned int abufpos;
+   unsigned int received_header;
    char attachedfile[RND_STR_LEN+SMALLBUFSIZE];
    char message_id[SMALLBUFSIZE];
    char message_id_hash[2*DIGEST_LENGTH+1];
    char miscbuf[MAX_TOKEN_LEN];
    char qpbuf[MAX_TOKEN_LEN];
+   char receivedbuf[SMALLBUFSIZE];
    unsigned long n_token;
    unsigned long n_subject_token;
    unsigned long n_body_token;
@@ -201,12 +215,15 @@ struct parser_state {
 
    char reference[SMALLBUFSIZE];
 
-   char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE];
+   char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_sender[SMALLBUFSIZE], b_sender_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE];
    char b_journal_to[MAXBUFSIZE];
 
-   int bodylen;
-   int tolen;
-   int journaltolen;
+   unsigned int bodylen;
+   unsigned int tolen;
+   unsigned int todomainlen;
+   unsigned int found_security_header;
+
+   long unsigned int journaltolen;
 
    int retention;
 };
@@ -236,11 +253,7 @@ struct session_data {
    int journal_envelope_length, journal_bottom_length;
    unsigned int sql_errno;
 #ifdef NEED_MYSQL
-   MYSQL mysql;
-#endif
-#ifdef NEED_PSQL
-   PGconn *psql;
-   char conninfo[SMALLBUFSIZE];
+   MYSQL mysql, sphx;
 #endif
 };
 
@@ -300,11 +313,14 @@ struct import {
    int keep_eml;
    int timeout;
    int cap_uidplus;
+   int fd;
    long total_size;
    int dryrun;
    int tot_msgs;
    int port;
    int seq;
+   int table_id;
+   int delay;
    char *server;
    char *username;
    char *password;
@@ -315,7 +331,7 @@ struct import {
    char *mboxdir;
    char *folder;
    char filename[SMALLBUFSIZE];
-   time_t started, updated, finished;
+   time_t started, updated, finished, after, before;
 };
 
 
@@ -354,6 +370,11 @@ struct data {
 };
 
 
+#if !defined(MARIADB_BASE_VERSION) && !defined(MARIADB_VERSION_ID) && \
+  MYSQL_VERSION_ID >= 80001 && MYSQL_VERSION_ID != 80002
+typedef bool my_bool;
+#endif
+
 struct sql {
 #ifdef NEED_MYSQL
    MYSQL_STMT *stmt;
@@ -381,20 +402,30 @@ struct counters {
 struct smtp_session {
    char ttmpfile[SMALLBUFSIZE];
    char mailfrom[SMALLBUFSIZE];
-   char buf[SMALLBUFSIZE];
-   char remote_host[INET6_ADDRSTRLEN];
+   char rcptto[MAX_RCPT_TO][SMALLBUFSIZE];
+   char remote_host[INET6_ADDRSTRLEN+1];
+   char nullbyte;
    time_t lasttime;
    int protocol_state;
    int slot;
    int fd;
    int bad;
-   int buflen;
    int tot_len;
-   int bdat_rounds;
-   int bdat_last_round;
    int bdat_bytes_to_read;
+   int num_of_rcpt_to;
    struct config *cfg;
    struct net net;
+   int max_message_size;
+   char *buf;
+   int buflen;
+   int bufsize;
+   int too_big;
+   int mail_size;
+};
+
+struct tls_protocol {
+   char *proto;
+   int version;
 };
 
 #endif /* _DEFS_H */

src/digest.c

@@ -34,25 +34,38 @@ int search_header_end(char *p, int n){
 
 
 int make_digests(struct session_data *sdata, struct config *cfg){
-   int i=0, n, fd, offset=3, hdr_len=0, len=0;
+   int n, fd, offset=3, hdr_len=0;
    char *body=NULL;
-   unsigned char buf[BIGBUFSIZE], md[DIGEST_LENGTH], md2[DIGEST_LENGTH];
-   SHA256_CTX context, context2;
+   unsigned char buf[BIGBUFSIZE];
+
+   EVP_MD_CTX *ctx, *ctx2;
+   const EVP_MD *md, *md2;
+   unsigned int i=0, md_len, md_len2;
+   unsigned char md_value[EVP_MAX_MD_SIZE], md_value2[EVP_MAX_MD_SIZE];
 
    memset(sdata->bodydigest, 0, 2*DIGEST_LENGTH+1);
    memset(sdata->digest, 0, 2*DIGEST_LENGTH+1);
-   SHA256_Init(&context);
-   SHA256_Init(&context2);
 
+   md = EVP_get_digestbyname("sha256");
+   md2 = EVP_get_digestbyname("sha256");
+   if(md == NULL || md2 == NULL){
+      syslog(LOG_PRIORITY, "ERROR: unknown message digest: sha256 in %s:%d", __func__, __LINE__);
+      return 1;
+   }
+
+   ctx = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx, md, NULL);
+
+   ctx2 = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx2, md2, NULL);
 
    fd = open(sdata->filename, O_RDONLY);
    if(fd == -1) return -1;
 
+   memset(buf, 0, sizeof(buf));
 
    while((n = read(fd, buf, sizeof(buf))) > 0){
-      len += n;
-
-      SHA256_Update(&context2, buf, n);
+      EVP_DigestUpdate(ctx2, buf, n);
 
       body = (char *)&buf[0];
 
@@ -69,7 +82,7 @@ int make_digests(struct session_data *sdata, struct config *cfg){
       }
 
 
-      SHA256_Update(&context, body, n);
+      EVP_DigestUpdate(ctx, body, n);
 
       i++;
    }
@@ -78,59 +91,93 @@ int make_digests(struct session_data *sdata, struct config *cfg){
 
    sdata->hdr_len = hdr_len;
 
-   SHA256_Final(md, &context);
-   SHA256_Final(md2, &context2);
+   EVP_DigestFinal_ex(ctx, md_value, &md_len);
+   EVP_MD_CTX_free(ctx);
+   EVP_DigestFinal_ex(ctx2, md_value2, &md_len2);
+   EVP_MD_CTX_free(ctx2);
 
-   for(i=0;i<DIGEST_LENGTH;i++){
-      snprintf(sdata->bodydigest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
-      snprintf(sdata->digest + i*2, 2*DIGEST_LENGTH, "%02x", md2[i]);
+   for(i=0;i<md_len;i++){
+      snprintf(sdata->bodydigest + i*2, 3, "%02x", md_value[i]);
+   }
+
+   for(i=0;i<md_len2;i++){
+      snprintf(sdata->digest + i*2, 3, "%02x", md_value2[i]);
    }
 
    return 0;
 }
 
 
-void digest_file(char *filename, char *digest){
-   int fd, i, n;
-   unsigned char buf[MAXBUFSIZE], md[DIGEST_LENGTH];
-   SHA256_CTX context;
+void raw_digest_file(char *digestname, char *filename, unsigned char *md_value){
+   int fd, n;
+   unsigned char buf[MAXBUFSIZE];
+   EVP_MD_CTX *ctx;
+   const EVP_MD *md;
+   unsigned int md_len;
 
-   memset(digest, 0, 2*DIGEST_LENGTH+1);
+   md = EVP_get_digestbyname(digestname);
+   if(md == NULL){
+      syslog(LOG_PRIORITY, "ERROR: unknown message digest: '%s' in %s:%d", digestname, __func__, __LINE__);
+      return;
+   }
 
    fd = open(filename, O_RDONLY);
    if(fd == -1) return;
 
-   SHA256_Init(&context);
+   ctx = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx, md, NULL);
 
    while((n = read(fd, buf, sizeof(buf))) > 0){
-      SHA256_Update(&context, buf, n);
+      EVP_DigestUpdate(ctx, buf, n);
    }
 
    close(fd);
 
-   SHA256_Final(md, &context);
-
-   for(i=0;i<DIGEST_LENGTH;i++)
-      snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
-
+   EVP_DigestFinal_ex(ctx, md_value, &md_len);
+   EVP_MD_CTX_free(ctx);
 }
 
 
-void digest_string(char *s, char *digest){
-   int i;
+void digest_file(char *filename, char *digest){
    unsigned char md[DIGEST_LENGTH];
-   SHA256_CTX context;
+
+   raw_digest_file("sha256", filename, &md[0]);
 
    memset(digest, 0, 2*DIGEST_LENGTH+1);
 
-   SHA256_Init(&context);
-
-   SHA256_Update(&context, s, strlen(s));
-
-   SHA256_Final(md, &context);
-
-   for(i=0;i<DIGEST_LENGTH;i++)
+   for(int i=0;i<SHA256_DIGEST_LENGTH;i++){
       snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
-
+   }
 }
 
+
+void digest_string(char *digestname, char *s, char *digest){
+   EVP_MD_CTX *ctx;
+   const EVP_MD *md;
+   unsigned int i, md_len;
+   unsigned char md_value[DIGEST_LENGTH];
+
+   memset(digest, 0, 2*DIGEST_LENGTH+2);
+
+   md = EVP_get_digestbyname(digestname);
+   if(md == NULL){
+      syslog(LOG_PRIORITY, "ERROR: unknown message digest: '%s' in %s:%d", digestname, __func__, __LINE__);
+      return;
+   }
+
+   ctx = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx, md, NULL);
+   EVP_DigestUpdate(ctx, s, strlen(s));
+   EVP_DigestFinal_ex(ctx, md_value, &md_len);
+   EVP_MD_CTX_free(ctx);
+
+   for(i=0;i<md_len;i++){
+      snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md_value[i]);
+   }
+}
+
+
+void create_md5_from_email_address(char *puf, char *md5buf){
+   digest_string("md5", puf, md5buf);
+   md5buf[2*MD5_DIGEST_LENGTH] = ' ';
+}

src/extract.c

@@ -19,40 +19,38 @@
 #define die(e) do { syslog(LOG_INFO, "error: helper: %s", e); exit(EXIT_FAILURE); } while (0);
 
 
-void remove_xml(char *buf, int *html){
+int remove_xml(char *src, char *dest, int destlen, int *html){
    int i=0;
-   char *p;
 
-   p = buf;
+   memset(dest, 0, destlen);
 
-   for(; *p; p++){
-      if(*p == '<'){ *html = 1; }
-
-      if(*html == 0){
-         *(buf+i) = *p;
-         i++;
+   for(; *src; src++){
+      if(*src == '<'){
+         *html = 1;
       }
-
-      if(*p == '>'){
+      else if(*src == '>'){
          *html = 0;
-
-         if(i > 2 && *(buf+i-1) != ' '){
-            *(buf+i) = ' '; i++;
-         }
-
+         // make sure there's a whitespace between tag contents
+         if(i < destlen && i > 0 && !isspace(dest[i-1]))
+            dest[i++] = ' ';
+      }
+      else{
+         if(*html == 0){
+            if(i < destlen) *(dest+i) = *src;
+            i++;
+         }
       }
-
    }
 
-   *(buf+i) = '\0';
+   return i;
 }
 
 
 #ifdef HAVE_ZIP
 int extract_opendocument(struct session_data *sdata, struct parser_state *state, char *filename, char *prefix){
-   int errorp, i=0, len=0, html=0;
-   int len2;
-   char buf[MAXBUFSIZE];
+   int errorp, i=0, html=0;
+   unsigned int len2;
+   char buf[4*MAXBUFSIZE], puf[4*MAXBUFSIZE];
    struct zip *z;
    struct zip_stat sb;
    struct zip_file *zf;
@@ -70,13 +68,13 @@ int extract_opendocument(struct session_data *sdata, struct parser_state *state,
 
          zf = zip_fopen_index(z, i, 0);
          if(zf){
+            int len;
             while((len = zip_fread(zf, buf, sizeof(buf)-2)) > 0){
 
-               remove_xml(buf, &html);
-               len2 = strlen(buf);
+               len2 = remove_xml(buf, puf, sizeof(puf), &html);
 
                if(len2 > 0 && state->bodylen < BIGBUFSIZE-len2-1){
-                  memcpy(&(state->b_body[state->bodylen]), buf, len2);
+                  memcpy(&(state->b_body[state->bodylen]), puf, len2);
                   state->bodylen += len2;
                }
 
@@ -100,7 +98,7 @@ int extract_opendocument(struct session_data *sdata, struct parser_state *state,
 
 
 int unzip_file(struct session_data *sdata, struct parser_state *state, char *filename, int *rec, struct config *cfg){
-   int errorp, i=0, len=0, fd;
+   int errorp, i=0, fd;
    char *p, extracted_filename[SMALLBUFSIZE], buf[MAXBUFSIZE];
    struct zip *z;
    struct zip_stat sb;
@@ -131,6 +129,7 @@ int unzip_file(struct session_data *sdata, struct parser_state *state, char *fil
             if(fd != -1){
                zf = zip_fopen_index(z, i, 0);
                if(zf){
+                  int len;
                   while((len = zip_fread(zf, buf, sizeof(buf))) > 0){
                      if(write(fd, buf, len) == -1) syslog(LOG_PRIORITY, "ERROR: error writing to fd in %s", __func__);
                   }
@@ -175,7 +174,7 @@ int extract_tnef(struct session_data *sdata, struct parser_state *state, char *f
    struct dirent **namelist;
 
    memset(tmpdir, 0, sizeof(tmpdir));
-   make_random_string(&tmpdir[0], sizeof(tmpdir)-3);
+   make_random_string((unsigned char *)&tmpdir[0], sizeof(tmpdir)-3);
 
    memcpy(&tmpdir[sizeof(tmpdir)-3], ".d", 2);
 
@@ -183,7 +182,7 @@ int extract_tnef(struct session_data *sdata, struct parser_state *state, char *f
 
    snprintf(buf, sizeof(buf)-1, "%s --unix-paths -C %s %s", HAVE_TNEF, tmpdir, filename);
 
-   system(buf);
+   if(system(buf) == -1) syslog(LOG_INFO, "error: running %s", buf);
 
    n = scandir(tmpdir, &namelist, NULL, alphasort);
    if(n < 0) syslog(LOG_INFO, "error: reading %s", tmpdir);
@@ -217,9 +216,8 @@ void kill_helper(){
 
 
 void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct config *cfg){
-   int link[2], n;
+   int link[2];
    pid_t pid;
-   char outbuf[MAXBUFSIZE];
 
    if(strcmp(type, "other") == 0 || strcmp(type, "text") == 0) return;
 
@@ -314,18 +312,19 @@ void extract_attachment_content(struct session_data *sdata, struct parser_state
    }
    else {
       close(link[1]);
+      ssize_t n;
+      char outbuf[MAXBUFSIZE];
+
       while((n = read(link[0], outbuf, sizeof(outbuf))) > 0){
          if(state->bodylen < BIGBUFSIZE-n-1){
             memcpy(&(state->b_body[state->bodylen]), outbuf, n);
             state->bodylen += n;
          }
-         //printf("Output: %.*s\n", n, outbuf);
       }
+      close(link[0]);
       wait(NULL);
       return;
    }
 
 
 }
-
-

src/hash.c

@@ -33,12 +33,10 @@ void clearhash(struct node *xhash[]){
          p = q;
 
          q = q->r;
-         if(p){
-            if(p->str){
-               free(p->str);
-            }
-            free(p);
+         if(p->str){
+            free(p->str);
          }
+         free(p);
       }
       xhash[i] = NULL;
    }
@@ -154,13 +152,12 @@ int is_substr_in_hash(struct node *xhash[], char *s){
 
 
 unsigned int DJBHash(char* str, unsigned int len){
-   unsigned int hash = 5381;
+   unsigned int hashval = 5381;
    unsigned int i    = 0;
 
    for(i=0; i < len; str++, i++){
-      hash = ((hash << 5) + hash) + (*str);
+      hashval = ((hashval << 5) + hashval) + (*str);
    }
 
-   return hash;
+   return hashval;
 }
-

src/htmlentities.h

@@ -285,4 +285,3 @@ struct mi htmlentities[] = {
 
 
 #endif /* _HTMLENTITIES_H */
-

src/imap.c

@@ -91,47 +91,12 @@ END:
 
 
 int connect_to_imap_server(struct data *data){
-   int n;
    char buf[MAXBUFSIZE];
-   X509* server_cert;
-   char *str;
 
    data->import->cap_uidplus = 0;
 
    if(data->net->use_ssl == 1){
-      SSL_library_init();
-      SSL_load_error_strings();
-
-   #if OPENSSL_VERSION_NUMBER < 0x10100000L
-      data->net->ctx = SSL_CTX_new(TLSv1_client_method());
-   #else
-      data->net->ctx = SSL_CTX_new(TLS_client_method());
-   #endif
-      CHK_NULL(data->net->ctx, "internal SSL error");
-
-      data->net->ssl = SSL_new(data->net->ctx);
-      CHK_NULL(data->net->ssl, "internal ssl error");
-
-      SSL_set_fd(data->net->ssl, data->net->socket);
-      n = SSL_connect(data->net->ssl);
-      CHK_SSL(n, "internal ssl error");
-
-      printf("Cipher: %s\n", SSL_get_cipher(data->net->ssl));
-
-      server_cert = SSL_get_peer_certificate(data->net->ssl);
-      CHK_NULL(server_cert, "server cert error");
-
-      str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0);
-      CHK_NULL(str, "error in server cert");
-      printf("server cert:\n\t subject: %s\n", str);
-      OPENSSL_free(str);
-
-      str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0);
-      CHK_NULL(str, "error in server cert");
-      printf("\t issuer: %s\n\n", str);
-      OPENSSL_free(str);
-
-      X509_free(server_cert);
+      init_ssl_to_server(data);
    }
 
 
@@ -140,7 +105,12 @@ int connect_to_imap_server(struct data *data){
 
    /* imap cmd: LOGIN */
 
-   snprintf(buf, sizeof(buf)-1, "A%d LOGIN %s \"%s\"\r\n", data->import->seq, data->import->username, data->import->password);
+   if(strcmp(data->import->username, "ZIMBRA") == 0){
+      snprintf(buf, sizeof(buf)-1, "A%d AUTHENTICATE PLAIN %s\r\n", data->import->seq, data->import->password);
+   }
+   else {
+      snprintf(buf, sizeof(buf)-1, "A%d LOGIN %s \"%s\"\r\n", data->import->seq, data->import->username, data->import->password);
+   }
 
    write1(data->net, buf, strlen(buf));
    if(read_response(buf, sizeof(buf), data) == 0){
@@ -194,7 +164,7 @@ int imap_select_cmd_on_folder(char *folder, struct data *data){
       }
    }
 
-   printf("found %d messages\n", messages);
+   if(data->quiet == 0){printf("found %d messages\n", messages); }
 
    data->import->total_messages += messages;
 
@@ -348,16 +318,16 @@ void imap_expunge_message(struct data *data){
 }
 
 
-int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct config *cfg){
-   int rc=ERR, i, messages=0;
+int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   int i, messages=0;
 
    messages = imap_select_cmd_on_folder(folder, data);
 
    if(messages <= 0) return OK;
 
    if(data->recursive_folder_names == 1){
-      data->folder = get_folder_id(sdata, data, folder, 0);
-      if(data->folder == ERR_FOLDER) data->folder = add_new_folder(sdata, data, folder, 0);
+      data->folder = get_folder_id(sdata, folder, 0);
+      if(data->folder == ERR_FOLDER) data->folder = add_new_folder(sdata, folder, 0);
    }
 
    for(i=data->import->start_position; i<=messages; i++){
@@ -365,7 +335,7 @@ int process_imap_folder(char *folder, struct session_data *sdata, struct data *d
          if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/messages); fflush(stdout); }
 
          if(data->import->dryrun == 0){
-            rc = import_message(sdata, data, cfg);
+            int rc = import_message(sdata, data, counters, cfg);
 
             if(data->import->remove_after_import == 1 && rc == OK){
                imap_delete_message(data, i);
@@ -391,14 +361,14 @@ int process_imap_folder(char *folder, struct session_data *sdata, struct data *d
    }
 
 
-   printf("\n");
+   if(data->quiet == 0){printf("\n"); }
 
    return OK;
 }
 
 
 void send_imap_close(struct data *data){
-   char puf[SMALLBUFSIZE];  
+   char puf[SMALLBUFSIZE];
    snprintf(puf, sizeof(puf)-1, "A%d CLOSE\r\n", data->import->seq);
 
    write1(data->net, puf, strlen(puf));
@@ -410,7 +380,7 @@ int list_folders(struct data *data){
    char attrs[SMALLBUFSIZE], folder[SMALLBUFSIZE];
    int len=MAXBUFSIZE+3, pos=0, n, rc=ERR, fldrlen=0, result;
 
-   printf("List of IMAP folders:\n");
+   if(data->quiet == 0){printf("List of IMAP folders:\n"); }
 
    buf = malloc(len);
    if(!buf) return rc;
@@ -451,7 +421,7 @@ int list_folders(struct data *data){
    }
 
    // trim the "A3 OK LIST completed" trailer off
-   if(p) *p = '\0';
+   if(p) *p = '\0'; //-V547
 
    memset(attrs, 0, sizeof(attrs));
 
@@ -482,23 +452,29 @@ int list_folders(struct data *data){
                q++;
                fldrlen = strtol(q, NULL, 10);
             } else {
-               
+
                if(fldrlen) {
-                  ruf = malloc(strlen(q) * 2 + 1);
-                  memset(ruf, 0, strlen(q) * 2 + 1);
-                  memcpy(ruf, q, strlen(q));
-                  r = ruf;
-                  while(*r != '\0') {
-                     if(*r == '\\') {
-                        memmove(r + 1, r, strlen(r));
+                  int ruflen = strlen(q) * 2;
+                  ruf = malloc(ruflen + 1);
+                  if(ruf){
+                     snprintf(ruf, ruflen, "%s", q);
+                     r = ruf;
+                     while(*r != '\0') {
+                        if(*r == '\\') {
+                           memmove(r + 1, r, strlen(r));
+                           r++;
+                        }
                         r++;
                      }
-                     r++;
+
+                     snprintf(folder, sizeof(folder)-1, "%s", ruf);
+
+                     free(ruf);
+                  }
+                  else {
+                     printf("error: ruf = malloc()\n");
                   }
 
-                  snprintf(folder, sizeof(folder)-1, "%s", ruf);
-
-                  free(ruf);
                   fldrlen = 0;
                } else {
                   snprintf(folder, sizeof(folder)-1, "%s", q);
@@ -507,9 +483,9 @@ int list_folders(struct data *data){
                if(!strstr(attrs, "\\Noselect")){
                   addnode(data->imapfolders, folder);
                }
-               else printf("skipping ");
+               else if(data->quiet == 0){printf("skipping "); }
 
-               printf("=> '%s [%s]'\n", folder, attrs);
+               if(data->quiet == 0){printf("=> '%s [%s]'\n", folder, attrs); }
 
                memset(attrs, 0, sizeof(attrs));
             }

src/import.c

@@ -18,21 +18,23 @@
 #include <piler.h>
 
 
-int import_message(struct session_data *sdata, struct data *data, struct config *cfg){
+int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    int rc=ERR;
-   char *p, *rule, newpath[SMALLBUFSIZE];
+   char *rule;
    struct stat st;
    struct parser_state state;
-   struct counters counters;
 
+   if(data->import->delay > 0){
+      struct timespec req;
+
+      req.tv_sec = 0;
+      req.tv_nsec = 1000000 * data->import->delay;
+
+      nanosleep(&req, NULL);
+   }
 
    init_session_data(sdata, cfg);
 
-   if(data->import->extra_recipient){
-      snprintf(sdata->rcptto[0], SMALLBUFSIZE-1, "%s", data->import->extra_recipient);
-      sdata->num_of_rcpt_to = 1;
-   }
-
    if(cfg->verbosity > 1) printf("processing: %s\n", data->import->filename);
 
    if(strcmp(data->import->filename, "-") == 0){
@@ -70,15 +72,14 @@ int import_message(struct session_data *sdata, struct data *data, struct config
 
    data->import->total_size += sdata->tot_len;
 
-   
+
    sdata->delivered = 0;
 
    sdata->import = 1;
 
    state = parse_message(sdata, 1, data, cfg);
    post_parse(sdata, &state, cfg);
-
-   rule = check_againt_ruleset(data->archiving_rules, &state, sdata->tot_len, sdata->spam_message);
+   rule = check_against_ruleset(data->archiving_rules, &state, sdata->tot_len, sdata->spam_message);
 
    if(rule){
       if(data->quiet == 0) printf("discarding %s by archiving policy: %s\n", data->import->filename, rule);
@@ -91,8 +92,22 @@ int import_message(struct session_data *sdata, struct data *data, struct config
          printf("%s: invalid message, hdr_len: %d\n", data->import->filename, sdata->hdr_len);
          rc = ERR;
       }
+      else if(data->import->after > 0 && sdata->sent < data->import->after){
+         if(cfg->verbosity > 1) printf("discarding older email: %s (%ld/%ld)\n", sdata->filename, sdata->sent, data->import->after);
+         rc = ERR_DISCARDED;
+      }
+      else if(data->import->before > 0 && sdata->sent > data->import->before){
+         if(cfg->verbosity > 1) printf("discarding newer email: %s (%ld/%ld)\n", sdata->filename, sdata->sent, data->import->before);
+         rc = ERR_DISCARDED;
+      }
       else {
+         // When importing emails, we should add the retention value (later) to the original sent value
+         sdata->retained = sdata->sent;
+
+         // backup original value of data->folder
+         int folder = data->folder;
          rc = process_message(sdata, &state, data, cfg);
+         data->folder = folder;
          unlink(state.message_id_hash);
       }
    }
@@ -106,36 +121,40 @@ int import_message(struct session_data *sdata, struct data *data, struct config
 
    switch(rc) {
       case OK:
-                        bzero(&counters, sizeof(counters));
-                        counters.c_rcvd = 1;
-                        counters.c_size += sdata->tot_len;
-                        counters.c_stored_size = sdata->stored_len;
-                        update_counters(sdata, data, &counters, cfg);
+                        counters->c_rcvd++;
+                        counters->c_size += sdata->tot_len;
+                        counters->c_stored_size += sdata->stored_len;
 
                         break;
 
       case ERR_EXISTS:
                         rc = OK;
 
-                        bzero(&counters, sizeof(counters));
-                        counters.c_duplicate = 1;
-                        update_counters(sdata, data, &counters, cfg);
+                        counters->c_duplicate++;
 
                         if(data->quiet == 0) printf("duplicate: %s (duplicate id: %llu)\n", data->import->filename, sdata->duplicate_id);
                         break;
 
+      case ERR_DISCARDED:
+                        rc = OK;
+
+                        counters->c_ignore++;
+
+                        break;
+
       default:
                         printf("failed to import: %s (id: %s)\n", data->import->filename, sdata->ttmpfile);
                         break;
-   } 
+   }
 
    if(rc != OK && data->import->failed_folder){
-      p = strrchr(data->import->filename, '/');
+      char *p = strrchr(data->import->filename, '/');
       if(p)
          p++;
       else
          p = data->import->filename;
 
+      char newpath[SMALLBUFSIZE];
       snprintf(newpath, sizeof(newpath)-2, "%s/%s", data->import->failed_folder, p);
 
       if(rename(data->import->filename, newpath))
@@ -146,7 +165,28 @@ int import_message(struct session_data *sdata, struct data *data, struct config
 }
 
 
-int get_folder_id(struct session_data *sdata, struct data *data, char *foldername, int parent_id){
+int update_import_table(struct session_data *sdata, struct data *data) {
+   int ret=ERR, status=2, started=1;
+   struct sql sql;
+
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE) == ERR) return ret;
+
+   p_bind_init(&sql);
+
+   sql.sql[sql.pos] = (char *)&(started); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&(status); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&(data->import->tot_msgs); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&(data->import->table_id); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+
+   if(p_exec_stmt(sdata, &sql) == OK) ret = OK;
+
+   close_prepared_statement(&sql);
+
+   return ret;
+}
+
+
+int get_folder_id(struct session_data *sdata, char *foldername, int parent_id){
    int id=ERR_FOLDER;
    struct sql sql;
 
@@ -174,7 +214,7 @@ int get_folder_id(struct session_data *sdata, struct data *data, char *foldernam
 }
 
 
-int add_new_folder(struct session_data *sdata, struct data *data, char *foldername, int parent_id){
+int add_new_folder(struct session_data *sdata, char *foldername, int parent_id){
    int id=ERR_FOLDER;
    struct sql sql;
 

src/import.h

@@ -6,21 +6,23 @@
    #define _IMPORT_H
 
 
-int import_message(struct session_data *sdata, struct data *data, struct config *cfg);
+int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+int update_import_table(struct session_data *sdata, struct data *data);
 
-int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct config *cfg);
-int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct config *cfg);
-int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct config *cfg);
-void import_from_pop3_server(struct session_data *sdata, struct data *data, struct config *cfg);
-int import_from_imap_server(struct session_data *sdata, struct data *data, struct config *cfg);
+int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg);
+int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 
 int connect_to_pop3_server(struct data *data);
-void process_pop3_emails(struct session_data *sdata, struct data *data, struct config *cfg);
+void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 
 int connect_to_imap_server(struct data *data);
 int list_folders(struct data *data);
-int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct config *cfg);
+int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 void send_imap_close(struct data *data);
 
-#endif /* _IMPORT_H */
+void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 
+#endif /* _IMPORT_H */

src/import_imap.c

@@ -22,7 +22,7 @@
 #include <piler.h>
 
 
-int import_from_imap_server(struct session_data *sdata, struct data *data, struct config *cfg){
+int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    int i, rc=ERR, ret=OK, skipmatch;
    char port_string[8], puf[SMALLBUFSIZE];
    struct addrinfo hints, *res;
@@ -73,7 +73,7 @@ int import_from_imap_server(struct session_data *sdata, struct data *data, struc
       q = data->imapfolders[i];
       while(q != NULL){
 
-         if(q && q->str && strlen(q->str) > 1){
+         if(q->str && strlen(q->str) > 1){
 
             skipmatch = 0;
 
@@ -88,7 +88,7 @@ int import_from_imap_server(struct session_data *sdata, struct data *data, struc
             else {
                if(data->quiet == 0) printf("processing folder: %s... ", (char *)q->str);
 
-               if(process_imap_folder(q->str, sdata, data, cfg) == ERR) ret = ERR;
+               if(process_imap_folder(q->str, sdata, data, counters, cfg) == ERR) ret = ERR;
             }
 
          }

src/import_mailbox.c

@@ -22,7 +22,7 @@
 #include <piler.h>
 
 
-int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct config *cfg){
+int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    FILE *F, *f=NULL;
    int rc=ERR, tot_msgs=0, ret=OK;
    char buf[MAXBUFSIZE];
@@ -44,7 +44,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *
          if(f){
             fclose(f);
             f = NULL;
-            rc = import_message(sdata, data, cfg);
+            rc = import_message(sdata, data, counters, cfg);
             if(rc == ERR){
                printf("error importing: '%s'\n", data->import->filename);
                ret = ERR;
@@ -64,7 +64,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *
 
    if(f){
       fclose(f);
-      rc = import_message(sdata, data, cfg);
+      rc = import_message(sdata, data, counters, cfg);
       if(rc == ERR){
          printf("ERROR: error importing: '%s'\n", data->import->filename);
          ret = ERR;
@@ -80,7 +80,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *
 }
 
 
-int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct config *cfg){
+int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    DIR *dir;
    struct dirent *de;
    int rc=ERR, ret=OK, i=0;
@@ -103,7 +103,7 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct dat
       if(stat(fname, &st) == 0){
          if(S_ISDIR(st.st_mode)){
             folder = data->folder;
-            rc = import_mbox_from_dir(fname, sdata, data, cfg);
+            rc = import_mbox_from_dir(fname, sdata, data, counters, cfg);
             data->folder = folder;
             if(rc == ERR) ret = ERR;
          }
@@ -111,9 +111,9 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct dat
 
             if(S_ISREG(st.st_mode)){
                if(i == 0 && data->recursive_folder_names == 1){
-                  folder = get_folder_id(sdata, data, fname, data->folder);
+                  folder = get_folder_id(sdata, fname, data->folder);
                   if(folder == ERR_FOLDER){
-                     folder = add_new_folder(sdata, data, fname, data->folder);
+                     folder = add_new_folder(sdata, fname, data->folder);
 
                      if(folder == ERR_FOLDER){
                         printf("error: cannot get/add folder '%s' to parent id: %d\n", fname, data->folder);
@@ -126,7 +126,7 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct dat
 
                }
 
-               rc = import_from_mailbox(fname, sdata, data, cfg);
+               rc = import_from_mailbox(fname, sdata, data, counters, cfg);
                if(rc == OK) (data->import->tot_msgs)++;
                else ret = ERR;
 

src/import_maildir.c

@@ -22,7 +22,7 @@
 #include <piler.h>
 
 
-int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct config *cfg){
+int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg){
    DIR *dir;
    struct dirent *de;
    int rc=ERR, ret=OK, i=0;
@@ -46,7 +46,7 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir
          if(S_ISDIR(st.st_mode)){
             folder = data->folder;
             snprintf(subdir, sizeof(subdir)-1, "%s", data->import->filename);
-            rc = import_from_maildir(sdata, data, subdir, cfg);
+            rc = import_from_maildir(sdata, data, subdir, counters, cfg);
             data->folder = folder;
             if(rc == ERR) ret = ERR;
          }
@@ -61,9 +61,9 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir
                      return ERR;
                   }
 
-                  folder = get_folder_id(sdata, data, p, data->folder);
+                  folder = get_folder_id(sdata, p, data->folder);
                   if(folder == ERR_FOLDER){
-                     folder = add_new_folder(sdata, data, p, data->folder);
+                     folder = add_new_folder(sdata, p, data->folder);
 
                      if(folder == ERR_FOLDER){
                         printf("error: cannot get/add folder '%s' to parent id: %d\n", p, data->folder);
@@ -76,14 +76,14 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir
 
                }
 
-               rc = import_message(sdata, data, cfg);
+               rc = import_message(sdata, data, counters, cfg);
 
                if(rc == OK) (data->import->tot_msgs)++;
                else if(rc == ERR){
                   printf("ERROR: error importing: '%s'\n", data->import->filename);
                   ret = ERR;
                }
- 
+
                if(data->import->remove_after_import == 1 && rc != ERR) unlink(data->import->filename);
 
                i++;
@@ -103,5 +103,9 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir
    }
    closedir(dir);
 
+   if(data->import->table_id > 0){
+      update_import_table(sdata, data);
+   }
+
    return ret;
 }

src/import_pilerexport.c

@@ -0,0 +1,114 @@
+/*
+ * import_pop3.c
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <getopt.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+void import_the_file(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   close(data->import->fd);
+   data->import->fd = -1;
+
+   if(import_message(sdata, data, counters, cfg) != ERR){
+      unlink(data->import->filename);
+   }
+}
+
+
+void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+
+   if(!strcmp(buf, PILEREXPORT_BEGIN_MARK)){
+      if((*count) > 0){
+         import_the_file(sdata, data, counters, cfg);
+      }
+
+      (*count)++;
+
+      snprintf(data->import->filename, SMALLBUFSIZE-1, "import-%llu", *count);
+
+      data->import->fd = open(data->import->filename, O_CREAT|O_WRONLY|O_TRUNC, S_IRUSR|S_IWUSR);
+      if(data->import->fd == -1){
+         // Do some error handling
+         printf("error: cannot open %s\n", data->import->filename);
+      }
+
+   } else {
+      if(write(data->import->fd, buf, buflen) != buflen){
+         printf("error: didnt write %d bytes\n", buflen);
+      }
+   }
+}
+
+
+void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   int n, rc, nullbyte, savedlen=0, puflen;
+   uint64 count=0;
+   char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE];
+
+   memset(savedbuf, 0, sizeof(savedbuf));
+
+   data->import->fd = -1;
+
+   do {
+      memset(buf, 0, sizeof(buf));
+      n = fread(buf, 1, sizeof(buf)-1, stdin);
+
+      int remaininglen = n;
+
+      if(savedlen > 0){
+         memset(copybuf, 0, sizeof(copybuf));
+
+         memcpy(copybuf, savedbuf, savedlen);
+         memcpy(&copybuf[savedlen], buf, n);
+
+         remaininglen += savedlen;
+
+         savedlen = 0;
+         memset(savedbuf, 0, sizeof(savedbuf));
+
+         p = &copybuf[0];
+      }
+      else {
+         p = &buf[0];
+      }
+
+      do {
+         puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf), &rc, &nullbyte);
+         p += puflen;
+         remaininglen -= puflen;
+
+         if(puflen > 0){
+            if(rc == OK){
+               process_buffer(puf, puflen, &count, sdata, data, counters, cfg);
+            }
+            else {
+               snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
+               savedlen = puflen;
+            }
+         }
+
+      } while(puflen > 0);
+
+   } while(n > 0);
+
+   if(data->import->fd != -1){
+      import_the_file(sdata, data, counters, cfg);
+   }
+}

src/import_pop3.c

@@ -22,7 +22,7 @@
 #include <piler.h>
 
 
-void import_from_pop3_server(struct session_data *sdata, struct data *data, struct config *cfg){
+void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    int rc;
    char port_string[8];
    struct addrinfo hints, *res;
@@ -58,7 +58,7 @@ void import_from_pop3_server(struct session_data *sdata, struct data *data, stru
       goto ENDE_POP3;
    }
 
-   process_pop3_emails(sdata, data, cfg);
+   process_pop3_emails(sdata, data, counters, cfg);
 
    close_connection(data->net);
 

src/list.c

@@ -1,97 +0,0 @@
-/*
- * list.c, SJ
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include "list.h"
-#include "config.h"
-
-
-int append_list(struct list **list, char *p){
-   struct list *q, *t, *u=NULL;
-
-   q = *list;
-
-   while(q){
-      if(strcmp(q->s, p) == 0)
-         return 0;
-
-      u = q;
-      q = q->r;
-   }
-
-   t = create_list_item(p);
-   if(t){
-      if(*list == NULL)
-         *list = t;
-      else if(u)
-         u->r = t;
-
-      return 1;
-   }
-
-   return -1;
-}
-
-
-struct list *create_list_item(char *s){
-   struct list *h=NULL;
-
-   if((h = malloc(sizeof(struct list))) == NULL)
-      return NULL;
-
-   snprintf(h->s, SMALLBUFSIZE-1, "%s", s);
-   h->r = NULL;
-
-   return h;
-}
-
-
-int is_string_on_list(struct list *list, char *s){
-   struct list *p;
-
-   p = list;
-
-   while(p != NULL){
-      if(strcmp(p->s, s) == 0) return 1;
-      p = p->r;
-   }
-
-   return 0;
-}
-
-
-int is_item_on_string(struct list *list, char *s){
-   struct list *p;
-
-   p = list;
-
-   while(p != NULL){
-      if(strstr(s, p->s)) return 1;
-      p = p->r;
-   }
-
-   return 0;
-}
-
-
-void free_list(struct list *list){
-   struct list *p, *q;
-
-   p = list;
-
-   while(p != NULL){
-      q = p->r;
-
-      if(p)
-         free(p);
-
-      p = q;
-   }
-}
-
-
-

src/list.h

@@ -1,17 +0,0 @@
-/*
- * list.h, SJ
- */
-
-#ifndef _LIST_H
- #define _LIST_H
-
-#include "defs.h"
-
-int append_list(struct list **list, char *p);
-struct list *create_list_item(char *s);
-int is_string_on_list(struct list *list, char *s);
-int is_item_on_string(struct list *list, char *s);
-void free_list(struct list *list);
-
-#endif /* _LIST_H */
-

src/memc.c

@@ -59,7 +59,7 @@ void memcached_init(struct memcached_server *ptr, char *server_ip, int server_po
 
 
 int set_socket_options(struct memcached_server *ptr){
-   int error, flag=1, flags, rval;
+   int error, flag=1, flags;
    struct timeval waittime;
    struct linger linger;
 
@@ -128,8 +128,7 @@ int set_socket_options(struct memcached_server *ptr){
 
 
    if((flags & O_NONBLOCK) == 0){
-      rval = fcntl(ptr->fd, F_SETFL, flags | O_NONBLOCK);
-      if(rval == -1) return MEMCACHED_FAILURE;
+      if(fcntl(ptr->fd, F_SETFL, flags | O_NONBLOCK) == -1) return MEMCACHED_FAILURE;
    }
 
    return MEMCACHED_SUCCESS;
@@ -197,7 +196,7 @@ int memcached_connect(struct memcached_server *ptr){
         {
           (void)close(ptr->fd);
           ptr->fd= -1;
-          
+
         }
 
       return MEMCACHED_FAILURE;
@@ -217,12 +216,13 @@ int memcached_shutdown(struct memcached_server *ptr){
 }
 
 
-int memcached_add(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry){
+int memcached_add(struct memcached_server *ptr, char *cmd, char *key, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry){
    int len=0;
 
    if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
 
-   snprintf(ptr->buf, MAXBUFSIZE-1, "add %s %d %ld %d \r\n", key, flags, expiry, valuelen);
+   // cmd could be either 'add' or 'set'
+   snprintf(ptr->buf, MAXBUFSIZE-1, "%s %s %u %lu %u \r\n", cmd, key, flags, expiry, valuelen);
    len = strlen(ptr->buf);
 
    strncat(ptr->buf, value, MAXBUFSIZE-strlen(ptr->buf)-1);
@@ -240,30 +240,7 @@ int memcached_add(struct memcached_server *ptr, char *key, unsigned int keylen,
 }
 
 
-int memcached_set(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry){
-   int len=0;
-
-   if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
-
-   snprintf(ptr->buf, MAXBUFSIZE-1, "set %s %d %ld %d \r\n", key, flags, expiry, valuelen);
-   len = strlen(ptr->buf);
-
-   strncat(ptr->buf, value, MAXBUFSIZE-strlen(ptr->buf)-1);
-   strncat(ptr->buf, "\r\n", MAXBUFSIZE-strlen(ptr->buf)-1);
-
-   len += valuelen + 2;
-
-   send(ptr->fd, ptr->buf, len, 0);
-
-   ptr->last_read_bytes = __recvtimeout(ptr->fd, ptr->buf, MAXBUFSIZE, ptr->rcv_timeout);
-
-   if(strcmp("STORED\r\n", ptr->buf)) return MEMCACHED_FAILURE;
-
-   return MEMCACHED_SUCCESS;
-}
-
-
-int memcached_increment(struct memcached_server *ptr, char *key, unsigned int keylen, unsigned long long value, unsigned long long *result){
+int memcached_increment(struct memcached_server *ptr, char *key, unsigned long long value, unsigned long long *result){
    char *p;
 
    if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
@@ -285,52 +262,6 @@ int memcached_increment(struct memcached_server *ptr, char *key, unsigned int ke
 }
 
 
-char *memcached_get(struct memcached_server *ptr, char *key, unsigned int *len, unsigned int *flags){
-   char *p;
-
-   if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return NULL;
-
-   snprintf(ptr->buf, MAXBUFSIZE, "get %s \r\n", key);
-   send(ptr->fd, ptr->buf, strlen(ptr->buf), 0);
-
-   ptr->last_read_bytes = __recvtimeout(ptr->fd, ptr->buf, MAXBUFSIZE, ptr->rcv_timeout);
-
-   if(ptr->last_read_bytes <= 0){
-      memcached_shutdown(ptr);
-      return NULL;
-   }
-
-
-   if(ptr->last_read_bytes < 10) return NULL;
-
-   if(strncmp("VALUE ", ptr->buf, 6)) return NULL;
-
-   p = strchr(ptr->buf, '\r');
-   if(!p) return NULL;
-   *p = '\0';
-   ptr->result = p + 2;
-
-   p = strrchr(ptr->buf + 6, ' ');
-   if(!p) return NULL;
-   *len = atoi(p+1);
-   *p = '\0';
-
-
-   p = strrchr(ptr->buf + 6, ' ');
-   if(!p) return NULL;
-   *flags = atoi(p+1);
-   *p = '\0';
-
- 
-   p = strchr(ptr->result, '\r');
-   if(!p) return NULL;
-
-   *p = '\0';
-
-   return ptr->result;
-}
-
-
 int memcached_mget(struct memcached_server *ptr, char *key){
 
    if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
@@ -394,5 +325,3 @@ char *memcached_fetch_result(struct memcached_server *ptr, char *key, char *valu
 
    return p;
 }
-
-

src/memc.h

@@ -9,9 +9,8 @@ void memcached_init(struct memcached_server *ptr, char *server_ip, int server_po
 int set_socket_options(struct memcached_server *ptr);
 int memcached_connect(struct memcached_server *ptr);
 int memcached_shutdown(struct memcached_server *ptr);
-int memcached_add(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry);
-int memcached_set(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry);
-int memcached_increment(struct memcached_server *ptr, char *key, unsigned int keylen, unsigned long long value, unsigned long long *result);
+int memcached_add(struct memcached_server *ptr, char *cmd, char *key, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry);
+int memcached_increment(struct memcached_server *ptr, char *key, unsigned long long value, unsigned long long *result);
 char *memcached_get(struct memcached_server *ptr, char *key, unsigned int *len, unsigned int *flags);
 int memcached_mget(struct memcached_server *ptr, char *key);
 char *memcached_fetch_result(struct memcached_server *ptr, char *key, char *value, unsigned int *flags);

src/message.c

@@ -10,6 +10,7 @@
 #include <sys/mman.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
+#include <errno.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <syslog.h>
@@ -19,7 +20,7 @@
 
 int store_index_data(struct session_data *sdata, struct parser_state *state, struct data *data, uint64 id, struct config *cfg){
    int rc=ERR;
-   char *subj;
+   char *subj, *sender=state->b_from, *sender_domain=state->b_from_domain;
    struct sql sql;
 
    if(data->folder == 0){
@@ -29,43 +30,108 @@ int store_index_data(struct session_data *sdata, struct parser_state *state, str
    subj = state->b_subject;
    if(*subj == ' ') subj++;
 
-
-   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc;
-
-
    fix_email_address_for_sphinx(state->b_from);
+   fix_email_address_for_sphinx(state->b_sender);
    fix_email_address_for_sphinx(state->b_to);
    fix_email_address_for_sphinx(state->b_from_domain);
+   fix_email_address_for_sphinx(state->b_sender_domain);
    fix_email_address_for_sphinx(state->b_to_domain);
 
+   if(state->b_sender_domain[0]){
+      sender = state->b_sender;
+      sender_domain = state->b_sender_domain;
+   }
 
-   p_bind_init(&sql);
+   if(cfg->rtindex){
+      // Manticore doesn't support prepared statements using sphinxQL
+      // so we have to go through a painful query assembly escaping
+      // the untrusted input
+      //
+      char a[4*MAXBUFSIZE+4*SMALLBUFSIZE];
+      char *query=NULL;
 
-   sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
-   sql.sql[sql.pos] = state->b_from; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = state->b_to; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = state->b_from_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = state->b_to_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = state->b_body; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&sdata->sent; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&sdata->tot_len; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&sdata->direction; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&data->folder; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&state->n_attachments; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = sdata->attachments; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      snprintf(a, sizeof(a)-1, "REPLACE INTO %s (id, arrived, sent, size, direction, folder, attachments, attachment_types, sender, rcpt, senderdomain, rcptdomain, subject, body) VALUES (%llu,%ld,%ld,%d,%d,%d,%d,'%s','", cfg->sphxdb, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, data->folder, state->n_attachments, sdata->attachments);
 
-   if(p_exec_stmt(sdata, &sql) == OK) rc = OK;
-   else syslog(LOG_PRIORITY, "ERROR: %s failed to store index data for id=%llu, sql_errno=%d", sdata->ttmpfile, id, sdata->sql_errno);
+      int ret = append_string_to_buffer(&query, a);
 
-   close_prepared_statement(&sql);
+      unsigned long len = strlen(sender);
+      char *s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, sender, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(state->b_to);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, state->b_to, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(sender_domain);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, sender_domain, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(state->b_to_domain);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, state->b_to_domain, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(subj);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, subj, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(state->b_body);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, state->b_body, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "')");
+
+      if(ret == 0 && mysql_real_query(&(sdata->sphx), query, strlen(query)) == OK) rc = OK;
+      else syslog(LOG_PRIORITY, "ERROR: %s failed to store index data for id=%llu, errno=%d, append ret=%d", sdata->ttmpfile, id, mysql_errno(&(sdata->sphx)), ret);
+
+      free(query);
+   }
+   else {
+      if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc;
+
+      p_bind_init(&sql);
+
+      sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
+      sql.sql[sql.pos] = sender; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = state->b_to; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = state->b_to_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = state->b_body; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = (char *)&sdata->sent; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = (char *)&sdata->tot_len; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = (char *)&sdata->direction; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = (char *)&data->folder; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = (char *)&state->n_attachments; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = sdata->attachments; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+
+      if(p_exec_stmt(sdata, &sql) == OK) rc = OK;
+      else syslog(LOG_PRIORITY, "ERROR: %s failed to store index data for id=%llu, sql_errno=%d", sdata->ttmpfile, id, sdata->sql_errno);
+
+      close_prepared_statement(&sql);
+   }
 
    return rc;
 }
 
 
-uint64 get_metaid_by_messageid(struct session_data *sdata, struct data *data, char *message_id, char *piler_id){
+uint64 get_metaid_by_messageid(struct session_data *sdata, char *message_id, char *piler_id){
    uint64 id=0;
    struct sql sql;
 
@@ -94,7 +160,7 @@ uint64 get_metaid_by_messageid(struct session_data *sdata, struct data *data, ch
 }
 
 
-int store_recipients(struct session_data *sdata, struct data *data, char *to, uint64 id, struct config *cfg){
+int store_recipients(struct session_data *sdata, char *to, uint64 id, struct config *cfg){
    int rc=OK, n=0;
    char *p, *q, puf[SMALLBUFSIZE];
    struct sql sql;
@@ -135,15 +201,6 @@ int store_recipients(struct session_data *sdata, struct data *data, char *to, ui
 }
 
 
-void remove_recipients(struct session_data *sdata, uint64 id){
-   char s[SMALLBUFSIZE];
-
-   snprintf(s, sizeof(s)-1, "DELETE FROM " SQL_RECIPIENT_TABLE " WHERE id=%llu", id);
-
-   p_query(sdata, s);
-}
-
-
 int store_folder_id(struct session_data *sdata, struct data *data, uint64 id, struct config *cfg){
    int rc=ERR;
    struct sql sql;
@@ -166,7 +223,7 @@ int store_folder_id(struct session_data *sdata, struct data *data, uint64 id, st
 }
 
 
-int update_metadata_reference(struct session_data *sdata, struct parser_state *state, struct data *data, char *ref, struct config *cfg){
+int update_metadata_reference(struct session_data *sdata, struct parser_state *state, char *ref, struct config *cfg){
    int ret = ERR;
    struct sql sql;
 
@@ -188,40 +245,64 @@ int update_metadata_reference(struct session_data *sdata, struct parser_state *s
 
 
 int store_meta_data(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
-   int rc=ERR, result;
-   char *subj, *p, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1];
+   int rc=ERR;
+   char *subj, *sender, *sender_domain, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1];
    uint64 id=0;
    struct sql sql;
 
    subj = state->b_subject;
    if(*subj == ' ') subj++;
 
-   snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, state->b_from, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
+   if(state->b_sender_domain[0]){
+      sender = state->b_sender;
+      sender_domain = state->b_sender_domain;
+      get_first_email_address_from_string(state->b_sender, s2, sizeof(s2));
+   } else {
+      sender = state->b_from;
+      sender_domain = state->b_from_domain;
+      get_first_email_address_from_string(state->b_from, s2, sizeof(s2));
+   }
 
-   digest_string(s, &vcode[0]);
+   snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, sender, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
+
+   digest_string("sha256", s, &vcode[0]);
 
    memset(ref, 0, sizeof(ref));
    if(strlen(state->reference) > 10){
-      digest_string(state->reference, &ref[0]);
-      update_metadata_reference(sdata, state, data, &ref[0], cfg);
+      digest_string("sha256", state->reference, &ref[0]);
+      update_metadata_reference(sdata, state, &ref[0], cfg);
+   }
+   else if(state->reference[0] == 0){
+      // during import, the order of messages is often random
+      // check if this is a message which is already referenced
+      uint64 count=0;
+
+      digest_string("sha256", state->message_id, &ref[0]);
+      if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_METADATA_REFERENCE) != ERR){
+         p_bind_init(&sql);
+
+         sql.sql[sql.pos] = &ref[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+
+         if(p_exec_stmt(sdata, &sql) == OK){
+            p_bind_init(&sql);
+
+            sql.sql[sql.pos] = (char *)&count; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+            p_store_results(&sql);
+            p_fetch_results(&sql);
+            p_free_results(&sql);
+         }
+      }
+
+      close_prepared_statement(&sql);
+
+      // no reference yet
+      if(count <= 0)
+         ref[0] = 0;
    }
 
 
    if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR;
 
-   memset(s2, 0, sizeof(s2));
-
-   p = state->b_from;
-   do {
-      memset(s2, 0, sizeof(s2));
-      p = split(p, ' ', s2, sizeof(s2)-1, &result);
-
-      if(s2[0] == '\0') continue;
-
-      if(does_it_seem_like_an_email_address(s2) == 1){ break; }
-   } while(p);
-
-
    if(strlen(state->b_to) < 5){
       snprintf(state->b_to, SMALLBUFSIZE-1, "undisclosed-recipients@no.domain");
    }
@@ -230,7 +311,7 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru
    p_bind_init(&sql);
 
    sql.sql[sql.pos] = &s2[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = state->b_from_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
    sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
    sql.sql[sql.pos] = (char *)&sdata->spam_message; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
    sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
@@ -250,7 +331,7 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru
    if(p_exec_stmt(sdata, &sql) == OK){
       id = p_get_insert_id(&sql);
 
-      if(store_recipients(sdata, data, state->b_to, id, cfg) == OK){
+      if(store_recipients(sdata, state->b_to, id, cfg) == OK){
 
          if(store_index_data(sdata, state, data, id, cfg) == OK) rc = OK;
 
@@ -313,20 +394,20 @@ void remove_stripped_attachments(struct parser_state *state){
 }
 
 
-int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
+int is_duplicated_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
    int fd;
    char piler_id[SMALLBUFSIZE];
 
    /* discard if existing message_id */
 
-   sdata->duplicate_id = get_metaid_by_messageid(sdata, data, state->message_id, piler_id);
+   sdata->duplicate_id = get_metaid_by_messageid(sdata, state->message_id, piler_id);
 
    if(sdata->duplicate_id > 0){
       remove_stripped_attachments(state);
 
-      if(strlen(state->b_journal_to) > 0){
+      if(sdata->restored_copy == 0 && strlen(state->b_journal_to) > 0){
          if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to add journal rcpt (%s) to id=%llu for message-id: '%s'", sdata->ttmpfile, state->b_journal_to, sdata->duplicate_id, state->message_id);
-         store_recipients(sdata, data, state->b_journal_to, sdata->duplicate_id, cfg);
+         store_recipients(sdata, state->b_journal_to, sdata->duplicate_id, cfg);
       }
 
       return ERR_EXISTS;
@@ -335,7 +416,7 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru
    fd = open(state->message_id_hash, O_CREAT|O_EXCL, S_IRUSR|S_IWUSR);
    if(fd == -1){
       remove_stripped_attachments(state);
-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: touch %s FAILED (%s)", sdata->ttmpfile, state->message_id_hash, state->message_id);
+      syslog(LOG_PRIORITY, "%s: touch %s FAILED (%s), error: %s", sdata->ttmpfile, state->message_id_hash, state->message_id, strerror(errno));
       return ERR_EXISTS;
    }
    close(fd);
@@ -343,12 +424,13 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru
    if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: touch %s OK (%s)", sdata->ttmpfile, state->message_id_hash, state->message_id);
 
 
-
    if(cfg->mmap_dedup_test == 1 && data->dedup != MAP_FAILED && data->child_serial >= 0 && data->child_serial < MAXCHILDREN){
 
       if(strstr(data->dedup, state->message_id_hash)){
-         if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: dedup string: %s", sdata->ttmpfile, data->dedup);
-         if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: message-id-hash=%s, serial=%d", sdata->ttmpfile, state->message_id_hash, data->child_serial);
+         if(cfg->verbosity >= _LOG_DEBUG){
+            syslog(LOG_INFO, "%s: dedup string: %s", sdata->ttmpfile, data->dedup);
+            syslog(LOG_INFO, "%s: message-id-hash=%s, serial=%d", sdata->ttmpfile, state->message_id_hash, data->child_serial);
+         }
 
          remove_stripped_attachments(state);
          return ERR_EXISTS;
@@ -357,11 +439,19 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru
       memcpy(data->dedup + data->child_serial*DIGEST_LENGTH*2, state->message_id_hash, DIGEST_LENGTH*2);
    }
 
+   return OK;
+}
+
+
+int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
+
+   if(is_duplicated_message(sdata, state, data, cfg) == ERR_EXISTS)
+      return ERR_EXISTS;
 
    sdata->retained += query_retain_period(data, state, sdata->tot_len, sdata->spam_message, cfg);
 
 
-   if(state->n_attachments > 0 && store_attachments(sdata, state, data, cfg) == ERR) return ERR;
+   if(state->n_attachments > 0 && store_attachments(sdata, state, cfg) == ERR) return ERR;
 
 
    if(store_file(sdata, sdata->tmpframe, 0, cfg) == 0){

src/misc.c

@@ -63,6 +63,10 @@ void get_extractor_list(){
       printf("%s ", HAVE_TNEF);
    #endif
 
+   #ifdef HAVE_ZIP
+      printf("libzip ");
+   #endif
+
    printf("\n\n");
 }
 
@@ -73,7 +77,7 @@ void __fatal(char *s){
 }
 
 /*
- * calculate the difference betwwen two timevals in [usec]
+ * calculate the difference between two timevals in [usec]
  */
 
 long tvdiff(struct timeval a, struct timeval b){
@@ -84,81 +88,6 @@ long tvdiff(struct timeval a, struct timeval b){
 }
 
 
-/*
- * search something in a buffer
- */
-
-int searchStringInBuffer(char *s, int len1, char *what, int len2){
-   int i, k, r;
-
-   for(i=0; i<len1; i++){
-      r = 0;
-
-      for(k=0; k<len2; k++){
-         if(*(s+i+k) == *(what+k))
-            r++;
-      }
-
-      if(r == len2)
-         return i;
-   }
-
-   return -1;
-}
-
-
-int search_char_backward(char *buf, int buflen, char c){
-   int n, m;
-
-   m = buflen - 1 - 5;
-   if(m < 0) m = 0;
-
-   for(n=m; n<buflen; n++){
-      if(*(buf + n) == c){
-         return n;
-      }
-   }
-   return -1;
-}
-
-
-/*
- * count a character in buffer
- */
-
-int countCharacterInBuffer(char *p, char c){
-   int i=0;
-
-   for(; *p; p++){
-      if(*p == c)
-         i++;
-   }
-
-   return i;
-}
-
-
-void replaceCharacterInBuffer(char *p, char from, char to){
-   int i, k=0;
-
-   for(i=0; i<strlen(p); i++){
-      if(p[i] == from){
-         if(to > 0){
-            p[k] = to;
-            k++;
-         }
-      }
-      else {
-         p[k] = p[i];
-         k++;
-      }
-
-   }
-
-   p[k] = '\0';
-}
-
-
 /*
  * split a string by a character as delimiter
  */
@@ -214,10 +143,8 @@ char *split_str(char *row, char *what, char *s, int size){
       r += strlen(what);
    }
 
-   if(s != NULL){
-      strncpy(s, row, len);
-      s[len] = '\0';
-   }
+   strncpy(s, row, len);
+   s[len] = '\0';
 
    return r;
 }
@@ -248,8 +175,7 @@ int trimBuffer(char *s){
 
 
 int extract_verp_address(char *email){
-   char *p, *p1, *p2;
-   char puf[SMALLBUFSIZE];
+   char *p1;
 
    // a VERP address is like archive+user=domain.com@myarchive.local
 
@@ -259,13 +185,14 @@ int extract_verp_address(char *email){
 
    p1 = strchr(email, '+');
    if(p1){
-      p2 = strchr(p1, '@');
+      char *p2 = strchr(p1, '@');
       if(p2 && p2 > p1 + 2){
          if(strchr(p1+1, '=')){
+            char puf[SMALLBUFSIZE];
             memset(puf, 0, sizeof(puf));
 
             memcpy(&puf[0], p1+1, p2-p1-1);
-            p = strchr(puf, '=');
+            char *p = strchr(puf, '=');
             if(p) *p = '@';
             strcpy(email, puf);
          }
@@ -302,19 +229,15 @@ int extractEmail(char *rawmail, char *email){
  * using the rand() function if not possible
  */
 
-void make_random_string(char *buf, int buflen){
-   int i, len, fd;
-   int urandom=0;
-   static char alphanum[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
-   unsigned char s[QUEUE_ID_LEN];
-
-   len = strlen(alphanum);
+void make_random_string(unsigned char *buf, int buflen){
+   const char alphanum[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+   int i, fd, urandom=0, len = sizeof(alphanum)-1;
 
    fd = open(RANDOM_POOL, O_RDONLY);
    if(fd != -1){
-      if(readFromEntropyPool(fd, s, sizeof(s)) == sizeof(s)){
-         for(i=0; i<QUEUE_ID_LEN; i++){
-            *(buf+i) = alphanum[s[i] % len];
+      if(readFromEntropyPool(fd, buf, buflen) == buflen){
+         for(i=0; i<buflen; i++){
+            *(buf+i) = alphanum[*(buf+i) % len];
          }
 
          urandom = 1;
@@ -339,11 +262,13 @@ void create_id(char *id, unsigned char server_id){
 
    get_random_bytes(buf, RND_STR_LEN/2, server_id);
 
+   // New encryption scheme using AES-256
+   buf[0] = 0x50;
+
    for(i=0; i < RND_STR_LEN/2; i++){
       sprintf(id, "%02x", buf[i]);
       id += 2;
    }
-
 }
 
 
@@ -361,7 +286,7 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
    taia_now(&now);
    taia_pack(nowpack, &now);
 
-   memcpy(buf, nowpack, 12);
+   memcpy(buf, nowpack, 12); //-V512
 
    fd = open(RANDOM_POOL, O_RDONLY);
    if(fd == -1) return ret;
@@ -371,7 +296,7 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
    if(readFromEntropyPool(fd, buf+12+1, len-12-1) != len-12-1){
       syslog(LOG_PRIORITY, "%s: %s", ERR_CANNOT_READ_FROM_POOL, RANDOM_POOL);
    }
-   
+
    close(fd);
    return ret;
 }
@@ -383,10 +308,10 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
 
 int readFromEntropyPool(int fd, void *_s, ssize_t n){
    char *s = _s;
-   ssize_t res, pos = 0;
+   ssize_t pos = 0;
 
    while(n > pos){
-      res = read(fd, s + pos, n - pos);
+      ssize_t res = read(fd, s + pos, n - pos);
       switch(res){
          case  -1: continue;
          case   0: return res;
@@ -444,28 +369,28 @@ int ssl_want_retry(SSL *ssl, int ret, int timeout){
    i = SSL_get_error(ssl, ret);
    if(i == SSL_ERROR_NONE)
       return 1;
- 
+
    tv.tv_sec = timeout/1000;
    tv.tv_usec = 0;
    FD_ZERO(&fds);
- 
+
    switch(i){
       case SSL_ERROR_WANT_READ: // pause until the socket is readable
           sock = SSL_get_rfd(ssl);
           FD_SET(sock, &fds);
           i = select(sock+1, &fds, 0, 0, &tv);
           break;
- 
+
       case SSL_ERROR_WANT_WRITE: // pause until the socket is writeable
          sock = SSL_get_wfd(ssl);
          FD_SET(sock, &fds);
          i = select(sock+1, 0, &fds, 0, &tv);
          break;
- 
+
       case SSL_ERROR_ZERO_RETURN: // the sock closed, just return quietly
          i = 0;
          break;
- 
+
       default: // ERROR - unexpected error code
          i = -1;
          break;
@@ -639,12 +564,6 @@ void strtolower(char *s){
 }
 
 
-void *get_in_addr(struct sockaddr *sa){
-   if(sa->sa_family == AF_INET) return &(((struct sockaddr_in*)sa)->sin_addr);
-   return &(((struct sockaddr_in6*)sa)->sin6_addr);
-}
-
-
 int make_socket_non_blocking(int fd){
    int flags, s;
 
@@ -733,9 +652,86 @@ void move_email(struct smtp_session *session){
 
    snprintf(buf, sizeof(buf)-1, "%d/%s", session->ttmpfile[0] % session->cfg->number_of_worker_processes, session->ttmpfile);
 
-   if(rename(session->ttmpfile, buf)){
-      syslog(LOG_PRIORITY, "ERROR: couldn't rename %s to %s", session->ttmpfile, buf);
+   if(session->nullbyte){
+      snprintf(buf, sizeof(buf)-1, "%s/%s", ERROR_DIR, session->ttmpfile);
+      syslog(LOG_PRIORITY, "ERROR: %s contains an invalid NUL-byte, moving it to %s", session->ttmpfile, ERROR_DIR);
    }
+
+   if(rename(session->ttmpfile, buf)){
+      syslog(LOG_PRIORITY, "ERROR: couldn't rename %s to %s (reason: %s)", session->ttmpfile, buf, strerror(errno));
+   }
+}
+
+
+int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte){
+   int i=0;
+
+   *rc = ERR;
+   *nullbyte = 0;
+
+   memset(buf, 0, buflen);
+
+   for(int j=0; j<slen; j++){
+      if(i<buflen-2){
+         if(*(s+j) == 0){
+            *nullbyte = 1;
+         }
+
+         buf[i] = *(s+j);
+         i++;
+
+         if(*(s+j) == c){
+            *rc = OK;
+            break;
+         }
+      }
+      else {
+         break;
+      }
+   }
+
+   return i;
+}
+
+
+int init_ssl_to_server(struct data *data){
+   int n;
+   X509* server_cert;
+   char *str;
+
+   SSL_library_init();
+   SSL_load_error_strings();
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+   data->net->ctx = SSL_CTX_new(TLSv1_2_client_method());
+#else
+   data->net->ctx = SSL_CTX_new(TLS_client_method());
+#endif
+   CHK_NULL(data->net->ctx, "internal SSL error");
+
+   data->net->ssl = SSL_new(data->net->ctx);
+   CHK_NULL(data->net->ssl, "internal ssl error");
+
+   SSL_set_fd(data->net->ssl, data->net->socket);
+   n = SSL_connect(data->net->ssl);
+   CHK_SSL(n, "internal ssl error");
+
+   //printf("Cipher: %s\n", SSL_get_cipher(data->net->ssl));
+
+   server_cert = SSL_get_peer_certificate(data->net->ssl);
+   CHK_NULL(server_cert, "server cert error");
+
+   str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0);
+   CHK_NULL(str, "error in server cert");
+   OPENSSL_free(str);
+
+   str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0);
+   CHK_NULL(str, "error in server cert");
+   OPENSSL_free(str);
+
+   X509_free(server_cert);
+
+   return OK;
 }
 
 
@@ -759,3 +755,55 @@ char *strcasestr(const char *s, const char *find){
    return((char*)s);
 }
 #endif
+
+
+int append_string_to_buffer(char **buffer, char *str){
+   int arglen;
+   char *s=NULL;
+
+   arglen = strlen(str);
+
+   if(!*buffer){
+      *buffer = malloc(arglen+1);
+      memset(*buffer, 0, arglen+1);
+      memcpy(*buffer, str, arglen);
+   }
+   else {
+      int len = strlen(*buffer);
+      s = realloc(*buffer, len + arglen+1);
+      if(!s) return 1;
+
+      *buffer = s;
+
+      memset(*buffer+len, 0, arglen+1);
+      memcpy(*buffer+len, str, arglen);
+   }
+
+   return 0;
+}
+
+
+int get_size_from_smtp_mail_from(char *s){
+   int size=0;
+   char *p;
+
+   p = strcasestr(s, "SIZE=");
+   if(p){
+      p += strlen("SIZE=");
+      char *q = p;
+      for(; *q; q++){
+         if(isspace(*q)) break;
+      }
+
+      // We extract max. 9 characters, which is just under 1GB
+      // and not overflowing an int variable
+      if(q - p <= 9){
+         char c = *q;
+         *q = '\0';
+         size = atoi(p);
+         *q = c;
+      }
+   }
+
+   return size;
+}

src/misc.h

@@ -19,16 +19,12 @@ int get_build();
 void get_extractor_list();
 void __fatal(char *s);
 long tvdiff(struct timeval a, struct timeval b);
-int searchStringInBuffer(char *s, int len1, char *what, int len2);
-int search_char_backward(char *buf, int buflen, char c);
-int countCharacterInBuffer(char *p, char c);
-void replaceCharacterInBuffer(char *p, char from, char to);
 char *split(char *str, int ch, char *buf, int buflen, int *result);
 char *split_str(char *row, char *what, char *s, int size);
 int trimBuffer(char *s);
 int extractEmail(char *rawmail, char *email);
 int extract_verp_address(char *email);
-void make_random_string(char *buf, int buflen);
+void make_random_string(unsigned char *buf, int buflen);
 void create_id(char *id, unsigned char server_id);
 int get_random_bytes(unsigned char *buf, int len, unsigned char server_id);
 int readFromEntropyPool(int fd, void *_s, ssize_t n);
@@ -44,16 +40,21 @@ void init_session_data(struct session_data *sdata, struct config *cfg);
 int read_from_stdin(struct session_data *sdata);
 void strtolower(char *s);
 
-void *get_in_addr(struct sockaddr *sa);
 int make_socket_non_blocking(int fd);
 int create_and_bind(char *listen_addr, int listen_port);
 
 int can_i_write_directory(char *dir);
 
 void move_email(struct smtp_session *session);
+int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte);
+
+int init_ssl_to_server(struct data *data);
 
 #ifndef _GNU_SOURCE
    char *strcasestr(const char *s, const char *find);
 #endif
 
+int append_string_to_buffer(char **buffer, char *str);
+int get_size_from_smtp_mail_from(char *s);
+
 #endif /* _MISC_H */

src/mydomains.c

@@ -11,7 +11,6 @@
 
 
 void load_mydomains(struct session_data *sdata, struct data *data, struct config *cfg){
-   int rc;
    char s[SMALLBUFSIZE];
    struct sql sql;
 
@@ -32,9 +31,7 @@ void load_mydomains(struct session_data *sdata, struct data *data, struct config
       p_store_results(&sql);
 
       while(p_fetch_results(&sql) == OK){
-         rc = addnode(data->mydomains, s);
-
-         if(rc == 0) syslog(LOG_PRIORITY, "failed to append mydomain: '%s'", s);
+         if(addnode(data->mydomains, s) == 0) syslog(LOG_PRIORITY, "failed to append mydomain: '%s'", s);
          else if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "added mydomain: '%s'", s);
 
          memset(s, 0, sizeof(s));

src/mysql.c

@@ -19,7 +19,7 @@ int open_database(struct session_data *sdata, struct config *cfg){
    mysql_options(&(sdata->mysql), MYSQL_OPT_RECONNECT, (const char*)&rc);
 
    if(mysql_real_connect(&(sdata->mysql), cfg->mysqlhost, cfg->mysqluser, cfg->mysqlpwd, cfg->mysqldb, cfg->mysqlport, cfg->mysqlsocket, 0) == 0){
-      printf("cant connect to mysql server\n");
+      syslog(LOG_PRIORITY, "cant connect to mysql server");
       return ERR;
    }
 
@@ -33,11 +33,40 @@ int open_database(struct session_data *sdata, struct config *cfg){
 }
 
 
+int open_sphx(struct session_data *sdata, struct config *cfg){
+   int rc=1;
+   char buf[BUFLEN];
+
+   mysql_init(&(sdata->sphx));
+
+   mysql_options(&(sdata->sphx), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg->mysql_connect_timeout);
+   mysql_options(&(sdata->sphx), MYSQL_OPT_RECONNECT, (const char*)&rc);
+
+   if(mysql_real_connect(&(sdata->sphx), cfg->sphxhost, "", "", cfg->sphxdb, cfg->sphxport, "", 0) == 0){
+      syslog(LOG_PRIORITY, "cant connect to %s:%d", cfg->sphxhost, cfg->sphxport);
+      return ERR;
+   }
+
+   snprintf(buf, sizeof(buf)-2, "SET NAMES %s", cfg->mysqlcharset);
+   mysql_real_query(&(sdata->sphx), buf, strlen(buf));
+
+   snprintf(buf, sizeof(buf)-2, "SET CHARACTER SET %s", cfg->mysqlcharset);
+   mysql_real_query(&(sdata->sphx), buf, strlen(buf));
+
+   return OK;
+}
+
+
 void close_database(struct session_data *sdata){
    mysql_close(&(sdata->mysql));
 }
 
 
+void close_sphx(struct session_data *sdata){
+   mysql_close(&(sdata->sphx));
+}
+
+
 void p_bind_init(struct sql *sql){
    int i;
 
@@ -104,7 +133,6 @@ int p_exec_stmt(struct session_data *sdata, struct sql *sql){
          bind[i].buffer = sql->sql[i];
          bind[i].is_null = 0;
 
-         
       }
       else { break; }
    }
@@ -222,7 +250,7 @@ int prepare_sql_statement(struct session_data *sdata, struct sql *sql, char *s){
       return ERR;
    }
 
-   return OK; 
+   return OK;
 }
 
 

src/parser.c

@@ -18,9 +18,7 @@
 
 struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct data *data, struct config *cfg){
    FILE *f;
-   int i;
-   unsigned int len;
-   char *p, buf[MAXBUFSIZE], puf[SMALLBUFSIZE];
+   char buf[MAXBUFSIZE];
    char writebuffer[MAXBUFSIZE], abuffer[MAXBUFSIZE];
    struct parser_state state;
 
@@ -32,44 +30,6 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec
       return state;
    }
 
-
-   if(sdata->num_of_rcpt_to > 0 && cfg->process_rcpt_to_addresses == 1){
-      for(i=0; i<sdata->num_of_rcpt_to; i++){
-
-         snprintf(puf, sizeof(puf)-1, "%s ", sdata->rcptto[i]);
-
-         if(does_it_seem_like_an_email_address(puf) == 1){
-            p = strstr(puf, cfg->hostid);
-            if(!p){
-
-               strtolower(puf);
-               len = strlen(puf);
-
-               if(state.tolen < MAXBUFSIZE-len-1){
-
-                  if(findnode(state.rcpt, puf) == NULL){
-                     addnode(state.journal_recipient, puf);
-                     addnode(state.rcpt, puf);
-
-                     memcpy(&(state.b_journal_to[state.journaltolen]), puf, len);
-                     state.journaltolen += len;
-
-                     memcpy(&(state.b_to[state.tolen]), puf, len);
-                     state.tolen += len;
-
-                     if(state.tolen < MAXBUFSIZE-len-1){
-                        split_email_address(puf);
-                        memcpy(&(state.b_to[state.tolen]), puf, len);
-                        state.tolen += len;
-                     }
-                  }
-               }
-            }
-         }
-
-      }
-   }
-
    if(take_into_pieces == 1){
       state.mfd = open(sdata->tmpframe, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR);
       if(state.mfd == -1){
@@ -85,30 +45,67 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec
 
    if(take_into_pieces == 1 && state.writebufpos > 0){
       if(write(state.mfd, writebuffer, state.writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: %s: write(), %s, %d, %s", sdata->ttmpfile, __func__, __LINE__, __FILE__);
-      memset(writebuffer, 0, sizeof(writebuffer));
+      memset(writebuffer, 0, sizeof(writebuffer)); //-V597
       state.writebufpos = 0;
    }
 
    if(take_into_pieces == 1){
       close(state.mfd); state.mfd = 0;
+
+      if(state.has_to_dump_whole_body == 1){
+         if(state.abufpos > 0){
+            flush_attachment_buffer(&state, &abuffer[0], sizeof(abuffer));
+         }
+         if(state.fd != -1) close(state.fd);
+         if(state.b64fd != -1) close(state.b64fd);
+      }
+
    }
 
    fclose(f);
 
+   if(data->import && data->import->extra_recipient){
+      char tmpbuf[SMALLBUFSIZE];
+      snprintf(tmpbuf, sizeof(tmpbuf)-1, "%s", data->import->extra_recipient);
+      add_recipient(tmpbuf, strlen(tmpbuf), sdata, &state, data, cfg);
+   }
+
+   // If both Sender: and From: headers exist, and they are different, then append
+   // the From: address to recipients list to give him access to this email as well
+
+   if(state.b_sender_domain[0]){
+      char frombuf[SMALLBUFSIZE];
+      char senderbuf[SMALLBUFSIZE];
+      get_first_email_address_from_string(state.b_from, frombuf, sizeof(frombuf));
+      get_first_email_address_from_string(state.b_sender, senderbuf, sizeof(senderbuf));
+      if(strcmp(frombuf, senderbuf)){
+         frombuf[strlen(frombuf)] = ' ';
+         add_recipient(frombuf, strlen(frombuf), sdata, &state, data, cfg);
+      }
+   }
+
    return state;
 }
 
 
 void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg){
-   int i, rec=0;
-   unsigned int len;
-   char *p;
+   int i;
 
    clearhash(state->boundaries);
    clearhash(state->rcpt);
    clearhash(state->rcpt_domain);
    clearhash(state->journal_recipient);
 
+   // Fix From: and Sender: lines if they are too long
+   if(strlen(state->b_from) > 255) state->b_from[255] = '\0';
+   if(strlen(state->b_from_domain) > 255) state->b_from_domain[255] = '\0';
+
+   if(strlen(state->b_sender) > 255) state->b_sender[255] = '\0';
+   if(strlen(state->b_sender_domain) > 255) state->b_sender_domain[255] = '\0';
+
+   // Truncate the message_id if it's >255 characters
+   if(strlen(state->message_id) > 255) state->message_id[255] = '\0';
+
    fixupEncodedHeaderLine(state->b_subject, MAXBUFSIZE);
    trimBuffer(state->b_subject);
 
@@ -119,20 +116,50 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c
       if(sdata->internal_recipient == 1 && sdata->external_recipient == 1) sdata->direction = DIRECTION_INTERNAL_AND_OUTGOING;
    }
 
+   char *q = strrchr(state->receivedbuf, ';');
+   // I've seen some odd 1st Received: lines where the date
+   // wasn't the last item after a semicolon, eg.
+   //
+   // Received: from some.server.local [172.16.102.18]
+   //   by some.server.com with POP3 (Message Archiver)
+   //   for <nobody@127.0.0.1> (single-drop); Fri, 22 Nov 2019 11:24:49 -0500 (EST)
+   //   smtp.auth=3Daaaaaaaa@yourdomain.com|Received:=20by=20yourdomain
+   //   .com=20with=20ESMTPA=20id=20md50124862414.msg=3B=20Fri,=2022=20
+   //   ...
+   //
+   // Such line messes with the parse_date_header() function
+   // so the workaround is to discard such Received: line
+   if(q && strlen(q+1) < 45){
+      time_t received_timestamp = parse_date_header(q+1);
+      if(received_timestamp > 10000000){
+         // If the calculated date based on Date: header line differs more than 1 week
+         // then we'll override it with the data parsed from the first Received: line
+         if(labs(received_timestamp - sdata->sent) > 604800) sdata->sent = received_timestamp;
+      }
+   }
 
    for(i=1; i<=state->n_attachments; i++){
+      char puf[SMALLBUFSIZE];
+      snprintf(puf, sizeof(puf)-1, "%s ", state->attachments[i].filename);
+
+      unsigned int len = strlen(puf);
+      if(state->bodylen < BIGBUFSIZE-len-1){
+         memcpy(&(state->b_body[state->bodylen]), puf, len);
+         state->bodylen += len;
+      }
+
       digest_file(state->attachments[i].internalname, &(state->attachments[i].digest[0]));
 
-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest);
+      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s, dumped: %d", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest, state->attachments[i].dumped);
 
-      p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type);
+      char *p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type);
       len = strlen(p);
       if(strlen(sdata->attachments) < SMALLBUFSIZE-len-1 && !strstr(sdata->attachments, p)) memcpy(&(sdata->attachments[strlen(sdata->attachments)]), p, len);
 
       if(state->attachments[i].dumped == 1){
-         rec = 0;
+         int rec = 0;
          if(cfg->extract_attachments == 1 && state->bodylen < BIGBUFSIZE-1024) extract_attachment_content(sdata, state, state->attachments[i].aname, get_attachment_extractor_by_filename(state->attachments[i].filename), &rec, cfg);
-      
+
          unlink(state->attachments[i].aname);
       }
 
@@ -146,7 +173,7 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c
    }
 
 
-   digest_string(state->message_id, &(state->message_id_hash[0]));
+   digest_string("sha256", state->message_id, &(state->message_id_hash[0]));
 
    if(sdata->sent == 0) sdata->sent = sdata->now;
 }
@@ -172,11 +199,29 @@ void storno_attachment(struct parser_state *state){
 }
 
 
-int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, int writebuffersize, char *abuffer, int abuffersize, struct data *data, struct config *cfg){
-   char *p, *q, puf[SMALLBUFSIZE];
-   unsigned char b64buffer[MAXBUFSIZE];
-   char tmpbuf[MAXBUFSIZE];
-   int n64, writelen, boundary_line=0, result;
+void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned int abuffersize){
+   if(write(state->fd, abuffer, state->abufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
+
+   if(state->b64fd != -1){
+      abuffer[state->abufpos] = '\0';
+      if(state->base64 == 1){
+         unsigned char b64buffer[MAXBUFSIZE];
+         int n64 = base64_decode_attachment_buffer(abuffer, &b64buffer[0], sizeof(b64buffer));
+         if(write(state->b64fd, b64buffer, n64) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
+      }
+      else if(write(state->b64fd, abuffer, state->abufpos) == -1){
+         syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
+      }
+   }
+
+   state->abufpos = 0;
+   memset(abuffer, 0, abuffersize);
+}
+
+
+int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg){
+   char *p;
+   int boundary_line=0;
    unsigned int len;
 
    if(cfg->debug == 1) printf("line: %s", buf);
@@ -194,6 +239,10 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
          sdata->restored_copy = 1;
       }
 
+      if(cfg->security_header[0] && state->found_security_header == 0 && strstr(buf, cfg->security_header)){
+         state->found_security_header = 1;
+      }
+
       if(*(cfg->piler_header_field) != 0 && strncmp(buf, cfg->piler_header_field, strlen(cfg->piler_header_field)) == 0){
          sdata->restored_copy = 1;
       }
@@ -203,8 +252,10 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
          memset(state->b_to, 0, MAXBUFSIZE);
          state->tolen = 0;
          memset(state->b_to_domain, 0, SMALLBUFSIZE);
+         state->todomainlen = 0;
 
          clearhash(state->rcpt);
+         clearhash(state->rcpt_domain);
 
          //if(sdata->import == 0){
             sdata->ms_journal = 1;
@@ -224,31 +275,19 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
 
 
    if(take_into_pieces == 1){
-      if(state->message_state == MSG_BODY && state->fd != -1 && is_substr_in_hash(state->boundaries, buf) == 0){
-         //n = write(state->fd, buf, len); // WRITE
+      if(state->message_state == MSG_BODY && state->fd != -1 && (state->has_to_dump_whole_body == 1 || is_substr_in_hash(state->boundaries, buf) == 0) ){
          if(len + state->abufpos > abuffersize-1){
-            if(write(state->fd, abuffer, state->abufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
-
-            if(state->b64fd != -1){
-               abuffer[state->abufpos] = '\0';
-               if(state->base64 == 1){
-                  n64 = base64_decode_attachment_buffer(abuffer, &b64buffer[0], sizeof(b64buffer));
-                  if(write(state->b64fd, b64buffer, n64) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
-               }
-               else {
-                  if(write(state->b64fd, abuffer, state->abufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
-               }
-            }
-
-            state->abufpos = 0; memset(abuffer, 0, abuffersize);
+            flush_attachment_buffer(state, abuffer, abuffersize);
          }
          memcpy(abuffer+state->abufpos, buf, len); state->abufpos += len;
 
          state->attachments[state->n_attachments].size += len;
+
+         // When processing the body and writing to an attachment file, then we finish here
+         return 0;
       }
       else {
          state->saved_size += len;
-         //n = write(state->mfd, buf, len); // WRITE
          if(len + state->writebufpos > writebuffersize-1){
             if(write(state->mfd, writebuffer, state->writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
             state->writebufpos = 0;
@@ -259,9 +298,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
    }
 
 
-   if(state->message_state == MSG_BODY && state->has_to_dump == 1 &&  state->pushed_pointer == 0){
-      //printf("####name: %s, type: %s, base64: %d\n", state->filename, state->type, state->base64);
-
+   if(state->message_state == MSG_BODY && state->has_to_dump == 1 && state->pushed_pointer == 0){
       state->pushed_pointer = 1;
 
 
@@ -274,8 +311,6 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
          snprintf(state->attachments[state->n_attachments].internalname, TINYBUFSIZE-1, "%s.a%d", sdata->ttmpfile, state->n_attachments);
          snprintf(state->attachments[state->n_attachments].aname, TINYBUFSIZE-1, "%s.a%d.bin", sdata->ttmpfile, state->n_attachments);
 
-         //printf("DUMP FILE: %s\n", state->attachments[state->n_attachments].internalname);
-
          if(take_into_pieces == 1){
             state->fd = open(state->attachments[state->n_attachments].internalname, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR);
 
@@ -284,7 +319,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
             p = get_attachment_extractor_by_filename(state->attachments[state->n_attachments].filename);
 
             snprintf(state->attachments[state->n_attachments].shorttype, TINYBUFSIZE-1, "%s", p);
- 
+
             if(strcmp("other", p)){
                state->b64fd = open(state->attachments[state->n_attachments].aname, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
                state->attachments[state->n_attachments].dumped = 1;
@@ -297,9 +332,9 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
                syslog(LOG_PRIORITY, "%s: error opening %s", sdata->ttmpfile, state->attachments[state->n_attachments].internalname);
             }
             else {
+               char puf[SMALLBUFSIZE];
                snprintf(puf, sizeof(puf)-1, "ATTACHMENT_POINTER_%s.a%d_XXX_PILER", sdata->ttmpfile, state->n_attachments);
-               //n = write(state->mfd, puf, strlen(puf)); // WRITE
-               writelen = strlen(puf);
+               int writelen = strlen(puf);
                if(writelen + state->writebufpos > writebuffersize-1){
                   if(write(state->mfd, writebuffer, state->writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
                   state->writebufpos = 0;
@@ -355,15 +390,35 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
          sdata->spam_message = 1;
       }
 
-      if(strncasecmp(buf, "From:", strlen("From:")) == 0){
+      if(strncasecmp(buf, "X-Piler-Envelope-To:", strlen("X-Piler-Envelope-To:")) == 0){
+         state->message_state = MSG_ENVELOPE_TO;
+         buf += strlen("X-Piler-Envelope-To:");
+      }
+      else if(strncasecmp(buf, "From:", strlen("From:")) == 0){
          state->message_state = MSG_FROM;
          buf += strlen("From:");
       }
+      else if(strncasecmp(buf, "Sender:", strlen("Sender:")) == 0){
+         state->message_state = MSG_SENDER;
+         buf += strlen("Sender:");
+      }
       else if(strncasecmp(buf, "Content-Type:", strlen("Content-Type:")) == 0){
          state->message_state = MSG_CONTENT_TYPE;
       }
-      else if(strncasecmp(buf, "Content-Transfer-Encoding:", strlen("Content-Transfer-Encoding:")) == 0) state->message_state = MSG_CONTENT_TRANSFER_ENCODING;
-      else if(strncasecmp(buf, "Content-Disposition:", strlen("Content-Disposition:")) == 0){
+      else if(strncasecmp(buf, "Content-Transfer-Encoding:", strlen("Content-Transfer-Encoding:")) == 0){
+         state->message_state = MSG_CONTENT_TRANSFER_ENCODING;
+         if(state->is_1st_header == 1 && strcasestr(buf, "base64")){
+            state->has_to_dump = 1;
+            state->has_to_dump_whole_body = 1;
+         }
+      }
+
+      /*
+       * We only enter MSG_CONTENT_DISPOSITION state if we couldn't find
+       * the filename in MSG_CONTENT_TYPE state. We also assume that
+       * Content-Type: comes first, then Content-Disposition:
+       */
+      else if(strncasecmp(buf, "Content-Disposition:", strlen("Content-Disposition:")) == 0 && strcasestr(state->attachment_name_buf, "name") == NULL){
          state->message_state = MSG_CONTENT_DISPOSITION;
       }
       else if(strncasecmp(buf, "To:", 3) == 0){
@@ -378,7 +433,10 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
          state->message_state = MSG_CC;
          buf += strlen("Bcc:");
       }
-      else if(strncasecmp(buf, "Message-Id:", 11) == 0) state->message_state = MSG_MESSAGE_ID;
+      else if(strncasecmp(buf, "Message-Id:", 11) == 0){
+         state->message_state = MSG_MESSAGE_ID;
+         buf += strlen("Message-Id:");
+      }
       else if(strncasecmp(buf, "References:", 11) == 0) state->message_state = MSG_REFERENCES;
       else if(strncasecmp(buf, "Subject:", strlen("Subject:")) == 0){
          state->message_state = MSG_SUBJECT;
@@ -402,26 +460,37 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
 
          if(strstr(buf, "=?") && strstr(buf, "?=")) fixupEncodedHeaderLine(buf, MAXBUFSIZE);
 
-         sdata->sent = parse_date_header(buf);
+         sdata->sent = parse_date_header(buf+5);
 
          /* allow +2 days drift in the parsed Date: value */
 
          if(sdata->sent - sdata->now > 2*86400) sdata->sent = sdata->now;
       }
 
-      else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf);
-      else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0) state->message_state = MSG_RECEIVED;
+      else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf+14);
+      else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0){
+         state->message_state = MSG_RECEIVED;
+         state->received_header++;
+      }
       else if(cfg->extra_to_field[0] != '\0' && strncasecmp(buf, cfg->extra_to_field, strlen(cfg->extra_to_field)) == 0){
-         state->message_state = MSG_TO;
+         state->message_state = MSG_RECIPIENT;
          buf += strlen(cfg->extra_to_field);
       }
 
       if(state->message_state == MSG_MESSAGE_ID && state->message_id[0] == 0){
-         p = strchr(buf+11, ' ');
-         if(p) p = buf + 12;
-         else p = buf + 11;
+         while(isspace(*buf)){
+            buf++;
+         }
 
-         snprintf(state->message_id, SMALLBUFSIZE-1, "%s", p);
+         snprintf(state->message_id, SMALLBUFSIZE-1, "%s", buf);
+      }
+
+      if(state->message_state == MSG_CONTENT_TYPE || state->message_state == MSG_CONTENT_DISPOSITION){
+         fill_attachment_name_buf(state, buf);
+      }
+
+      if(state->received_header == 1 && state->message_state == MSG_RECEIVED && strlen(state->receivedbuf) + len < sizeof(state->receivedbuf)){
+         memcpy(&(state->receivedbuf[strlen(state->receivedbuf)]), buf, len);
       }
 
       /* we are interested in only From:, To:, Subject:, Received:, Content-*: header lines */
@@ -436,7 +505,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
    }
 
 
-   /* 
+   /*
     * A normal journal looks like this:
     *
     *   Sender: sender@domain
@@ -521,6 +590,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
          state->message_rfc822 = 1;
          state->is_header = 1;
 
+         state->has_to_dump = 0;
+
          if(sdata->ms_journal == 1){
             state->is_1st_header = 1;
 
@@ -530,6 +601,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
             memset(state->b_body, 0, BIGBUFSIZE);
             memset(state->b_from, 0, SMALLBUFSIZE);
             memset(state->b_from_domain, 0, SMALLBUFSIZE);
+            memset(state->b_sender, 0, SMALLBUFSIZE);
+            memset(state->b_sender_domain, 0, SMALLBUFSIZE);
             memset(state->message_id, 0, SMALLBUFSIZE);
 
             sdata->ms_journal = 0;
@@ -542,21 +615,6 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
    }
 
 
-   if(state->message_state == MSG_CONTENT_TYPE){
-      p = &buf[0];
-      for(; *p; p++){
-         if(*p != ' ' && *p != '\t') break;
-      }
-
-      len = strlen(p);
-
-      if(len + state->anamepos < SMALLBUFSIZE-2){
-         memcpy(&(state->attachment_name_buf[state->anamepos]), p, len);
-         state->anamepos += len;
-      }
-   }
-
-
    if(state->message_state == MSG_CONTENT_TRANSFER_ENCODING){
       if(strcasestr(buf, "base64")) state->base64 = 1;
       if(strcasestr(buf, "quoted-printable")) state->qp = 1;
@@ -577,20 +635,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
       if(state->has_to_dump == 1){
          if(take_into_pieces == 1 && state->fd != -1){
             if(state->abufpos > 0){
-               if(write(state->fd, abuffer, state->abufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
-
-               if(state->b64fd != -1){
-                  abuffer[state->abufpos] = '\0';
-                  if(state->base64 == 1){
-                     n64 = base64_decode_attachment_buffer(abuffer, &b64buffer[0], sizeof(b64buffer));
-                     if(write(state->b64fd, b64buffer, n64) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
-                  }
-                  else {
-                     if(write(state->b64fd, abuffer, state->abufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
-                  }
-               }
-
-               state->abufpos = 0; memset(abuffer, 0, abuffersize); 
+               flush_attachment_buffer(state, abuffer, abuffersize);
             }
             close(state->fd);
             close(state->b64fd);
@@ -612,14 +657,14 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
       state->pushed_pointer = 0;
 
       memset(state->type, 0, TINYBUFSIZE);
-      snprintf(state->charset, TINYBUFSIZE-1, "unknown");
+      memset(state->charset, 0, TINYBUFSIZE);
 
       memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
       state->anamepos = 0;
 
       state->message_state = MSG_UNDEF;
 
-      return 0;      
+      return 0;
    }
 
    if(boundary_line == 1){ return 0; }
@@ -629,7 +674,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
 
 
    /* skip irrelevant headers */
-   if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT) return 0;
+   if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_SENDER && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT && state->message_state != MSG_ENVELOPE_TO) return 0;
 
 
    /* don't process body if it's not a text or html part */
@@ -655,127 +700,27 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
    if(state->texthtml == 1 && state->message_state == MSG_BODY) markHTML(buf, state);
 
 
-   if(state->texthtml == 1) decodeHTML(buf, state->utf8);
+   if(state->texthtml == 1){
+      size_t buflen = strlen(buf);
+      decodeHTML(buf, state->utf8);
+      /* decodeHTML converted some entities to iso-8859-1 */
+      if(state->utf8 != 1 && strlen(buf) != buflen){
+        /* no charset or us-ascii: switch to iso-8859-1 */
+        if (state->charset[0] == 0 || strcasecmp(state->charset, "us-ascii") == 0){
+          syslog(LOG_PRIORITY, "%s: assuming iso-8859-1 encoding for HTML (was '%s')", sdata->ttmpfile, state->charset);
+          snprintf(state->charset, TINYBUFSIZE-1, "ISO8859-1");
+        }
+      }
+   }
 
    /* encode the body if it's not utf-8 encoded */
    if(state->message_state == MSG_BODY && state->utf8 != 1){
-      result = utf8_encode(buf, strlen(buf), &tmpbuf[0], sizeof(tmpbuf), state->charset);
+      char tmpbuf[MAXBUFSIZE];
+      int result = utf8_encode(buf, strlen(buf), &tmpbuf[0], sizeof(tmpbuf), state->charset);
       if(result == OK) snprintf(buf, MAXBUFSIZE-1, "%s", tmpbuf);
    }
 
-   translateLine((unsigned char*)buf, state);
-
-   reassembleToken(buf);
-
-
-   p = buf;
-
-   //printf("a: %d/%d/%d/%d/j=%d %s\n", state->is_1st_header, state->is_header, state->message_rfc822, state->message_state, sdata->ms_journal, buf);
-
-   do {
-      memset(puf, 0, sizeof(puf));
-      p = split(p, ' ', puf, sizeof(puf)-1, &result);
-
-      if(puf[0] == '\0') continue;
-
-      degenerateToken((unsigned char*)puf);
-
-      if(puf[0] == '\0') continue;
-
-      strncat(puf, " ", sizeof(puf)-strlen(puf)-1);
-
-      if(strncasecmp(puf, "http://", 7) == 0 || strncasecmp(puf, "https://", 8) == 0) fixURL(puf, sizeof(puf)-1);
-
-      len = strlen(puf);
-
-      // skip body tokens if not an URL && (empty token || too long)
-      if(state->is_header == 0 && strncmp(puf, "__URL__", 7) && (puf[0] == ' ' || (len > MAX_WORD_LEN && cfg->enable_cjk == 0)) ){
-         continue;
-      }
-
-      if(state->message_state == MSG_FROM && state->is_1st_header == 1 && strlen(state->b_from) < SMALLBUFSIZE-len-1){
-         strtolower(puf);
-
-         q = strchr(puf, '@');
-         if(q) fix_plus_sign_in_email_address(puf, &q, &len);
-
-         memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
-
-         if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(puf) == 1 && state->b_from_domain[0] == '\0'){
-            if(q && strlen(q) > 5){
-               memcpy(&(state->b_from_domain), q+1, strlen(q+1)-1);
-               if(strstr(sdata->mailfrom, "<>")){
-                  snprintf(sdata->fromemail, SMALLBUFSIZE-1, "%s", puf);
-                  sdata->fromemail[len-1] = '\0';
-               }
-            }
-
-            if(is_email_address_on_my_domains(puf, data) == 1) sdata->internal_sender = 1;
-
-            if(strlen(state->b_from) < SMALLBUFSIZE-len-1){
-               split_email_address(puf);
-               memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
-            }
-         }
-      }
-      else if((state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && state->is_1st_header == 1 && state->tolen < MAXBUFSIZE-len-1){
-         strtolower(puf);
-
-         /* fix aaa+bbb@ccc.fu address to aaa@ccc.fu, 2017.02.04, SJ */
-         q = strchr(puf, '@');
-         if(q) fix_plus_sign_in_email_address(puf, &q, &len);
-
-         if(state->message_state == MSG_RECIPIENT && findnode(state->journal_recipient, puf) == NULL){
-            addnode(state->journal_recipient, puf);
-            memcpy(&(state->b_journal_to[state->journaltolen]), puf, len);
-            if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: journal rcpt: '%s'", sdata->ttmpfile, puf);
-         }
-
-
-         if(findnode(state->rcpt, puf) == NULL){
-
-            /* skip any address matching ...@cfg->hostid, 2013.10.29, SJ */
-            if(q && strncmp(q+1, cfg->hostid, cfg->hostid_len) == 0){
-               continue;
-            }
-
-            addnode(state->rcpt, puf);
-            memcpy(&(state->b_to[state->tolen]), puf, len);
-            state->tolen += len;
-
-            if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(puf) == 1){
-
-               if(is_email_address_on_my_domains(puf, data) == 1) sdata->internal_recipient = 1;
-               else sdata->external_recipient = 1;
-
-               if(q){
-                  if(findnode(state->rcpt_domain, q+1) == NULL){
-                     addnode(state->rcpt_domain, q+1);
-                     memcpy(&(state->b_to_domain[strlen(state->b_to_domain)]), q+1, strlen(q+1));
-                  }
-               }
-
-               if(state->tolen < MAXBUFSIZE-len-1){
-                  split_email_address(puf);
-                  memcpy(&(state->b_to[state->tolen]), puf, len);
-                  state->tolen += len;
-               }
-
-            }
-         }
-
-      }
-      else if(state->message_state == MSG_BODY && len >= cfg->min_word_len && state->bodylen < BIGBUFSIZE-len-1){
-         // 99% of email addresses are longer than 8 characters
-         if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(puf)){
-            fix_email_address_for_sphinx(puf);
-         }
-
-         memcpy(&(state->b_body[state->bodylen]), puf, len);
-         state->bodylen += len;
-      }
-
-   } while(p);
+   tokenize(buf, state, sdata, data, cfg);
 
    return 0;
 }

src/parser.h

@@ -11,7 +11,7 @@
 
 struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct data *data, struct config *cfg);
 void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg);
-int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, int writebuffersize, char *abuffer, int abuffersize, struct data *data, struct config *cfg);
+int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg);
 
 void init_state(struct parser_state *state);
 time_t parse_date_header(char *s);
@@ -20,11 +20,12 @@ void fixupEncodedHeaderLine(char *buf, int buflen);
 void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state);
 void fixupBase64EncodedLine(char *buf, struct parser_state *state);
 void markHTML(char *buf, struct parser_state *state);
-int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state);
+void setStateHTML(char *htmlbuf, int pos, struct parser_state *state);
 void translateLine(unsigned char *p, struct parser_state *state);
 void fix_email_address_for_sphinx(char *s);
 void split_email_address(char *s);
 int does_it_seem_like_an_email_address(char *email);
+void add_recipient(char *email, unsigned int len, struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
 void reassembleToken(char *p);
 void degenerateToken(unsigned char *p);
 void fixURL(char *buf, int buflen);
@@ -34,5 +35,9 @@ char *get_attachment_extractor_by_filename(char *filename);
 void parse_reference(struct parser_state *state, char *s);
 int base64_decode_attachment_buffer(char *p, unsigned char *b, int blen);
 void fix_plus_sign_in_email_address(char *puf, char **at_sign, unsigned int *len);
+void tokenize(char *buf, struct parser_state *state, struct session_data *sdata, struct data *data, struct config *cfg);
+void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned int abuffersize);
+void fill_attachment_name_buf(struct parser_state *state, char *buf);
+int get_first_email_address_from_string(char *str, char *buf, int buflen);
 
 #endif /* _PARSER_H */

src/parser_utils.c

@@ -40,6 +40,7 @@ void init_state(struct parser_state *state){
 
    state->htmltag = 0;
    state->style = 0;
+   state->meta_content_type = 0;
 
    state->skip_html = 0;
 
@@ -49,13 +50,16 @@ void init_state(struct parser_state *state){
    memset(state->message_id_hash, 0, 2*DIGEST_LENGTH+1);
    memset(state->miscbuf, 0, MAX_TOKEN_LEN);
    memset(state->qpbuf, 0, MAX_TOKEN_LEN);
+   memset(state->receivedbuf, 0, sizeof(state->receivedbuf));
 
    memset(state->type, 0, TINYBUFSIZE);
+   memset(state->charset, 0, TINYBUFSIZE);
 
    memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
    state->anamepos = 0;
 
    state->has_to_dump = 0;
+   state->has_to_dump_whole_body = 0;
    state->fd = -1;
    state->b64fd = -1;
    state->mfd = -1;
@@ -66,6 +70,7 @@ void init_state(struct parser_state *state){
 
    state->writebufpos = 0;
    state->abufpos = 0;
+   state->received_header = 0;
 
    inithash(state->boundaries);
    inithash(state->rcpt);
@@ -89,6 +94,8 @@ void init_state(struct parser_state *state){
 
    memset(state->b_from, 0, SMALLBUFSIZE);
    memset(state->b_from_domain, 0, SMALLBUFSIZE);
+   memset(state->b_sender, 0, SMALLBUFSIZE);
+   memset(state->b_sender_domain, 0, SMALLBUFSIZE);
    memset(state->b_to, 0, MAXBUFSIZE);
    memset(state->b_to_domain, 0, SMALLBUFSIZE);
    memset(state->b_subject, 0, MAXBUFSIZE);
@@ -96,10 +103,13 @@ void init_state(struct parser_state *state){
    memset(state->b_journal_to, 0, MAXBUFSIZE);
 
    state->tolen = 0;
+   state->todomainlen = 0;
    state->bodylen = 0;
    state->journaltolen = 0;
 
    state->retention = 0;
+
+   state->found_security_header = 0;
 }
 
 
@@ -112,13 +122,12 @@ long get_local_timezone_offset(){
 
 
 time_t parse_date_header(char *datestr){
-   int n=0, len;
+   int n=0;
    long offset=0;
    time_t ts=0;
    char *p, *q, *r, *tz, s[SMALLBUFSIZE], tzh[4], tzm[3];
    struct tm tm;
 
-   datestr += 5;
    p = datestr;
 
    for(; *datestr; datestr++){
@@ -148,7 +157,7 @@ time_t parse_date_header(char *datestr){
    do {
       p = split_str(p, " ", s, sizeof(s)-1);
 
-      len = strlen(s);
+      int len = strlen(s);
 
       if(len > 0){
          n++;
@@ -245,7 +254,7 @@ time_t parse_date_header(char *datestr){
             } else {
                tm.tm_min = atoi(r);
             }
-            continue; 
+            continue;
          }
       }
 
@@ -257,12 +266,24 @@ time_t parse_date_header(char *datestr){
 
    ts += get_local_timezone_offset() - offset;
 
+   if(ts < 700000000){
+      // If the Date: field contains some garbage, eg.
+      // "Date: [mail_datw]" or similar, and the date
+      // is before Sat Mar  7 20:26:40 UTC 1992, then
+      // return the current timestamp
+
+      time_t now;
+      time(&now);
+
+      return now;
+   }
+
    return ts;
 }
 
 
 int extract_boundary(char *p, struct parser_state *state){
-   char *q, *q2;
+   char *q;
 
    p += strlen("boundary");
 
@@ -289,7 +310,7 @@ int extract_boundary(char *p, struct parser_state *state){
             break;
       }
 
-      q2 = strchr(p, ';');
+      char *q2 = strchr(p, ';');
       if(q2) *q2 = '\0';
 
       q = strrchr(p, '"');
@@ -313,14 +334,13 @@ int extract_boundary(char *p, struct parser_state *state){
 
 
 void fixupEncodedHeaderLine(char *buf, int buflen){
-   char *p, *q, *r, *s, *e, *end;
+   char *q, *r, *s, *e, *end;
    /*
     * I thought SMALLBUFSIZE would be enough for v, encoding and tmpbuf(2*),
     * but then I saw a 6-7000 byte long subject line, so I've switched to MAXBUFSIZE
     */
    char v[MAXBUFSIZE], u[MAXBUFSIZE], puf[MAXBUFSIZE], encoding[MAXBUFSIZE], tmpbuf[2*MAXBUFSIZE];
    int need_encoding, ret, prev_encoded=0, n_tokens=0;
-   int b64=0, qp=0;
 
    if(buflen < 5) return;
 
@@ -331,7 +351,7 @@ void fixupEncodedHeaderLine(char *buf, int buflen){
    do {
       q = split_str(q, " ", v, sizeof(v)-1);
 
-      p = v;
+      char *p = v;
 
       do {
          memset(u, 0, sizeof(u));
@@ -347,7 +367,7 @@ void fixupEncodedHeaderLine(char *buf, int buflen){
           *    Happy New Year! =?utf-8?q?=F0=9F=8E=86?=
           */
 
-         b64 = qp = 0;
+         int b64=0, qp=0;
          memset(encoding, 0, sizeof(encoding));
 
          r = strstr(p, "=?");
@@ -422,6 +442,8 @@ void fixupEncodedHeaderLine(char *buf, int buflen){
             }
 
             if(need_encoding == 1 && ret == OK){
+               #pragma GCC diagnostic push
+               #pragma GCC diagnostic ignored "-Wstringop-truncation"
                strncat(puf, tmpbuf, sizeof(puf)-strlen(puf)-1);
             }
             else {
@@ -442,9 +464,9 @@ void fixupEncodedHeaderLine(char *buf, int buflen){
 
 void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state){
    int i=0;
-   char *p, puf[MAXBUFSIZE];
 
    if(strlen(state->qpbuf) > 0){
+      char puf[MAXBUFSIZE];
       memset(puf, 0, sizeof(puf));
       snprintf(puf, sizeof(puf)-1, "%s%s", state->qpbuf, buf);
       snprintf(buf, MAXBUFSIZE-1, "%s", puf);
@@ -457,7 +479,7 @@ void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state){
    }
 
    if(i == 1){
-      p = strrchr(buf, ' ');
+      char *p = strrchr(buf, ' ');
       if(p){
          memset(state->qpbuf, 0, MAX_TOKEN_LEN);
          if(strlen(p) < MAX_TOKEN_LEN-1){
@@ -471,9 +493,8 @@ void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state){
 
 
 void fixupBase64EncodedLine(char *buf, struct parser_state *state){
-   char *p, puf[MAXBUFSIZE];
-
    if(strlen(state->miscbuf) > 0){
+      char puf[MAXBUFSIZE];
       memset(puf, 0, sizeof(puf));
       strncpy(puf, state->miscbuf, sizeof(puf)-strlen(puf)-1);
       strncat(puf, buf, sizeof(puf)-strlen(puf)-1);
@@ -485,7 +506,7 @@ void fixupBase64EncodedLine(char *buf, struct parser_state *state){
    }
 
    if(buf[strlen(buf)-1] != '\n'){
-      p = strrchr(buf, ' ');
+      char *p = strrchr(buf, ' ');
       if(p){
          memcpy(&(state->miscbuf[0]), p+1, MAX_TOKEN_LEN-1);
          *p = '\0';
@@ -517,13 +538,13 @@ void markHTML(char *buf, struct parser_state *state){
       }
 
       if(state->htmltag == 1){
-    
+
          if(j == 0 && *s == '!'){
             state->skip_html = 1;
             //printf("skiphtml=1\n");
          }
 
-         if(state->skip_html == 0){ 
+         if(state->skip_html == 0){
             if(*s != '>' && *s != '<' && *s != '"'){
                //printf("j=%d/%c", j, *s);
                html[j] = tolower(*s);
@@ -532,7 +553,7 @@ void markHTML(char *buf, struct parser_state *state){
 
             if(isspace(*s)){
                if(j > 0){
-                  k += appendHTMLTag(puf, html, pos, state);
+                  setStateHTML(html, pos, state);
                   memset(html, 0, SMALLBUFSIZE); j=0;
                }
                pos++;
@@ -557,55 +578,51 @@ void markHTML(char *buf, struct parser_state *state){
 
          if(j > 0){
             strncat(html, " ", SMALLBUFSIZE-1);
-            k += appendHTMLTag(puf, html, pos, state);
+            setStateHTML(html, pos, state);
             memset(html, 0, SMALLBUFSIZE); j=0;
          }
+         state->meta_content_type = 0;
       }
 
    }
 
    //printf("append last in line:*%s*, html=+%s+, j=%d\n", puf, html, j);
-   if(j > 0){ appendHTMLTag(puf, html, pos, state); }
+   if(j > 0){ setStateHTML(html, pos, state); }
 
    strcpy(buf, puf);
 }
 
 
-int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state){
-   char *p, html[SMALLBUFSIZE];
-   int len;
-
+void setStateHTML(char *htmlbuf, int pos, struct parser_state *state){
    if(pos == 0 && strncmp(htmlbuf, "style ", 6) == 0) state->style = 1;
    if(pos == 0 && strncmp(htmlbuf, "/style ", 7) == 0) state->style = 0;
 
-   return 0;
+   if(pos == 0 && state->charset[0] == 0 && strncmp(htmlbuf, "meta ", 5) == 0) state->meta_content_type = 0x1;
+   if(state->meta_content_type){
+     if((state->meta_content_type & 0x2) == 0 && strstr(htmlbuf, "http-equiv=content-type "))
+       state->meta_content_type |= 0x2;
 
-   //printf("appendHTML: pos:%d, +%s+\n", pos, htmlbuf);
+     if((state->meta_content_type & 0x4) == 0 && strstr(htmlbuf, "content=text/html;"))
+       state->meta_content_type |= 0x4;
 
-   if(state->style == 1) return 0;
+     if(state->meta_content_type == 0x7){
+       char *p, *q;
 
-   if(strlen(htmlbuf) == 0) return 0;
+       p = strstr(htmlbuf, "charset=");
+       if(p){
+         p += 8;
+         for(q = p; isalnum(*q) || index("-_", *q); q++)
+           ;
 
-   snprintf(html, SMALLBUFSIZE-1, "HTML*%s", htmlbuf);
-   len = strlen(html);
-
-   if(len > 8 && strchr(html, '=')){
-      p = strstr(html, "cid:");
-      if(p){
-         *(p+3) = '\0';
-         strncat(html, " ", SMALLBUFSIZE-1);
-      }
-
-      strncat(buf, html, MAXBUFSIZE-1);
-      return len;
+         if(q > p && q-p+1 < (int) sizeof(state->charset)){
+           syslog(LOG_PRIORITY, "Changing HTML charset from '%s' to '%*s' due to meta tag", state->charset, (int)(q-p), p);
+           strncpy(state->charset, p, q-p);
+           state->charset[q-p+1] = '\0';
+           state->meta_content_type = 0;
+         }
+       }
+     }
    }
-
-   if(strstr(html, "http") ){
-      strncat(buf, html+5, MAXBUFSIZE-1);
-      return len-5;
-   }
-
-   return 0;
 }
 
 
@@ -618,9 +635,9 @@ void translateLine(unsigned char *p, struct parser_state *state){
 
    for(; *p; p++){
 
-      if( (state->message_state == MSG_RECEIVED || state->message_state == MSG_FROM || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && *p == '@'){ continue; }
+      if( (state->message_state == MSG_RECEIVED || state->message_state == MSG_FROM || state->message_state == MSG_SENDER || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && *p == '@'){ continue; }
 
-      if(state->message_state == MSG_FROM || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT){
+      if(state->message_state == MSG_FROM || state->message_state == MSG_SENDER || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT){
 
          /* To fix some unusual addresses, eg.
           *    "'user@domain'"    -> user@domain
@@ -692,6 +709,47 @@ int does_it_seem_like_an_email_address(char *email){
 }
 
 
+void add_recipient(char *email, unsigned int len, struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
+   if(findnode(state->rcpt, email) == NULL){
+
+      char *q = strchr(email, '@');
+
+      /* skip any address matching ...@cfg->hostid, 2013.10.29, SJ */
+      if(q && strncmp(q+1, cfg->hostid, cfg->hostid_len) == 0){
+         return;
+      }
+
+      addnode(state->rcpt, email);
+      memcpy(&(state->b_to[state->tolen]), email, len);
+      state->tolen += len;
+
+      if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(email) == 1){
+
+         if(is_email_address_on_my_domains(email, data) == 1) sdata->internal_recipient = 1;
+         else sdata->external_recipient = 1;
+
+         if(q){
+            if(findnode(state->rcpt_domain, q+1) == NULL){
+               addnode(state->rcpt_domain, q+1);
+               unsigned int domainlen = strlen(q+1);
+
+               if(state->todomainlen < SMALLBUFSIZE-domainlen-1){
+                  memcpy(&(state->b_to_domain[state->todomainlen]), q+1, domainlen);
+                  state->todomainlen += domainlen;
+               }
+            }
+         }
+
+         if(state->tolen < MAXBUFSIZE-len-1){
+            split_email_address(email);
+            memcpy(&(state->b_to[state->tolen]), email, len);
+            state->tolen += len;
+         }
+      }
+   }
+}
+
+
 /*
  * reassemble 'V i a g r a' to 'Viagra'
  */
@@ -722,7 +780,7 @@ void degenerateToken(unsigned char *p){
    int i=1, d=0, dp=0;
    unsigned char *s;
 
-   /* quit if this the string does not end with a punctuation character */
+   /* quit if the string does not end with a punctuation character */
 
    if(!ispunct(*(p+strlen((char *)p)-1)))
       return;
@@ -778,8 +836,7 @@ void fixURL(char *buf, int buflen){
 
 
 void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultbuflen){
-   int extended=0;
-   char buf[SMALLBUFSIZE], puf[SMALLBUFSIZE], *p, *q, *encoding;
+   char buf[SMALLBUFSIZE], *p, *q;
 
    snprintf(buf, sizeof(buf)-1, "%s", s);
 
@@ -815,6 +872,8 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb
        *
        */
 
+      int extended=0;
+
       p += strlen(name);
       if(*p == '*'){
          extended = 1;
@@ -824,24 +883,25 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb
       if(p){
          p++;
 
-         // If the line has the 'name' more than once, then truncate the subsequent parts, ie.
-         // utf-8''P;LAN%20Holden%204.docx;filename="P;LAN Holden 4.docx" ==> utf-8''P;LAN%20Holden%204.docx
-         q = strstr(p, name);
-         if(q) *q = '\0';
+         // skip any whitespace after name=, ie. name = "
+         while(*p==' ' || *p=='\t') p++;
 
-         q = strrchr(p, ';');
-         if(q) *q = '\0';
-         q = strrchr(p, '"');
-         if(q){
-            *q = '\0';
-            p = strchr(p, '"');
-            if(p){
-               p++;
-            }
+         // if there's a double quote after the equal symbol (=), ie. name*="utf-8....
+         if(*p == '"'){
+            p++;
+            q = strchr(p, '"');
+
+            if(q) *q = '\0';
+         }
+         else {
+            // no " after =, so split on ;
+            q = strchr(p, ';');
+            if(q) *q = '\0';
          }
 
+
          if(extended == 1){
-            encoding = p;
+            char *encoding = p;
             q = strchr(p, '\'');
             if(q){
                *q = '\0';
@@ -858,6 +918,7 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb
                snprintf(resultbuf, resultbuflen-2, "%s", p);
          }
          else {
+            char puf[SMALLBUFSIZE];
             snprintf(puf, sizeof(puf)-1, "%s", p);
             fixupEncodedHeaderLine(puf, sizeof(puf));
 
@@ -871,8 +932,6 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb
 
 
 char *determine_attachment_type(char *filename, char *type){
-   char *p;
-
    if(strncasecmp(type, "text/", strlen("text/")) == 0) return "text,";
    if(strncasecmp(type, "image/", strlen("image/")) == 0) return "image,";
    if(strncasecmp(type, "audio/", strlen("audio/")) == 0) return "audio,";
@@ -903,7 +962,7 @@ char *determine_attachment_type(char *filename, char *type){
 
    if(strncasecmp(type, "application/", 12) == 0){
 
-      p = strrchr(filename, '.');
+      char *p = strrchr(filename, '.');
       if(p){
          p++;
 
@@ -913,7 +972,7 @@ char *determine_attachment_type(char *filename, char *type){
          if(strncasecmp(p, "rar", 3) == 0) return "compressed,";
 
          // tar.gz has the same type
-         if(strncasecmp(p, "x-gzip", 3) == 0) return "compressed,";
+         if(strncasecmp(p, "gz", 2) == 0) return "compressed,";
 
          if(strncasecmp(p, "rtf", 3) == 0) return "word,";
          if(strncasecmp(p, "doc", 3) == 0) return "word,";
@@ -928,7 +987,7 @@ char *determine_attachment_type(char *filename, char *type){
          if(strncasecmp(p, "jpg", 3) == 0) return "image,";
          if(strncasecmp(p, "jpeg", 4) == 0) return "image,";
          if(strncasecmp(p, "tiff", 4) == 0) return "image,";
-      } 
+      }
    }
 
    return "other,";
@@ -965,14 +1024,13 @@ char *get_attachment_extractor_by_filename(char *filename){
 
 
 void parse_reference(struct parser_state *state, char *s){
-   int len;
    char puf[SMALLBUFSIZE];
 
    if(strlen(state->reference) > 10) return;
 
    do {
       s = split_str(s, " ", puf, sizeof(puf)-1);
-      len = strlen(puf);
+      int len = strlen(puf);
 
       if(len > 10 && len < SMALLBUFSIZE-1){
          memcpy(&(state->reference[strlen(state->reference)]), puf, len);
@@ -998,12 +1056,11 @@ int base64_decode_attachment_buffer(char *p, unsigned char *b, int blen){
 
 
 void fix_plus_sign_in_email_address(char *puf, char **at_sign, unsigned int *len){
-   int n;
    char *r;
 
    r = strchr(puf, '+');
    if(r){
-      n = strlen(*at_sign);
+      int n = strlen(*at_sign);
       memmove(r, *at_sign, n);
       *(r+n) = '\0';
       *len = strlen(puf);
@@ -1011,3 +1068,40 @@ void fix_plus_sign_in_email_address(char *puf, char **at_sign, unsigned int *len
    }
 }
 
+
+void fill_attachment_name_buf(struct parser_state *state, char *buf){
+   char *p = &buf[0];
+
+   for(; *p; p++){
+      if(*p != ' ' && *p != '\t') break;
+   }
+
+   int len = strlen(p);
+
+   if(len + state->anamepos < SMALLBUFSIZE-3){
+      memcpy(&(state->attachment_name_buf[state->anamepos]), p, len);
+      state->anamepos += len;
+
+      // add a trailing separator semicolon to make sure there's separation
+      // with the next item
+      state->attachment_name_buf[state->anamepos] = ';';
+      state->anamepos++;
+   }
+}
+
+
+int get_first_email_address_from_string(char *str, char *buf, int buflen){
+   int result;
+
+   char *p = str;
+   do {
+      memset(buf, 0, buflen);
+      p = split(p, ' ', buf, buflen-1, &result);
+
+      if(*buf == '\0') continue;
+
+      if(does_it_seem_like_an_email_address(buf) == 1){ return 1; }
+   } while(p);
+
+   return 0;
+}

src/piler-smtp.c

@@ -31,15 +31,17 @@ extern char *optarg;
 extern int optind;
 
 struct epoll_event event, *events=NULL;
-int timeout = 20; // checking for timeout this often [sec]
 int num_connections = 0;
 int listenerfd = -1;
+int loglevel = 1;
 
 char *configfile = CONFIG_FILE;
 struct config cfg;
 struct passwd *pwd;
 struct smtp_session *session, **sessions=NULL;
+struct smtp_acl *smtp_acl[MAXHASH];
 
+time_t prev_timeout_check = 0;
 
 void usage(){
    printf("\nusage: piler\n\n");
@@ -47,18 +49,19 @@ void usage(){
    printf("    -d                                Fork to the background\n");
    printf("    -v                                Return the version and build number\n");
    printf("    -V                                Return the version and some build parameters\n");
+   printf("    -L <log level>                    Set the log level: 1-5\n");
 
    exit(0);
 }
 
 
-void p_clean_exit(){
-   int i;
+void p_clean_exit(int sig){
+   if(sig > 0) syslog(LOG_PRIORITY, "got signal: %d, %s", sig, strsignal(sig));
 
    if(listenerfd != -1) close(listenerfd);
 
    if(sessions){
-      for(i=0; i<cfg.max_connections; i++){
+      for(int i=0; i<cfg.max_connections; i++){
          if(sessions[i]) free_smtp_session(sessions[i]);
       }
 
@@ -67,6 +70,8 @@ void p_clean_exit(){
 
    if(events) free(events);
 
+   clear_smtp_acl(smtp_acl);
+
    syslog(LOG_PRIORITY, "%s has been terminated", PROGNAME);
 
    ERR_free_strings();
@@ -77,26 +82,29 @@ void p_clean_exit(){
 
 void fatal(char *s){
    syslog(LOG_PRIORITY, "%s", s);
-   p_clean_exit();
+   p_clean_exit(0);
 }
 
 
 void check_for_client_timeout(){
    time_t now;
-   int i;
+
+   time(&now);
+
+   if(cfg.verbosity >= LOG_DEBUG) syslog(LOG_PRIORITY, "%s @%ld", __func__, now);
+
+   if(now - prev_timeout_check < cfg.smtp_timeout) return;
 
    if(num_connections > 0){
-      time(&now);
-
-      for(i=0; i<cfg.max_connections; i++){
+      for(int i=0; i<cfg.max_connections; i++){
          if(sessions[i] && now - sessions[i]->lasttime >= cfg.smtp_timeout){
-            syslog(LOG_PRIORITY, "client %s timeout", sessions[i]->remote_host);
-            tear_down_session(sessions, sessions[i]->slot, &num_connections);
+            syslog(LOG_PRIORITY, "client %s timeout, lasttime: %ld", sessions[i]->remote_host, sessions[i]->lasttime);
+            tear_down_session(sessions, sessions[i]->slot, &num_connections, "timeout");
          }
       }
    }
 
-   alarm(timeout);
+   time(&prev_timeout_check);
 }
 
 
@@ -120,27 +128,32 @@ void initialise_configuration(){
    setlocale(LC_MESSAGES, cfg.locale);
    setlocale(LC_CTYPE, cfg.locale);
 
+   load_smtp_acl(smtp_acl);
+
    syslog(LOG_PRIORITY, "reloaded config: %s", configfile);
 }
 
 
 int main(int argc, char **argv){
-   int listenerfd, client_sockfd;
-   int i, n, daemonise=0;
+   int client_sockfd;
+   int i, daemonise=0;
    int client_len = sizeof(struct sockaddr_storage);
    ssize_t readlen;
    struct sockaddr_storage client_address;
-   char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
-   char readbuf[BIGBUFSIZE];
+   char readbuf[REALLYBIGBUFSIZE];
    int efd;
 
-   while((i = getopt(argc, argv, "c:dvVh")) > 0){
+   while((i = getopt(argc, argv, "c:L:dvVh")) > 0){
       switch(i){
 
         case 'c' :
                    configfile = optarg;
                    break;
 
+        case 'L':
+                   loglevel = atoi(optarg);
+                   break;
+
         case 'd' :
                    daemonise = 1;
                    break;
@@ -151,7 +164,7 @@ int main(int argc, char **argv){
                    return 0;
 
         case 'h' :
-        default  : 
+        default  :
                    usage();
       }
    }
@@ -188,7 +201,12 @@ int main(int argc, char **argv){
 
    set_signal_handler(SIGINT, p_clean_exit);
    set_signal_handler(SIGTERM, p_clean_exit);
-   set_signal_handler(SIGALRM, check_for_client_timeout);
+   set_signal_handler(SIGKILL, p_clean_exit);
+   set_signal_handler(SIGSEGV, p_clean_exit);
+
+   set_signal_handler(SIGPIPE, SIG_IGN);
+   set_signal_handler(SIGALRM, SIG_IGN);
+
    set_signal_handler(SIGHUP, initialise_configuration);
 
    // calloc() initialitizes the allocated memory
@@ -209,10 +227,8 @@ int main(int argc, char **argv){
    if(daemonise == 1 && daemon(1, 0) == -1) fatal(ERR_DAEMON);
 #endif
 
-   alarm(timeout);
-
    for(;;){
-      n = epoll_wait(efd, events, cfg.max_connections, -1);
+      int n = epoll_wait(efd, events, cfg.max_connections, 1000);
       for(i=0; i<n; i++){
 
          // Office365 sometimes behaves oddly: when it receives the 250 OK
@@ -223,7 +239,7 @@ int main(int argc, char **argv){
             if(cfg.verbosity >= _LOG_EXTREME) syslog(LOG_PRIORITY, "ERROR: the remote end hung up without sending QUIT");
             session = get_session_by_socket(sessions, cfg.max_connections, events[i].data.fd);
             if(session)
-               tear_down_session(sessions, session->slot, &num_connections);
+               tear_down_session(sessions, session->slot, &num_connections, "hungup");
             else
                close(events[i].data.fd);
             continue;
@@ -242,15 +258,19 @@ int main(int argc, char **argv){
                      break;
                   }
                   else {
-                     syslog(LOG_PRIORITY, "ERROR: accept()");
+                     syslog(LOG_PRIORITY, "ERROR: accept(): '%s'", strerror(errno));
                      break;
                   }
                }
 
+               char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
+               memset(hbuf, 0, sizeof(hbuf));
+               memset(sbuf, 0, sizeof(sbuf));
+
                if(getnameinfo((struct sockaddr *)&client_address, client_len, hbuf, sizeof(hbuf), sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV) == 0){
                   // Strictly speaking it's not correct to log num_connections+1 connections
                   // but it still gives a good clue how many connections we have at the moment
-                  syslog(LOG_PRIORITY, "connected from %s:%s on descriptor %d (active connections: %d)", hbuf, sbuf, client_sockfd, num_connections + 1);
+                  syslog(LOG_PRIORITY, "connected from %s:%s on fd=%d (active connections: %d)", hbuf, sbuf, client_sockfd, num_connections + 1);
                }
 
                if(make_socket_non_blocking(client_sockfd) == -1){
@@ -265,7 +285,7 @@ int main(int argc, char **argv){
                   break;
                }
 
-               start_new_session(sessions, client_sockfd, &num_connections, &cfg);
+               start_new_session(sessions, client_sockfd, &num_connections, smtp_acl, hbuf, &cfg);
             }
 
             continue;
@@ -287,8 +307,6 @@ int main(int argc, char **argv){
             time(&(session->lasttime));
 
             while(1){
-               memset(readbuf, 0, sizeof(readbuf));
-
                if(session->net.use_ssl == 1)
                   readlen = SSL_read(session->net.ssl, (char*)&readbuf[0], sizeof(readbuf)-1);
                else
@@ -309,7 +327,8 @@ int main(int argc, char **argv){
                   break;
                }
 
-               handle_data(session, &readbuf[0], readlen);
+               readbuf[readlen] = '\0';
+               handle_data(session, &readbuf[0], readlen, &cfg);
 
                if(session->protocol_state == SMTP_STATE_BDAT && session->bad == 1){
                   done = 1;
@@ -320,12 +339,14 @@ int main(int argc, char **argv){
             /* Don't wait until the remote client closes the connection after he sent the QUIT command */
 
             if(done || session->protocol_state == SMTP_STATE_FINISHED){
-               tear_down_session(sessions, session->slot, &num_connections);
+               tear_down_session(sessions, session->slot, &num_connections, "done");
             }
          }
 
 
       }
+
+      check_for_client_timeout();
    }
 
    return 0;

src/piler.c

@@ -41,6 +41,9 @@ struct passwd *pwd;
 struct child children[MAXCHILDREN];
 
 
+void p_clean_exit();
+
+
 void usage(){
    printf("\nusage: piler\n\n");
    printf("    -c <config file>                  Config file to use if not the default\n");
@@ -98,12 +101,47 @@ void child_sighup_handler(int sig){
 }
 
 
+int perform_checks(char *filename, struct session_data *sdata, struct data *data, struct parser_state *parser_state, struct config *cfg){
+
+   if(cfg->security_header[0] && parser_state->found_security_header == 0){
+      syslog(LOG_PRIORITY, "%s: discarding: missing security header", filename);
+      return ERR_DISCARDED;
+   }
+
+   char *arule = check_against_ruleset(data->archiving_rules, parser_state, sdata->tot_len, sdata->spam_message);
+
+   if(arule){
+      syslog(LOG_PRIORITY, "%s: discarding: archiving policy: *%s*", filename, arule);
+      return ERR_DISCARDED;
+   }
+
+
+   if(cfg->archive_only_mydomains == 1 && sdata->internal_sender == 0 && sdata->internal_recipient == 0){
+      syslog(LOG_PRIORITY, "%s: discarding: not on mydomains", filename);
+      return ERR_DISCARDED;
+   }
+
+   make_digests(sdata, cfg);
+
+   // A normal header is much bigger than 10 bytes. We get here for header-only
+   // messages without a Message-ID: line. I believe that no such message is valid, and
+   // it's a reasonable to discard it, and not allowing it to fill up the error directory.
+
+   if(sdata->hdr_len < 10){
+      syslog(LOG_PRIORITY, "%s: discarding: a header-only message without a Message-ID line", filename);
+      return ERR_DISCARDED;
+   }
+
+   int rc = process_message(sdata, parser_state, data, cfg);
+   unlink(parser_state->message_id_hash);
+
+   return rc;
+}
+
+
 int process_email(char *filename, struct session_data *sdata, struct data *data, int size, struct config *cfg){
-   int rc;
    char tmpbuf[SMALLBUFSIZE];
    char *status=S_STATUS_UNDEF;
-   char *arule;
-   char *rcpt;
    char *p;
    struct timezone tz;
    struct timeval tv1, tv2;
@@ -128,10 +166,11 @@ int process_email(char *filename, struct session_data *sdata, struct data *data,
    snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
 
    parser_state = parse_message(sdata, 1, data, cfg);
+
    post_parse(sdata, &parser_state, cfg);
 
    if(cfg->syslog_recipients == 1){
-      rcpt = parser_state.b_to;
+      char *rcpt = parser_state.b_to;
       do {
          rcpt = split_str(rcpt, " ", tmpbuf, sizeof(tmpbuf)-1);
 
@@ -141,26 +180,7 @@ int process_email(char *filename, struct session_data *sdata, struct data *data,
       } while(rcpt);
    }
 
-
-   arule = check_againt_ruleset(data->archiving_rules, &parser_state, sdata->tot_len, sdata->spam_message);
-
-   if(arule){
-      syslog(LOG_PRIORITY, "%s: discarding: archiving policy: *%s*", filename, arule);
-      rc = ERR_DISCARDED;
-      remove_stripped_attachments(&parser_state);
-   }
-   else {
-      make_digests(sdata, cfg);
-
-      if(sdata->hdr_len < 10){
-         syslog(LOG_PRIORITY, "%s: invalid message, hdr_len: %d", filename, sdata->hdr_len);
-         rc = ERR;
-      }
-
-      rc = process_message(sdata, &parser_state, data, cfg);
-      unlink(parser_state.message_id_hash);
-   }
-
+   int rc = perform_checks(filename, sdata, data, &parser_state, cfg);
    unlink(sdata->tmpframe);
 
    remove_stripped_attachments(&parser_state);
@@ -267,10 +287,19 @@ void child_main(struct child *ptr){
 
       sig_block(SIGHUP);
 
-      if(open_database(&sdata, &cfg) == OK){
+      int sphxopen = 0;
+      if(cfg.rtindex && open_sphx(&sdata, &cfg) == OK){
+         sphxopen = 1;
+      }
+
+      if((cfg.rtindex == 0 || sphxopen == 1) && open_database(&sdata, &cfg) == OK){
          ptr->messages += process_dir(dir, &sdata, &data, &cfg);
          close_database(&sdata);
 
+         if(cfg.rtindex){
+            close_sphx(&sdata);
+         }
+
          sleep(1);
       }
       else {
@@ -280,13 +309,13 @@ void child_main(struct child *ptr){
 
       sig_unblock(SIGHUP);
 
-      // TODO: do we want to quit after processing a certain number of messages?
-
-      //if(cfg.max_requests_per_child > 0 && ptr->messages >= cfg.max_requests_per_child){
-      //   if(cfg.verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "child (pid: %d, serial: %d) served enough: %d", getpid(), ptr->messages, ptr->serial);
-      //   break;
-      //}
+      // Let the child quit after processing max_requests_per_child messages
 
+      if(cfg.max_requests_per_child > 0 && ptr->messages >= cfg.max_requests_per_child){
+         if(cfg.verbosity >= _LOG_DEBUG)
+            syslog(LOG_PRIORITY, "child (pid: %d, serial: %d) served enough: %d", getpid(), ptr->messages, ptr->serial);
+         break;
+      }
    }
 
 #ifdef HAVE_MEMCACHED
@@ -441,9 +470,9 @@ void initialise_configuration(){
       return;
    }
 
-   load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
-   load_rules(&sdata, &data, data.retention_rules, SQL_RETENTION_RULE_TABLE);
-   load_rules(&sdata, &data, data.folder_rules, SQL_FOLDER_RULE_TABLE);
+   load_rules(&sdata, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
+   load_rules(&sdata, data.retention_rules, SQL_RETENTION_RULE_TABLE);
+   load_rules(&sdata, data.folder_rules, SQL_FOLDER_RULE_TABLE);
 
    load_mydomains(&sdata, &data, &cfg);
 
@@ -461,7 +490,7 @@ void initialise_configuration(){
 
 
 int main(int argc, char **argv){
-   int i, daemonise=0, dedupfd;
+   int i, daemonise=0;
    struct stat st;
 
 
@@ -500,6 +529,7 @@ int main(int argc, char **argv){
    initrules(data.retention_rules);
    initrules(data.folder_rules);
    data.dedup = MAP_FAILED;
+   data.import = NULL;
 
    initialise_configuration();
 
@@ -514,7 +544,7 @@ int main(int argc, char **argv){
    if(stat(cfg.pidfile, &st) == 0) fatal(ERR_PID_FILE_EXISTS);
 
    if(cfg.mmap_dedup_test == 1){
-      dedupfd = open(MESSAGE_ID_DEDUP_FILE, O_RDWR);
+      int dedupfd = open(MESSAGE_ID_DEDUP_FILE, O_RDWR);
       if(dedupfd == -1) fatal(ERR_OPEN_DEDUP_FILE);
 
       data.dedup = mmap(NULL, MAXCHILDREN*DIGEST_LENGTH*2, PROT_READ|PROT_WRITE, MAP_SHARED, dedupfd, 0);

src/piler.h

@@ -17,6 +17,7 @@
 #include <sig.h>
 #include <av.h>
 #include <rules.h>
+#include <screen.h>
 #include <sql.h>
 #include <import.h>
 #include <smtp.h>
@@ -36,7 +37,8 @@ int do_av_check(char *filename, struct config *cfg);
 
 int make_digests(struct session_data *sdata, struct config *cfg);
 void digest_file(char *filename, char *digest);
-void digest_string(char *s, char *digest);
+void digest_string(char *digestname, char *s, char *digest);
+void create_md5_from_email_address(char *puf, char *md5buf);
 
 void remove_stripped_attachments(struct parser_state *state);
 int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
@@ -44,20 +46,21 @@ void rollback(struct session_data *sdata, struct parser_state *state, uint64 id,
 int reimport_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
 int store_file(struct session_data *sdata, char *filename, int len, struct config *cfg);
 int remove_stored_message_files(struct session_data *sdata, struct parser_state *state, struct config *cfg);
-int store_attachments(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
-int query_attachments(struct session_data *sdata, struct data *data, struct ptr_array *ptr_arr);
+int store_attachments(struct session_data *sdata, struct parser_state *state, struct config *cfg);
+int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr);
 
 struct config read_config(char *configfile);
 
+void createdir(char *path, uid_t uid, gid_t gid, mode_t mode);
 void check_and_create_directories(struct config *cfg, uid_t uid, gid_t gid);
 
 void update_counters(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 
-int retrieve_email_from_archive(struct session_data *sdata, struct data *data, FILE *dest, struct config *cfg);
+int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct config *cfg);
 int file_from_archive_to_network(char *filename, int sd, int tls_enable, struct data *data, struct config *cfg);
 
-int get_folder_id(struct session_data *sdata, struct data *data, char *foldername, int parent_id);
-int add_new_folder(struct session_data *sdata, struct data *data, char *foldername, int parent_id);
+int get_folder_id(struct session_data *sdata, char *foldername, int parent_id);
+int add_new_folder(struct session_data *sdata, char *foldername, int parent_id);
 
 int store_index_data(struct session_data *sdata, struct parser_state *state, struct data *data, uint64 id, struct config *cfg);
 
@@ -68,10 +71,11 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
 void load_mydomains(struct session_data *sdata, struct data *data, struct config *cfg);
 int is_email_address_on_my_domains(char *email, struct data *data);
 
-int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct config *cfg);
-void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections);
+int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg);
+void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections, char *reason);
 struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int max_connections, int socket);
-void handle_data(struct smtp_session *session, char *readbuf, int readlen);
+void write_envelope_addresses(struct smtp_session *session, struct config *cfg);
+void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg);
 void free_smtp_session(struct smtp_session *session);
 
 void child_sighup_handler(int sig);
@@ -79,7 +83,6 @@ void child_main(struct child *ptr);
 pid_t child_make(struct child *ptr);
 int search_slot_by_pid(pid_t pid);
 void kill_children(int sig);
-void p_clean_exit();
 void fatal(char *s);
 void initialise_configuration();
 

src/pileraget.c

@@ -46,7 +46,7 @@ int main(int argc, char **argv){
       return 1;
    }
 
-   snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
+   snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, argv[1][24], argv[1][25], argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
    if(stat(filename, &st)){
       snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][RND_STR_LEN-6], argv[1][RND_STR_LEN-5], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
@@ -57,5 +57,3 @@ int main(int argc, char **argv){
 
    return 0;
 }
-
-

src/pilerconf.c

@@ -41,7 +41,7 @@ int main(int argc, char **argv){
                     break;
 
 
-         default  : 
+         default  :
                     break;
        }
    }

src/pilerexport.c

@@ -8,11 +8,13 @@
 #include <sys/time.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/mman.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <time.h>
 #include <locale.h>
 #include <syslog.h>
+#include <zip.h>
 #include <getopt.h>
 #include <piler.h>
 
@@ -22,15 +24,22 @@ extern int optind;
 
 int dryrun = 0;
 int exportall = 0;
-int rc = 0;
+int verification_status = 0;
+int export_to_stdout = 0;
 char *query=NULL;
 int verbosity = 0;
 int max_matches = 1000;
 char *index_list = "main1,dailydelta1,delta1";
+struct passwd *pwd;
 regex_t regexp;
+char *zipfile = NULL;
+struct zip *zip = NULL;
+uint64 *zip_ids = NULL;
+int zip_counter = 0;
+int zip_batch = 2000;
+int max_files_in_export_dir = 0;
 
-
-int export_emails_matching_to_query(struct session_data *sdata, struct data *data, char *s, struct config *cfg);
+int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg);
 
 
 void usage(){
@@ -48,7 +57,13 @@ void usage(){
    printf("    -w <where condition>              Where condition to pass to sphinx, eg. \"match('@subject: piler')\"\n");
    printf("    -m <max. matches>                 Max. matches to apply to sphinx query (default: %d)\n", max_matches);
    printf("    -i <index list>                   Sphinx indices to use  (default: %s)\n", index_list);
+#if LIBZIP_VERSION_MAJOR >= 1
+   printf("    -z <zip file>                     Write exported EML files to a zip file\n");
+   printf("    -Z <batch size>                   Zip batch size. Valid range: 10-10000, default: 2000\n");
+#endif
    printf("    -A                                Export all emails from archive\n");
+   printf("    -D <max files>                    Max. number of files to put in a single directory, default: 2000\n");
+   printf("    -o                                Export emails to stdout\n");
    printf("    -d                                Dry run\n");
 
    regfree(&regexp);
@@ -87,15 +102,12 @@ unsigned long convert_time(char *yyyymmdd, int h, int m, int s){
 
    tm.tm_mday = atoi(yyyymmdd);
 
-
-   tm.tm_isdst = -1;
-
    return mktime(&tm);
 }
 
 
 int append_email_to_buffer(char **buffer, char *email){
-   int len, arglen;
+   int arglen;
    char *s=NULL, emailaddress[SMALLBUFSIZE];
 
    snprintf(emailaddress, sizeof(emailaddress)-1, "'%s'", email);
@@ -107,7 +119,7 @@ int append_email_to_buffer(char **buffer, char *email){
       memcpy(*buffer, emailaddress, arglen);
    }
    else {
-      len = strlen(*buffer);
+      int len = strlen(*buffer);
       s = realloc(*buffer, len + arglen+2);
       if(!s){
          printf("malloc problem!\n");
@@ -125,50 +137,29 @@ int append_email_to_buffer(char **buffer, char *email){
 }
 
 
-int append_string_to_buffer(char **buffer, char *str){
-   int len, arglen;
-   char *s=NULL;
-
-   arglen = strlen(str);
-
-   if(!*buffer){
-      *buffer = malloc(arglen+1);
-      memset(*buffer, 0, arglen+1);
-      memcpy(*buffer, str, arglen);
-   }
-   else {
-      len = strlen(*buffer);
-      s = realloc(*buffer, len + arglen+1);
-      if(!s) return 1;
-
-      *buffer = s;
-
-      memset(*buffer+len, 0, arglen+1);
-      memcpy(*buffer+len, str, arglen);
-   }
-
-   return 0;
-}
-
-
-uint64 run_query(struct session_data *sdata, struct session_data *sdata2, struct data *data, char *where_condition, uint64 last_id, int *num, struct config *cfg){
-   MYSQL_RES *res;
+uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *where_condition, uint64 last_id, int *num, struct config *cfg){
    MYSQL_ROW row;
-   int rc=0;
    uint64 id=0;
-   char s[SMALLBUFSIZE];
+   char s[MAXBUFSIZE];
+   int rc=0;
 
    *num = 0;
 
    if(!where_condition) return id;
 
-   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE id IN (", SQL_METADATA_TABLE);
+   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE id IN (", SQL_METADATA_TABLE);
    rc += append_string_to_buffer(&query, s);
 
    snprintf(s, sizeof(s)-1, "SELECT id FROM %s WHERE %s AND id > %llu ORDER BY id ASC LIMIT 0,%d", index_list, where_condition, last_id, max_matches);
 
+   if(dryrun){
+      printf("sphinx query: %s\n", s);
+   }
+
+   syslog(LOG_PRIORITY, "sphinx query: %s", s);
+
    if(mysql_real_query(&(sdata2->mysql), s, strlen(s)) == 0){
-      res = mysql_store_result(&(sdata2->mysql));
+      MYSQL_RES *res = mysql_store_result(&(sdata2->mysql));
       if(res != NULL){
          while((row = mysql_fetch_row(res))){
             id = strtoull(row[0], NULL, 10);
@@ -183,7 +174,8 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, struct
       }
    }
 
-   if(!rc) export_emails_matching_to_query(sdata, data, query, cfg);
+   if(!rc) export_emails_matching_to_query(sdata, query, cfg);
+   else printf("error: append_string_to_buffer() in run_query()\n");
 
    free(query);
    query = NULL;
@@ -194,12 +186,11 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, struct
 
 
 uint64 get_total_found(struct session_data *sdata){
-   MYSQL_RES *res;
    MYSQL_ROW row;
    uint64 total_found=0;
 
    if(mysql_real_query(&(sdata->mysql), "SHOW META LIKE 'total_found'", 28) == 0){
-      res = mysql_store_result(&(sdata->mysql));
+      MYSQL_RES *res = mysql_store_result(&(sdata->mysql));
       if(res != NULL){
          while((row = mysql_fetch_row(res))){
             total_found = strtoull(row[1], NULL, 10);
@@ -212,17 +203,17 @@ uint64 get_total_found(struct session_data *sdata){
 }
 
 
-void export_emails_matching_id_list(struct session_data *sdata, struct session_data *sdata2, struct data *data, char *where_condition, struct config *cfg){
+void export_emails_matching_id_list(struct session_data *sdata, struct session_data *sdata2, char *where_condition, struct config *cfg){
    int n;
    uint64 count=0, last_id=0, total_found=0;
 
-   last_id = run_query(sdata, sdata2, data, where_condition, last_id, &n, cfg);
+   last_id = run_query(sdata, sdata2, where_condition, last_id, &n, cfg);
    count += n;
 
    total_found = get_total_found(sdata2);
 
    while(count < total_found){
-      last_id = run_query(sdata, sdata2, data, where_condition, last_id, &n, cfg);
+      last_id = run_query(sdata, sdata2, where_condition, last_id, &n, cfg);
       count += n;
    }
 
@@ -230,102 +221,86 @@ void export_emails_matching_id_list(struct session_data *sdata, struct session_d
 
 
 int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain, int minsize, int maxsize, unsigned long startdate, unsigned long stopdate){
-   int where_condition=1;
    char s[SMALLBUFSIZE];
+   int rc=0;
 
    if(exportall == 1){
-      rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM ");
+      rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM ");
       rc += append_string_to_buffer(&query, SQL_METADATA_TABLE);
       rc += append_string_to_buffer(&query, " WHERE deleted=0 ");
       return rc;
    }
 
-   snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE deleted=0 ", SQL_MESSAGES_VIEW);
+   snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE deleted=0 ", SQL_MESSAGES_VIEW);
 
    rc = append_string_to_buffer(&query, s);
 
    if(from){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`from` IN (");
       rc += append_string_to_buffer(&query, from);
       rc += append_string_to_buffer(&query, ")");
 
       free(from);
-
-      where_condition++;
    }
 
    if(to){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`to` IN (");
       rc += append_string_to_buffer(&query, to);
       rc += append_string_to_buffer(&query, ")");
 
       free(to);
-
-      where_condition++;
    }
 
    if(fromdomain){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`fromdomain` IN (");
       rc += append_string_to_buffer(&query, fromdomain);
       rc += append_string_to_buffer(&query, ")");
 
       free(fromdomain);
-
-      where_condition++;
    }
 
 
    if(todomain){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`todomain` IN (");
       rc += append_string_to_buffer(&query, todomain);
       rc += append_string_to_buffer(&query, ")");
 
       free(todomain);
-
-      where_condition++;
    }
 
    if(minsize > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
       snprintf(s, sizeof(s)-1, " `size` >= %d", minsize);
       rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
    }
 
 
    if(maxsize > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
       snprintf(s, sizeof(s)-1, " `size` <= %d", maxsize);
       rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
    }
 
 
    if(startdate > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
-      snprintf(s, sizeof(s)-1, " `sent` >= %ld", startdate);
+      rc = append_string_to_buffer(&query, " AND ");
+      snprintf(s, sizeof(s)-1, " `sent` >= %lu", startdate);
       rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
    }
 
 
    if(stopdate > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
-      snprintf(s, sizeof(s)-1, " `sent` <= %ld", stopdate);
+      rc = append_string_to_buffer(&query, " AND ");
+      snprintf(s, sizeof(s)-1, " `sent` <= %lu", stopdate);
       rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
    }
 
 
@@ -334,14 +309,37 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain
    return rc;
 }
 
+#if LIBZIP_VERSION_MAJOR >= 1
+void zip_flush(){
+   zip_close(zip);
 
-int export_emails_matching_to_query(struct session_data *sdata, struct data *data, char *s, struct config *cfg){
+   zip = NULL;
+   zip_counter = 0;
+
+   if(!zip_ids) return;
+
+   for(int i=0; i<zip_batch; i++){
+      if(*(zip_ids+i)){
+         char filename[SMALLBUFSIZE];
+         snprintf(filename, sizeof(filename)-1, "%llu.eml", *(zip_ids+i));
+         unlink(filename);
+      }
+   }
+
+   free(zip_ids);
+   zip_ids = NULL;
+}
+#endif
+
+int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg){
    FILE *f;
-   uint64 id, n=0;
+   uint64 id, n=0, dir_counter=0;
    char digest[SMALLBUFSIZE], bodydigest[SMALLBUFSIZE];
    char filename[SMALLBUFSIZE];
-   int rc=0;
+   char export_subdir[SMALLBUFSIZE];
    struct sql sql;
+   int errorp, rc=0, attachments;
+   unsigned long total_attachments=0;
 
    if(prepare_sql_statement(sdata, &sql, s) == ERR) return ERR;
 
@@ -358,6 +356,7 @@ int export_emails_matching_to_query(struct session_data *sdata, struct data *dat
    sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++;
    sql.sql[sql.pos] = &digest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(digest)-2; sql.pos++;
    sql.sql[sql.pos] = &bodydigest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(bodydigest)-2; sql.pos++;
+   sql.sql[sql.pos] = (char *)&attachments; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++;
 
    p_store_results(&sql);
 
@@ -367,11 +366,30 @@ int export_emails_matching_to_query(struct session_data *sdata, struct data *dat
 
          if(dryrun == 0){
 
+            if(export_to_stdout){
+               printf("%s", PILEREXPORT_BEGIN_MARK);
+               rc = retrieve_email_from_archive(sdata, stdout, cfg);
+               continue;
+            }
+
+            if(max_files_in_export_dir > 0 && n % max_files_in_export_dir == 0){
+               dir_counter++;
+               snprintf(export_subdir, sizeof(export_subdir)-1, "export-%llu", dir_counter);
+               if(n > 0 && chdir("..")){
+                  p_clean_exit("error chdir(\"..\")", 1);
+               }
+
+               createdir(export_subdir, pwd->pw_uid, pwd->pw_gid, 0700);
+               if(chdir(export_subdir)){
+                  p_clean_exit("error chdir to export-* dir", 1);
+               }
+            }
+
             snprintf(filename, sizeof(filename)-1, "%llu.eml", id);
 
             f = fopen(filename, "w");
             if(f){
-               rc = retrieve_email_from_archive(sdata, data, f, cfg);
+               rc = retrieve_email_from_archive(sdata, f, cfg);
                fclose(f);
 
                n++;
@@ -383,13 +401,48 @@ int export_emails_matching_to_query(struct session_data *sdata, struct data *dat
                if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0){
                   printf("exported: %10llu\r", n); fflush(stdout);
                }
-               else
+               else {
                   printf("verification FAILED. %s\n", filename);
+                  verification_status = 1;
+               }
 
+               if(zipfile){
+               #if LIBZIP_VERSION_MAJOR >= 1
+                  // Open zip file if handler is NULL
+                  if(!zip){
+                     zip = zip_open(zipfile, ZIP_CREATE, &errorp);
+                     if(!zip){
+                        printf("error: error creating zip file=%s, error code=%d\n", zipfile, errorp);
+                        return ERR;
+                     }
+                  }
+
+                  if(!zip_ids) zip_ids = (uint64*) calloc(sizeof(uint64), zip_batch);
+
+                  if(!zip_ids){
+                     printf("calloc error for zip_ids\n");
+                     return ERR;
+                  }
+
+                  zip_source_t *zs = zip_source_file(zip, filename, 0, 0);
+                  if(zs && zip_file_add(zip, filename, zs, ZIP_FL_ENC_UTF_8) >= 0){
+                     *(zip_ids+zip_counter) = id;
+                     zip_counter++;
+                  } else {
+                     printf("error adding file %s: %s\n", filename, zip_strerror(zip));
+                     return ERR;
+                  }
+
+                  if(zip_counter == zip_batch){
+                     zip_flush();
+                  }
+               #endif
+               }
             }
             else printf("cannot open: %s\n", filename);
          }
          else {
+            total_attachments += attachments;
             printf("id:%llu\n", id);
          }
 
@@ -402,6 +455,9 @@ int export_emails_matching_to_query(struct session_data *sdata, struct data *dat
 ENDE:
    close_prepared_statement(&sql);
 
+   if(dryrun){
+      printf("attachments: %lu\n", total_attachments);
+   }
 
    printf("\n");
 
@@ -410,13 +466,12 @@ ENDE:
 
 
 int main(int argc, char **argv){
-   int c, minsize=0, maxsize=0;
+   int minsize=0, maxsize=0;
    size_t nmatch=0;
    unsigned long startdate=0, stopdate=0;
    char *configfile=CONFIG_FILE;
    char *to=NULL, *from=NULL, *todomain=NULL, *fromdomain=NULL, *where_condition=NULL;
    struct session_data sdata, sdata2;
-   struct data data;
    struct config cfg;
 
 
@@ -436,6 +491,7 @@ int main(int argc, char **argv){
             {"all",          no_argument,        0,  'A' },
             {"dry-run",      no_argument,        0,  'd' },
             {"dryrun",       no_argument,        0,  'd' },
+            {"stdout",       no_argument,        0,  'o' },
             {"help",         no_argument,        0,  'h' },
             {"version",      no_argument,        0,  'v' },
             {"from",         required_argument,  0,  'f' },
@@ -444,7 +500,10 @@ int main(int argc, char **argv){
             {"to-domain",    required_argument,  0,  'R' },
             {"start-date",   required_argument,  0,  'a' },
             {"stop-date",    required_argument,  0,  'b' },
+            {"zip",          required_argument,  0,  'z' },
+            {"zip-batch",    required_argument,  0,  'Z' },
             {"where-condition", required_argument,  0,  'w' },
+            {"max-files",    required_argument,  0,  'D' },
             {"max-matches",  required_argument,  0,  'm' },
             {"index-list",   required_argument,  0,  'i' },
             {0,0,0,0}
@@ -452,9 +511,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:D:oAdhv?", long_options, &option_index);
 #else
-      c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?");
+      int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:D:oAdhv?");
 #endif
 
       if(c == -1) break;
@@ -485,7 +544,10 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&from, optarg);
+                    if(append_email_to_buffer(&from, optarg)){
+                       printf("error: append_email_to_buffer() for from\n");
+                       return 1;
+                    }
 
                     break;
 
@@ -496,7 +558,10 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&to, optarg);
+                    if(append_email_to_buffer(&to, optarg)){
+                       printf("error: append_email_to_buffer() for to\n");
+                       return 1;
+                    }
 
                     break;
 
@@ -507,9 +572,12 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&fromdomain, optarg);
+                    if(append_email_to_buffer(&fromdomain, optarg)){
+                       printf("error: append_email_to_buffer() for fromdomain\n");
+                       return 1;
+                    }
 
-                    break;           
+                    break;
 
          case 'R' :
 
@@ -518,10 +586,13 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&todomain, optarg);
+                    if(append_email_to_buffer(&todomain, optarg)){
+                       printf("error: append_email_to_buffer() for todomain\n");
+                       return 1;
+                    }
+
+                    break;
 
-                    break;                     
-                    
          case 'a' :
                     startdate = convert_time(optarg, 0, 0, 0);
                     break;
@@ -543,6 +614,23 @@ int main(int argc, char **argv){
                     index_list = optarg;
                     break;
 
+         case 'z':  zipfile = optarg;
+                    break;
+
+         case 'Z':  zip_batch = atoi(optarg);
+                    if(zip_batch < 10 || zip_batch > 10000)
+                       zip_batch = 2000;
+                    break;
+
+         case 'D':  max_files_in_export_dir = atoi(optarg);
+                    if(max_files_in_export_dir < 10 || max_files_in_export_dir > 100000)
+                       max_files_in_export_dir = 2000;
+                    break;
+
+         case 'o':
+                    export_to_stdout = 1;
+                    break;
+
          case 'd' :
                     dryrun = 1;
                     break;
@@ -570,6 +658,11 @@ int main(int argc, char **argv){
 
    if(read_key(&cfg)) p_clean_exit(ERR_READING_KEY, 1);
 
+   if(strlen(cfg.username) > 1){
+      pwd = getpwnam(cfg.username);
+      if(!pwd) __fatal(ERR_NON_EXISTENT_USER);
+   }
+
 
    init_session_data(&sdata, &cfg);
 
@@ -591,19 +684,23 @@ int main(int argc, char **argv){
          p_clean_exit("cannot connect to 127.0.0.1:9306", 1);
       }
 
-      export_emails_matching_id_list(&sdata, &sdata2, &data, where_condition, &cfg);
+      export_emails_matching_id_list(&sdata, &sdata2, where_condition, &cfg);
 
       close_database(&sdata2);
    }
    else {
       if(build_query_from_args(from, to, fromdomain, todomain, minsize, maxsize, startdate, stopdate) > 0) p_clean_exit("malloc problem building query", 1);
-      export_emails_matching_to_query(&sdata, &data, query, &cfg);
+      export_emails_matching_to_query(&sdata, query, &cfg);
       free(query);
    }
 
    close_database(&sdata);
 
-   return 0;
+   if(zipfile){
+   #if LIBZIP_VERSION_MAJOR >= 1
+      zip_flush();
+   #endif
+   }
+
+   return verification_status;
 }
-
-

src/pilerget.c

@@ -19,7 +19,6 @@
 int main(int argc, char **argv){
    int readkey=1;
    struct session_data sdata;
-   struct data data;
    struct config cfg;
 
 
@@ -48,12 +47,10 @@ int main(int argc, char **argv){
 
    snprintf(sdata.ttmpfile, SMALLBUFSIZE-1, "%s", argv[1]);
    snprintf(sdata.filename, SMALLBUFSIZE-1, "%s", sdata.ttmpfile);
-   retrieve_email_from_archive(&sdata, &data, stdout, &cfg);
+   retrieve_email_from_archive(&sdata, stdout, &cfg);
 
 
    close_database(&sdata);
 
    return 0;
 }
-
-

src/pilerimport.c

@@ -52,24 +52,32 @@ void usage(){
    printf("    -s <start position>               Start importing POP3 emails from this position\n");
    printf("    -j <failed folder>                Move failed to import emails to this folder\n");
    printf("    -a <recipient>                    Add recipient to the To:/Cc: list\n");
+   printf("    -T <id>                           Update import table at id=<id>\n");
+   printf("    -Z <ms>                           Delay Z milliseconds in between emails being imported\n");
    printf("    -D                                Dry-run, do not import anything\n");
+   printf("    -y                                Read pilerexport data from stdin\n");
    printf("    -o                                Only download emails for POP3/IMAP import\n");
    printf("    -r                                Remove imported emails\n");
    printf("    -q                                Quiet mode\n");
+   printf("    -A <timestamp>                    Import emails sent after this timestamp\n");
+   printf("    -B <timestamp>                    Import emails sent before this timestamp\n");
 
    exit(0);
 }
 
 
 int main(int argc, char **argv){
-   int i, c, n_mbox=0;
+   int i, n_mbox=0, read_from_pilerexport=0;
    char *configfile=CONFIG_FILE, *mbox[MBOX_ARGS], *directory=NULL;
-   char *imapserver=NULL, *pop3server=NULL;
+   char puf[SMALLBUFSIZE], *imapserver=NULL, *pop3server=NULL;
    struct session_data sdata;
    struct config cfg;
    struct data data;
    struct import import;
    struct net net;
+   struct counters counters;
+
+   bzero(&counters, sizeof(counters));
 
    for(i=0; i<MBOX_ARGS; i++) mbox[i] = NULL;
 
@@ -96,7 +104,11 @@ int main(int argc, char **argv){
    memset(import.filename, 0, SMALLBUFSIZE);
    import.mboxdir = NULL;
    import.tot_msgs = 0;
+   import.table_id = 0;
    import.folder = NULL;
+   import.delay = 0;
+   import.after = 0;
+   import.before = 0;
 
    data.import = &import;
 
@@ -132,12 +144,17 @@ int main(int argc, char **argv){
             {"batch-limit",  required_argument,  0,  'b' },
             {"timeout",      required_argument,  0,  't' },
             {"start-position",required_argument,  0,  's' },
+            {"table-id",     required_argument,  0,  'T' },
+            {"delay",        required_argument,  0,  'Z' },
             {"quiet",        no_argument,        0,  'q' },
-            {"recursive",    required_argument,  0,  'R' },
+            {"recursive",    no_argument,        0,  'R' },
             {"remove-after-import",no_argument,  0,  'r' },
             {"failed-folder",  required_argument,  0,  'j' },
             {"move-folder",  required_argument,  0,  'g' },
+            {"after",        required_argument,  0,  'A' },
+            {"before",       required_argument,  0,  'B' },
             {"only-download",no_argument,        0,  'o' },
+            {"read-from-export",no_argument,     0,  'y' },
             {"dry-run",      no_argument,        0,  'D' },
             {"help",         no_argument,        0,  'h' },
             {0,0,0,0}
@@ -145,9 +162,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:DRroqh?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:A:B:yDRroqh?", long_options, &option_index);
 #else
-      c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:DRroqh?");
+      int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:A:B:yDRroqh?");
 #endif
 
       if(c == -1) break;
@@ -253,7 +270,30 @@ int main(int argc, char **argv){
                     break;
 
          case 'a' :
-                    data.import->extra_recipient = optarg;
+                    snprintf(puf, sizeof(puf)-1, "%s ", optarg);
+                    data.import->extra_recipient = puf;
+                    break;
+
+         case 'T' :
+                    if(atoi(optarg) < 1){
+                       printf("invalid import table id: %s\n", optarg);
+                       return -1;
+                    }
+
+                    data.import->table_id = atoi(optarg);
+                    break;
+
+         case 'Z' :
+                    if(atoi(optarg) < 1){
+                       printf("invalid delay value: %s\n", optarg);
+                       return -1;
+                    }
+
+                    data.import->delay = atoi(optarg);
+                    break;
+
+         case 'y' :
+                    read_from_pilerexport = 1;
                     break;
 
          case 'D' :
@@ -264,19 +304,25 @@ int main(int argc, char **argv){
                     data.quiet = 1;
                     break;
 
+         case 'A' : data.import->after = atol(optarg);
+                    break;
+
+         case 'B' : data.import->before = atol(optarg);
+                    break;
+
          case 'h' :
          case '?' :
                     usage();
                     break;
 
 
-         default  : 
+         default  :
                     break;
        }
    }
 
 
-   if(!mbox[0] && !data.import->mboxdir && !data.import->filename && !directory && !imapserver && !pop3server) usage();
+   if(!mbox[0] && !data.import->mboxdir && !data.import->filename[0] && !directory && !imapserver && !pop3server && !read_from_pilerexport) usage();
 
    if(data.import->failed_folder && !can_i_write_directory(data.import->failed_folder)){
       printf("cannot write failed directory '%s'\n", data.import->failed_folder);
@@ -287,6 +333,8 @@ int main(int argc, char **argv){
 
    cfg = read_config(configfile);
 
+   memset(cfg.security_header, 0, MAXVAL);
+
    if((data.recursive_folder_names == 1 || data.import->folder) && cfg.enable_folders == 0){
       printf("please set enable_folders=1 in piler.conf to use the folder options\n");
       return ERR;
@@ -295,6 +343,9 @@ int main(int argc, char **argv){
    /* make sure we don't discard messages without a valid Message-Id when importing manually */
    cfg.archive_emails_not_having_message_id = 1;
 
+   /* The mmap_dedup_test feature is expected to work with the piler daemon only */
+   cfg.mmap_dedup_test = 0;
+
    if(read_key(&cfg)){
       printf("%s\n", ERR_READING_KEY);
       return ERR;
@@ -305,6 +356,7 @@ int main(int argc, char **argv){
 
    if(open_database(&sdata, &cfg) == ERR) return 0;
 
+   if(cfg.rtindex && open_sphx(&sdata, &cfg) == ERR) return 0;
 
    setlocale(LC_CTYPE, cfg.locale);
 
@@ -315,10 +367,10 @@ int main(int argc, char **argv){
 #endif
 
    if(data.import->folder){
-      data.folder = get_folder_id(&sdata, &data, data.import->folder, 0);
+      data.folder = get_folder_id(&sdata, data.import->folder, 0);
 
       if(data.folder == ERR_FOLDER){
-         data.folder = add_new_folder(&sdata, &data, data.import->folder, 0);
+         data.folder = add_new_folder(&sdata, data.import->folder, 0);
       }
 
       if(data.folder == ERR_FOLDER){
@@ -329,23 +381,24 @@ int main(int argc, char **argv){
 
    }
 
-   load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
-   load_rules(&sdata, &data, data.retention_rules, SQL_RETENTION_RULE_TABLE);
-   load_rules(&sdata, &data, data.folder_rules, SQL_FOLDER_RULE_TABLE);
+   load_rules(&sdata, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
+   load_rules(&sdata, data.retention_rules, SQL_RETENTION_RULE_TABLE);
+   load_rules(&sdata, data.folder_rules, SQL_FOLDER_RULE_TABLE);
 
    load_mydomains(&sdata, &data, &cfg);
 
-   if(data.import->filename[0] != '\0') import_message(&sdata, &data, &cfg);
+   if(data.import->filename[0] != '\0') import_message(&sdata, &data, &counters, &cfg);
 
    if(mbox[0]){
       for(i=0; i<n_mbox; i++){
-         import_from_mailbox(mbox[i], &sdata, &data, &cfg);
+         import_from_mailbox(mbox[i], &sdata, &data, &counters, &cfg);
       }
    }
-   if(data.import->mboxdir) import_mbox_from_dir(data.import->mboxdir, &sdata, &data, &cfg);
-   if(directory) import_from_maildir(&sdata, &data, directory, &cfg);
-   if(imapserver) import_from_imap_server(&sdata, &data, &cfg);
-   if(pop3server) import_from_pop3_server(&sdata, &data, &cfg);
+   if(data.import->mboxdir) import_mbox_from_dir(data.import->mboxdir, &sdata, &data, &counters, &cfg);
+   if(directory) import_from_maildir(&sdata, &data, directory, &counters, &cfg);
+   if(imapserver) import_from_imap_server(&sdata, &data, &counters, &cfg);
+   if(pop3server) import_from_pop3_server(&sdata, &data, &counters, &cfg);
+   if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &counters, &cfg);
 
    clearrules(data.archiving_rules);
    clearrules(data.retention_rules);
@@ -353,8 +406,14 @@ int main(int argc, char **argv){
 
    clearhash(data.mydomains);
 
+   update_counters(&sdata, &data, &counters, &cfg);
+
+   syslog(LOG_PRIORITY, "server=%s, user=%s, directory=%s, imported=%lld, duplicated=%lld, discarded=%lld", data.import->server, data.import->username, directory, counters.c_rcvd, counters.c_duplicate, counters.c_ignore);
+
    close_database(&sdata);
 
+   if(cfg.rtindex) close_sphx(&sdata);
+
    if(data.quiet == 0) printf("\n");
 
    return 0;

src/pop3.c

@@ -23,59 +23,13 @@
 #include <piler.h>
 
 
-int is_last_complete_pop3_packet(char *s, int len){
-
-   if(*(s+len-5) == '\r' && *(s+len-4) == '\n' && *(s+len-3) == '.' && *(s+len-2) == '\r' && *(s+len-1) == '\n'){
-      return 1;
-   }
-
-   return 0;
-}
-
-
 int connect_to_pop3_server(struct data *data){
-   int n;
    char buf[MAXBUFSIZE];
-   X509* server_cert;
-   char *str;
-
 
    if(data->net->use_ssl == 1){
-
-      SSL_library_init();
-      SSL_load_error_strings();
-
-   #if OPENSSL_VERSION_NUMBER < 0x10100000L
-      data->net->ctx = SSL_CTX_new(TLSv1_client_method());
-   #else
-      data->net->ctx = SSL_CTX_new(TLS_client_method());
-   #endif
-      CHK_NULL(data->net->ctx, "internal SSL error");
-
-      data->net->ssl = SSL_new(data->net->ctx);
-      CHK_NULL(data->net->ssl, "internal ssl error");
-
-      SSL_set_fd(data->net->ssl, data->net->socket);
-      n = SSL_connect(data->net->ssl);
-      CHK_SSL(n, "internal ssl error");
-
-      printf("Cipher: %s\n", SSL_get_cipher(data->net->ssl));
-
-      server_cert = SSL_get_peer_certificate(data->net->ssl);
-      CHK_NULL(server_cert, "server cert error");
-
-      str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0);
-      CHK_NULL(str, "error in server cert");
-      OPENSSL_free(str);
-
-      str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0);
-      CHK_NULL(str, "error in server cert");
-      OPENSSL_free(str);
-
-      X509_free(server_cert);
+      init_ssl_to_server(data);
    }
 
-
    recvtimeoutssl(data->net, buf, sizeof(buf));
 
 
@@ -99,7 +53,7 @@ int connect_to_pop3_server(struct data *data){
 
 
 void get_number_of_total_messages(struct data *data){
-   char *p, buf[MAXBUFSIZE];
+   char buf[MAXBUFSIZE];
 
    data->import->total_messages = 0;
 
@@ -109,7 +63,7 @@ void get_number_of_total_messages(struct data *data){
    recvtimeoutssl(data->net, buf, sizeof(buf));
 
    if(strncmp(buf, "+OK ", 4) == 0){
-      p = strchr(&buf[4], ' ');
+      char *p = strchr(&buf[4], ' ');
       if(p){
          *p = '\0';
          data->import->total_messages = atoi(&buf[4]);
@@ -122,67 +76,86 @@ void get_number_of_total_messages(struct data *data){
 
 
 int pop3_download_email(struct data *data, int i){
-   int n, fd, pos=0, readlen=0, lastpos=0, nreads=0;
-   char *p, buf[MAXBUFSIZE];
-   char aggrbuf[3*MAXBUFSIZE];
+   char *p, buf[MAXBUFSIZE], savedbuf[MAXBUFSIZE], copybuf[2*MAXBUFSIZE];
 
    data->import->processed_messages++;
 
    snprintf(data->import->filename, SMALLBUFSIZE-1, "pop3-tmp-%d-%d.txt", getpid(), i);
    unlink(data->import->filename);
 
-   fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
+   int fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
    if(fd == -1){
       printf("cannot open: %s\n", data->import->filename);
       return ERR;
    }
 
+   memset(savedbuf, 0, sizeof(savedbuf));
+
    snprintf(buf, sizeof(buf)-1, "RETR %d\r\n", i);
    write1(data->net, buf, strlen(buf));
 
-   memset(aggrbuf, 0, sizeof(aggrbuf));
+   int nlines = 0;
+   int endofmessage = 0;
+   int savedlen = 0;
+   int n = 0;
 
    while((n = recvtimeoutssl(data->net, buf, sizeof(buf))) > 0){
-      nreads++;
-      readlen += n;
+      int remaininglen = n;
 
-      if(nreads == 1){
+      if(savedlen){
+         memset(copybuf, 0, sizeof(copybuf));
+         memcpy(copybuf, savedbuf, savedlen);
+         memcpy(&copybuf[savedlen], buf, n);
 
-         if(strncmp(buf, "+OK", 3) == 0){
-            p = strchr(&buf[3], '\n');
-            if(p){
-               *p = '\0';
-               pos = strlen(buf)+1;
-               *p = '\n';
+         remaininglen += savedlen;
+
+         savedlen = 0;
+         memset(savedbuf, 0, sizeof(savedbuf));
+
+         p = &copybuf[0];
+      } else {
+         p = &buf[0];
+      }
+
+      int puflen=0;
+      int rc=OK;
+      int nullbyte=0;
+
+      do {
+         char puf[MAXBUFSIZE];
+
+         puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf)-1, &rc, &nullbyte);
+         remaininglen -= puflen;
+         nlines++;
+
+         if(nlines == 1){
+            if(strncmp(puf, "+OK", 3)){
+               printf("error: %s", puf);
+               return ERR;
+            }
+         } else {
+            if(puf[puflen-1] == '\n'){
+               if(puflen == 3 && puf[0] == '.' && puf[1] == '\r' && puf[2] == '\n'){
+                  endofmessage = 1;
+                  break;
+               }
+
+               int dotstuff = 0;
+               if(puf[0] == '.' && puf[1] != '\r' && puf[1] != '\n') dotstuff = 1;
+
+               if(write(fd, &puf[dotstuff], puflen-dotstuff) == -1) printf("ERROR: writing to fd\n");
+
+            } else if(puflen > 0) {
+               savedlen = puflen;
+               snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
             }
          }
-         else { printf("error: %s", buf); return ERR; }
 
-      }
+         p += puflen;
 
-      if((uint)(lastpos + 1 + n) < sizeof(aggrbuf)){
+      } while(puflen > 0);
 
-         if(nreads == 1){
-            memcpy(aggrbuf+lastpos, buf+pos, n-pos);
-            lastpos += n-pos;
-         }
-         else {
-            memcpy(aggrbuf+lastpos, buf, n);
-            lastpos += n;
-         }
-      }
-      else {
-         if(write(fd, aggrbuf, sizeof(buf)) == -1) printf("ERROR: writing to fd\n");
-
-         memmove(aggrbuf, aggrbuf+sizeof(buf), lastpos-sizeof(buf));
-         lastpos -= sizeof(buf);
-
-         memcpy(aggrbuf+lastpos, buf, n);
-         lastpos += n;
-      }
-
-      if(is_last_complete_pop3_packet(aggrbuf, lastpos) == 1){
-         if(write(fd, aggrbuf, lastpos-3) == -1) printf("ERROR: writing to fd\n");
+      if(endofmessage){
          break;
       }
    }
@@ -202,8 +175,7 @@ void pop3_delete_message(struct data *data, int i){
 }
 
 
-void process_pop3_emails(struct session_data *sdata, struct data *data, struct config *cfg){
-   int i=0, rc=ERR;
+void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    char buf[MAXBUFSIZE];
 
    data->import->processed_messages = 0;
@@ -214,12 +186,12 @@ void process_pop3_emails(struct session_data *sdata, struct data *data, struct c
 
    if(data->import->total_messages <= 0) return;
 
-   for(i=data->import->start_position; i<=data->import->total_messages; i++){
+   for(int i=data->import->start_position; i<=data->import->total_messages; i++){
       if(pop3_download_email(data, i) == OK){
          if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/data->import->total_messages); fflush(stdout); }
 
          if(data->import->dryrun == 0){
-            rc = import_message(sdata, data, cfg);
+            int rc = import_message(sdata, data, counters, cfg);
 
             if(data->import->remove_after_import == 1 && rc == OK){
                pop3_delete_message(data, i);

src/reindex.c

@@ -42,7 +42,7 @@ void p_clean_exit(char *msg, int rc){
 }
 
 
-uint64 get_max_meta_id(struct session_data *sdata, struct data *data){
+uint64 get_max_meta_id(struct session_data *sdata){
    char s[SMALLBUFSIZE];
    uint64 id=0;
    struct sql sql;
@@ -74,16 +74,11 @@ uint64 get_max_meta_id(struct session_data *sdata, struct data *data){
 
 
 uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *data, uint64 from_id, uint64 to_id, struct config *cfg){
-   FILE *f;
-   char filename[SMALLBUFSIZE];
    char s[SMALLBUFSIZE];
-   int rc=0;
-   uint64 stored_id=0, reindexed=0, delta;
+   uint64 stored_id=0, reindexed=0;
    struct parser_state state;
    struct sql sql;
 
-   delta = to_id - from_id;
-
    if(cfg->enable_folders == 1)
       snprintf(s, sizeof(s)-1, "SELECT m.`id`, `piler_id`, `arrived`, `sent`, f.folder_id FROM %s m, %s f WHERE m.id=f.id AND (m.id BETWEEN %llu AND %llu) AND `deleted`=0", SQL_METADATA_TABLE, SQL_FOLDER_MESSAGE_TABLE, from_id, to_id);
    else
@@ -110,45 +105,54 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da
 
       while(p_fetch_results(&sql) == OK){
 
-         if(stored_id > 0){
+         char filename[SMALLBUFSIZE];
+         snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id);
 
-            snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id);
-
-            f = fopen(filename, "w");
-            if(f){
-               rc = retrieve_email_from_archive(sdata, data, f, cfg);
-               fclose(f);
-
-               if(rc){
-                  printf("cannot retrieve: %s\n", filename);
-                  unlink(filename);
-                  continue;
-               }
-
-               snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
-
-               state = parse_message(sdata, 0, data, cfg);
-               post_parse(sdata, &state, cfg);
-
-               rc = store_index_data(sdata, &state, data, stored_id, cfg);
-
-               if(rc == OK) reindexed++;
-               else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename);
+         FILE *f = fopen(filename, "w");
+         if(f){
+            int rc = retrieve_email_from_archive(sdata, f, cfg);
+            fclose(f);
 
+            if(rc){
+               printf("cannot retrieve: %s\n", filename);
                unlink(filename);
-
-               if(progressbar){
-                  printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta));
-                  fflush(stdout);
-               }
-
+               continue;
+            }
+
+            snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
+
+            struct stat st;
+            sdata->tot_len = stat(filename, &st) == 0 ? st.st_size : 0;
+
+            sdata->internal_sender = sdata->internal_recipient = sdata->external_recipient = sdata->direction = 0;
+            memset(sdata->attachments, 0, SMALLBUFSIZE);
+
+            state = parse_message(sdata, 1, data, cfg);
+            post_parse(sdata, &state, cfg);
+
+            rc = store_index_data(sdata, &state, data, stored_id, cfg);
+
+            unlink(sdata->tmpframe);
+            remove_stripped_attachments(&state);
+
+            if(rc == OK) reindexed++;
+            else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename);
+
+            unlink(filename);
+
+            if(progressbar){
+               uint64 delta = to_id - from_id + 1;
+
+               printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta));
+               fflush(stdout);
             }
-            else printf("cannot open: %s\n", filename);
 
          }
+         else printf("cannot open: %s\n", filename);
 
       }
 
+
       p_free_results(&sql);
    }
 
@@ -162,7 +166,7 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da
 
 
 int main(int argc, char **argv){
-   int c, all=0;
+   int all=0;
    uint64 from_id=0, to_id=0, n=0;
    char *configfile=CONFIG_FILE, *folder=NULL;
    struct session_data sdata;
@@ -171,7 +175,7 @@ int main(int argc, char **argv){
 
 
    while(1){
-      c = getopt(argc, argv, "c:f:t:F:pahv?");
+      int c = getopt(argc, argv, "c:f:t:F:pahv?");
 
       if(c == -1) break;
 
@@ -211,7 +215,7 @@ int main(int argc, char **argv){
    }
 
 
-   if(all == 0 && (from_id <= 0 || to_id <= 0) ) usage();
+   if(all == 0 && (from_id == 0 || to_id == 0) ) usage();
 
    if(!can_i_write_directory(NULL)) __fatal("cannot write current directory!");
 
@@ -241,10 +245,15 @@ int main(int argc, char **argv){
       p_clean_exit("cannot connect to mysql server", 1);
    }
 
-   load_rules(&sdata, &data, data.folder_rules, SQL_FOLDER_RULE_TABLE);
+   if(cfg.rtindex && open_sphx(&sdata, &cfg) == ERR){
+      p_clean_exit("cannot connect to 127.0.0.1:9306", 1);
+   }
+
+
+   load_rules(&sdata, data.folder_rules, SQL_FOLDER_RULE_TABLE);
 
    if(folder){
-      data.folder = get_folder_id(&sdata, &data, folder, 0);
+      data.folder = get_folder_id(&sdata, folder, 0);
       if(data.folder == 0){
          printf("error: could not get folder id for '%s'\n", folder);
          return 0;
@@ -256,7 +265,7 @@ int main(int argc, char **argv){
 
    if(all == 1){
       from_id = 1;
-      to_id = get_max_meta_id(&sdata, &data);
+      to_id = get_max_meta_id(&sdata);
    }
 
    n = retrieve_email_by_metadata_id(&sdata, &data, from_id, to_id, &cfg);
@@ -268,6 +277,7 @@ int main(int argc, char **argv){
    clearhash(data.mydomains);
 
    close_database(&sdata);
+   if(cfg.rtindex) close_sphx(&sdata);
 
    return 0;
 }

src/rules.c

@@ -10,22 +10,31 @@
 #include "rules.h"
 
 
-void load_rules(struct session_data *sdata, struct data *data, struct node *xhash[], char *table){
+void reset_rule_condition(struct rule_cond *rule_cond){
+   memset(rule_cond->domain, 0, SMALLBUFSIZE);
+   memset(rule_cond->from, 0, SMALLBUFSIZE);
+   memset(rule_cond->to, 0, SMALLBUFSIZE);
+   memset(rule_cond->subject, 0, SMALLBUFSIZE);
+   memset(rule_cond->body, 0, SMALLBUFSIZE);
+   memset(rule_cond->_size, 0, SMALLBUFSIZE);
+   memset(rule_cond->attachment_name, 0, SMALLBUFSIZE);
+   memset(rule_cond->attachment_type, 0, SMALLBUFSIZE);
+   memset(rule_cond->_attachment_size, 0, SMALLBUFSIZE);
+
+   rule_cond->size = 0;
+   rule_cond->attachment_size = 0;
+   rule_cond->spam = 0;
+   rule_cond->days = 0;
+   rule_cond->folder_id = 0;
+}
+
+
+void load_rules(struct session_data *sdata, struct node *xhash[], char *table){
    char s[SMALLBUFSIZE];
    struct rule_cond rule_cond;
    struct sql sql;
 
-   memset(rule_cond.domain, 0, SMALLBUFSIZE);
-   memset(rule_cond.from, 0, SMALLBUFSIZE);
-   memset(rule_cond.to, 0, SMALLBUFSIZE);
-   memset(rule_cond.subject, 0, SMALLBUFSIZE);
-   memset(rule_cond.body, 0, SMALLBUFSIZE);
-   memset(rule_cond._size, 0, SMALLBUFSIZE);
-   memset(rule_cond.attachment_name, 0, SMALLBUFSIZE);
-   memset(rule_cond.attachment_type, 0, SMALLBUFSIZE);
-   memset(rule_cond._attachment_size, 0, SMALLBUFSIZE);
-
-   rule_cond.size = rule_cond.attachment_size = rule_cond.spam = rule_cond.days = rule_cond.folder_id = 0;
+   reset_rule_condition(&rule_cond);
 
    snprintf(s, sizeof(s)-1, "SELECT `domain`, `from`, `to`, `subject`, `body`, `_size`, `size`, `attachment_name`, `attachment_type`, `_attachment_size`, `attachment_size`, `spam`, `days`, `folder_id` FROM `%s`", table);
 
@@ -58,18 +67,7 @@ void load_rules(struct session_data *sdata, struct data *data, struct node *xhas
 
       while(p_fetch_results(&sql) == OK){
          append_rule(xhash, &rule_cond);
-
-         memset(rule_cond.domain, 0, SMALLBUFSIZE);
-         memset(rule_cond.from, 0, SMALLBUFSIZE);
-         memset(rule_cond.to, 0, SMALLBUFSIZE);
-         memset(rule_cond.subject, 0, SMALLBUFSIZE);
-         memset(rule_cond.body, 0, SMALLBUFSIZE);
-         memset(rule_cond._size, 0, SMALLBUFSIZE);
-         memset(rule_cond.attachment_name, 0, SMALLBUFSIZE);
-         memset(rule_cond.attachment_type, 0, SMALLBUFSIZE);
-         memset(rule_cond._attachment_size, 0, SMALLBUFSIZE);
-
-         rule_cond.size = rule_cond.attachment_size = rule_cond.spam = rule_cond.days = rule_cond.folder_id = 0;
+         reset_rule_condition(&rule_cond);
       }
 
       p_free_results(&sql);
@@ -120,7 +118,6 @@ int append_rule(struct node *xhash[], struct rule_cond *rule_cond){
 
 struct rule *create_rule_item(struct rule_cond *rule_cond){
    struct rule *h=NULL;
-   char empty = '\0';
    int len;
 
    if(rule_cond == NULL) return NULL;
@@ -146,16 +143,16 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){
    h->emptyfrom = h->emptyto = h->emptysubject = h->emptyaname = h->emptyatype = 0;
 
 
-   if(rule_cond->from == NULL || strlen(rule_cond->from) < 1){ rule_cond->from[0] = empty; h->emptyfrom = 1; }
+   if(rule_cond->from[0] == 0){ h->emptyfrom = 1; }
    if(regcomp(&(h->from), rule_cond->from, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->to == NULL || strlen(rule_cond->to) < 1){ rule_cond->to[0] = empty; h->emptyto = 1; }
+   if(rule_cond->to[0] == 0){ h->emptyto = 1; }
    if(regcomp(&(h->to), rule_cond->to, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->subject == NULL || strlen(rule_cond->subject) < 1){ rule_cond->subject[0] = empty; h->emptysubject = 1; }
+   if(rule_cond->subject[0] == 0){ h->emptysubject = 1; }
    if(regcomp(&(h->subject), rule_cond->subject, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->body == NULL || strlen(rule_cond->body) < 1){ rule_cond->body[0] = empty; h->emptybody = 1; }
+   if(rule_cond->body[0] == 0){ h->emptybody = 1; }
    if(regcomp(&(h->body), rule_cond->body, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
    h->spam = rule_cond->spam;
@@ -163,20 +160,16 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){
    h->folder_id = rule_cond->folder_id;
 
    h->size = rule_cond->size;
-
-   if(rule_cond->_size == NULL) rule_cond->_size[0] = empty;
    snprintf(h->_size, 3, "%s", rule_cond->_size);
 
-   if(rule_cond->attachment_name == NULL || strlen(rule_cond->attachment_name) < 1){ rule_cond->attachment_name[0] = empty; h->emptyaname = 1; }
+   if(rule_cond->attachment_name[0] == 0){ h->emptyaname = 1; }
    if(regcomp(&(h->attachment_name), rule_cond->attachment_name, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->attachment_type == NULL || strlen(rule_cond->attachment_type) < 1){ rule_cond->attachment_type[0] = empty; h->emptyatype = 1; }
+   if(rule_cond->attachment_type[0] == 0){ h->emptyatype = 1; }
    if(regcomp(&(h->attachment_type), rule_cond->attachment_type, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
 
    h->attachment_size = rule_cond->attachment_size;
-
-   if(rule_cond->_attachment_size == NULL) rule_cond->_attachment_size[0] = empty;
    snprintf(h->_attachment_size, 3, "%s", rule_cond->_attachment_size);
 
    len = strlen(rule_cond->domain)+8 + strlen(rule_cond->from)+6 + strlen(rule_cond->to)+4 + strlen(rule_cond->subject)+9 + strlen(rule_cond->body)+6 + strlen(rule_cond->_size)+6 + strlen(rule_cond->attachment_name)+10 + strlen(rule_cond->attachment_type)+10 + strlen(rule_cond->_attachment_size)+10 + 9 + 15 + 15;
@@ -184,7 +177,10 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){
 
 
 
-   if(h->rulestr) snprintf(h->rulestr, len-1, "domain=%s,from=%s,to=%s,subject=%s,body=%s,size%s%d,att.name=%s,att.type=%s,att.size%s%d,spam=%d", rule_cond->domain, rule_cond->from, rule_cond->to, rule_cond->subject, rule_cond->body, rule_cond->_size, rule_cond->size, rule_cond->attachment_name, rule_cond->attachment_type, rule_cond->_attachment_size, rule_cond->attachment_size, rule_cond->spam);
+   if(h->rulestr){
+      snprintf(h->rulestr, len-1, "domain=%s,from=%s,to=%s,subject=%s,body=%s,size%s%d,att.name=%s,att.type=%s,att.size%s%d,spam=%d", rule_cond->domain, rule_cond->from, rule_cond->to, rule_cond->subject, rule_cond->body, rule_cond->_size, rule_cond->size, rule_cond->attachment_name, rule_cond->attachment_type, rule_cond->_attachment_size, rule_cond->attachment_size, rule_cond->spam);
+      syslog(LOG_INFO, "adding rule: %s", h->rulestr);
+   }
    else h->compiled = 0;
 
    h->r = NULL;
@@ -193,11 +189,44 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){
 }
 
 
-char *check_againt_ruleset(struct node *xhash[], struct parser_state *state, int size, int spam){
-   size_t nmatch=0;
+int count_match(struct rule *p, struct parser_state *state, int size, int spam){
+   int ismatch=0;
+
+   ismatch += check_spam_rule(spam, p->spam);
+   ismatch += check_size_rule(size, p->size, p->_size);
+   ismatch += check_attachment_rule(state, p);
+
+   if(p->compiled == 1){
+      size_t nmatch=0;
+
+      if(p->emptyfrom == 1){
+         ismatch += RULE_UNDEF;
+      }
+      else if(regexec(&(p->from), state->b_from, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+      if(p->emptyto == 1){
+         ismatch += RULE_UNDEF;
+      }
+      else if(regexec(&(p->to), state->b_to, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+      if(p->emptysubject == 1){
+         ismatch += RULE_UNDEF;
+      }
+      else if(regexec(&(p->subject), state->b_subject, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+      if(p->emptybody == 1){
+         ismatch += RULE_UNDEF;
+      }
+      else if(regexec(&(p->body), state->b_body, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+   }
+
+   return ismatch;
+}
+
+
+char *check_against_ruleset(struct node *xhash[], struct parser_state *state, int size, int spam){
    struct rule *p;
    struct node *q;
-   int ismatch;
 
    q = xhash[0];
 
@@ -206,40 +235,8 @@ char *check_againt_ruleset(struct node *xhash[], struct parser_state *state, int
       if(q->str){
          p = q->str;
 
-         if(p){
-            ismatch = 0;
-
-            ismatch += check_spam_rule(spam, p->spam);
-            ismatch += check_size_rule(size, p->size, p->_size);
-            ismatch += check_attachment_rule(state, p);
-
-            if(p->compiled == 1){
-               if(p->emptyfrom == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->from), state->b_from, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptyto == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->to), state->b_to, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptysubject == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->subject), state->b_subject, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptybody == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->body), state->b_body, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-            }
-
-            if(ismatch > 0){
-               return p->rulestr;
-            }
-
+         if(count_match(p, state, size, spam) > 0){
+            return p->rulestr;
          }
       }
 
@@ -251,10 +248,8 @@ char *check_againt_ruleset(struct node *xhash[], struct parser_state *state, int
 
 
 time_t query_retain_period(struct data *data, struct parser_state *state, int size, int spam, struct config *cfg){
-   size_t nmatch=0;
    struct rule *p;
    struct node *q;
-   int ismatch;
 
    q = data->retention_rules[0];
 
@@ -263,47 +258,15 @@ time_t query_retain_period(struct data *data, struct parser_state *state, int si
       if(q->str){
          p = q->str;
 
-         ismatch = 0;
-
          if(p->domainlen > 2){
             if(strcasestr(state->b_to_domain, p->domain) || strcasestr(state->b_from_domain, p->domain)){
                state->retention = p->days;
-               return (time_t)p->days * (time_t)86400;
+               return (time_t)(state->retention) * (time_t)86400;
             }
          }
-         else {
-
-            ismatch += check_spam_rule(spam, p->spam);
-            ismatch += check_size_rule(size, p->size, p->_size);
-            ismatch += check_attachment_rule(state, p);
-
-            if(p->compiled == 1){
-               if(p->emptyfrom == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->from), state->b_from, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptyto == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->to), state->b_to, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptysubject == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->subject), state->b_subject, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptybody == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->body), state->b_body, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-            }
-
-            if(ismatch > 0){
-               state->retention = p->days;
-               return (time_t)p->days * (time_t)86400;
-            }
+         else if(count_match(p, state, size, spam) > 0){
+            state->retention = p->days;
+            return (time_t)(state->retention) * (time_t)86400;
          }
 
       }
@@ -314,15 +277,13 @@ time_t query_retain_period(struct data *data, struct parser_state *state, int si
 
    state->retention = cfg->default_retention_days;
 
-   return (time_t)cfg->default_retention_days * (time_t)86400;
+   return (time_t)(state->retention) * (time_t)86400;
 }
 
 
 int get_folder_id_by_rule(struct data *data, struct parser_state *state, int size, int spam, struct config *cfg){
-   size_t nmatch=0;
    struct rule *p;
    struct node *q;
-   int ismatch;
 
    if(cfg->enable_folders == 0) return 0;
 
@@ -333,45 +294,13 @@ int get_folder_id_by_rule(struct data *data, struct parser_state *state, int siz
       if(q->str){
          p = q->str;
 
-         ismatch = 0;
-
          if(p->domainlen > 2){
             if(strcasestr(state->b_to_domain, p->domain) || strcasestr(state->b_from_domain, p->domain)){
                return p->folder_id;
             }
          }
-         else {
-
-            ismatch += check_spam_rule(spam, p->spam);
-            ismatch += check_size_rule(size, p->size, p->_size);
-            ismatch += check_attachment_rule(state, p);
-
-            if(p->compiled == 1){
-               if(p->emptyfrom == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->from), state->b_from, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptyto == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->to), state->b_to, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptysubject == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->subject), state->b_subject, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptybody == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->body), state->b_body, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-            }
-
-            if(ismatch > 0){
-               return p->folder_id;
-            }
+         else if(count_match(p, state, size, spam) > 0){
+            return p->folder_id;
          }
 
       }
@@ -405,14 +334,16 @@ int check_spam_rule(int is_spam, int spam){
 int check_attachment_rule(struct parser_state *state, struct rule *rule){
    int i;
    size_t nmatch=0;
-   int ismatch = 0;
 
-   if(state->n_attachments == 0) return RULE_UNDEF;
+   // If no attachment rule, then return RULE_UNDEF
+   if(rule->emptyaname == 1 && rule->emptyatype == 1 && rule->attachment_size == 0) return RULE_UNDEF;
+
+   // If we have attachments, but no attachment rules, then return RULE_NO_MATCH
+   if(state->n_attachments == 0 && (rule->emptyaname == 0 || rule->emptyatype == 0 || rule->attachment_size)) return RULE_NO_MATCH;
 
-   if(rule->emptyaname == 1 && rule->emptyatype == 1) return RULE_UNDEF;
 
    for(i=1; i<=state->n_attachments; i++){
-      ismatch = 0;
+      int ismatch = 0;
 
       if(rule->emptyaname == 0){
          if(regexec(&(rule->attachment_name), state->attachments[i].filename, nmatch, NULL, 0) == 0)
@@ -443,36 +374,33 @@ void initrules(struct node *xhash[]){
 
 
 void clearrules(struct node *xhash[]){
-   struct node *p, *q;
+   struct node *q;
    struct rule *rule;
 
    q = xhash[0];
 
    while(q != NULL){
-      p = q;
+      struct node *p = q;
       q = q->r;
 
-      if(p){
-         if(p->str){
-            rule = (struct rule*)p->str;
+      if(p->str){
+         rule = (struct rule*)p->str;
 
-            regfree(&(rule->from));
-            regfree(&(rule->to));
-            regfree(&(rule->subject));
-            regfree(&(rule->body));
-            regfree(&(rule->attachment_name));
-            regfree(&(rule->attachment_type));
+         regfree(&(rule->from));
+         regfree(&(rule->to));
+         regfree(&(rule->subject));
+         regfree(&(rule->body));
+         regfree(&(rule->attachment_name));
+         regfree(&(rule->attachment_type));
 
-            free(rule->rulestr);
+         free(rule->rulestr);
 
-            if(rule->domain) free(rule->domain);
+         if(rule->domain) free(rule->domain);
 
-            free(rule);
-         }
-         free(p);
+         free(rule);
       }
+      free(p);
    }
 
    xhash[0] = NULL;
 }
-

src/rules.h

@@ -7,10 +7,10 @@
 
 #include "defs.h"
 
-void load_rules(struct session_data *sdata, struct data *data, struct node *xhash[], char *table);
+void load_rules(struct session_data *sdata, struct node *xhash[], char *table);
 int append_rule(struct node *xhash[], struct rule_cond *rule_cond);
 struct rule *create_rule_item(struct rule_cond *rule_cond);
-char *check_againt_ruleset(struct node *xhash[], struct parser_state *state, int size, int spam);
+char *check_against_ruleset(struct node *xhash[], struct parser_state *state, int size, int spam);
 time_t query_retain_period(struct data *data, struct parser_state *state, int size, int spam, struct config *cfg);
 int get_folder_id_by_rule(struct data *data, struct parser_state *state, int size, int spam, struct config *cfg);
 int check_size_rule(int message_size, int size, char *_size);
@@ -21,4 +21,3 @@ void initrules(struct node *xhash[]);
 void clearrules(struct node *xhash[]);
 
 #endif /* _RULES_H */
-

src/screen.c

@@ -0,0 +1,254 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <sys/types.h>
+#include <netdb.h>
+#include <piler.h>
+
+
+void init_smtp_acl(struct smtp_acl *smtp_acl[]){
+   smtp_acl[0] = NULL;
+}
+
+
+void clear_smtp_acl(struct smtp_acl *smtp_acl[]){
+   struct smtp_acl *q;
+
+   q = smtp_acl[0];
+
+   while(q){
+      struct smtp_acl *p = q;
+      q = q->r;
+
+      free(p);
+   }
+
+   smtp_acl[0] = NULL;
+}
+
+
+int add_smtp_acl(struct smtp_acl *smtp_acl[], char *network_str, struct smtp_acl *acl){
+   struct smtp_acl *q, *p=NULL, *node;
+
+   if((node = malloc(sizeof(struct smtp_acl))) == NULL) return 0;
+
+   memset(node, 0, sizeof(struct smtp_acl));
+
+   node->low = acl->low;
+   node->high = acl->high;
+   node->prefix = acl->prefix;
+   node->rejected = acl->rejected;
+   snprintf(node->network_str, sizeof(node->network_str)-1, "%s", network_str);
+
+   node->r = NULL;
+
+   q = smtp_acl[0];
+
+   while(q){
+      p = q;
+      q = q->r;
+   }
+
+   if(!p){
+      smtp_acl[0] = node;
+   } else {
+      p->r = node;
+   }
+
+   return 1;
+}
+
+
+int is_valid_line(char *line){
+   // Currently we support ipv4 stuff only, ie. valid characters are: 0-9./
+   // and a line should look like "1.2.3.4/24 permit" or similar (without quotes)
+
+   if(!strchr(line, '.') || !strchr(line, '/') || (!strchr(line, ' ') && !strchr(line, '\t')) ){
+      return 0;
+   }
+
+   if(!strstr(line, "permit") && !strstr(line, "reject")){
+      return 0;
+   }
+
+   // ascii values:
+   //   46: .
+   //   47: /
+   //   48-57: 0-9
+   //   97-122: a-z
+   //
+   for(; *line; line++){
+      if(isalnum(*line) == 0 && isblank(*line) == 0 && *line != 46 && *line != 47) return 0;
+   }
+
+   return 1;
+}
+
+
+int a_to_hl(char *ipstr, in_addr_t *addr){
+  struct in_addr in;
+
+  if(inet_aton(ipstr, &in) == 1){
+     *addr = ntohl(in.s_addr);
+     return 1;
+  }
+
+  syslog(LOG_PRIORITY, "invalid ipv4 address string: *%s*", ipstr);
+
+  return 0;
+}
+
+
+in_addr_t netmask(int prefix){
+   if(prefix == 0)
+      return( ~((in_addr_t) -1) );
+   else
+      return ~((1 << (32 - prefix)) - 1);
+}
+
+
+in_addr_t network(in_addr_t addr, int prefix){
+   return addr & netmask(prefix);
+}
+
+
+in_addr_t broadcast(in_addr_t addr, int prefix){
+   return addr | ~netmask(prefix);
+}
+
+
+int str_to_net_range(char *network_addr_prefix, struct smtp_acl *smtp_acl){
+   in_addr_t net = 0;
+   int prefix = 0;
+
+   smtp_acl->low = 0;
+   smtp_acl->high = 0;
+
+   // By default we permit unless you specify "reject" (without quotes)
+   // To be on the safer side we permit even if you misspell the word "reject"
+
+   smtp_acl->rejected = 0;
+
+   if(strcasestr(network_addr_prefix, "reject")){
+      smtp_acl->rejected = 1;
+   }
+
+   char *p = strchr(network_addr_prefix, '/');
+   if(!p) return 0;
+
+   if(strlen(network_addr_prefix) > sizeof(smtp_acl->network_str)){
+      syslog(LOG_PRIORITY, "line *%s* is longer than %ld bytes, discarded", network_addr_prefix, sizeof(smtp_acl->network_str));
+      return 0;
+   }
+
+   char buf[SMALLBUFSIZE];
+   snprintf(buf, sizeof(buf)-1, "%s", network_addr_prefix);
+
+   *p = '\0';
+   p++;
+
+   // Even though the remaining part of the acl line continues with some number
+   // then whitespace characters and the permit/reject action atoi() can still
+   // figure out the numeric part properly, that's why I'm lazy here.
+   prefix = atoi(p);
+
+   // The prefix string (p) must start with a digit and the prefix integer must be in 0..32 range
+   if(*p < 48 || *p > 57 || prefix < 0 || prefix > 32){
+      syslog(LOG_PRIORITY, "error: invalid prefix: %s", p);
+      return 0;
+   }
+
+   if(a_to_hl(network_addr_prefix, &net)){
+      smtp_acl->low = network(net, prefix);
+      smtp_acl->high = broadcast(net, prefix);
+      smtp_acl->prefix = prefix;
+
+      syslog(LOG_PRIORITY, "info: parsed acl *%s*", buf);
+
+      return 1;
+   }
+
+   return 0;
+}
+
+
+void load_smtp_acl(struct smtp_acl *smtp_acl[]){
+   int count=0;
+
+   clear_smtp_acl(smtp_acl);
+   init_smtp_acl(smtp_acl);
+
+   FILE *f = fopen(SMTP_ACL_FILE, "r");
+   if(!f){
+      syslog(LOG_PRIORITY, "info: cannot open %s, piler-smtp accepts smtp connections from everywhere", SMTP_ACL_FILE);
+      return;
+   }
+
+   char line[SMALLBUFSIZE];
+   struct smtp_acl acl;
+
+   while(fgets(line, sizeof(line)-1, f)){
+      // Skip comments
+      if(line[0] == ';' || line[0] == '#') continue;
+
+      trimBuffer(line);
+
+      // Skip empty line
+      if(line[0] == 0) continue;
+
+      char line2[SMALLBUFSIZE];
+      int rc = 0;
+      snprintf(line2, sizeof(line2)-1, "%s", line);
+
+      if(is_valid_line(line) == 1 && str_to_net_range(line, &acl) == 1){
+         add_smtp_acl(smtp_acl, line, &acl);
+         count++;
+         rc = 1;
+      }
+
+      if(!rc) syslog(LOG_PRIORITY, "error: failed to parse line: *%s*", line2);
+   }
+
+   fclose(f);
+
+   // If we have entries on the smtp acl list, then add 127.0.0.1/8
+   // to let the GUI health page connect to the piler-smtp daemon
+   if(count){
+      snprintf(line, sizeof(line)-1, "127.0.0.1/8 permit");
+
+      if(str_to_net_range(line, &acl) == 1){
+         add_smtp_acl(smtp_acl, line, &acl);
+      }
+   }
+}
+
+
+int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr){
+   struct smtp_acl *q=smtp_acl[0];
+   in_addr_t addr = 0;
+
+   // Empty acl, let it pass
+   if(!q) return 0;
+
+   if(a_to_hl(ipaddr, &addr) == 0){
+      syslog(LOG_PRIORITY, "error: invalid smtp client address: *%s*", ipaddr);
+      return 1;
+   }
+
+   while(q){
+      if(addr >= q->low && addr <= q->high){
+         if(q->rejected) syslog(LOG_PRIORITY, "denied connection from %s, acl: %s/%d reject", ipaddr, q->network_str, q->prefix);
+         return q->rejected;
+      }
+
+      q = q->r;
+   }
+
+   syslog(LOG_PRIORITY, "denied connection from %s by implicit default deny", ipaddr);
+
+   return 1;
+}

src/screen.h

@@ -0,0 +1,16 @@
+/*
+ * screen.h, SJ
+ */
+
+#ifndef _SCREEN_H
+ #define _SCREEN_H
+
+#include "defs.h"
+
+void init_smtp_acl(struct smtp_acl *smtp_acl[]);
+void clear_smtp_acl(struct smtp_acl *smtp_acl[]);
+int add_smtp_acl(struct smtp_acl *smtp_acl[], char *network_str, struct smtp_acl *acl);
+void load_smtp_acl(struct smtp_acl *smtp_acl[]);
+int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr);
+
+#endif /* _SCREEN_H */

src/session.c

@@ -2,34 +2,45 @@
 #include <stdlib.h>
 #include <string.h>
 #include <netdb.h>
+#include <signal.h>
 #include <piler.h>
 
 
 int get_session_slot(struct smtp_session **sessions, int max_connections);
-void init_smtp_session(struct smtp_session *session, int slot, int sd, struct config *cfg);
+void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg);
 
+#define GOT_CRLF_DOT_CRLF(p) *p == '\r' && *(p+1) == '\n' && *(p+2) == '.' && *(p+3) == '\r' && *(p+4) == '\n' ? 1 : 0
 
-#ifdef HAVE_LIBWRAP
-int is_blocked_by_tcp_wrappers(int sd){
-   struct request_info req;
+uint64 get_sessions_total_memory(struct smtp_session **sessions, int max_connections){
+   uint64 total = 0;
 
-   request_init(&req, RQ_DAEMON, "piler", RQ_FILE, sd, 0);
-
-   fromhost(&req);
-
-   if(!hosts_access(&req)){
-      send(sd, SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY, strlen(SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY), 0);
-      syslog(LOG_PRIORITY, "denied connection from %s by tcp_wrappers", eval_client(&req));
-      return 1;
+   for(int i=0; i<max_connections; i++){
+      if(sessions[i]) total += sessions[i]->bufsize;
    }
 
-   return 0;
+   return total;
 }
-#endif
 
 
-int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct config *cfg){
-   char smtp_banner[SMALLBUFSIZE];
+/*
+ * If the sending party sets the email size when it sends the "mail from"
+ * part in the smtp transaction, eg. MAIL FROM:<jajaja@akakak.lo> size=509603
+ * then piler-smtp could know the email size in advance and could do
+ * a better estimate on the allowed number of smtp sessions.
+ */
+
+uint64 get_sessions_total_expected_mail_size(struct smtp_session **sessions, int max_connections){
+   uint64 total = 0;
+
+   for(int i=0; i<max_connections; i++){
+      if(sessions[i]) total += sessions[i]->mail_size;
+   }
+
+   return total;
+}
+
+
+int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg){
    int slot;
 
    /*
@@ -43,19 +54,36 @@ int start_new_session(struct smtp_session **sessions, int socket, int *num_conne
       return -1;
    }
 
-#ifdef HAVE_LIBWRAP
-   if(is_blocked_by_tcp_wrappers(socket) == 1){
+   // Check remote client against the allowed network ranges
+   if(cfg->smtp_access_list && is_blocked_by_pilerscreen(smtp_acl, client_addr)){
+      send(socket, SMTP_RESP_550_ERR, strlen(SMTP_RESP_550_ERR), 0);
       close(socket);
       return -1;
    }
-#endif
+
+   /*
+    * We are under the max_smtp_memory threshold
+    */
+
+   uint64 expected_total_mail_size = get_sessions_total_expected_mail_size(sessions, cfg->max_connections);
+   uint64 total_memory = get_sessions_total_memory(sessions, cfg->max_connections);
+
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "DEBUG: total smtp memory allocated: %llu, expected total size: %llu", total_memory, expected_total_mail_size);
+
+   if(total_memory > cfg->max_smtp_memory || expected_total_mail_size > cfg->max_smtp_memory){
+      syslog(LOG_PRIORITY, "ERROR: too much memory consumption: %llu", total_memory);
+      send(socket, SMTP_RESP_451_ERR_TOO_MANY_REQUESTS, strlen(SMTP_RESP_451_ERR_TOO_MANY_REQUESTS), 0);
+      return -1;
+   }
 
    slot = get_session_slot(sessions, cfg->max_connections);
 
    if(slot >= 0 && sessions[slot] == NULL){
       sessions[slot] = malloc(sizeof(struct smtp_session));
       if(sessions[slot]){
-         init_smtp_session(sessions[slot], slot, socket, cfg);
+         init_smtp_session(sessions[slot], slot, socket, client_addr, cfg);
+
+         char smtp_banner[SMALLBUFSIZE];
          snprintf(smtp_banner, sizeof(smtp_banner)-1, SMTP_RESP_220_BANNER, cfg->hostid);
          send(socket, smtp_banner, strlen(smtp_banner), 0);
 
@@ -100,14 +128,9 @@ struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int m
 }
 
 
-void init_smtp_session(struct smtp_session *session, int slot, int sd, struct config *cfg){
-   struct sockaddr_in addr;
-   socklen_t addr_size = sizeof(struct sockaddr_in);
-   char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
-
+void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg){
    session->slot = slot;
 
-   session->buflen = 0;
    session->protocol_state = SMTP_STATE_INIT;
 
    session->cfg = cfg;
@@ -118,108 +141,197 @@ void init_smtp_session(struct smtp_session *session, int slot, int sd, struct co
    session->net.ctx = NULL;
    session->net.ssl = NULL;
 
-   session->fd = -1;
+   session->nullbyte = 0;
 
-   memset(session->buf, 0, SMALLBUFSIZE);
-   memset(session->remote_host, 0, INET6_ADDRSTRLEN);
+   snprintf(session->remote_host, sizeof(session->remote_host)-1, "%s", client_addr);
 
-   reset_bdat_counters(session);
+   session->buf = NULL;
+   session->buflen = 0;
+   session->bufsize = 0;
 
-   time(&(session->lasttime));
-
-   if(getpeername(session->net.socket, (struct sockaddr *)&addr, &addr_size) == 0 &&
-      getnameinfo((struct sockaddr *)&addr, addr_size, hbuf, sizeof(hbuf), sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV) == 0){
-         snprintf(session->remote_host, INET6_ADDRSTRLEN-1, "%s", hbuf);
-   }
+   reset_smtp_session(session);
 }
 
 
 void free_smtp_session(struct smtp_session *session){
-
    if(session){
 
+      if(session->buf != NULL){
+         free(session->buf);
+      }
+
       if(session->net.use_ssl == 1){
          SSL_shutdown(session->net.ssl);
          SSL_free(session->net.ssl);
       }
 
-      if(session->net.ctx) SSL_CTX_free(session->net.ctx);
+      if(session->net.ctx){
+         SSL_CTX_free(session->net.ctx);
+      }
 
       free(session);
    }
 }
 
 
-void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections){
-   syslog(LOG_PRIORITY, "disconnected from %s on descriptor %d (%d active connections)", sessions[slot]->remote_host, sessions[slot]->net.socket, (*num_connections)-1);
+void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections, char *reason){
+   if(sessions[slot] == NULL){
+      syslog(LOG_PRIORITY, "session already torn down, slot=%d, reason=%s (%d active connections)", slot, reason, *num_connections);
+      return;
+   }
+
+   if(*num_connections > 0) (*num_connections)--;
+
+   syslog(LOG_PRIORITY, "disconnected from %s on fd=%d, slot=%d, reason=%s (%d active connections)",
+          sessions[slot]->remote_host, sessions[slot]->net.socket, slot, reason, *num_connections);
 
    close(sessions[slot]->net.socket);
 
+   if(sessions[slot]->fd != -1){
+      syslog(LOG_PRIORITY, "Removing %s", sessions[slot]->ttmpfile);
+      close(sessions[slot]->fd);
+      unlink(sessions[slot]->ttmpfile);
+      sessions[slot]->fd = -1;
+   }
+
    free_smtp_session(sessions[slot]);
    sessions[slot] = NULL;
-
-   (*num_connections)--;
 }
 
 
-void handle_data(struct smtp_session *session, char *readbuf, int readlen){
-   char *p, puf[MAXBUFSIZE];
-   int result;
+inline int get_last_newline_position(char *buf, int buflen){
+   int i;
 
-   // process BDAT stuff
+   for(i=buflen; i>0; i--){
+      if(*(buf+i) == '\n'){
+         i++;
+         break;
+      }
+   }
+
+   return i;
+}
+
+
+void flush_buffer(struct smtp_session *session){
+   // In the DATA phase skip the 1st character if it's a dot (.)
+   // and there are more characters before the trailing CR-LF
+   //
+   // See https://www.ietf.org/rfc/rfc5321.html#section-4.5.2 for more
+   for(int i=0; i<session->buflen; i++){
+      if(*(session->buf+i) == '\n' && *(session->buf+i+1) == '.' && *(session->buf+i+2) == '.'){
+         int dst = i + 2;
+         int src = dst + 1;
+         int l = session->buflen - src;
+         memmove(session->buf + dst, session->buf + src, l);
+         session->buflen -= 1;
+      }
+   }
+
+   // Exclude the trailing \r\n.\r\n sequence
+
+   session->buflen -= 5;
+
+   if(write(session->fd, session->buf, session->buflen) != session->buflen){
+      session->bad = 1;
+      syslog(LOG_PRIORITY, "ERROR (line: %d) %s: failed to write %d bytes", __LINE__, __func__, session->buflen);
+   }
+
+   session->tot_len = session->buflen;
+}
+
+
+void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg){
+   // Update lasttime if we have something to process
+   time(&(session->lasttime));
 
    if(session->protocol_state == SMTP_STATE_BDAT){
+      process_bdat(session, readbuf, readlen, cfg);
+      return;
+   }
+
+   // realloc memory if the new chunk doesn't fit in
+
+   if(session->buflen + readlen + 10 > session->bufsize){
+      // Handle if the current memory allocation for this email is above the max_message_size threshold
+
+      if(session->buflen > cfg->max_message_size){
+         if(session->too_big == 0) syslog(LOG_PRIORITY, "ERROR: too big email: %d vs %d", session->buflen, cfg->max_message_size);
+         session->bad = 1;
+         session->too_big = 1;
+      }
+
+      if(session->bad == 0){
+         char *q = realloc(session->buf, session->bufsize + SMTPBUFSIZE);
+         if(q){
+            session->buf = q;
+            memset(session->buf+session->bufsize, 0, SMTPBUFSIZE);
+            session->bufsize += SMTPBUFSIZE;
+         } else {
+            syslog(LOG_PRIORITY, "ERROR: realloc %s %s %d", session->ttmpfile, __func__, __LINE__);
+            session->bad = 1;
+         }
+      }
+   }
+
+   // process smtp command
+   if(session->protocol_state != SMTP_STATE_DATA){
+
+      // We got ~2 MB of garbage and no valid smtp command
+      // Terminate the connection
+      if(session->buflen + readlen > SMTPBUFSIZE - 10){
+         session->bad = 1;
+      }
+
+      // We are at the beginning of the smtp transaction
       if(session->bad == 1){
-         // something bad happened in the BDAT processing
+         write1(&(session->net), SMTP_RESP_451_ERR, strlen(SMTP_RESP_451_ERR));
+         syslog(LOG_PRIORITY, "ERROR: sent 451 temp error back to client %s", session->ttmpfile);
          return;
       }
 
-      process_bdat(session, readbuf, readlen);
+      //printf("got %d *%s*\n", readlen, readbuf);
+
+      memcpy(session->buf + session->buflen, readbuf, readlen);
+      session->buflen += readlen;
+
+      int pos = get_last_newline_position(session->buf, session->buflen);
+
+      if(pos < readlen) return; // no complete command
+
+      process_smtp_command(session, cfg);
+
+      memset(session->buf, 0, session->bufsize);
+      session->buflen = 0;
+
+      return;
    }
 
-   // process DATA
+   if(session->bad == 0){
+      memcpy(session->buf + session->buflen, readbuf, readlen);
+      session->buflen += readlen;
 
-   else if(session->protocol_state == SMTP_STATE_DATA){
-      process_data(session, readbuf, readlen);
-   }
-
-   // process other SMTP commands
-
-   else {
-
-      if(session->buflen > 0){
-         snprintf(puf, sizeof(puf)-1, "%s%s", session->buf, readbuf);
-         snprintf(readbuf, BIGBUFSIZE-1, "%s", puf);
-
-         session->buflen = 0;
-         memset(session->buf, 0, SMALLBUFSIZE);
+      char *p = session->buf + session->buflen - 5;
+      if(session->buflen >= 5 && GOT_CRLF_DOT_CRLF(p)){
+         flush_buffer(session);
+         process_command_period(session);
       }
-
-      p = readbuf;
-
-      do {
-         memset(puf, 0, sizeof(puf));
-         p = split(p, '\n', puf, sizeof(puf)-1, &result);
-
-         if(puf[0] == '\0') continue;
-
-         if(result == 1){
-            process_smtp_command(session, puf);
-
-            // if chunking is enabled and we have data after BDAT <len>
-            // then process the rest
-
-            if(session->cfg->enable_chunking == 1 && p && session->protocol_state == SMTP_STATE_BDAT){
-               process_bdat(session, p, strlen(p));
-               break;
-            }
-         }
-         else {
-            snprintf(session->buf, SMALLBUFSIZE-1, "%s", puf);
-            session->buflen = strlen(puf);
-         }
-      } while(p);
-
+   } else if(strstr(readbuf, "\r\n.\r\n")){
+      process_command_period(session);
    }
 
 }
+
+
+void write_envelope_addresses(struct smtp_session *session, struct config *cfg){
+   if(session->fd == -1) return;
+
+   for(int i=0; i<session->num_of_rcpt_to; i++){
+      char *p = strchr(session->rcptto[i], '@');
+      if(p && strncmp(p+1, cfg->hostid, cfg->hostid_len)){
+         char s[SMALLBUFSIZE];
+         snprintf(s, sizeof(s)-1, "X-Piler-Envelope-To: %s\n", session->rcptto[i]);
+         if(write(session->fd, s, strlen(s)) == -1) syslog(LOG_PRIORITY, "ERROR: %s: cannot write envelope to address", session->ttmpfile);
+      }
+   }
+}

src/sig.c

@@ -56,4 +56,3 @@ signal_func *set_signal_handler(int signo, signal_func * func){
 
    return oact.sa_handler;
 }
-

src/smtp.c

@@ -15,53 +15,59 @@
 #include "smtp.h"
 
 
-void process_smtp_command(struct smtp_session *session, char *buf){
+void process_smtp_command(struct smtp_session *session, struct config *cfg){
    char response[SMALLBUFSIZE];
 
-   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "processing command: *%s*", buf);
+   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "got on fd=%d: *%s*", session->net.socket, session->buf);
 
-   if(strncasecmp(buf, SMTP_CMD_HELO, strlen(SMTP_CMD_HELO)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_HELO, strlen(SMTP_CMD_HELO)) == 0){
       process_command_helo(session, response, sizeof(response));
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_EHLO, strlen(SMTP_CMD_EHLO)) == 0 ||
-         strncasecmp(buf, LMTP_CMD_LHLO, strlen(LMTP_CMD_LHLO)) == 0){
-      process_command_ehlo_lhlo(session, response, sizeof(response));
+   if(strncasecmp(session->buf, SMTP_CMD_EHLO, strlen(SMTP_CMD_EHLO)) == 0 ||
+         strncasecmp(session->buf, LMTP_CMD_LHLO, strlen(LMTP_CMD_LHLO)) == 0){
+      process_command_ehlo_lhlo(session, response, sizeof(response), cfg);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){
-      process_command_mail_from(session, buf);
+   if(strncasecmp(session->buf, SMTP_CMD_HELP, strlen(SMTP_CMD_HELP)) == 0){
+      send_smtp_response(session, SMTP_RESP_221_PILER_SMTP_OK);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){
-      process_command_rcpt_to(session, buf);
+   if(strncasecmp(session->buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){
+      process_command_mail_from(session);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_DATA, strlen(SMTP_CMD_DATA)) == 0){
-      process_command_data(session);
+   if(strncasecmp(session->buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){
+      process_command_rcpt_to(session, cfg);
       return;
    }
 
-   if(session->cfg->enable_chunking == 1 && strncasecmp(buf, SMTP_CMD_BDAT, strlen(SMTP_CMD_BDAT)) == 0){
-      get_bdat_size_to_read(session, buf);
+   if(strncasecmp(session->buf, SMTP_CMD_DATA, strlen(SMTP_CMD_DATA)) == 0){
+      process_command_data(session, cfg);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_QUIT, strlen(SMTP_CMD_QUIT)) == 0){
+   /* Support only BDAT xxxx LAST command */
+   if(session->cfg->enable_chunking == 1 && strncasecmp(session->buf, SMTP_CMD_BDAT, strlen(SMTP_CMD_BDAT)) == 0 && strcasestr(session->buf, "LAST")){
+      get_bdat_size_to_read(session);
+      return;
+   }
+
+   if(strncasecmp(session->buf, SMTP_CMD_QUIT, strlen(SMTP_CMD_QUIT)) == 0){
       process_command_quit(session, response, sizeof(response));
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_RESET, strlen(SMTP_CMD_RESET)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_RESET, strlen(SMTP_CMD_RESET)) == 0){
       process_command_reset(session);
       return;
    }
 
-   if(session->cfg->tls_enable == 1 && strncasecmp(buf, SMTP_CMD_STARTTLS, strlen(SMTP_CMD_STARTTLS)) == 0 && session->net.use_ssl == 0){
+   if(session->cfg->tls_enable == 1 && strncasecmp(session->buf, SMTP_CMD_STARTTLS, strlen(SMTP_CMD_STARTTLS)) == 0 && session->net.use_ssl == 0){
       process_command_starttls(session);
       return;
    }
@@ -70,67 +76,8 @@ void process_smtp_command(struct smtp_session *session, char *buf){
 }
 
 
-void process_data(struct smtp_session *session, char *readbuf, int readlen){
-   char puf[SMALLBUFSIZE+BIGBUFSIZE];
-   int n, pos, len, writelen;
-
-   memset(puf, 0, sizeof(puf));
-   len = readlen;
-
-   if(session->buflen > 0){
-      memcpy(&puf[0], session->buf, session->buflen);
-      memcpy(&puf[session->buflen], readbuf, readlen);
-
-      len += session->buflen;
-   }
-   else {
-      memcpy(&puf[0], readbuf, readlen);
-   }
-
-   pos = searchStringInBuffer(&puf[0], len, SMTP_CMD_PERIOD, 5);
-
-   if(pos > 0){
-      if(write(session->fd, puf, pos) != -1){
-         session->tot_len += pos;
-         process_command_period(session);
-      }
-      else syslog(LOG_PRIORITY, "ERROR (line: %d): process_data(): failed to write %d bytes", __LINE__, pos);
-   }
-   else if(pos < 0){
-      n = search_char_backward(&puf[0], len, '\r');
-
-      if(n == -1 || len - n > 4){
-
-         if(n == -1) writelen = len;
-         else writelen = n-1;
-
-         if(writelen > 0){
-            if(write(session->fd, puf, writelen) != -1){
-               session->tot_len += writelen;
-            }
-            else syslog(LOG_PRIORITY, "ERROR (line: %d): process_data(): failed to write %d bytes", __LINE__, writelen);
-         }
-      }
-      else {
-         if(write(session->fd, puf, n) != -1){
-            session->tot_len += n;
-         }
-         else syslog(LOG_PRIORITY, "ERROR (line: %d) process_data(): failed to write %d bytes", __LINE__, n);
-
-         snprintf(session->buf, SMALLBUFSIZE-1, "%s", &puf[n]);
-         session->buflen = len - n;
-      }
-   }
-   else {
-      // The buffer contains only the CR-LF-DOT-CR-LF sequence
-      process_command_period(session);
-   }
-}
-
-
 void wait_for_ssl_accept(struct smtp_session *session){
    int rc;
-   char ssl_error[SMALLBUFSIZE];
 
    if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "waiting for ssl handshake");
 
@@ -147,6 +94,8 @@ void wait_for_ssl_accept(struct smtp_session *session){
    }
 
    if(session->cfg->verbosity >= _LOG_DEBUG || session->net.use_ssl == 0){
+      char ssl_error[SMALLBUFSIZE];
+
       ERR_error_string_n(ERR_get_error(), ssl_error, SMALLBUFSIZE);
       syslog(LOG_PRIORITY, "SSL_accept() result, rc=%d, errorcode: %d, error text: %s",
              rc, SSL_get_error(session->net.ssl, rc), ssl_error);
@@ -156,7 +105,7 @@ void wait_for_ssl_accept(struct smtp_session *session){
 
 void send_smtp_response(struct smtp_session *session, char *buf){
    write1(&(session->net), buf, strlen(buf));
-   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "sent: %s", buf);
+   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "sent on fd=%d: %s", session->net.socket, buf);
 }
 
 
@@ -168,7 +117,7 @@ void process_command_helo(struct smtp_session *session, char *buf, int buflen){
 }
 
 
-void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen){
+void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen, struct config *cfg){
    char extensions[SMALLBUFSIZE];
 
    memset(extensions, 0, sizeof(extensions));
@@ -179,7 +128,9 @@ void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int bufl
    if(session->net.use_ssl == 0 && session->cfg->tls_enable == 1) snprintf(extensions, sizeof(extensions)-1, "%s", SMTP_EXTENSION_STARTTLS);
    if(session->cfg->enable_chunking == 1) strncat(extensions, SMTP_EXTENSION_CHUNKING, sizeof(extensions)-strlen(extensions)-2);
 
-   snprintf(buf, buflen-1, SMTP_RESP_250_EXTENSIONS, session->cfg->hostid, extensions);
+   //#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n%s250-SIZE %d\r\n250 8BITMIME\r\n"
+
+   snprintf(buf, buflen-1, SMTP_RESP_250_EXTENSIONS, session->cfg->hostid, cfg->max_message_size, extensions);
 
    send_smtp_response(session, buf);
 }
@@ -197,6 +148,15 @@ int init_ssl(struct smtp_session *session){
       return 0;
    }
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+   SSL_CTX_set_options(session->net.ctx, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3|SSL_OP_NO_TLSv1|SSL_OP_NO_TLSv1_1);
+#else
+   if(SSL_CTX_set_min_proto_version(session->net.ctx, session->cfg->tls_min_version_number) == 0){
+      syslog(LOG_PRIORITY, "failed SSL_CTX_set_min_proto_version() to %s/%d", session->cfg->tls_min_version, session->cfg->tls_min_version_number);
+      return 0;
+   }
+#endif
+
    if(SSL_CTX_set_cipher_list(session->net.ctx, session->cfg->cipher_list) == 0){
       syslog(LOG_PRIORITY, "failed to set cipher list: '%s'", session->cfg->cipher_list);
       return 0;
@@ -224,8 +184,6 @@ void process_command_starttls(struct smtp_session *session){
       session->net.ssl = SSL_new(session->net.ctx);
       if(session->net.ssl){
 
-         SSL_set_options(session->net.ssl, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3);
-
          if(SSL_set_fd(session->net.ssl, session->net.socket) == 1){
             session->net.starttls = 1;
             send_smtp_response(session, SMTP_RESP_220_READY_TO_START_TLS);
@@ -235,42 +193,57 @@ void process_command_starttls(struct smtp_session *session){
                wait_for_ssl_accept(session);
 
             return;
-         } syslog(LOG_PRIORITY, "%s: SSL_set_fd() failed", session->ttmpfile);
-      } syslog(LOG_PRIORITY, "%s: SSL_new() failed", session->ttmpfile);
-   } syslog(LOG_PRIORITY, "SSL ctx is null!");
+         } syslog(LOG_PRIORITY, "ERROR: %s: SSL_set_fd() failed", session->ttmpfile);
+      } syslog(LOG_PRIORITY, "ERROR: %s: SSL_new() failed", session->ttmpfile);
+   } syslog(LOG_PRIORITY, "ERROR: init_ssl()");
 
    send_smtp_response(session, SMTP_RESP_454_ERR_TLS_TEMP_ERROR);
 }
 
 
-void process_command_mail_from(struct smtp_session *session, char *buf){
+void process_command_mail_from(struct smtp_session *session){
    memset(session->mailfrom, 0, SMALLBUFSIZE);
 
    if(session->protocol_state != SMTP_STATE_HELO && session->protocol_state != SMTP_STATE_PERIOD && session->protocol_state != SMTP_STATE_BDAT){
       send(session->net.socket, SMTP_RESP_503_ERR, strlen(SMTP_RESP_503_ERR), 0);
    }
    else {
-      memset(&(session->ttmpfile[0]), 0, SMALLBUFSIZE);
-      make_random_string(&(session->ttmpfile[0]), QUEUE_ID_LEN);
       session->protocol_state = SMTP_STATE_MAIL_FROM;
 
-      extractEmail(buf, session->mailfrom);
+      extractEmail(session->buf, session->mailfrom);
 
-      reset_bdat_counters(session);
-      session->tot_len = 0;
+      int mailsize = get_size_from_smtp_mail_from(session->buf);
+
+      reset_smtp_session(session);
+
+      session->mail_size = mailsize;
 
       send_smtp_response(session, SMTP_RESP_250_OK);
    }
 }
 
 
-void process_command_rcpt_to(struct smtp_session *session, char *buf){
+void process_command_rcpt_to(struct smtp_session *session, struct config *cfg){
 
    if(session->protocol_state == SMTP_STATE_MAIL_FROM || session->protocol_state == SMTP_STATE_RCPT_TO){
 
       // For now, we are not interested in the envelope recipients
 
       session->protocol_state = SMTP_STATE_RCPT_TO;
+
+      if(session->num_of_rcpt_to < MAX_RCPT_TO){
+         extractEmail(session->buf, session->rcptto[session->num_of_rcpt_to]);
+
+         // Check if we should accept archive_address only
+         if(cfg->archive_address[0] && !strstr(cfg->archive_address, session->rcptto[session->num_of_rcpt_to])){
+            syslog(LOG_PRIORITY, "ERROR: Invalid recipient: *%s*", session->rcptto[session->num_of_rcpt_to]);
+            send_smtp_response(session, SMTP_RESP_550_ERR_INVALID_RECIPIENT);
+            return;
+         }
+
+         session->num_of_rcpt_to++;
+      }
+
       send_smtp_response(session, SMTP_RESP_250_OK);
    }
    else {
@@ -279,7 +252,7 @@ void process_command_rcpt_to(struct smtp_session *session, char *buf){
 }
 
 
-void process_command_data(struct smtp_session *session){
+void process_command_data(struct smtp_session *session, struct config *cfg){
    session->tot_len = 0;
 
    if(session->protocol_state != SMTP_STATE_RCPT_TO){
@@ -294,6 +267,8 @@ void process_command_data(struct smtp_session *session){
       else {
          session->protocol_state = SMTP_STATE_DATA;
          send_smtp_response(session, SMTP_RESP_354_DATA_OK);
+
+         if(cfg->process_rcpt_to_addresses == 1) write_envelope_addresses(session, cfg);
       }
    }
 
@@ -301,24 +276,39 @@ void process_command_data(struct smtp_session *session){
 
 void process_command_period(struct smtp_session *session){
    char buf[SMALLBUFSIZE];
+   struct timezone tz;
+   struct timeval tv1, tv2;
 
    session->protocol_state = SMTP_STATE_PERIOD;
 
    // TODO: add some error handling
 
+   gettimeofday(&tv1, &tz);
    fsync(session->fd);
    close(session->fd);
+   gettimeofday(&tv2, &tz);
 
    session->fd = -1;
 
-   syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d, client=%s", session->ttmpfile, session->mailfrom, session->tot_len, session->remote_host);
+   syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d, client=%s, fd=%d, fsync=%ld", session->ttmpfile, session->mailfrom, session->tot_len, session->remote_host, session->net.socket, tvdiff(tv2, tv1));
 
-   move_email(session);
+   if(session->bad == 1 || session->too_big == 1){
+      snprintf(buf, sizeof(buf)-1, SMTP_RESP_451_ERR);
+      unlink(session->ttmpfile);
+      syslog(LOG_PRIORITY, "ERROR: problem in processing, removing %s", session->ttmpfile);
+      if(session->too_big)
+         snprintf(buf, sizeof(buf)-1, "%s", SMTP_RESP_552_ERR_TOO_BIG_EMAIL);
+      else
+         snprintf(buf, sizeof(buf)-1, "%s", SMTP_RESP_451_ERR);
+   } else {
+      move_email(session);
+      snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
+   }
 
-   snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
-
-   session->buflen = 0;
-   memset(session->buf, 0, SMALLBUFSIZE);
+   if(session->buf){
+      memset(session->buf, 0, session->bufsize);
+      session->buflen = 0;
+   }
 
    send_smtp_response(session, buf);
 }
@@ -334,14 +324,32 @@ void process_command_quit(struct smtp_session *session, char *buf, int buflen){
 
 
 void process_command_reset(struct smtp_session *session){
+   reset_smtp_session(session);
    send_smtp_response(session, SMTP_RESP_250_OK);
-
-   session->tot_len = 0;
-   session->fd = -1;
    session->protocol_state = SMTP_STATE_HELO;
+}
+
+
+void reset_smtp_session(struct smtp_session *session){
+   session->tot_len = 0;
+   session->mail_size = 0;
+   session->too_big = 0;
+   session->bad = 0;
+
+   session->fd = -1;
+
+   session->num_of_rcpt_to = 0;
+   for(int i=0; i<MAX_RCPT_TO; i++) memset(session->rcptto[i], 0, SMALLBUFSIZE);
 
    reset_bdat_counters(session);
 
+   time(&(session->lasttime));
+
    memset(&(session->ttmpfile[0]), 0, SMALLBUFSIZE);
-   make_random_string(&(session->ttmpfile[0]), QUEUE_ID_LEN);
+   make_random_string((unsigned char*)&(session->ttmpfile[0]), QUEUE_ID_LEN);
+
+   if(session->buf){
+      memset(session->buf, 0, session->bufsize);
+   }
+   session->buflen = 0;
 }

src/smtp.h

@@ -7,22 +7,24 @@
 
 #include <piler.h>
 
-void process_smtp_command(struct smtp_session *session, char *buf);
-void process_data(struct smtp_session *session, char *readbuf, int readlen);
+void process_smtp_command(struct smtp_session *session, struct config *cfg);
+void process_data(struct smtp_session *session, char *buf, int buflen);
 
 void send_smtp_response(struct smtp_session *session, char *buf);
 void process_command_helo(struct smtp_session *session, char *buf, int buflen);
-void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen);
+void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen, struct config *cfg);
 void process_command_quit(struct smtp_session *session, char *buf, int buflen);
 void process_command_reset(struct smtp_session *session);
-void process_command_mail_from(struct smtp_session *session, char *buf);
-void process_command_rcpt_to(struct smtp_session *session, char *buf);
-void process_command_data(struct smtp_session *session);
+void process_command_mail_from(struct smtp_session *session);
+void process_command_rcpt_to(struct smtp_session *session, struct config *cfg);
+void process_command_data(struct smtp_session *session, struct config *cfg);
 void process_command_period(struct smtp_session *session);
 void process_command_starttls(struct smtp_session *session);
 
 void reset_bdat_counters(struct smtp_session *session);
-void get_bdat_size_to_read(struct smtp_session *session, char *buf);
-void process_bdat(struct smtp_session *session, char *readbuf, int readlen);
+void get_bdat_size_to_read(struct smtp_session *session);
+void process_bdat(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg);
+
+void reset_smtp_session(struct smtp_session *session);
 
 #endif

src/smtpcodes.h

@@ -18,6 +18,7 @@
 
 #define SMTP_CMD_HELO "HELO"
 #define SMTP_CMD_EHLO "EHLO"
+#define SMTP_CMD_HELP "HELP"
 #define SMTP_CMD_MAIL_FROM "MAIL FROM:"
 #define SMTP_CMD_RCPT_TO "RCPT TO:"
 #define SMTP_CMD_DATA "DATA"
@@ -32,11 +33,12 @@
 
 // SMTP responses
 
+#define SMTP_RESP_221_PILER_SMTP_OK "221 piler-smtp is OK\r\n"
 #define SMTP_RESP_220_BANNER "220 %s ESMTP\r\n"
 #define SMTP_RESP_220_READY_TO_START_TLS "220 Ready to start TLS\r\n"
 #define SMTP_RESP_221_GOODBYE "221 %s Goodbye\r\n"
 #define SMTP_RESP_250_OK "250 Ok\r\n"
-#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n250-PIPELINING\r\n%s250-SIZE\r\n250 8BITMIME\r\n"
+#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n250-SIZE %d\r\n%s250 8BITMIME\r\n"
 
 #define SMTP_EXTENSION_STARTTLS "250-STARTTLS\r\n"
 #define SMTP_EXTENSION_CHUNKING "250-CHUNKING\r\n"
@@ -49,12 +51,16 @@
 #define SMTP_RESP_421_ERR_TMP "421 %s service not available\r\n"
 #define SMTP_RESP_421_ERR_WRITE_FAILED "421 writing queue file failed\r\n"
 #define SMTP_RESP_421_ERR_ALL_PORTS_ARE_BUSY "421 All server ports are busy\r\n"
+#define SMTP_RESP_451_ERR_TOO_MANY_REQUESTS "451 Too many requests, try again later\r\n"
 #define SMTP_RESP_451_ERR "451 Error in processing, try again later\r\n"
 #define SMTP_RESP_454_ERR_TLS_TEMP_ERROR "454 TLS not available currently\r\n"
 
 #define SMTP_RESP_502_ERR "502 Command not implemented\r\n"
 #define SMTP_RESP_503_ERR "503 Bad command sequence\r\n"
+#define SMTP_RESP_550_ERR_INVALID_RECIPIENT "550 Invalid recipient\r\n"
 #define SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY "550 You are banned by local policy\r\n"
+#define SMTP_RESP_550_ERR "550 Service currently unavailable\r\n"
+#define SMTP_RESP_552_ERR_TOO_BIG_EMAIL "552 Too big email\r\n"
 
 
 // LMTP commands
@@ -64,4 +70,3 @@
 // LMTP responses
 
 #define LMTP_RESP_220_BANNER "220 %s LMTP\r\n"
-

src/sql.h

@@ -7,7 +7,9 @@
 
 
 int open_database(struct session_data *sdata, struct config *cfg);
+int open_sphx(struct session_data *sdata, struct config *cfg);
 void close_database(struct session_data *sdata);
+void close_sphx(struct session_data *sdata);
 int prepare_sql_statement(struct session_data *sdata, struct sql *sql, char *s);
 void p_query(struct session_data *sdata, char *s);
 int p_exec_stmt(struct session_data *sdata, struct sql *sql);

src/stats.c

@@ -0,0 +1,264 @@
+/*
+ * stats.c, SJ
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <netdb.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <syslog.h>
+#include <getopt.h>
+#include <piler.h>
+
+#define SMTP_TIMEOUT 5
+
+extern char *optarg;
+extern int optind;
+
+struct stats {
+   uint64 rcvd;
+   uint64 size;
+   uint64 ssize;
+
+   uint64 sphx;
+   uint64 ram_bytes;
+   uint64 disk_bytes;
+
+   uint64 error_emails;
+   long last_email;
+   float smtp_response_time;
+};
+
+
+int query_counters(struct session_data *sdata, struct stats *stats){
+   int rc=ERR;
+   struct sql sql;
+
+   if(prepare_sql_statement(sdata, &sql, "select rcvd, size, stored_size from counter") == ERR) return rc;
+
+   p_bind_init(&sql);
+
+   if(p_exec_stmt(sdata, &sql) == OK){
+      p_bind_init(&sql);
+
+      sql.sql[sql.pos] = (char *)&(stats->rcvd); sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+      sql.sql[sql.pos] = (char *)&(stats->size); sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+      sql.sql[sql.pos] = (char *)&(stats->ssize); sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+
+      p_store_results(&sql);
+
+      if(p_fetch_results(&sql) == OK) rc = OK;
+
+      p_free_results(&sql);
+   }
+
+   close_prepared_statement(&sql);
+
+   return rc;
+}
+
+
+int get_last_email_archived_timestamp(struct session_data *sdata, struct stats *stats){
+   int rc=ERR;
+   unsigned long arrived=86400;
+   struct sql sql;
+
+   // By default we haven't archived a mail a day ago, and this value should represent an error
+   time(&(sdata->now));
+   arrived = sdata->now - 86400;
+
+   if(prepare_sql_statement(sdata, &sql, "select arrived from metadata order by id desc limit 1") == ERR) return rc;
+
+   p_bind_init(&sql);
+
+   if(p_exec_stmt(sdata, &sql) == OK){
+      p_bind_init(&sql);
+
+      sql.sql[sql.pos] = (char *)&arrived; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(arrived); sql.pos++;
+
+      p_store_results(&sql);
+
+      if(p_fetch_results(&sql) == OK) rc = OK;
+
+      p_free_results(&sql);
+   }
+
+   close_prepared_statement(&sql);
+
+   stats->last_email = sdata->now - arrived;
+
+   return rc;
+}
+
+
+void sphinx_queries(struct session_data *sdata, struct stats *stats){
+   MYSQL_RES *result;
+   MYSQL_ROW row;
+
+   p_query(sdata, "SHOW STATUS LIKE 'queries'");
+
+   result = mysql_store_result(&(sdata->mysql));
+   if(result){
+      row = mysql_fetch_row(result);
+
+      if(row){
+         if(mysql_num_fields(result) == 2){
+            stats->sphx = strtoull(row[1], NULL, 10);
+         }
+      }
+
+      mysql_free_result(result);
+   }
+
+   p_query(sdata, "SHOW INDEX main1 STATUS");
+
+   result = mysql_store_result(&(sdata->mysql));
+   if(result){
+      while((row = mysql_fetch_row(result))){
+         if(strcmp((char*)row[0], "ram_bytes") == 0) stats->ram_bytes = strtoull(row[1], NULL, 10);
+         if(strcmp((char*)row[0], "disk_bytes") == 0) stats->disk_bytes = strtoull(row[1], NULL, 10);
+      }
+
+      mysql_free_result(result);
+   }
+}
+
+
+void count_error_emails(struct stats *stats){
+   DIR *dir;
+   struct stat st;
+
+   dir = opendir(ERROR_DIR);
+   if(dir){
+      struct dirent *de;
+      while((de = readdir(dir))){
+         if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
+
+         char buf[SMALLBUFSIZE];
+         snprintf(buf, sizeof(buf)-1, "%s/%s", ERROR_DIR, de->d_name);
+
+         if(stat(buf, &st) == 0 && S_ISREG(st.st_mode)){
+            stats->error_emails++;
+         }
+      }
+
+      closedir(dir);
+   }
+}
+
+
+void check_smtp_status(struct stats *stats, struct config *cfg){
+   int sd, rc;
+   char port_string[8];
+   char buf[SMALLBUFSIZE];
+   struct addrinfo hints, *res;
+   struct timezone tz;
+   struct timeval tv1, tv2;
+
+   // Set this to a very high number, 1 hour in ms
+   stats->smtp_response_time = 3600000;
+
+   memset(buf, 0, sizeof(buf));
+
+   snprintf(port_string, sizeof(port_string)-1, "%d", cfg->listen_port);
+
+   memset(&hints, 0, sizeof(hints));
+   hints.ai_family = AF_UNSPEC;
+   hints.ai_socktype = SOCK_STREAM;
+
+   if((rc = getaddrinfo(cfg->listen_addr, port_string, &hints, &res)) != 0){
+      fprintf(stderr, "getaddrinfo for '%s': %s\n", cfg->listen_addr, gai_strerror(rc));
+      return;
+   }
+
+   if((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
+      fprintf(stderr, "cannot create socket\n");
+      goto ENDE_CHECK_SMTP_STATUS;
+   }
+
+   gettimeofday(&tv1, &tz);
+
+   if(connect(sd, res->ai_addr, res->ai_addrlen) == -1){
+      fprintf(stderr, "connect()\n");
+      goto ENDE_CHECK_SMTP_STATUS;
+   }
+
+   recvtimeout(sd, buf, sizeof(buf)-1, SMTP_TIMEOUT);
+
+   close(sd);
+
+   gettimeofday(&tv2, &tz);
+
+   if(strncmp(buf, "220 ", 4) == 0){
+      stats->smtp_response_time = tvdiff(tv2, tv1) / 1000.0; // response time in ms
+   }
+
+ENDE_CHECK_SMTP_STATUS:
+   freeaddrinfo(res);
+}
+
+
+void print_json_results(struct stats *stats){
+   printf("{\n");
+   printf("\t\"rcvd\": %llu,\n", stats->rcvd);
+   printf("\t\"size\": %llu,\n", stats->size);
+   printf("\t\"ssize\": %llu,\n", stats->ssize);
+   printf("\t\"sphx\": %llu,\n", stats->sphx);
+   printf("\t\"ram_bytes\": %llu,\n", stats->ram_bytes);
+   printf("\t\"disk_bytes\": %llu,\n", stats->disk_bytes);
+   printf("\t\"error_emails\": %llu,\n", stats->error_emails);
+   printf("\t\"last_email\": %ld,\n", stats->last_email);
+   printf("\t\"smtp_response\": %.2f\n", stats->smtp_response_time);
+
+   printf("}\n");
+}
+
+
+int main(){
+   struct session_data sdata;
+   struct stats stats;
+   struct config cfg;
+   char *configfile=CONFIG_FILE;
+
+   memset(&stats, 0, sizeof(stats));
+
+   srand(getpid());
+
+   (void) openlog("pilerstat", LOG_PID, LOG_MAIL);
+
+   cfg = read_config(configfile);
+
+   if(open_database(&sdata, &cfg) == ERR) return 0;
+
+   query_counters(&sdata, &stats);
+   get_last_email_archived_timestamp(&sdata, &stats);
+
+   close_database(&sdata);
+
+
+   cfg.mysqlsocket[0] = '\0';
+   snprintf(cfg.mysqlhost, MAXVAL-2, "127.0.0.1");
+   cfg.mysqlport = 9306;
+
+   if(open_database(&sdata, &cfg) == ERR) return 0;
+
+   sphinx_queries(&sdata, &stats);
+
+   close_database(&sdata);
+
+   count_error_emails(&stats);
+
+   check_smtp_status(&stats, &cfg);
+
+   print_json_results(&stats);
+
+   return 0;
+}

src/store.c

@@ -51,6 +51,8 @@ int store_file(struct session_data *sdata, char *filename, int len, struct confi
 #else
    EVP_CIPHER_CTX *ctx;
 #endif
+   int blocklen;
+   unsigned char rnd[EVP_MAX_BLOCK_LENGTH];
    unsigned char *outbuf=NULL;
    int outlen=0, writelen, tmplen;
 
@@ -107,26 +109,46 @@ int store_file(struct session_data *sdata, char *filename, int len, struct confi
 
    #if OPENSSL_VERSION_NUMBER < 0x10100000L
       EVP_CIPHER_CTX_init(&ctx);
-      EVP_EncryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      EVP_EncryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+      blocklen = EVP_CIPHER_CTX_block_size(&ctx);
    #else
       ctx = EVP_CIPHER_CTX_new();
       if(!ctx) goto ENDE;
 
       EVP_CIPHER_CTX_init(ctx);
-      EVP_EncryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      EVP_EncryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+      blocklen = EVP_CIPHER_CTX_block_size(ctx);
    #endif
 
-      outbuf = malloc(dstlen + EVP_MAX_BLOCK_LENGTH);
+      // prepend a block with random data as replacement for dynamic iv
+      // see e.g. https://crypto.stackexchange.com/questions/5421/using-cbc-with-a-fixed-iv-and-a-random-first-plaintext-block
+      fd = open(RANDOM_POOL, O_RDONLY);
+      if(fd == -1) goto ENDE;
+      tmplen = readFromEntropyPool(fd, rnd, blocklen);
+      close(fd);
+      if(tmplen != blocklen) goto ENDE;
+      // make sure, random data does not start with zlib magic 0x78
+      if(rnd[0] == 0x78) rnd[0] =~ rnd[0];
+
+      outbuf = malloc(dstlen + blocklen * 2);
       if(outbuf == NULL) goto ENDE;
 
    #if OPENSSL_VERSION_NUMBER < 0x10100000L
-      if(!EVP_EncryptUpdate(&ctx, outbuf, &outlen, z, dstlen)) goto ENDE;
+      if(!EVP_EncryptUpdate(&ctx, outbuf, &outlen, rnd, blocklen)) goto ENDE;
+      if(!EVP_EncryptUpdate(&ctx, outbuf + outlen, &tmplen, z, dstlen)) goto ENDE;
+   #else
+      if(!EVP_EncryptUpdate(ctx, outbuf, &outlen, rnd, blocklen)) goto ENDE;
+      if(!EVP_EncryptUpdate(ctx, outbuf + outlen, &tmplen, z, dstlen)) goto ENDE;
+   #endif
+      outlen += tmplen;
+
+   #if OPENSSL_VERSION_NUMBER < 0x10100000L
       if(!EVP_EncryptFinal_ex(&ctx, outbuf + outlen, &tmplen)) goto ENDE;
    #else
-      if(!EVP_EncryptUpdate(ctx, outbuf, &outlen, z, dstlen)) goto ENDE;
       if(!EVP_EncryptFinal_ex(ctx, outbuf + outlen, &tmplen)) goto ENDE;
    #endif
       outlen += tmplen;
+
    #if OPENSSL_VERSION_NUMBER < 0x10100000L
       EVP_CIPHER_CTX_cleanup(&ctx);
    #else
@@ -206,19 +228,18 @@ int store_file(struct session_data *sdata, char *filename, int len, struct confi
 
 ENDE:
    if(outbuf) free(outbuf);
-   if(z) free(z);
+   if(z) free(z); //-V547
 
    return ret;
 }
 
 
 int remove_stored_message_files(struct session_data *sdata, struct parser_state *state, struct config *cfg){
-   int i;
    char s[SMALLBUFSIZE];
 
    if(state->n_attachments > 0){
 
-      for(i=1; i<=state->n_attachments; i++){
+      for(int i=1; i<=state->n_attachments; i++){
          snprintf(s, sizeof(s)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, sdata->ttmpfile[8], sdata->ttmpfile[9], sdata->ttmpfile[10], sdata->ttmpfile[RND_STR_LEN-4], sdata->ttmpfile[RND_STR_LEN-3], sdata->ttmpfile[RND_STR_LEN-2], sdata->ttmpfile[RND_STR_LEN-1], sdata->ttmpfile, i);
 
          if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: unlinking %s", sdata->ttmpfile, s);
@@ -235,5 +256,3 @@ int remove_stored_message_files(struct session_data *sdata, struct parser_state
 
    return 0;
 }
-
-

src/tai.c

@@ -4,7 +4,6 @@
 #include <sys/time.h>
 #include "tai.h"
 
-static char hex[16] = "0123456789abcdef";
 
 
 void tai_pack(char *s, struct tai *t){
@@ -50,21 +49,3 @@ void taia_now(struct taia *t){
    t->nano = 1000 * now.tv_usec + 500;
    t->atto = 0;
 }
-
-
-void tai_timestamp(char *s){
-   struct tai now;
-   char nowpack[TAI_PACK];
-   int i;
-
-   now.x = 4611686018427387914ULL + (uint64)time((long *) 0);
-
-   tai_pack(nowpack, &now);
-
-   for (i = 0;i < 8;++i) {
-      *(s+i*2) = hex[(nowpack[i] >> 4) & 15];
-      *(s+i*2+1) = hex[nowpack[i] & 15];
-   }
-
-   *(s+2*TAI_PACK) = '\0';
-}

src/tai.h

@@ -5,12 +5,12 @@
 #ifndef _TAI_H
    #define _TAI_H
 
+#include "config.h"
+
 #define TAI_PACK 8
 #define TAIA_PACK 16
 #define TIMESTAMP 25
 
-typedef unsigned long long uint64;
-
 struct tai {
    uint64 x;
 };
@@ -26,4 +26,3 @@ void taia_now(struct taia *t);
 void taia_pack(char *s, struct taia *t);
 
 #endif /* _TAI_H */
-

src/test.c

@@ -12,9 +12,21 @@
 #include <time.h>
 #include <locale.h>
 #include <syslog.h>
+#include <getopt.h>
 #include <piler.h>
 
 
+void usage(){
+   printf("\nusage: pilertest\n\n");
+
+   printf("    [-c <config file>]                Config file to use if not the default\n");
+   printf("    -m <message eml file>             Message in EML format\n");
+   printf("    -a <extra recipient>              Extra recipient\n");
+
+   exit(0);
+}
+
+
 int main(int argc, char **argv){
    int i;
    time_t retention_seconds=0;
@@ -24,25 +36,77 @@ int main(int argc, char **argv){
    struct config cfg;
    struct data data;
    struct import import;
-   char *rule;
+   char *configfile=CONFIG_FILE, *rule, *emlfile=NULL;
+   char puf[SMALLBUFSIZE];
+
+   import.extra_recipient = NULL;
 
    srand(getpid());
 
-   if(argc < 2){
-      fprintf(stderr, "usage: %s <message> [<extra recipient>]\n", argv[0]);
-      exit(1);
+   while(1){
+
+#ifdef _GNU_SOURCE
+      static struct option long_options[] =
+         {
+            {"config",          required_argument,  0,  'c' },
+            {"message",         required_argument,  0,  'm' },
+            {"extra-recipient", required_argument,  0,  'a' },
+            {0,0,0,0}
+         };
+
+      int option_index = 0;
+
+
+      int c = getopt_long(argc, argv, "c:m:a:hv?", long_options, &option_index);
+#else
+      int c = getopt(argc, argv, "c:m:a:hv?");
+#endif
+
+      if(c == -1) break;
+
+      switch(c){
+
+         case 'c' :
+                    configfile = optarg;
+                    break;
+
+         case 'm' :
+                    emlfile = optarg;
+                    break;
+
+         case 'a':
+                    snprintf(puf, sizeof(puf)-1, "%s ", optarg);
+                    import.extra_recipient = puf;
+                    break;
+
+         default  :
+                    usage();
+                    break;
+      }
+
    }
 
+
+   if(emlfile == NULL) usage();
+
    if(!can_i_write_directory(NULL)) __fatal("cannot write current directory!");
 
-   if(stat(argv[1], &st) != 0){
-      fprintf(stderr, "%s is not found\n", argv[1]);
+   if(stat(emlfile, &st) != 0){
+      fprintf(stderr, "%s is not found\n", emlfile);
+      return 0;
+   }
+
+
+   if(!can_i_write_directory(NULL)) __fatal("cannot write current directory!");
+
+   if(stat(emlfile, &st) != 0){
+      fprintf(stderr, "%s is not found\n", emlfile);
       return 0;
    }
 
    (void) openlog("test", LOG_PID, LOG_MAIL);
 
-   cfg = read_config(CONFIG_FILE);
+   cfg = read_config(configfile);
 
    if(open_database(&sdata, &cfg) == ERR) return 0;
 
@@ -51,10 +115,6 @@ int main(int argc, char **argv){
 
    printf("build: %d\n", get_build());
 
-   import.extra_recipient = NULL;
-
-   if(argc > 2) import.extra_recipient = argv[2];
-
    data.import = &import;
 
    data.folder = 0;
@@ -68,20 +128,20 @@ int main(int argc, char **argv){
 
    load_mydomains(&sdata, &data, &cfg);
 
-   load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
-   load_rules(&sdata, &data, data.retention_rules, SQL_RETENTION_RULE_TABLE);
-   load_rules(&sdata, &data, data.folder_rules, SQL_FOLDER_RULE_TABLE);
+   load_rules(&sdata, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
+   load_rules(&sdata, data.retention_rules, SQL_RETENTION_RULE_TABLE);
+   load_rules(&sdata, data.folder_rules, SQL_FOLDER_RULE_TABLE);
 
 
    init_session_data(&sdata, &cfg);
- 
+
    sdata.delivered = 0;
    sdata.tot_len = st.st_size;
    sdata.import = 1;
 
-   snprintf(sdata.ttmpfile, SMALLBUFSIZE-1, "%s", argv[1]);
-   snprintf(sdata.filename, SMALLBUFSIZE-1, "%s", argv[1]);
-   snprintf(sdata.tmpframe, SMALLBUFSIZE-1, "%s.m", argv[1]);
+   snprintf(sdata.ttmpfile, SMALLBUFSIZE-1, "%s", emlfile);
+   snprintf(sdata.filename, SMALLBUFSIZE-1, "%s", emlfile);
+   snprintf(sdata.tmpframe, SMALLBUFSIZE-1, "%s.m", emlfile);
 
    printf("parsing...\n");
    state = parse_message(&sdata, 1, &data, &cfg);
@@ -91,7 +151,9 @@ int main(int argc, char **argv){
 
    printf("message-id: %s / %s\n", state.message_id, state.message_id_hash);
    printf("from: *%s (%s)*\n", state.b_from, state.b_from_domain);
+   printf("sender: *%s (%s)*\n", state.b_sender, state.b_sender_domain);
    printf("to: *%s (%s)*\n", state.b_to, state.b_to_domain);
+   printf("journal recipients: *%s*\n", state.b_journal_to);
    printf("reference: *%s*\n", state.reference);
    printf("subject: *%s*\n", state.b_subject);
    printf("body: *%s*\n", state.b_body);
@@ -102,14 +164,14 @@ int main(int argc, char **argv){
 
    printf("hdr len: %d\n", sdata.hdr_len);
 
-   rule = check_againt_ruleset(data.archiving_rules, &state, st.st_size, sdata.spam_message);
- 
+   rule = check_against_ruleset(data.archiving_rules, &state, st.st_size, sdata.spam_message);
+
    printf("body digest: %s\n", sdata.bodydigest);
 
    printf("rules check: %s\n", rule);
 
    retention_seconds = query_retain_period(&data, &state, st.st_size, sdata.spam_message, &cfg);
-   sdata.retained = sdata.now + retention_seconds;
+   sdata.retained = sdata.sent + retention_seconds;
 
    printf("folder: %d\n", get_folder_id_by_rule(&data, &state, st.st_size, sdata.spam_message, &cfg));
 
@@ -122,7 +184,7 @@ int main(int argc, char **argv){
    clearhash(data.mydomains);
 
    for(i=1; i<=state.n_attachments; i++){
-      printf("i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s\n", i, state.attachments[i].filename, state.attachments[i].type, state.attachments[i].size, state.attachments[i].internalname, state.attachments[i].digest);
+      printf("i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, dumped: %d, digest: %s\n", i, state.attachments[i].filename, state.attachments[i].type, state.attachments[i].size, state.attachments[i].internalname, state.attachments[i].dumped, state.attachments[i].digest);
       unlink(state.attachments[i].internalname);
    }
 
@@ -134,6 +196,8 @@ int main(int argc, char **argv){
 
    printf("spam: %d\n", sdata.spam_message);
 
+   printf("1st received line: %s\n", state.receivedbuf);
+
    if(sdata.internal_sender == 0 && sdata.internal_recipient == 0) printf("NOT IN mydomains\n");
 
    printf("\n\n");

src/tokenizer.c

@@ -0,0 +1,142 @@
+/*
+ * tokenizer.c, SJ
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <piler.h>
+
+void tokenize(char *buf, struct parser_state *state, struct session_data *sdata, struct data *data, struct config *cfg){
+   char *p, *q, puf[SMALLBUFSIZE];
+   int result;
+   unsigned int len;
+
+   translateLine((unsigned char*)buf, state);
+
+   reassembleToken(buf);
+
+
+   p = buf;
+
+   //printf("a: %d/%d/%d/%d/j=%d %s\n", state->is_1st_header, state->is_header, state->message_rfc822, state->message_state, sdata->ms_journal, buf);
+
+   do {
+      memset(puf, 0, sizeof(puf));
+      p = split(p, ' ', puf, sizeof(puf)-1, &result);
+
+      if(puf[0] == '\0') continue;
+
+      degenerateToken((unsigned char*)puf);
+
+      if(puf[0] == '\0') continue;
+
+      strncat(puf, " ", sizeof(puf)-strlen(puf)-1);
+
+      if(strncasecmp(puf, "http://", 7) == 0 || strncasecmp(puf, "https://", 8) == 0) fixURL(puf, sizeof(puf)-1);
+
+      len = strlen(puf);
+
+      // skip body tokens if not an URL && (empty token || too long)
+      if(state->is_header == 0 && strncmp(puf, "__URL__", 7) && (puf[0] == ' ' || (len > MAX_WORD_LEN && cfg->enable_cjk == 0)) ){
+         continue;
+      }
+
+      char md5buf[2*DIGEST_LENGTH+2];
+
+      if(state->message_state == MSG_FROM && state->is_1st_header == 1 && strlen(state->b_from) < SMALLBUFSIZE-len-1){
+         strtolower(puf);
+
+         q = strchr(puf, '@');
+         if(q) fix_plus_sign_in_email_address(puf, &q, &len);
+
+         memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
+
+         if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(puf) == 1 && state->b_from_domain[0] == '\0'){
+            if(q && strlen(q) > 5){
+               memcpy(&(state->b_from_domain), q+1, strlen(q+1)-1);
+               if(strstr(sdata->mailfrom, "<>")){
+                  snprintf(sdata->fromemail, SMALLBUFSIZE-1, "%s", puf);
+                  sdata->fromemail[len-1] = '\0';
+               }
+            }
+
+            if(is_email_address_on_my_domains(puf, data) == 1) sdata->internal_sender = 1;
+
+            if(len >= MAX_EMAIL_ADDRESS_SPHINX_LEN && strlen(state->b_from) < SMALLBUFSIZE-len-1){
+               create_md5_from_email_address(puf, md5buf);
+               memcpy(&(state->b_from[strlen(state->b_from)]), md5buf, strlen(md5buf));
+            }
+
+            if(strlen(state->b_from) < SMALLBUFSIZE-len-1){
+               split_email_address(puf);
+               memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
+            }
+         }
+      }
+      else if(state->message_state == MSG_SENDER && state->is_1st_header == 1 && strlen(state->b_sender) < SMALLBUFSIZE-len-1){
+         strtolower(puf);
+
+         q = strchr(puf, '@');
+         if(q) fix_plus_sign_in_email_address(puf, &q, &len);
+
+         memcpy(&(state->b_sender[strlen(state->b_sender)]), puf, len);
+
+         if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(puf) == 1 && state->b_sender_domain[0] == '\0'){
+            if(q && strlen(q) > 5){
+               memcpy(&(state->b_sender_domain), q+1, strlen(q+1)-1);
+            }
+
+            if(len >= MAX_EMAIL_ADDRESS_SPHINX_LEN && strlen(state->b_sender) < SMALLBUFSIZE-len-1){
+               create_md5_from_email_address(puf, md5buf);
+               memcpy(&(state->b_sender[strlen(state->b_sender)]), md5buf, strlen(md5buf));
+            }
+
+            if(strlen(state->b_sender) < SMALLBUFSIZE-len-1){
+               split_email_address(puf);
+               memcpy(&(state->b_sender[strlen(state->b_sender)]), puf, len);
+            }
+         }
+      }
+      else if((state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT || state->message_state == MSG_ENVELOPE_TO) && state->is_1st_header == 1 && state->tolen < MAXBUFSIZE-len-1){
+         strtolower(puf);
+
+         /* fix aaa+bbb@ccc.fu address to aaa@ccc.fu, 2017.02.04, SJ */
+         q = strchr(puf, '@');
+         if(q) fix_plus_sign_in_email_address(puf, &q, &len);
+
+         if((state->message_state == MSG_RECIPIENT || state->message_state == MSG_ENVELOPE_TO) && findnode(state->journal_recipient, puf) == NULL && state->journaltolen < sizeof(state->b_journal_to)-len-1){
+            addnode(state->journal_recipient, puf);
+            memcpy(&(state->b_journal_to[state->journaltolen]), puf, len);
+            state->journaltolen += len;
+            if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: journal rcpt: '%s'", sdata->ttmpfile, puf);
+         }
+
+         if(len >= MAX_EMAIL_ADDRESS_SPHINX_LEN){
+            create_md5_from_email_address(puf, md5buf);
+            add_recipient(md5buf, strlen(md5buf), sdata, state, data, cfg);
+         }
+
+         add_recipient(puf, len, sdata, state, data, cfg);
+      }
+      else if(state->message_state == MSG_BODY && len >= (unsigned int)(cfg->min_word_len) && state->bodylen < BIGBUFSIZE-len-1){
+         // 99% of email addresses are longer than 8 characters
+         if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(puf)){
+            fix_email_address_for_sphinx(puf);
+         }
+
+         memcpy(&(state->b_body[state->bodylen]), puf, len);
+         state->bodylen += len;
+      }
+
+   } while(p);
+
+}