Bumped up version number to 1.4.5

Signed-off-by: Janos SUTO <sj@acts.hu>

.gitignore

@@ -0,0 +1,26 @@
+Makefile
+*.o
+*.so
+*.a
+*.deb
+src/libpiler.so.0*
+src/piler
+src/pilerconf
+src/piler-smtp
+src/pileraget
+src/pilerexport
+src/pilerget
+src/pilerimport
+src/pilerstats
+src/pilertest
+src/reindex
+unit_tests/check_attachments
+unit_tests/check_decoder
+unit_tests/check_digest
+unit_tests/check_hash
+unit_tests/check_misc
+unit_tests/check_mydomains
+unit_tests/check_parser
+unit_tests/check_parser_utils
+unit_tests/check_rules
+unit_tests/smtp

G

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+read -r version < VERSION
+
+tarball="piler-${version}.tar.gz"
+
+git archive --prefix="piler-${version}/" HEAD | gzip -c > "$tarball"
+sha256sum "$tarball"

LICENSE

@@ -1,6 +1,6 @@
 piler, an enterprise level email archiving application
 
-Copyright (C) 2012-2016, Janos SUTO <sj@acts.hu>
+Copyright (C) 2012-2023, Janos SUTO <sj@acts.hu>
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

Makefile.in

@@ -33,7 +33,7 @@ INSTALL = @INSTALL@
 am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \
 	piler-config.h stamp.h stamp-h1 params.h
 
-all: all-recursive
+all: all-recursive config-php
 install: installdirs install-recursive
 
 
@@ -63,6 +63,10 @@ $(RECURSIVE_TARGETS):
 	fi; test -z "$$fail"
 
 
+config-php:
+	sed -e "s%SYSCONFDIR%$(sysconfdir)%g" -e "s%SBINDIR%$(sbindir)%g" config.php.in > webui/config.php
+	sed -e "s%BINDIR%$(bindir)%g" -i webui/config.php
+
 
 installdirs: mkinstalldirs
 	$(srcdir)/mkinstalldirs \
@@ -75,15 +79,20 @@ installdirs: mkinstalldirs
 		$(DESTDIR)$(sysconfdir)/piler \
 		$(DESTDIR)/etc/init.d \
 		$(DESTDIR)$(localstatedir)/piler/store \
-		$(DESTDIR)$(localstatedir)/piler/stat $(DESTDIR)$(localstatedir)/piler/tmp \
+		$(DESTDIR)$(localstatedir)/piler/stat \
-		$(DESTDIR)$(localstatedir)/piler/sphinx
+		$(DESTDIR)$(localstatedir)/piler/tmp \
+		$(DESTDIR)$(localstatedir)/piler/error \
+		$(DESTDIR)$(localstatedir)/piler/export \
+		$(DESTDIR)$(localstatedir)/piler/manticore
 
 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/run/piler
 	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/store
 	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/imap
 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/stat
 	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/tmp
-	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/sphinx
+	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/error
+	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/export
+	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/manticore
 
 
 install-am:
@@ -117,7 +126,7 @@ key:
 	dd if=/dev/urandom bs=56 count=1 of=piler.key
 
 postinstall:
-	@sh util/postinstall.sh $(RUNNING_USER) $(RUNNING_GROUP) $(sysconfdir) $(localstatedir) $(libexecdir)
+	@bash util/postinstall.sh $(RUNNING_USER) $(RUNNING_GROUP) $(sysconfdir) $(localstatedir) $(libexecdir)
 
 
 

RELEASE_NOTES

@@ -1,3 +1,129 @@
+1.4.5:
+------
+
+- Introduced 2 new variables in /etc/piler/piler.conf affecting piler-smtp
+
+  ; max message size in bytes
+  ; piler-smtp will reject any message that's bigger than this number
+  max_message_size=50000000
+
+  ; max memory in bytes piler-smtp uses for buffering messages
+  ; when this limit is exceeded, no new emails will be accepted
+  ; until the used memory for all in progress emails decreases
+  ; below this level
+  max_smtp_memory=500000000
+
+  Be sure to adjust these values to your environment!
+
+- Added read-only connection support for manticore
+  If using sphinx, add the following to config-site.php:
+  $config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9306';
+
+- pilerimport supports Zimbra IMAP impersonation
+
+  Generate the following base64 encoded string:
+  (Be sure to use the actual usernames and password):
+
+  pw="$( printf '%s\0%s\0%s' 'username' 'zimbra_admin_username' 'zimbra_admin_password' | base64 )"
+
+  Then specify -u ZIMBRA -p "$pw" for pilerimport, eg.
+
+  pilerimport -i imap.server -u ZIMBRA -p "$pw" ...
+
+  Note that "ZIMBRA" is a special username, it indicates for pilerimport
+  to actually use the imap impersonation for Zimbra.
+
+
+1.4.4:
+------
+
+- Renamed "group" table to "usergroup"
+  Be sure to run util/db-upgrade.sql on the mysql piler database
+
+
+1.4.3:
+------
+
+- Improved real-time index support (no need to use rtindex.py any more)
+  See https://www.mailpiler.org/wiki/current:manticore for more
+
+
+1.4.2:
+------
+
+- Fixed docker image
+- Fixed php socket path in piler-nginx.conf
+- Fixed manticore index data settings
+
+
+1.4.1:
+------
+
+- Replaced sphinx with manticore 5.0.2, see https://www.mailpiler.org/wiki/current:manticore
+- Replaced obsoleted libchart library with chart.js
+- Added real-time index support
+- Added oauth2 support to imapfetch.py
+- Added support for pilerexport to spread files among several directories. See the -D option
+
+
+1.3.12:
+-------
+
+- Introduced new piler.conf variable: tls_min_version
+
+  It sets the minimum TLS protocol version the piler-smtp daemon supports.
+
+  Possible values:
+  - TLSv1   (not recommended)
+  - TLSv1.1 (not recommended)
+  - TLSv1.2 (default)
+  - TLSv1.3
+
+- Introduced the archive_address feature, see etc/example.conf for the details
+- Introduced the raw: search label
+- Added Italian translation. Credits: Stefano Gatto
+- timestamp signing sorts by 'id' column
+- timestamp hash value defaults to sha256
+- Minor fixes
+- Added support for php 8.1
+- Fixed handling long email addresses
+
+Be sure to apply util/db-upgrade.sql
+
+
+1.3.11:
+-------
+
+- [BUGFIX] Refactored the smtp timeout check
+- Obsoleted the LDAP port parameter. Specify the ldap host in the form
+  of protocol://hostname:port, eg. ldaps://ldap.example.com:636
+
+1.3.10:
+-------
+
+- Added security header feature
+- Introduced the smtp acl list, and obsoleted the tcp_wrappers check
+- Switched from Blowfish encryption to AES-256
+
+1.3.9:
+------
+
+- Added a separator to searching for attachment names
+- [BUGFIX] Render multiple mail parts in mail view instead of only the last part
+- Use TLS v1.2 with openssl 1.0.x for connecting remote pop3/imap servers
+- Instant search results to the gui when the search page loads
+- Support sphinx-3.3.1, introduced sphinx strict mode variable
+- GUI domain fixes
+- gcc 9 fixes
+- Fix permission on sphinx data dir to 700
+- pilerpurge.py should honor the mysqlhost value
+- Password change enabled by default
+- Health page fixes
+- GUI mime parser fixes
+- Start/stop script fix
+- Optimized search page for mobile devices, set $config['ENABLE_MOBILE_PREVIEW'] = 1; in config-site.php to enable it
+
+
 1.3.0:
 -----
 

SECURITY.md

@@ -0,0 +1,20 @@
+Disclosure policy
+
+If you find a security issue, please contact the project owner at sj@acts.hu
+with the details (ie. piler version, details of the setup, how to exploit the
+vulnerability, etc).
+
+Please provide 30 days for verifying the vulnerability, fixing the issue, and
+notifying the piler users.
+
+Security update policy
+
+If a security vulnerability has found, the details, possible mitigations,
+workarounds, etc. will be shared on the piler mailing list (piler-user@list.acts.hu)
+and on the wiki: https://www.mailpiler.org/
+
+Security configurations
+
+ - Use https for the GUI
+ - Reset the default passwords for admin and auditor
+ - Use the smtp acl feature to restrict SMTP access to the archive, see https://mailpiler.com/smtp-acl-list/

VERSION

@@ -1 +1 @@
-1.3.0
+1.4.5

bitbucket-pipelines.yml

@@ -3,10 +3,20 @@
 # Only use spaces to indent your .yml configuration.
 # -----
 # You can specify a custom docker image from Docker Hub as your build environment.
-# image: docker-image:tag
+
+image: sutoj/builder:bionic
+clone:
+  depth: 5
 
 pipelines:
   default:
     - step:
         script:
-          - echo "Everything is awesome!"
+          - service mysql start
+          - ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --with-database=mariadb
+          - make clean all install
+          - mysql -u piler -ppiler123 piler1 < /usr/share/piler/db-mysql.sql
+          - cd unit_tests
+          - ./run.sh
+          - cd ../webui
+          - phpunit

config.php.in

@@ -1,9 +1,14 @@
 <?php
 
+ini_set('session.cookie_httponly', true);
+ini_set('session.use_strict_mode', 1);
+ini_set('session.use_only_cookies', 1);
+
+
 define('NORMAL', 1);
 define('DEBUG', 5);
 
-$config = array();
+$config = [];
 
 
 /*
@@ -11,9 +16,15 @@ $config = array();
  * variable to be overridden in config-site.php
  */
 
+$config['ENABLE_MOBILE_PREVIEW'] = 0;
+
+$config['SITE_LOGO_LG'] = '/view/theme/default/assets/images/archive-logo-lg.png';
+$config['SITE_LOGO_SM'] = '/view/theme/default/assets/images/archive-logo-sm.png';
+$config['COMPATIBILITY'] = 'Which browsers are supported, etc';
+
 $config['BRANDING_TEXT'] = '';
 $config['BRANDING_URL'] = '';
-$config['BRANDING_LOGO'] = '';
+$config['BRANDING_LOGO'] = $config['SITE_LOGO_SM'];
 $config['BRANDING_BACKGROUND_COLOUR'] = '';
 $config['BRANDING_TEXT_COLOUR'] = '';
 $config['BRANDING_FAVICON'] = '/view/theme/default/assets/ico/favicon.png';
@@ -23,14 +34,21 @@ $config['TITLE_PREFIX'] = '';
 $config['CUSTOM_PRE_AUTH_FUNCTION'] = '';
 $config['CUSTOM_EMAIL_QUERY_FUNCTION'] = '';
 
+$config['DOMAIN_REGEX'] = '/^[a-zA-Z0-9]+[a-zA-Z0-9-_\.]{0,}\.[a-zA-Z0-9]{2,20}$/';
+
 $config['BOOTSTRAP_THEME'] = '-cosmo';
 
 $config['DEFAULT_LANG'] = 'en';
 
 $config['THEME'] = 'default';
 
-$config['SITE_NAME'] = 'piler.yourdomain.com';
+$config['SITE_NAME'] = 'piler.example.com';
-$config['SITE_URL'] = 'http://piler.yourdomain.com/';
+$config['SITE_URL'] = 'http://piler.example.com/';
+
+$config['EXTERNAL_DASHBOARD_URL'] = '';
+
+$config['SESSION_EXPIRY'] = 3600;
+$config['DELTA_INDEXER_PERIOD'] = 1800;
 
 $config['ENABLE_SAAS'] = 0;
 $config['CAPTCHA_FAILED_LOGIN_COUNT'] = 0;
@@ -46,6 +64,8 @@ $config['SHOW_MENU_FOR_OUTLOOK'] = 0;
 
 $config['SEARCH_QUERY_QUOTING'] = 0;
 
+$config['LOCALIZE_MESSAGE_HEADERS_IN_PREVIEW'] = 1;
+
 $config['TIMEZONE'] = 'Europe/Budapest';
 
 $config['PROVIDED_BY'] = 'www.mailpiler.org';
@@ -55,16 +75,22 @@ $config['SITE_DESCRIPTION'] = 'piler email archiver';
 $config['INDEXER_BEACON'] = '/var/piler/stat/indexer';
 $config['PURGE_BEACON'] = '/var/piler/stat/purge';
 
+$config['ENABLE_PDF_DOWNLOAD'] = 0;
+// You may need to run "Xvfb :1 -screen 0 1024x768x16"
+// In this case specify WKHTMLTOPDF_COMMAND = "DISPLAY=:1.0 wkhtmltopdf";
+$config['WKHTMLTOPDF_COMMAND'] = "wkhtmltopdf";
+
 // authentication against an ldap directory (disabled by default)
 
 $config['ENABLE_LDAP_AUTH'] = 0;
-$config['LDAP_HOST'] = 'ldap.yourdomain.com';
+$config['LDAP_HOST'] = 'ldap.example.com';
 $config['LDAP_HELPER_DN'] = 'cn=....';
 $config['LDAP_HELPER_PASSWORD'] = 'xxxxxxx';
 $config['LDAP_MAIL_ATTR'] = 'mail';
 $config['LDAP_AUDITOR_MEMBER_DN'] = '';
 $config['LDAP_ADMIN_MEMBER_DN'] = '';
 $config['LDAP_BASE_DN'] = '';
+$config['LDAP_USE_START_TLS'] = 0;
 
 // AD specific settings
 //
@@ -88,14 +114,18 @@ $config['LDAP_MAIL_ATTR'] = 'proxyAddresses';
 //$config['LDAP_MAIL_ATTR'] = 'mail';
 
 // iredmail specific settings
-//$config['LDAP_HELPER_DN'] = 'cn=vmailadmin,dc=yourdomain,dc=com';
+//$config['LDAP_HELPER_DN'] = 'cn=vmailadmin,dc=example,dc=com';
 //$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'mailUser';
-//$config['LDAP_BASE_DN'] = 'o=domains,dc=yourdomain,dc=com';
+//$config['LDAP_BASE_DN'] = 'o=domains,dc=example,dc=com';
 //$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'mailList';
 //$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'memberOfGroup';
 //$config['LDAP_MAIL_ATTR'] = 'mail';
 
-
+// Uninvention specific settings
+//$config['LDAP_MAIL_ATTR'] = 'mailPrimaryAddress';
+//$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'person';
+//$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'person';
+//$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mailAlternativeAddress';
 
 
 // enable single sign-on (disabled by default)
@@ -109,14 +139,14 @@ $config['ENABLE_IMAP_AUTH'] = 0;
 $config['RESTORE_OVER_IMAP'] = 0;
 $config['IMAP_RESTORE_FOLDER_INBOX'] = 'INBOX';
 $config['IMAP_RESTORE_FOLDER_SENT'] = 'Sent';
-$config['IMAP_HOST'] = 'mail.yourdomain.com';
+$config['IMAP_HOST'] = 'mail.example.com';
 $config['IMAP_PORT'] =  993;
 $config['IMAP_SSL'] = true;
 
 // enable authentication against a pop3 server (disabled by default)
 
 $config['ENABLE_POP3_AUTH'] = 0;
-$config['POP3_HOST'] = 'mail.yourdomain.com';
+$config['POP3_HOST'] = 'mail.example.com';
 $config['POP3_PORT'] = 995;
 $config['POP3_SSL'] = true;
 
@@ -132,21 +162,22 @@ $config['GOOGLE_DEVELOPER_KEY'] = 'xxxxxxxxxxxx';
 $config['GOOGLE_APPLICATION_NAME'] = 'piler enterprise email archiver';
 $config['GOOGLE_ALL_MAIL'] = '[Gmail]/All Mail';
 
-$config['SITE_LOGO_LG'] = 'view/theme/default/assets/images/archive-logo-lg.png';
-$config['SITE_LOGO_SM'] = 'view/theme/default/assets/images/archive-logo-sm.png';
-$config['COMPATIBILITY'] = 'Which browsers are supported, etc';
-
 
 $config['ENABLE_AUDIT'] = 1;
 $config['MEMCACHED_ENABLED'] = 0;
-$config['PASSWORD_CHANGE_ENABLED'] = 0;
+$config['PASSWORD_CHANGE_ENABLED'] = 1;
 $config['ENABLE_STATISTICS'] = 1;
 $config['ENABLE_HISTORY'] = 1;
 $config['ENABLE_DELETE'] = 0;
+$config['NEED_TO_APPROVE_DELETE'] = 0;
 $config['ENABLE_REMOTE_IMAGES'] = '0';
 $config['ENABLE_ON_THE_FLY_VERIFICATION'] = 0;
 $config['ENABLE_LDAP_IMPORT_FEATURE'] = 0;
 $config['ENABLE_FOLDER_RESTRICTIONS'] = 0;
+
+// When enabled the search page auto loads the search results
+// ie. as if the user had clicked on the Search button
+$config['ENABLE_INSTANT_SEARCH'] = 0;
 $config['SEARCH_RESULT_CHECKBOX_CHECKED'] = 0;
 $config['HELPER_URL'] = '';
 $config['LOG_LEVEL'] = NORMAL;
@@ -159,11 +190,13 @@ $config['BULK_DOWNLOAD_FOR_USERS'] = 1;
 $config['MAX_DOWNLOAD_PER_HOUR'] = 0;
 $config['MAX_RESTORE_PER_HOUR'] = 0;
 
-$config['MAX_CGI_FROM_SUBJ_LEN'] = 34;
+$config['MAX_CGI_FROM_SUBJ_LEN'] = 48;
 $config['PAGE_LEN'] = 20;
 $config['MAX_NUMBER_OF_FROM_ITEMS'] = 5;
 $config['MAX_SEARCH_HITS'] = 1000;
 
+$config['SPHINX_MAIN_INDEX_THRESHOLD'] = 2000000000;
+
 $config['DEFAULT_RETENTION'] = 0;
 
 $config['LOCALHOST'] = '127.0.0.1';
@@ -171,23 +204,25 @@ $config['PILER_HOST'] = '0.0.0.0';
 $config['PILER_PORT'] = 25;
 $config['SMARTHOST'] = '';
 $config['SMARTHOST_PORT'] = 25;
-$config['SMTP_DOMAIN'] = 'yourdomain.com';
+$config['SMARTHOST_USER'] = '';
-$config['SMTP_FROMADDR'] = 'no-reply@yourdomain.com';
+$config['SMARTHOST_PASSWORD'] = '';
-$config['ADMIN_EMAIL'] = 'admin@yourdomain.com';
+$config['SMTP_DOMAIN'] = 'example.com';
+$config['SMTP_FROMADDR'] = 'no-reply@example.com';
+$config['ADMIN_EMAIL'] = 'admin@example.com';
 
 $config['PILER_HEADER_FIELD'] = 'X-piler-id: ';
 
 $config['DEFAULT_POLICY'] = 'default_policy';
 
-$config['DIR_BASE'] = '/var/piler/www/';
+$config['DIR_BASE'] = dirname(__FILE__) . '/';
 
-$config['DIR_SPHINX'] = '/var/piler/sphinx/';
+$config['DIR_SPHINX'] = '/var/piler/manticore/';
 $config['DIR_STAT'] = '/var/piler/stat';
 $config['DIR_IMAP'] = '/var/piler/imap';
 $config['DIR_TMP'] = '/var/piler/tmp';
 
-$config['DECRYPT_BINARY'] = '/usr/local/bin/pilerget';
+$config['DECRYPT_BINARY'] = 'BINDIR/pilerget';
-$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/local/bin/pileraget';
+$config['DECRYPT_ATTACHMENT_BINARY'] = 'BINDIR/pileraget';
 $config['DECRYPT_BUFFER_LENGTH'] = 65536;
 
 $config['OPENSSL_BINARY'] = '/usr/bin/openssl';
@@ -195,6 +230,13 @@ $config['TSA_URL'] = '';
 $config['TSA_PUBLIC_KEY_FILE'] = '';
 $config['TSA_START_ID'] = 1;
 $config['TSA_STAMP_REQUEST_UNIT_SIZE'] = 10000;
+$config['TSA_VERIFY_CERTIFICATE'] = true;
+$config['TSA_RELAXED_CHECK'] = false;
+$config['TSA_AUTH_USER'] = '';
+$config['TSA_AUTH_PASSWORD'] = '';
+$config['TSA_AUTH_CERT_FILE'] = '';
+$config['TSA_AUTH_KEY_FILE'] = '';
+$config['TSA_AUTH_KEY_PASSWORD'] = '';
 
 $config['DB_DRIVER'] = 'mysql';
 $config['DB_PREFIX'] = '';
@@ -207,15 +249,19 @@ $config['DB_CHARSET'] = 'utf8mb4';
 $config['SPHINX_DRIVER'] = 'sphinx';
 $config['SPHINX_DATABASE'] = 'sphinx';
 $config['SPHINX_HOSTNAME'] = '127.0.0.1:9306';
+$config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9307';
 $config['SPHINX_MAIN_INDEX'] = 'main1,dailydelta1,delta1';
 $config['SPHINX_ATTACHMENT_INDEX'] = 'att1';
 $config['SPHINX_TAG_INDEX'] = 'tag1';
 $config['SPHINX_NOTE_INDEX'] = 'note1';
+$config['SPHINX_STRICT_SCHEMA'] = 1;
+$config['RT'] = 0;
+$config['MAX_EMAIL_LEN'] = 41;
 
 $config['RELOAD_COMMAND'] = 'sudo -n /etc/init.d/rc.piler reload';
 $config['PILERIMPORT_IMAP_COMMAND'] = '/usr/local/bin/pilerimport -d /var/piler/imap -q -r';
 $config['CPU_USAGE_COMMAND'] = "LC_ALL=C mpstat | tail -1 | rev | awk '{ print $1 }' | rev";
-$config['PILER_BINARY'] = "/usr/local/sbin/piler";
+$config['PILER_BINARY'] = "SBINDIR/piler";
 
 $config['LDAP_IMPORT_CONFIG_FILE'] = '/usr/local/etc/ldap-import.cfg';
 
@@ -234,7 +280,10 @@ $config['SIZE_Y'] = 250;
 $config['DATE_TEMPLATE'] = 'Y.m.d';
 $config['DATE_FORMAT'] = 'YYYY-MM-DD';
 $config['JQUERY_DATE_FORMAT'] = 'yy-mm-dd';
+$config['DECIMAL_SEPARATOR'] = "."; // See https://www.php.net/manual/en/function.number-format
+$config['THOUSANDS_SEPARATOR'] = ","; // for the format options
 
+$config['ENABLE_GB2312_FIX'] = 1;
 
 $config['FROM_LENGTH_TO_SHOW'] = 28;
 
@@ -248,15 +297,26 @@ $config['MIN_PASSWORD_LENGTH'] = 6;
 
 $config['MIN_PREFIX_LEN'] = 5;
 
-$config['CGI_INPUT_FIELD_WIDTH'] = 50;
-$config['CGI_INPUT_FIELD_HEIGHT'] = 7;
-
 $config['ADMIN_CAN_POWER_SEARCH'] = 0;
 $config['FOUR_EYES_LOGIN_FOR_AUDITOR'] = 0;
 
 $config['MEMCACHED_PREFIX'] = '_piler:';
 $config['MEMCACHED_TTL'] = 900;
 
+$config['JS_CODE'] = '
+    <script src="//code.jquery.com/jquery-2.2.4.min.js" integrity="sha256-BbhdlvQf/xTY9gja0Dq3HiwQF8LaCRTXxZKRutelT44=" crossorigin="anonymous"></script>
+    <script src="//code.jquery.com/ui/1.12.1/jquery-ui.min.js" integrity="sha256-VazP97ZCwtekAsvgPBSUwPFKdrwD3unUfSGVYrahUqU=" crossorigin="anonymous"></script>
+    <script src="//stackpath.bootstrapcdn.com/bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha384-aJ21OjlMXNL5UyIl/XNwTMqvzeRMZH2w8c5cRVpzpU8Y5bApTppSuUkhZXN0VxHd" crossorigin="anonymous"></script>
+
+    <script type="text/javascript" src="/view/javascript/rc-splitter.js"></script>
+    <script type="text/javascript" src="/view/javascript/piler.js"></script>
+';
+
+$config['CSS_CODE'] = '
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css" integrity="sha384-HSMxcRTRxnN+Bdg0JdbxYKrThecOKuH5zCYotlSAcp1+c8xmyTe9GYg1l9a69psu" crossorigin="anonymous">
+   <link rel="stylesheet" href="/view/theme/default/assets/css/piler.css" />
+';
+
 $SUPPRESS_RECIPIENTS = array();
 
 
@@ -271,6 +331,8 @@ $config['DELIMITER'] = "\t";
 
 $config['TRACKING_CODE'] = '';
 
+$mailattrs = ["mail", "mailalternateaddress", "proxyaddresses", "zimbraMailForwardingAddress", "member", "memberOfGroup", "othermailbox", "mailprimaryaddress", "mailalternativeaddress"];
+
 $langs = array(
                 'cz',
                 'de',
@@ -278,16 +340,17 @@ $langs = array(
                 'es',
                 'fr',
                 'hu',
+                'it',
                 'pl',
                 'pt',
                 'ru',
-                'tr'
+                'tr',
+                'tw'
                );
 
 
 $themes = array(
                 'default',
-                'mobile'
                );
 
 
@@ -311,7 +374,9 @@ define('NOW', time());
  * normally you don't have to change anything below
  */
 
-require_once 'config-site.php';
+if(file_exists('SYSCONFDIR/piler/config-site.php')) { require_once 'SYSCONFDIR/piler/config-site.php'; }
+
+ini_set('session.cookie_lifetime', $config['SESSION_EXPIRY']);
 
 require($config['DIR_BASE'] . "/system/registry.php");
 require($config['DIR_BASE'] . "/system/request.php");
@@ -324,8 +389,6 @@ if($session->get("theme") && preg_match("/^([a-zA-Z0-9\-\_]+)$/", $session->get(
 
 include("system/helper/detectmobilebrowser.php");
 
-if(MOBILE_DEVICE == 1) { $config['THEME'] = 'mobile'; }
-
 // make sure auditors are restricted in a saas environment
 if($config['ENABLE_SAAS'] == 1) { $config['RESTRICTED_AUDITOR'] = 1; }
 if($session->get("username") == 'auditor@local' || isset($_SERVER['argv'][2]) ) { $config['RESTRICTED_AUDITOR'] = 0; }
@@ -338,7 +401,7 @@ foreach ($config as $k => $v) {
 
 
 define('TABLE_USER', 'user');
-define('TABLE_GROUP', 'group');
+define('TABLE_GROUP', 'usergroup');
 define('TABLE_GROUP_USER', 'group_user');
 define('TABLE_GROUP_EMAIL', 'group_email');
 define('TABLE_FOLDER', 'folder');
@@ -375,9 +438,10 @@ define('TABLE_AUTOSEARCH', 'autosearch');
 define('TABLE_LEGAL_HOLD', 'legal_hold');
 define('TABLE_TIMESTAMP', 'timestamp');
 define('TABLE_PRIVATE', 'private');
+define('TABLE_DELETED', 'deleted');
 define('VIEW_MESSAGES', 'v_messages');
 
-define('EOL', "\n");
+define('EOL', "\r\n");
 
 define('DIR_SYSTEM', DIR_BASE . 'system/');
 define('DIR_MODEL', DIR_BASE . 'model/');
@@ -410,6 +474,8 @@ define('QSHAPE_DEFERRED_SENDER', DIR_STAT . '/deferred-sender');
 define('CPUSTAT', DIR_STAT . '/cpu.stat');
 define('AD_SYNC_STAT', DIR_STAT . '/adsync.stat');
 define('ARCHIVE_SIZE', DIR_STAT . '/archive.size');
+define('SPHINX_CURRENT_MAIN_INDEX_SIZE', DIR_STAT . '/current_main_index_size');
+define('SPHINX_TOTAL_INDEX_SIZE', DIR_STAT . '/total_index_size');
 define('LOCK_FILE', DIR_LOG . 'lock');
 
 define('SEARCH_HELPER_URL', SITE_URL . 'search-helper.php');
@@ -425,6 +491,8 @@ define('HEALTH_WORKER_URL', SITE_URL . 'index.php?route=health/worker');
 
 define('LDAP_TYPE_GENERIC', 'generic_ldap');
 
+define('ATTACHMENT_DUMP_CHECKPOINT', 'attachment_dump_checkpoint');
+
 define('ACTION_ALL', 0);
 define('ACTION_UNKNOWN', 1);
 define('ACTION_LOGIN', 2);
@@ -445,6 +513,8 @@ define('ACTION_UNAUTHORIZED_DOWNLOAD_ATTACHMENT', 16);
 define('ACTION_VIEW_JOURNAL', 17);
 define('ACTION_NOT_SPAM', 18);
 define('ACTION_MARK_AS_PRIVATE', 19);
+define('ACTION_MARK_MESSAGE_FOR_REMOVAL', 20);
+define('ACTION_REJECT_REMOVAL', 21);
 
 $actions = array(
                   'unknown' => 1,
@@ -459,7 +529,9 @@ $actions = array(
                   'save_search' => 11,
                   'download_attachment' => 15,
                   'journal' => 17,
-                  'private' => 19
+                  'private' => 19,
+                  'marked_for_removal', 20,
+                  'reject_removal', 21
                 );
 
 
@@ -477,5 +549,14 @@ if(!isset($health_smtp_servers)) {
    $health_smtp_servers = array( array(PILER_HOST, PILER_PORT, "piler"), array(SMARTHOST, SMARTHOST_PORT, "smarthost") );
 }
 
-
+if(SPHINX_STRICT_SCHEMA) {
-?>
+   define('FROM_TOKEN', '@sender');
+   define('FROMDOMAIN_TOKEN', '@senderdomain');
+   define('TO_TOKEN', '@rcpt');
+   define('TODOMAIN_TOKEN', '@rcptdomain');
+} else {
+   define('FROM_TOKEN', '@from');
+   define('FROMDOMAIN_TOKEN', '@fromdomain');
+   define('TO_TOKEN', '@to');
+   define('TODOMAIN_TOKEN', '@todomain');
+}

configure.in

@@ -85,7 +85,7 @@ fi
 echo "\"Configure command: ./configure $PARAMS\"" >> $CONFIGURE_PARAMS_FILE
 
 
-SUBDIRS="src etc util init.d unit_tests"
+SUBDIRS="src etc util init.d systemd unit_tests webui"
 
 
 dnl static build
@@ -115,18 +115,13 @@ AC_ARG_ENABLE(memcached,
         [  --enable-memcached      build memcached support], want_memcached=$enableval, want_memcached="no")
 
 
-AC_ARG_ENABLE(tcpwrappers,
+dnl tcpwrappers library
-	[  --enable-tcpwrappers    build tcpwrappers support], want_tcpwrappers=$enableval, want_tcpwrappers="no")
 
+AC_CHECK_HEADERS(tcpd.h, have_tcpwrappers=yes, echo "tcpd.h is not found")
+AC_CHECK_LIB([wrap],[hosts_access],[AC_CHECK_LIB(wrap, hosts_access, have_tcpwrappers=yes, echo "libwrap is not found"; have_tcpwrappers=no)],[],[])ac_cv_lib_wrap=ac_cv_lib_wrap_main
 
-if test "$want_tcpwrappers" = "yes"; then
+if test "$have_tcpwrappers" = "no"; then
-   AC_CHECK_HEADERS(tcpd.h, have_tcpwrappers=yes, have_tcpwrappers=no)
+   echo "can't find either tcpd.h or libwrap. Tcpwrappers support is disabled";
-   AC_CHECK_LIB([wrap],[hosts_access],[AC_CHECK_LIB(wrap, hosts_access, have_tcpwrappers=yes, echo "libwrap is not found"; have_tcpwrappers=no)],[],[])ac_cv_lib_wrap=ac_cv_lib_wrap_main
-
-   if test "$have_tcpwrappers" = "no"; then
-      echo "can't find either tcpd.h or libwrap";
-      exit 1;
-   fi
 fi
 
 
@@ -263,6 +258,10 @@ if test z`which timeout 2>/dev/null` != "z"; then
 fi
 
 
+read -r version < VERSION
+AC_DEFINE_UNQUOTED(VERSION, "$version")
+
+
 dnl user running piler
 
 AC_ARG_WITH(piler-user,
@@ -356,9 +355,6 @@ AC_SUBST(DATAROOTDIR)
 AC_DEFINE_UNQUOTED(DATAROOTDIR,"$dataroot_dir",[where to look for the share data files])
 
 
-AC_DEFINE_UNQUOTED(VIRUS_TEMPLATE, "$my_prefix/share/clapf/template.virus", [where the virus template is])
-AC_DEFINE_UNQUOTED(ZOMBIE_NET_REGEX, "$my_prefix/share/clapf/zombienets.regex", [where the virus template is])
-
 
 if test "$have_mysql" = "no" && test "$have_psql" = "no"; then
    echo
@@ -387,11 +383,6 @@ if test "$os" = "SunOS"; then
    if test -x /usr/xpg4/bin/id; then id_bin="/usr/xpg4/bin/id"; fi
 fi
 
-if test `echo $os | grep -c CYGWIN` -eq 1; then
-   defs="$defs -DCYGWIN"
-   AC_DEFINE_UNQUOTED(ZOMBIE_NET_REGEX, "zombienets.regex", [where the virus template is])
-fi
-
 dnl whether we have antivirus support
 
 if test "$have_clamd" = "yes" ; then
@@ -438,19 +429,8 @@ if test "$have_zip" = "yes"; then
    echo "zip library: yes"
    AC_DEFINE_UNQUOTED(HAVE_ZIP, 1, [libzip support])
    antispam_libs="$antispam_libs -lzip"
-fi
-
-if test "$have_tcpwrappers" = "yes"; then
-   echo "tcpwrappers support: yes"
-   AC_DEFINE_UNQUOTED(HAVE_LIBWRAP, 1, [tcpwrappers support])
-
-   if test "$os" = "FreeBSD"; then
-      antispam_libs="$antispam_libs -lwrap"
-   else
-      antispam_libs="$antispam_libs -lwrap -lnsl"
-   fi
 else
-   echo "tcpwrappers support: no"
+   echo "zip library: no"
 fi
 
 
@@ -542,16 +522,27 @@ if test $? -eq 1; then echo "the user \"$RUNNING_USER\" does not exists, please
 
 echo; echo
 
-CFLAGS="$static -O2 -Wall -g"
+gcc_version="$(gcc -dumpversion)"
-LIBS="$antispam_libs $sunos_libs "
+extra_cflags=""
-OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o mydomains.o $objs"
+if [[ "${gcc_version:0:1}" -gt 6 ]]; then
+   extra_cflags="-Wimplicit-fallthrough=2"
+fi
 
-AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile unit_tests/Makefile contrib/imap/Makefile])
+CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra $extra_cflags -Wuninitialized -Wimplicit-fallthrough=2 -Wno-format-truncation -g"
+LIBS="$antispam_libs $sunos_libs "
+OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o screen.o $objs"
+
+AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile])
 AC_OUTPUT
 
+echo
+echo
+echo "IMPORTANT! If you upgrade, be sure to read https://www.mailpiler.org/upgrade-instructions/"
+echo
+echo
 
 echo
-echo
+echo "Did you know that piler has an enterprise edition as well?"
-echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/en/upgrade.html"
+echo "Check out what it can do for you at https://mailpiler.com/#features"
 echo
 echo

contrib/export-attachments/export-attachments.php

@@ -0,0 +1,103 @@
+#!/usr/bin/php
+<?php
+
+define('EXPORT_LOCK_FILE', '/var/piler/tmp/export-attachments.lock');
+
+$webuidir = "";
+
+$outdir = '';
+
+
+$opts = 'hw:d:';
+$lopts = [
+   'webui:',
+   'dir:',
+   'help'
+];
+
+
+if($options = getopt($opts, $lopts)) {
+   if(isset($options['webui'])) { $webuidir = $options['webui']; }
+   if(isset($options['dir'])) { $outdir = $options['dir']; }
+   if(isset($options['help'])) { usage(); }
+}
+
+
+if($webuidir == '' || $outdir == '') { usage(); }
+
+
+
+require_once("$webuidir/config.php");
+
+require(DIR_SYSTEM . "/startup.php");
+
+$request = new Request();
+Registry::set("request", $request);
+
+
+$start = NULL;
+
+$loader = new Loader();
+Registry::set('load', $loader);
+
+$loader->load->model('domain/domain');
+$loader->load->model('search/search');
+$loader->load->model('search/message');
+$loader->load->model('message/attachment');
+
+
+$db = new DB(DB_DRIVER, DB_HOSTNAME, DB_USERNAME, DB_PASSWORD, DB_DATABASE, DB_PREFIX);
+Registry::set('db', $db);
+
+Registry::set('auditor_user', 1);
+
+openlog("export-attachments", LOG_PID, LOG_MAIL);
+
+$fp = fopen(EXPORT_LOCK_FILE, "w");
+if(!flock($fp, LOCK_EX)) {
+   syslog(LOG_INFO, "WARN: couldn't get a lock on " . EXPORT_LOCK_FILE);
+   exit;
+}
+
+
+$domain = new ModelDomainDomain();
+$attachment = new ModelMessageAttachment();
+$message = new ModelSearchMessage();
+
+$domains = $domain->get_mapped_domains();
+
+$last_id = $attachment->get_last_attachment_id();
+$start_id = $attachment->get_checkpoint();
+
+syslog(LOG_INFO, "start: $start_id, limit: $last_id");
+
+
+for($i=$start_id; $i<$last_id; $i++) {
+   $a = $attachment->get_attachment_by_id($i);
+   $m = $message->get_message_addresses_by_piler_id($a['piler_id'], $domains);
+
+   $attachment->dump_attachment($outdir, "out", $m['sender'], $i, $a);
+
+   foreach($m['rcpt'] as $rcpt) {
+      $attachment->dump_attachment($outdir, "in", $rcpt, $i, $a);
+   }
+
+   if($i % 100 == 0) { $attachment->update_checkpoint($i); }
+}
+
+$attachment->update_checkpoint($i);
+
+// Release lock
+flock($fp, LOCK_UN);
+fclose($fp);
+
+
+
+function usage() {
+   print "\nUsage: " . __FILE__ . "\n\n";
+   print "\t--webui <path to webui directory>\n";
+   print "\t--dir <basedir to write attachments>\n";
+   print "\t--help\n\n";
+
+   exit;
+}

contrib/installer/focal.sh

@@ -0,0 +1,257 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+
+MY_IP="1.2.3.4"
+PILER_HOSTNAME="${PILER_HOSTNAME:-archive.example.com}"
+MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
+MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
+SERVER_ID="${SERVER_ID:-0}"
+USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
+SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
+PHP_FPM_SOCKET="/var/run/php/php7.4-fpm.sock"
+
+MYSQL_HOSTNAME="localhost"
+MYSQL_DATABASE="piler"
+MYSQL_USERNAME="piler"
+
+SPHINX_TARGZ="sphinx-3.3.1-bin.tar.gz"
+DOWNLOAD_URL="https://download.mailpiler.com"
+PILER_DEB="piler_1.3.12-focal-eb2b22b_amd64.deb"
+PILER_USER="piler"
+CONFIG_SITE_PHP="/etc/piler/config-site.php"
+
+export DEBIAN_FRONTEND=noninteractive
+
+install_prerequisites() {
+   apt-get update
+   apt-get -y --no-install-recommends install \
+      wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \
+      php7.4-gd php7.4-curl php7.4-xml ca-certificates zip catdoc unrtf poppler-utils nginx tnef libzip5 \
+      libtre5 libwrap0 cron libmariadb-dev python3 python3-mysqldb libmariadb-dev mariadb-client-core-10.3 \
+      mariadb-server-10.3
+
+   wget -q -O "/tmp/${SPHINX_TARGZ}" "${DOWNLOAD_URL}/generic-local/${SPHINX_TARGZ}"
+   tar -C / -zxvf "/tmp/${SPHINX_TARGZ}"
+   wget -O /usr/local/bin/traefik "${DOWNLOAD_URL}/generic-local/traefik"
+   chmod +x /usr/local/bin/traefik
+   setcap cap_net_bind_service+ep /usr/local/bin/traefik
+}
+
+
+fix_mysql_settings() {
+   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
+[mysqld]
+
+innodb_buffer_pool_size=512M
+innodb_flush_log_at_trx_commit=1
+innodb_log_buffer_size=64M
+innodb_log_file_size=64M
+innodb_read_io_threads=4
+innodb_write_io_threads=4
+innodb_log_files_in_group=2
+
+innodb_file_per_table
+PILER_CNF
+}
+
+start_mysql() {
+   fix_mysql_settings
+
+   service mysql restart
+   mysqladmin -u root password "${MYSQL_ROOT_PASSWORD}"
+}
+
+
+install_piler() {
+   wget "${DOWNLOAD_URL}/piler/${PILER_DEB}"
+   dpkg -i "$PILER_DEB"
+   rm -f "$PILER_DEB"
+
+   crontab -u "$PILER_USER" /usr/share/piler/piler.cron
+
+   rm -f "$PILER_DEB" /etc/nginx/sites-enabled/default
+   ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf
+
+   touch /var/piler/.bash_history
+   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
+}
+
+
+create_my_cnf() {
+   local user=$1
+   local password=$2
+   local my_cnf=$3
+
+   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
+   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
+   chown $PILER_USER:$PILER_USER "$my_cnf"
+   chmod 600 "$my_cnf"
+}
+
+
+fix_config_site_php() {
+   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"
+
+   {
+      echo "\$config['SERVER_ID'] = $SERVER_ID;"
+      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
+      echo "\$config['SPHINX_VERSION'] = 331;"
+   } >> "$CONFIG_SITE_PHP"
+
+   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
+      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
+   fi
+
+   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
+}
+
+
+init_db() {
+   sed -e "s%MYSQL_HOSTNAME%$MYSQL_HOSTNAME%g" -e "s%MYSQL_DATABASE%$MYSQL_DATABASE%g" -e "s%MYSQL_USERNAME%$MYSQL_USERNAME%g" -e "s%MYSQL_PASSWORD%$MYSQL_PILER_PASSWORD%g" \
+      /usr/share/piler/db-mysql-root.sql.in | mysql --defaults-file=/etc/piler/.my.cnf-root -h $MYSQL_HOSTNAME
+   mysql --defaults-file=/etc/piler/.my.cnf -h $MYSQL_HOSTNAME $MYSQL_DATABASE < /usr/share/piler/db-mysql.sql
+}
+
+
+add_systemd_services() {
+   pushd /etc/systemd/system
+
+   ln -sf /usr/libexec/piler/piler.service .
+   ln -sf /usr/libexec/piler/piler-smtp.service .
+   ln -sf /usr/libexec/piler/pilersearch.service .
+
+   popd
+
+   systemctl daemon-reload
+
+   systemctl enable piler
+   systemctl enable piler-smtp
+   systemctl enable pilersearch
+   systemctl enable traefik
+}
+
+
+fix_configs() {
+   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
+      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
+      nginx -t
+      nginx -s reload
+
+      sed -i 's/server {/server {\n\tlisten 127.0.0.1:80;/' /etc/piler/piler-nginx.conf
+      systemctl stop nginx
+      sleep 5
+      systemctl start nginx
+   fi
+
+   if [[ ! -f /etc/piler/piler.conf ]]; then
+      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.example.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > /etc/piler/piler.conf
+      chmod 600 /etc/piler/piler.conf
+      chown $PILER_USER:$PILER_USER /etc/piler/piler.conf
+   fi
+
+   sed -i 's/tls_enable=0/tls_enable=1/g' /etc/piler/piler.conf
+
+   sed -i -e "s/MYSQL_HOSTNAME/${MYSQL_HOSTNAME}/g" \
+          -e "s/MYSQL_DATABASE/${MYSQL_DATABASE}/g" \
+          -e "s/MYSQL_USERNAME/${MYSQL_USERNAME}/g" \
+          -e "s/MYSQL_PASSWORD/${MYSQL_PILER_PASSWORD}/g" \
+          /etc/piler/sphinx.conf
+}
+
+setup_traefik() {
+   wget -O /etc/systemd/system/traefik.service "${DOWNLOAD_URL}/generic-local/traefik.service"
+
+   mkdir -p /usr/local/etc/traefik
+   touch /usr/local/etc/traefik/acme.json
+   chmod 600 /usr/local/etc/traefik/acme.json
+   chown www-data:www-data /usr/local/etc/traefik/acme.json
+
+   cat > /usr/local/etc/traefik/traefik.yaml << TRAEFIK
+log:
+  level: INFO
+
+entryPoints:
+  web:
+    address: "$MY_IP:80"
+    http:
+      redirections:
+        entryPoint:
+          to: websecure
+          scheme: https
+          permanent: true
+  websecure:
+    address: "$MY_IP:443"
+
+providers:
+  file:
+    filename: "/usr/local/etc/traefik/traefik.yaml"
+
+certificatesResolvers:
+  le:
+    acme:
+      storage: "/usr/local/etc/traefik/acme.json"
+      email: admin@$PILER_HOSTNAME
+      httpChallenge:
+        entryPoint: web
+
+tls:
+  options:
+    default:
+      minVersion: VersionTLS13
+
+http:
+  middlewares:
+    piler_headers:
+      headers:
+        customResponseHeaders:
+          Server: ""
+          Strict-Transport-Security: "max-age=31536000"
+          X-Content-Type-Optionsi: "nosniff"
+          Referrer-Policy: "same-origin"
+  routers:
+    master:
+      rule: "Host(\`$PILER_HOSTNAME\`)"
+      service: www
+      middlewares:
+        - "piler_headers"
+      tls:
+        certResolver: le
+  services:
+    www:
+      loadBalancer:
+        servers:
+        - url: "http://127.0.0.1:80/"
+TRAEFIK
+}
+
+
+install_prerequisites
+
+start_mysql
+
+install_piler
+
+create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
+create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf
+
+fix_config_site_php
+
+setup_traefik
+
+add_systemd_services
+
+fix_configs
+init_db
+
+su -c "indexer --all -c /etc/piler/sphinx.conf" $PILER_USER
+
+[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler
+
+systemctl start pilersearch
+systemctl start piler
+systemctl start piler-smtp
+systemctl start traefik

contrib/installer/jammy.sh

@@ -0,0 +1,258 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+
+MY_IP="1.2.3.4"
+PILER_HOSTNAME="${PILER_HOSTNAME:-archive.example.com}"
+MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
+MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
+SERVER_ID="${SERVER_ID:-0}"
+USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
+SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
+PHP_FPM_SOCKET="/var/run/php/php8.1-fpm.sock"
+
+MYSQL_HOSTNAME="localhost"
+MYSQL_DATABASE="piler"
+MYSQL_USERNAME="piler"
+
+DOWNLOAD_URL="https://download.mailpiler.com"
+PILER_DEB="piler_1.4.4-jammy-0573c6da_amd64.deb"
+PILER_USER="piler"
+PILER_CONF="/etc/piler/piler.conf"
+CONFIG_SITE_PHP="/etc/piler/config-site.php"
+SEARCHCFG="/etc/piler/manticore.conf"
+
+TRAEFIK_VERSION="v2.11.0"
+ARCH="amd64"
+
+export DEBIAN_FRONTEND=noninteractive
+
+install_prerequisites() {
+   apt-get update
+   apt-get -y --no-install-recommends install \
+      wget rsyslog openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql \
+      php8.1-fpm php8.1-zip php8.1-ldap php8.1-gd php8.1-curl php8.1-xml \
+      php8.1-mbstring ca-certificates zip catdoc unrtf poppler-utils \
+      nginx tnef libzip4 libtre5 libwrap0 cron libmariadb-dev python3 \
+      python3-mysqldb libmariadb-dev mariadb-client-core-10.6 \
+      mariadb-server-10.6
+
+   wget "https://github.com/traefik/traefik/releases/download/${TRAEFIK_VERSION}/traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz"
+   tar zxvf "traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz" traefik
+   chown root:root traefik
+   chmod 755 traefik
+   mv traefik /usr/local/bin
+   setcap cap_net_bind_service+ep /usr/local/bin/traefik
+
+   wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
+   dpkg -i manticore-repo.noarch.deb && \
+   apt-get update && \
+   apt-get install -y manticore manticore-columnar-lib manticore-extra && \
+   rm -f manticore-repo.noarch.deb
+   systemctl stop manticore
+}
+
+
+fix_mysql_settings() {
+   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
+[mysqld]
+
+innodb_buffer_pool_size=512M
+innodb_flush_log_at_trx_commit=1
+innodb_log_buffer_size=64M
+innodb_log_file_size=64M
+innodb_read_io_threads=4
+innodb_write_io_threads=4
+innodb_log_files_in_group=2
+
+innodb_file_per_table
+PILER_CNF
+}
+
+start_mysql() {
+   fix_mysql_settings
+
+   service mysql restart
+   mysqladmin -u root password "${MYSQL_ROOT_PASSWORD}"
+}
+
+
+install_piler() {
+   wget "https://bitbucket.org/jsuto/piler/downloads/${PILER_DEB}"
+   dpkg -i "$PILER_DEB"
+   rm -f "$PILER_DEB"
+
+   sed -i 's/.*indexer.*/###/g' /usr/share/piler/piler.cron
+   crontab -u "$PILER_USER" /usr/share/piler/piler.cron
+
+   rm -f "$PILER_DEB" /etc/nginx/sites-enabled/default
+   ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf
+
+   touch /var/piler/.bash_history
+   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
+}
+
+
+create_my_cnf() {
+   local user=$1
+   local password=$2
+   local my_cnf=$3
+
+   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
+   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
+   chown $PILER_USER:$PILER_USER "$my_cnf"
+   chmod 600 "$my_cnf"
+}
+
+
+fix_config_site_php() {
+   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"
+
+   {
+      echo "\$config['SERVER_ID'] = $SERVER_ID;"
+      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
+   } >> "$CONFIG_SITE_PHP"
+
+   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
+      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
+   fi
+
+   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
+   echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
+}
+
+
+init_db() {
+   sed -e "s%MYSQL_HOSTNAME%$MYSQL_HOSTNAME%g" -e "s%MYSQL_DATABASE%$MYSQL_DATABASE%g" -e "s%MYSQL_USERNAME%$MYSQL_USERNAME%g" -e "s%MYSQL_PASSWORD%$MYSQL_PILER_PASSWORD%g" \
+      /usr/share/piler/db-mysql-root.sql.in | mysql --defaults-file=/etc/piler/.my.cnf-root -h $MYSQL_HOSTNAME
+   mysql --defaults-file=/etc/piler/.my.cnf -h $MYSQL_HOSTNAME $MYSQL_DATABASE < /usr/share/piler/db-mysql.sql
+}
+
+
+add_systemd_services() {
+   pushd /etc/systemd/system
+
+   ln -sf /usr/libexec/piler/piler.service .
+   ln -sf /usr/libexec/piler/piler-smtp.service .
+   ln -sf /usr/libexec/piler/pilersearch.service .
+
+   popd
+
+   systemctl daemon-reload
+
+   systemctl enable piler
+   systemctl enable piler-smtp
+   systemctl enable pilersearch
+   systemctl enable traefik
+}
+
+
+fix_configs() {
+   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
+      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
+      nginx -t
+      nginx -s reload
+
+      sed -i 's/server {/server {\n\tlisten 127.0.0.1:80;/' /etc/piler/piler-nginx.conf
+      systemctl stop nginx
+      sleep 5
+      systemctl start nginx
+   fi
+
+   if [[ ! -f "$PILER_CONF" ]]; then
+      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.example.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > "$PILER_CONF"
+      chmod 600 "$PILER_CONF"
+      chown $PILER_USER:$PILER_USER "$PILER_CONF"
+   fi
+
+   sed -i 's/tls_enable=0/tls_enable=1/g' "$PILER_CONF"
+   sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
+
+   sed -i "s/define('RT.*/define('RT', 1);/" "$SEARCHCFG"
+}
+
+setup_traefik() {
+   wget -O /etc/systemd/system/traefik.service "${DOWNLOAD_URL}/generic-local/traefik.service"
+
+   mkdir -p /usr/local/etc/traefik
+   touch /usr/local/etc/traefik/acme.json
+   chmod 600 /usr/local/etc/traefik/acme.json
+   chown www-data:www-data /usr/local/etc/traefik/acme.json
+
+   cat > /usr/local/etc/traefik/traefik.yaml << TRAEFIK
+log:
+  level: INFO
+
+entryPoints:
+  websecure:
+    address: "$MY_IP:443"
+
+providers:
+  file:
+    filename: "/usr/local/etc/traefik/traefik.yaml"
+
+certificatesResolvers:
+  le:
+    acme:
+      storage: "/usr/local/etc/traefik/acme.json"
+      email: admin@$PILER_HOSTNAME
+      tlsChallenge: {}
+
+tls:
+  options:
+    default:
+      minVersion: VersionTLS13
+
+http:
+  middlewares:
+    piler_headers:
+      headers:
+        customResponseHeaders:
+          Server: ""
+          Strict-Transport-Security: "max-age=31536000"
+          X-Content-Type-Optionsi: "nosniff"
+          Referrer-Policy: "same-origin"
+  routers:
+    master:
+      rule: "Host(\`$PILER_HOSTNAME\`)"
+      service: www
+      middlewares:
+        - "piler_headers"
+      tls:
+        certResolver: le
+  services:
+    www:
+      loadBalancer:
+        servers:
+        - url: "http://127.0.0.1:80/"
+TRAEFIK
+}
+
+
+install_prerequisites
+
+start_mysql
+
+install_piler
+
+create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
+create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf
+
+fix_config_site_php
+
+setup_traefik
+
+add_systemd_services
+
+fix_configs
+init_db
+
+[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler
+
+systemctl start pilersearch
+systemctl start piler
+systemctl start piler-smtp
+systemctl start traefik

contrib/proxmox-lxc/README

@@ -0,0 +1,9 @@
+You nedd proxmox with lxc
+
+Fill out conf file first and provide public key
+
+It automatically
+- downloads the latest debian template
+- provides a container
+- installs piler until the post installation step
+- gives a self signed certificate

contrib/proxmox-lxc/create_lxc.sh

@@ -0,0 +1,195 @@
+#!/bin/bash
+
+# This script wil create and fire up a standard debian buster lxc container on your proxmox pve.
+# The Script will look for the next free lxc number and take the next free and use it. So take
+# care that behind your last number is place for it. 
+
+#### SOME VARIABLES TO ADJUST ####
+
+# Storage with templates
+LXC_TMP="local"
+
+# Size and pool of rootfs / in GB
+SIZ_ROT="100"
+S_ROT_P="local-zfs"
+
+# Size and pool of Filestorage in GB will mounted to /share
+SIZ_FIL="100"
+S_FIL_P="local-zfs"
+
+#Weather or not (1 and 0) the container will createt as unpriviliged LXC
+LXC_UNP="1"
+
+# Size of the RAM assigned to the LXC
+LXC_MEM="1024"
+
+# Size of the SWAP assigned to the LXC
+LXC_SWA="1024"
+
+# The hostname (eq. zamba1 or mailpiler1)
+LXC_HOST="zamba"
+
+# The domainname (searchdomain /etc/resolf.conf & hosts)
+LXC_SDN="zmb.local"
+
+# IP-address and subnet
+LXC_IP="10.10.80.20/24"
+
+# Gateway
+LXC_GW="10.10.80.10"
+
+# DNS-server and here shoud be your AD-DC
+LXC_DNS="10.10.80.10"
+
+# Networkbridge for this machine
+LXC_BRD="vmbr80"
+
+# root password - take care to delete from this file
+LXC_PWD="MYPASSWD"
+
+LXC_KEY="ssh-rsa xxxxxxxx"
+
+############### Zamba-Server-Section ###############
+
+# Domain Entries to samba/smb.conf. Will be also uses for samba domain-provisioning when zmb-pdc will choosen.
+ZMB_REA="ZMB.LOCAL"
+ZMB_DOM="ZMB"
+
+# THE Domain-Admin and passwd for zamba-install
+ZMB_ADA="Administrator"
+ZMB_APW="MYPASSWORD"
+
+############### Mailpiler-Section ###############
+
+# The FQDN vor the Hostname. This must be exactly the same like the LXC_HOST / LXC_SDN at section above.
+PILER_DOM="piler.zmb.rocks"
+SMARTHOST="10.10.80.20"
+PILER_VER="1.3.10"
+SPHINX_VER="3.3.1"
+PHP_VER="7.4"
+
+############### Matrix-Section ###############
+
+# The FQDN vor the Hostname. This should be the same like the LXC_HOST / LXC_SDN at section above.
+MRX_DOM="matrix.zmb.rocks"
+ELE_DOM="element.zmb.rocks"
+ELE_VER="v1.7.21"
+JIT_DOM="meet.zmb.rocks"
+
+#################################
+
+# CHeck is the newest template available, else download it.
+
+DEB_LOC=$(pveam list $LXC_TMP | grep debian-10-standard | cut -d'_' -f2)
+
+DEB_REP=$(pveam available --section system | grep debian-10-standard | cut -d'_' -f2)
+
+if [[ $DEB_LOC == $DEB_REP ]];
+then
+  echo "Newest Version of Debian 10 Standard $DEP_REP exists.";
+else
+  echo "Will now download newest Debian 10 Standard $DEP_REP.";
+  pveam download $LXC_TMP debian-10-standard_$DEB_REP\_amd64.tar.gz
+fi
+
+# Get next free LXC-number
+LXC_LST=$( lxc-ls | egrep -o '.{1,5}$' )
+LXC_CHK=$((LXC_LST+1));
+
+if  [ $LXC_CHK -lt 100 ] || [ -f /etc/pve/qemu-server/$LXC_CHK.conf ]; then
+  LXC_NBR=$(pvesh get /cluster/nextid);
+else
+  LXC_NBR=$LXC_CHK;
+fi
+
+echo "Will now create LXC Container $LXC_NBR!";
+
+# Create the container
+pct create $LXC_NBR -unprivileged $LXC_UNP $LXC_TMP:vztmpl/debian-10-standard_$DEB_REP\_amd64.tar.gz -rootfs $S_ROT_P:$SIZ_ROT;
+sleep 2;
+
+pct set $LXC_NBR -memory $LXC_MEM -swap $LXC_SWA -hostname $LXC_HOST \-nameserver $LXC_DNS -searchdomain $LXC_SDN -onboot 1 -timezone Europe/Berlin -net0 name=eth0,bridge=$LXC_BRD,firewall=1,gw=$LXC_GW,ip=$LXC_IP,type=veth;
+sleep 2;
+
+PS3="Select the Server-Function: "
+
+select opt in just_lxc zmb-standalone zmb-member zmb-pdc mailpiler matrix quit; do
+  case $opt in
+    just_lxc)
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      echo "Should be ready!"
+      break
+      ;;
+    zmb-standalone)
+      break
+      ;;
+    zmb-member)
+      echo "Make some additions to LXC for AD-Member-Server!"
+      pct set $LXC_NBR -mp0 $S_FIL_P:$SIZ_FIL,mp=/tank
+      sleep 2;
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      cp /root/zmb_mem.orig /root/zmb_mem.sh
+      sed -i "s|#ZMB_VAR|#ZMB_VAR\nZMB_REA='$ZMB_REA'\nZMB_DOM='$ZMB_DOM'\nZMB_ADA='$ZMB_ADA'\nZMB_APW='$ZMB_APW'|" /root/zmb_mem.sh
+      pct push $LXC_NBR /root/zmb_mem.sh /root/zmb_mem.sh
+      echo "Install zamba as AD-Member-Server!"
+      lxc-attach -n$LXC_NBR bash /root/zmb_mem.sh
+      break
+      ;;
+    zmb-pdc)
+      break
+      ;;
+    mailpiler)
+      echo "Make some additions to LXC for Mailpiler!"
+      pct set $LXC_NBR -features nesting=1
+      sleep 2;
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      cp /root/mailpiler.orig /root/mailpiler.sh
+      sed -i "s|#PILER_VAR|#PILER_VAR\nPILER_DOM='$PILER_DOM'\nSMARTHOST='$SMARTHOST'\nPILER_VER='$PILER_VER'\nSPHINX_VER='$SPHINX_VER'\nPHP_VER='$PHP_VER'|" /root/mailpiler.sh
+      pct push $LXC_NBR /root/mailpiler.sh /root/mailpiler.sh
+      echo "Install Mailpiler mailarchiv!"
+      lxc-attach -n$LXC_NBR bash mailpiler.sh
+      break
+      ;;
+    matrix)
+      echo "Make some additions to LXC for Matrix!"
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      cp /root/matrix.orig /root/matrix.sh
+      sed -i "s|#MATRIX_VAR|#Matrix_VAR\nMRX_DOM='$MRX_DOM'\nELE_DOM='$ELE_DOM'\nELE_VER='$ELE_VER'\nJIT_DOM='$JIT_DOM'|" /root/matrix.sh
+      pct push $LXC_NBR /root/matrix.sh /root/matrix.sh
+      echo "Install Matrix Chatserver!"
+      lxc-attach -n$LXC_NBR bash matrix.sh
+      break
+      ;;
+    quit)
+      break
+      ;;
+    *)
+      echo "Invalid option!"
+      ;;
+    esac
+done
+

contrib/proxmox-lxc/mailpiler.orig

@@ -0,0 +1,179 @@
+#!/bin/bash
+
+#Variables will be filled in from the mainscript:
+
+#PILER_VAR
+
+
+HOSTNAME=$(hostname -f)
+
+echo "Ensure your Hostname is set to your Piler FQDN!"
+
+echo $HOSTNAME
+
+if 
+    [ "$HOSTNAME" != "$PILER_DOM" ]
+then
+        echo "Hostname doesn't match Piler_Domain! Check install.sh, /etc/hosts, /etc/hostname." && exit
+else
+        echo "Hostname matches PILER_DOMAIN, so starting installation."
+fi
+
+apt install -y gpg apt-transport-https lsb-release
+
+wget -q https://packages.sury.org/php/apt.gpg -O- | apt-key add -
+echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" | tee /etc/apt/sources.list.d/php.list
+
+apt update && apt full-upgrade -y
+
+apt install -y mc sysstat build-essential libwrap0-dev libpst-dev tnef libytnef0-dev unrtf catdoc libtre-dev tre-agrep poppler-utils libzip-dev unixodbc libpq5 software-properties-common libpoppler-dev openssl libssl-dev memcached telnet nginx mariadb-server default-libmysqlclient-dev python-mysqldb gcc libwrap0 libzip4 latex2rtf latex2html catdoc tnef libpq5 zipcmp zipmerge ziptool libsodium23
+
+apt update && apt install -y php$PHP_VER-{fpm,common,ldap,mysql,cli,opcache,phpdbg,gd,memcache,json,readline,zip}
+
+apt purge -y postfix
+
+cat > /etc/mysql/conf.d/mailpiler.conf <<EOF
+innodb_buffer_pool_size=256M
+innodb_flush_log_at_trx_commit=1
+innodb_log_buffer_size=64M
+innodb_log_file_size=16M
+query_cache_size=0
+query_cache_type=0
+query_cache_limit=2M
+EOF
+
+systemctl restart mariadb
+
+cd /tmp
+wget https://download.mailpiler.com/generic-local/sphinx-$SPHINX_VER-bin.tar.gz
+tar -xvzf sphinx-$SPHINX_VER-bin.tar.gz -C /
+
+groupadd piler
+useradd -g piler -m -s /bin/bash -d /var/piler piler
+usermod -L piler
+chmod 755 /var/piler
+
+wget https://bitbucket.org/jsuto/piler/downloads/piler-$PILER_VER.tar.gz
+tar -xvzf piler-$PILER_VER.tar.gz
+cd piler-$PILER_VER/
+./configure --localstatedir=/var --with-database=mysql --enable-tcpwrappers --enable-memcached
+make
+make install
+ldconfig
+
+cp util/postinstall.sh util/postinstall.sh.bak
+sed -i "s/   SMARTHOST=.*/   SMARTHOST="\"$SMARTHOST\""/" util/postinstall.sh
+sed -i 's/   WWWGROUP=.*/   WWWGROUP="www-data"/' util/postinstall.sh
+
+make postinstall
+
+cp /usr/local/etc/piler/piler.conf /usr/local/etc/piler/piler.conf.bak
+sed -i "s/hostid=.*/hostid=$PILER_DOM/" /usr/local/etc/piler/piler.conf
+sed -i "s/update_counters_to_memcached=.*/update_counters_to_memcached=1/" /usr/local/etc/piler/piler.conf
+
+su piler -c "indexer --all --config /usr/local/etc/piler/sphinx.conf"
+
+/etc/init.d/rc.piler start
+/etc/init.d/rc.searchd start
+
+update-rc.d rc.piler defaults
+update-rc.d rc.searchd defaults
+
+mkdir -p /etc/nginx/ssl
+openssl req -x509 -nodes -days 3650 -newkey rsa:4096 -keyout /etc/nginx/ssl/piler.key -out /etc/nginx/ssl/piler.crt -subj "/CN=$PILER_DOM" -addext "subjectAltName=DNS:$PILER_DOM"
+
+cd /etc/nginx/sites-available
+cp /tmp/piler-$PILER_VER/contrib/webserver/piler-nginx.conf /etc/nginx/sites-available/
+ln -s /etc/nginx/sites-available/piler-nginx.conf /etc/nginx/sites-enabled/piler-nginx.conf
+
+sed -i "s|PILER_HOST|$PILER_DOM|g" /etc/nginx/sites-available/piler-nginx.conf
+sed -i "s|/var/run/php/php7.4-fpm.sock|/var/run/php/php$PHP_VER-fpm.sock|g" /etc/nginx/sites-available/piler-nginx.conf
+
+sed -i "/server_name.*/a \\
+        listen 443 ssl http2;\n\n\
+        ssl_certificate /etc/nginx/ssl/piler.crt;\n\
+        ssl_certificate_key /etc/nginx/ssl/piler.key;\n\n\
+        ssl_session_timeout 1d;\n\
+        ssl_session_cache shared:SSL:15m;\n\
+        ssl_session_tickets off;\n\n\
+        # modern configuration of Mozilla SSL configurator. Tweak to your needs.\n\
+        ssl_protocols TLSv1.2 TLSv1.3;\n\
+        ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;\n\
+        ssl_prefer_server_ciphers off;\n\n\
+        add_header X-Frame-Options SAMEORIGIN;\n\
+        add_header X-Content-Type-Options nosniff;" /etc/nginx/sites-available/piler-nginx.conf
+
+sed -i "/^server {.*/i\
+server {\n\
+        listen 80;\n\
+        server_name $PILER_DOM;\n\
+        server_tokens off;\n\
+        # HTTP to HTTPS redirect.\n\
+        return 301 https://$PILER_DOM;\n\
+}" /etc/nginx/sites-available/piler-nginx.conf
+
+cp /usr/local/etc/piler/config-site.php /usr/local/etc/piler/config-site.php.bak
+sed -i "s|\$config\['SITE_URL'\] = .*|\$config\['SITE_URL'\] = 'https://$PILER_DOM/';|" /usr/local/etc/piler/config-site.php
+cat >> /usr/local/etc/piler/config-site.php <<EOF
+
+// CUSTOM
+\$config['PROVIDED_BY'] = '$PILER_DOM';
+\$config['SUPPORT_LINK'] = 'https://$PILER_DOM';
+\$config['COMPATIBILITY'] = '';
+
+// fancy features.
+\$config['ENABLE_INSTANT_SEARCH'] = 1;
+\$config['ENABLE_TABLE_RESIZE'] = 1;
+
+\$config['ENABLE_DELETE'] = 1;
+\$config['ENABLE_ON_THE_FLY_VERIFICATION'] = 1;
+
+// general settings.
+\$config['TIMEZONE'] = 'Europe/Berlin';
+
+// authentication
+// Enable authentication against an imap server
+//\$config['ENABLE_IMAP_AUTH'] = 1;
+//\$config['RESTORE_OVER_IMAP'] = 1;
+//\$config['IMAP_RESTORE_FOLDER_INBOX'] = 'INBOX';
+//\$config['IMAP_RESTORE_FOLDER_SENT'] = 'Sent';
+//\$config['IMAP_HOST'] = '$SMARTHOST';
+//\$config['IMAP_PORT'] =  993;
+//\$config['IMAP_SSL'] = true;
+
+// authentication against an ldap directory (disabled by default)
+//\$config['ENABLE_LDAP_AUTH'] = 1;
+//\$config['LDAP_HOST'] = '$SMARTHOST';
+//\$config['LDAP_PORT'] = 389;
+//\$config['LDAP_HELPER_DN'] = 'cn=administrator,cn=users,dc=mydomain,dc=local';
+//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
+//\$config['LDAP_MAIL_ATTR'] = 'mail';
+//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
+//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
+//\$config['LDAP_BASE_DN'] = 'ou=Benutzer,dc=krs,dc=local';
+
+// authentication against an Uninvention based ldap directory 
+//\$config['ENABLE_LDAP_AUTH'] = 1;
+//\$config['LDAP_HOST'] = '$SMARTHOST';
+//\$config['LDAP_PORT'] = 7389;
+//\$config['LDAP_HELPER_DN'] = 'uid=ldap-search-user,cn=users,dc=mydomain,dc=local';
+//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
+//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
+//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
+//\$config['LDAP_BASE_DN'] = 'cn=users,dc=mydomain,dc=local';
+//\$config['LDAP_MAIL_ATTR'] = 'mailPrimaryAddress';
+//\$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'person';
+//\$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'person';
+//\$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mailAlternativeAddress';
+
+// special settings.
+\$config['MEMCACHED_ENABLED'] = 1;
+\$config['SPHINX_STRICT_SCHEMA'] = 1; // required for Sphinx $SPHINX_VER, see https://bitbucket.org/jsuto/piler/issues/1085/sphinx-331.
+EOF
+
+nginx -t && systemctl restart nginx
+
+apt autoremove -y
+apt clean -y
+
+

contrib/proxmox-lxc/matrix.orig

@@ -0,0 +1,150 @@
+#!/bin/bash
+
+#MATRIX_VAR
+
+
+MRX_PKE=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+
+ELE_DBNAME="synapse_db"
+ELE_DBUSER="synapse_user"
+ELE_DBPASS=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+
+apt update && apt full-upgrade -y
+
+apt install -y lsb-release apt-transport-https curl gpg software-properties-common net-tools nginx mc postgresql python3-psycopg2
+
+wget wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/matrix-org.list
+apt update && apt install -y matrix-synapse-py3
+systemctl enable matrix-synapse
+
+ss -tulpen
+
+mkdir /etc/nginx/ssl
+openssl req -x509 -nodes -days 3650 -newkey rsa:4096 -keyout /etc/nginx/ssl/matrix.key -out /etc/nginx/ssl/matrix.crt -subj "/CN=$MRX_DOM" -addext "subjectAltName=DNS:$MRX_DOM"
+
+cat > /etc/nginx/sites-available/$MRX_DOM <<EOF
+# Virtual Host configuration for example.com
+#
+# You can move that to a different file under sites-available/ and symlink that
+# to sites-enabled/ to enable it.
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name $MRX_DOM;
+
+    return 301 https://$MRX_DOM;
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    server_name $MRX_DOM;
+
+    ssl on;
+    ssl_certificate /etc/nginx/ssl/matrix.crt;
+    ssl_certificate_key /etc/nginx/ssl/matrix.key;
+
+    location / {
+      proxy_pass http://127.0.0.1:8008;
+      proxy_set_header X-Forwarded-For \$remote_addr;
+    }
+}
+
+server {
+    listen 8448 ssl;
+    listen [::]:8448 ssl;
+    server_name $MRX_DOM;
+
+    ssl on;
+    ssl_certificate /etc/nginx/ssl/matrix.crt;
+    ssl_certificate_key /etc/nginx/ssl/matrix.key;
+
+    # If you don't wanna serve a site, comment this out
+    root /var/www/$MRX_DOM;
+    index index.html index.htm;
+
+    location / {
+        proxy_pass http://127.0.0.1:8008;
+        proxy_set_header X-Forwarded-For \$remote_addr;
+    }
+}
+
+EOF
+ln -s /etc/nginx/sites-available/$MRX_DOM /etc/nginx/sites-enabled/$MRX_DOM
+
+cat > /etc/nginx/sites-available/$ELE_DOM <<EOF
+# Virtual Host configuration for example.com
+#
+# You can move that to a different file under sites-available/ and symlink that
+# to sites-enabled/ to enable it.
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name $ELE_DOM;
+    return 301 https://$ELE_DOM;
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    server_name $ELE_DOM;
+
+    ssl on;
+    ssl_certificate /etc/nginx/ssl/matrix.crt;
+    ssl_certificate_key /etc/nginx/ssl/matrix.key;
+
+    # If you don't wanna serve a site, comment this out
+    root /var/www/$ELE_DOM/element;
+    index index.html index.htm;
+} 
+
+EOF
+
+ln -s /etc/nginx/sites-available/$ELE_DOM /etc/nginx/sites-enabled/$ELE_DOM
+
+systemctl restart nginx
+
+mkdir /var/www/$ELE_DOM
+cd /var/www/$ELE_DOM
+wget https://packages.riot.im/element-release-key.asc
+gpg --import element-release-key.asc
+
+wget https://github.com/vector-im/element-web/releases/download/$ELE_VER/element-$ELE_VER.tar.gz
+wget https://github.com/vector-im/element-web/releases/download/$ELE_VER/element-$ELE_VER.tar.gz.asc
+gpg --verify element-$ELE_VER.tar.gz.asc
+
+tar -xzvf element-$ELE_VER.tar.gz
+ln -s element-$ELE_VER element
+chown www-data:www-data -R element
+cp ./element/config.sample.json ./element/config.json
+sed -i "s|https://matrix-client.matrix.org|https://$MRX_DOM|" ./element/config.json
+sed -i "s|\"server_name\": \"matrix.org\"|\"server_name\": \"$MRX_DOM\"|" ./element/config.json
+
+su postgres <<EOF
+psql -c "CREATE USER $ELE_DBUSER WITH PASSWORD '$ELE_DBPASS';"
+psql -c "CREATE DATABASE $ELE_DBNAME ENCODING 'UTF8' LC_COLLATE='C' LC_CTYPE='C' template=template0 OWNER $ELE_DBUSER;"
+echo "Postgres User '$ELE_DBUSER' and database '$ELE_DBNAME' created."
+EOF
+
+cd /
+sed -i "s|#registration_shared_secret: <PRIVATE STRING>|registration_shared_secret: \"$MRX_PKE\"|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|#public_baseurl: https://example.com/|public_baseurl: https://$MRX_DOM/|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|#enable_registration: false|enable_registration: true|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|name: sqlite3|name: psycopg2|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|database: /var/lib/matrix-synapse/homeserver.db|database: $ELE_DBNAME\n    user: $ELE_DBUSER\n    password: $ELE_DBPASS\n    host: 127.0.0.1\n    cp_min: 5\n    cp_max: 10|" /etc/matrix-synapse/homeserver.yaml
+
+systemctl restart matrix-synapse
+
+register_new_matrix_user -c /etc/matrix-synapse/homeserver.yaml http://127.0.0.1:8008
+
+#curl https://download.jitsi.org/jitsi-key.gpg.key | sh -c 'gpg --dearmor > /usr/share/keyrings/jitsi-keyring.gpg'
+#echo 'deb [signed-by=/usr/share/keyrings/jitsi-keyring.gpg] https://download.jitsi.org stable/' | tee /etc/apt/sources.list.d/jitsi-stable.list > /dev/null
+
+#apt update
+#apt install -y jitsi-meet
+
+
+

contrib/proxmox-lxc/zmb_mem.orig

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+#ZMB_VAR
+
+
+
+apt update && apt full-upgrade -y
+echo -ne '\n' | apt install -y acl dnsutils mc samba winbind libpam-winbind libnss-winbind krb5-user krb5-config samba-dsdb-modules samba-vfs-modules 
+
+mv /etc/krb5.conf /etc/krb5.conf.bak
+cat > /etc/krb5.conf <<EOF
+[libdefaults]
+	default_realm = $ZMB_REA
+    ticket_lifetime = 600
+	dns_lookup_realm = true
+	dns_lookup_kdc = true
+	renew_lifetime = 7d
+EOF
+
+echo -e "$ZMB_APW" | kinit -V $ZMB_ADA
+klist
+
+mv /etc/samba/smb.conf /etc/samba/smb.conf.bak
+cat > /etc/samba/smb.conf <<EOF
+[global]
+	workgroup = $ZMB_DOM
+	security = ADS
+	realm = $ZMB_REA
+	server string = %h server
+
+	vfs objects = acl_xattr shadow_copy2
+    map acl inherit = Yes
+    store dos attributes = Yes
+	idmap config *:backend = tdb
+	idmap config *:range = 3000000-4000000
+	idmap config *:schema_mode = rfc2307
+
+	winbind refresh tickets = Yes
+	winbind use default domain = Yes
+	winbind separator = /
+	winbind nested groups = yes
+	winbind nss info = rfc2307
+
+	pam password change = Yes
+	passwd chat = *Enter\snew\s*\spassword:* %n\n *Retype\snew\s*\spassword:* %n\n *password\supdated\ssuccessfully* .
+	passwd program = /usr/bin/passwd %u
+
+	template homedir = /home/%U
+	template shell = /bin/bash
+	bind interfaces only = Yes
+	interfaces = lo eth0
+	log file = /var/log/samba/log.%m
+	logging = syslog
+	max log size = 1000
+	panic action = /usr/share/samba/panic-action %d
+
+	load printers = No
+	printcap name = /dev/null
+	printing = bsd
+	disable spoolss = Yes
+
+	allow trusted domains = No
+	dns proxy = No
+	shadow: snapdir = .zfs/snapshot
+	shadow: sort = desc
+	shadow: format = -%Y-%m-%d-%H%M
+	shadow: snapprefix = ^zfs-auto-snap_\(frequent\)\{0,1\}\(hourly\)\{0,1\}\(daily\)\{0,1\}\(monthly\)\{0,1\}
+	shadow: delimiter = -20
+
+[share]
+	comment = Main Share
+	path = /tank/share
+	read only = No
+	create mask = 0660
+	directory mask = 0770
+	inherit acls = Yes
+
+
+
+EOF
+
+systemctl restart smbd
+
+echo -e "$ZMB_APW" | net ads join -U $ZMB_ADA createcomputer=Computers
+sed -i "s|files systemd|files systemd winbind|g" /etc/nsswitch.conf
+sed -i "s|#WINBINDD_OPTS=|WINBINDD_OPTS=|" /etc/default/winbind
+echo -e "session optional        pam_mkhomedir.so skel=/etc/skel umask=077" >> /etc/pam.d/common-session
+
+systemctl restart winbind nmbd
+wbinfo -u
+wbinfo -g
+
+mkdir /tank/share
+chown 'administrator':'domain users' /tank/share
+
+setfacl -Rm u:administrator:rwx,g::-,o::- /tank/share
+setfacl -Rdm u:administrator:rwx,g::-,o::- /tank/share
+
+systemctl restart smbd nmbd winbind
+

contrib/smtp/smtp-source.py

@@ -168,4 +168,4 @@ while i < args.count:
 
 
 if args.debug == 0:
-    print
+    print('')

contrib/webserver/piler-apache-2.x.conf

@@ -4,8 +4,7 @@
     DocumentRoot "/var/piler/www"
 
     <Directory /var/piler/www>
-       Order allow,deny
+       Require all granted
-       Allow from all
 
        AllowOverride all
     </Directory>

contrib/webserver/piler-nginx.conf

@@ -4,10 +4,13 @@ server {
 
         root /var/piler/www;
 
-        access_log  /var/log/nginx/access.log;
+        server_tokens off;
-        error_log   /var/log/nginx/error.log;
 
-        gzip  on;
+        add_header X-XSS-Protection "1; mode=block";
+        add_header X-Content-Type-Options "nosniff";
+        add_header Referrer-Policy "same-origin";
+
+        gzip on;
         gzip_types text/plain application/xml text/css;
         gzip_vary on;
 
@@ -31,8 +34,7 @@ server {
                return 404;
             }
 
-            #fastcgi_pass unix:/var/run/php5-fpm.sock;
+            fastcgi_pass unix:/var/run/php/php8.1-fpm.sock;
-            fastcgi_pass unix:/var/run/php/php7.0-fpm.sock;
             fastcgi_index index.php;
             include fastcgi_params;
         }
@@ -49,6 +51,7 @@ server {
         rewrite /message.php /index.php?route=message/view;
         rewrite /bulkrestore.php /index.php?route=message/bulkrestore;
         rewrite /bulkremove.php /index.php?route=message/bulkremove;
+        rewrite /rejectremove.php /index.php?route=message/rejectremove;
         rewrite /bulkpdf.php /index.php?route=message/bulkpdf;
         rewrite /folders.php /index.php?route=folder/list&;
         rewrite /settings.php /index.php?route=user/settings;

cppcheck.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+cppcheck -DHAVE_PDFTOTEXT -DHAVE_PPTHTML -DHAVE_TNEF -DHAVE_UNRTF -DHAVE_XLS2CSV -DHAVE_CATPPT -DHAVE_CATDOC -DHAVE_ZIP -D_GNU_SOURCE -DHAVE_DAEMON -DHAVE_TRE -DHAVE_CLAMD -DHAVE_LIBCLAMAV -DNEED_MYSQL --error-exitcode=1 --enable=all --suppressions-list=suppressions.txt --force src/ unit_tests/

docker/Dockerfile

@@ -0,0 +1,43 @@
+FROM ubuntu:22.04
+
+ARG PACKAGE
+
+LABEL description="piler ubuntu jammy image" \
+      maintainer="Janos SUTO, sj@acts.hu" \
+      package="${PACKAGE}"
+
+ENV DEBIAN_FRONTEND="noninteractive" \
+    DISTRO="jammy" \
+    PILER_USER="piler" \
+    MYSQL_DATABASE="piler"
+
+COPY ${PACKAGE} /
+
+RUN apt-get update && \
+    apt-get -y --no-install-recommends install \
+       wget openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql php8.1-fpm php8.1-zip php8.1-ldap \
+       php8.1-gd php8.1-curl php8.1-xml php8.1-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip4 \
+       libtre5 cron libmariadb-dev mariadb-client-core-10.6 python3 python3-mysqldb ca-certificates curl rsyslog && \
+    wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
+    dpkg -i manticore-repo.noarch.deb && \
+    rm -f manticore-repo.noarch.deb && \
+    apt-get update && \
+    apt-get install -y manticore manticore-columnar-lib && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    sed -i '/session    required     pam_loginuid.so/c\#session    required     pam_loginuid.so' /etc/pam.d/cron && \
+    dpkg -i ${PACKAGE} && \
+    touch /etc/piler/MANTICORE && \
+    ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled && \
+    rm -f ${PACKAGE} /etc/nginx/sites-enabled/default /etc/piler/piler.key /etc/piler/piler.pem /etc/piler/config-site.php && \
+    crontab -u $PILER_USER /usr/share/piler/piler.cron
+
+VOLUME ["/etc/piler"]
+VOLUME ["/var/piler/store"]
+VOLUME ["/var/piler/manticore"]
+
+EXPOSE 25 80 443
+
+COPY start.sh /start.sh
+
+CMD ["/start.sh"]

docker/README.md

@@ -0,0 +1,16 @@
+## How to run piler
+
+Edit the variables in docker-compose.yaml, then run
+
+```
+docker-compose up -d
+```
+
+## How to build the image for yourself
+
+Pick the latest deb package from Bitbucket download page (https://bitbucket.org/jsuto/piler/downloads/)
+and use it as the PACKAGE build argument, eg.
+
+```
+docker build --build-arg PACKAGE=piler_1.4.1-jammy-c860ca67_amd64.deb -t piler:1.4.1 .
+```

docker/build.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+IMAGE_NAME="sutoj/piler:1.4.4"
+
+if [[ $# -ne 1 ]]; then echo "ERROR: missing package name" 1>&2; exit 1; fi
+
+docker build --pull --build-arg PACKAGE="$1" -t "$IMAGE_NAME" .

docker/docker-compose.yaml

@@ -0,0 +1,59 @@
+version: "3"
+services:
+  mysql:
+    image: mariadb:11.1.2
+    container_name: mysql
+    restart: unless-stopped
+    cap_drop:
+      - ALL
+    cap_add:
+      - dac_override
+      - setuid
+      - setgid
+    environment:
+      - MYSQL_DATABASE=piler
+      - MYSQL_USER=piler
+      - MYSQL_PASSWORD=piler123
+      - MYSQL_RANDOM_ROOT_PASSWORD=yes
+    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
+    volumes:
+      - db_data:/var/lib/mysql
+  piler:
+    image: sutoj/piler:1.4.4
+    container_name: piler
+    init: true
+    environment:
+      - MYSQL_HOSTNAME=mysql
+      - MYSQL_DATABASE=piler
+      - MYSQL_USER=piler
+      - MYSQL_PASSWORD=piler123
+      - PILER_HOSTNAME=cust1.acts.hu
+      - RT=1
+    ports:
+      - "25:25"
+      - "80:80"
+    volumes:
+      - piler_etc:/etc/piler
+      - piler_manticore:/var/piler/manticore
+      - piler_store:/var/piler/store
+    healthcheck:
+      test: curl -s smtp://localhost/
+      interval: "60s"
+      timeout: "3s"
+      start_period: "15s"
+      retries: 3
+    deploy:
+      resources:
+        reservations:
+          memory: 512M
+        limits:
+          memory: 512M
+
+    depends_on:
+      - "mysql"
+
+volumes:
+  db_data: {}
+  piler_etc: {}
+  piler_manticore: {}
+  piler_store: {}

docker/start.sh

@@ -0,0 +1,216 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+CONFIG_DIR="/etc/piler"
+VOLUME_DIR="/var/piler"
+PILER_CONF="${CONFIG_DIR}/piler.conf"
+PILER_KEY="${CONFIG_DIR}/piler.key"
+PILER_PEM="${CONFIG_DIR}/piler.pem"
+PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf"
+SPHINX_CONF="${CONFIG_DIR}/manticore.conf"
+CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php"
+PILER_MY_CNF="${CONFIG_DIR}/.my.cnf"
+RT="${RT:-0}"
+
+error() {
+   echo "ERROR:" "$*" 1>&2
+   exit 1
+}
+
+
+log() {
+   echo "DEBUG:" "$*"
+}
+
+
+pre_flight_check() {
+   [[ -v PILER_HOSTNAME ]] || error "Missing PILER_HOSTNAME env variable"
+   [[ -v MYSQL_HOSTNAME ]] || error "Missing MYSQL_HOSTNAME env variable"
+   [[ -v MYSQL_DATABASE ]] || error "Missing MYSQL_DATABASE env variable"
+   [[ -v MYSQL_USER ]]     || error "Missing MYSQL_USER env variable"
+   [[ -v MYSQL_PASSWORD ]] || error "Missing MYSQL_PASSWORD env variable"
+}
+
+
+give_it_to_piler() {
+   local f="$1"
+
+   [[ -f "$f" ]] || error "${f} does not exist, aborting"
+
+   chown "${PILER_USER}:${PILER_USER}" "$f"
+   chmod 600 "$f"
+}
+
+
+make_certificate() {
+   local f="$1"
+   local crt="/tmp/1.cert"
+   local SSL_CERT_DATA="/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com"
+
+   log "Making an ssl certificate"
+
+   openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 -subj "$SSL_CERT_DATA" -keyout "$f" -out "$crt" -sha1 2>/dev/null
+   cat "$crt" >> "$f"
+   rm -f "$crt"
+
+   give_it_to_piler "$f"
+}
+
+
+make_piler_key() {
+   local f="$1"
+
+   log "Generating piler.key"
+
+   dd if=/dev/urandom bs=56 count=1 of="$f" 2>/dev/null
+   [[ $(stat -c '%s' "$f") -eq 56 ]] || error "could not read 56 bytes from /dev/urandom to ${f}"
+
+   give_it_to_piler "$f"
+}
+
+
+fix_configs() {
+   [[ -f "$PILER_KEY" ]] || make_piler_key "$PILER_KEY"
+   [[ -f "$PILER_PEM" ]] || make_certificate "$PILER_PEM"
+
+   if [[ ! -f "$PILER_NGINX_CONF" ]]; then
+      log "Writing ${PILER_NGINX_CONF}"
+
+      cp "${PILER_NGINX_CONF}.dist" "$PILER_NGINX_CONF"
+      sed -i "s%PILER_HOST%${PILER_HOSTNAME}%" "$PILER_NGINX_CONF"
+   fi
+
+   if [[ ! -f "$PILER_CONF" ]]; then
+      log "Writing ${PILER_CONF}"
+
+      sed \
+         -e "s/mysqluser=.*/mysqluser=${MYSQL_USER}/g" \
+         -e "s/mysqldb=.*/mysqldb=${MYSQL_DATABASE}/g" \
+         -e "s/verystrongpassword/${MYSQL_PASSWORD}/g" \
+         -e "s/hostid=.*/hostid=${PILER_HOSTNAME}/g" \
+         -e "s/tls_enable=.*/tls_enable=1/g" \
+         -e "s/mysqlsocket=.*/mysqlsocket=/g" "${PILER_CONF}.dist" > "$PILER_CONF"
+
+      {
+         echo "mysqlhost=${MYSQL_HOSTNAME}"
+      } >> "$PILER_CONF"
+
+      give_it_to_piler "$PILER_CONF"
+   fi
+
+   if [[ ! -f "$CONFIG_SITE_PHP" ]]; then
+      log "Writing ${CONFIG_SITE_PHP}"
+
+      cp "${CONFIG_DIR}/config-site.dist.php" "$CONFIG_SITE_PHP"
+
+      sed -i "s%HOSTNAME%${PILER_HOSTNAME}%" "$CONFIG_SITE_PHP"
+
+      {
+         echo "\$config['DECRYPT_BINARY'] = '/usr/bin/pilerget';"
+         echo "\$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/bin/pileraget';"
+         echo "\$config['PILER_BINARY'] = '/usr/sbin/piler';"
+         echo "\$config['DB_HOSTNAME'] = '$MYSQL_HOSTNAME';"
+         echo "\$config['DB_DATABASE'] = '$MYSQL_DATABASE';"
+         echo "\$config['DB_USERNAME'] = '$MYSQL_USER';"
+         echo "\$config['DB_PASSWORD'] = '$MYSQL_PASSWORD';"
+         echo "\$config['ENABLE_MEMCACHED'] = 1;"
+         echo "\$memcached_server = ['memcached', 11211];"
+      } >> "$CONFIG_SITE_PHP"
+   fi
+
+   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \
+       -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \
+       -e "s%MYSQL_USERNAME%${MYSQL_USER}%" \
+       -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" \
+       -i "$SPHINX_CONF"
+
+   # Fixes for RT index
+
+   if [[ $RT -eq 1 ]]; then
+      sed -i "s/define('RT', 0)/define('RT', 1)/" "$SPHINX_CONF"
+      if ! grep "'RT'" "$CONFIG_SITE_PHP"; then
+         echo "\$config['RT'] = 1;" >> "$CONFIG_SITE_PHP"
+      fi
+
+      if ! grep "'SPHINX_MAIN_INDEX'" "$CONFIG_SITE_PHP"; then
+         echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
+      fi
+
+      sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
+   fi
+}
+
+
+wait_until_mysql_server_is_ready() {
+   while true; do if mysql "--defaults-file=${PILER_MY_CNF}" <<< "show databases"; then break; fi; log "${MYSQL_HOSTNAME} is not ready"; sleep 5; done
+
+   log "${MYSQL_HOSTNAME} is ready"
+}
+
+
+init_database() {
+   local table
+   local has_metadata_table=0
+
+   wait_until_mysql_server_is_ready
+
+   while read -r table; do
+      if [[ "$table" == metadata ]]; then has_metadata_table=1; fi
+   done < <(mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< 'show tables')
+
+   if [[ $has_metadata_table -eq 0 ]]; then
+      log "no metadata table, creating tables"
+
+      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" < /usr/share/piler/db-mysql.sql
+   else
+      log "metadata table exists"
+   fi
+
+   if [[ -v ADMIN_USER_PASSWORD_HASH ]]; then
+      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< "update user set password='${ADMIN_USER_PASSWORD_HASH}' where uid=0"
+   fi
+}
+
+
+create_my_cnf_files() {
+   printf "[client]\nhost = %s\nuser = %s\npassword = %s\n[mysqldump]\nhost = %s\nuser = %s\npassword = %s\n" \
+      "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" \
+      > "$PILER_MY_CNF"
+
+   give_it_to_piler "$PILER_MY_CNF"
+}
+
+
+start_services() {
+   service cron start
+   service php8.1-fpm start
+   service nginx start
+   rsyslogd
+}
+
+
+start_piler() {
+   if [[ $RT -eq 0 && ! -f "${VOLUME_DIR}/manticore/main1.spp" ]]; then
+      log "main1.spp does not exist, creating index files"
+      su -c "indexer --all --config ${SPHINX_CONF}" "$PILER_USER"
+   fi
+
+   # No pid file should exist for piler
+   rm -f /var/run/piler/*pid
+
+   /etc/init.d/rc.searchd start
+   /etc/init.d/rc.piler start
+}
+
+
+pre_flight_check
+fix_configs
+create_my_cnf_files
+init_database
+start_services
+start_piler
+
+sleep infinity

etc/Makefile.in

@@ -31,8 +31,10 @@ all:
 
 install:
 	$(INSTALL) -m 0640 -g $(RUNNING_GROUP) $(srcdir)/piler.conf $(DESTDIR)$(sysconfdir)/piler/piler.conf.dist
-	sed -e 's%LOCALSTATEDIR%$(localstatedir)%g' $(srcdir)/sphinx.conf.in > sphinx.conf.dist
+	sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/sphinx.conf.in > sphinx.conf.dist
-	$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/sphinx.conf.dist $(DESTDIR)$(sysconfdir)/piler/sphinx.conf.dist
+	sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/manticore.conf.in > manticore.conf.dist
+	$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/sphinx.conf.dist $(DESTDIR)$(sysconfdir)/piler/sphinx.conf.dist
+	$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/manticore.conf.dist $(DESTDIR)$(sysconfdir)/piler/manticore.conf.dist
 	$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/config-site.dist.php $(DESTDIR)$(sysconfdir)/piler/config-site.dist.php
 	$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/cron.jobs.in $(DESTDIR)$(datarootdir)/piler/piler.cron
 	sed -i -e 's%LOCALSTATEDIR%$(localstatedir)%g' $(DESTDIR)$(datarootdir)/piler/piler.cron

etc/config-site.dist.php

@@ -1,11 +1,13 @@
 <?php
 
-$config['SITE_NAME'] = 'HOSTNAME';
+define('SITE_NAME_CONST', 'SITE_NAME');
-$config['SITE_URL'] = 'http://' . $config['SITE_NAME'] . '/';
 
-$config['SMTP_DOMAIN'] = $config['SITE_NAME'];
+$config[SITE_NAME_CONST] = 'HOSTNAME';
-$config['SMTP_FROMADDR'] = 'no-reply@' . $config['SITE_NAME'];
+$config['SITE_URL'] = 'http://' . $config[SITE_NAME_CONST] . '/';
-$config['ADMIN_EMAIL'] = 'admin@' . $config['SITE_NAME'];
+
+$config['SMTP_DOMAIN'] = $config[SITE_NAME_CONST];
+$config['SMTP_FROMADDR'] = 'no-reply@' . $config[SITE_NAME_CONST];
+$config['ADMIN_EMAIL'] = 'admin@' . $config[SITE_NAME_CONST];
 
 $config['DB_PASSWORD'] = 'MYSQL_PASSWORD';
 

etc/cron.jobs.in

@@ -1,19 +1,14 @@
 ### PILERSTART
 5,35 * * * * LIBEXECDIR/piler/indexer.delta.sh
 30   2 * * * LIBEXECDIR/piler/indexer.main.sh
-15,45 * * * * LIBEXECDIR/piler/indexer.attachment.sh
+40 3 * * * LIBEXECDIR/piler/purge.sh
-*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet tag1 --rotate
+*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet tag1 --rotate
-*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet note1 --rotate
+*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet note1 --rotate
 */5 * * * *  /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \;
+*/5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error
 
-### optional: the same report you can see on the health page
+### optional
-###30 7 * * * /usr/bin/php LIBEXECDIR/piler/daily-report.php --webui LOCALSTATEDIR/piler/www
+###30 6 * * * /usr/bin/php LIBEXECDIR/piler/generate_stats.php --webui LOCALSTATEDIR/piler/www >/dev/null
-
+###*/5 * * * * LIBEXECDIR/piler/import.sh
-### optional: populate accouting data
-###30 6 * * * /usr/bin/php LIBEXECDIR/piler/generate_stats.php --webui LOCALSTATEDIR/piler/www
-
-### optional: purge aged emails
-###2 0 * * * BINDIR/pilerpurge
-
 
 ### PILEREND

etc/example.conf

@@ -22,10 +22,11 @@ username=piler
 ; The default is 7 years + 2 days (=7*365+2=2557 days)
 default_retention_days=2557
 
-; this is a 16 character long vector
+; The initialization vector for encryption.
-; after the installation you must not change it ever
+; By now it has become obsolete. Don't use it for
-; otherwise you can't access your emails
+; new installations. However, if you used it before
-;iv=****************
+; then you must keep it as it is.
+;iv=
 
 ; whether to encrypt messages (1) or not (0).
 ; Make sure to set this value to your needs right after installing piler,
@@ -45,12 +46,12 @@ number_of_worker_processes=2
 max_connections=64
 
 ; number of processed emails per each piler process
-max_requests_per_child=1000
+max_requests_per_child=10000
 
 ; SMTP HELO identification string
 ; this should be the FQDN part of the email address
-; where you copy emails, eg. archive@piler.yourdomain.com -> piler.yourdomain.com
+; where you copy emails, eg. archive@piler.example.com -> piler.example.com
-hostid=piler.yourdomain.com
+hostid=piler.example.com
 
 ; whether to process rcpt to addresses and add them to rcpt table (1) or not (0)
 process_rcpt_to_addresses=0
@@ -66,6 +67,9 @@ listen_port=25
 
 clamd_socket=/tmp/clamd
 
+; check for client timeout interval. Default: 20 sec
+check_for_client_timeout_interval=20
+
 ; smtp timeout. Default: 60 sec
 smtp_timeout=60
 
@@ -103,6 +107,13 @@ pemfile=
 cipher_list=ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS
 
 
+; set the minimum TLS protocol version for piler-smtp daemon
+;
+; Valid values: TLSv1, TLSv1.1, TLSv1.2, TLSv1.3
+; TLSv1 and TLSv1.1 are not recommended for security reasons
+tls_min_version=TLSv1.2
+
+
 ; piler's own header to indicate previously archived messages
 piler_header_field=X-piler-id:
 
@@ -198,7 +209,7 @@ memcached_to_db_interval=900
 // this can be either utf8 or utf8mb4. Make sure to match the value
 // to the charset of the piler database! Also, make sure to set this
 // value in sphinx.conf
-mysqlcharset=utfmb4
+mysqlcharset=utf8mb4
 ;mysqlhost=127.0.0.1
 ;mysqlport=3306
 mysqlsocket=/var/run/mysqld/mysqld.sock
@@ -208,6 +219,14 @@ mysqldb=piler
 mysql_connect_timeout=2
 
 
+;
+; manticore stuff
+;
+sphxhost=127.0.0.1
+sphxport=9306
+; whether to enable (1) or not (0) the rt index. If so, then piler
+; writes the index data directly to manticore's sql API
+rtindex=0
 
 ; if you want to change the 'sent' time as you archive the message
 ; set this in seconds. This can be a postive or negative value.
@@ -218,3 +237,38 @@ tweak_sent_time_offset=0
 ; whether to enable (1) or not (0) the extra mmap dedup test feature
 ; if you change it, be sure to stop, then start piler
 mmap_dedup_test=0
+
+; security header that must be present in the first mail header of
+; the message. If the security_header value is not empty, then the
+; parser checks for this header line. Unless it's found it will discard
+; the given email. Note that this feature is supposed to be a security
+; mechanism against unwanted email in the archive if limiting smtp
+; clients via an IP-address list is not feasible.
+security_header=
+
+; By default the archive accepts any envelope recipient addresses.
+; If your archive's port 25 is wide open to the Internet (which it
+; shouldn't be, then spammers may find it, and fill it with spam.
+;
+; By setting this variable you may restrict the envelope address
+; to a single email address, eg. some-random-address-12345@archive.example.com
+; Then the archive will reject any other envelope recipients
+archive_address=
+
+; whether to enable (1) or not (0) an smtp access list similar to
+; postfix's postscreen. Valid actions in the acl file are "permit"
+; and "reject" (without quotes). See smtp.acl.example for more.
+;
+; Important! There's an implicit default deny at the end of the
+; rules. In other words if you decide to use the acl file, then
+; everyone is not explicitly permitted is denied.
+smtp_access_list=0
+
+; max message size in bytes
+; piler-smtp will reject any message that's bigger than this number
+max_message_size=50000000
+
+; max memory in bytes piler-smtp uses for buffering messages
+; when this limit is exceeded, no new emails will be accepted
+; until the used memory decreases below this level
+max_smtp_memory=500000000

etc/manticore.conf.in

@@ -0,0 +1,268 @@
+#!/usr/bin/php
+<?php
+
+define('LOCALSTATEDIR', '/var');
+define('NGRAM_CONFIG', "        #ngram_len               = 1\n        #ngram_chars             = U+3000..U+2FA1F\n");
+
+# See http://sphinxsearch.com/wiki/doku.php?id=charset_tables for more on the charset_table settings
+# The following settings contains English and some Latin extras
+define('CHARSET_TABLE', "0..9, english, _, \
+                                  U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
+                                  U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
+                                  U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
+                                  U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171\n");
+
+define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
+
+define('RT', 0);
+
+?>
+
+#
+# minimal manticore configuration suited to piler
+#
+
+<?php if(RT == 0) { ?>
+
+source base
+{
+   type = mysql
+   sql_host = MYSQL_HOSTNAME
+   sql_db = MYSQL_DATABASE
+   sql_user = MYSQL_USERNAME
+   sql_pass = MYSQL_PASSWORD
+
+   sql_attr_uint = size
+   sql_attr_uint = sent
+   sql_attr_uint = attachments
+}
+
+source delta : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query_pre  = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index
+   sql_query_post_index  = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
+
+   sql_query_killlist = SELECT `id` FROM `metadata` WHERE `deleted`=1
+}
+
+source main1 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source main2 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source main3 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source main4 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source dailydelta : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+index main1
+{
+        source                  = main1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index main2
+{
+        source                  = main2
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main2
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index main3
+{
+        source                  = main3
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main3
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index main4
+{
+        source                  = main4
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main4
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index dailydelta1
+{
+        source                  = dailydelta
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/dailydelta1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+        killlist_target         = main1:kl, main2:kl, main3:kl, main4:kl, dailydelta1:kl
+<?php print NGRAM_CONFIG; ?>
+}
+
+index delta1
+{
+        source                  = delta
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/delta1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+source tag : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query  = SELECT `_id`, `id` AS iid, `uid`, `tag` FROM `tag`
+
+   sql_attr_uint = iid
+   sql_attr_uint = uid
+}
+
+source note : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query  = SELECT `_id`, `id` AS iid, `uid`, `note` FROM `note`
+
+   sql_attr_uint = iid
+   sql_attr_uint = uid
+}
+
+index tag1
+{
+        source                  = tag
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/tag1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+        stored_fields           =
+<?php print NGRAM_CONFIG; ?>
+}
+
+
+index note1
+{
+        source                  = note
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/note1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+        stored_fields           =
+<?php print NGRAM_CONFIG; ?>
+}
+
+<?php } else { ?>
+
+index piler1
+{
+    type = rt
+    path = /var/piler/manticore/piler1
+    rt_mem_limit = 512M
+    stored_fields =
+    min_word_len = 1
+    min_prefix_len = 5
+    charset_table  = <?php print CHARSET_TABLE; ?>
+    # See https://manual.manticoresearch.com/Creating_an_index/Data_types#Row-wise-and-columnar-attribute-storages
+    # if you want to enable columnar storage
+    # columnar_attrs = *
+    rt_field = sender
+    rt_field = rcpt
+    rt_field = senderdomain
+    rt_field = rcptdomain
+    rt_field = subject
+    rt_field = body
+    rt_field = attachment_types
+    rt_attr_bigint = arrived
+    rt_attr_bigint = sent
+    rt_attr_uint = size
+    rt_attr_uint = direction
+    rt_attr_uint = folder
+    rt_attr_uint = attachments
+}
+
+index tag1
+{
+    type = rt
+    path = /var/piler/manticore/tag1
+    rt_mem_limit = 16M
+    stored_fields = tag
+    min_word_len = 2
+    min_prefix_len = 5
+    charset_table  = <?php print CHARSET_TABLE; ?>
+    rt_field = tag
+    rt_attr_bigint = mid
+    rt_attr_uint = uid
+}
+
+index note1
+{
+    type = rt
+    path = /var/piler/manticore/note1
+    rt_mem_limit = 16M
+    stored_fields = note
+    min_word_len = 2
+    min_prefix_len = 5
+    charset_table  = <?php print CHARSET_TABLE; ?>
+    rt_field = note
+    rt_attr_bigint = mid
+    rt_attr_uint = uid
+}
+
+<?php } ?>
+
+
+searchd
+{
+        listen                  = 127.0.0.1:9312
+        listen                  = 127.0.0.1:9306:mysql
+        listen                  = 127.0.0.1:9307:mysql_readonly
+        log                     = /var/piler/manticore/manticore.log
+        binlog_max_log_size     = 256M
+        binlog_path             = /var/piler/manticore
+        binlog_flush            = 2
+        query_log               = /var/piler/manticore/query.log
+        network_timeout         = 5
+        pid_file                = /var/run/piler/searchd.pid
+        seamless_rotate         = 1
+        preopen_tables          = 1
+        unlink_old              = 1
+        thread_stack            = 512k
+<?php if(RT) { ?>
+        rt_flush_period         = 300
+<?php } ?>
+
+}

etc/smtp.acl.example

@@ -0,0 +1,9 @@
+# Allow office365 servers. See the below URI for more
+# https://docs.microsoft.com/en-us/microsoft-365/enterprise/urls-and-ip-address-ranges?view=o365-worldwide
+#
+# Anything is not listed below will be rejected
+#
+40.92.0.0/15 permit
+40.107.0.0/16 permit
+52.100.0.0/14 permit
+104.47.0.0/17 permit

etc/sphinx.conf.in

@@ -1,3 +1,34 @@
+#!/usr/bin/php
+<?php
+
+define('SPHINX_VERSION', 331); // If you have sphinx-3.3.1, then set SPHINX_VERSION to 331
+define('LOCALSTATEDIR', '@LOCALSTATEDIR@');
+define('NGRAM_CONFIG', "        #ngram_len               = 1\n        #ngram_chars             = U+3000..U+2FA1F\n");
+
+# See http://sphinxsearch.com/wiki/doku.php?id=charset_tables for more on the charset_table settings
+# The following settings contains English and some Latin extras
+define('SPHINX_CHARSET_TABLE', "0..9, english, _, \
+                                  U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
+                                  U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
+                                  U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
+                                  U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171\n");
+
+// Sphinx 3.3.1 introduced some strict rules for fulltext search column names
+// In order to comply with it you must enable SPHINX_STRICT_SCHEMA variable
+// Be sure to check out http://www.mailpiler.org/wiki/current:sphinx3 for more
+// NB: The SPHINX_STRICT_SCHEMA in sphinx.conf MUST BE THE SAME as in config.php (or in config-site.php)
+//
+define('SPHINX_STRICT_SCHEMA', 1);
+define('RT', 0);
+
+if(SPHINX_STRICT_SCHEMA) {
+   define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
+} else {
+   define('SELECT_FIELDS', 'id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
+}
+
+?>
+
 #
 # minimal sphinx configuration suited to piler
 #
@@ -15,46 +46,52 @@ source base
    sql_attr_uint = attachments
 }
 
+<?php if(!RT) { ?>
+
 source delta : base
 {
    sql_query_pre = SET NAMES utf8mb4
    sql_query_pre  = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index
    sql_query_post_index  = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index \
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
-        WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
 
+<?php if(SPHINX_VERSION < 300) { ?>
    sql_query_killlist = SELECT `id` FROM `metadata` WHERE `deleted`=1
+<?php } else { ?>
+   sql_query_kbatch = SELECT `id` FROM `metadata` WHERE `deleted`=1
+<?php } ?>
 }
 
 source main1 : base
 {
    sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
 }
 
 source main2 : base
 {
    sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
 }
 
 source main3 : base
 {
    sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
 }
 
 source main4 : base
 {
    sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
 }
 
 source dailydelta : base
 {
    sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
 }
+<?php } ?>
 
 source tag : base
 {
@@ -76,148 +113,171 @@ source note : base
 
 }
 
-source att : base
+<?php if(!RT) { ?>
-{
-
-   sql_query_pre = SET NAMES utf8mb4
-   sql_query  =  select a.id as aid, m.id as mid, a.name AS aname, a.size, REPLACE(REPLACE(m.`from`, '@', 'X'), '.', 'X') as `from`, REPLACE(REPLACE((select group_concat(rcpt.`to` ORDER BY `to` ASC SEPARATOR ' ') from rcpt where rcpt.id=mid group by rcpt.id), '@', 'X'), '.', 'X') as `to` from attachment a, metadata m where m.piler_id=a.piler_id
-
-   sql_attr_uint = size
-   sql_attr_uint = mid
-}
-
-
 index main1
 {
-	source			= main1
+        source                  = main1
-	path			= LOCALSTATEDIR/piler/sphinx/main1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main1
-	docinfo			= extern
+<?php if(SPHINX_VERSION < 300) { ?>
-	dict			= keywords
+        docinfo                 = extern
+        dict                    = keywords
+<?php } ?>
         min_prefix_len          = 5
         min_word_len            = 1
-        #ngram_len               = 1
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-        #ngram_chars             = U+3000..U+2FA1F
+<?php print NGRAM_CONFIG; ?>
 }
 
 index main2
 {
         source                  = main2
-        path                    = LOCALSTATEDIR/piler/sphinx/main2
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main2
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-	dict			= keywords
+        dict                    = keywords
+<?php } ?>
         min_prefix_len          = 5
         min_word_len            = 1
-        #ngram_len               = 1
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-        #ngram_chars             = U+3000..U+2FA1F
+<?php print NGRAM_CONFIG; ?>
 }
 
 index main3
 {
         source                  = main3
-        path                    = LOCALSTATEDIR/piler/sphinx/main3
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main3
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-	dict			= keywords
+        dict                    = keywords
+<?php } ?>
         min_prefix_len          = 5
         min_word_len            = 1
-        #ngram_len               = 1
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-        #ngram_chars             = U+3000..U+2FA1F
+<?php print NGRAM_CONFIG; ?>
 }
 
 index main4
 {
         source                  = main4
-        path                    = LOCALSTATEDIR/piler/sphinx/main4
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main4
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-	dict			= keywords
+        dict                    = keywords
+<?php } ?>
         min_prefix_len          = 5
         min_word_len            = 1
-        #ngram_len               = 1
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-        #ngram_chars             = U+3000..U+2FA1F
+<?php print NGRAM_CONFIG; ?>
 }
 
 index dailydelta1
 {
         source                  = dailydelta
-        path                    = LOCALSTATEDIR/piler/sphinx/dailydelta1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/dailydelta1
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-	dict			= keywords
+        dict                    = keywords
+<?php } ?>
         min_prefix_len          = 5
         min_word_len            = 1
-        #ngram_len               = 1
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-        #ngram_chars             = U+3000..U+2FA1F
+<?php print NGRAM_CONFIG; ?>
 }
 
 index delta1
 {
         source                  = delta
-        path                    = LOCALSTATEDIR/piler/sphinx/delta1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/delta1
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-	dict			= keywords
+        dict                    = keywords
+<?php } ?>
         min_prefix_len          = 5
         min_word_len            = 1
-        #ngram_len               = 1
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-        #ngram_chars             = U+3000..U+2FA1F
+<?php print NGRAM_CONFIG; ?>
+<?php if(SPHINX_VERSION >= 310) { ?>
+        kbatch = main1, main2, main3, main4, dailydelta1
+<?php } ?>
 }
 
+<?php } ?>
 
 index tag1
 {
         source                  = tag
-        path                    = LOCALSTATEDIR/piler/sphinx/tag1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/tag1
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-	dict			= keywords
+        dict                    = keywords
+<?php } ?>
         min_prefix_len          = 5
         min_word_len            = 1
-        #ngram_len               = 1
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-        #ngram_chars             = U+3000..U+2FA1F
+<?php print NGRAM_CONFIG; ?>
 }
 
 
 index note1
 {
         source                  = note
-        path                    = LOCALSTATEDIR/piler/sphinx/note1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/note1
-        docinfo                 = extern
+<?php if(SPHINX_VERSION < 300) { ?>
-	dict			= keywords
-        min_prefix_len          = 5
-        min_word_len            = 1
-        #ngram_len               = 1
-        #ngram_chars             = U+3000..U+2FA1F
-}
-
-
-index att1
-{
-        source                  = att
-        path                    = /var/piler/sphinx/att1
         docinfo                 = extern
         dict                    = keywords
-        min_prefix_len          = 6
+<?php } ?>
+        min_prefix_len          = 5
         min_word_len            = 1
-        ngram_len               = 1
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-        ngram_chars             = U+1100..U+2FA1F
+<?php print NGRAM_CONFIG; ?>
 }
 
+<?php if(RT) { ?>
+index piler1
+{
+        type = rt
+        path = /var/piler/sphinx/piler1
+        rt_mem_limit = 128M
+        rt_field = sender
+        rt_field = rcpt
+        rt_field = senderdomain
+        rt_field = rcptdomain
+        rt_field = subject
+        rt_field = body
+        rt_field = attachment_types
+        rt_attr_bigint = arrived
+        rt_attr_bigint = sent
+        rt_attr_uint = size
+        rt_attr_uint = direction
+        rt_attr_uint = folder
+        rt_attr_uint = attachments
+}
+<?php } ?>
 
 indexer
 {
-	mem_limit		= 256M
+        mem_limit               = 256M
 }
 
 
 searchd
 {
-	listen			= 127.0.0.1:9312
+        listen                  = 127.0.0.1:9312
-	listen			= 127.0.0.1:9306:mysql41
+        listen                  = 127.0.0.1:9306:mysql41
-	log			= /dev/null
+        log                     = /dev/null
-	binlog_path             = 
+        binlog_path             = /var/piler/sphinx
-	##query_log		=
+        binlog_max_log_size     = 16M
-	read_timeout		= 5
+        binlog_flush            = 2
-	max_children		= 30
+<?php if(RT) { ?>
-	pid_file		= /var/run/piler/searchd.pid
+        rt_flush_period         = 300
-	seamless_rotate		= 1
+<?php } ?>
-	preopen_indexes		= 1
+        ##query_log             =
-	unlink_old		= 1
+        read_timeout            = 5
-	thread_stack		= 512k
+        max_children            = 30
-	workers			= threads # for RT to work
+        pid_file                = /var/run/piler/searchd.pid
+        seamless_rotate         = 1
+        preopen_indexes         = 1
+        unlink_old              = 1
+        thread_stack            = 512k
+        workers                 = thread_pool
 }

init.d/rc.piler.in

@@ -15,27 +15,30 @@
 
 NAME=piler
 OPTIONS=""
-PID_FILE=`SBINDIR/pilerconf $OPTIONS -q pidfile | cut -f2 -d=`
+PID_FILE="$(SBINDIR/pilerconf $OPTIONS -q pidfile | cut -f2 -d=)"
-PID_NUMBER=`test -f ${PID_FILE} && cat ${PID_FILE}`
+PID_NUMBER="$(test -f "$PID_FILE" && cat "$PID_FILE")"
-PILER_SMTP_PID=$(ps uaxw | grep -w piler-smtp | grep -v grep | awk '{print $2}')
+PILER_SMTP_PID="$(pgrep piler-smtp)"
 
 start() {
-	echo "starting piler-smtp . . . "
+   echo "starting piler-smtp . . . "
-	SBINDIR/piler-smtp -d
+   SBINDIR/piler-smtp -d
 
-	echo "starting $NAME . . ."
+   echo "starting $NAME . . ."
-	SBINDIR/piler -d $OPTIONS
+   SBINDIR/piler -d $OPTIONS
 }
 
 stop() {
-	echo "stopping $NAME"
+   if [ "$PID_NUMBER" != "" ]; then echo "stopping piler"; kill "$PID_NUMBER"; fi
-	kill ${PID_NUMBER}
 
-	if [ $PILER_SMTP_PID != '' ]; then echo "stopping piler-smtp"; kill $PILER_SMTP_PID; fi
+   if [ "$PILER_SMTP_PID" != "" ]; then echo "stopping piler-smtp"; kill "$PILER_SMTP_PID"; fi
 }
 
 check_status(){
-        test -f /proc/${PID_NUMBER}/status
+        if [ -f "/proc/${PID_NUMBER}/status" ]; then
+           echo "piler is running, pid: ${PID_NUMBER}";
+        else
+           echo "piler is NOT running";
+        fi
 
         if [ "${PILER_SMTP_PID}" != '' ]; then
            echo "piler-smtp is running, pid: ${PILER_SMTP_PID}";
@@ -45,37 +48,34 @@ check_status(){
 }
 
 case "$1" in
-	start)
+        start)
-		start;
+                start;
-   	;;
+        ;;
 
-	stop)
+        stop)
-		stop;
+                stop;
-   	;;
+        ;;
 
         status)
-                if check_status;
+                if check_status; then
-                 then
+                   exit 0
-                    echo "${NAME} is running."
+                else
-                    exit 0
+                   exit 1
-                 else
-                    echo "${NAME} is not running."
-                    exit 1
                 fi
         ;;
 
-	restart)
+        restart)
-		stop;
+                stop;
-		sleep 1;
+                sleep 1;
-		start;
+                start;
-   	;;
+        ;;
 
-	reload)
+        reload)
-		kill -HUP $PID_NUMBER
+                kill -HUP "$PID_NUMBER"
-		echo "reloaded"
+                echo "reloaded"
-	;;
+        ;;
 
-	*)
+        *)
-		echo "Usage: $0 start|stop|restart|reload|status"
+                echo "Usage: $0 start|stop|restart|reload|status"
+        ;;
 esac
-

init.d/rc.searchd.in

@@ -16,7 +16,13 @@ export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
 
 NAME=searchd
 PID_FILE=/var/run/piler/searchd.pid
-PID_NUMBER=`test -f ${PID_FILE} && cat ${PID_FILE}`
+PID_NUMBER=$(test -f ${PID_FILE} && cat ${PID_FILE})
+CONFIG_FILE=sphinx.conf
+
+if [ -f SYSCONFDIR/piler/MANTICORE ]; then
+   CONFIG_FILE=manticore.conf
+fi
+
 
 start() {
 	echo "starting searchd . . ."
@@ -26,8 +32,8 @@ start() {
 		chown piler:piler /var/run/piler
 	fi
 
-        if [ `id -u` -eq 0 ]; then
+        if [ $(id -u) -eq 0 ]; then
-		su piler -c "searchd --config SYSCONFDIR/piler/sphinx.conf"
+		su piler -c "searchd --config SYSCONFDIR/piler/${CONFIG_FILE}"
 	else
 		searchd
 	fi
@@ -35,7 +41,7 @@ start() {
 
 stop() {
 	echo "stopping searchd"
-	kill ${PID_NUMBER}
+	kill "$PID_NUMBER"
 }
 
 check_status(){

piler-config.h.in

@@ -2,6 +2,8 @@
  * piler-config.h.in, SJ
  */
 
+#define VERSION "x.x.x"
+
 #define CONFDIR "/usr/local/etc"
 #define DATADIR "/usr/local/var"
 #define DATAROOTDIR "/usr/local/share"
@@ -24,9 +26,6 @@
 #undef HAVE_TNEF
 #undef HAVE_ZIP
 
-#undef HAVE_LIBWRAP
-
 #undef HAVE_TWEAK_SENT_TIME
 
 #undef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
-

src/Makefile.in

@@ -33,7 +33,7 @@ MAKE = `which make`
 
 INSTALL = @INSTALL@
 
-all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport reindex test piler-smtp
+all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport reindex test stats piler-smtp
 install: install-piler
 
 
@@ -43,12 +43,12 @@ piler: piler.c libpiler.a
 libpiler.a: $(OBJS) $(SQL_OBJS)
 	ar cr libpiler.a $(OBJS) $(SQL_OBJS)
 	ranlib libpiler.a
-	$(CC) -shared -o libpiler.so.$(LIBPILER_VERSION) $(OBJS) $(SQL_OBJS)
+	$(CC) -shared -Wl,-soname,libpiler.so.$(LIBPILER_VERSION) -o libpiler.so.$(LIBPILER_VERSION) $(OBJS) $(SQL_OBJS)
 	ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so
 	ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so.$(PILER_VERSION)
 
 piler-smtp: piler-smtp.c libpiler.a
-	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< cfg.o misc.o tai.o smtp.o session.o dirs.o sig.o bdat.o $(LIBS) $(LIBDIR)
+	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< cfg.o misc.o tai.o smtp.o session.o dirs.o sig.o bdat.o screen.o $(LIBS) $(LIBDIR)
 
 pilerget: pilerget.c libpiler.a
 	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
@@ -71,8 +71,11 @@ reindex: reindex.c libpiler.a
 test: test.c libpiler.a
 	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o pilertest $< -lpiler $(LIBS) $(LIBDIR) @LDFLAGS@
 
+stats: stats.c libpiler.a
+	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o pilerstats $< -lpiler $(LIBS) $(LIBDIR) @LDFLAGS@
+
 %.o: $(srcdir)/%.c
-	$(CC) $(CFLAGS) -fPIC $(INCDIR) $(DEFS) -c $< -o $@
+	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -c $< -o $@
 
 
 install-piler:
@@ -90,9 +93,10 @@ install-piler:
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerexport $(DESTDIR)$(bindir)
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) reindex $(DESTDIR)$(bindir)
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilertest $(DESTDIR)$(bindir)
+	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerstats $(DESTDIR)$(bindir)
 
 clean:
-	rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilertest reindex piler-smtp
+	rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilertest pilerstats reindex piler-smtp
 
 distclean: clean
 	rm -f Makefile

src/archive.c

@@ -16,6 +16,9 @@
 #include <syslog.h>
 #include <openssl/blowfish.h>
 #include <openssl/evp.h>
+#if OPENSSL_VERSION_MAJOR >= 3
+   #include <openssl/provider.h>
+#endif
 #include <zlib.h>
 #include <assert.h>
 #include <piler.h>
@@ -54,11 +57,14 @@ void zerr(int ret){
 
 int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
    int ret, pos=0;
-   unsigned have;
    z_stream strm;
    char *new_ptr;
    unsigned char out[REALLYBIGBUFSIZE];
 
+   /* expecting deflate with 32k window size (0x78) */
+   if(len > 0 && in[0] != 0x78)
+      return Z_DATA_ERROR;
+
    /* allocate inflate state */
 
    strm.zalloc = Z_NULL;
@@ -96,7 +102,7 @@ int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
                 return ret;
       }
 
-      have = REALLYBIGBUFSIZE - strm.avail_out;
+      unsigned have = REALLYBIGBUFSIZE - strm.avail_out;
 
       /*
        * write the uncompressed result either to stdout
@@ -131,11 +137,16 @@ int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
 }
 
 
-int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct __config *cfg){
+int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct config *cfg){
    int rc=0, n, olen, tlen, len, fd=-1;
    unsigned char *s=NULL, *addr=NULL, inbuf[REALLYBIGBUFSIZE];
    struct stat st;
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
    EVP_CIPHER_CTX ctx;
+#else
+   EVP_CIPHER_CTX *ctx=NULL;
+#endif
+   int blocklen;
 
 
    if(filename == NULL) return 1;
@@ -154,12 +165,47 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
       return 1;
    }
 
+   // The new encryption scheme uses piler id starting with 5000....
 
    if(cfg->encrypt_messages == 1){
+   #if OPENSSL_VERSION_NUMBER < 0x10100000L
       EVP_CIPHER_CTX_init(&ctx);
-      EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      if(strstr(filename, "/5000")){
+         rc = EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+      } else {
+         rc = EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      }
 
-      len = st.st_size+EVP_MAX_BLOCK_LENGTH;
+      if(!rc){
+         syslog(LOG_PRIORITY, "ERROR: EVP_DecryptInit_ex()");
+         goto CLEANUP;
+      }
+
+      blocklen = EVP_CIPHER_CTX_block_size(&ctx);
+   #else
+      ctx = EVP_CIPHER_CTX_new();
+      if(!ctx) goto CLEANUP;
+
+      EVP_CIPHER_CTX_init(ctx);
+      if(strstr(filename, "/5000")){
+         rc = EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+      } else {
+      #if OPENSSL_VERSION_MAJOR >= 3
+         OSSL_PROVIDER_load(NULL, "legacy");
+         OSSL_PROVIDER_load(NULL, "default");
+      #endif
+         rc = EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      }
+
+      if(!rc){
+         syslog(LOG_PRIORITY, "ERROR: EVP_DecryptInit_ex()");
+         goto CLEANUP;
+      }
+
+      blocklen = EVP_CIPHER_CTX_block_size(ctx);
+   #endif
+
+      len = st.st_size+blocklen;
 
       s = malloc(len);
 
@@ -172,7 +218,11 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
 
       while((n = read(fd, inbuf, sizeof(inbuf)))){
 
+      #if OPENSSL_VERSION_NUMBER < 0x10100000L
          if(!EVP_DecryptUpdate(&ctx, s+tlen, &olen, inbuf, n)){
+      #else
+         if(!EVP_DecryptUpdate(ctx, s+tlen, &olen, inbuf, n)){
+      #endif
             syslog(LOG_PRIORITY, "%s: EVP_DecryptUpdate()", filename);
             goto CLEANUP;
          }
@@ -181,14 +231,24 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
       }
 
 
+   #if OPENSSL_VERSION_NUMBER < 0x10100000L
       if(EVP_DecryptFinal(&ctx, s + tlen, &olen) != 1){
+   #else
+      if(EVP_DecryptFinal(ctx, s + tlen, &olen) != 1){
+   #endif
          syslog(LOG_PRIORITY, "%s: EVP_DecryptFinal()", filename);
          goto CLEANUP;
       }
 
 
       tlen += olen;
+
+      // old fileformat with static IV
       rc = inf(s, tlen, mode, buffer, dest);
+      // new fileformat, starting with blocklen bytes of garbage
+      if(rc != Z_OK && tlen >= blocklen){
+         rc = inf(s+blocklen, tlen-blocklen, mode, buffer, dest);
+      }
    }
    else {
       addr = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
@@ -201,17 +261,22 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
 
 
 CLEANUP:
-   if(fd != -1) close(fd);
+   if(fd != -1) close(fd); //-V547
    if(s) free(s);
-   if(cfg->encrypt_messages == 1) EVP_CIPHER_CTX_cleanup(&ctx);
+   if(cfg->encrypt_messages == 1)
+   #if OPENSSL_VERSION_NUMBER < 0x10100000L
+      EVP_CIPHER_CTX_cleanup(&ctx);
+   #else
+      if(ctx) EVP_CIPHER_CTX_free(ctx);
+   #endif
 
    return 0;
 }
 
 
-int retrieve_email_from_archive(struct session_data *sdata, struct __data *data, FILE *dest, struct __config *cfg){
+int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct config *cfg){
-   int i, attachments;
+   int attachments;
-   char *buffer=NULL, *saved_buffer, *p, filename[SMALLBUFSIZE], pointer[SMALLBUFSIZE];
+   char *buffer=NULL, *saved_buffer, *p, filename[SMALLBUFSIZE];
    struct ptr_array ptr_arr[MAX_ATTACHMENTS];
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
    struct stat st;
@@ -222,14 +287,14 @@ int retrieve_email_from_archive(struct session_data *sdata, struct __data *data,
       return 1;
    }
 
-   attachments = query_attachments(sdata, data, &ptr_arr[0]);
+   attachments = query_attachments(sdata, &ptr_arr[0]);
 
    if(attachments == -1){
       printf("problem querying the attachment of %s\n", sdata->ttmpfile);
       return 1;
    }
 
-   snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
+   snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, *(sdata->ttmpfile+24), *(sdata->ttmpfile+25), *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
    if(stat(filename, &st)){
       snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+RND_STR_LEN-6), *(sdata->ttmpfile+RND_STR_LEN-5), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
@@ -245,7 +310,8 @@ int retrieve_email_from_archive(struct session_data *sdata, struct __data *data,
       if(buffer){
          saved_buffer = buffer;
 
-         for(i=1; i<=attachments; i++){
+         for(int i=1; i<=attachments; i++){
+            char pointer[SMALLBUFSIZE];
             snprintf(pointer, sizeof(pointer)-1, "ATTACHMENT_POINTER_%s.a%d_XXX_PILER", sdata->ttmpfile, i);
 
             p = strstr(buffer, pointer);
@@ -255,7 +321,7 @@ int retrieve_email_from_archive(struct session_data *sdata, struct __data *data,
                buffer = p + strlen(pointer);
 
                if(strlen(ptr_arr[i].piler_id) == RND_STR_LEN){
-                  snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);
+                  snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, ptr_arr[i].piler_id[24], ptr_arr[i].piler_id[25], ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);
 
                #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
                   if(stat(filename, &st)){
@@ -280,5 +346,3 @@ int retrieve_email_from_archive(struct session_data *sdata, struct __data *data,
 
    return 0;
 }
-
-

src/attachment.c

@@ -16,35 +16,34 @@
 #include <piler.h>
 
 
-int store_attachments(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg){
+int store_attachments(struct session_data *sdata, struct parser_state *state, struct config *cfg){
-   uint64 id=0;
    int i, rc=1, found, affected_rows;
+   struct sql sql, sql2;
 
-
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE) == ERR) return rc;
-   if(prepare_sql_statement(sdata, &(data->stmt_insert_into_attachment_table), SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE) == ERR) return rc;
+   if(prepare_sql_statement(sdata, &sql2, SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE) == ERR) return rc;
-   if(prepare_sql_statement(sdata, &(data->stmt_get_attachment_id_by_signature), SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE) == ERR) return rc;
 
 
    for(i=1; i<=state->n_attachments; i++){
       found = 0;
-      id = 0;
+      uint64 id = 0;
 
       if(state->attachments[i].size > 0){
 
-         p_bind_init(data);
+         p_bind_init(&sql2);
 
-         data->sql[data->pos] = state->attachments[i].digest; data->type[data->pos] = TYPE_STRING; data->pos++;
+         sql2.sql[sql2.pos] = state->attachments[i].digest; sql2.type[sql2.pos] = TYPE_STRING; sql2.pos++;
-         data->sql[data->pos] = (char *)&(state->attachments[i].size); data->type[data->pos] = TYPE_LONG; data->pos++;
+         sql2.sql[sql2.pos] = (char *)&(state->attachments[i].size); sql2.type[sql2.pos] = TYPE_LONG; sql2.pos++;
 
-         if(p_exec_query(sdata, data->stmt_get_attachment_id_by_signature, data) == OK){
+         if(p_exec_stmt(sdata, &sql2) == OK){
 
-            p_bind_init(data);
+            p_bind_init(&sql2);
 
-            data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+            sql2.sql[sql2.pos] = (char *)&id; sql2.type[sql2.pos] = TYPE_LONGLONG; sql2.len[sql2.pos] = sizeof(uint64); sql2.pos++;
 
-            p_store_results(data->stmt_get_attachment_id_by_signature, data);
+            p_store_results(&sql2);
-            if(p_fetch_results(data->stmt_get_attachment_id_by_signature) == OK) found = 1;
+            if(p_fetch_results(&sql2) == OK) found = 1;
-            p_free_results(data->stmt_get_attachment_id_by_signature);
+            p_free_results(&sql2);
          }
 
          if(found == 0){
@@ -55,20 +54,20 @@ int store_attachments(struct session_data *sdata, struct parser_state *state, st
          }
 
 
-         p_bind_init(data);
+         p_bind_init(&sql);
 
-         data->sql[data->pos] = sdata->ttmpfile; data->type[data->pos] = TYPE_STRING; data->pos++;
+         sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-         data->sql[data->pos] = (char *)&i; data->type[data->pos] = TYPE_LONG; data->pos++;
+         sql.sql[sql.pos] = (char *)&i; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-         data->sql[data->pos] = state->attachments[i].digest; data->type[data->pos] = TYPE_STRING; data->pos++;
+         sql.sql[sql.pos] = state->attachments[i].digest; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-         data->sql[data->pos] = state->attachments[i].filename; data->type[data->pos] = TYPE_STRING; data->pos++;
+         sql.sql[sql.pos] = state->attachments[i].filename; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-         data->sql[data->pos] = state->attachments[i].type; data->type[data->pos] = TYPE_STRING; data->pos++;
+         sql.sql[sql.pos] = state->attachments[i].type; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-         data->sql[data->pos] = (char *)&(state->attachments[i].size); data->type[data->pos] = TYPE_LONG; data->pos++;
+         sql.sql[sql.pos] = (char *)&(state->attachments[i].size); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-         data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
+         sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
 
-         if(p_exec_query(sdata, data->stmt_insert_into_attachment_table, data) == ERR) goto CLOSE;
+         if(p_exec_stmt(sdata, &sql) == ERR) goto CLOSE;
 
 
-         affected_rows = p_get_affected_rows(data->stmt_insert_into_attachment_table);
+         affected_rows = p_get_affected_rows(&sql);
          if(affected_rows != 1){
             syslog(LOG_PRIORITY, "%s attachment sql error: affected rows: %d", sdata->ttmpfile, affected_rows);
             goto CLOSE;
@@ -84,69 +83,70 @@ int store_attachments(struct session_data *sdata, struct parser_state *state, st
    rc = 0;
 
 CLOSE:
-   close_prepared_statement(data->stmt_insert_into_attachment_table);
+   close_prepared_statement(&sql);
-   close_prepared_statement(data->stmt_get_attachment_id_by_signature);
+   close_prepared_statement(&sql2);
 
    return rc;
 }
 
 
-int query_attachment_pointers(struct session_data *sdata, struct __data *data, uint64 ptr, char *piler_id, int *id){
+int query_attachment_pointers(struct session_data *sdata, uint64 ptr, char *piler_id, int *id){
    int rc=0;
+   struct sql sql;
 
-   if(prepare_sql_statement(sdata, &(data->stmt_get_attachment_pointer), SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER) == ERR) return rc;
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER) == ERR) return rc;
 
+   p_bind_init(&sql);
 
-   p_bind_init(data);
+   sql.sql[sql.pos] = (char *)&ptr; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
 
-   data->sql[data->pos] = (char *)&ptr; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
+   if(p_exec_stmt(sdata, &sql) == OK){
 
-   if(p_exec_query(sdata, data->stmt_get_attachment_pointer, data) == OK){
+      p_bind_init(&sql);
 
-      p_bind_init(data);
+      sql.sql[sql.pos] = piler_id; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++;
+      sql.sql[sql.pos] = (char *)id; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++;
 
-      data->sql[data->pos] = piler_id; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = RND_STR_LEN; data->pos++;
+      p_store_results(&sql);
-      data->sql[data->pos] = (char *)id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
 
-      p_store_results(data->stmt_get_attachment_pointer, data);
+      if(p_fetch_results(&sql) == OK) rc = 1;
-
+      p_free_results(&sql);
-      if(p_fetch_results(data->stmt_get_attachment_pointer) == OK) rc = 1;
-      p_free_results(data->stmt_get_attachment_pointer);
    }
 
-   close_prepared_statement(data->stmt_get_attachment_pointer);
+   close_prepared_statement(&sql);
 
    return rc;
 }
 
 
-int query_attachments(struct session_data *sdata, struct __data *data, struct ptr_array *ptr_arr){
+int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr){
    int i, rc, id, attachments=0;
    uint64 ptr;
+   struct sql sql;
 
    for(i=0; i<MAX_ATTACHMENTS; i++) memset((char*)&ptr_arr[i], 0, sizeof(struct ptr_array));
 
-   if(prepare_sql_statement(sdata, &(data->stmt_query_attachment), SQL_PREPARED_STMT_QUERY_ATTACHMENT) == ERR) return attachments;
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_QUERY_ATTACHMENT) == ERR) return attachments;
 
-   p_bind_init(data);
+   p_bind_init(&sql);
 
-   data->sql[data->pos] = sdata->ttmpfile; data->type[data->pos] = TYPE_STRING; data->pos++;
+   sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
 
-   if(p_exec_query(sdata, data->stmt_query_attachment, data) == ERR) goto CLOSE;
+   if(p_exec_stmt(sdata, &sql) == ERR) goto CLOSE;
 
-   p_bind_init(data);
+   p_bind_init(&sql);
 
-   data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
+   sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++;
-   data->sql[data->pos] = (char *)&ptr; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+   sql.sql[sql.pos] = (char *)&ptr; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
 
-   p_store_results(data->stmt_query_attachment, data);
+   p_store_results(&sql);
 
-   while(p_fetch_results(data->stmt_query_attachment) == OK){
+   while(p_fetch_results(&sql) == OK){
 
       if(id > 0 && id < MAX_ATTACHMENTS){
          if(ptr > 0){
             ptr_arr[id].ptr = ptr;
-            rc = query_attachment_pointers(sdata, data, ptr, &(ptr_arr[id].piler_id[0]), &(ptr_arr[id].attachment_id));
+            rc = query_attachment_pointers(sdata, ptr, &(ptr_arr[id].piler_id[0]), &(ptr_arr[id].attachment_id));
             if(!rc){
                attachments = -1;
                goto CLOSE;
@@ -161,12 +161,10 @@ int query_attachments(struct session_data *sdata, struct __data *data, struct pt
       }
    }
 
-   p_free_results(data->stmt_query_attachment);
+   p_free_results(&sql);
 
 CLOSE:
-   close_prepared_statement(data->stmt_query_attachment);
+   close_prepared_statement(&sql);
 
    return attachments;
 }
-
-

src/av.h

@@ -20,6 +20,6 @@
 #define CLAMD_RESP_INFECTED "FOUND"
 #define CLAMD_RESP_ERROR "ERROR"
 
-int clamd_scan(char *tmpfile, struct __config *cfg);
+int clamd_scan(char *tmpfile, struct config *cfg);
 
 #endif /* _AV_H */

src/avir.c

@@ -10,7 +10,7 @@
 #include <piler.h>
 
 
-int do_av_check(char *filename, struct __config *cfg){
+int do_av_check(char *filename, struct config *cfg){
    int rav = AVIR_OK;
 
    if(clamd_scan(filename, cfg) == AV_VIRUS) rav = AVIR_VIRUS;

src/base64.c

@@ -1,79 +0,0 @@
-/*
- * base64.c, SJ
- */
-
-#include <stdio.h>
-#include <string.h>
-
-
-char base64_value(char c){
-   static const char *base64_table = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-
-   if((int)c > 63) return '=';
-
-   return base64_table[(int)c];
-}
-
-
-void base64_encode_block(unsigned char *in, int inlen, char *out){
-   char a, b, c, d, fragment;
-
-   sprintf(out, "====");
-
-   if(inlen <= 0) return;
-
-   fragment = *in & 0x3;
-
-   a = *in >> 2;
-
-   out[0] = base64_value(a);
-
-   b = fragment << 4;
-
-   if(inlen > 1)
-      b += *(in+1) >> 4;
-
-   out[1] = base64_value(b);
-
-   if(inlen == 1) return;
-
-
-   c = *(in+1) & 0xf;
-   c = c << 2;
-
-   if(inlen > 2){
-      fragment = *(in+2) & 0xfc;
-      c += fragment >> 6;
-
-      d = *(in+2) & 0x3f;
-      out[3] = base64_value(d);
-   }
-
-   out[2] = base64_value(c);
-}
-
-
-void base64_encode(unsigned char *in, int inlen, char *out, int outlen){
-   int i=0, j, pos=0;
-   unsigned char buf[3];
-
-   memset(buf, 0, 3);
-   memset(out, 0, outlen);
-
-   for(j=0; j<inlen; j++){
-
-      if(i == 3){
-         base64_encode_block(buf, 3, &out[pos]); pos += 4;
-
-         memset(buf, 0, 3);
-
-         i = 0;
-      }
-
-      buf[i] = *(in+j);
-      i++;       
-   }
-
-   base64_encode_block(buf, i, &out[pos]);
-}
-

src/bdat.c

@@ -21,67 +21,59 @@
 
 
 void reset_bdat_counters(struct smtp_session *session){
-   session->bdat_rounds = 0;
-   session->bdat_last_round = 0;
    session->bdat_bytes_to_read = 0;
    session->bad = 0;
 }
 
 
-void get_bdat_size_to_read(struct smtp_session *session, char *buf){
+void get_bdat_size_to_read(struct smtp_session *session){
    char *p;
 
-   session->bdat_rounds++;
    session->bdat_bytes_to_read = 0;
 
    session->protocol_state = SMTP_STATE_BDAT;
 
-   // determine if this is the last BDAT command
+   p = strcasestr(session->buf, " LAST");
-
-   p = strcasestr(buf, " LAST");
    if(p){
-      session->bdat_last_round = 1;
-      syslog(LOG_INFO, "%s: BDAT LAST", session->ttmpfile);
       *p = '\0';
    }
 
    // determine the size to be read
 
-   p = strchr(buf, ' ');
+   p = strchr(session->buf, ' ');
    if(p){
       session->bdat_bytes_to_read = atoi(p);
-      if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: BDAT len=%d", session->ttmpfile, session->bdat_bytes_to_read);
+      if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "fd=%d: BDAT len=%d", session->net.socket, session->bdat_bytes_to_read);
    }
 
    if(!p || session->bdat_bytes_to_read <= 0){
       session->bdat_bytes_to_read = 0;
-      syslog(LOG_INFO, "%s: malformed BDAT command", session->ttmpfile);
+      syslog(LOG_INFO, "%s: ERROR: malformed BDAT command", session->ttmpfile);
    }
 }
 
 
-void process_bdat(struct smtp_session *session, char *readbuf, int readlen){
+void process_bdat(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg){
-   int i;
-   char buf[SMALLBUFSIZE];
-
    if(readlen <= 0) return;
 
-   //printf("readbuf in process_bdat (%d): *%s*\n", readlen, readbuf);
+   if(session->fd == -1){
-
-   if(session->bdat_rounds == 1){
       session->fd = open(session->ttmpfile, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR|S_IRGRP);
       if(session->fd == -1){
          syslog(LOG_PRIORITY, "%s: %s", ERR_OPEN_TMP_FILE, session->ttmpfile);
       }
+
+      if(cfg->process_rcpt_to_addresses == 1) write_envelope_addresses(session, cfg);
    }
 
    session->bdat_bytes_to_read -= readlen;
 
    if(session->fd != -1){
-      write(session->fd, readbuf, readlen);
+      if(write(session->fd, readbuf, readlen) != -1){
-      session->tot_len += readlen;
+         session->tot_len += readlen;
 
-      if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: wrote %d bytes, %d bytes to go", session->ttmpfile, readlen, session->bdat_bytes_to_read);
+         if(session->cfg->verbosity >= _LOG_EXTREME) syslog(LOG_INFO, "%s: wrote %d bytes, %d bytes to go", session->ttmpfile, readlen, session->bdat_bytes_to_read);
+      }
+      else syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
    }
 
 
@@ -92,45 +84,34 @@ void process_bdat(struct smtp_session *session, char *readbuf, int readlen){
 
       close(session->fd);
       unlink(session->ttmpfile);
+
+      session->fd = -1;
    }
 
-   if(session->bdat_bytes_to_read == 0){
 
-      if(session->bdat_last_round == 1){
+   // If there's nothing more to read, then send response to smtp client
-         if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: read all bdat data in %d rounds", session->ttmpfile, session->bdat_rounds);
 
-         // send back the smtp answers
+   if(session->bdat_bytes_to_read <= 0){
-         for(i=0; i<session->bdat_rounds; i++){
-            if(session->fd == -1){
-               send_smtp_response(session, SMTP_RESP_421_ERR_WRITE_FAILED);
-            }
-            else {
-               if(i == 0){
-                  fsync(session->fd);
-                  close(session->fd);
 
-                  move_email(session);
+      if(session->fd == -1){
-
+         send_smtp_response(session, SMTP_RESP_421_ERR_WRITE_FAILED);
-                  snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
+      }
-                  send_smtp_response(session, buf);
+      else {
-                  syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d", session->ttmpfile, session->mailfrom, session->tot_len);
+         fsync(session->fd);
-               }
+         close(session->fd);
-               else send_smtp_response(session, SMTP_RESP_250_BDAT);
-            }
-         }
-
-         // technically we are not in the PERIOD state, but it's good enough
-         // to quit the BDAT processing state
-         session->protocol_state = SMTP_STATE_PERIOD;
 
          session->fd = -1;
+
+         move_email(session);
+
+         char buf[SMALLBUFSIZE];
+         snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
+         send_smtp_response(session, buf);
+         syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d, client=%s, fd=%d", session->ttmpfile, session->mailfrom, session->tot_len, session->remote_host, session->net.socket);
       }
 
-      else {
+      // technically we are not in the PERIOD state, but it's good enough
-         // this is not the last BDAT round, let's go back
+      // to quit the BDAT processing state
-         // after the rcpt state, and wait for the next
+      session->protocol_state = SMTP_STATE_PERIOD;
-         // BDAT command
-         session->protocol_state = SMTP_STATE_RCPT_TO;
-      }
    }
 }

src/cfg.c

@@ -17,30 +17,16 @@ int string_parser(char *src, char *target, int limit){
    return 0;
 };
 
-int multi_line_string_parser(char *src, char *target, unsigned int limit){
-   if(strlen(src) > 0 && strlen(target) + strlen(src) + 3 < limit){
-      strncat(target, src, limit-strlen(target));
-      strncat(target, "\r\n", limit-strlen(target));
-
-      return 0;
-   }
-
-   return 1;
-};
-
 int int_parser(char *src, int *target){
    *target = strtol(src, (char **) NULL, 10);
 
    return 0;
 };
 
-
+int uint64_parser(char *src, uint64 *target){
-int float_parser(char *src, float *target){
+   *target = strtoull(src, (char**)NULL, 10);
-   *target = strtof(src, (char **) NULL);
-
    return 0;
-};
+}
-
 
 struct _parse_rule {
    char *name;
@@ -58,61 +44,71 @@ struct _parse_rule {
 
 struct _parse_rule config_parse_rules[] =
 {
-
+   { "archive_address", "string", (void*) string_parser, offsetof(struct config, archive_address), "", MAXVAL-1},
-   { "archive_emails_not_having_message_id", "integer", (void*) int_parser, offsetof(struct __config, archive_emails_not_having_message_id), "0", sizeof(int)},
+   { "archive_emails_not_having_message_id", "integer", (void*) int_parser, offsetof(struct config, archive_emails_not_having_message_id), "0", sizeof(int)},
-   { "archive_only_mydomains", "integer", (void*) int_parser, offsetof(struct __config, archive_only_mydomains), "0", sizeof(int)},
+   { "archive_only_mydomains", "integer", (void*) int_parser, offsetof(struct config, archive_only_mydomains), "0", sizeof(int)},
-   { "backlog", "integer", (void*) int_parser, offsetof(struct __config, backlog), "20", sizeof(int)},
+   { "backlog", "integer", (void*) int_parser, offsetof(struct config, backlog), "20", sizeof(int)},
-   { "cipher_list", "string", (void*) string_parser, offsetof(struct __config, cipher_list), "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS", MAXVAL-1},
+   { "check_for_client_timeout_interval", "integer", (void*) int_parser, offsetof(struct config, check_for_client_timeout_interval), "20", sizeof(int)},
-   { "clamd_addr", "string", (void*) string_parser, offsetof(struct __config, clamd_addr), "", MAXVAL-1},
+   { "cipher_list", "string", (void*) string_parser, offsetof(struct config, cipher_list), "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS", MAXVAL-1},
-   { "clamd_port", "integer", (void*) int_parser, offsetof(struct __config, clamd_port), "0", sizeof(int)},
+   { "clamd_addr", "string", (void*) string_parser, offsetof(struct config, clamd_addr), "", MAXVAL-1},
-   { "clamd_socket", "string", (void*) string_parser, offsetof(struct __config, clamd_socket), CLAMD_SOCKET, MAXVAL-1},
+   { "clamd_port", "integer", (void*) int_parser, offsetof(struct config, clamd_port), "0", sizeof(int)},
-   { "debug", "integer", (void*) int_parser, offsetof(struct __config, debug), "0", sizeof(int)},
+   { "clamd_socket", "string", (void*) string_parser, offsetof(struct config, clamd_socket), CLAMD_SOCKET, MAXVAL-1},
-   { "default_retention_days", "integer", (void*) int_parser, offsetof(struct __config, default_retention_days), "2557", sizeof(int)},
+   { "debug", "integer", (void*) int_parser, offsetof(struct config, debug), "0", sizeof(int)},
-   { "enable_chunking", "integer", (void*) int_parser, offsetof(struct __config, enable_chunking), "0", sizeof(int)},
+   { "default_retention_days", "integer", (void*) int_parser, offsetof(struct config, default_retention_days), "2557", sizeof(int)},
-   { "enable_cjk", "integer", (void*) int_parser, offsetof(struct __config, enable_cjk), "0", sizeof(int)},
+   { "enable_chunking", "integer", (void*) int_parser, offsetof(struct config, enable_chunking), "0", sizeof(int)},
-   { "enable_folders", "integer", (void*) int_parser, offsetof(struct __config, enable_folders), "0", sizeof(int)},
+   { "enable_cjk", "integer", (void*) int_parser, offsetof(struct config, enable_cjk), "0", sizeof(int)},
-   { "encrypt_messages", "integer", (void*) int_parser, offsetof(struct __config, encrypt_messages), "1", sizeof(int)},
+   { "enable_folders", "integer", (void*) int_parser, offsetof(struct config, enable_folders), "0", sizeof(int)},
-   { "extra_to_field", "string", (void*) string_parser, offsetof(struct __config, extra_to_field), "", MAXVAL-1},
+   { "encrypt_messages", "integer", (void*) int_parser, offsetof(struct config, encrypt_messages), "1", sizeof(int)},
-   { "extract_attachments", "integer", (void*) int_parser, offsetof(struct __config, extract_attachments), "1", sizeof(int)},
+   { "extra_to_field", "string", (void*) string_parser, offsetof(struct config, extra_to_field), "", MAXVAL-1},
-   { "helper_timeout", "integer", (void*) int_parser, offsetof(struct __config, helper_timeout), "20", sizeof(int)},
+   { "extract_attachments", "integer", (void*) int_parser, offsetof(struct config, extract_attachments), "1", sizeof(int)},
-   { "hostid", "string", (void*) string_parser, offsetof(struct __config, hostid), HOSTID, MAXVAL-1},
+   { "helper_timeout", "integer", (void*) int_parser, offsetof(struct config, helper_timeout), "20", sizeof(int)},
-   { "iv", "string", (void*) string_parser, offsetof(struct __config, iv), "", MAXVAL-1},
+   { "hostid", "string", (void*) string_parser, offsetof(struct config, hostid), HOSTID, MAXVAL-1},
-   { "listen_addr", "string", (void*) string_parser, offsetof(struct __config, listen_addr), "0.0.0.0", MAXVAL-1},
+   { "iv", "string", (void*) string_parser, offsetof(struct config, iv), "", MAXVAL-1},
-   { "listen_port", "integer", (void*) int_parser, offsetof(struct __config, listen_port), "25", sizeof(int)},
+   { "listen_addr", "string", (void*) string_parser, offsetof(struct config, listen_addr), "0.0.0.0", MAXVAL-1},
-   { "locale", "string", (void*) string_parser, offsetof(struct __config, locale), "", MAXVAL-1},
+   { "listen_port", "integer", (void*) int_parser, offsetof(struct config, listen_port), "25", sizeof(int)},
-   { "max_connections", "integer", (void*) int_parser, offsetof(struct __config, max_connections), "64", sizeof(int)},
+   { "locale", "string", (void*) string_parser, offsetof(struct config, locale), "", MAXVAL-1},
-   { "max_requests_per_child", "integer", (void*) int_parser, offsetof(struct __config, max_requests_per_child), "1000", sizeof(int)},
+   { "max_connections", "integer", (void*) int_parser, offsetof(struct config, max_connections), "64", sizeof(int)},
-   { "memcached_servers", "string", (void*) string_parser, offsetof(struct __config, memcached_servers), "127.0.0.1", MAXVAL-1},
+   { "max_message_size", "integer", (void*) int_parser, offsetof(struct config, max_message_size), "50000000", sizeof(int)},
-   { "memcached_to_db_interval", "integer", (void*) int_parser, offsetof(struct __config, memcached_to_db_interval), "900", sizeof(int)},
+   { "max_requests_per_child", "integer", (void*) int_parser, offsetof(struct config, max_requests_per_child), "10000", sizeof(int)},
-   { "memcached_ttl", "integer", (void*) int_parser, offsetof(struct __config, memcached_ttl), "86400", sizeof(int)},
+   { "max_smtp_memory", "uint64", (void*) uint64_parser, offsetof(struct config, max_smtp_memory), "500000000", sizeof(uint64)},
-   { "min_message_size", "integer", (void*) int_parser, offsetof(struct __config, min_message_size), "100", sizeof(int)},
+   { "memcached_servers", "string", (void*) string_parser, offsetof(struct config, memcached_servers), "127.0.0.1", MAXVAL-1},
-   { "min_word_len", "integer", (void*) int_parser, offsetof(struct __config, min_word_len), "1", sizeof(int)},
+   { "memcached_to_db_interval", "integer", (void*) int_parser, offsetof(struct config, memcached_to_db_interval), "900", sizeof(int)},
-   { "mmap_dedup_test", "integer", (void*) int_parser, offsetof(struct __config, mmap_dedup_test), "0", sizeof(int)},
+   { "memcached_ttl", "integer", (void*) int_parser, offsetof(struct config, memcached_ttl), "86400", sizeof(int)},
-   { "mysqlcharset", "string", (void*) string_parser, offsetof(struct __config, mysqlcharset), "utf8mb4", MAXVAL-1},
+   { "min_message_size", "integer", (void*) int_parser, offsetof(struct config, min_message_size), "100", sizeof(int)},
-   { "mysqlhost", "string", (void*) string_parser, offsetof(struct __config, mysqlhost), "", MAXVAL-1},
+   { "min_word_len", "integer", (void*) int_parser, offsetof(struct config, min_word_len), "1", sizeof(int)},
-   { "mysqlport", "integer", (void*) int_parser, offsetof(struct __config, mysqlport), "", sizeof(int)},
+   { "mmap_dedup_test", "integer", (void*) int_parser, offsetof(struct config, mmap_dedup_test), "0", sizeof(int)},
-   { "mysqlsocket", "string", (void*) string_parser, offsetof(struct __config, mysqlsocket), "/tmp/mysql.sock", MAXVAL-1},
+   { "mysqlcharset", "string", (void*) string_parser, offsetof(struct config, mysqlcharset), "utf8mb4", MAXVAL-1},
-   { "mysqluser", "string", (void*) string_parser, offsetof(struct __config, mysqluser), "piler", MAXVAL-1},
+   { "mysqlhost", "string", (void*) string_parser, offsetof(struct config, mysqlhost), "", MAXVAL-1},
-   { "mysqlpwd", "string", (void*) string_parser, offsetof(struct __config, mysqlpwd), "", MAXVAL-1},
+   { "mysqlport", "integer", (void*) int_parser, offsetof(struct config, mysqlport), "", sizeof(int)},
-   { "mysqldb", "string", (void*) string_parser, offsetof(struct __config, mysqldb), "piler", MAXVAL-1},
+   { "mysqlsocket", "string", (void*) string_parser, offsetof(struct config, mysqlsocket), "/tmp/mysql.sock", MAXVAL-1},
-   { "mysql_connect_timeout", "integer", (void*) int_parser, offsetof(struct __config, mysql_connect_timeout), "2", sizeof(int)},
+   { "mysqluser", "string", (void*) string_parser, offsetof(struct config, mysqluser), "piler", MAXVAL-1},
-   { "number_of_worker_processes", "integer", (void*) int_parser, offsetof(struct __config, number_of_worker_processes), "2", sizeof(int)},
+   { "mysqlpwd", "string", (void*) string_parser, offsetof(struct config, mysqlpwd), "", MAXVAL-1},
-   { "pemfile", "string", (void*) string_parser, offsetof(struct __config, pemfile), "", MAXVAL-1},
+   { "mysqldb", "string", (void*) string_parser, offsetof(struct config, mysqldb), "piler", MAXVAL-1},
-   { "pidfile", "string", (void*) string_parser, offsetof(struct __config, pidfile), PIDFILE, MAXVAL-1},
+   { "mysql_connect_timeout", "integer", (void*) int_parser, offsetof(struct config, mysql_connect_timeout), "2", sizeof(int)},
-   { "piler_header_field", "string", (void*) string_parser, offsetof(struct __config, piler_header_field), "X-piler-id:", MAXVAL-1},
+   { "number_of_worker_processes", "integer", (void*) int_parser, offsetof(struct config, number_of_worker_processes), "2", sizeof(int)},
-   { "process_rcpt_to_addresses", "integer", (void*) int_parser, offsetof(struct __config, process_rcpt_to_addresses), "0", sizeof(int)},
+   { "pemfile", "string", (void*) string_parser, offsetof(struct config, pemfile), "", MAXVAL-1},
-   { "queuedir", "string", (void*) string_parser, offsetof(struct __config, queuedir), QUEUE_DIR, MAXVAL-1},
+   { "pidfile", "string", (void*) string_parser, offsetof(struct config, pidfile), PIDFILE, MAXVAL-1},
-   { "server_id", "integer", (void*) int_parser, offsetof(struct __config, server_id), "0", sizeof(int)},
+   { "piler_header_field", "string", (void*) string_parser, offsetof(struct config, piler_header_field), "X-piler-id:", MAXVAL-1},
-   { "smtp_timeout", "integer", (void*) int_parser, offsetof(struct __config, smtp_timeout), "60", sizeof(int)},
+   { "process_rcpt_to_addresses", "integer", (void*) int_parser, offsetof(struct config, process_rcpt_to_addresses), "0", sizeof(int)},
-   { "spam_header_line", "string", (void*) string_parser, offsetof(struct __config, spam_header_line), "", MAXVAL-1},
+   { "queuedir", "string", (void*) string_parser, offsetof(struct config, queuedir), QUEUE_DIR, MAXVAL-1},
-   { "syslog_recipients", "integer", (void*) int_parser, offsetof(struct __config, syslog_recipients), "0", sizeof(int)},
+   { "rtindex", "integer", (void*) int_parser, offsetof(struct config, rtindex), "0", sizeof(int)},
-   { "tls_enable", "integer", (void*) int_parser, offsetof(struct __config, tls_enable), "0", sizeof(int)},
+   { "security_header", "string", (void*) string_parser, offsetof(struct config, security_header), "", MAXVAL-1},
-   { "tweak_sent_time_offset", "integer", (void*) int_parser, offsetof(struct __config, tweak_sent_time_offset), "0", sizeof(int)},
+   { "server_id", "integer", (void*) int_parser, offsetof(struct config, server_id), "0", sizeof(int)},
-   { "update_counters_to_memcached", "integer", (void*) int_parser, offsetof(struct __config, update_counters_to_memcached), "0", sizeof(int)},
+   { "sphxdb", "string", (void*) string_parser, offsetof(struct config, sphxdb), "piler1", MAXVAL-1},
-   { "username", "string", (void*) string_parser, offsetof(struct __config, username), "piler", MAXVAL-1},
+   { "sphxhost", "string", (void*) string_parser, offsetof(struct config, sphxhost), "127.0.0.1", MAXVAL-1},
-   { "use_antivirus", "integer", (void*) int_parser, offsetof(struct __config, use_antivirus), "1", sizeof(int)},
+   { "sphxport", "integer", (void*) int_parser, offsetof(struct config, sphxport), "9306", sizeof(int)},
-   { "verbosity", "integer", (void*) int_parser, offsetof(struct __config, verbosity), "1", sizeof(int)},
+   { "smtp_access_list", "integer", (void*) int_parser, offsetof(struct config, smtp_access_list), "0", sizeof(int)},
-   { "workdir", "string", (void*) string_parser, offsetof(struct __config, workdir), WORK_DIR, MAXVAL-1},
+   { "smtp_timeout", "integer", (void*) int_parser, offsetof(struct config, smtp_timeout), "60", sizeof(int)},
+   { "spam_header_line", "string", (void*) string_parser, offsetof(struct config, spam_header_line), "", MAXVAL-1},
+   { "syslog_recipients", "integer", (void*) int_parser, offsetof(struct config, syslog_recipients), "0", sizeof(int)},
+   { "tls_enable", "integer", (void*) int_parser, offsetof(struct config, tls_enable), "0", sizeof(int)},
+   { "tls_min_version", "string", (void*) string_parser, offsetof(struct config, tls_min_version), "TLSv1.2", MAXVAL-1},
+   { "tweak_sent_time_offset", "integer", (void*) int_parser, offsetof(struct config, tweak_sent_time_offset), "0", sizeof(int)},
+   { "update_counters_to_memcached", "integer", (void*) int_parser, offsetof(struct config, update_counters_to_memcached), "0", sizeof(int)},
+   { "username", "string", (void*) string_parser, offsetof(struct config, username), "piler", MAXVAL-1},
+   { "use_antivirus", "integer", (void*) int_parser, offsetof(struct config, use_antivirus), "1", sizeof(int)},
+   { "verbosity", "integer", (void*) int_parser, offsetof(struct config, verbosity), "1", sizeof(int)},
+   { "workdir", "string", (void*) string_parser, offsetof(struct config, workdir), WORK_DIR, MAXVAL-1},
 
    {NULL, NULL, NULL, 0, 0, 0}
 };
@@ -122,7 +118,7 @@ struct _parse_rule config_parse_rules[] =
  * parse configfile
  */
 
-int parse_config_file(char *configfile, struct __config *target_cfg, struct _parse_rule *rules){
+int parse_config_file(char *configfile, struct config *target_cfg, struct _parse_rule *rules){
    char line[MAXVAL], *chpos;
    FILE *f;
 
@@ -162,7 +158,27 @@ int parse_config_file(char *configfile, struct __config *target_cfg, struct _par
 }
 
 
-int load_default_config(struct __config *cfg, struct _parse_rule *rules){
+int get_tls_protocol_number(char *protocol){
+   struct tls_protocol tls_protocols[] = {
+      { "TLSv1", TLS1_VERSION },
+      { "TLSv1.1", TLS1_1_VERSION },
+      { "TLSv1.2", TLS1_2_VERSION },
+   #ifdef TLS1_3_VERSION
+      { "TLSv1.3", TLS1_3_VERSION },
+   #endif
+   };
+
+   for(unsigned int i=0; i<sizeof(tls_protocols)/sizeof(struct tls_protocol); i++){
+      if(!strcmp(protocol, tls_protocols[i].proto)) {
+         return tls_protocols[i].version;
+      }
+   }
+
+   return 0;
+}
+
+
+int load_default_config(struct config *cfg, struct _parse_rule *rules){
    int i=0;
 
    while(rules[i].name){
@@ -178,12 +194,12 @@ int load_default_config(struct __config *cfg, struct _parse_rule *rules){
  * read configuration file variables
  */
 
-struct __config read_config(char *configfile){
+struct config read_config(char *configfile){
-   struct __config cfg;
+   struct config cfg;
 
    /* reset config structure and fill it with defaults */
 
-   memset((char *)&cfg, 0, sizeof(struct __config));
+   memset((char *)&cfg, 0, sizeof(struct config));
 
    load_default_config(&cfg, config_parse_rules);
 
@@ -194,6 +210,9 @@ struct __config read_config(char *configfile){
 
    cfg.hostid_len = strlen(cfg.hostid);
 
+   // Get the TLS protocol constant from string, ie. TLSv1.3 -> 772
+   cfg.tls_min_version_number = get_tls_protocol_number(cfg.tls_min_version);
+
    return cfg;
 }
 
@@ -202,9 +221,10 @@ struct __config read_config(char *configfile){
  * print a single configuration item as key=value
  */
 
-void print_config_item(struct __config *cfg, struct _parse_rule *rules, int i){
+void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
    int j;
    float f;
+   uint64 u;
    char *p, buf[MAXVAL];
 
    p = (char*)cfg + rules[i].offset;
@@ -213,6 +233,10 @@ void print_config_item(struct __config *cfg, struct _parse_rule *rules, int i){
       memcpy((char*)&j, p, sizeof(int));
       printf("%s=%d\n", rules[i].name, j);
    }
+   else if(strcmp(rules[i].type, "uint64") == 0){
+      memcpy((char*)&u, p, sizeof(uint64));
+      printf("%s=%llu\n", rules[i].name, u);
+   }
    else if(strcmp(rules[i].type, "float") == 0){
       memcpy((char*)&f, p, sizeof(float));
       printf("%s=%.4f\n", rules[i].name, f);
@@ -237,7 +261,7 @@ void print_config_item(struct __config *cfg, struct _parse_rule *rules, int i){
  * print all known configuration items
  */
 
-void print_config_all(struct __config *cfg, char *key){
+void print_config_all(struct config *cfg, char *key){
    int i=0;
    struct _parse_rule *rules;
 
@@ -261,7 +285,7 @@ void print_config_all(struct __config *cfg, char *key){
  * print all configuration items found in configfile
  */
 
-void print_config(char *configfile, struct __config *cfg){
+void print_config(char *configfile, struct config *cfg){
    FILE *f;
    char line[MAXVAL], *chpos, previtem[MAXVAL];
    struct _parse_rule *rules;
@@ -303,5 +327,3 @@ void print_config(char *configfile, struct __config *cfg){
 
    fclose(f);
 }
-
-

src/cfg.h

@@ -7,7 +7,7 @@
 
 #include "config.h"
 
-struct __config {
+struct config {
    int server_id;
    char username[MAXVAL];
 
@@ -29,6 +29,8 @@ struct __config {
    int tls_enable;
    char pemfile[MAXVAL];
    char cipher_list[MAXVAL];
+   char tls_min_version[MAXVAL];
+   int tls_min_version_number;
 
    int use_antivirus;
 
@@ -49,6 +51,7 @@ struct __config {
    int verbosity;
    char locale[MAXVAL];
 
+   int check_for_client_timeout_interval;
    int smtp_timeout;
    int helper_timeout;
    int extract_attachments;
@@ -63,6 +66,9 @@ struct __config {
 
    int default_retention_days;
 
+   char security_header[MAXVAL];
+   char archive_address[MAXVAL];
+
    // mysql stuff
 
    char mysqlcharset[MAXVAL];
@@ -74,6 +80,12 @@ struct __config {
    char mysqldb[MAXVAL];
    int mysql_connect_timeout;
 
+   // manticore stuff
+   char sphxhost[MAXVAL];
+   int sphxport;
+   char sphxdb[MAXVAL];
+   int rtindex;
+
    int update_counters_to_memcached;
    int memcached_to_db_interval;
 
@@ -96,6 +108,11 @@ struct __config {
    int enable_folders;
 
    int debug;
+
+   int smtp_access_list;
+
+   int max_message_size;
+   uint64 max_smtp_memory;
 };
 
 

src/clamd.c

@@ -18,9 +18,9 @@
 #include <piler.h>
 
 
-int clamd_scan(char *tmpfile, struct __config *cfg){
+int clamd_scan(char *tmpfile, struct config *cfg){
    int s, n;
-   char *p, *q, buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE];
+   char buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE];
    struct sockaddr_un server;
 
    chmod(tmpfile, 0644);
@@ -58,9 +58,9 @@ int clamd_scan(char *tmpfile, struct __config *cfg){
    if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: CLAMD DEBUG: %d %s", tmpfile, n, buf);
 
    if(strcasestr(buf, CLAMD_RESP_INFECTED)){
-      p = strchr(buf, ' ');
+      char *p = strchr(buf, ' ');
       if(p){
-         q = strrchr(p, ' ');
+         char *q = strrchr(p, ' ');
          if(q){
             *q = '\0';
             p++;

src/config.h

@@ -9,15 +9,17 @@
 #include "piler-config.h"
 #include "params.h"
 
-#define VERSION "1.3.0-epoll"
+typedef unsigned long long uint64;
 
-#define BUILD 977
+#define BUILD 1001
 
 #define HOSTID "mailarchiver"
 
 #define CONFIG_FILE CONFDIR "/piler/piler.conf"
+#define SMTP_ACL_FILE CONFDIR "/piler/smtp.acl"
 #define WORK_DIR DATADIR "/piler/tmp"
 #define QUEUE_DIR DATADIR "/piler/store"
+#define ERROR_DIR DATADIR "/piler/error"
 
 #define CLAMD_SOCKET "/tmp/clamd"
 
@@ -30,6 +32,7 @@
 #define SMALLBUFSIZE 512
 #define BIGBUFSIZE 131072
 #define REALLYBIGBUFSIZE 524288
+#define SMTPBUFSIZE 2048000
 #define TINYBUFSIZE 128
 #define MAXVAL 256
 #define RANDOM_POOL "/dev/urandom"
@@ -39,6 +42,20 @@
 #define IPLEN 16+1
 #define KEYLEN 56
 #define MIN_EMAIL_ADDRESS_LEN 9
+// Sphinx 3.x has an issue with tokens longer than 41 characters.
+//
+// When a regular user executes a query, then his default email address filter
+// causes the query to fail with the below error message, even when the query
+// itself is correct:
+//
+// SELECT id FROM main1,dailydelta1,delta1 WHERE MATCH(' ( (@sender thisisanextremelylongemailaddressyesareallylongoneyeahitolyouXaddressXcom ) | (@rcpt thisisanextremelylongemailaddressyesareallylongoneyeahitolyouXaddressXcom) ) ') ORDER BY `sent` DESC LIMIT 0,20 OPTION max_matches=1000'
+//
+// ERROR 1064 (42000): index dailydelta1,delta1,main1: syntax error, unexpected $end near ' '
+//
+// Note that we use 42, because the parser adds a trailing space to the tokens
+// See https://www.mailpiler.org/wiki/current:sphinx3 and
+// https://bitbucket.org/jsuto/piler/issues/1082/no-sphinx-results-with-long-email for more
+#define MAX_EMAIL_ADDRESS_SPHINX_LEN 42
 
 #define CRLF "\n"
 
@@ -58,10 +75,13 @@
 #define MEMCACHED_MSGS_STORED_SIZE MEMCACHED_CLAPF_PREFIX "stored_size"
 
 
+#define PILEREXPORT_BEGIN_MARK "x-exported-by-pilerexport: start\n"
+
 #define LOG_PRIORITY LOG_INFO
 
 #define _LOG_INFO 3
 #define _LOG_DEBUG 5
+#define _LOG_EXTREME 100
 
 #define MAX_RCPT_TO 128
 
@@ -105,8 +125,10 @@
 #define SQL_PREPARED_STMT_GET_FOLDER_ID              "SELECT `id` FROM " SQL_FOLDER_TABLE " WHERE `name`=? AND `parent_id`=?"
 #define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE   "INSERT INTO `" SQL_FOLDER_TABLE "` (`name`, `parent_id`) VALUES(?,?)"
 #define SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE  "UPDATE " SQL_METADATA_TABLE " SET reference=? WHERE message_id=? AND reference=''"
+#define SQL_PREPARED_STMT_GET_METADATA_REFERENCE     "SELECT COUNT(*) AS count FROM " SQL_METADATA_TABLE " WHERE reference=?"
 #define SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS        "SELECT id, type, username, password, server FROM " SQL_IMPORT_TABLE " WHERE started=0 ORDER BY id LIMIT 0,1"
 #define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE      "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `id`) VALUES(?,?)"
+#define SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE        "UPDATE " SQL_IMPORT_TABLE " SET started=?, status=?, imported=? WHERE id=?"
 
 /* Error codes */
 
@@ -136,4 +158,3 @@
 #define S_STATUS_ERROR "error"
 
 #endif /* _CONFIG_H */
-

src/counters.c

@@ -11,64 +11,66 @@
 #include <piler.h>
 
 
-struct counters load_counters(struct session_data *sdata, struct __data *data){
+struct counters load_counters(struct session_data *sdata){
    char buf[SMALLBUFSIZE];
    struct counters counters;
+   struct sql sql;
 
    bzero(&counters, sizeof(counters));
 
    snprintf(buf, SMALLBUFSIZE-1, "SELECT `rcvd`, `virus`, `duplicate`, `ignore`, `size`, `stored_size` FROM `%s`", SQL_COUNTER_TABLE);
 
 
-   if(prepare_sql_statement(sdata, &(data->stmt_generic), buf) == ERR) return counters;
+   if(prepare_sql_statement(sdata, &sql, buf) == ERR) return counters;
 
 
-   p_bind_init(data);
+   p_bind_init(&sql);
 
-   if(p_exec_query(sdata, data->stmt_generic, data) == OK){
+   if(p_exec_stmt(sdata, &sql) == OK){
 
-      p_bind_init(data);
+      p_bind_init(&sql);
 
-      data->sql[data->pos] = (char *)&counters.c_rcvd; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+      sql.sql[sql.pos] = (char *)&counters.c_rcvd; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-      data->sql[data->pos] = (char *)&counters.c_virus; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+      sql.sql[sql.pos] = (char *)&counters.c_virus; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-      data->sql[data->pos] = (char *)&counters.c_duplicate; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+      sql.sql[sql.pos] = (char *)&counters.c_duplicate; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-      data->sql[data->pos] = (char *)&counters.c_ignore; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+      sql.sql[sql.pos] = (char *)&counters.c_ignore; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-      data->sql[data->pos] = (char *)&counters.c_size; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+      sql.sql[sql.pos] = (char *)&counters.c_size; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-      data->sql[data->pos] = (char *)&counters.c_stored_size; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+      sql.sql[sql.pos] = (char *)&counters.c_stored_size; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
 
-      p_store_results(data->stmt_generic, data);
+      p_store_results(&sql);
-      p_fetch_results(data->stmt_generic);
+      p_fetch_results(&sql);
-      p_free_results(data->stmt_generic);
+      p_free_results(&sql);
    }
 
-   close_prepared_statement(data->stmt_generic);
+   close_prepared_statement(&sql);
 
    return counters;
 }
 
 
-void update_counters(struct session_data *sdata, struct __data *data, struct counters *counters, struct __config *cfg){
+void update_counters(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    char buf[MAXBUFSIZE];
 #ifdef HAVE_MEMCACHED
-   unsigned long long mc, rcvd;
+   unsigned long long mc;
    struct counters c;
-   char key[MAX_MEMCACHED_KEY_LEN];
    unsigned int flags=0;
+#endif
 
-   if(counters->c_virus + counters->c_duplicate + counters->c_ignore + counters->c_size + counters->c_stored_size <= 0) return;
+   if(counters->c_virus + counters->c_duplicate + counters->c_ignore + counters->c_size + counters->c_stored_size == 0) return;
 
+#ifdef HAVE_MEMCACHED
    if(cfg->update_counters_to_memcached == 1){
 
       /* increment counters to memcached */
 
-      if(memcached_increment(&(data->memc), MEMCACHED_MSGS_RCVD, strlen(MEMCACHED_MSGS_RCVD), counters->c_rcvd, &mc) == MEMCACHED_SUCCESS){
+      if(memcached_increment(&(data->memc), MEMCACHED_MSGS_RCVD, counters->c_rcvd, &mc) == MEMCACHED_SUCCESS){
-         rcvd = mc;
+         unsigned long long rcvd = mc;
 
-         if(counters->c_virus > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), counters->c_virus, &mc);
+         if(counters->c_virus > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_VIRUS, counters->c_virus, &mc);
-         if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), counters->c_duplicate, &mc);
+         if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_DUPLICATE, counters->c_duplicate, &mc);
-         if(counters->c_ignore > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), counters->c_ignore, &mc);
+         if(counters->c_ignore > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_IGNORE, counters->c_ignore, &mc);
-         if(counters->c_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_SIZE, strlen(MEMCACHED_MSGS_SIZE), counters->c_size, &mc);
+         if(counters->c_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_SIZE, counters->c_size, &mc);
-         if(counters->c_stored_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_STORED_SIZE, strlen(MEMCACHED_MSGS_STORED_SIZE), counters->c_stored_size, &mc);
+         if(counters->c_stored_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_STORED_SIZE, counters->c_stored_size, &mc);
 
 
          bzero(&c, sizeof(c));
@@ -76,6 +78,8 @@ void update_counters(struct session_data *sdata, struct __data *data, struct cou
          snprintf(buf, MAXBUFSIZE-1, "%s %s %s %s %s %s %s", MEMCACHED_MSGS_RCVD, MEMCACHED_MSGS_VIRUS, MEMCACHED_MSGS_DUPLICATE, MEMCACHED_MSGS_IGNORE, MEMCACHED_MSGS_SIZE, MEMCACHED_MSGS_STORED_SIZE, MEMCACHED_COUNTERS_LAST_UPDATE);
 
          if(memcached_mget(&(data->memc), buf) == MEMCACHED_SUCCESS){
+            char key[MAX_MEMCACHED_KEY_LEN];
+
             while((memcached_fetch_result(&(data->memc), &key[0], &buf[0], &flags))){
                if(!strcmp(key, MEMCACHED_MSGS_RCVD)) c.c_rcvd = strtoull(buf, NULL, 10);
                else if(!strcmp(key, MEMCACHED_MSGS_VIRUS)) c.c_virus = strtoull(buf, NULL, 10);
@@ -87,8 +91,8 @@ void update_counters(struct session_data *sdata, struct __data *data, struct cou
             }
 
 
-            if(sdata->now - mc > cfg->memcached_to_db_interval && c.c_rcvd > 0 && c.c_rcvd >= rcvd){
+            if(sdata->now - mc > (unsigned long long)cfg->memcached_to_db_interval && c.c_rcvd > 0 && c.c_rcvd >= rcvd){
-               snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_set(&(data->memc), MEMCACHED_COUNTERS_LAST_UPDATE, strlen(MEMCACHED_COUNTERS_LAST_UPDATE), buf, strlen(buf), 0, 0);
+               snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), "set", MEMCACHED_COUNTERS_LAST_UPDATE, buf, strlen(buf), 0, 0);
 
                snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET `rcvd`=%llu, `virus`=%llu, `duplicate`=%llu, `ignore`=%llu, `size`=%llu, `stored_size`=%llu", SQL_COUNTER_TABLE, c.c_rcvd, c.c_virus, c.c_duplicate, c.c_ignore, c.c_size, c.c_stored_size);
 
@@ -101,23 +105,21 @@ void update_counters(struct session_data *sdata, struct __data *data, struct cou
       }
       else {
 
-         c = load_counters(sdata, data);
+         c = load_counters(sdata);
 
-         snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), MEMCACHED_COUNTERS_LAST_UPDATE, strlen(MEMCACHED_COUNTERS_LAST_UPDATE), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), "add", MEMCACHED_COUNTERS_LAST_UPDATE, buf, strlen(buf), 0, 0);
 
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_virus + counters->c_virus); memcached_add(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_virus + counters->c_virus); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_VIRUS, buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_rcvd + counters->c_rcvd); memcached_add(&(data->memc), MEMCACHED_MSGS_RCVD, strlen(MEMCACHED_MSGS_RCVD), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_rcvd + counters->c_rcvd); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_RCVD, buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_duplicate + counters->c_duplicate); memcached_add(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_duplicate + counters->c_duplicate); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_DUPLICATE, buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_ignore + counters->c_ignore); memcached_add(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_ignore + counters->c_ignore); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_IGNORE, buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_size + counters->c_size); memcached_add(&(data->memc), MEMCACHED_MSGS_SIZE, strlen(MEMCACHED_MSGS_SIZE), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_size + counters->c_size); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_SIZE, buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_stored_size + counters->c_stored_size); memcached_add(&(data->memc), MEMCACHED_MSGS_STORED_SIZE, strlen(MEMCACHED_MSGS_STORED_SIZE), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_stored_size + counters->c_stored_size); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_STORED_SIZE, buf, strlen(buf), 0, 0);
       }
 
    }
    else {
 #endif
-      if(counters->c_virus + counters->c_duplicate + counters->c_ignore + counters->c_size + counters->c_stored_size <= 0) return;
-
       snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET `rcvd`=`rcvd`+%llu, `virus`=`virus`+%llu, `duplicate`=`duplicate`+%llu, `ignore`=`ignore`+%llu, `size`=`size`+%llu, `stored_size`=`stored_size`+%llu", SQL_COUNTER_TABLE, counters->c_rcvd, counters->c_virus, counters->c_duplicate, counters->c_ignore, counters->c_size, counters->c_stored_size);
       p_query(sdata, buf);
 
@@ -126,5 +128,3 @@ void update_counters(struct session_data *sdata, struct __data *data, struct cou
 #endif
 
 }
-
-

src/decoder.c

@@ -78,6 +78,8 @@ inline void utf8_encode_char(unsigned char c, unsigned char *buf, int buflen, in
        * U+0000..U+007F      00..7F
        * U+0080..U+07FF      C2..DF      80..BF
        * U+0800..U+0FFF      E0          A0..BF      80..BF
+       *
+       * FIXME: See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#G7404 for valid sequences
        */
 
       if(c <= 0x7F){
@@ -96,20 +98,6 @@ inline void utf8_encode_char(unsigned char c, unsigned char *buf, int buflen, in
 }
 
 
-void sanitiseBase64(char *s){
-   char *p1;
-
-   if(s == NULL) return;
-
-   for(; *s; s++){
-      if(b64[(unsigned int)(*s & 0xFF)] == 255){
-         for(p1 = s; p1[0] != '\0'; p1++)
-            p1[0] = p1[1];
-      }
-   }
-}
-
-
 inline static void pack_4_into_3(char *s, char *s2){
    int j, n[4], k1, k2;
 
@@ -155,7 +143,7 @@ int decodeBase64(char *p){
 
 
 int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
-   int i, len=0, decodedlen;
+   int i, len=0;
    char s[5], s2[3];
 
    if(plen < 4 || plen > blen)
@@ -164,7 +152,7 @@ int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
    for(i=0; i<plen; i+=4){
       memcpy(s, p+i, 4);
       s[4] = '\0';
-      decodedlen = 3;
+      int decodedlen = 3;
 
       /* safety check against abnormally long lines */
 
@@ -189,12 +177,11 @@ int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
 void decodeQP(char *p){
    unsigned int i;
    int k=0, a, b;
-   char c;
 
    if(p == NULL) return;
 
    for(i=0; i<strlen((char*)p); i++){
-      c = p[i];
+      char c = p[i];
 
       if(p[i] == '=' && isxdigit(p[i+1]) && isxdigit(p[i+2])){
          a = p[i+1];
@@ -204,9 +191,13 @@ void decodeQP(char *p){
 
          i += 2;
       }
+      else if(p[i] == '_'){
+         c = ' ';
+      }
 
       p[k] = c;
       k++;
+
    }
 
    p[k] = '\0';
@@ -323,7 +314,18 @@ int utf8_encode(char *inbuf, int inbuflen, char *outbuf, int outbuflen, char *en
 
    memset(outbuf, 0, outbuflen);
 
-   cd = iconv_open("utf-8", encoding);
+   // Iconv sometimes produces an invalid utf8 sequence for gb2312.
+   // The fix is to use cp936, instead of gb2312 encoding.
+   //
+   // If there will be more similar exceptions, then we have to use
+   // a more efficient lookup method
+
+   if(strcasecmp(encoding, "gb2312") == 0)
+      cd = iconv_open("utf-8", "cp936");
+   else if(strcasecmp(encoding, "ks_c_5601-1987") == 0)
+      cd = iconv_open("utf-8", "EUC-KR");
+   else
+      cd = iconv_open("utf-8", encoding);
 
    if(cd != (iconv_t)-1){
       inbytesleft = inbuflen;
@@ -339,4 +341,3 @@ int utf8_encode(char *inbuf, int inbuflen, char *outbuf, int outbuflen, char *en
 
    return ret;
 }
-

src/decoder.h

@@ -5,9 +5,6 @@
 #ifndef _DECODER_H
  #define _DECODER_H
 
-void base64_encode(unsigned char *in, int inlen, char *out, int outlen);
-
-void sanitiseBase64(char *s);
 int decodeBase64(char *p);
 int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen);
 void decodeQP(char *p);

src/defs.h

@@ -9,17 +9,12 @@
   #include <mysql.h>
   #include <mysqld_error.h>
 #endif
-#ifdef NEED_PSQL
-  #include <libpq-fe.h>
-#endif
 #ifdef HAVE_TRE
    #include <tre/tre.h>
    #include <tre/regex.h>
 #endif
-#ifdef HAVE_LIBWRAP
-   #include <tcpd.h>
-#endif
 
+#include <openssl/md5.h>
 #include <openssl/sha.h>
 #include <openssl/ssl.h>
 #include <netinet/in.h>
@@ -30,21 +25,24 @@
 #define MSG_BODY 0
 #define MSG_RECEIVED 1
 #define MSG_FROM 2
-#define MSG_TO 3
+#define MSG_SENDER 3
-#define MSG_CC 4
+#define MSG_TO 4
-#define MSG_SUBJECT 5
+#define MSG_CC 5
-#define MSG_CONTENT_TYPE 6
+#define MSG_SUBJECT 6
-#define MSG_CONTENT_TRANSFER_ENCODING 7
+#define MSG_CONTENT_TYPE 7
-#define MSG_CONTENT_DISPOSITION 8
+#define MSG_CONTENT_TRANSFER_ENCODING 8
-#define MSG_MESSAGE_ID 9
+#define MSG_CONTENT_DISPOSITION 9
-#define MSG_REFERENCES 10
+#define MSG_MESSAGE_ID 10
-#define MSG_RECIPIENT 11
+#define MSG_REFERENCES 11
+#define MSG_RECIPIENT 12
+#define MSG_ENVELOPE_TO 13
 
 #define MAXHASH 277
 
 #define BASE64_RATIO 1.33333333
 
-#define DIGEST_LENGTH SHA256_DIGEST_LENGTH
+#define DIGEST_LENGTH EVP_MAX_MD_SIZE
+#define DIGEST_HEX_LENGTH 2*DIGEST_LENGTH+1
 
 #define UNDEF 0
 #define READY 1
@@ -78,7 +76,7 @@ struct attachment {
    char type[TINYBUFSIZE];
    char shorttype[TINYBUFSIZE];
    char aname[TINYBUFSIZE];
-   char filename[TINYBUFSIZE];
+   char filename[SMALLBUFSIZE];
    char internalname[TINYBUFSIZE];
    char digest[2*DIGEST_LENGTH+1];
    char dumped;
@@ -99,6 +97,25 @@ struct node {
 };
 
 
+struct smtp_acl {
+   char network_str[BUFLEN];
+   in_addr_t low, high;
+   int prefix;
+   int rejected;
+   struct smtp_acl *r;
+};
+
+
+struct net {
+   int socket;
+   int use_ssl;
+   int starttls;
+   int timeout;
+   SSL_CTX *ctx;
+   SSL *ssl;
+};
+
+
 struct rule {
 #ifdef HAVE_TRE
    regex_t from;
@@ -156,8 +173,10 @@ struct parser_state {
    int qp;
    int htmltag;
    int style;
+   int meta_content_type;
    int skip_html;
    int has_to_dump;
+   int has_to_dump_whole_body;
    int fd;
    int b64fd;
    int mfd;
@@ -166,19 +185,20 @@ struct parser_state {
    int content_type_is_set;
    int pushed_pointer;
    int saved_size;
-   int writebufpos;
+   unsigned int writebufpos;
-   int abufpos;
+   unsigned int abufpos;
+   unsigned int received_header;
    char attachedfile[RND_STR_LEN+SMALLBUFSIZE];
    char message_id[SMALLBUFSIZE];
    char message_id_hash[2*DIGEST_LENGTH+1];
    char miscbuf[MAX_TOKEN_LEN];
    char qpbuf[MAX_TOKEN_LEN];
+   char receivedbuf[SMALLBUFSIZE];
    unsigned long n_token;
    unsigned long n_subject_token;
    unsigned long n_body_token;
    unsigned long n_chain_token;
 
-   char filename[TINYBUFSIZE];
    char type[TINYBUFSIZE];
    char charset[TINYBUFSIZE];
 
@@ -195,12 +215,15 @@ struct parser_state {
 
    char reference[SMALLBUFSIZE];
 
-   char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE];
+   char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_sender[SMALLBUFSIZE], b_sender_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE];
    char b_journal_to[MAXBUFSIZE];
 
-   int bodylen;
+   unsigned int bodylen;
-   int tolen;
+   unsigned int tolen;
-   int journaltolen;
+   unsigned int todomainlen;
+   unsigned int found_security_header;
+
+   long unsigned int journaltolen;
 
    int retention;
 };
@@ -230,11 +253,7 @@ struct session_data {
    int journal_envelope_length, journal_bottom_length;
    unsigned int sql_errno;
 #ifdef NEED_MYSQL
-   MYSQL mysql;
+   MYSQL mysql, sphx;
-#endif
-#ifdef NEED_PSQL
-   PGconn *psql;
-   char conninfo[SMALLBUFSIZE];
 #endif
 };
 
@@ -294,8 +313,25 @@ struct import {
    int keep_eml;
    int timeout;
    int cap_uidplus;
+   int fd;
    long total_size;
-   time_t started, updated, finished;
+   int dryrun;
+   int tot_msgs;
+   int port;
+   int seq;
+   int table_id;
+   int delay;
+   char *server;
+   char *username;
+   char *password;
+   char *database;
+   char *skiplist;
+   char *folder_imap;
+   char *folder_name;
+   char *mboxdir;
+   char *folder;
+   char filename[SMALLBUFSIZE];
+   time_t started, updated, finished, after, before;
 };
 
 
@@ -308,7 +344,7 @@ struct licence {
 };
 
 
-struct __data {
+struct data {
    int folder, quiet;
    char recursive_folder_names;
    char starttls[TINYBUFSIZE];
@@ -318,32 +354,6 @@ struct __data {
    struct licence licence;
    char *dedup;
    int child_serial;
-
-#ifdef NEED_MYSQL
-   MYSQL_STMT *stmt_generic;
-   MYSQL_STMT *stmt_get_meta_id_by_message_id;
-   MYSQL_STMT *stmt_insert_into_rcpt_table;
-   MYSQL_STMT *stmt_insert_into_sphinx_table;
-   MYSQL_STMT *stmt_insert_into_meta_table;
-   MYSQL_STMT *stmt_insert_into_attachment_table;
-   MYSQL_STMT *stmt_insert_into_folder_message_table;
-   MYSQL_STMT *stmt_get_attachment_id_by_signature;
-   MYSQL_STMT *stmt_get_attachment_pointer;
-   MYSQL_STMT *stmt_query_attachment;
-   MYSQL_STMT *stmt_get_folder_id;
-   MYSQL_STMT *stmt_insert_into_folder_table;
-   MYSQL_STMT *stmt_update_metadata_reference;
-   MYSQL_STMT *stmt_update_metadata;
-   MYSQL_STMT *stmt_select_from_meta_table;
-   MYSQL_STMT *stmt_select_non_referenced_attachments;
-#endif
-
-   char *sql[MAX_SQL_VARS];
-   int type[MAX_SQL_VARS];
-   int len[MAX_SQL_VARS];
-   unsigned long length[MAX_SQL_VARS];
-   my_bool is_null[MAX_SQL_VARS];
-   my_bool error[MAX_SQL_VARS];
    int pos;
 
 #ifdef HAVE_TRE
@@ -355,8 +365,27 @@ struct __data {
 #ifdef HAVE_MEMCACHED
    struct memcached_server memc;
 #endif
-   SSL_CTX *ctx;
+
-   SSL *ssl;
+   struct net *net;
+};
+
+
+#if !defined(MARIADB_BASE_VERSION) && !defined(MARIADB_VERSION_ID) && \
+  MYSQL_VERSION_ID >= 80001 && MYSQL_VERSION_ID != 80002
+typedef bool my_bool;
+#endif
+
+struct sql {
+#ifdef NEED_MYSQL
+   MYSQL_STMT *stmt;
+#endif
+   char *sql[MAX_SQL_VARS];
+   int type[MAX_SQL_VARS];
+   int len[MAX_SQL_VARS];
+   unsigned long length[MAX_SQL_VARS];
+   my_bool is_null[MAX_SQL_VARS];
+   my_bool error[MAX_SQL_VARS];
+   int pos;
 };
 
 
@@ -373,24 +402,30 @@ struct counters {
 struct smtp_session {
    char ttmpfile[SMALLBUFSIZE];
    char mailfrom[SMALLBUFSIZE];
-   char buf[SMALLBUFSIZE];
+   char rcptto[MAX_RCPT_TO][SMALLBUFSIZE];
-   char remote_host[INET6_ADDRSTRLEN];
+   char remote_host[INET6_ADDRSTRLEN+1];
+   char nullbyte;
    time_t lasttime;
    int protocol_state;
    int slot;
    int fd;
    int bad;
-   int buflen;
    int tot_len;
-   int bdat_rounds;
-   int bdat_last_round;
    int bdat_bytes_to_read;
-   int socket;
+   int num_of_rcpt_to;
-   struct __config *cfg;
+   struct config *cfg;
-   SSL_CTX *ctx;
+   struct net net;
-   SSL *ssl;
+   int max_message_size;
-   int use_ssl;
+   char *buf;
-   int starttls;
+   int buflen;
+   int bufsize;
+   int too_big;
+   int mail_size;
+};
+
+struct tls_protocol {
+   char *proto;
+   int version;
 };
 
 #endif /* _DEFS_H */

src/digest.c

@@ -33,26 +33,39 @@ int search_header_end(char *p, int n){
 }
 
 
-int make_digests(struct session_data *sdata, struct __config *cfg){
+int make_digests(struct session_data *sdata, struct config *cfg){
-   int i=0, n, fd, offset=3, hdr_len=0, len=0;
+   int n, fd, offset=3, hdr_len=0;
    char *body=NULL;
-   unsigned char buf[BIGBUFSIZE], md[DIGEST_LENGTH], md2[DIGEST_LENGTH];
+   unsigned char buf[BIGBUFSIZE];
-   SHA256_CTX context, context2;
+
+   EVP_MD_CTX *ctx, *ctx2;
+   const EVP_MD *md, *md2;
+   unsigned int i=0, md_len, md_len2;
+   unsigned char md_value[EVP_MAX_MD_SIZE], md_value2[EVP_MAX_MD_SIZE];
 
    memset(sdata->bodydigest, 0, 2*DIGEST_LENGTH+1);
    memset(sdata->digest, 0, 2*DIGEST_LENGTH+1);
-   SHA256_Init(&context);
-   SHA256_Init(&context2);
 
+   md = EVP_get_digestbyname("sha256");
+   md2 = EVP_get_digestbyname("sha256");
+   if(md == NULL || md2 == NULL){
+      syslog(LOG_PRIORITY, "ERROR: unknown message digest: sha256 in %s:%d", __func__, __LINE__);
+      return 1;
+   }
+
+   ctx = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx, md, NULL);
+
+   ctx2 = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx2, md2, NULL);
 
    fd = open(sdata->filename, O_RDONLY);
    if(fd == -1) return -1;
 
+   memset(buf, 0, sizeof(buf));
 
    while((n = read(fd, buf, sizeof(buf))) > 0){
-      len += n;
+      EVP_DigestUpdate(ctx2, buf, n);
-
-      SHA256_Update(&context2, buf, n);
 
       body = (char *)&buf[0];
 
@@ -69,7 +82,7 @@ int make_digests(struct session_data *sdata, struct __config *cfg){
       }
 
 
-      SHA256_Update(&context, body, n);
+      EVP_DigestUpdate(ctx, body, n);
 
       i++;
    }
@@ -78,59 +91,93 @@ int make_digests(struct session_data *sdata, struct __config *cfg){
 
    sdata->hdr_len = hdr_len;
 
-   SHA256_Final(md, &context);
+   EVP_DigestFinal_ex(ctx, md_value, &md_len);
-   SHA256_Final(md2, &context2);
+   EVP_MD_CTX_free(ctx);
+   EVP_DigestFinal_ex(ctx2, md_value2, &md_len2);
+   EVP_MD_CTX_free(ctx2);
 
-   for(i=0;i<DIGEST_LENGTH;i++){
+   for(i=0;i<md_len;i++){
-      snprintf(sdata->bodydigest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
+      snprintf(sdata->bodydigest + i*2, 3, "%02x", md_value[i]);
-      snprintf(sdata->digest + i*2, 2*DIGEST_LENGTH, "%02x", md2[i]);
+   }
+
+   for(i=0;i<md_len2;i++){
+      snprintf(sdata->digest + i*2, 3, "%02x", md_value2[i]);
    }
 
    return 0;
 }
 
 
-void digest_file(char *filename, char *digest){
+void raw_digest_file(char *digestname, char *filename, unsigned char *md_value){
-   int fd, i, n;
+   int fd, n;
-   unsigned char buf[MAXBUFSIZE], md[DIGEST_LENGTH];
+   unsigned char buf[MAXBUFSIZE];
-   SHA256_CTX context;
+   EVP_MD_CTX *ctx;
+   const EVP_MD *md;
+   unsigned int md_len;
 
-   memset(digest, 0, 2*DIGEST_LENGTH+1);
+   md = EVP_get_digestbyname(digestname);
+   if(md == NULL){
+      syslog(LOG_PRIORITY, "ERROR: unknown message digest: '%s' in %s:%d", digestname, __func__, __LINE__);
+      return;
+   }
 
    fd = open(filename, O_RDONLY);
    if(fd == -1) return;
 
-   SHA256_Init(&context);
+   ctx = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx, md, NULL);
 
    while((n = read(fd, buf, sizeof(buf))) > 0){
-      SHA256_Update(&context, buf, n);
+      EVP_DigestUpdate(ctx, buf, n);
    }
 
    close(fd);
 
-   SHA256_Final(md, &context);
+   EVP_DigestFinal_ex(ctx, md_value, &md_len);
-
+   EVP_MD_CTX_free(ctx);
-   for(i=0;i<DIGEST_LENGTH;i++)
-      snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
-
 }
 
 
-void digest_string(char *s, char *digest){
+void digest_file(char *filename, char *digest){
-   int i;
    unsigned char md[DIGEST_LENGTH];
-   SHA256_CTX context;
+
+   raw_digest_file("sha256", filename, &md[0]);
 
    memset(digest, 0, 2*DIGEST_LENGTH+1);
 
-   SHA256_Init(&context);
+   for(int i=0;i<SHA256_DIGEST_LENGTH;i++){
-
-   SHA256_Update(&context, s, strlen(s));
-
-   SHA256_Final(md, &context);
-
-   for(i=0;i<DIGEST_LENGTH;i++)
       snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
-
+   }
 }
 
+
+void digest_string(char *digestname, char *s, char *digest){
+   EVP_MD_CTX *ctx;
+   const EVP_MD *md;
+   unsigned int i, md_len;
+   unsigned char md_value[DIGEST_LENGTH];
+
+   memset(digest, 0, 2*DIGEST_LENGTH+2);
+
+   md = EVP_get_digestbyname(digestname);
+   if(md == NULL){
+      syslog(LOG_PRIORITY, "ERROR: unknown message digest: '%s' in %s:%d", digestname, __func__, __LINE__);
+      return;
+   }
+
+   ctx = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx, md, NULL);
+   EVP_DigestUpdate(ctx, s, strlen(s));
+   EVP_DigestFinal_ex(ctx, md_value, &md_len);
+   EVP_MD_CTX_free(ctx);
+
+   for(i=0;i<md_len;i++){
+      snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md_value[i]);
+   }
+}
+
+
+void create_md5_from_email_address(char *puf, char *md5buf){
+   digest_string("md5", puf, md5buf);
+   md5buf[2*MD5_DIGEST_LENGTH] = ' ';
+}

src/dirs.c

@@ -13,10 +13,25 @@
 #include <piler.h>
 
 
-void createdir(char *path, uid_t uid, gid_t gid, mode_t mode);
+void createdir(char *path, uid_t uid, gid_t gid, mode_t mode){
+   struct stat st;
+
+   if(strlen(path) > 2){
+      if(path[strlen(path)-1] == '/') path[strlen(path)-1] = '\0';
+
+      if(stat(path, &st)){
+         if(mkdir(path, mode) == 0){
+            if(chown(path, uid, gid))
+               syslog(LOG_PRIORITY, "ERROR: createdir(): chown() failed on %s", path);
+            syslog(LOG_PRIORITY, "created directory: *%s*", path);
+         }
+         else syslog(LOG_PRIORITY, "ERROR: could not create directory: *%s*", path);
+      }
+   }
+}
 
 
-void check_and_create_directories(struct __config *cfg, uid_t uid, gid_t gid){
+void check_and_create_directories(struct config *cfg, uid_t uid, gid_t gid){
    char *p, s[SMALLBUFSIZE];
    int i;
 
@@ -56,23 +71,3 @@ void check_and_create_directories(struct __config *cfg, uid_t uid, gid_t gid){
    }
 
 }
-
-
-void createdir(char *path, uid_t uid, gid_t gid, mode_t mode){
-   struct stat st;
-
-   if(strlen(path) > 2){
-      if(path[strlen(path)-1] == '/') path[strlen(path)-1] = '\0';
-
-      if(stat(path, &st)){
-         if(mkdir(path, mode) == 0){
-            chown(path, uid, gid);
-            syslog(LOG_PRIORITY, "created directory: *%s*", path);
-         }
-         else syslog(LOG_PRIORITY, "could not create directory: *%s*", path);
-      }
-
-   }
-}
-
-

src/errmsg.h

@@ -20,6 +20,7 @@
 #define ERR_TIMED_OUT "ERR: timed out"
 #define ERR_FORK_FAILED "ERR: cannot fork()"
 #define ERR_OPEN_DEDUP_FILE "ERR: cannot open dedup file"
+#define ERR_PID_FILE_EXISTS "ERR: pidfile exists. If piler daemon is not running, and the pidfile exists (unclean shutdown?) then remove it, and start piler again"
 
 #define ERR_MYSQL_CONNECT "Cannot connect to mysql server"
 #define ERR_PSQL_CONNECT "Cannot connect to PSql server"

src/extract.c

@@ -19,40 +19,38 @@
 #define die(e) do { syslog(LOG_INFO, "error: helper: %s", e); exit(EXIT_FAILURE); } while (0);
 
 
-void remove_xml(char *buf, int *html){
+int remove_xml(char *src, char *dest, int destlen, int *html){
    int i=0;
-   char *p;
 
-   p = buf;
+   memset(dest, 0, destlen);
 
-   for(; *p; p++){
+   for(; *src; src++){
-      if(*p == '<'){ *html = 1; }
+      if(*src == '<'){
-
+         *html = 1;
-      if(*html == 0){
-         *(buf+i) = *p;
-         i++;
       }
-
+      else if(*src == '>'){
-      if(*p == '>'){
          *html = 0;
-
+         // make sure there's a whitespace between tag contents
-         if(i > 2 && *(buf+i-1) != ' '){
+         if(i < destlen && i > 0 && !isspace(dest[i-1]))
-            *(buf+i) = ' '; i++;
+            dest[i++] = ' ';
-         }
+      }
-
+      else{
+         if(*html == 0){
+            if(i < destlen) *(dest+i) = *src;
+            i++;
+         }
       }
-
    }
 
-   *(buf+i) = '\0';
+   return i;
 }
 
 
 #ifdef HAVE_ZIP
 int extract_opendocument(struct session_data *sdata, struct parser_state *state, char *filename, char *prefix){
-   int errorp, i=0, len=0, html=0;
+   int errorp, i=0, html=0;
-   int len2;
+   unsigned int len2;
-   char buf[MAXBUFSIZE];
+   char buf[4*MAXBUFSIZE], puf[4*MAXBUFSIZE];
    struct zip *z;
    struct zip_stat sb;
    struct zip_file *zf;
@@ -70,13 +68,13 @@ int extract_opendocument(struct session_data *sdata, struct parser_state *state,
 
          zf = zip_fopen_index(z, i, 0);
          if(zf){
+            int len;
             while((len = zip_fread(zf, buf, sizeof(buf)-2)) > 0){
 
-               remove_xml(buf, &html);
+               len2 = remove_xml(buf, puf, sizeof(puf), &html);
-               len2 = strlen(buf);
 
                if(len2 > 0 && state->bodylen < BIGBUFSIZE-len2-1){
-                  memcpy(&(state->b_body[state->bodylen]), buf, len2);
+                  memcpy(&(state->b_body[state->bodylen]), puf, len2);
                   state->bodylen += len2;
                }
 
@@ -99,8 +97,8 @@ int extract_opendocument(struct session_data *sdata, struct parser_state *state,
 }
 
 
-int unzip_file(struct session_data *sdata, struct parser_state *state, char *filename, int *rec, struct __config *cfg){
+int unzip_file(struct session_data *sdata, struct parser_state *state, char *filename, int *rec, struct config *cfg){
-   int errorp, i=0, len=0, fd;
+   int errorp, i=0, fd;
    char *p, extracted_filename[SMALLBUFSIZE], buf[MAXBUFSIZE];
    struct zip *z;
    struct zip_stat sb;
@@ -131,8 +129,9 @@ int unzip_file(struct session_data *sdata, struct parser_state *state, char *fil
             if(fd != -1){
                zf = zip_fopen_index(z, i, 0);
                if(zf){
+                  int len;
                   while((len = zip_fread(zf, buf, sizeof(buf))) > 0){
-                     write(fd, buf, len);
+                     if(write(fd, buf, len) == -1) syslog(LOG_PRIORITY, "ERROR: error writing to fd in %s", __func__);
                   }
                   zip_fclose(zf);
                }
@@ -169,13 +168,13 @@ int unzip_file(struct session_data *sdata, struct parser_state *state, char *fil
 
 #ifdef HAVE_TNEF
 
-int extract_tnef(struct session_data *sdata, struct parser_state *state, char *filename, struct __config *cfg){
+int extract_tnef(struct session_data *sdata, struct parser_state *state, char *filename, struct config *cfg){
    int rc=0, n, rec=1;
    char tmpdir[BUFLEN], buf[SMALLBUFSIZE];
    struct dirent **namelist;
 
    memset(tmpdir, 0, sizeof(tmpdir));
-   make_random_string(&tmpdir[0], sizeof(tmpdir)-3);
+   make_random_string((unsigned char *)&tmpdir[0], sizeof(tmpdir)-3);
 
    memcpy(&tmpdir[sizeof(tmpdir)-3], ".d", 2);
 
@@ -183,7 +182,7 @@ int extract_tnef(struct session_data *sdata, struct parser_state *state, char *f
 
    snprintf(buf, sizeof(buf)-1, "%s --unix-paths -C %s %s", HAVE_TNEF, tmpdir, filename);
 
-   system(buf);
+   if(system(buf) == -1) syslog(LOG_INFO, "error: running %s", buf);
 
    n = scandir(tmpdir, &namelist, NULL, alphasort);
    if(n < 0) syslog(LOG_INFO, "error: reading %s", tmpdir);
@@ -216,10 +215,9 @@ void kill_helper(){
 }
 
 
-void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct __config *cfg){
+void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct config *cfg){
-   int link[2], n;
+   int link[2];
    pid_t pid;
-   char outbuf[MAXBUFSIZE];
 
    if(strcmp(type, "other") == 0 || strcmp(type, "text") == 0) return;
 
@@ -314,18 +312,19 @@ void extract_attachment_content(struct session_data *sdata, struct parser_state
    }
    else {
       close(link[1]);
+      ssize_t n;
+      char outbuf[MAXBUFSIZE];
+
       while((n = read(link[0], outbuf, sizeof(outbuf))) > 0){
          if(state->bodylen < BIGBUFSIZE-n-1){
             memcpy(&(state->b_body[state->bodylen]), outbuf, n);
             state->bodylen += n;
          }
-         //printf("Output: %.*s\n", n, outbuf);
       }
+      close(link[0]);
       wait(NULL);
       return;
    }
 
 
 }
-
-

src/hash.c

@@ -33,12 +33,10 @@ void clearhash(struct node *xhash[]){
          p = q;
 
          q = q->r;
-         if(p){
+         if(p->str){
-            if(p->str){
+            free(p->str);
-               free(p->str);
-            }
-            free(p);
          }
+         free(p);
       }
       xhash[i] = NULL;
    }
@@ -154,13 +152,12 @@ int is_substr_in_hash(struct node *xhash[], char *s){
 
 
 unsigned int DJBHash(char* str, unsigned int len){
-   unsigned int hash = 5381;
+   unsigned int hashval = 5381;
    unsigned int i    = 0;
 
    for(i=0; i < len; str++, i++){
-      hash = ((hash << 5) + hash) + (*str);
+      hashval = ((hashval << 5) + hashval) + (*str);
    }
 
-   return hash;
+   return hashval;
 }
-

src/htmlentities.h

@@ -285,4 +285,3 @@ struct mi htmlentities[] = {
 
 
 #endif /* _HTMLENTITIES_H */
-

src/imap.c

@@ -23,9 +23,6 @@
 #include <piler.h>
 
 
-void update_import_job_stat(struct session_data *sdata, struct __data *data);
-
-
 int get_message_length_from_imap_answer(char *s){
    char *p, *q;
    int len=0;
@@ -54,18 +51,18 @@ int get_message_length_from_imap_answer(char *s){
 }
 
 
-int read_response(int sd, char *buf, int buflen, int *seq, struct __data *data, int use_ssl){
+int read_response(char *buf, int buflen, struct data *data){
    int i=0, n, len=0, rc=0;
    char puf[MAXBUFSIZE], tagok[SMALLBUFSIZE], tagno[SMALLBUFSIZE], tagbad[SMALLBUFSIZE];
 
-   snprintf(tagok, sizeof(tagok)-1, "A%d OK", *seq);
+   snprintf(tagok, sizeof(tagok)-1, "A%d OK", data->import->seq);
-   snprintf(tagno, sizeof(tagno)-1, "A%d NO", *seq);
+   snprintf(tagno, sizeof(tagno)-1, "A%d NO", data->import->seq);
-   snprintf(tagbad, sizeof(tagbad)-1, "A%d BAD", *seq);
+   snprintf(tagbad, sizeof(tagbad)-1, "A%d BAD", data->import->seq);
 
    memset(buf, 0, buflen);
 
    while(!strstr(buf, tagok)){
-      n = recvtimeoutssl(sd, puf, sizeof(puf), data->import->timeout, use_ssl, data->ssl);
+      n = recvtimeoutssl(data->net, puf, sizeof(puf));
 
       if(n + len < buflen) strncat(buf, puf, n);
       else goto END;
@@ -87,28 +84,74 @@ int read_response(int sd, char *buf, int buflen, int *seq, struct __data *data,
 
 END:
 
-   (*seq)++;
+   (data->import->seq)++;
 
    return rc;
 }
 
 
-int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg){
+int connect_to_imap_server(struct data *data){
-   int rc=ERR, i, n, messages=0, len, readlen, fd, nreads, readpos, finished, msglen, msg_written_len, tagoklen, tagbadlen, result;
+   char buf[MAXBUFSIZE];
-   char *p, tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], tagbad[SMALLBUFSIZE], buf[MAXBUFSIZE], puf[MAXBUFSIZE], filename[SMALLBUFSIZE];
 
-   /* imap cmd: SELECT */
+   data->import->cap_uidplus = 0;
+
+   if(data->net->use_ssl == 1){
+      init_ssl_to_server(data);
+   }
+
+
+   recvtimeoutssl(data->net, buf, sizeof(buf));
+
+
+   /* imap cmd: LOGIN */
+
+   if(strcmp(data->import->username, "ZIMBRA") == 0){
+      snprintf(buf, sizeof(buf)-1, "A%d AUTHENTICATE PLAIN %s\r\n", data->import->seq, data->import->password);
+   }
+   else {
+      snprintf(buf, sizeof(buf)-1, "A%d LOGIN %s \"%s\"\r\n", data->import->seq, data->import->username, data->import->password);
+   }
+
+   write1(data->net, buf, strlen(buf));
+   if(read_response(buf, sizeof(buf), data) == 0){
+      printf("login failed, server reponse: %s\n", buf);
+      return ERR;
+   }
+
+   if(strstr(buf, "UIDPLUS")){
+      data->import->cap_uidplus = 1;
+   }
+   else {
+
+      /* run the CAPABILITY command if the reply doesn't contain the UIDPLUS capability */
+
+      snprintf(buf, sizeof(buf)-1, "A%d CAPABILITY\r\n", data->import->seq);
+
+      write1(data->net, buf, strlen(buf));
+      read_response(buf, sizeof(buf), data);
+
+      if(strstr(buf, "UIDPLUS")) data->import->cap_uidplus = 1;
+   }
+
+
+   return OK;
+}
+
+
+int imap_select_cmd_on_folder(char *folder, struct data *data){
+   int messages=0;
+   char *p, buf[MAXBUFSIZE];
 
    if(strchr(folder, '"'))
-      snprintf(buf, sizeof(buf)-1, "A%d SELECT %s\r\n", *seq, folder);
+      snprintf(buf, sizeof(buf)-1, "A%d SELECT %s\r\n", data->import->seq, folder);
    else
-      snprintf(buf, sizeof(buf)-1, "A%d SELECT \"%s\"\r\n", *seq, folder);
+      snprintf(buf, sizeof(buf)-1, "A%d SELECT \"%s\"\r\n", data->import->seq, folder);
 
-   write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+   write1(data->net, buf, strlen(buf));
-   if(read_response(sd, buf, sizeof(buf), seq, data, use_ssl) == 0){
+   if(read_response(buf, sizeof(buf), data) == 0){
       trimBuffer(buf);
-      printf("select cmd error: %s\n", buf);
+      printf("ERROR: select cmd error: %s\n", buf);
-      return rc;
+      return messages;
    }
 
    p = strstr(buf, " EXISTS");
@@ -121,304 +164,241 @@ int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sda
       }
    }
 
-   printf("found %d messages\n", messages);
+   if(data->quiet == 0){printf("found %d messages\n", messages); }
-
-   if(messages <= 0) return OK;
-
-
-   if(data->recursive_folder_names == 1){
-      data->folder = get_folder_id(sdata, data, folder, 0);
-      if(data->folder == ERR_FOLDER) data->folder = add_new_folder(sdata, data, folder, 0);
-   }
-
 
    data->import->total_messages += messages;
 
+   return messages;
+}
+
+
+int imap_download_email(struct data *data, int i){
+   int fd, len, result, tagoklen, tagbadlen;
+   int n, readlen=0, nreads=0, readpos=0, finished=0, msglen=0, msg_written_len=0;
+   char *p, buf[MAXBUFSIZE], puf[MAXBUFSIZE], tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], tagbad[SMALLBUFSIZE];
+
+   data->import->processed_messages++;
+
+   snprintf(data->import->filename, SMALLBUFSIZE-1, "%d-imap-%d.txt", getpid(), data->import->processed_messages);
+
+   unlink(data->import->filename);
+
+   fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
+   if(fd == -1){
+      printf("cannot open: %s\n", data->import->filename);
+      return ERR;
+   }
+
+   snprintf(tag, sizeof(tag)-1, "A%d", data->import->seq);
+   snprintf(tagok, sizeof(tagok)-1, "A%d OK", (data->import->seq)++);
+   snprintf(tagbad, sizeof(tagbad)-1, "%s BAD", tag);
+
+   tagoklen = strlen(tagok);
+   tagbadlen = strlen(tagbad);
+
+   snprintf(buf, sizeof(buf)-1, "%s FETCH %d (BODY.PEEK[])\r\n", tag, i);
+   write1(data->net, buf, strlen(buf));
+
+   while((n = recvtimeoutssl(data->net, &buf[readpos], sizeof(buf)-readpos)) > 0){
+
+      readlen += n;
+
+      if(strchr(buf, '\n')){
+         readpos = 0;
+         p = &buf[0];
+         do {
+            nreads++;
+            memset(puf, 0, sizeof(puf));
+            p = split(p, '\n', puf, sizeof(puf)-1, &result);
+            len = strlen(puf);
+
+            if(result == 1){
+               // process a complete line
+
+               if(nreads == 1){
+
+                  if(strcasestr(puf, " FETCH ")){
+                     msglen = get_message_length_from_imap_answer(puf);
+
+                     if(msglen == 0){
+                        finished = 1;
+                        break;
+                     }
+                     continue;
+                  }
+
+                  if(strcasestr(puf, " BYE")){
+                     printf("imap server sent BYE response: '%s'\n", puf);
+                     close(fd);
+                     unlink(data->import->filename);
+                     return ERR;
+                  }
+
+               }
+
+               if(len > 0 && msg_written_len < msglen){
+                  if(write(fd, puf, len) == -1) printf("ERROR: writing to fd\n");
+                  if(write(fd, "\n", 1) == -1) printf("ERROR: writing to fd\n");
+                  msg_written_len += len + 1;
+               }
+
+               if(strncmp(puf, tagok, tagoklen) == 0){
+                  finished = 1;
+                  break;
+               }
+
+               if(strncmp(puf, tagbad, tagbadlen) == 0){
+                  printf("ERROR happened reading the message!\n");
+                  finished = 1;
+                  break;
+               }
+
+            }
+            else {
+               // prepend the last incomplete line back to 'buf'
+
+               snprintf(buf, sizeof(buf)-2, "%s", puf);
+               readpos = len;
+               break;
+            }
+
+         } while(p);
+
+
+      }
+      else {
+         readpos += n;
+      }
+
+      if(finished == 1) break;
+   }
+
+   close(fd);
+
+   if(msglen > 10) return OK;
+
+   return ERR;
+}
+
+
+void imap_delete_message(struct data *data, int i){
+   char buf[SMALLBUFSIZE];
+
+   snprintf(buf, sizeof(buf)-1, "A%d STORE %d +FLAGS.SILENT (\\Deleted)\r\n", data->import->seq, i);
+   write1(data->net, buf, strlen(buf));
+   read_response(buf, sizeof(buf), data);
+}
+
+
+void imap_move_message_to_folder(struct data *data, int i){
+   int tagoklen;
+   char buf[SMALLBUFSIZE], tagok[SMALLBUFSIZE];
+
+   snprintf(tagok, sizeof(tagok)-1, "A%d OK", data->import->seq);
+   tagoklen = strlen(tagok);
+
+   snprintf(buf, sizeof(buf)-1, "A%d COPY %d %s\r\n", data->import->seq, i, data->import->move_folder);
+   write1(data->net, buf, strlen(buf));
+   read_response(buf, sizeof(buf), data);
+
+   if(strncmp(buf, tagok, tagoklen) == 0){
+      snprintf(buf, sizeof(buf)-1, "A%d STORE %d +FLAGS.SILENT (\\Deleted)\r\n", data->import->seq, i);
+      write1(data->net, buf, strlen(buf));
+      read_response(buf, sizeof(buf), data);
+   }
+}
+
+
+void imap_expunge_message(struct data *data){
+   char buf[SMALLBUFSIZE];
+
+   snprintf(buf, sizeof(buf)-1, "A%d EXPUNGE\r\n", data->import->seq);
+   write1(data->net, buf, strlen(buf));
+   read_response(buf, sizeof(buf), data);
+}
+
+
+int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   int i, messages=0;
+
+   messages = imap_select_cmd_on_folder(folder, data);
+
+   if(messages <= 0) return OK;
+
+   if(data->recursive_folder_names == 1){
+      data->folder = get_folder_id(sdata, folder, 0);
+      if(data->folder == ERR_FOLDER) data->folder = add_new_folder(sdata, folder, 0);
+   }
+
    for(i=data->import->start_position; i<=messages; i++){
+      if(imap_download_email(data, i) == OK){
+         if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/messages); fflush(stdout); }
+
+         if(data->import->dryrun == 0){
+            int rc = import_message(sdata, data, counters, cfg);
+
+            if(data->import->remove_after_import == 1 && rc == OK){
+               imap_delete_message(data, i);
+            }
+
+            if(data->import->move_folder && data->import->cap_uidplus == 1){
+               imap_move_message_to_folder(data, i);
+            }
+         }
+
+         if(data->import->download_only == 0) unlink(data->import->filename);
+      }
 
       /* whether to quit after processing a batch of messages */
 
       if(data->import->batch_processing_limit > 0 && data->import->processed_messages >= data->import->batch_processing_limit){
          break;
       }
-
-      data->import->processed_messages++;
-      if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/messages); fflush(stdout); }
-
-      snprintf(tag, sizeof(tag)-1, "A%d", *seq);
-      snprintf(tagok, sizeof(tagok)-1, "A%d OK", (*seq)++);
-      snprintf(tagbad, sizeof(tagbad)-1, "%s BAD", tag);
-
-      tagoklen = strlen(tagok);
-      tagbadlen = strlen(tagbad);
-
-      snprintf(buf, sizeof(buf)-1, "%s FETCH %d (BODY.PEEK[])\r\n", tag, i);
-
-      snprintf(filename, sizeof(filename)-1, "%d-imap-%d.txt", getpid(), data->import->processed_messages);
-      unlink(filename);
-
-      fd = open(filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
-      if(fd == -1){
-         printf("cannot open: %s\n", filename);
-         return rc;
-      }
-
-
-      write1(sd, buf, strlen(buf), use_ssl, data->ssl);
-
-      readlen = 0;
-      nreads = 0;
-      readpos = 0;
-      finished = 0;
-      msglen = 0;
-      msg_written_len = 0;
-
-      while((n = recvtimeoutssl(sd, &buf[readpos], sizeof(buf)-readpos, data->import->timeout, use_ssl, data->ssl)) > 0){
-
-         readlen += n;
-
-         if(strchr(buf, '\n')){
-            readpos = 0;
-            p = &buf[0];
-            do {
-               nreads++;
-               memset(puf, 0, sizeof(puf));
-               p = split(p, '\n', puf, sizeof(puf)-1, &result);
-               len = strlen(puf);
-
-               if(result == 1){
-                  // process a complete line
-
-                  if(nreads == 1){
-
-                     if(strcasestr(puf, " FETCH ")){
-                        msglen = get_message_length_from_imap_answer(puf);
-
-                        if(msglen == 0){
-                           finished = 1;
-                           break;
-                        }
-                        continue;
-                     }
-
-                     if(strcasestr(puf, " BYE")){
-                        printf("imap server sent BYE response: '%s'\n", puf);
-                        close(fd);
-                        unlink(filename);
-                        return ERR;
-                     }
-
-                  }
- 
-                  if(len > 0 && msg_written_len < msglen){
-                     write(fd, puf, len);
-                     write(fd, "\n", 1);
-                     msg_written_len += len + 1;
-                  }
-
-                  if(strncmp(puf, tagok, tagoklen) == 0){
-                     finished = 1;
-                     break;
-                  }
-
-                  if(strncmp(puf, tagbad, tagbadlen) == 0){
-                     printf("ERROR happened reading the message!\n");
-                     finished = 1;
-                     break;
-                  }
-
-               }
-               else {
-                  // prepend the last incomplete line back to 'buf'
-
-                  snprintf(buf, sizeof(buf)-2, "%s", puf);
-                  readpos = len;
-                  break;
-               }
-
-            } while(p);
-
-
-
-         }
-         else {
-            readpos += n;
-         }
-
-         if(finished == 1) break;
-      }
-
-
-      close(fd);
-
-      if(dryrun == 0 && msglen > 10){
-         rc = import_message(filename, sdata, data, cfg);
-
-         if(data->import->processed_messages % 100 == 0){
-            time(&(data->import->updated));
-            update_import_job_stat(sdata, data);
-         }
-      }
-      else rc = OK;
-
-
-      if(rc == ERR) printf("error importing '%s'\n", filename);
-      else {
-
-         if(data->import->remove_after_import == 1 && dryrun == 0){
-            snprintf(buf, sizeof(buf)-1, "A%d STORE %d +FLAGS.SILENT (\\Deleted)\r\n", *seq, i);
-            write1(sd, buf, strlen(buf), use_ssl, data->ssl);
-            read_response(sd, buf, sizeof(buf), seq, data, use_ssl);
-         }
-
-
-         if(data->import->move_folder && data->import->cap_uidplus == 1 && dryrun == 0){
-
-            snprintf(tagok, sizeof(tagok)-1, "A%d OK", *seq);
-            tagoklen = strlen(tagok);
-
-            snprintf(buf, sizeof(buf)-1, "A%d COPY %d %s\r\n", *seq, i, data->import->move_folder);
-            write1(sd, buf, strlen(buf), use_ssl, data->ssl);
-            read_response(sd, buf, sizeof(buf), seq, data, use_ssl);
-
-            if(strncmp(buf, tagok, tagoklen) == 0){
-               snprintf(buf, sizeof(buf)-1, "A%d STORE %d +FLAGS.SILENT (\\Deleted)\r\n", *seq, i);
-               write1(sd, buf, strlen(buf), use_ssl, data->ssl);
-               read_response(sd, buf, sizeof(buf), seq, data, use_ssl);
-
-            }
-         }
-
-
-
-         if(data->import->download_only == 0) unlink(filename);
-      }
-
-
    }
 
-   if((data->import->remove_after_import == 1 || data->import->move_folder) && dryrun == 0){
+   if((data->import->remove_after_import == 1 || data->import->move_folder) && data->import->dryrun == 0){
-      snprintf(buf, sizeof(buf)-1, "A%d EXPUNGE\r\n", *seq);
+      imap_expunge_message(data);
-      write1(sd, buf, strlen(buf), use_ssl, data->ssl);
-      read_response(sd, buf, sizeof(buf), seq, data, use_ssl);
    }
 
 
-   printf("\n");
+   if(data->quiet == 0){printf("\n"); }
 
    return OK;
 }
 
 
-int connect_to_imap_server(int sd, int *seq, char *username, char *password, struct __data *data, int use_ssl){
+void send_imap_close(struct data *data){
-   int n;
-   char buf[MAXBUFSIZE];
-   X509* server_cert;
-   char *str;
-
-   data->import->cap_uidplus = 0;
-
-   if(use_ssl == 1){
-
-      SSL_library_init();
-      SSL_load_error_strings();
-
-      data->ctx = SSL_CTX_new(TLSv1_client_method());
-      CHK_NULL(data->ctx, "internal SSL error");
-
-      data->ssl = SSL_new(data->ctx);
-      CHK_NULL(data->ssl, "internal ssl error");
-
-      SSL_set_fd(data->ssl, sd);
-      n = SSL_connect(data->ssl);
-      CHK_SSL(n, "internal ssl error");
-
-      printf("Cipher: %s\n", SSL_get_cipher(data->ssl));
-
-      server_cert = SSL_get_peer_certificate(data->ssl);
-      CHK_NULL(server_cert, "server cert error");
-
-      //if(verbose){
-         str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0);
-         CHK_NULL(str, "error in server cert");
-         printf("server cert:\n\t subject: %s\n", str);
-         OPENSSL_free(str);
-
-         str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0);
-         CHK_NULL(str, "error in server cert");
-         printf("\t issuer: %s\n\n", str);
-         OPENSSL_free(str);
-      //}
-
-      X509_free(server_cert);
-   }
-
-
-   recvtimeoutssl(sd, buf, sizeof(buf), data->import->timeout, use_ssl, data->ssl);
-
-
-   /* imap cmd: LOGIN */
-
-   snprintf(buf, sizeof(buf)-1, "A%d LOGIN %s \"%s\"\r\n", *seq, username, password);
-
-   write1(sd, buf, strlen(buf), use_ssl, data->ssl);
-   if(read_response(sd, buf, sizeof(buf), seq, data, use_ssl) == 0){
-      printf("login failed, server reponse: %s\n", buf);
-      return ERR;
-   }
-
-   if(strstr(buf, "UIDPLUS")){
-      data->import->cap_uidplus = 1;
-   }
-   else {
-
-      /* run the CAPABILITY command if the reply doesn't contain the UIDPLUS capability */
-
-      snprintf(buf, sizeof(buf)-1, "A%d CAPABILITY\r\n", *seq);
-
-      write1(sd, buf, strlen(buf), use_ssl, data->ssl);
-      read_response(sd, buf, sizeof(buf), seq, data, use_ssl);
-
-      if(strstr(buf, "UIDPLUS")) data->import->cap_uidplus = 1;
-   }
-
-
-   return OK;
-}
-
-
-void send_imap_close(int sd, int *seq, struct __data *data, int use_ssl){
    char puf[SMALLBUFSIZE];
-   snprintf(puf, sizeof(puf)-1, "A%d CLOSE\r\n", *seq);
+   snprintf(puf, sizeof(puf)-1, "A%d CLOSE\r\n", data->import->seq);
 
-   write1(sd, puf, strlen(puf), use_ssl, data->ssl);
+   write1(data->net, puf, strlen(puf));
 }
 
 
-int list_folders(int sd, int *seq, int use_ssl, char *folder_name, struct __data *data){
+int list_folders(struct data *data){
    char *p, *q, *r, *buf, *ruf, tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], puf[MAXBUFSIZE];
    char attrs[SMALLBUFSIZE], folder[SMALLBUFSIZE];
    int len=MAXBUFSIZE+3, pos=0, n, rc=ERR, fldrlen=0, result;
 
-   printf("List of IMAP folders:\n");
+   if(data->quiet == 0){printf("List of IMAP folders:\n"); }
 
    buf = malloc(len);
    if(!buf) return rc;
 
    memset(buf, 0, len);
 
-   snprintf(tag, sizeof(tag)-1, "A%d", *seq); snprintf(tagok, sizeof(tagok)-1, "A%d OK", (*seq)++);
+   snprintf(tag, sizeof(tag)-1, "A%d", data->import->seq); snprintf(tagok, sizeof(tagok)-1, "A%d OK", (data->import->seq)++);
-   if(folder_name == NULL)
+   if(data->import->folder_imap == NULL)
       snprintf(puf, sizeof(puf)-1, "%s LIST \"\" \"*\"\r\n", tag);
    else
-      snprintf(puf, sizeof(puf)-1, "%s LIST \"%s\" \"*\"\r\n", tag, folder_name);
+      snprintf(puf, sizeof(puf)-1, "%s LIST \"%s\" \"*\"\r\n", tag, data->import->folder_imap);
 
-   write1(sd, puf, strlen(puf), use_ssl, data->ssl);
+   write1(data->net, puf, strlen(puf));
 
    p = NULL;
 
    while(1){
-      n = recvtimeoutssl(sd, puf, sizeof(puf), data->import->timeout, use_ssl, data->ssl);
+      n = recvtimeoutssl(data->net, puf, sizeof(puf));
       if(n < 0) return ERR;
 
       if(pos + n >= len){
@@ -441,7 +421,7 @@ int list_folders(int sd, int *seq, int use_ssl, char *folder_name, struct __data
    }
 
    // trim the "A3 OK LIST completed" trailer off
-   if(p) *p = '\0';
+   if(p) *p = '\0'; //-V547
 
    memset(attrs, 0, sizeof(attrs));
 
@@ -474,21 +454,27 @@ int list_folders(int sd, int *seq, int use_ssl, char *folder_name, struct __data
             } else {
 
                if(fldrlen) {
-                  ruf = malloc(strlen(q) * 2 + 1);
+                  int ruflen = strlen(q) * 2;
-                  memset(ruf, 0, strlen(q) * 2 + 1);
+                  ruf = malloc(ruflen + 1);
-                  memcpy(ruf, q, strlen(q));
+                  if(ruf){
-                  r = ruf;
+                     snprintf(ruf, ruflen, "%s", q);
-                  while(*r != '\0') {
+                     r = ruf;
-                     if(*r == '\\') {
+                     while(*r != '\0') {
-                        memmove(r + 1, r, strlen(r));
+                        if(*r == '\\') {
+                           memmove(r + 1, r, strlen(r));
+                           r++;
+                        }
                         r++;
                      }
-                     r++;
+
+                     snprintf(folder, sizeof(folder)-1, "%s", ruf);
+
+                     free(ruf);
+                  }
+                  else {
+                     printf("error: ruf = malloc()\n");
                   }
 
-                  snprintf(folder, sizeof(folder)-1, "%s", ruf);
-
-                  free(ruf);
                   fldrlen = 0;
                } else {
                   snprintf(folder, sizeof(folder)-1, "%s", q);
@@ -497,9 +483,9 @@ int list_folders(int sd, int *seq, int use_ssl, char *folder_name, struct __data
                if(!strstr(attrs, "\\Noselect")){
                   addnode(data->imapfolders, folder);
                }
-               else printf("skipping ");
+               else if(data->quiet == 0){printf("skipping "); }
 
-               printf("=> '%s [%s]'\n", folder, attrs);
+               if(data->quiet == 0){printf("=> '%s [%s]'\n", folder, attrs); }
 
                memset(attrs, 0, sizeof(attrs));
             }
@@ -518,5 +504,3 @@ ENDE_FOLDERS:
 
    return rc;
 }
-
-

src/import.c

@@ -18,27 +18,29 @@
 #include <piler.h>
 
 
-int import_message(char *filename, struct session_data *sdata, struct __data *data, struct __config *cfg){
+int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    int rc=ERR;
-   char *p, *rule, newpath[SMALLBUFSIZE];
+   char *rule;
    struct stat st;
    struct parser_state state;
-   struct counters counters;
 
+   if(data->import->delay > 0){
+      struct timespec req;
+
+      req.tv_sec = 0;
+      req.tv_nsec = 1000000 * data->import->delay;
+
+      nanosleep(&req, NULL);
+   }
 
    init_session_data(sdata, cfg);
 
-   if(data->import->extra_recipient){
+   if(cfg->verbosity > 1) printf("processing: %s\n", data->import->filename);
-      snprintf(sdata->rcptto[0], SMALLBUFSIZE-1, "%s", data->import->extra_recipient);
-      sdata->num_of_rcpt_to = 1;
-   }
 
-   if(cfg->verbosity > 1) printf("processing: %s\n", filename);
+   if(strcmp(data->import->filename, "-") == 0){
-
-   if(strcmp(filename, "-") == 0){
 
       if(read_from_stdin(sdata) == ERR){
-         printf("error reading from stdin\n");
+         printf("ERROR: error reading from stdin\n");
          return rc;
       }
 
@@ -47,17 +49,17 @@ int import_message(char *filename, struct session_data *sdata, struct __data *da
    }
    else {
 
-      if(stat(filename, &st) != 0){
+      if(stat(data->import->filename, &st) != 0){
-         printf("cannot stat() %s\n", filename);
+         printf("ERROR: cannot stat() %s\n", data->import->filename);
          return rc;
       }
 
       if(S_ISREG(st.st_mode) == 0){
-         printf("%s is not a file\n", filename);
+         printf("ERROR: %s is not a file\n", data->import->filename);
          return rc;
       }
 
-      snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
+      snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", data->import->filename);
 
       sdata->tot_len = st.st_size;
    }
@@ -77,125 +79,159 @@ int import_message(char *filename, struct session_data *sdata, struct __data *da
 
    state = parse_message(sdata, 1, data, cfg);
    post_parse(sdata, &state, cfg);
-
+   rule = check_against_ruleset(data->archiving_rules, &state, sdata->tot_len, sdata->spam_message);
-   rule = check_againt_ruleset(data->archiving_rules, &state, sdata->tot_len, sdata->spam_message);
 
    if(rule){
-      if(data->quiet == 0) printf("discarding %s by archiving policy: %s\n", filename, rule);
+      if(data->quiet == 0) printf("discarding %s by archiving policy: %s\n", data->import->filename, rule);
       rc = OK;
    }
    else {
       make_digests(sdata, cfg);
 
       if(sdata->hdr_len < 10){
-         printf("%s: invalid message, hdr_len: %d\n", filename, sdata->hdr_len);
+         printf("%s: invalid message, hdr_len: %d\n", data->import->filename, sdata->hdr_len);
-         return ERR;
+         rc = ERR;
       }
+      else if(data->import->after > 0 && sdata->sent < data->import->after){
+         if(cfg->verbosity > 1) printf("discarding older email: %s (%ld/%ld)\n", sdata->filename, sdata->sent, data->import->after);
+         rc = ERR_DISCARDED;
+      }
+      else if(data->import->before > 0 && sdata->sent > data->import->before){
+         if(cfg->verbosity > 1) printf("discarding newer email: %s (%ld/%ld)\n", sdata->filename, sdata->sent, data->import->before);
+         rc = ERR_DISCARDED;
+      }
+      else {
+         // When importing emails, we should add the retention value (later) to the original sent value
+         sdata->retained = sdata->sent;
 
-      rc = process_message(sdata, &state, data, cfg);
+         // backup original value of data->folder
-      unlink(state.message_id_hash);
+         int folder = data->folder;
+         rc = process_message(sdata, &state, data, cfg);
+         data->folder = folder;
+         unlink(state.message_id_hash);
+      }
    }
 
    unlink(sdata->tmpframe);
 
-   if(strcmp(filename, "-") == 0) unlink(sdata->ttmpfile);
+   remove_stripped_attachments(&state);
+
+   if(strcmp(data->import->filename, "-") == 0) unlink(sdata->ttmpfile);
 
 
    switch(rc) {
       case OK:
-                        bzero(&counters, sizeof(counters));
+                        counters->c_rcvd++;
-                        counters.c_rcvd = 1;
+                        counters->c_size += sdata->tot_len;
-                        counters.c_size += sdata->tot_len;
+                        counters->c_stored_size += sdata->stored_len;
-                        counters.c_stored_size = sdata->stored_len;
-                        update_counters(sdata, data, &counters, cfg);
 
                         break;
 
       case ERR_EXISTS:
                         rc = OK;
 
-                        bzero(&counters, sizeof(counters));
+                        counters->c_duplicate++;
-                        counters.c_duplicate = 1;
+
-                        update_counters(sdata, data, &counters, cfg);
+                        if(data->quiet == 0) printf("duplicate: %s (duplicate id: %llu)\n", data->import->filename, sdata->duplicate_id);
+                        break;
+
+      case ERR_DISCARDED:
+                        rc = OK;
+
+                        counters->c_ignore++;
 
-                        if(data->quiet == 0) printf("duplicate: %s (duplicate id: %llu)\n", filename, sdata->duplicate_id);
                         break;
 
       default:
-                        printf("failed to import: %s (id: %s)\n", filename, sdata->ttmpfile);
+                        printf("failed to import: %s (id: %s)\n", data->import->filename, sdata->ttmpfile);
                         break;
    }
 
    if(rc != OK && data->import->failed_folder){
-      p = strrchr(filename, '/');
+      char *p = strrchr(data->import->filename, '/');
       if(p)
          p++;
       else
-         p = filename;
+         p = data->import->filename;
 
+      char newpath[SMALLBUFSIZE];
       snprintf(newpath, sizeof(newpath)-2, "%s/%s", data->import->failed_folder, p);
 
-      if(rename(filename, newpath))
+      if(rename(data->import->filename, newpath))
-         printf("cannot move %s to %s\n", filename, newpath);
+         printf("cannot move %s to %s\n", data->import->filename, newpath);
    }
 
    return rc;
 }
 
 
-int get_folder_id(struct session_data *sdata, struct __data *data, char *foldername, int parent_id){
+int update_import_table(struct session_data *sdata, struct data *data) {
+   int ret=ERR, status=2, started=1;
+   struct sql sql;
+
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE) == ERR) return ret;
+
+   p_bind_init(&sql);
+
+   sql.sql[sql.pos] = (char *)&(started); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&(status); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&(data->import->tot_msgs); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&(data->import->table_id); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+
+   if(p_exec_stmt(sdata, &sql) == OK) ret = OK;
+
+   close_prepared_statement(&sql);
+
+   return ret;
+}
+
+
+int get_folder_id(struct session_data *sdata, char *foldername, int parent_id){
    int id=ERR_FOLDER;
+   struct sql sql;
 
-   if(prepare_sql_statement(sdata, &(data->stmt_get_folder_id), SQL_PREPARED_STMT_GET_FOLDER_ID) == ERR) return id;
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_FOLDER_ID) == ERR) return id;
 
-   p_bind_init(data);
+   p_bind_init(&sql);
-   data->sql[data->pos] = foldername; data->type[data->pos] = TYPE_STRING; data->pos++;
-   data->sql[data->pos] = (char *)&parent_id; data->type[data->pos] = TYPE_LONG; data->pos++;
 
-   if(p_exec_query(sdata, data->stmt_get_folder_id, data) == OK){
+   sql.sql[sql.pos] = foldername; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = (char *)&parent_id; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
 
-      p_bind_init(data);
+   if(p_exec_stmt(sdata, &sql) == OK){
-      data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(unsigned long); data->pos++;
 
-      p_store_results(data->stmt_get_folder_id, data);
+      p_bind_init(&sql);
-      p_fetch_results(data->stmt_get_folder_id);
+
-      p_free_results(data->stmt_get_folder_id);
+      sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(unsigned long); sql.pos++;
+
+      p_store_results(&sql);
+      p_fetch_results(&sql);
+      p_free_results(&sql);
    }
 
-   close_prepared_statement(data->stmt_get_folder_id);
+   close_prepared_statement(&sql);
 
    return id;
 }
 
 
-int add_new_folder(struct session_data *sdata, struct __data *data, char *foldername, int parent_id){
+int add_new_folder(struct session_data *sdata, char *foldername, int parent_id){
    int id=ERR_FOLDER;
+   struct sql sql;
 
    if(foldername == NULL) return id;
 
-   if(prepare_sql_statement(sdata, &(data->stmt_insert_into_folder_table), SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE) == ERR) return id;
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE) == ERR) return id;
 
-   p_bind_init(data);
+   p_bind_init(&sql);
-   data->sql[data->pos] = foldername; data->type[data->pos] = TYPE_STRING; data->pos++;
-   data->sql[data->pos] = (char *)&parent_id; data->type[data->pos] = TYPE_LONG; data->pos++;
 
-   if(p_exec_query(sdata, data->stmt_insert_into_folder_table, data) == OK){
+   sql.sql[sql.pos] = foldername; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-      id = p_get_insert_id(data->stmt_insert_into_folder_table);
+   sql.sql[sql.pos] = (char *)&parent_id; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+
+   if(p_exec_stmt(sdata, &sql) == OK){
+      id = p_get_insert_id(&sql);
    }
 
-   close_prepared_statement(data->stmt_insert_into_folder_table);
+   close_prepared_statement(&sql);
 
    return id;
 }
-
-
-void update_import_job_stat(struct session_data *sdata, struct __data *data){
-   char buf[SMALLBUFSIZE];
-
-   snprintf(buf, sizeof(buf)-1, "update import set status=%d, started=%ld, updated=%ld, finished=%ld, total=%d, imported=%d where id=%d", data->import->status, data->import->started, data->import->updated, data->import->finished, data->import->total_messages, data->import->processed_messages, data->import->import_job_id);
-
-   p_query(sdata, buf);
-}
-
-
-

src/import.h

@@ -6,24 +6,23 @@
    #define _IMPORT_H
 
 
-int import_message(char *filename, struct session_data *sdata, struct __data *data, struct __config *cfg);
+int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
-void update_import_job_stat(struct session_data *sdata, struct __data *data);
+int update_import_table(struct session_data *sdata, struct data *data);
 
-int read_gui_import_data(struct session_data *sdata, struct __data *data, char *folder_imap, char *skiplist, int dryrun, struct __config *cfg);
+int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg);
+int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 
-int import_from_maildir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg);
+int connect_to_pop3_server(struct data *data);
-int import_from_mailbox(char *mailbox, struct session_data *sdata, struct __data *data, struct __config *cfg);
+void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
-int import_mbox_from_dir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg);
-int import_from_pop3_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, int dryrun, struct __config *cfg);
-int import_from_imap_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, char *folder_imap, char *skiplist, int dryrun, struct __config *cfg);
 
-int connect_to_pop3_server(int sd, char *username, char *password, struct __data *data, int use_ssl);
+int connect_to_imap_server(struct data *data);
-int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg);
+int list_folders(struct data *data);
+int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+void send_imap_close(struct data *data);
 
-int connect_to_imap_server(int sd, int *seq, char *username, char *password, struct __data *data, int use_ssl);
+void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
-int list_folders(int sd, int *seq, int use_ssl, char *folder_name, struct __data *data);
-int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg);
-void send_imap_close(int sd, int *seq, struct __data *data, int use_ssl);
 
 #endif /* _IMPORT_H */
-

src/import_gui.c

@@ -1,80 +0,0 @@
-/*
- * import_gui.c, SJ
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/socket.h>
-#include <netdb.h>
-#include <arpa/inet.h>
-#include <netinet/in.h>
-#include <fcntl.h>
-#include <dirent.h>
-#include <unistd.h>
-#include <time.h>
-#include <locale.h>
-#include <getopt.h>
-#include <syslog.h>
-#include <piler.h>
-
-
-int read_gui_import_data(struct session_data *sdata, struct __data *data, char *folder_imap, char *skiplist, int dryrun, struct __config *cfg){
-   int rc=ERR;
-   char s_type[SMALLBUFSIZE], s_username[SMALLBUFSIZE], s_password[SMALLBUFSIZE], s_server[SMALLBUFSIZE];
-
-   memset(s_type, 0, sizeof(s_type));
-   memset(s_username, 0, sizeof(s_username));
-   memset(s_password, 0, sizeof(s_password));
-   memset(s_server, 0, sizeof(s_server));
-
-   if(prepare_sql_statement(sdata, &(data->stmt_generic), SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS) == ERR) return ERR;
-
-   p_bind_init(data);
-
-   if(p_exec_query(sdata, data->stmt_generic, data) == OK){
-
-      p_bind_init(data);
-
-      data->sql[data->pos] = (char *)&(data->import->import_job_id); data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
-      data->sql[data->pos] = &s_type[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_type)-2; data->pos++;
-      data->sql[data->pos] = &s_username[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_username)-2; data->pos++;
-      data->sql[data->pos] = &s_password[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_password)-2; data->pos++;
-      data->sql[data->pos] = &s_server[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_server)-2; data->pos++;
-
-      p_store_results(data->stmt_generic, data);
-
-      if(p_fetch_results(data->stmt_generic) == OK) rc = OK;
-
-      p_free_results(data->stmt_generic);
-   }
-
-   close_prepared_statement(data->stmt_generic);
-
-   data->import->processed_messages = 0;
-   data->import->total_messages = 0;
-
-   time(&(data->import->started));
-   data->import->status = 1;
-   update_import_job_stat(sdata, data);  
-
-   if(strcmp(s_type, "pop3") == 0){
-      rc = import_from_pop3_server(s_server, s_username, s_password, 110, sdata, data, dryrun, cfg);
-   }
-
-   if(strcmp(s_type, "imap") == 0){
-      rc = import_from_imap_server(s_server, s_username, s_password, 143, sdata, data, folder_imap, skiplist, dryrun, cfg);
-   }
-
-   update_import_job_stat(sdata, data);
-
-   // don't set error in case of a problem, because it
-   // will scare users looking at the gui progressbar
-
-   return rc;
-}
-
-

src/import_imap.c

@@ -22,61 +22,64 @@
 #include <piler.h>
 
 
-int import_from_imap_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, char *folder_imap, char *skiplist, int dryrun, struct __config *cfg){
+int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
-   int i, rc=ERR, ret=OK, sd, seq=1, skipmatch, use_ssl=0;
+   int i, rc=ERR, ret=OK, skipmatch;
    char port_string[8], puf[SMALLBUFSIZE];
    struct addrinfo hints, *res;
    struct node *q;
 
+   data->net->use_ssl = 0;
+   data->import->seq = 1;
 
    inithash(data->imapfolders);
 
-   snprintf(port_string, sizeof(port_string)-1, "%d", port);
+   snprintf(port_string, sizeof(port_string)-1, "%d", data->import->port);
 
    memset(&hints, 0, sizeof(hints));
    hints.ai_family = AF_UNSPEC;
    hints.ai_socktype = SOCK_STREAM;
 
-   if((rc = getaddrinfo(server, port_string, &hints, &res)) != 0){
+   if((rc = getaddrinfo(data->import->server, port_string, &hints, &res)) != 0){
-      printf("getaddrinfo for '%s': %s\n", server, gai_strerror(rc));
+      printf("getaddrinfo for '%s': %s\n", data->import->server, gai_strerror(rc));
       return ERR;
    }
 
-   if(port == 993) use_ssl = 1;
+   if(data->import->port == 993) data->net->use_ssl = 1;
 
 
-   if((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
+   if((data->net->socket = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
       printf("cannot create socket\n");
       ret = ERR;
       goto ENDE_IMAP;
    }
 
-   if(connect(sd, res->ai_addr, res->ai_addrlen) == -1){
+   if(connect(data->net->socket, res->ai_addr, res->ai_addrlen) == -1){
       printf("connect()\n");
       ret = ERR;
       goto ENDE_IMAP;
    }
 
-   if(connect_to_imap_server(sd, &seq, username, password, data, use_ssl) == ERR){
+   if(connect_to_imap_server(data) == ERR){
-      close(sd);
+      close(data->net->socket);
       ret = ERR;
       goto ENDE_IMAP;
    }
 
 
-   if(list_folders(sd, &seq, use_ssl, folder_imap, data) == ERR) goto ENDE_IMAP;
+   if(list_folders(data) == ERR) goto ENDE_IMAP;
+
 
    for(i=0;i<MAXHASH;i++){
       q = data->imapfolders[i];
       while(q != NULL){
 
-         if(q && q->str && strlen(q->str) > 1){
+         if(q->str && strlen(q->str) > 1){
 
             skipmatch = 0;
 
-            if(skiplist && strlen(skiplist) > 0){
+            if(data->import->skiplist && strlen(data->import->skiplist) > 0){
                snprintf(puf, sizeof(puf)-1, "%s,", (char *)q->str);
-               if(strstr(skiplist, puf)) skipmatch = 1;
+               if(strstr(data->import->skiplist, puf)) skipmatch = 1;
             }
 
             if(skipmatch == 1){
@@ -85,7 +88,7 @@ int import_from_imap_server(char *server, char *username, char *password, int po
             else {
                if(data->quiet == 0) printf("processing folder: %s... ", (char *)q->str);
 
-               if(process_imap_folder(sd, &seq, q->str, sdata, data, use_ssl, dryrun, cfg) == ERR) ret = ERR;
+               if(process_imap_folder(q->str, sdata, data, counters, cfg) == ERR) ret = ERR;
             }
 
          }
@@ -95,9 +98,9 @@ int import_from_imap_server(char *server, char *username, char *password, int po
       }
    }
 
-   send_imap_close(sd, &seq, data, use_ssl);
+   send_imap_close(data);
 
-   close_connection(sd, data, use_ssl);
+   close_connection(data->net);
 
 ENDE_IMAP:
    freeaddrinfo(res);
@@ -108,5 +111,3 @@ ENDE_IMAP:
 
    return ret;
 }
-
-

src/import_mailbox.c

@@ -22,10 +22,10 @@
 #include <piler.h>
 
 
-int import_from_mailbox(char *mailbox, struct session_data *sdata, struct __data *data, struct __config *cfg){
+int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    FILE *F, *f=NULL;
    int rc=ERR, tot_msgs=0, ret=OK;
-   char buf[MAXBUFSIZE], fname[SMALLBUFSIZE];
+   char buf[MAXBUFSIZE];
    time_t t;
 
 
@@ -44,18 +44,18 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct __data
          if(f){
             fclose(f);
             f = NULL;
-            rc = import_message(fname, sdata, data, cfg);
+            rc = import_message(sdata, data, counters, cfg);
             if(rc == ERR){
-               printf("error importing: '%s'\n", fname);
+               printf("error importing: '%s'\n", data->import->filename);
                ret = ERR;
             }
-            else unlink(fname);
+            else unlink(data->import->filename);
 
-            if(data->quiet == 0) printf("processed: %7d\r", tot_msgs); fflush(stdout);
+            if(data->quiet == 0){ printf("processed: %7d\r", data->import->tot_msgs); fflush(stdout); }
          }
 
-         snprintf(fname, sizeof(fname)-1, "%ld-%d", t, tot_msgs);
+         snprintf(data->import->filename, sizeof(data->import->filename)-1, "%ld-%d", t, data->import->tot_msgs);
-         f = fopen(fname, "w+");
+         f = fopen(data->import->filename, "w+");
          continue;
       }
 
@@ -64,14 +64,14 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct __data
 
    if(f){
       fclose(f);
-      rc = import_message(fname, sdata, data, cfg);
+      rc = import_message(sdata, data, counters, cfg);
       if(rc == ERR){
-         printf("error importing: '%s'\n", fname);
+         printf("ERROR: error importing: '%s'\n", data->import->filename);
          ret = ERR;
       }
-      else unlink(fname);
+      else unlink(data->import->filename);
 
-      if(data->quiet == 0) printf("processed: %7d\r", tot_msgs); fflush(stdout);
+      if(data->quiet == 0){ printf("processed: %7d\r", data->import->tot_msgs); fflush(stdout); }
    }
 
    fclose(F);
@@ -80,7 +80,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct __data
 }
 
 
-int import_mbox_from_dir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg){
+int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    DIR *dir;
    struct dirent *de;
    int rc=ERR, ret=OK, i=0;
@@ -103,7 +103,7 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct __d
       if(stat(fname, &st) == 0){
          if(S_ISDIR(st.st_mode)){
             folder = data->folder;
-            rc = import_mbox_from_dir(fname, sdata, data, tot_msgs, cfg);
+            rc = import_mbox_from_dir(fname, sdata, data, counters, cfg);
             data->folder = folder;
             if(rc == ERR) ret = ERR;
          }
@@ -111,9 +111,9 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct __d
 
             if(S_ISREG(st.st_mode)){
                if(i == 0 && data->recursive_folder_names == 1){
-                  folder = get_folder_id(sdata, data, fname, data->folder);
+                  folder = get_folder_id(sdata, fname, data->folder);
                   if(folder == ERR_FOLDER){
-                     folder = add_new_folder(sdata, data, fname, data->folder);
+                     folder = add_new_folder(sdata, fname, data->folder);
 
                      if(folder == ERR_FOLDER){
                         printf("error: cannot get/add folder '%s' to parent id: %d\n", fname, data->folder);
@@ -126,8 +126,8 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct __d
 
                }
 
-               rc = import_from_mailbox(fname, sdata, data, cfg);
+               rc = import_from_mailbox(fname, sdata, data, counters, cfg);
-               if(rc == OK) (*tot_msgs)++;
+               if(rc == OK) (data->import->tot_msgs)++;
                else ret = ERR;
 
                i++;

src/import_maildir.c

@@ -22,12 +22,12 @@
 #include <piler.h>
 
 
-int import_from_maildir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg){
+int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg){
    DIR *dir;
    struct dirent *de;
    int rc=ERR, ret=OK, i=0;
    int folder;
-   char *p, fname[SMALLBUFSIZE];
+   char *p, subdir[SMALLBUFSIZE];
    struct stat st;
 
    dir = opendir(directory);
@@ -40,12 +40,13 @@ int import_from_maildir(char *directory, struct session_data *sdata, struct __da
    while((de = readdir(dir))){
       if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
 
-      snprintf(fname, sizeof(fname)-1, "%s/%s", directory, de->d_name);
+      snprintf(data->import->filename, SMALLBUFSIZE-1, "%s/%s", directory, de->d_name);
 
-      if(stat(fname, &st) == 0){
+      if(stat(data->import->filename, &st) == 0){
          if(S_ISDIR(st.st_mode)){
             folder = data->folder;
-            rc = import_from_maildir(fname, sdata, data, tot_msgs, cfg);
+            snprintf(subdir, sizeof(subdir)-1, "%s", data->import->filename);
+            rc = import_from_maildir(sdata, data, subdir, counters, cfg);
             data->folder = folder;
             if(rc == ERR) ret = ERR;
          }
@@ -56,13 +57,13 @@ int import_from_maildir(char *directory, struct session_data *sdata, struct __da
                   p = strrchr(directory, '/');
                   if(p) p++;
                   else {
-                     printf("invalid directory name: '%s'\n", directory);
+                     printf("ERROR: invalid directory name: '%s'\n", directory);
                      return ERR;
                   }
 
-                  folder = get_folder_id(sdata, data, p, data->folder);
+                  folder = get_folder_id(sdata, p, data->folder);
                   if(folder == ERR_FOLDER){
-                     folder = add_new_folder(sdata, data, p, data->folder);
+                     folder = add_new_folder(sdata, p, data->folder);
 
                      if(folder == ERR_FOLDER){
                         printf("error: cannot get/add folder '%s' to parent id: %d\n", p, data->folder);
@@ -75,34 +76,36 @@ int import_from_maildir(char *directory, struct session_data *sdata, struct __da
 
                }
 
-               rc = import_message(fname, sdata, data, cfg);
+               rc = import_message(sdata, data, counters, cfg);
 
-               if(rc == OK) (*tot_msgs)++;
+               if(rc == OK) (data->import->tot_msgs)++;
                else if(rc == ERR){
-                  printf("error importing: '%s'\n", fname);
+                  printf("ERROR: error importing: '%s'\n", data->import->filename);
                   ret = ERR;
                }
 
-               if(data->import->remove_after_import == 1 && rc != ERR) unlink(fname);
+               if(data->import->remove_after_import == 1 && rc != ERR) unlink(data->import->filename);
 
                i++;
 
-               if(data->quiet == 0) printf("processed: %7d\r", *tot_msgs); fflush(stdout);
+               if(data->quiet == 0){ printf("processed: %7d\r", data->import->tot_msgs); fflush(stdout); }
             }
             else {
-               printf("%s is not a file\n", fname);
+               printf("%s is not a file\n", data->import->filename);
             }
 
          }
       }
       else {
-         printf("cannot stat() %s\n", fname);
+         printf("cannot stat() %s\n", data->import->filename);
       }
 
    }
    closedir(dir);
 
+   if(data->import->table_id > 0){
+      update_import_table(sdata, data);
+   }
+
    return ret;
 }
-
-

src/import_pilerexport.c

@@ -0,0 +1,114 @@
+/*
+ * import_pop3.c
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <getopt.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+void import_the_file(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   close(data->import->fd);
+   data->import->fd = -1;
+
+   if(import_message(sdata, data, counters, cfg) != ERR){
+      unlink(data->import->filename);
+   }
+}
+
+
+void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+
+   if(!strcmp(buf, PILEREXPORT_BEGIN_MARK)){
+      if((*count) > 0){
+         import_the_file(sdata, data, counters, cfg);
+      }
+
+      (*count)++;
+
+      snprintf(data->import->filename, SMALLBUFSIZE-1, "import-%llu", *count);
+
+      data->import->fd = open(data->import->filename, O_CREAT|O_WRONLY|O_TRUNC, S_IRUSR|S_IWUSR);
+      if(data->import->fd == -1){
+         // Do some error handling
+         printf("error: cannot open %s\n", data->import->filename);
+      }
+
+   } else {
+      if(write(data->import->fd, buf, buflen) != buflen){
+         printf("error: didnt write %d bytes\n", buflen);
+      }
+   }
+}
+
+
+void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   int n, rc, nullbyte, savedlen=0, puflen;
+   uint64 count=0;
+   char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE];
+
+   memset(savedbuf, 0, sizeof(savedbuf));
+
+   data->import->fd = -1;
+
+   do {
+      memset(buf, 0, sizeof(buf));
+      n = fread(buf, 1, sizeof(buf)-1, stdin);
+
+      int remaininglen = n;
+
+      if(savedlen > 0){
+         memset(copybuf, 0, sizeof(copybuf));
+
+         memcpy(copybuf, savedbuf, savedlen);
+         memcpy(&copybuf[savedlen], buf, n);
+
+         remaininglen += savedlen;
+
+         savedlen = 0;
+         memset(savedbuf, 0, sizeof(savedbuf));
+
+         p = &copybuf[0];
+      }
+      else {
+         p = &buf[0];
+      }
+
+      do {
+         puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf), &rc, &nullbyte);
+         p += puflen;
+         remaininglen -= puflen;
+
+         if(puflen > 0){
+            if(rc == OK){
+               process_buffer(puf, puflen, &count, sdata, data, counters, cfg);
+            }
+            else {
+               snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
+               savedlen = puflen;
+            }
+         }
+
+      } while(puflen > 0);
+
+   } while(n > 0);
+
+   if(data->import->fd != -1){
+      import_the_file(sdata, data, counters, cfg);
+   }
+}

src/import_pop3.c

@@ -22,50 +22,46 @@
 #include <piler.h>
 
 
-int import_from_pop3_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, int dryrun, struct __config *cfg){
+void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
-   int rc, ret=OK, sd, use_ssl=0;
+   int rc;
    char port_string[8];
    struct addrinfo hints, *res;
 
-   snprintf(port_string, sizeof(port_string)-1, "%d", port);
+   data->net->use_ssl = 0;
+
+   snprintf(port_string, sizeof(port_string)-1, "%d", data->import->port);
 
    memset(&hints, 0, sizeof(hints));
    hints.ai_family = AF_UNSPEC;
    hints.ai_socktype = SOCK_STREAM;
 
-   if((rc = getaddrinfo(server, port_string, &hints, &res)) != 0){
+   if((rc = getaddrinfo(data->import->server, port_string, &hints, &res)) != 0){
-      printf("getaddrinfo for '%s': %s\n", server, gai_strerror(rc));
+      printf("getaddrinfo for '%s': %s\n", data->import->server, gai_strerror(rc));
-      return ERR;
+      return;
    }
 
-   if(port == 995) use_ssl = 1;
+   if(data->import->port == 995) data->net->use_ssl = 1;
 
-   if((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
+   if((data->net->socket = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
       printf("cannot create socket\n");
-      ret = ERR;
       goto ENDE_POP3;
    }
 
-   if(connect(sd, res->ai_addr, res->ai_addrlen) == -1){
+   if(connect(data->net->socket, res->ai_addr, res->ai_addrlen) == -1){
       printf("connect()\n");
-      ret = ERR;
       goto ENDE_POP3;
    }
 
 
-   if(connect_to_pop3_server(sd, username, password, data, use_ssl) == ERR){
+   if(connect_to_pop3_server(data) == ERR){
-      close(sd);
+      close(data->net->socket);
-      ret = ERR;
       goto ENDE_POP3;
    }
 
-   if(process_pop3_emails(sd, sdata, data, use_ssl, dryrun, cfg) == ERR) ret = ERR;
+   process_pop3_emails(sdata, data, counters, cfg);
 
-   close_connection(sd, data, use_ssl);
+   close_connection(data->net);
 
 ENDE_POP3:
    freeaddrinfo(res);
-
-   return ret;
 }
-

src/list.c

@@ -1,97 +0,0 @@
-/*
- * list.c, SJ
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include "list.h"
-#include "config.h"
-
-
-int append_list(struct list **list, char *p){
-   struct list *q, *t, *u=NULL;
-
-   q = *list;
-
-   while(q){
-      if(strcmp(q->s, p) == 0)
-         return 0;
-
-      u = q;
-      q = q->r;
-   }
-
-   t = create_list_item(p);
-   if(t){
-      if(*list == NULL)
-         *list = t;
-      else if(u)
-         u->r = t;
-
-      return 1;
-   }
-
-   return -1;
-}
-
-
-struct list *create_list_item(char *s){
-   struct list *h=NULL;
-
-   if((h = malloc(sizeof(struct list))) == NULL)
-      return NULL;
-
-   snprintf(h->s, SMALLBUFSIZE-1, "%s", s);
-   h->r = NULL;
-
-   return h;
-}
-
-
-int is_string_on_list(struct list *list, char *s){
-   struct list *p;
-
-   p = list;
-
-   while(p != NULL){
-      if(strcmp(p->s, s) == 0) return 1;
-      p = p->r;
-   }
-
-   return 0;
-}
-
-
-int is_item_on_string(struct list *list, char *s){
-   struct list *p;
-
-   p = list;
-
-   while(p != NULL){
-      if(strstr(s, p->s)) return 1;
-      p = p->r;
-   }
-
-   return 0;
-}
-
-
-void free_list(struct list *list){
-   struct list *p, *q;
-
-   p = list;
-
-   while(p != NULL){
-      q = p->r;
-
-      if(p)
-         free(p);
-
-      p = q;
-   }
-}
-
-
-

src/list.h

@@ -1,17 +0,0 @@
-/*
- * list.h, SJ
- */
-
-#ifndef _LIST_H
- #define _LIST_H
-
-#include "defs.h"
-
-int append_list(struct list **list, char *p);
-struct list *create_list_item(char *s);
-int is_string_on_list(struct list *list, char *s);
-int is_item_on_string(struct list *list, char *s);
-void free_list(struct list *list);
-
-#endif /* _LIST_H */
-

src/memc.c

@@ -59,7 +59,7 @@ void memcached_init(struct memcached_server *ptr, char *server_ip, int server_po
 
 
 int set_socket_options(struct memcached_server *ptr){
-   int error, flag=1, flags, rval;
+   int error, flag=1, flags;
    struct timeval waittime;
    struct linger linger;
 
@@ -128,8 +128,7 @@ int set_socket_options(struct memcached_server *ptr){
 
 
    if((flags & O_NONBLOCK) == 0){
-      rval = fcntl(ptr->fd, F_SETFL, flags | O_NONBLOCK);
+      if(fcntl(ptr->fd, F_SETFL, flags | O_NONBLOCK) == -1) return MEMCACHED_FAILURE;
-      if(rval == -1) return MEMCACHED_FAILURE;
    }
 
    return MEMCACHED_SUCCESS;
@@ -217,12 +216,13 @@ int memcached_shutdown(struct memcached_server *ptr){
 }
 
 
-int memcached_add(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry){
+int memcached_add(struct memcached_server *ptr, char *cmd, char *key, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry){
    int len=0;
 
    if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
 
-   snprintf(ptr->buf, MAXBUFSIZE-1, "add %s %d %ld %d \r\n", key, flags, expiry, valuelen);
+   // cmd could be either 'add' or 'set'
+   snprintf(ptr->buf, MAXBUFSIZE-1, "%s %s %u %lu %u \r\n", cmd, key, flags, expiry, valuelen);
    len = strlen(ptr->buf);
 
    strncat(ptr->buf, value, MAXBUFSIZE-strlen(ptr->buf)-1);
@@ -240,30 +240,7 @@ int memcached_add(struct memcached_server *ptr, char *key, unsigned int keylen,
 }
 
 
-int memcached_set(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry){
+int memcached_increment(struct memcached_server *ptr, char *key, unsigned long long value, unsigned long long *result){
-   int len=0;
-
-   if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
-
-   snprintf(ptr->buf, MAXBUFSIZE-1, "set %s %d %ld %d \r\n", key, flags, expiry, valuelen);
-   len = strlen(ptr->buf);
-
-   strncat(ptr->buf, value, MAXBUFSIZE-strlen(ptr->buf)-1);
-   strncat(ptr->buf, "\r\n", MAXBUFSIZE-strlen(ptr->buf)-1);
-
-   len += valuelen + 2;
-
-   send(ptr->fd, ptr->buf, len, 0);
-
-   ptr->last_read_bytes = __recvtimeout(ptr->fd, ptr->buf, MAXBUFSIZE, ptr->rcv_timeout);
-
-   if(strcmp("STORED\r\n", ptr->buf)) return MEMCACHED_FAILURE;
-
-   return MEMCACHED_SUCCESS;
-}
-
-
-int memcached_increment(struct memcached_server *ptr, char *key, unsigned int keylen, unsigned long long value, unsigned long long *result){
    char *p;
 
    if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
@@ -285,52 +262,6 @@ int memcached_increment(struct memcached_server *ptr, char *key, unsigned int ke
 }
 
 
-char *memcached_get(struct memcached_server *ptr, char *key, unsigned int *len, unsigned int *flags){
-   char *p;
-
-   if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return NULL;
-
-   snprintf(ptr->buf, MAXBUFSIZE, "get %s \r\n", key);
-   send(ptr->fd, ptr->buf, strlen(ptr->buf), 0);
-
-   ptr->last_read_bytes = __recvtimeout(ptr->fd, ptr->buf, MAXBUFSIZE, ptr->rcv_timeout);
-
-   if(ptr->last_read_bytes <= 0){
-      memcached_shutdown(ptr);
-      return NULL;
-   }
-
-
-   if(ptr->last_read_bytes < 10) return NULL;
-
-   if(strncmp("VALUE ", ptr->buf, 6)) return NULL;
-
-   p = strchr(ptr->buf, '\r');
-   if(!p) return NULL;
-   *p = '\0';
-   ptr->result = p + 2;
-
-   p = strrchr(ptr->buf + 6, ' ');
-   if(!p) return NULL;
-   *len = atoi(p+1);
-   *p = '\0';
-
-
-   p = strrchr(ptr->buf + 6, ' ');
-   if(!p) return NULL;
-   *flags = atoi(p+1);
-   *p = '\0';
-
- 
-   p = strchr(ptr->result, '\r');
-   if(!p) return NULL;
-
-   *p = '\0';
-
-   return ptr->result;
-}
-
-
 int memcached_mget(struct memcached_server *ptr, char *key){
 
    if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
@@ -394,5 +325,3 @@ char *memcached_fetch_result(struct memcached_server *ptr, char *key, char *valu
 
    return p;
 }
-
-

src/memc.h

@@ -9,9 +9,8 @@ void memcached_init(struct memcached_server *ptr, char *server_ip, int server_po
 int set_socket_options(struct memcached_server *ptr);
 int memcached_connect(struct memcached_server *ptr);
 int memcached_shutdown(struct memcached_server *ptr);
-int memcached_add(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry);
+int memcached_add(struct memcached_server *ptr, char *cmd, char *key, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry);
-int memcached_set(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry);
+int memcached_increment(struct memcached_server *ptr, char *key, unsigned long long value, unsigned long long *result);
-int memcached_increment(struct memcached_server *ptr, char *key, unsigned int keylen, unsigned long long value, unsigned long long *result);
 char *memcached_get(struct memcached_server *ptr, char *key, unsigned int *len, unsigned int *flags);
 int memcached_mget(struct memcached_server *ptr, char *key);
 char *memcached_fetch_result(struct memcached_server *ptr, char *key, char *value, unsigned int *flags);

src/message.c

@@ -10,6 +10,7 @@
 #include <sys/mman.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
+#include <errno.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <syslog.h>
@@ -17,9 +18,10 @@
 #include <zlib.h>
 
 
-int store_index_data(struct session_data *sdata, struct parser_state *state, struct __data *data, uint64 id, struct __config *cfg){
+int store_index_data(struct session_data *sdata, struct parser_state *state, struct data *data, uint64 id, struct config *cfg){
    int rc=ERR;
-   char *subj;
+   char *subj, *sender=state->b_from, *sender_domain=state->b_from_domain;
+   struct sql sql;
 
    if(data->folder == 0){
       data->folder = get_folder_id_by_rule(data, state, sdata->tot_len, sdata->spam_message, cfg);
@@ -28,74 +30,142 @@ int store_index_data(struct session_data *sdata, struct parser_state *state, str
    subj = state->b_subject;
    if(*subj == ' ') subj++;
 
-
-   if(prepare_sql_statement(sdata, &(data->stmt_insert_into_sphinx_table), SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc;
-
-
    fix_email_address_for_sphinx(state->b_from);
+   fix_email_address_for_sphinx(state->b_sender);
    fix_email_address_for_sphinx(state->b_to);
    fix_email_address_for_sphinx(state->b_from_domain);
+   fix_email_address_for_sphinx(state->b_sender_domain);
    fix_email_address_for_sphinx(state->b_to_domain);
 
+   if(state->b_sender_domain[0]){
+      sender = state->b_sender;
+      sender_domain = state->b_sender_domain;
+   }
 
-   p_bind_init(data);
+   if(cfg->rtindex){
+      // Manticore doesn't support prepared statements using sphinxQL
+      // so we have to go through a painful query assembly escaping
+      // the untrusted input
+      //
+      char a[4*MAXBUFSIZE+4*SMALLBUFSIZE];
+      char *query=NULL;
 
-   data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
+      snprintf(a, sizeof(a)-1, "REPLACE INTO %s (id, arrived, sent, size, direction, folder, attachments, attachment_types, sender, rcpt, senderdomain, rcptdomain, subject, body) VALUES (%llu,%ld,%ld,%d,%d,%d,%d,'%s','", cfg->sphxdb, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, data->folder, state->n_attachments, sdata->attachments);
-   data->sql[data->pos] = state->b_from; data->type[data->pos] = TYPE_STRING; data->pos++;
-   data->sql[data->pos] = state->b_to; data->type[data->pos] = TYPE_STRING; data->pos++;
-   data->sql[data->pos] = state->b_from_domain; data->type[data->pos] = TYPE_STRING; data->pos++;
-   data->sql[data->pos] = state->b_to_domain; data->type[data->pos] = TYPE_STRING; data->pos++;
-   data->sql[data->pos] = subj; data->type[data->pos] = TYPE_STRING; data->pos++;
-   data->sql[data->pos] = state->b_body; data->type[data->pos] = TYPE_STRING; data->pos++;
-   data->sql[data->pos] = (char *)&sdata->now; data->type[data->pos] = TYPE_LONG; data->pos++;
-   data->sql[data->pos] = (char *)&sdata->sent; data->type[data->pos] = TYPE_LONG; data->pos++;
-   data->sql[data->pos] = (char *)&sdata->tot_len; data->type[data->pos] = TYPE_LONG; data->pos++;
-   data->sql[data->pos] = (char *)&sdata->direction; data->type[data->pos] = TYPE_LONG; data->pos++;
-   data->sql[data->pos] = (char *)&data->folder; data->type[data->pos] = TYPE_LONG; data->pos++;
-   data->sql[data->pos] = (char *)&state->n_attachments; data->type[data->pos] = TYPE_LONG; data->pos++;
-   data->sql[data->pos] = sdata->attachments; data->type[data->pos] = TYPE_STRING; data->pos++;
 
-   if(p_exec_query(sdata, data->stmt_insert_into_sphinx_table, data) == OK) rc = OK;
+      int ret = append_string_to_buffer(&query, a);
 
-   close_prepared_statement(data->stmt_insert_into_sphinx_table);
+      unsigned long len = strlen(sender);
+      char *s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, sender, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
 
+      len = strlen(state->b_to);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, state->b_to, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(sender_domain);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, sender_domain, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(state->b_to_domain);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, state->b_to_domain, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(subj);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, subj, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(state->b_body);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, state->b_body, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "')");
+
+      if(ret == 0 && mysql_real_query(&(sdata->sphx), query, strlen(query)) == OK) rc = OK;
+      else syslog(LOG_PRIORITY, "ERROR: %s failed to store index data for id=%llu, errno=%d, append ret=%d", sdata->ttmpfile, id, mysql_errno(&(sdata->sphx)), ret);
+
+      free(query);
+   }
+   else {
+      if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc;
+
+      p_bind_init(&sql);
+
+      sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
+      sql.sql[sql.pos] = sender; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = state->b_to; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = state->b_to_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = state->b_body; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = (char *)&sdata->sent; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = (char *)&sdata->tot_len; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = (char *)&sdata->direction; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = (char *)&data->folder; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = (char *)&state->n_attachments; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = sdata->attachments; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+
+      if(p_exec_stmt(sdata, &sql) == OK) rc = OK;
+      else syslog(LOG_PRIORITY, "ERROR: %s failed to store index data for id=%llu, sql_errno=%d", sdata->ttmpfile, id, sdata->sql_errno);
+
+      close_prepared_statement(&sql);
+   }
 
    return rc;
 }
 
 
-uint64 get_metaid_by_messageid(struct session_data *sdata, struct __data *data, char *message_id, char *piler_id){
+uint64 get_metaid_by_messageid(struct session_data *sdata, char *message_id, char *piler_id){
    uint64 id=0;
+   struct sql sql;
 
-   if(prepare_sql_statement(sdata, &(data->stmt_get_meta_id_by_message_id), SQL_PREPARED_STMT_GET_META_ID_BY_MESSAGE_ID) == ERR) return id;
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_META_ID_BY_MESSAGE_ID) == ERR) return id;
 
-   p_bind_init(data);
+   p_bind_init(&sql);
-   data->sql[data->pos] = message_id; data->type[data->pos] = TYPE_STRING; data->pos++;
+   sql.sql[sql.pos] = message_id; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
 
-   if(p_exec_query(sdata, data->stmt_get_meta_id_by_message_id, data) == OK){
+   if(p_exec_stmt(sdata, &sql) == OK){
 
-      p_bind_init(data);
+      p_bind_init(&sql);
-      data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
-      data->sql[data->pos] = piler_id; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = RND_STR_LEN; data->pos++;
 
-      p_store_results(data->stmt_get_meta_id_by_message_id, data);
+      sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+      sql.sql[sql.pos] = piler_id; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++;
 
-      p_fetch_results(data->stmt_get_meta_id_by_message_id);
+      p_store_results(&sql);
 
-      p_free_results(data->stmt_get_meta_id_by_message_id);
+      p_fetch_results(&sql);
+
+      p_free_results(&sql);
    }
 
-   mysql_stmt_close(data->stmt_get_meta_id_by_message_id);
+   close_prepared_statement(&sql);
 
    return id;
 }
 
 
-int store_recipients(struct session_data *sdata, struct __data *data, char *to, uint64 id, struct __config *cfg){
+int store_recipients(struct session_data *sdata, char *to, uint64 id, struct config *cfg){
-   int ret=OK, n=0;
+   int rc=OK, n=0;
    char *p, *q, puf[SMALLBUFSIZE];
+   struct sql sql;
 
-   if(prepare_sql_statement(sdata, &(data->stmt_insert_into_rcpt_table), SQL_PREPARED_STMT_INSERT_INTO_RCPT_TABLE) == ERR) return ret;
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_RCPT_TABLE) == ERR) return ERR;
 
    p = to;
    do {
@@ -106,79 +176,67 @@ int store_recipients(struct session_data *sdata, struct __data *data, char *to,
       if(q && strlen(q) > 3 && does_it_seem_like_an_email_address(puf) == 1){
          q++;
 
-         p_bind_init(data);
+         p_bind_init(&sql);
 
-         data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
+         sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
-         data->sql[data->pos] = &puf[0]; data->type[data->pos] = TYPE_STRING; data->pos++;
+         sql.sql[sql.pos] = &puf[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-         data->sql[data->pos] = q; data->type[data->pos] = TYPE_STRING; data->pos++;
+         sql.sql[sql.pos] = q; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
 
-
+         if(p_exec_stmt(sdata, &sql) == ERR){
-         if(p_exec_query(sdata, data->stmt_insert_into_rcpt_table, data) == ERR){
+            if(sdata->sql_errno != ER_DUP_ENTRY){
-            if(sdata->sql_errno != ER_DUP_ENTRY) ret = ERR;
+               syslog(LOG_PRIORITY, "ERROR: %s: failed to add '%s' for id=%llu to rcpt table, sql_errno=%d", sdata->ttmpfile, puf, id, sdata->sql_errno);
+               rc = ERR;
+            }
          }
          else n++;
       }
 
    } while(p);
 
-   mysql_stmt_close(data->stmt_insert_into_rcpt_table);
+   close_prepared_statement(&sql);
 
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: added %d recipients", sdata->ttmpfile, n);
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored %d recipients, rc=%d", sdata->ttmpfile, n, rc);
-
-   return ret;
-}
-
-
-void remove_recipients(struct session_data *sdata, uint64 id){
-   char s[SMALLBUFSIZE];
-
-   snprintf(s, sizeof(s)-1, "DELETE FROM " SQL_RECIPIENT_TABLE " WHERE id=%llu", id);
-
-   p_query(sdata, s);
-}
-
-
-int store_folder_id(struct session_data *sdata, struct __data *data, uint64 id){
-   int rc = ERR;
-
-   if(data->folder == ERR_FOLDER) return rc;
-
-   if(prepare_sql_statement(sdata, &(data->stmt_insert_into_folder_message_table), SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE) == ERR) return rc;
-
-   p_bind_init(data);
-
-   data->sql[data->pos] = (char *)&data->folder; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
-   data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
-
-   if(p_exec_query(sdata, data->stmt_insert_into_folder_message_table, data) == OK) rc = OK;
-   close_prepared_statement(data->stmt_insert_into_folder_message_table);
 
    return rc;
 }
 
 
-void remove_folder_id(struct session_data *sdata, uint64 id){
+int store_folder_id(struct session_data *sdata, struct data *data, uint64 id, struct config *cfg){
-   char s[SMALLBUFSIZE];
+   int rc=ERR;
+   struct sql sql;
 
-   snprintf(s, sizeof(s)-1, "DELETE FROM " SQL_FOLDER_MESSAGE_TABLE " WHERE id=%llu", id);
+   if(data->folder == ERR_FOLDER) return rc;
 
-   p_query(sdata, s);
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE) == ERR) return rc;
+
+   p_bind_init(&sql);
+
+   sql.sql[sql.pos] = (char *)&data->folder; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
+
+   if(p_exec_stmt(sdata, &sql) == OK) rc = OK;
+   close_prepared_statement(&sql);
+
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored folderdata, rc=%d", sdata->ttmpfile, rc);
+
+   return rc;
 }
 
 
-int update_metadata_reference(struct session_data *sdata, struct parser_state *state, struct __data *data, char *ref, struct __config *cfg){
+int update_metadata_reference(struct session_data *sdata, struct parser_state *state, char *ref, struct config *cfg){
    int ret = ERR;
+   struct sql sql;
 
-   if(prepare_sql_statement(sdata, &(data->stmt_update_metadata_reference), SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE) == ERR) return ret;
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE) == ERR) return ret;
 
-   p_bind_init(data);
+   p_bind_init(&sql);
 
-   data->sql[data->pos] = ref; data->type[data->pos] = TYPE_STRING; data->pos++;
+   sql.sql[sql.pos] = ref; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   data->sql[data->pos] = state->reference; data->type[data->pos] = TYPE_STRING; data->pos++;
+   sql.sql[sql.pos] = state->reference; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
 
-   if(p_exec_query(sdata, data->stmt_update_metadata_reference, data) == OK) ret = OK;
+   if(p_exec_stmt(sdata, &sql) == OK) ret = OK;
 
-   mysql_stmt_close(data->stmt_update_metadata_reference);
+   close_prepared_statement(&sql);
 
    if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: updated meta reference for '%s', rc=%d", sdata->ttmpfile, state->reference, ret);
 
@@ -186,90 +244,146 @@ int update_metadata_reference(struct session_data *sdata, struct parser_state *s
 }
 
 
-int store_meta_data(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg){
+int store_meta_data(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
-   int rc, ret=ERR, result;
+   int rc=ERR;
-   char *subj, *p, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1];
+   char *subj, *sender, *sender_domain, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1];
    uint64 id=0;
-
+   struct sql sql;
 
    subj = state->b_subject;
    if(*subj == ' ') subj++;
 
-   snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, state->b_from, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
+   if(state->b_sender_domain[0]){
+      sender = state->b_sender;
+      sender_domain = state->b_sender_domain;
+      get_first_email_address_from_string(state->b_sender, s2, sizeof(s2));
+   } else {
+      sender = state->b_from;
+      sender_domain = state->b_from_domain;
+      get_first_email_address_from_string(state->b_from, s2, sizeof(s2));
+   }
 
-   digest_string(s, &vcode[0]);
+   snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, sender, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
+
+   digest_string("sha256", s, &vcode[0]);
 
    memset(ref, 0, sizeof(ref));
    if(strlen(state->reference) > 10){
-      digest_string(state->reference, &ref[0]);
+      digest_string("sha256", state->reference, &ref[0]);
-      update_metadata_reference(sdata, state, data, &ref[0], cfg);
+      update_metadata_reference(sdata, state, &ref[0], cfg);
+   }
+   else if(state->reference[0] == 0){
+      // during import, the order of messages is often random
+      // check if this is a message which is already referenced
+      uint64 count=0;
+
+      digest_string("sha256", state->message_id, &ref[0]);
+      if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_METADATA_REFERENCE) != ERR){
+         p_bind_init(&sql);
+
+         sql.sql[sql.pos] = &ref[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+
+         if(p_exec_stmt(sdata, &sql) == OK){
+            p_bind_init(&sql);
+
+            sql.sql[sql.pos] = (char *)&count; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+            p_store_results(&sql);
+            p_fetch_results(&sql);
+            p_free_results(&sql);
+         }
+      }
+
+      close_prepared_statement(&sql);
+
+      // no reference yet
+      if(count <= 0)
+         ref[0] = 0;
    }
 
 
-   if(prepare_sql_statement(sdata, &(data->stmt_insert_into_meta_table), SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR;
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR;
-
-   memset(s2, 0, sizeof(s2));
-
-   p = state->b_from;
-   do {
-      memset(s2, 0, sizeof(s2));
-      p = split(p, ' ', s2, sizeof(s2)-1, &result);
-
-      if(s2[0] == '\0') continue;
-
-      if(does_it_seem_like_an_email_address(s2) == 1){ break; }
-   } while(p);
-
 
    if(strlen(state->b_to) < 5){
       snprintf(state->b_to, SMALLBUFSIZE-1, "undisclosed-recipients@no.domain");
    }
 
 
-   p_bind_init(data);
+   p_bind_init(&sql);
 
-   data->sql[data->pos] = &s2[0]; data->type[data->pos] = TYPE_STRING; data->pos++;
+   sql.sql[sql.pos] = &s2[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   data->sql[data->pos] = state->b_from_domain; data->type[data->pos] = TYPE_STRING; data->pos++;
+   sql.sql[sql.pos] = sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   data->sql[data->pos] = subj; data->type[data->pos] = TYPE_STRING; data->pos++;
+   sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   data->sql[data->pos] = (char *)&sdata->spam_message; data->type[data->pos] = TYPE_LONG; data->pos++;
+   sql.sql[sql.pos] = (char *)&sdata->spam_message; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   data->sql[data->pos] = (char *)&sdata->now; data->type[data->pos] = TYPE_LONG; data->pos++;
+   sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   data->sql[data->pos] = (char *)&sdata->sent; data->type[data->pos] = TYPE_LONG; data->pos++;
+   sql.sql[sql.pos] = (char *)&sdata->sent; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   data->sql[data->pos] = (char *)&sdata->retained; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
+   sql.sql[sql.pos] = (char *)&sdata->retained; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
-   data->sql[data->pos] = (char *)&sdata->tot_len; data->type[data->pos] = TYPE_LONG; data->pos++;
+   sql.sql[sql.pos] = (char *)&sdata->tot_len; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   data->sql[data->pos] = (char *)&sdata->hdr_len; data->type[data->pos] = TYPE_LONG; data->pos++;
+   sql.sql[sql.pos] = (char *)&sdata->hdr_len; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   data->sql[data->pos] = (char *)&sdata->direction; data->type[data->pos] = TYPE_LONG; data->pos++;
+   sql.sql[sql.pos] = (char *)&sdata->direction; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   data->sql[data->pos] = (char *)&state->n_attachments; data->type[data->pos] = TYPE_LONG; data->pos++;
+   sql.sql[sql.pos] = (char *)&state->n_attachments; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   data->sql[data->pos] = sdata->ttmpfile; data->type[data->pos] = TYPE_STRING; data->pos++;
+   sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   data->sql[data->pos] = state->message_id; data->type[data->pos] = TYPE_STRING; data->pos++;
+   sql.sql[sql.pos] = state->message_id; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   data->sql[data->pos] = &ref[0]; data->type[data->pos] = TYPE_STRING; data->pos++;
+   sql.sql[sql.pos] = &ref[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   data->sql[data->pos] = sdata->digest; data->type[data->pos] = TYPE_STRING; data->pos++;
+   sql.sql[sql.pos] = sdata->digest; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   data->sql[data->pos] = sdata->bodydigest; data->type[data->pos] = TYPE_STRING; data->pos++;
+   sql.sql[sql.pos] = sdata->bodydigest; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   data->sql[data->pos] = &vcode[0]; data->type[data->pos] = TYPE_STRING; data->pos++;
+   sql.sql[sql.pos] = &vcode[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
 
-   if(p_exec_query(sdata, data->stmt_insert_into_meta_table, data) == ERR){
+   if(p_exec_stmt(sdata, &sql) == OK){
-      ret = ERR_EXISTS;
+      id = p_get_insert_id(&sql);
-   }
-   else {
-      id = p_get_insert_id(data->stmt_insert_into_meta_table);
-      rc = store_recipients(sdata, data, state->b_to, id, cfg);
 
-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored recipients, rc=%d", sdata->ttmpfile, rc);
+      if(store_recipients(sdata, state->b_to, id, cfg) == OK){
 
-      rc = store_index_data(sdata, state, data, id, cfg);
+         if(store_index_data(sdata, state, data, id, cfg) == OK) rc = OK;
 
-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored indexdata, rc=%d", sdata->ttmpfile, rc);
+         if(cfg->enable_folders == 1){
-
+            rc = store_folder_id(sdata, data, id, cfg);
-      if(cfg->enable_folders == 1){
+         }
-         rc = store_folder_id(sdata, data, id);
-         if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored folderdata, rc=%d", sdata->ttmpfile, rc);
       }
 
-      ret = OK;
+      // There were some sql errors, so we should rollback everything
+      if(rc == ERR){
+         rollback(sdata, state, id, cfg);
+      }
+
+   }
+   else {
+      syslog(LOG_PRIORITY, "ERROR: %s storing metadata, sql_errno=%d", sdata->ttmpfile, sdata->sql_errno);
    }
 
-   close_prepared_statement(data->stmt_insert_into_meta_table);
+   close_prepared_statement(&sql);
 
-   return ret;
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored metadata, rc=%d",  sdata->ttmpfile, rc);
+
+   return rc;
+}
+
+
+void rollback(struct session_data *sdata, struct parser_state *state, uint64 id, struct config *cfg){
+   char buf[SMALLBUFSIZE];
+
+   snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE id=%llu", SQL_SPHINX_TABLE, id);
+   p_query(sdata, buf);
+   syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
+
+   snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE id=%llu", SQL_RECIPIENT_TABLE, id);
+   p_query(sdata, buf);
+   syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
+
+   snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE id=%llu", SQL_METADATA_TABLE, id);
+   p_query(sdata, buf);
+   syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
+
+   snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE piler_id='%s'", SQL_ATTACHMENT_TABLE, sdata->ttmpfile);
+   p_query(sdata, buf);
+   syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
+
+   if(cfg->enable_folders == 1){
+      snprintf(buf, sizeof(buf)-1, "DELETE FROM " SQL_FOLDER_MESSAGE_TABLE " WHERE id=%llu", id);
+      p_query(sdata, buf);
+      syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
+   }
+
+   remove_stored_message_files(sdata, state, cfg);
 }
 
 
@@ -280,20 +394,20 @@ void remove_stripped_attachments(struct parser_state *state){
 }
 
 
-int process_message(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg){
+int is_duplicated_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
-   int rc, fd;
+   int fd;
    char piler_id[SMALLBUFSIZE];
 
    /* discard if existing message_id */
 
-   sdata->duplicate_id = get_metaid_by_messageid(sdata, data, state->message_id, piler_id);
+   sdata->duplicate_id = get_metaid_by_messageid(sdata, state->message_id, piler_id);
 
    if(sdata->duplicate_id > 0){
       remove_stripped_attachments(state);
 
-      if(strlen(state->b_journal_to) > 0){
+      if(sdata->restored_copy == 0 && strlen(state->b_journal_to) > 0){
          if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to add journal rcpt (%s) to id=%llu for message-id: '%s'", sdata->ttmpfile, state->b_journal_to, sdata->duplicate_id, state->message_id);
-         store_recipients(sdata, data, state->b_journal_to, sdata->duplicate_id, cfg);
+         store_recipients(sdata, state->b_journal_to, sdata->duplicate_id, cfg);
       }
 
       return ERR_EXISTS;
@@ -302,7 +416,7 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru
    fd = open(state->message_id_hash, O_CREAT|O_EXCL, S_IRUSR|S_IWUSR);
    if(fd == -1){
       remove_stripped_attachments(state);
-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: touch %s FAILED (%s)", sdata->ttmpfile, state->message_id_hash, state->message_id);
+      syslog(LOG_PRIORITY, "%s: touch %s FAILED (%s), error: %s", sdata->ttmpfile, state->message_id_hash, state->message_id, strerror(errno));
       return ERR_EXISTS;
    }
    close(fd);
@@ -310,12 +424,13 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru
    if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: touch %s OK (%s)", sdata->ttmpfile, state->message_id_hash, state->message_id);
 
 
-
    if(cfg->mmap_dedup_test == 1 && data->dedup != MAP_FAILED && data->child_serial >= 0 && data->child_serial < MAXCHILDREN){
 
       if(strstr(data->dedup, state->message_id_hash)){
-         if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: dedup string: %s", sdata->ttmpfile, data->dedup);
+         if(cfg->verbosity >= _LOG_DEBUG){
-         if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: message-id-hash=%s, serial=%d", sdata->ttmpfile, state->message_id_hash, data->child_serial);
+            syslog(LOG_INFO, "%s: dedup string: %s", sdata->ttmpfile, data->dedup);
+            syslog(LOG_INFO, "%s: message-id-hash=%s, serial=%d", sdata->ttmpfile, state->message_id_hash, data->child_serial);
+         }
 
          remove_stripped_attachments(state);
          return ERR_EXISTS;
@@ -324,37 +439,26 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru
       memcpy(data->dedup + data->child_serial*DIGEST_LENGTH*2, state->message_id_hash, DIGEST_LENGTH*2);
    }
 
+   return OK;
+}
 
 
+int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
 
-   /* store base64 encoded file attachments */
+   if(is_duplicated_message(sdata, state, data, cfg) == ERR_EXISTS)
+      return ERR_EXISTS;
 
-   if(state->n_attachments > 0){
+   sdata->retained += query_retain_period(data, state, sdata->tot_len, sdata->spam_message, cfg);
-      rc = store_attachments(sdata, state, data, cfg);
-
-      remove_stripped_attachments(state);
-
-      if(rc) return ERR;
-   }
 
 
-   rc = store_file(sdata, sdata->tmpframe, 0, cfg);
+   if(state->n_attachments > 0 && store_attachments(sdata, state, cfg) == ERR) return ERR;
-   if(rc == 0){
+
-      syslog(LOG_PRIORITY, "%s: error storing message: %s", sdata->ttmpfile, sdata->tmpframe);
+
+   if(store_file(sdata, sdata->tmpframe, 0, cfg) == 0){
+      syslog(LOG_PRIORITY, "ERROR: %s: failed to store message: %s", sdata->ttmpfile, sdata->tmpframe);
       return ERR;
    }
 
 
-   sdata->retained += query_retain_period(data, state, sdata->tot_len, sdata->spam_message, cfg);
+   return store_meta_data(sdata, state, data, cfg);
-
-   rc = store_meta_data(sdata, state, data, cfg);
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored metadata, rc=%d",  sdata->ttmpfile, rc);
-   if(rc == ERR_EXISTS){
-
-      remove_stored_message_files(sdata, state, cfg);
-
-      return ERR_EXISTS;
-   }
-
-   return OK;
 }

src/misc.c

@@ -63,6 +63,10 @@ void get_extractor_list(){
       printf("%s ", HAVE_TNEF);
    #endif
 
+   #ifdef HAVE_ZIP
+      printf("libzip ");
+   #endif
+
    printf("\n\n");
 }
 
@@ -73,7 +77,7 @@ void __fatal(char *s){
 }
 
 /*
- * calculate the difference betwwen two timevals in [usec]
+ * calculate the difference between two timevals in [usec]
  */
 
 long tvdiff(struct timeval a, struct timeval b){
@@ -84,81 +88,6 @@ long tvdiff(struct timeval a, struct timeval b){
 }
 
 
-/*
- * search something in a buffer
- */
-
-int searchStringInBuffer(char *s, int len1, char *what, int len2){
-   int i, k, r;
-
-   for(i=0; i<len1; i++){
-      r = 0;
-
-      for(k=0; k<len2; k++){
-         if(*(s+i+k) == *(what+k))
-            r++;
-      }
-
-      if(r == len2)
-         return i;
-   }
-
-   return 0;
-}
-
-
-int search_char_backward(char *buf, int buflen, char c){
-   int n, m;
-
-   m = buflen - 1 - 5;
-   if(m < 0) m = 0;
-
-   for(n=m; n<buflen; n++){
-      if(*(buf + n) == c){
-         return n;
-      }
-   }
-   return -1;
-}
-
-
-/*
- * count a character in buffer
- */
-
-int countCharacterInBuffer(char *p, char c){
-   int i=0;
-
-   for(; *p; p++){
-      if(*p == c)
-         i++;
-   }
-
-   return i;
-}
-
-
-void replaceCharacterInBuffer(char *p, char from, char to){
-   int i, k=0;
-
-   for(i=0; i<strlen(p); i++){
-      if(p[i] == from){
-         if(to > 0){
-            p[k] = to;
-            k++;
-         }
-      }
-      else {
-         p[k] = p[i];
-         k++;
-      }
-
-   }
-
-   p[k] = '\0';
-}
-
-
 /*
  * split a string by a character as delimiter
  */
@@ -214,10 +143,8 @@ char *split_str(char *row, char *what, char *s, int size){
       r += strlen(what);
    }
 
-   if(s != NULL){
+   strncpy(s, row, len);
-      strncpy(s, row, len);
+   s[len] = '\0';
-      s[len] = '\0';
-   }
 
    return r;
 }
@@ -231,13 +158,13 @@ int trimBuffer(char *s){
    int n=0;
    char *p;
 
-   p = strrchr(s, '\n');
+   p = strchr(s, '\n');
    if(p){
       *p = '\0';
       n++;
    }
 
-   p = strrchr(s, '\r');
+   p = strchr(s, '\r');
    if(p){
       *p = '\0';
       n++;
@@ -248,8 +175,7 @@ int trimBuffer(char *s){
 
 
 int extract_verp_address(char *email){
-   char *p, *p1, *p2;
+   char *p1;
-   char puf[SMALLBUFSIZE];
 
    // a VERP address is like archive+user=domain.com@myarchive.local
 
@@ -259,13 +185,14 @@ int extract_verp_address(char *email){
 
    p1 = strchr(email, '+');
    if(p1){
-      p2 = strchr(p1, '@');
+      char *p2 = strchr(p1, '@');
       if(p2 && p2 > p1 + 2){
          if(strchr(p1+1, '=')){
+            char puf[SMALLBUFSIZE];
             memset(puf, 0, sizeof(puf));
 
             memcpy(&puf[0], p1+1, p2-p1-1);
-            p = strchr(puf, '=');
+            char *p = strchr(puf, '=');
             if(p) *p = '@';
             strcpy(email, puf);
          }
@@ -302,19 +229,15 @@ int extractEmail(char *rawmail, char *email){
  * using the rand() function if not possible
  */
 
-void make_random_string(char *buf, int buflen){
+void make_random_string(unsigned char *buf, int buflen){
-   int i, len, fd;
+   const char alphanum[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
-   int urandom=0;
+   int i, fd, urandom=0, len = sizeof(alphanum)-1;
-   static char alphanum[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
-   unsigned char s[QUEUE_ID_LEN];
-
-   len = strlen(alphanum);
 
    fd = open(RANDOM_POOL, O_RDONLY);
    if(fd != -1){
-      if(readFromEntropyPool(fd, s, sizeof(s)) == sizeof(s)){
+      if(readFromEntropyPool(fd, buf, buflen) == buflen){
-         for(i=0; i<QUEUE_ID_LEN; i++){
+         for(i=0; i<buflen; i++){
-            *(buf+i) = alphanum[s[i] % len];
+            *(buf+i) = alphanum[*(buf+i) % len];
          }
 
          urandom = 1;
@@ -339,11 +262,13 @@ void create_id(char *id, unsigned char server_id){
 
    get_random_bytes(buf, RND_STR_LEN/2, server_id);
 
+   // New encryption scheme using AES-256
+   buf[0] = 0x50;
+
    for(i=0; i < RND_STR_LEN/2; i++){
       sprintf(id, "%02x", buf[i]);
       id += 2;
    }
-
 }
 
 
@@ -361,7 +286,7 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
    taia_now(&now);
    taia_pack(nowpack, &now);
 
-   memcpy(buf, nowpack, 12);
+   memcpy(buf, nowpack, 12); //-V512
 
    fd = open(RANDOM_POOL, O_RDONLY);
    if(fd == -1) return ret;
@@ -383,10 +308,10 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
 
 int readFromEntropyPool(int fd, void *_s, ssize_t n){
    char *s = _s;
-   ssize_t res, pos = 0;
+   ssize_t pos = 0;
 
    while(n > pos){
-      res = read(fd, s + pos, n - pos);
+      ssize_t res = read(fd, s + pos, n - pos);
       switch(res){
          case  -1: continue;
          case   0: return res;
@@ -422,13 +347,13 @@ int recvtimeout(int s, char *buf, int len, int timeout){
 }
 
 
-int write1(int sd, void *buf, int buflen, int use_ssl, SSL *ssl){
+int write1(struct net *net, void *buf, int buflen){
    int n;
 
-   if(use_ssl == 1)
+   if(net->use_ssl == 1)
-      n = SSL_write(ssl, buf, buflen);
+      n = SSL_write(net->ssl, buf, buflen);
    else
-      n = send(sd, buf, buflen, 0);
+      n = send(net->socket, buf, buflen, 0);
 
    return n;
 }
@@ -489,26 +414,26 @@ int ssl_read_timeout(SSL *ssl, void *buf, int len, int timeout){
 }
 
 
-int recvtimeoutssl(int s, char *buf, int len, int timeout, int use_ssl, SSL *ssl){
+int recvtimeoutssl(struct net *net, char *buf, int len){
 
     memset(buf, 0, len);
 
-    if(use_ssl == 1){
+    if(net->use_ssl == 1){
-       return ssl_read_timeout(ssl, buf, len-1, timeout);
+       return ssl_read_timeout(net->ssl, buf, len-1, net->timeout);
     }
     else {
-       return recvtimeout(s, buf, len-1, timeout);
+       return recvtimeout(net->socket, buf, len-1, net->timeout);
     }
 }
 
 
-void close_connection(int sd, struct __data *data, int use_ssl){
+void close_connection(struct net *net){
-   close(sd);
+   close(net->socket);
 
-   if(use_ssl == 1){
+   if(net->use_ssl == 1){
-      SSL_shutdown(data->ssl);
+      SSL_shutdown(net->ssl);
-      SSL_free(data->ssl);
+      SSL_free(net->ssl);
-      SSL_CTX_free(data->ctx);
+      SSL_CTX_free(net->ctx);
       ERR_free_strings();
    }
 }
@@ -544,7 +469,7 @@ int drop_privileges(struct passwd *pwd){
 }
 
 
-void init_session_data(struct session_data *sdata, struct __config *cfg){
+void init_session_data(struct session_data *sdata, struct config *cfg){
    int i;
 
 
@@ -639,12 +564,6 @@ void strtolower(char *s){
 }
 
 
-void *get_in_addr(struct sockaddr *sa){
-   if(sa->sa_family == AF_INET) return &(((struct sockaddr_in*)sa)->sin_addr);
-   return &(((struct sockaddr_in6*)sa)->sin6_addr);
-}
-
-
 int make_socket_non_blocking(int fd){
    int flags, s;
 
@@ -733,9 +652,86 @@ void move_email(struct smtp_session *session){
 
    snprintf(buf, sizeof(buf)-1, "%d/%s", session->ttmpfile[0] % session->cfg->number_of_worker_processes, session->ttmpfile);
 
-   if(rename(session->ttmpfile, buf)){
+   if(session->nullbyte){
-      syslog(LOG_PRIORITY, "ERROR: couldn't rename %s to %s", session->ttmpfile, buf);
+      snprintf(buf, sizeof(buf)-1, "%s/%s", ERROR_DIR, session->ttmpfile);
+      syslog(LOG_PRIORITY, "ERROR: %s contains an invalid NUL-byte, moving it to %s", session->ttmpfile, ERROR_DIR);
    }
+
+   if(rename(session->ttmpfile, buf)){
+      syslog(LOG_PRIORITY, "ERROR: couldn't rename %s to %s (reason: %s)", session->ttmpfile, buf, strerror(errno));
+   }
+}
+
+
+int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte){
+   int i=0;
+
+   *rc = ERR;
+   *nullbyte = 0;
+
+   memset(buf, 0, buflen);
+
+   for(int j=0; j<slen; j++){
+      if(i<buflen-2){
+         if(*(s+j) == 0){
+            *nullbyte = 1;
+         }
+
+         buf[i] = *(s+j);
+         i++;
+
+         if(*(s+j) == c){
+            *rc = OK;
+            break;
+         }
+      }
+      else {
+         break;
+      }
+   }
+
+   return i;
+}
+
+
+int init_ssl_to_server(struct data *data){
+   int n;
+   X509* server_cert;
+   char *str;
+
+   SSL_library_init();
+   SSL_load_error_strings();
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+   data->net->ctx = SSL_CTX_new(TLSv1_2_client_method());
+#else
+   data->net->ctx = SSL_CTX_new(TLS_client_method());
+#endif
+   CHK_NULL(data->net->ctx, "internal SSL error");
+
+   data->net->ssl = SSL_new(data->net->ctx);
+   CHK_NULL(data->net->ssl, "internal ssl error");
+
+   SSL_set_fd(data->net->ssl, data->net->socket);
+   n = SSL_connect(data->net->ssl);
+   CHK_SSL(n, "internal ssl error");
+
+   //printf("Cipher: %s\n", SSL_get_cipher(data->net->ssl));
+
+   server_cert = SSL_get_peer_certificate(data->net->ssl);
+   CHK_NULL(server_cert, "server cert error");
+
+   str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0);
+   CHK_NULL(str, "error in server cert");
+   OPENSSL_free(str);
+
+   str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0);
+   CHK_NULL(str, "error in server cert");
+   OPENSSL_free(str);
+
+   X509_free(server_cert);
+
+   return OK;
 }
 
 
@@ -759,3 +755,55 @@ char *strcasestr(const char *s, const char *find){
    return((char*)s);
 }
 #endif
+
+
+int append_string_to_buffer(char **buffer, char *str){
+   int arglen;
+   char *s=NULL;
+
+   arglen = strlen(str);
+
+   if(!*buffer){
+      *buffer = malloc(arglen+1);
+      memset(*buffer, 0, arglen+1);
+      memcpy(*buffer, str, arglen);
+   }
+   else {
+      int len = strlen(*buffer);
+      s = realloc(*buffer, len + arglen+1);
+      if(!s) return 1;
+
+      *buffer = s;
+
+      memset(*buffer+len, 0, arglen+1);
+      memcpy(*buffer+len, str, arglen);
+   }
+
+   return 0;
+}
+
+
+int get_size_from_smtp_mail_from(char *s){
+   int size=0;
+   char *p;
+
+   p = strcasestr(s, "SIZE=");
+   if(p){
+      p += strlen("SIZE=");
+      char *q = p;
+      for(; *q; q++){
+         if(isspace(*q)) break;
+      }
+
+      // We extract max. 9 characters, which is just under 1GB
+      // and not overflowing an int variable
+      if(q - p <= 9){
+         char c = *q;
+         *q = '\0';
+         size = atoi(p);
+         *q = c;
+      }
+   }
+
+   return size;
+}

src/misc.h

@@ -19,40 +19,42 @@ int get_build();
 void get_extractor_list();
 void __fatal(char *s);
 long tvdiff(struct timeval a, struct timeval b);
-int searchStringInBuffer(char *s, int len1, char *what, int len2);
-int search_char_backward(char *buf, int buflen, char c);
-int countCharacterInBuffer(char *p, char c);
-void replaceCharacterInBuffer(char *p, char from, char to);
 char *split(char *str, int ch, char *buf, int buflen, int *result);
 char *split_str(char *row, char *what, char *s, int size);
 int trimBuffer(char *s);
 int extractEmail(char *rawmail, char *email);
-void make_random_string(char *buf, int buflen);
+int extract_verp_address(char *email);
+void make_random_string(unsigned char *buf, int buflen);
 void create_id(char *id, unsigned char server_id);
 int get_random_bytes(unsigned char *buf, int len, unsigned char server_id);
 int readFromEntropyPool(int fd, void *_s, ssize_t n);
 int recvtimeout(int s, char *buf, int len, int timeout);
-int write1(int sd, void *buf, int buflen, int use_ssl, SSL *ssl);
+int write1(struct net *net, void *buf, int buflen);
-int recvtimeoutssl(int s, char *buf, int len, int timeout, int use_ssl, SSL *ssl);
+int recvtimeoutssl(struct net *net, char *buf, int len);
-void close_connection(int sd, struct __data *data, int use_ssl);
+void close_connection(struct net *net);
 
 void write_pid_file(char *pidfile);
 int drop_privileges(struct passwd *pwd);
 
-void init_session_data(struct session_data *sdata, struct __config *cfg);
+void init_session_data(struct session_data *sdata, struct config *cfg);
 int read_from_stdin(struct session_data *sdata);
 void strtolower(char *s);
 
-void *get_in_addr(struct sockaddr *sa);
 int make_socket_non_blocking(int fd);
 int create_and_bind(char *listen_addr, int listen_port);
 
 int can_i_write_directory(char *dir);
 
 void move_email(struct smtp_session *session);
+int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte);
+
+int init_ssl_to_server(struct data *data);
 
 #ifndef _GNU_SOURCE
    char *strcasestr(const char *s, const char *find);
 #endif
 
+int append_string_to_buffer(char **buffer, char *str);
+int get_size_from_smtp_mail_from(char *s);
+
 #endif /* _MISC_H */

src/mydomains.c

@@ -10,43 +10,41 @@
 #include <piler.h>
 
 
-void load_mydomains(struct session_data *sdata, struct __data *data, struct __config *cfg){
+void load_mydomains(struct session_data *sdata, struct data *data, struct config *cfg){
-   int rc;
    char s[SMALLBUFSIZE];
+   struct sql sql;
 
    memset(s, 0, sizeof(s));
 
 
-   if(prepare_sql_statement(sdata, &(data->stmt_generic), SQL_PREPARED_STMT_GET_DOMAINS) == ERR) return;
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_DOMAINS) == ERR) return;
 
 
-   p_bind_init(data);
+   p_bind_init(&sql);
 
-   if(p_exec_query(sdata, data->stmt_generic, data) == OK){
+   if(p_exec_stmt(sdata, &sql) == OK){
 
-      p_bind_init(data);
+      p_bind_init(&sql);
 
-      data->sql[data->pos] = &s[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s)-2; data->pos++;
+      sql.sql[sql.pos] = &s[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(s)-2; sql.pos++;
 
-      p_store_results(data->stmt_generic, data);
+      p_store_results(&sql);
 
-      while(p_fetch_results(data->stmt_generic) == OK){
+      while(p_fetch_results(&sql) == OK){
-         rc = addnode(data->mydomains, s);
+         if(addnode(data->mydomains, s) == 0) syslog(LOG_PRIORITY, "failed to append mydomain: '%s'", s);
-
-         if(rc == 0) syslog(LOG_PRIORITY, "failed to append mydomain: '%s'", s);
          else if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "added mydomain: '%s'", s);
 
          memset(s, 0, sizeof(s));
       }
 
-      p_free_results(data->stmt_generic);
+      p_free_results(&sql);
    }
 
-   close_prepared_statement(data->stmt_generic);
+   close_prepared_statement(&sql);
 }
 
 
-int is_email_address_on_my_domains(char *email, struct __data *data){
+int is_email_address_on_my_domains(char *email, struct data *data){
    int rc=0;
    char *q, *s;
 
@@ -65,5 +63,3 @@ int is_email_address_on_my_domains(char *email, struct __data *data){
 
    return rc;
 }
-
-

src/mysql.c

@@ -9,7 +9,7 @@
 #include <piler.h>
 
 
-int open_database(struct session_data *sdata, struct __config *cfg){
+int open_database(struct session_data *sdata, struct config *cfg){
    int rc=1;
    char buf[BUFLEN];
 
@@ -19,7 +19,7 @@ int open_database(struct session_data *sdata, struct __config *cfg){
    mysql_options(&(sdata->mysql), MYSQL_OPT_RECONNECT, (const char*)&rc);
 
    if(mysql_real_connect(&(sdata->mysql), cfg->mysqlhost, cfg->mysqluser, cfg->mysqlpwd, cfg->mysqldb, cfg->mysqlport, cfg->mysqlsocket, 0) == 0){
-      printf("cant connect to mysql server\n");
+      syslog(LOG_PRIORITY, "cant connect to mysql server");
       return ERR;
    }
 
@@ -33,20 +33,49 @@ int open_database(struct session_data *sdata, struct __config *cfg){
 }
 
 
+int open_sphx(struct session_data *sdata, struct config *cfg){
+   int rc=1;
+   char buf[BUFLEN];
+
+   mysql_init(&(sdata->sphx));
+
+   mysql_options(&(sdata->sphx), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg->mysql_connect_timeout);
+   mysql_options(&(sdata->sphx), MYSQL_OPT_RECONNECT, (const char*)&rc);
+
+   if(mysql_real_connect(&(sdata->sphx), cfg->sphxhost, "", "", cfg->sphxdb, cfg->sphxport, "", 0) == 0){
+      syslog(LOG_PRIORITY, "cant connect to %s:%d", cfg->sphxhost, cfg->sphxport);
+      return ERR;
+   }
+
+   snprintf(buf, sizeof(buf)-2, "SET NAMES %s", cfg->mysqlcharset);
+   mysql_real_query(&(sdata->sphx), buf, strlen(buf));
+
+   snprintf(buf, sizeof(buf)-2, "SET CHARACTER SET %s", cfg->mysqlcharset);
+   mysql_real_query(&(sdata->sphx), buf, strlen(buf));
+
+   return OK;
+}
+
+
 void close_database(struct session_data *sdata){
    mysql_close(&(sdata->mysql));
 }
 
 
-void p_bind_init(struct __data *data){
+void close_sphx(struct session_data *sdata){
+   mysql_close(&(sdata->sphx));
+}
+
+
+void p_bind_init(struct sql *sql){
    int i;
 
-   data->pos = 0;
+   sql->pos = 0;
 
    for(i=0; i<MAX_SQL_VARS; i++){
-      data->sql[i] = NULL;
+      sql->sql[i] = NULL;
-      data->type[i] = TYPE_UNDEF;
+      sql->type[i] = TYPE_UNDEF;
-      data->len[i] = 0;
+      sql->len[i] = 0;
    }
 }
 
@@ -56,7 +85,7 @@ void p_query(struct session_data *sdata, char *s){
 }
 
 
-int p_exec_query(struct session_data *sdata, MYSQL_STMT *stmt, struct __data *data){
+int p_exec_stmt(struct session_data *sdata, struct sql *sql){
    MYSQL_BIND bind[MAX_SQL_VARS];
    unsigned long length[MAX_SQL_VARS];
    int i, ret=ERR;
@@ -65,10 +94,10 @@ int p_exec_query(struct session_data *sdata, MYSQL_STMT *stmt, struct __data *da
    memset(bind, 0, sizeof(bind));
 
    for(i=0; i<MAX_SQL_VARS; i++){
-      if(data->type[i] > TYPE_UNDEF){
+      if(sql->type[i] > TYPE_UNDEF){
 
 
-         switch(data->type[i]) {
+         switch(sql->type[i]) {
              case TYPE_SHORT:
                                   bind[i].buffer_type = MYSQL_TYPE_SHORT;
                                   bind[i].length = 0;
@@ -88,7 +117,7 @@ int p_exec_query(struct session_data *sdata, MYSQL_STMT *stmt, struct __data *da
 
              case TYPE_STRING:
                                   bind[i].buffer_type = MYSQL_TYPE_STRING;
-                                  length[i] = strlen(data->sql[i]);
+                                  length[i] = strlen(sql->sql[i]);
                                   bind[i].length = &length[i];
                                   break;
 
@@ -101,23 +130,22 @@ int p_exec_query(struct session_data *sdata, MYSQL_STMT *stmt, struct __data *da
          };
 
 
-         bind[i].buffer = data->sql[i];
+         bind[i].buffer = sql->sql[i];
          bind[i].is_null = 0;
 
-         
       }
       else { break; }
    }
 
-   if(mysql_stmt_bind_param(stmt, bind)){
+   if(mysql_stmt_bind_param(sql->stmt, bind)){
-      sdata->sql_errno = mysql_stmt_errno(stmt);
+      sdata->sql_errno = mysql_stmt_errno(sql->stmt);
-      syslog(LOG_PRIORITY, "%s: mysql_stmt_bind_param() error: %s (errno: %d)", sdata->ttmpfile, mysql_stmt_error(stmt), sdata->sql_errno);
+      syslog(LOG_PRIORITY, "ERROR: %s: mysql_stmt_bind_param() '%s' (errno: %d)", sdata->ttmpfile, mysql_stmt_error(sql->stmt), sdata->sql_errno);
       return ret;
    }
 
-   if(mysql_stmt_execute(stmt)){
+   if(mysql_stmt_execute(sql->stmt)){
-      sdata->sql_errno = mysql_stmt_errno(stmt);
+      sdata->sql_errno = mysql_stmt_errno(sql->stmt);
-      syslog(LOG_PRIORITY, "%s: mysql_stmt_execute error: *%s* (errno: %d)", sdata->ttmpfile, mysql_error(&(sdata->mysql)), sdata->sql_errno);
+      syslog(LOG_PRIORITY, "ERROR: %s: mysql_stmt_execute() '%s' (errno: %d)", sdata->ttmpfile, mysql_error(&(sdata->mysql)), sdata->sql_errno);
       return ret;
    }
 
@@ -127,16 +155,16 @@ int p_exec_query(struct session_data *sdata, MYSQL_STMT *stmt, struct __data *da
 }
 
 
-int p_store_results(MYSQL_STMT *stmt, struct __data *data){
+int p_store_results(struct sql *sql){
    MYSQL_BIND bind[MAX_SQL_VARS];
    int i, ret=ERR;
 
    memset(bind, 0, sizeof(bind));
 
    for(i=0; i<MAX_SQL_VARS; i++){
-      if(data->type[i] > TYPE_UNDEF){
+      if(sql->type[i] > TYPE_UNDEF){
 
-         switch(data->type[i]) {
+         switch(sql->type[i]) {
              case TYPE_SHORT:     bind[i].buffer_type = MYSQL_TYPE_SHORT;
                                   break;
 
@@ -152,7 +180,7 @@ int p_store_results(MYSQL_STMT *stmt, struct __data *data){
 
              case TYPE_STRING:
                                   bind[i].buffer_type = MYSQL_TYPE_STRING;
-                                  bind[i].buffer_length = data->len[i];
+                                  bind[i].buffer_length = sql->len[i];
                                   break;
 
              default:
@@ -162,21 +190,21 @@ int p_store_results(MYSQL_STMT *stmt, struct __data *data){
          };
 
 
-         bind[i].buffer = (char *)data->sql[i];
+         bind[i].buffer = (char *)sql->sql[i];
-         bind[i].is_null = &(data->is_null[i]);
+         bind[i].is_null = &(sql->is_null[i]);
-         bind[i].length = &(data->length[i]);
+         bind[i].length = &(sql->length[i]);
-         bind[i].error = &(data->error[i]);
+         bind[i].error = &(sql->error[i]);
 
       }
       else { break; }
    }
 
-   if(mysql_stmt_bind_result(stmt, bind)){
+   if(mysql_stmt_bind_result(sql->stmt, bind)){
       return ret;
    }
 
 
-   if(mysql_stmt_store_result(stmt)){
+   if(mysql_stmt_store_result(sql->stmt)){
       return ret;
    }
 
@@ -186,39 +214,39 @@ int p_store_results(MYSQL_STMT *stmt, struct __data *data){
 }
 
 
-int p_fetch_results(MYSQL_STMT *stmt){
+int p_fetch_results(struct sql *sql){
 
-   if(mysql_stmt_fetch(stmt) == 0) return OK;
+   if(mysql_stmt_fetch(sql->stmt) == 0) return OK;
 
    return ERR;
 }
 
 
-void p_free_results(MYSQL_STMT *stmt){
+void p_free_results(struct sql *sql){
-   mysql_stmt_free_result(stmt);
+   mysql_stmt_free_result(sql->stmt);
 }
 
 
-uint64 p_get_insert_id(MYSQL_STMT *stmt){
+uint64 p_get_insert_id(struct sql *sql){
-   return mysql_stmt_insert_id(stmt);
+   return mysql_stmt_insert_id(sql->stmt);
 }
 
 
-int p_get_affected_rows(MYSQL_STMT *stmt){
+int p_get_affected_rows(struct sql *sql){
-   return mysql_stmt_affected_rows(stmt);
+   return mysql_stmt_affected_rows(sql->stmt);
 }
 
 
-int prepare_sql_statement(struct session_data *sdata, MYSQL_STMT **stmt, char *s){
+int prepare_sql_statement(struct session_data *sdata, struct sql *sql, char *s){
 
-   *stmt = mysql_stmt_init(&(sdata->mysql));
+   sql->stmt = mysql_stmt_init(&(sdata->mysql));
-   if(!*stmt){
+   if(!(sql->stmt)){
-      syslog(LOG_PRIORITY, "%s: mysql_stmt_init() error", sdata->ttmpfile);
+      syslog(LOG_PRIORITY, "%s: error: mysql_stmt_init()", sdata->ttmpfile);
       return ERR;
    }
 
-   if(mysql_stmt_prepare(*stmt, s, strlen(s))){
+   if(mysql_stmt_prepare(sql->stmt, s, strlen(s))){
-      syslog(LOG_PRIORITY, "%s: mysql_stmt_prepare() error: %s => sql: %s", sdata->ttmpfile, mysql_stmt_error(*stmt), s);
+      syslog(LOG_PRIORITY, "%s: error: mysql_stmt_prepare() %s => sql: %s", sdata->ttmpfile, mysql_stmt_error(sql->stmt), s);
       return ERR;
    }
 
@@ -226,6 +254,11 @@ int prepare_sql_statement(struct session_data *sdata, MYSQL_STMT **stmt, char *s
 }
 
 
+void close_prepared_statement(struct sql *sql){
+   if(sql->stmt) mysql_stmt_close(sql->stmt);
+}
+
+
 void insert_offset(struct session_data *sdata, int server_id){
    char s[SMALLBUFSIZE];
    uint64 id = server_id * 1000000000000ULL;
@@ -234,10 +267,3 @@ void insert_offset(struct session_data *sdata, int server_id){
 
    mysql_real_query(&(sdata->mysql), s, strlen(s));
 }
-
-
-void close_prepared_statement(MYSQL_STMT *stmt){
-   if(stmt) mysql_stmt_close(stmt);
-}
-
-

src/parser.c

@@ -16,11 +16,9 @@
 #include <piler.h>
 
 
-struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct __data *data, struct __config *cfg){
+struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct data *data, struct config *cfg){
    FILE *f;
-   int i;
+   char buf[MAXBUFSIZE];
-   unsigned int len;
-   char *p, buf[MAXBUFSIZE], puf[SMALLBUFSIZE];
    char writebuffer[MAXBUFSIZE], abuffer[MAXBUFSIZE];
    struct parser_state state;
 
@@ -32,44 +30,6 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec
       return state;
    }
 
-
-   if(sdata->num_of_rcpt_to > 0 && cfg->process_rcpt_to_addresses == 1){
-      for(i=0; i<sdata->num_of_rcpt_to; i++){
-
-         snprintf(puf, sizeof(puf)-1, "%s ", sdata->rcptto[i]);
-
-         if(does_it_seem_like_an_email_address(puf) == 1){
-            p = strstr(puf, cfg->hostid);
-            if(!p){
-
-               strtolower(puf);
-               len = strlen(puf);
-
-               if(state.tolen < MAXBUFSIZE-len-1){
-
-                  if(findnode(state.rcpt, puf) == NULL){
-                     addnode(state.journal_recipient, puf);
-                     addnode(state.rcpt, puf);
-
-                     memcpy(&(state.b_journal_to[state.journaltolen]), puf, len);
-                     state.journaltolen += len;
-
-                     memcpy(&(state.b_to[state.tolen]), puf, len);
-                     state.tolen += len;
-
-                     if(state.tolen < MAXBUFSIZE-len-1){
-                        split_email_address(puf);
-                        memcpy(&(state.b_to[state.tolen]), puf, len);
-                        state.tolen += len;
-                     }
-                  }
-               }
-            }
-         }
-
-      }
-   }
-
    if(take_into_pieces == 1){
       state.mfd = open(sdata->tmpframe, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR);
       if(state.mfd == -1){
@@ -84,31 +44,69 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec
    }
 
    if(take_into_pieces == 1 && state.writebufpos > 0){
-      write(state.mfd, writebuffer, state.writebufpos);
+      if(write(state.mfd, writebuffer, state.writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: %s: write(), %s, %d, %s", sdata->ttmpfile, __func__, __LINE__, __FILE__);
-      memset(writebuffer, 0, sizeof(writebuffer));
+      memset(writebuffer, 0, sizeof(writebuffer)); //-V597
       state.writebufpos = 0;
    }
 
    if(take_into_pieces == 1){
       close(state.mfd); state.mfd = 0;
+
+      if(state.has_to_dump_whole_body == 1){
+         if(state.abufpos > 0){
+            flush_attachment_buffer(&state, &abuffer[0], sizeof(abuffer));
+         }
+         if(state.fd != -1) close(state.fd);
+         if(state.b64fd != -1) close(state.b64fd);
+      }
+
    }
 
    fclose(f);
 
+   if(data->import && data->import->extra_recipient){
+      char tmpbuf[SMALLBUFSIZE];
+      snprintf(tmpbuf, sizeof(tmpbuf)-1, "%s", data->import->extra_recipient);
+      add_recipient(tmpbuf, strlen(tmpbuf), sdata, &state, data, cfg);
+   }
+
+   // If both Sender: and From: headers exist, and they are different, then append
+   // the From: address to recipients list to give him access to this email as well
+
+   if(state.b_sender_domain[0]){
+      char frombuf[SMALLBUFSIZE];
+      char senderbuf[SMALLBUFSIZE];
+      get_first_email_address_from_string(state.b_from, frombuf, sizeof(frombuf));
+      get_first_email_address_from_string(state.b_sender, senderbuf, sizeof(senderbuf));
+      if(strcmp(frombuf, senderbuf)){
+         frombuf[strlen(frombuf)] = ' ';
+         add_recipient(frombuf, strlen(frombuf), sdata, &state, data, cfg);
+      }
+   }
+
    return state;
 }
 
 
-void post_parse(struct session_data *sdata, struct parser_state *state, struct __config *cfg){
+void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg){
-   int i, rec=0;
+   int i;
-   unsigned int len;
-   char *p;
 
    clearhash(state->boundaries);
    clearhash(state->rcpt);
    clearhash(state->rcpt_domain);
    clearhash(state->journal_recipient);
 
+   // Fix From: and Sender: lines if they are too long
+   if(strlen(state->b_from) > 255) state->b_from[255] = '\0';
+   if(strlen(state->b_from_domain) > 255) state->b_from_domain[255] = '\0';
+
+   if(strlen(state->b_sender) > 255) state->b_sender[255] = '\0';
+   if(strlen(state->b_sender_domain) > 255) state->b_sender_domain[255] = '\0';
+
+   // Truncate the message_id if it's >255 characters
+   if(strlen(state->message_id) > 255) state->message_id[255] = '\0';
+
+   fixupEncodedHeaderLine(state->b_subject, MAXBUFSIZE);
    trimBuffer(state->b_subject);
 
    if(sdata->internal_sender == 0) sdata->direction = DIRECTION_INCOMING;
@@ -118,18 +116,48 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct _
       if(sdata->internal_recipient == 1 && sdata->external_recipient == 1) sdata->direction = DIRECTION_INTERNAL_AND_OUTGOING;
    }
 
+   char *q = strrchr(state->receivedbuf, ';');
+   // I've seen some odd 1st Received: lines where the date
+   // wasn't the last item after a semicolon, eg.
+   //
+   // Received: from some.server.local [172.16.102.18]
+   //   by some.server.com with POP3 (Message Archiver)
+   //   for <nobody@127.0.0.1> (single-drop); Fri, 22 Nov 2019 11:24:49 -0500 (EST)
+   //   smtp.auth=3Daaaaaaaa@yourdomain.com|Received:=20by=20yourdomain
+   //   .com=20with=20ESMTPA=20id=20md50124862414.msg=3B=20Fri,=2022=20
+   //   ...
+   //
+   // Such line messes with the parse_date_header() function
+   // so the workaround is to discard such Received: line
+   if(q && strlen(q+1) < 45){
+      time_t received_timestamp = parse_date_header(q+1);
+      if(received_timestamp > 10000000){
+         // If the calculated date based on Date: header line differs more than 1 week
+         // then we'll override it with the data parsed from the first Received: line
+         if(labs(received_timestamp - sdata->sent) > 604800) sdata->sent = received_timestamp;
+      }
+   }
 
    for(i=1; i<=state->n_attachments; i++){
+      char puf[SMALLBUFSIZE];
+      snprintf(puf, sizeof(puf)-1, "%s ", state->attachments[i].filename);
+
+      unsigned int len = strlen(puf);
+      if(state->bodylen < BIGBUFSIZE-len-1){
+         memcpy(&(state->b_body[state->bodylen]), puf, len);
+         state->bodylen += len;
+      }
+
       digest_file(state->attachments[i].internalname, &(state->attachments[i].digest[0]));
 
-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest);
+      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s, dumped: %d", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest, state->attachments[i].dumped);
 
-      p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type);
+      char *p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type);
       len = strlen(p);
       if(strlen(sdata->attachments) < SMALLBUFSIZE-len-1 && !strstr(sdata->attachments, p)) memcpy(&(sdata->attachments[strlen(sdata->attachments)]), p, len);
 
       if(state->attachments[i].dumped == 1){
-         rec = 0;
+         int rec = 0;
          if(cfg->extract_attachments == 1 && state->bodylen < BIGBUFSIZE-1024) extract_attachment_content(sdata, state, state->attachments[i].aname, get_attachment_extractor_by_filename(state->attachments[i].filename), &rec, cfg);
 
          unlink(state->attachments[i].aname);
@@ -145,7 +173,7 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct _
    }
 
 
-   digest_string(state->message_id, &(state->message_id_hash[0]));
+   digest_string("sha256", state->message_id, &(state->message_id_hash[0]));
 
    if(sdata->sent == 0) sdata->sent = sdata->now;
 }
@@ -162,7 +190,7 @@ void storno_attachment(struct parser_state *state){
    memset(state->attachments[state->n_attachments].type, 0, TINYBUFSIZE);
    memset(state->attachments[state->n_attachments].shorttype, 0, TINYBUFSIZE);
    memset(state->attachments[state->n_attachments].aname, 0, TINYBUFSIZE);
-   memset(state->attachments[state->n_attachments].filename, 0, TINYBUFSIZE);
+   memset(state->attachments[state->n_attachments].filename, 0, SMALLBUFSIZE);
    memset(state->attachments[state->n_attachments].internalname, 0, TINYBUFSIZE);
    memset(state->attachments[state->n_attachments].digest, 0, 2*DIGEST_LENGTH+1);
 
@@ -171,11 +199,29 @@ void storno_attachment(struct parser_state *state){
 }
 
 
-int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, int writebuffersize, char *abuffer, int abuffersize, struct __data *data, struct __config *cfg){
+void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned int abuffersize){
-   char *p, *q, puf[SMALLBUFSIZE];
+   if(write(state->fd, abuffer, state->abufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
-   unsigned char b64buffer[MAXBUFSIZE];
+
-   char tmpbuf[MAXBUFSIZE];
+   if(state->b64fd != -1){
-   int n64, writelen, boundary_line=0, result;
+      abuffer[state->abufpos] = '\0';
+      if(state->base64 == 1){
+         unsigned char b64buffer[MAXBUFSIZE];
+         int n64 = base64_decode_attachment_buffer(abuffer, &b64buffer[0], sizeof(b64buffer));
+         if(write(state->b64fd, b64buffer, n64) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
+      }
+      else if(write(state->b64fd, abuffer, state->abufpos) == -1){
+         syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
+      }
+   }
+
+   state->abufpos = 0;
+   memset(abuffer, 0, abuffersize);
+}
+
+
+int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg){
+   char *p;
+   int boundary_line=0;
    unsigned int len;
 
    if(cfg->debug == 1) printf("line: %s", buf);
@@ -193,11 +239,24 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
          sdata->restored_copy = 1;
       }
 
+      if(cfg->security_header[0] && state->found_security_header == 0 && strstr(buf, cfg->security_header)){
+         state->found_security_header = 1;
+      }
+
       if(*(cfg->piler_header_field) != 0 && strncmp(buf, cfg->piler_header_field, strlen(cfg->piler_header_field)) == 0){
          sdata->restored_copy = 1;
       }
 
-      if(sdata->ms_journal == 0 && strncmp(buf, "X-MS-Journal-Report:", strlen("X-MS-Journal-Report:")) == 0){
+      if(sdata->ms_journal == 0 && ( strncmp(buf, "X-MS-Journal-Report:", strlen("X-MS-Journal-Report:")) == 0 || strncmp(buf, "X-WM-Journal-Report: journal", strlen("X-WM-Journal-Report: journal")) == 0) ){
+
+         memset(state->b_to, 0, MAXBUFSIZE);
+         state->tolen = 0;
+         memset(state->b_to_domain, 0, SMALLBUFSIZE);
+         state->todomainlen = 0;
+
+         clearhash(state->rcpt);
+         clearhash(state->rcpt_domain);
+
          //if(sdata->import == 0){
             sdata->ms_journal = 1;
             memset(state->message_id, 0, SMALLBUFSIZE);
@@ -212,69 +271,50 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
 
       if(state->is_header == 1) state->is_header = 0;
       state->is_1st_header = 0;
-
-      if(state->anamepos > 0){
-         extractNameFromHeaderLine(state->attachment_name_buf, "name", state->filename);
-      }
-
    }
 
 
    if(take_into_pieces == 1){
-      if(state->message_state == MSG_BODY && state->fd != -1 && is_substr_in_hash(state->boundaries, buf) == 0){
+      if(state->message_state == MSG_BODY && state->fd != -1 && (state->has_to_dump_whole_body == 1 || is_substr_in_hash(state->boundaries, buf) == 0) ){
-         //n = write(state->fd, buf, len); // WRITE
          if(len + state->abufpos > abuffersize-1){
-            write(state->fd, abuffer, state->abufpos); 
+            flush_attachment_buffer(state, abuffer, abuffersize);
-
-            if(state->b64fd != -1){
-               abuffer[state->abufpos] = '\0';
-               if(state->base64 == 1){
-                  n64 = base64_decode_attachment_buffer(abuffer, &b64buffer[0], sizeof(b64buffer));
-                  write(state->b64fd, b64buffer, n64);
-               }
-               else {
-                  write(state->b64fd, abuffer, state->abufpos);
-               }
-            }
-
-            state->abufpos = 0; memset(abuffer, 0, abuffersize);
          }
          memcpy(abuffer+state->abufpos, buf, len); state->abufpos += len;
 
          state->attachments[state->n_attachments].size += len;
+
+         // When processing the body and writing to an attachment file, then we finish here
+         return 0;
       }
       else {
          state->saved_size += len;
-         //n = write(state->mfd, buf, len); // WRITE
          if(len + state->writebufpos > writebuffersize-1){
-            write(state->mfd, writebuffer, state->writebufpos); state->writebufpos = 0; memset(writebuffer, 0, writebuffersize);
+            if(write(state->mfd, writebuffer, state->writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
+            state->writebufpos = 0;
+            memset(writebuffer, 0, writebuffersize);
          }
          memcpy(writebuffer+state->writebufpos, buf, len); state->writebufpos += len;
       }
    }
 
 
-   if(state->message_state == MSG_BODY && state->has_to_dump == 1 &&  state->pushed_pointer == 0){
+   if(state->message_state == MSG_BODY && state->has_to_dump == 1 && state->pushed_pointer == 0){
-      //printf("####name: %s, type: %s, base64: %d\n", state->filename, state->type, state->base64);
-
       state->pushed_pointer = 1;
 
 
       // this is a real attachment to dump, it doesn't have to be base64 encoded!
-      if(strlen(state->filename) > 4 && strlen(state->type) > 3 && state->n_attachments < MAX_ATTACHMENTS-1){
+      if(state->attachment_name_buf[0] != 0 && strcasestr(state->attachment_name_buf, "name") && strlen(state->type) > 3 && state->n_attachments < MAX_ATTACHMENTS-1){
          state->n_attachments++;
 
-         snprintf(state->attachments[state->n_attachments].filename, TINYBUFSIZE-1, "%s", state->filename);
+         extractNameFromHeaderLine(state->attachment_name_buf, "name", state->attachments[state->n_attachments].filename, SMALLBUFSIZE);
          snprintf(state->attachments[state->n_attachments].type, TINYBUFSIZE-1, "%s", state->type);
          snprintf(state->attachments[state->n_attachments].internalname, TINYBUFSIZE-1, "%s.a%d", sdata->ttmpfile, state->n_attachments);
          snprintf(state->attachments[state->n_attachments].aname, TINYBUFSIZE-1, "%s.a%d.bin", sdata->ttmpfile, state->n_attachments);
 
-         //printf("DUMP FILE: %s\n", state->attachments[state->n_attachments].internalname);
-
          if(take_into_pieces == 1){
             state->fd = open(state->attachments[state->n_attachments].internalname, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR);
 
-            fixupEncodedHeaderLine(state->attachments[state->n_attachments].filename, TINYBUFSIZE);
+            fixupEncodedHeaderLine(state->attachments[state->n_attachments].filename, SMALLBUFSIZE);
 
             p = get_attachment_extractor_by_filename(state->attachments[state->n_attachments].filename);
 
@@ -292,13 +332,16 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
                syslog(LOG_PRIORITY, "%s: error opening %s", sdata->ttmpfile, state->attachments[state->n_attachments].internalname);
             }
             else {
+               char puf[SMALLBUFSIZE];
                snprintf(puf, sizeof(puf)-1, "ATTACHMENT_POINTER_%s.a%d_XXX_PILER", sdata->ttmpfile, state->n_attachments);
-               //n = write(state->mfd, puf, strlen(puf)); // WRITE
+               int writelen = strlen(puf);
-               writelen = strlen(puf);
                if(writelen + state->writebufpos > writebuffersize-1){
-                  write(state->mfd, writebuffer, state->writebufpos); state->writebufpos = 0; memset(writebuffer, 0, writebuffersize);
+                  if(write(state->mfd, writebuffer, state->writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
+                  state->writebufpos = 0;
+                  memset(writebuffer, 0, writebuffersize);
                }
-               memcpy(writebuffer+state->writebufpos, puf, writelen); state->writebufpos += writelen;
+               memcpy(writebuffer+state->writebufpos, puf, writelen);
+               state->writebufpos += writelen;
             }
          }
 
@@ -347,37 +390,62 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
          sdata->spam_message = 1;
       }
 
-      if(strncasecmp(buf, "From:", strlen("From:")) == 0) state->message_state = MSG_FROM;
+      if(strncasecmp(buf, "X-Piler-Envelope-To:", strlen("X-Piler-Envelope-To:")) == 0){
-
+         state->message_state = MSG_ENVELOPE_TO;
+         buf += strlen("X-Piler-Envelope-To:");
+      }
+      else if(strncasecmp(buf, "From:", strlen("From:")) == 0){
+         state->message_state = MSG_FROM;
+         buf += strlen("From:");
+      }
+      else if(strncasecmp(buf, "Sender:", strlen("Sender:")) == 0){
+         state->message_state = MSG_SENDER;
+         buf += strlen("Sender:");
+      }
       else if(strncasecmp(buf, "Content-Type:", strlen("Content-Type:")) == 0){
          state->message_state = MSG_CONTENT_TYPE;
-
-         if(state->anamepos > 0){
-            extractNameFromHeaderLine(state->attachment_name_buf, "name", state->filename);
-            memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
-            state->anamepos = 0;
-         }
-
       }
-      else if(strncasecmp(buf, "Content-Transfer-Encoding:", strlen("Content-Transfer-Encoding:")) == 0) state->message_state = MSG_CONTENT_TRANSFER_ENCODING;
+      else if(strncasecmp(buf, "Content-Transfer-Encoding:", strlen("Content-Transfer-Encoding:")) == 0){
-      else if(strncasecmp(buf, "Content-Disposition:", strlen("Content-Disposition:")) == 0){
+         state->message_state = MSG_CONTENT_TRANSFER_ENCODING;
+         if(state->is_1st_header == 1 && strcasestr(buf, "base64")){
+            state->has_to_dump = 1;
+            state->has_to_dump_whole_body = 1;
+         }
+      }
+
+      /*
+       * We only enter MSG_CONTENT_DISPOSITION state if we couldn't find
+       * the filename in MSG_CONTENT_TYPE state. We also assume that
+       * Content-Type: comes first, then Content-Disposition:
+       */
+      else if(strncasecmp(buf, "Content-Disposition:", strlen("Content-Disposition:")) == 0 && strcasestr(state->attachment_name_buf, "name") == NULL){
          state->message_state = MSG_CONTENT_DISPOSITION;
-
-         if(state->anamepos > 0){
-            extractNameFromHeaderLine(state->attachment_name_buf, "name", state->filename);
-            memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
-            state->anamepos = 0;
-         }
-
       }
-      else if(strncasecmp(buf, "To:", 3) == 0) state->message_state = MSG_TO;
+      else if(strncasecmp(buf, "To:", 3) == 0){
-      else if(strncasecmp(buf, "Cc:", 3) == 0) state->message_state = MSG_CC;
+         state->message_state = MSG_TO;
-      else if(strncasecmp(buf, "Bcc:", 4) == 0) state->message_state = MSG_CC;
+         buf += strlen("To:");
-      else if(strncasecmp(buf, "Message-Id:", 11) == 0) state->message_state = MSG_MESSAGE_ID;
+      }
+      else if(strncasecmp(buf, "Cc:", 3) == 0){
+         state->message_state = MSG_CC;
+         buf += strlen("Cc:");
+      }
+      else if(strncasecmp(buf, "Bcc:", 4) == 0){
+         state->message_state = MSG_CC;
+         buf += strlen("Bcc:");
+      }
+      else if(strncasecmp(buf, "Message-Id:", 11) == 0){
+         state->message_state = MSG_MESSAGE_ID;
+         buf += strlen("Message-Id:");
+      }
       else if(strncasecmp(buf, "References:", 11) == 0) state->message_state = MSG_REFERENCES;
-      else if(strncasecmp(buf, "Subject:", strlen("Subject:")) == 0) state->message_state = MSG_SUBJECT;
+      else if(strncasecmp(buf, "Subject:", strlen("Subject:")) == 0){
-      else if(strncasecmp(buf, "Recipient:", strlen("Recipient:")) == 0) state->message_state = MSG_RECIPIENT;
+         state->message_state = MSG_SUBJECT;
-
+         buf += strlen("Subject:");
+      }
+      else if(strncasecmp(buf, "Recipient:", strlen("Recipient:")) == 0){
+         state->message_state = MSG_RECIPIENT;
+         buf += strlen("Recipient:");
+      }
       if(sdata->ms_journal == 1 && (state->message_state == MSG_TO || state->message_state == MSG_RECIPIENT) ){
          p = strstr(buf, "Expanded:");
          if(p) *p = '\0';
@@ -392,23 +460,37 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
 
          if(strstr(buf, "=?") && strstr(buf, "?=")) fixupEncodedHeaderLine(buf, MAXBUFSIZE);
 
-         sdata->sent = parse_date_header(buf);
+         sdata->sent = parse_date_header(buf+5);
 
          /* allow +2 days drift in the parsed Date: value */
 
          if(sdata->sent - sdata->now > 2*86400) sdata->sent = sdata->now;
       }
 
-      else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf);
+      else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf+14);
-      else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0) state->message_state = MSG_RECEIVED;
+      else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0){
-      else if(cfg->extra_to_field[0] != '\0' && strncasecmp(buf, cfg->extra_to_field, strlen(cfg->extra_to_field)) == 0) state->message_state = MSG_TO;
+         state->message_state = MSG_RECEIVED;
+         state->received_header++;
+      }
+      else if(cfg->extra_to_field[0] != '\0' && strncasecmp(buf, cfg->extra_to_field, strlen(cfg->extra_to_field)) == 0){
+         state->message_state = MSG_RECIPIENT;
+         buf += strlen(cfg->extra_to_field);
+      }
 
       if(state->message_state == MSG_MESSAGE_ID && state->message_id[0] == 0){
-         p = strchr(buf+11, ' ');
+         while(isspace(*buf)){
-         if(p) p = buf + 12;
+            buf++;
-         else p = buf + 11;
+         }
 
-         snprintf(state->message_id, SMALLBUFSIZE-1, "%s", p);
+         snprintf(state->message_id, SMALLBUFSIZE-1, "%s", buf);
+      }
+
+      if(state->message_state == MSG_CONTENT_TYPE || state->message_state == MSG_CONTENT_DISPOSITION){
+         fill_attachment_name_buf(state, buf);
+      }
+
+      if(state->received_header == 1 && state->message_state == MSG_RECEIVED && strlen(state->receivedbuf) + len < sizeof(state->receivedbuf)){
+         memcpy(&(state->receivedbuf[strlen(state->receivedbuf)]), buf, len);
       }
 
       /* we are interested in only From:, To:, Subject:, Received:, Content-*: header lines */
@@ -463,31 +545,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
    if(state->is_1st_header == 1){
 
       if(state->message_state == MSG_SUBJECT && strlen(state->b_subject) + strlen(buf) < MAXBUFSIZE-1){
-
+         // buffer the subject lines, and decode it later
-         if(state->b_subject[0] == '\0'){
+         strncat(state->b_subject, buf, MAXBUFSIZE-strlen(state->b_subject)-1);
-            p = &buf[0];
-            if(strncmp(buf, "Subject:", strlen("Subject:")) == 0) p += strlen("Subject:");
-            if(*p == ' ') p++;
-
-            fixupEncodedHeaderLine(p, MAXBUFSIZE);
-            strncat(state->b_subject, p, MAXBUFSIZE-strlen(state->b_subject)-1);
-         }
-         else {
-
-            /*
-             * if the next subject line is encoded, then strip the whitespace characters at the beginning of the line
-             */
-
-            p = buf;
-
-            if(strcasestr(buf, "?Q?") || strcasestr(buf, "?B?")){
-               while(isspace(*p)) p++;
-            }
-
-            fixupEncodedHeaderLine(p, MAXBUFSIZE);
-
-            strncat(state->b_subject, p, MAXBUFSIZE-strlen(state->b_subject)-1);
-         }
       }
       else { fixupEncodedHeaderLine(buf, MAXBUFSIZE); }
    }
@@ -531,6 +590,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
          state->message_rfc822 = 1;
          state->is_header = 1;
 
+         state->has_to_dump = 0;
+
          if(sdata->ms_journal == 1){
             state->is_1st_header = 1;
 
@@ -540,6 +601,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
             memset(state->b_body, 0, BIGBUFSIZE);
             memset(state->b_from, 0, SMALLBUFSIZE);
             memset(state->b_from_domain, 0, SMALLBUFSIZE);
+            memset(state->b_sender, 0, SMALLBUFSIZE);
+            memset(state->b_sender_domain, 0, SMALLBUFSIZE);
             memset(state->message_id, 0, SMALLBUFSIZE);
 
             sdata->ms_journal = 0;
@@ -547,27 +610,11 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
       }
 
 
-      if(strcasestr(buf, "charset")) extractNameFromHeaderLine(buf, "charset", state->charset);
+      if(strcasestr(buf, "charset")) extractNameFromHeaderLine(buf, "charset", state->charset, TINYBUFSIZE);
       if(strcasestr(state->charset, "UTF-8")) state->utf8 = 1;
    }
 
 
-   if((state->message_state == MSG_CONTENT_TYPE || state->message_state == MSG_CONTENT_DISPOSITION) && strlen(state->filename) < 5){
-
-      p = &buf[0];
-      for(; *p; p++){
-         if(*p != ' ' && *p != '\t') break;
-      }
-
-      len = strlen(p);
-
-      if(len + state->anamepos < SMALLBUFSIZE-2){
-         memcpy(&(state->attachment_name_buf[state->anamepos]), p, len);
-         state->anamepos += len;
-      }
-   }
-
-
    if(state->message_state == MSG_CONTENT_TRANSFER_ENCODING){
       if(strcasestr(buf, "base64")) state->base64 = 1;
       if(strcasestr(buf, "quoted-printable")) state->qp = 1;
@@ -588,20 +635,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
       if(state->has_to_dump == 1){
          if(take_into_pieces == 1 && state->fd != -1){
             if(state->abufpos > 0){
-               write(state->fd, abuffer, state->abufpos);
+               flush_attachment_buffer(state, abuffer, abuffersize);
-
-               if(state->b64fd != -1){
-                  abuffer[state->abufpos] = '\0';
-                  if(state->base64 == 1){
-                     n64 = base64_decode_attachment_buffer(abuffer, &b64buffer[0], sizeof(b64buffer));
-                     write(state->b64fd, b64buffer, n64);
-                  }
-                  else {
-                     write(state->b64fd, abuffer, state->abufpos);
-                  }
-               }
-
-               state->abufpos = 0; memset(abuffer, 0, abuffersize); 
             }
             close(state->fd);
             close(state->b64fd);
@@ -622,9 +656,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
 
       state->pushed_pointer = 0;
 
-      memset(state->filename, 0, TINYBUFSIZE);
       memset(state->type, 0, TINYBUFSIZE);
-      snprintf(state->charset, TINYBUFSIZE-1, "unknown");
+      memset(state->charset, 0, TINYBUFSIZE);
 
       memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
       state->anamepos = 0;
@@ -641,7 +674,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
 
 
    /* skip irrelevant headers */
-   if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT) return 0;
+   if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_SENDER && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT && state->message_state != MSG_ENVELOPE_TO) return 0;
 
 
    /* don't process body if it's not a text or html part */
@@ -654,8 +687,6 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
    }
 
 
-   /* remove all HTML tags */
-   if(state->texthtml == 1 && state->message_state == MSG_BODY) markHTML(buf, state);
 
    if(state->message_state == MSG_BODY && state->qp == 1){
       fixupSoftBreakInQuotedPritableLine(buf, state); // 2011.12.07
@@ -665,128 +696,31 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
    /* I believe that we can live without this function call */
    //decodeURL(buf);
 
-   if(state->texthtml == 1) decodeHTML(buf, state->utf8);
+   /* remove all HTML tags */
+   if(state->texthtml == 1 && state->message_state == MSG_BODY) markHTML(buf, state);
+
+
+   if(state->texthtml == 1){
+      size_t buflen = strlen(buf);
+      decodeHTML(buf, state->utf8);
+      /* decodeHTML converted some entities to iso-8859-1 */
+      if(state->utf8 != 1 && strlen(buf) != buflen){
+        /* no charset or us-ascii: switch to iso-8859-1 */
+        if (state->charset[0] == 0 || strcasecmp(state->charset, "us-ascii") == 0){
+          syslog(LOG_PRIORITY, "%s: assuming iso-8859-1 encoding for HTML (was '%s')", sdata->ttmpfile, state->charset);
+          snprintf(state->charset, TINYBUFSIZE-1, "ISO8859-1");
+        }
+      }
+   }
 
    /* encode the body if it's not utf-8 encoded */
    if(state->message_state == MSG_BODY && state->utf8 != 1){
-      result = utf8_encode(buf, strlen(buf), &tmpbuf[0], sizeof(tmpbuf), state->charset);
+      char tmpbuf[MAXBUFSIZE];
+      int result = utf8_encode(buf, strlen(buf), &tmpbuf[0], sizeof(tmpbuf), state->charset);
       if(result == OK) snprintf(buf, MAXBUFSIZE-1, "%s", tmpbuf);
    }
 
-   translateLine((unsigned char*)buf, state);
+   tokenize(buf, state, sdata, data, cfg);
-
-   reassembleToken(buf);
-
-
-   if(state->is_header == 1) p = strchr(buf, ' ');
-   else p = buf;
-
-   //printf("a: %d/%d/%d/%d/j=%d %s\n", state->is_1st_header, state->is_header, state->message_rfc822, state->message_state, sdata->ms_journal, buf);
-
-   do {
-      memset(puf, 0, sizeof(puf));
-      p = split(p, ' ', puf, sizeof(puf)-1, &result);
-
-      if(puf[0] == '\0') continue;
-
-      degenerateToken((unsigned char*)puf);
-
-      if(puf[0] == '\0') continue;
-
-      strncat(puf, " ", sizeof(puf)-strlen(puf)-1);
-
-      if(strncasecmp(puf, "http://", 7) == 0 || strncasecmp(puf, "https://", 8) == 0) fixURL(puf, sizeof(puf)-1);
-
-      len = strlen(puf);
-
-      // skip body tokens if not an URL && (empty token || too long)
-      if(state->is_header == 0 && strncmp(puf, "__URL__", 7) && (puf[0] == ' ' || (len > MAX_WORD_LEN && cfg->enable_cjk == 0)) ){
-         continue;
-      }
-
-      if(state->message_state == MSG_FROM && state->is_1st_header == 1 && strlen(state->b_from) < SMALLBUFSIZE-len-1){
-         strtolower(puf);
-
-         q = strchr(puf, '@');
-         if(q) fix_plus_sign_in_email_address(puf, &q, &len);
-
-         memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
-
-         if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(puf) == 1 && state->b_from_domain[0] == '\0'){
-            if(q && strlen(q) > 5){
-               memcpy(&(state->b_from_domain), q+1, strlen(q+1)-1);
-               if(strstr(sdata->mailfrom, "<>")){
-                  snprintf(sdata->fromemail, SMALLBUFSIZE-1, "%s", puf);
-                  sdata->fromemail[len-1] = '\0';
-               }
-            }
-
-            if(is_email_address_on_my_domains(puf, data) == 1) sdata->internal_sender = 1;
-
-            if(strlen(state->b_from) < SMALLBUFSIZE-len-1){
-               split_email_address(puf);
-               memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
-            }
-         }
-      }
-      else if((state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && state->is_1st_header == 1 && state->tolen < MAXBUFSIZE-len-1){
-         strtolower(puf);
-
-         /* fix aaa+bbb@ccc.fu address to aaa@ccc.fu, 2017.02.04, SJ */
-         q = strchr(puf, '@');
-         if(q) fix_plus_sign_in_email_address(puf, &q, &len);
-
-         if(state->message_state == MSG_RECIPIENT && findnode(state->journal_recipient, puf) == NULL){
-            addnode(state->journal_recipient, puf);
-            memcpy(&(state->b_journal_to[state->journaltolen]), puf, len);
-            if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: journal rcpt: '%s'", sdata->ttmpfile, puf);
-         }
-
-
-         if(findnode(state->rcpt, puf) == NULL){
-
-            /* skip any address matching ...@cfg->hostid, 2013.10.29, SJ */
-            if(q && strncmp(q+1, cfg->hostid, cfg->hostid_len) == 0){
-               continue;
-            }
-
-            addnode(state->rcpt, puf);
-            memcpy(&(state->b_to[state->tolen]), puf, len);
-            state->tolen += len;
-
-            if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(puf) == 1){
-
-               if(is_email_address_on_my_domains(puf, data) == 1) sdata->internal_recipient = 1;
-               else sdata->external_recipient = 1;
-
-               if(q){
-                  if(findnode(state->rcpt_domain, q+1) == NULL){
-                     addnode(state->rcpt_domain, q+1);
-                     memcpy(&(state->b_to_domain[strlen(state->b_to_domain)]), q+1, strlen(q+1));
-                  }
-               }
-
-               if(state->tolen < MAXBUFSIZE-len-1){
-                  split_email_address(puf);
-                  memcpy(&(state->b_to[state->tolen]), puf, len);
-                  state->tolen += len;
-               }
-
-            }
-         }
-
-      }
-      else if(state->message_state == MSG_BODY && len >= cfg->min_word_len && state->bodylen < BIGBUFSIZE-len-1){
-         // 99% of email addresses are longer than 8 characters
-         if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(puf)){
-            fix_email_address_for_sphinx(puf);
-         }
-
-         memcpy(&(state->b_body[state->bodylen]), puf, len);
-         state->bodylen += len;
-      }
-
-   } while(p);
 
    return 0;
 }

src/parser.h

@@ -9,9 +9,9 @@
 #include "config.h"
 #include "defs.h"
 
-struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct __data *data, struct __config *cfg);
+struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct data *data, struct config *cfg);
-void post_parse(struct session_data *sdata, struct parser_state *state, struct __config *cfg);
+void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg);
-int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, int writebuffersize, char *abuffer, int abuffersize, struct __data *data, struct __config *cfg);
+int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg);
 
 void init_state(struct parser_state *state);
 time_t parse_date_header(char *s);
@@ -20,19 +20,24 @@ void fixupEncodedHeaderLine(char *buf, int buflen);
 void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state);
 void fixupBase64EncodedLine(char *buf, struct parser_state *state);
 void markHTML(char *buf, struct parser_state *state);
-int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state);
+void setStateHTML(char *htmlbuf, int pos, struct parser_state *state);
 void translateLine(unsigned char *p, struct parser_state *state);
 void fix_email_address_for_sphinx(char *s);
 void split_email_address(char *s);
 int does_it_seem_like_an_email_address(char *email);
+void add_recipient(char *email, unsigned int len, struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
 void reassembleToken(char *p);
 void degenerateToken(unsigned char *p);
 void fixURL(char *buf, int buflen);
-void extractNameFromHeaderLine(char *s, char *name, char *resultbuf);
+void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultbuflen);
 char *determine_attachment_type(char *filename, char *type);
 char *get_attachment_extractor_by_filename(char *filename);
 void parse_reference(struct parser_state *state, char *s);
 int base64_decode_attachment_buffer(char *p, unsigned char *b, int blen);
 void fix_plus_sign_in_email_address(char *puf, char **at_sign, unsigned int *len);
+void tokenize(char *buf, struct parser_state *state, struct session_data *sdata, struct data *data, struct config *cfg);
+void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned int abuffersize);
+void fill_attachment_name_buf(struct parser_state *state, char *buf);
+int get_first_email_address_from_string(char *str, char *buf, int buflen);
 
 #endif /* _PARSER_H */

src/parser_utils.c

@@ -40,6 +40,7 @@ void init_state(struct parser_state *state){
 
    state->htmltag = 0;
    state->style = 0;
+   state->meta_content_type = 0;
 
    state->skip_html = 0;
 
@@ -49,14 +50,16 @@ void init_state(struct parser_state *state){
    memset(state->message_id_hash, 0, 2*DIGEST_LENGTH+1);
    memset(state->miscbuf, 0, MAX_TOKEN_LEN);
    memset(state->qpbuf, 0, MAX_TOKEN_LEN);
+   memset(state->receivedbuf, 0, sizeof(state->receivedbuf));
 
-   memset(state->filename, 0, TINYBUFSIZE);
    memset(state->type, 0, TINYBUFSIZE);
+   memset(state->charset, 0, TINYBUFSIZE);
 
    memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
    state->anamepos = 0;
 
    state->has_to_dump = 0;
+   state->has_to_dump_whole_body = 0;
    state->fd = -1;
    state->b64fd = -1;
    state->mfd = -1;
@@ -67,6 +70,7 @@ void init_state(struct parser_state *state){
 
    state->writebufpos = 0;
    state->abufpos = 0;
+   state->received_header = 0;
 
    inithash(state->boundaries);
    inithash(state->rcpt);
@@ -81,7 +85,7 @@ void init_state(struct parser_state *state){
       memset(state->attachments[i].type, 0, TINYBUFSIZE);
       memset(state->attachments[i].shorttype, 0, TINYBUFSIZE);
       memset(state->attachments[i].aname, 0, TINYBUFSIZE);
-      memset(state->attachments[i].filename, 0, TINYBUFSIZE);
+      memset(state->attachments[i].filename, 0, SMALLBUFSIZE);
       memset(state->attachments[i].internalname, 0, TINYBUFSIZE);
       memset(state->attachments[i].digest, 0, 2*DIGEST_LENGTH+1);
    }
@@ -90,6 +94,8 @@ void init_state(struct parser_state *state){
 
    memset(state->b_from, 0, SMALLBUFSIZE);
    memset(state->b_from_domain, 0, SMALLBUFSIZE);
+   memset(state->b_sender, 0, SMALLBUFSIZE);
+   memset(state->b_sender_domain, 0, SMALLBUFSIZE);
    memset(state->b_to, 0, MAXBUFSIZE);
    memset(state->b_to_domain, 0, SMALLBUFSIZE);
    memset(state->b_subject, 0, MAXBUFSIZE);
@@ -97,10 +103,13 @@ void init_state(struct parser_state *state){
    memset(state->b_journal_to, 0, MAXBUFSIZE);
 
    state->tolen = 0;
+   state->todomainlen = 0;
    state->bodylen = 0;
    state->journaltolen = 0;
 
    state->retention = 0;
+
+   state->found_security_header = 0;
 }
 
 
@@ -113,13 +122,12 @@ long get_local_timezone_offset(){
 
 
 time_t parse_date_header(char *datestr){
-   int n=0, len;
+   int n=0;
    long offset=0;
    time_t ts=0;
    char *p, *q, *r, *tz, s[SMALLBUFSIZE], tzh[4], tzm[3];
    struct tm tm;
 
-   datestr += 5;
    p = datestr;
 
    for(; *datestr; datestr++){
@@ -149,7 +157,7 @@ time_t parse_date_header(char *datestr){
    do {
       p = split_str(p, " ", s, sizeof(s)-1);
 
-      len = strlen(s);
+      int len = strlen(s);
 
       if(len > 0){
          n++;
@@ -196,6 +204,7 @@ time_t parse_date_header(char *datestr){
          else if(strncasecmp(s, "Sat", 3) == 0) tm.tm_wday = 6;
          else if(strncasecmp(s, "Sun", 3) == 0) tm.tm_wday = 0;
 
+
          if(len <= 2 && tm.tm_mday == 0){ tm.tm_mday = atoi(s); continue; }
 
          if(len <= 2 && tm.tm_mon == -1){ tm.tm_mon = atoi(s) - 1; continue; }
@@ -257,12 +266,24 @@ time_t parse_date_header(char *datestr){
 
    ts += get_local_timezone_offset() - offset;
 
+   if(ts < 700000000){
+      // If the Date: field contains some garbage, eg.
+      // "Date: [mail_datw]" or similar, and the date
+      // is before Sat Mar  7 20:26:40 UTC 1992, then
+      // return the current timestamp
+
+      time_t now;
+      time(&now);
+
+      return now;
+   }
+
    return ts;
 }
 
 
 int extract_boundary(char *p, struct parser_state *state){
-   char *q, *q2;
+   char *q;
 
    p += strlen("boundary");
 
@@ -289,7 +310,7 @@ int extract_boundary(char *p, struct parser_state *state){
             break;
       }
 
-      q2 = strchr(p, ';');
+      char *q2 = strchr(p, ';');
       if(q2) *q2 = '\0';
 
       q = strrchr(p, '"');
@@ -313,13 +334,13 @@ int extract_boundary(char *p, struct parser_state *state){
 
 
 void fixupEncodedHeaderLine(char *buf, int buflen){
-   char *sb, *sq, *p, *q, *r, *s, *e, *start, *end;
+   char *q, *r, *s, *e, *end;
    /*
     * I thought SMALLBUFSIZE would be enough for v, encoding and tmpbuf(2*),
     * but then I saw a 6-7000 byte long subject line, so I've switched to MAXBUFSIZE
     */
-   char v[MAXBUFSIZE], puf[MAXBUFSIZE], encoding[MAXBUFSIZE], tmpbuf[2*MAXBUFSIZE];
+   char v[MAXBUFSIZE], u[MAXBUFSIZE], puf[MAXBUFSIZE], encoding[MAXBUFSIZE], tmpbuf[2*MAXBUFSIZE];
-   int need_encoding, ret;
+   int need_encoding, ret, prev_encoded=0, n_tokens=0;
 
    if(buflen < 5) return;
 
@@ -330,72 +351,111 @@ void fixupEncodedHeaderLine(char *buf, int buflen){
    do {
       q = split_str(q, " ", v, sizeof(v)-1);
 
-      p = v;
+      char *p = v;
-
-      memset(encoding, 0, sizeof(encoding));
 
       do {
-         start = strstr(p, "=?");
+         memset(u, 0, sizeof(u));
-         if(start){
-            *start = '\0';
-            if(strlen(p) > 0){
-               strncat(puf, p, sizeof(puf)-strlen(puf)-1);
-            }
 
-            start++;
+         /*
+          * We can't use split_str(p, "=?", ...) it will fail with the following pattern
+          *    =?UTF-8?B?SG9neWFuIMOtcmp1bmsgcGFuYXN6bGV2ZWxldD8=?=
+          *
+          * Also the below pattern requires special care:
+          *    =?gb2312?B?<something>?==?gb2312?Q?<something else>?=
+          *
+          * And we have to check the following cases as well:
+          *    Happy New Year! =?utf-8?q?=F0=9F=8E=86?=
+          */
 
-            e = strchr(start+2, '?');
+         int b64=0, qp=0;
+         memset(encoding, 0, sizeof(encoding));
+
+         r = strstr(p, "=?");
+         if(r){
+            p = r + 2;
+
+            e = strchr(p, '?');
             if(e){
                *e = '\0';
-               snprintf(encoding, sizeof(encoding)-1, "%s", start+1);
+               snprintf(encoding, sizeof(encoding)-1, "%s", p);
                *e = '?';
-            }
 
-            s = NULL;
+               s = strcasestr(e, "?B?");
-            sb = strcasestr(start, "?B?"); if(sb) s = sb;
+               if(s){
-            sq = strcasestr(start, "?Q?"); if(sq) s = sq;
+                  b64 = 1;
-
+                  p = s + 3;
-            if(s){
+               }
-               end = strstr(s+3, "?=");
+               else {
-               if(end){
+                  s = strcasestr(e, "?Q?");
-                  *end = '\0';
+                  if(s){
-
+                     qp = 1;
-                  if(sb){ decodeBase64(s+3); }
+                     p = s + 3;
-                  if(sq){ decodeQP(s+3); r = s + 3; for(; *r; r++){ if(*r == '_') *r = ' '; } }
-
-                  /* encode everything if it's not utf-8 encoded */
-
-                  need_encoding = 0;
-                  ret = ERR;
-
-                  if(strlen(encoding) > 2 && strcasecmp(encoding, "utf-8")){
-                     need_encoding = 1;
-                     ret = utf8_encode(s+3, strlen(s+3), &tmpbuf[0], sizeof(tmpbuf), encoding);
                   }
-
-                  if(need_encoding == 1 && ret == OK)
-                     strncat(puf, tmpbuf, sizeof(puf)-strlen(puf)-1);
-                  else 
-                     strncat(puf, s+3, sizeof(puf)-strlen(puf)-1);
-
-                  p = end + 2;
                }
             }
-            else {
-               strncat(puf, start, sizeof(puf)-strlen(puf)-1);
 
-               break;
+            end = strstr(p, "?=");
+            if(end){
+               *end = '\0';
+            }
+
+
+            snprintf(u, sizeof(u)-1, "%s", p);
+
+            if(end) {
+               p = end + 2;
             }
          }
          else {
-            strncat(puf, p, sizeof(puf)-strlen(puf)-1);
+            snprintf(u, sizeof(u)-1, "%s", p);
-            break;
+            p = NULL;
+         }
+
+         if(u[0] == 0) continue;
+
+         n_tokens++;
+
+         if(b64 == 1) decodeBase64(u);
+         else if(qp == 1) decodeQP(u);
+
+
+         /*
+          * https://www.ietf.org/rfc/rfc2047.txt says that
+          *
+          * "When displaying a particular header field that contains multiple
+          *  'encoded-word's, any 'linear-white-space' that separates a pair of
+          *  adjacent 'encoded-word's is ignored." (6.2)
+          */
+         if(prev_encoded == 1 && (b64 == 1 || qp == 1)) {}
+         else if(n_tokens > 1){
+            strncat(puf, " ", sizeof(puf)-strlen(puf)-1);
+         }
+
+         if(b64 == 1 || qp == 1){
+            prev_encoded = 1;
+            need_encoding = 0;
+            ret = ERR;
+
+            if(encoding[0] && strcasecmp(encoding, "utf-8")){
+               need_encoding = 1;
+               ret = utf8_encode(u, strlen(u), &tmpbuf[0], sizeof(tmpbuf), encoding);
+            }
+
+            if(need_encoding == 1 && ret == OK){
+               #pragma GCC diagnostic push
+               #pragma GCC diagnostic ignored "-Wstringop-truncation"
+               strncat(puf, tmpbuf, sizeof(puf)-strlen(puf)-1);
+            }
+            else {
+               strncat(puf, u, sizeof(puf)-strlen(puf)-1);
+            }
+         }
+         else {
+            strncat(puf, u, sizeof(puf)-strlen(puf)-1);
          }
 
       } while(p);
 
-      if(q) strncat(puf, " ", sizeof(puf)-strlen(puf)-1);
-
    } while(q);
 
    snprintf(buf, buflen-1, "%s", puf);
@@ -404,9 +464,9 @@ void fixupEncodedHeaderLine(char *buf, int buflen){
 
 void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state){
    int i=0;
-   char *p, puf[MAXBUFSIZE];
 
    if(strlen(state->qpbuf) > 0){
+      char puf[MAXBUFSIZE];
       memset(puf, 0, sizeof(puf));
       snprintf(puf, sizeof(puf)-1, "%s%s", state->qpbuf, buf);
       snprintf(buf, MAXBUFSIZE-1, "%s", puf);
@@ -419,7 +479,7 @@ void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state){
    }
 
    if(i == 1){
-      p = strrchr(buf, ' ');
+      char *p = strrchr(buf, ' ');
       if(p){
          memset(state->qpbuf, 0, MAX_TOKEN_LEN);
          if(strlen(p) < MAX_TOKEN_LEN-1){
@@ -433,9 +493,8 @@ void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state){
 
 
 void fixupBase64EncodedLine(char *buf, struct parser_state *state){
-   char *p, puf[MAXBUFSIZE];
-
    if(strlen(state->miscbuf) > 0){
+      char puf[MAXBUFSIZE];
       memset(puf, 0, sizeof(puf));
       strncpy(puf, state->miscbuf, sizeof(puf)-strlen(puf)-1);
       strncat(puf, buf, sizeof(puf)-strlen(puf)-1);
@@ -447,7 +506,7 @@ void fixupBase64EncodedLine(char *buf, struct parser_state *state){
    }
 
    if(buf[strlen(buf)-1] != '\n'){
-      p = strrchr(buf, ' ');
+      char *p = strrchr(buf, ' ');
       if(p){
          memcpy(&(state->miscbuf[0]), p+1, MAX_TOKEN_LEN-1);
          *p = '\0';
@@ -494,7 +553,7 @@ void markHTML(char *buf, struct parser_state *state){
 
             if(isspace(*s)){
                if(j > 0){
-                  k += appendHTMLTag(puf, html, pos, state);
+                  setStateHTML(html, pos, state);
                   memset(html, 0, SMALLBUFSIZE); j=0;
                }
                pos++;
@@ -519,55 +578,51 @@ void markHTML(char *buf, struct parser_state *state){
 
          if(j > 0){
             strncat(html, " ", SMALLBUFSIZE-1);
-            k += appendHTMLTag(puf, html, pos, state);
+            setStateHTML(html, pos, state);
             memset(html, 0, SMALLBUFSIZE); j=0;
          }
+         state->meta_content_type = 0;
       }
 
    }
 
    //printf("append last in line:*%s*, html=+%s+, j=%d\n", puf, html, j);
-   if(j > 0){ appendHTMLTag(puf, html, pos, state); }
+   if(j > 0){ setStateHTML(html, pos, state); }
 
    strcpy(buf, puf);
 }
 
 
-int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state){
+void setStateHTML(char *htmlbuf, int pos, struct parser_state *state){
-   char *p, html[SMALLBUFSIZE];
-   int len;
-
    if(pos == 0 && strncmp(htmlbuf, "style ", 6) == 0) state->style = 1;
    if(pos == 0 && strncmp(htmlbuf, "/style ", 7) == 0) state->style = 0;
 
-   return 0;
+   if(pos == 0 && state->charset[0] == 0 && strncmp(htmlbuf, "meta ", 5) == 0) state->meta_content_type = 0x1;
+   if(state->meta_content_type){
+     if((state->meta_content_type & 0x2) == 0 && strstr(htmlbuf, "http-equiv=content-type "))
+       state->meta_content_type |= 0x2;
 
-   //printf("appendHTML: pos:%d, +%s+\n", pos, htmlbuf);
+     if((state->meta_content_type & 0x4) == 0 && strstr(htmlbuf, "content=text/html;"))
+       state->meta_content_type |= 0x4;
 
-   if(state->style == 1) return 0;
+     if(state->meta_content_type == 0x7){
+       char *p, *q;
 
-   if(strlen(htmlbuf) == 0) return 0;
+       p = strstr(htmlbuf, "charset=");
+       if(p){
+         p += 8;
+         for(q = p; isalnum(*q) || index("-_", *q); q++)
+           ;
 
-   snprintf(html, SMALLBUFSIZE-1, "HTML*%s", htmlbuf);
+         if(q > p && q-p+1 < (int) sizeof(state->charset)){
-   len = strlen(html);
+           syslog(LOG_PRIORITY, "Changing HTML charset from '%s' to '%*s' due to meta tag", state->charset, (int)(q-p), p);
-
+           strncpy(state->charset, p, q-p);
-   if(len > 8 && strchr(html, '=')){
+           state->charset[q-p+1] = '\0';
-      p = strstr(html, "cid:");
+           state->meta_content_type = 0;
-      if(p){
+         }
-         *(p+3) = '\0';
+       }
-         strncat(html, " ", SMALLBUFSIZE-1);
+     }
-      }
-
-      strncat(buf, html, MAXBUFSIZE-1);
-      return len;
    }
-
-   if(strstr(html, "http") ){
-      strncat(buf, html+5, MAXBUFSIZE-1);
-      return len-5;
-   }
-
-   return 0;
 }
 
 
@@ -580,9 +635,9 @@ void translateLine(unsigned char *p, struct parser_state *state){
 
    for(; *p; p++){
 
-      if( (state->message_state == MSG_RECEIVED || state->message_state == MSG_FROM || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && *p == '@'){ continue; }
+      if( (state->message_state == MSG_RECEIVED || state->message_state == MSG_FROM || state->message_state == MSG_SENDER || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && *p == '@'){ continue; }
 
-      if(state->message_state == MSG_FROM || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT){
+      if(state->message_state == MSG_FROM || state->message_state == MSG_SENDER || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT){
 
          /* To fix some unusual addresses, eg.
           *    "'user@domain'"    -> user@domain
@@ -599,6 +654,7 @@ void translateLine(unsigned char *p, struct parser_state *state){
          prev = *p;
       }
 
+
       if(state->message_state == MSG_SUBJECT && (*p == '%' || *p == '_' || *p == '&') ){ continue; }
 
       if(state->message_state == MSG_CONTENT_TYPE && *p == '_' ){ continue; }
@@ -653,13 +709,53 @@ int does_it_seem_like_an_email_address(char *email){
 }
 
 
+void add_recipient(char *email, unsigned int len, struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
+   if(findnode(state->rcpt, email) == NULL){
+
+      char *q = strchr(email, '@');
+
+      /* skip any address matching ...@cfg->hostid, 2013.10.29, SJ */
+      if(q && strncmp(q+1, cfg->hostid, cfg->hostid_len) == 0){
+         return;
+      }
+
+      addnode(state->rcpt, email);
+      memcpy(&(state->b_to[state->tolen]), email, len);
+      state->tolen += len;
+
+      if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(email) == 1){
+
+         if(is_email_address_on_my_domains(email, data) == 1) sdata->internal_recipient = 1;
+         else sdata->external_recipient = 1;
+
+         if(q){
+            if(findnode(state->rcpt_domain, q+1) == NULL){
+               addnode(state->rcpt_domain, q+1);
+               unsigned int domainlen = strlen(q+1);
+
+               if(state->todomainlen < SMALLBUFSIZE-domainlen-1){
+                  memcpy(&(state->b_to_domain[state->todomainlen]), q+1, domainlen);
+                  state->todomainlen += domainlen;
+               }
+            }
+         }
+
+         if(state->tolen < MAXBUFSIZE-len-1){
+            split_email_address(email);
+            memcpy(&(state->b_to[state->tolen]), email, len);
+            state->tolen += len;
+         }
+      }
+   }
+}
+
+
 /*
  * reassemble 'V i a g r a' to 'Viagra'
  */
 
 void reassembleToken(char *p){
-   unsigned int i;
+   unsigned int i, k=0;
-   int k=0;
 
    for(i=0; i<strlen(p); i++){
 
@@ -684,7 +780,7 @@ void degenerateToken(unsigned char *p){
    int i=1, d=0, dp=0;
    unsigned char *s;
 
-   /* quit if this the string does not end with a punctuation character */
+   /* quit if the string does not end with a punctuation character */
 
    if(!ispunct(*(p+strlen((char *)p)-1)))
       return;
@@ -739,12 +835,13 @@ void fixURL(char *buf, int buflen){
 }
 
 
-void extractNameFromHeaderLine(char *s, char *name, char *resultbuf){
+void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultbuflen){
-   int extended=0;
+   char buf[SMALLBUFSIZE], *p, *q;
-   char buf[SMALLBUFSIZE], puf[SMALLBUFSIZE], *p, *q, *encoding;
 
    snprintf(buf, sizeof(buf)-1, "%s", s);
 
+   memset(resultbuf, 0, resultbuflen);
+
    p = strstr(buf, name);
    if(p){
 
@@ -768,8 +865,15 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf){
        *
        *      foo: bar; title*=UTF-8''%c2%a3%20and%20%e2%82%ac%20rates
        *
+       *   Odd one having two filename definitions, and having a semicolon (;) in the filename:
+       *
+       *      filename*=utf-8''P;LAN%20Holden%204.docx;filename="P;LAN Holden 4.docx"*
+       *
+       *
        */
 
+      int extended=0;
+
       p += strlen(name);
       if(*p == '*'){
          extended = 1;
@@ -778,19 +882,26 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf){
       p = strchr(p, '=');
       if(p){
          p++;
-         q = strrchr(p, ';');
+
-         if(q) *q = '\0';
+         // skip any whitespace after name=, ie. name = "
-         q = strrchr(p, '"');
+         while(*p==' ' || *p=='\t') p++;
-         if(q){
+
-            *q = '\0';
+         // if there's a double quote after the equal symbol (=), ie. name*="utf-8....
-            p = strchr(p, '"');
+         if(*p == '"'){
-            if(p){
+            p++;
-               p++;
+            q = strchr(p, '"');
-            }
+
+            if(q) *q = '\0';
+         }
+         else {
+            // no " after =, so split on ;
+            q = strchr(p, ';');
+            if(q) *q = '\0';
          }
 
+
          if(extended == 1){
-            encoding = p;
+            char *encoding = p;
             q = strchr(p, '\'');
             if(q){
                *q = '\0';
@@ -802,15 +913,16 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf){
             decodeURL(p);
 
             if(strlen(encoding) > 2 && strcasecmp(encoding, "utf-8"))
-               utf8_encode(p, strlen(p), resultbuf, TINYBUFSIZE-1, encoding);
+               utf8_encode(p, strlen(p), resultbuf, resultbuflen-2, encoding);
             else
-               snprintf(resultbuf, TINYBUFSIZE-1, "%s", p);
+               snprintf(resultbuf, resultbuflen-2, "%s", p);
          }
          else {
+            char puf[SMALLBUFSIZE];
             snprintf(puf, sizeof(puf)-1, "%s", p);
             fixupEncodedHeaderLine(puf, sizeof(puf));
 
-            snprintf(resultbuf, TINYBUFSIZE-1, "%s", puf);
+            snprintf(resultbuf, resultbuflen-2, "%s", puf);
          }
 
       }
@@ -820,8 +932,6 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf){
 
 
 char *determine_attachment_type(char *filename, char *type){
-   char *p;
-
    if(strncasecmp(type, "text/", strlen("text/")) == 0) return "text,";
    if(strncasecmp(type, "image/", strlen("image/")) == 0) return "image,";
    if(strncasecmp(type, "audio/", strlen("audio/")) == 0) return "audio,";
@@ -852,7 +962,7 @@ char *determine_attachment_type(char *filename, char *type){
 
    if(strncasecmp(type, "application/", 12) == 0){
 
-      p = strrchr(filename, '.');
+      char *p = strrchr(filename, '.');
       if(p){
          p++;
 
@@ -862,7 +972,7 @@ char *determine_attachment_type(char *filename, char *type){
          if(strncasecmp(p, "rar", 3) == 0) return "compressed,";
 
          // tar.gz has the same type
-         if(strncasecmp(p, "x-gzip", 3) == 0) return "compressed,";
+         if(strncasecmp(p, "gz", 2) == 0) return "compressed,";
 
          if(strncasecmp(p, "rtf", 3) == 0) return "word,";
          if(strncasecmp(p, "doc", 3) == 0) return "word,";
@@ -914,14 +1024,13 @@ char *get_attachment_extractor_by_filename(char *filename){
 
 
 void parse_reference(struct parser_state *state, char *s){
-   int len;
    char puf[SMALLBUFSIZE];
 
    if(strlen(state->reference) > 10) return;
 
    do {
       s = split_str(s, " ", puf, sizeof(puf)-1);
-      len = strlen(puf);
+      int len = strlen(puf);
 
       if(len > 10 && len < SMALLBUFSIZE-1){
          memcpy(&(state->reference[strlen(state->reference)]), puf, len);
@@ -947,15 +1056,52 @@ int base64_decode_attachment_buffer(char *p, unsigned char *b, int blen){
 
 
 void fix_plus_sign_in_email_address(char *puf, char **at_sign, unsigned int *len){
-   int n;
    char *r;
 
    r = strchr(puf, '+');
    if(r){
-      n = strlen(*at_sign);
+      int n = strlen(*at_sign);
       memmove(r, *at_sign, n);
       *(r+n) = '\0';
       *len = strlen(puf);
       *at_sign = r;
    }
 }
+
+
+void fill_attachment_name_buf(struct parser_state *state, char *buf){
+   char *p = &buf[0];
+
+   for(; *p; p++){
+      if(*p != ' ' && *p != '\t') break;
+   }
+
+   int len = strlen(p);
+
+   if(len + state->anamepos < SMALLBUFSIZE-3){
+      memcpy(&(state->attachment_name_buf[state->anamepos]), p, len);
+      state->anamepos += len;
+
+      // add a trailing separator semicolon to make sure there's separation
+      // with the next item
+      state->attachment_name_buf[state->anamepos] = ';';
+      state->anamepos++;
+   }
+}
+
+
+int get_first_email_address_from_string(char *str, char *buf, int buflen){
+   int result;
+
+   char *p = str;
+   do {
+      memset(buf, 0, buflen);
+      p = split(p, ' ', buf, buflen-1, &result);
+
+      if(*buf == '\0') continue;
+
+      if(does_it_seem_like_an_email_address(buf) == 1){ return 1; }
+   } while(p);
+
+   return 0;
+}

src/piler-smtp.c

@@ -31,23 +31,37 @@ extern char *optarg;
 extern int optind;
 
 struct epoll_event event, *events=NULL;
-int timeout = 20; // checking for timeout this often [sec]
 int num_connections = 0;
 int listenerfd = -1;
+int loglevel = 1;
 
 char *configfile = CONFIG_FILE;
-struct __config cfg;
+struct config cfg;
 struct passwd *pwd;
 struct smtp_session *session, **sessions=NULL;
+struct smtp_acl *smtp_acl[MAXHASH];
+
+time_t prev_timeout_check = 0;
+
+void usage(){
+   printf("\nusage: piler\n\n");
+   printf("    -c <config file>                  Config file to use if not the default\n");
+   printf("    -d                                Fork to the background\n");
+   printf("    -v                                Return the version and build number\n");
+   printf("    -V                                Return the version and some build parameters\n");
+   printf("    -L <log level>                    Set the log level: 1-5\n");
+
+   exit(0);
+}
 
 
-void p_clean_exit(){
+void p_clean_exit(int sig){
-   int i;
+   if(sig > 0) syslog(LOG_PRIORITY, "got signal: %d, %s", sig, strsignal(sig));
 
    if(listenerfd != -1) close(listenerfd);
 
    if(sessions){
-      for(i=0; i<cfg.max_connections; i++){
+      for(int i=0; i<cfg.max_connections; i++){
          if(sessions[i]) free_smtp_session(sessions[i]);
       }
 
@@ -56,9 +70,9 @@ void p_clean_exit(){
 
    if(events) free(events);
 
-   syslog(LOG_PRIORITY, "%s has been terminated", PROGNAME);
+   clear_smtp_acl(smtp_acl);
 
-   //unlink(cfg.pidfile);
+   syslog(LOG_PRIORITY, "%s has been terminated", PROGNAME);
 
    ERR_free_strings();
 
@@ -68,26 +82,29 @@ void p_clean_exit(){
 
 void fatal(char *s){
    syslog(LOG_PRIORITY, "%s", s);
-   p_clean_exit();
+   p_clean_exit(0);
 }
 
 
 void check_for_client_timeout(){
    time_t now;
-   int i;
+
+   time(&now);
+
+   if(cfg.verbosity >= LOG_DEBUG) syslog(LOG_PRIORITY, "%s @%ld", __func__, now);
+
+   if(now - prev_timeout_check < cfg.smtp_timeout) return;
 
    if(num_connections > 0){
-      time(&now);
+      for(int i=0; i<cfg.max_connections; i++){
-
-      for(i=0; i<cfg.max_connections; i++){
          if(sessions[i] && now - sessions[i]->lasttime >= cfg.smtp_timeout){
-            syslog(LOG_PRIORITY, "client %s timeout", sessions[i]->remote_host);
+            syslog(LOG_PRIORITY, "client %s timeout, lasttime: %ld", sessions[i]->remote_host, sessions[i]->lasttime);
-            tear_down_session(sessions, sessions[i]->slot, &num_connections);
+            tear_down_session(sessions, sessions[i]->slot, &num_connections, "timeout");
          }
       }
    }
 
-   alarm(timeout);
+   time(&prev_timeout_check);
 }
 
 
@@ -111,27 +128,32 @@ void initialise_configuration(){
    setlocale(LC_MESSAGES, cfg.locale);
    setlocale(LC_CTYPE, cfg.locale);
 
+   load_smtp_acl(smtp_acl);
+
    syslog(LOG_PRIORITY, "reloaded config: %s", configfile);
 }
 
 
 int main(int argc, char **argv){
-   int listenerfd, client_sockfd;
+   int client_sockfd;
-   int i, n, daemonise=0;
+   int i, daemonise=0;
    int client_len = sizeof(struct sockaddr_storage);
    ssize_t readlen;
    struct sockaddr_storage client_address;
-   char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
+   char readbuf[REALLYBIGBUFSIZE];
-   char readbuf[BIGBUFSIZE];
    int efd;
 
-   while((i = getopt(argc, argv, "c:dvVh")) > 0){
+   while((i = getopt(argc, argv, "c:L:dvVh")) > 0){
       switch(i){
 
         case 'c' :
                    configfile = optarg;
                    break;
 
+        case 'L':
+                   loglevel = atoi(optarg);
+                   break;
+
         case 'd' :
                    daemonise = 1;
                    break;
@@ -143,7 +165,7 @@ int main(int argc, char **argv){
 
         case 'h' :
         default  :
-                   __fatal("usage: ...");
+                   usage();
       }
    }
 
@@ -179,10 +201,13 @@ int main(int argc, char **argv){
 
    set_signal_handler(SIGINT, p_clean_exit);
    set_signal_handler(SIGTERM, p_clean_exit);
-   set_signal_handler(SIGALRM, check_for_client_timeout);
+   set_signal_handler(SIGKILL, p_clean_exit);
-   set_signal_handler(SIGHUP, initialise_configuration);
+   set_signal_handler(SIGSEGV, p_clean_exit);
 
-   alarm(timeout);
+   set_signal_handler(SIGPIPE, SIG_IGN);
+   set_signal_handler(SIGALRM, SIG_IGN);
+
+   set_signal_handler(SIGHUP, initialise_configuration);
 
    // calloc() initialitizes the allocated memory
 
@@ -203,12 +228,20 @@ int main(int argc, char **argv){
 #endif
 
    for(;;){
-      n = epoll_wait(efd, events, cfg.max_connections, -1);
+      int n = epoll_wait(efd, events, cfg.max_connections, 1000);
       for(i=0; i<n; i++){
 
+         // Office365 sometimes behaves oddly: when it receives the 250 OK
+         // message after sending the email, it doesn't send the QUIT command
+         // rather it aborts the connection
+
          if((events[i].events & EPOLLERR) || (events[i].events & EPOLLHUP) || (!(events[i].events & EPOLLIN))){
-            syslog(LOG_PRIORITY, "ERROR: epoll error");
+            if(cfg.verbosity >= _LOG_EXTREME) syslog(LOG_PRIORITY, "ERROR: the remote end hung up without sending QUIT");
-            close(events[i].data.fd);
+            session = get_session_by_socket(sessions, cfg.max_connections, events[i].data.fd);
+            if(session)
+               tear_down_session(sessions, session->slot, &num_connections, "hungup");
+            else
+               close(events[i].data.fd);
             continue;
          }
 
@@ -225,13 +258,19 @@ int main(int argc, char **argv){
                      break;
                   }
                   else {
-                     syslog(LOG_PRIORITY, "ERROR: accept()");
+                     syslog(LOG_PRIORITY, "ERROR: accept(): '%s'", strerror(errno));
                      break;
                   }
                }
 
+               char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
+               memset(hbuf, 0, sizeof(hbuf));
+               memset(sbuf, 0, sizeof(sbuf));
+
                if(getnameinfo((struct sockaddr *)&client_address, client_len, hbuf, sizeof(hbuf), sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV) == 0){
-                  syslog(LOG_PRIORITY, "connected from %s:%s on descriptor %d", hbuf, sbuf, client_sockfd);
+                  // Strictly speaking it's not correct to log num_connections+1 connections
+                  // but it still gives a good clue how many connections we have at the moment
+                  syslog(LOG_PRIORITY, "connected from %s:%s on fd=%d (active connections: %d)", hbuf, sbuf, client_sockfd, num_connections + 1);
                }
 
                if(make_socket_non_blocking(client_sockfd) == -1){
@@ -246,7 +285,7 @@ int main(int argc, char **argv){
                   break;
                }
 
-               start_new_session(sessions, client_sockfd, &num_connections, &cfg);
+               start_new_session(sessions, client_sockfd, &num_connections, smtp_acl, hbuf, &cfg);
             }
 
             continue;
@@ -268,14 +307,12 @@ int main(int argc, char **argv){
             time(&(session->lasttime));
 
             while(1){
-               memset(readbuf, 0, sizeof(readbuf));
+               if(session->net.use_ssl == 1)
-
+                  readlen = SSL_read(session->net.ssl, (char*)&readbuf[0], sizeof(readbuf)-1);
-               if(session->use_ssl == 1)
-                  readlen = SSL_read(session->ssl, (char*)&readbuf[0], sizeof(readbuf)-1);
                else
                   readlen = read(events[i].data.fd, (char*)&readbuf[0], sizeof(readbuf)-1);
 
-               if(cfg.verbosity >= _LOG_DEBUG && readlen > 0) syslog(LOG_PRIORITY, "got %ld bytes to read", readlen);
+               if(cfg.verbosity >= _LOG_EXTREME && readlen > 0) syslog(LOG_PRIORITY, "got %ld bytes to read", readlen);
 
                if(readlen == -1){
                   /* If errno == EAGAIN, that means we have read all data. So go back to the main loop. */
@@ -290,7 +327,8 @@ int main(int argc, char **argv){
                   break;
                }
 
-               handle_data(session, &readbuf[0], readlen);
+               readbuf[readlen] = '\0';
+               handle_data(session, &readbuf[0], readlen, &cfg);
 
                if(session->protocol_state == SMTP_STATE_BDAT && session->bad == 1){
                   done = 1;
@@ -301,12 +339,14 @@ int main(int argc, char **argv){
             /* Don't wait until the remote client closes the connection after he sent the QUIT command */
 
             if(done || session->protocol_state == SMTP_STATE_FINISHED){
-               tear_down_session(sessions, session->slot, &num_connections);
+               tear_down_session(sessions, session->slot, &num_connections, "done");
             }
          }
 
 
       }
+
+      check_for_client_timeout();
    }
 
    return 0;

src/piler.c

@@ -34,13 +34,27 @@ extern int optind;
 int quit = 0;
 int received_sighup = 0;
 char *configfile = CONFIG_FILE;
-struct __config cfg;
+struct config cfg;
-struct __data data;
+struct data data;
 struct passwd *pwd;
 
 struct child children[MAXCHILDREN];
 
 
+void p_clean_exit();
+
+
+void usage(){
+   printf("\nusage: piler\n\n");
+   printf("    -c <config file>                  Config file to use if not the default\n");
+   printf("    -d                                Fork to the background\n");
+   printf("    -v                                Return the version and build number\n");
+   printf("    -V                                Return the version and some build parameters\n");
+
+   exit(0);
+}
+
+
 void takesig(int sig){
    int i, status;
    pid_t pid;
@@ -87,12 +101,48 @@ void child_sighup_handler(int sig){
 }
 
 
-int process_email(char *filename, struct session_data *sdata, struct __data *data, int size, struct __config *cfg){
+int perform_checks(char *filename, struct session_data *sdata, struct data *data, struct parser_state *parser_state, struct config *cfg){
-   int rc;
+
+   if(cfg->security_header[0] && parser_state->found_security_header == 0){
+      syslog(LOG_PRIORITY, "%s: discarding: missing security header", filename);
+      return ERR_DISCARDED;
+   }
+
+   char *arule = check_against_ruleset(data->archiving_rules, parser_state, sdata->tot_len, sdata->spam_message);
+
+   if(arule){
+      syslog(LOG_PRIORITY, "%s: discarding: archiving policy: *%s*", filename, arule);
+      return ERR_DISCARDED;
+   }
+
+
+   if(cfg->archive_only_mydomains == 1 && sdata->internal_sender == 0 && sdata->internal_recipient == 0){
+      syslog(LOG_PRIORITY, "%s: discarding: not on mydomains", filename);
+      return ERR_DISCARDED;
+   }
+
+   make_digests(sdata, cfg);
+
+   // A normal header is much bigger than 10 bytes. We get here for header-only
+   // messages without a Message-ID: line. I believe that no such message is valid, and
+   // it's a reasonable to discard it, and not allowing it to fill up the error directory.
+
+   if(sdata->hdr_len < 10){
+      syslog(LOG_PRIORITY, "%s: discarding: a header-only message without a Message-ID line", filename);
+      return ERR_DISCARDED;
+   }
+
+   int rc = process_message(sdata, parser_state, data, cfg);
+   unlink(parser_state->message_id_hash);
+
+   return rc;
+}
+
+
+int process_email(char *filename, struct session_data *sdata, struct data *data, int size, struct config *cfg){
    char tmpbuf[SMALLBUFSIZE];
    char *status=S_STATUS_UNDEF;
-   char *arule;
+   char *p;
-   char *rcpt;
    struct timezone tz;
    struct timeval tv1, tv2;
    struct parser_state parser_state;
@@ -116,10 +166,11 @@ int process_email(char *filename, struct session_data *sdata, struct __data *dat
    snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
 
    parser_state = parse_message(sdata, 1, data, cfg);
+
    post_parse(sdata, &parser_state, cfg);
 
    if(cfg->syslog_recipients == 1){
-      rcpt = parser_state.b_to;
+      char *rcpt = parser_state.b_to;
       do {
          rcpt = split_str(rcpt, " ", tmpbuf, sizeof(tmpbuf)-1);
 
@@ -129,28 +180,10 @@ int process_email(char *filename, struct session_data *sdata, struct __data *dat
       } while(rcpt);
    }
 
-
+   int rc = perform_checks(filename, sdata, data, &parser_state, cfg);
-   arule = check_againt_ruleset(data->archiving_rules, &parser_state, sdata->tot_len, sdata->spam_message);
-
-   if(arule){
-      syslog(LOG_PRIORITY, "%s: discarding: archiving policy: *%s*", filename, arule);
-      rc = ERR_DISCARDED;
-      remove_stripped_attachments(&parser_state);
-   }
-   else {
-      make_digests(sdata, cfg);
-
-      if(sdata->hdr_len < 10){
-         syslog(LOG_PRIORITY, "%s: invalid message, hdr_len: %d", filename, sdata->hdr_len);
-         rc = ERR;
-      }
-
-      rc = process_message(sdata, &parser_state, data, cfg);
-      unlink(parser_state.message_id_hash);
-   }
-
    unlink(sdata->tmpframe);
 
+   remove_stripped_attachments(&parser_state);
 
    if(rc == OK){
       status = S_STATUS_STORED;
@@ -169,6 +202,19 @@ int process_email(char *filename, struct session_data *sdata, struct __data *dat
    }
    else {
       status = S_STATUS_ERROR;
+
+      // move the file from piler/tmp/[0-xxx] dir to piler/error directory
+      p = strchr(filename, '/');
+      if(p)
+         p++;
+      else
+         p = filename;
+
+      snprintf(tmpbuf, sizeof(tmpbuf)-1, "%s/%s", ERROR_DIR, p);
+      if(rename(filename, tmpbuf) == 0)
+         syslog(LOG_PRIORITY, "%s: moved to %s", filename, tmpbuf);
+      else
+         syslog(LOG_PRIORITY, "%s: failed to moved to %s", filename, tmpbuf);
    }
 
    if(rc != ERR) unlink(filename);
@@ -187,7 +233,7 @@ int process_email(char *filename, struct session_data *sdata, struct __data *dat
 }
 
 
-int process_dir(char *directory, struct session_data *sdata, struct __data *data, struct __config *cfg){
+int process_dir(char *directory, struct session_data *sdata, struct data *data, struct config *cfg){
    DIR *dir;
    struct dirent *de;
    int tot_msgs=0;
@@ -241,10 +287,19 @@ void child_main(struct child *ptr){
 
       sig_block(SIGHUP);
 
-      if(open_database(&sdata, &cfg) == OK){
+      int sphxopen = 0;
+      if(cfg.rtindex && open_sphx(&sdata, &cfg) == OK){
+         sphxopen = 1;
+      }
+
+      if((cfg.rtindex == 0 || sphxopen == 1) && open_database(&sdata, &cfg) == OK){
          ptr->messages += process_dir(dir, &sdata, &data, &cfg);
          close_database(&sdata);
 
+         if(cfg.rtindex){
+            close_sphx(&sdata);
+         }
+
          sleep(1);
       }
       else {
@@ -254,13 +309,13 @@ void child_main(struct child *ptr){
 
       sig_unblock(SIGHUP);
 
-      // TODO: do we want to quit after processing a certain number of messages?
+      // Let the child quit after processing max_requests_per_child messages
-
-      //if(cfg.max_requests_per_child > 0 && ptr->messages >= cfg.max_requests_per_child){
-      //   if(cfg.verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "child (pid: %d, serial: %d) served enough: %d", getpid(), ptr->messages, ptr->serial);
-      //   break;
-      //}
 
+      if(cfg.max_requests_per_child > 0 && ptr->messages >= cfg.max_requests_per_child){
+         if(cfg.verbosity >= _LOG_DEBUG)
+            syslog(LOG_PRIORITY, "child (pid: %d, serial: %d) served enough: %d", getpid(), ptr->messages, ptr->serial);
+         break;
+      }
    }
 
 #ifdef HAVE_MEMCACHED
@@ -415,9 +470,9 @@ void initialise_configuration(){
       return;
    }
 
-   load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
+   load_rules(&sdata, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
-   load_rules(&sdata, &data, data.retention_rules, SQL_RETENTION_RULE_TABLE);
+   load_rules(&sdata, data.retention_rules, SQL_RETENTION_RULE_TABLE);
-   load_rules(&sdata, &data, data.folder_rules, SQL_FOLDER_RULE_TABLE);
+   load_rules(&sdata, data.folder_rules, SQL_FOLDER_RULE_TABLE);
 
    load_mydomains(&sdata, &data, &cfg);
 
@@ -435,7 +490,8 @@ void initialise_configuration(){
 
 
 int main(int argc, char **argv){
-   int i, daemonise=0, dedupfd;
+   int i, daemonise=0;
+   struct stat st;
 
 
    while((i = getopt(argc, argv, "c:dvVh")) > 0){
@@ -460,7 +516,7 @@ int main(int argc, char **argv){
 
         case 'h' :
         default  : 
-                   __fatal("usage: ...");
+                   usage();
       }
    }
 
@@ -473,6 +529,7 @@ int main(int argc, char **argv){
    initrules(data.retention_rules);
    initrules(data.folder_rules);
    data.dedup = MAP_FAILED;
+   data.import = NULL;
 
    initialise_configuration();
 
@@ -484,8 +541,10 @@ int main(int argc, char **argv){
 
    if(drop_privileges(pwd)) fatal(ERR_SETUID);
 
+   if(stat(cfg.pidfile, &st) == 0) fatal(ERR_PID_FILE_EXISTS);
+
    if(cfg.mmap_dedup_test == 1){
-      dedupfd = open(MESSAGE_ID_DEDUP_FILE, O_RDWR);
+      int dedupfd = open(MESSAGE_ID_DEDUP_FILE, O_RDWR);
       if(dedupfd == -1) fatal(ERR_OPEN_DEDUP_FILE);
 
       data.dedup = mmap(NULL, MAXCHILDREN*DIGEST_LENGTH*2, PROT_READ|PROT_WRITE, MAP_SHARED, dedupfd, 0);

src/piler.h

@@ -17,6 +17,7 @@
 #include <sig.h>
 #include <av.h>
 #include <rules.h>
+#include <screen.h>
 #include <sql.h>
 #include <import.h>
 #include <smtp.h>
@@ -27,50 +28,54 @@
    #include "memc.h"
 #endif
 
-int read_key(struct __config *cfg);
+int read_key(struct config *cfg);
 void insert_offset(struct session_data *sdata, int server_id);
 
 void tear_down_client(int n);
 
-int do_av_check(char *filename, struct __config *cfg);
+int do_av_check(char *filename, struct config *cfg);
 
-int make_digests(struct session_data *sdata, struct __config *cfg);
+int make_digests(struct session_data *sdata, struct config *cfg);
 void digest_file(char *filename, char *digest);
-void digest_string(char *s, char *digest);
+void digest_string(char *digestname, char *s, char *digest);
+void create_md5_from_email_address(char *puf, char *md5buf);
 
 void remove_stripped_attachments(struct parser_state *state);
-int process_message(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg);
+int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
-int reimport_message(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg);
+void rollback(struct session_data *sdata, struct parser_state *state, uint64 id, struct config *cfg);
-int store_file(struct session_data *sdata, char *filename, int len, struct __config *cfg);
+int reimport_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
-int remove_stored_message_files(struct session_data *sdata, struct parser_state *state, struct __config *cfg);
+int store_file(struct session_data *sdata, char *filename, int len, struct config *cfg);
-int store_attachments(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg);
+int remove_stored_message_files(struct session_data *sdata, struct parser_state *state, struct config *cfg);
-int query_attachments(struct session_data *sdata, struct __data *data, struct ptr_array *ptr_arr);
+int store_attachments(struct session_data *sdata, struct parser_state *state, struct config *cfg);
+int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr);
 
-struct __config read_config(char *configfile);
+struct config read_config(char *configfile);
 
-void check_and_create_directories(struct __config *cfg, uid_t uid, gid_t gid);
+void createdir(char *path, uid_t uid, gid_t gid, mode_t mode);
+void check_and_create_directories(struct config *cfg, uid_t uid, gid_t gid);
 
-void update_counters(struct session_data *sdata, struct __data *data, struct counters *counters, struct __config *cfg);
+void update_counters(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 
-int retrieve_email_from_archive(struct session_data *sdata, struct __data *data, FILE *dest, struct __config *cfg);
+int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct config *cfg);
-int file_from_archive_to_network(char *filename, int sd, int tls_enable, struct __data *data, struct __config *cfg);
+int file_from_archive_to_network(char *filename, int sd, int tls_enable, struct data *data, struct config *cfg);
 
-int get_folder_id(struct session_data *sdata, struct __data *data, char *foldername, int parent_id);
+int get_folder_id(struct session_data *sdata, char *foldername, int parent_id);
-int add_new_folder(struct session_data *sdata, struct __data *data, char *foldername, int parent_id);
+int add_new_folder(struct session_data *sdata, char *foldername, int parent_id);
 
-int store_index_data(struct session_data *sdata, struct parser_state *state, struct __data *data, uint64 id, struct __config *cfg);
+int store_index_data(struct session_data *sdata, struct parser_state *state, struct data *data, uint64 id, struct config *cfg);
 
-void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct __config *cfg);
+void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct config *cfg);
 
-int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct __config *cfg);
+int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct config *cfg);
 
-void load_mydomains(struct session_data *sdata, struct __data *data, struct __config *cfg);
+void load_mydomains(struct session_data *sdata, struct data *data, struct config *cfg);
-int is_email_address_on_my_domains(char *email, struct __data *data);
+int is_email_address_on_my_domains(char *email, struct data *data);
 
-int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct __config *cfg);
+int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg);
-void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections);
+void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections, char *reason);
 struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int max_connections, int socket);
-void handle_data(struct smtp_session *session, char *readbuf, int readlen);
+void write_envelope_addresses(struct smtp_session *session, struct config *cfg);
+void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg);
 void free_smtp_session(struct smtp_session *session);
 
 void child_sighup_handler(int sig);
@@ -78,7 +83,6 @@ void child_main(struct child *ptr);
 pid_t child_make(struct child *ptr);
 int search_slot_by_pid(pid_t pid);
 void kill_children(int sig);
-void p_clean_exit();
 void fatal(char *s);
 void initialise_configuration();
 

src/pileraget.c

@@ -22,7 +22,7 @@ int main(int argc, char **argv){
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
    struct stat st;
 #endif
-   struct __config cfg;
+   struct config cfg;
 
 
    if(argc < 3){
@@ -46,7 +46,7 @@ int main(int argc, char **argv){
       return 1;
    }
 
-   snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
+   snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, argv[1][24], argv[1][25], argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
    if(stat(filename, &st)){
       snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][RND_STR_LEN-6], argv[1][RND_STR_LEN-5], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
@@ -57,5 +57,3 @@ int main(int argc, char **argv){
 
    return 0;
 }
-
-

src/pilerconf.c

@@ -12,13 +12,13 @@ extern char *optarg;
 extern int optind;
 
 
-void print_config_all(struct __config *cfg, char *key);
+void print_config_all(struct config *cfg, char *key);
-void print_config(char *configfile, struct __config *cfg);
+void print_config(char *configfile, struct config *cfg);
 
 int main(int argc, char **argv){
    int i, print_from_file=0;
    char *configfile=CONFIG_FILE, *query=NULL;
-   struct __config cfg;
+   struct config cfg;
 
    while((i = getopt(argc, argv, "c:q:nh?")) > 0){
        switch(i){

src/pilerexport.c

@@ -8,11 +8,13 @@
 #include <sys/time.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/mman.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <time.h>
 #include <locale.h>
 #include <syslog.h>
+#include <zip.h>
 #include <getopt.h>
 #include <piler.h>
 
@@ -22,15 +24,22 @@ extern int optind;
 
 int dryrun = 0;
 int exportall = 0;
-int rc = 0;
+int verification_status = 0;
+int export_to_stdout = 0;
 char *query=NULL;
 int verbosity = 0;
 int max_matches = 1000;
 char *index_list = "main1,dailydelta1,delta1";
+struct passwd *pwd;
 regex_t regexp;
+char *zipfile = NULL;
+struct zip *zip = NULL;
+uint64 *zip_ids = NULL;
+int zip_counter = 0;
+int zip_batch = 2000;
+int max_files_in_export_dir = 0;
 
-
+int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg);
-int export_emails_matching_to_query(struct session_data *sdata, struct __data *data, char *s, struct __config *cfg);
 
 
 void usage(){
@@ -48,7 +57,13 @@ void usage(){
    printf("    -w <where condition>              Where condition to pass to sphinx, eg. \"match('@subject: piler')\"\n");
    printf("    -m <max. matches>                 Max. matches to apply to sphinx query (default: %d)\n", max_matches);
    printf("    -i <index list>                   Sphinx indices to use  (default: %s)\n", index_list);
+#if LIBZIP_VERSION_MAJOR >= 1
+   printf("    -z <zip file>                     Write exported EML files to a zip file\n");
+   printf("    -Z <batch size>                   Zip batch size. Valid range: 10-10000, default: 2000\n");
+#endif
    printf("    -A                                Export all emails from archive\n");
+   printf("    -D <max files>                    Max. number of files to put in a single directory, default: 2000\n");
+   printf("    -o                                Export emails to stdout\n");
    printf("    -d                                Dry run\n");
 
    regfree(&regexp);
@@ -87,15 +102,12 @@ unsigned long convert_time(char *yyyymmdd, int h, int m, int s){
 
    tm.tm_mday = atoi(yyyymmdd);
 
-
-   tm.tm_isdst = -1;
-
    return mktime(&tm);
 }
 
 
 int append_email_to_buffer(char **buffer, char *email){
-   int len, arglen;
+   int arglen;
    char *s=NULL, emailaddress[SMALLBUFSIZE];
 
    snprintf(emailaddress, sizeof(emailaddress)-1, "'%s'", email);
@@ -107,7 +119,7 @@ int append_email_to_buffer(char **buffer, char *email){
       memcpy(*buffer, emailaddress, arglen);
    }
    else {
-      len = strlen(*buffer);
+      int len = strlen(*buffer);
       s = realloc(*buffer, len + arglen+2);
       if(!s){
          printf("malloc problem!\n");
@@ -125,50 +137,29 @@ int append_email_to_buffer(char **buffer, char *email){
 }
 
 
-int append_string_to_buffer(char **buffer, char *str){
+uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *where_condition, uint64 last_id, int *num, struct config *cfg){
-   int len, arglen;
-   char *s=NULL;
-
-   arglen = strlen(str);
-
-   if(!*buffer){
-      *buffer = malloc(arglen+1);
-      memset(*buffer, 0, arglen+1);
-      memcpy(*buffer, str, arglen);
-   }
-   else {
-      len = strlen(*buffer);
-      s = realloc(*buffer, len + arglen+1);
-      if(!s) return 1;
-
-      *buffer = s;
-
-      memset(*buffer+len, 0, arglen+1);
-      memcpy(*buffer+len, str, arglen);
-   }
-
-   return 0;
-}
-
-
-uint64 run_query(struct session_data *sdata, struct session_data *sdata2, struct __data *data, char *where_condition, uint64 last_id, int *num, struct __config *cfg){
-   MYSQL_RES *res;
    MYSQL_ROW row;
-   int rc=0;
    uint64 id=0;
-   char s[SMALLBUFSIZE];
+   char s[MAXBUFSIZE];
+   int rc=0;
 
    *num = 0;
 
    if(!where_condition) return id;
 
-   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE id IN (", SQL_METADATA_TABLE);
+   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE id IN (", SQL_METADATA_TABLE);
    rc += append_string_to_buffer(&query, s);
 
    snprintf(s, sizeof(s)-1, "SELECT id FROM %s WHERE %s AND id > %llu ORDER BY id ASC LIMIT 0,%d", index_list, where_condition, last_id, max_matches);
 
+   if(dryrun){
+      printf("sphinx query: %s\n", s);
+   }
+
+   syslog(LOG_PRIORITY, "sphinx query: %s", s);
+
    if(mysql_real_query(&(sdata2->mysql), s, strlen(s)) == 0){
-      res = mysql_store_result(&(sdata2->mysql));
+      MYSQL_RES *res = mysql_store_result(&(sdata2->mysql));
       if(res != NULL){
          while((row = mysql_fetch_row(res))){
             id = strtoull(row[0], NULL, 10);
@@ -183,7 +174,8 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, struct
       }
    }
 
-   if(!rc) export_emails_matching_to_query(sdata, data, query, cfg);
+   if(!rc) export_emails_matching_to_query(sdata, query, cfg);
+   else printf("error: append_string_to_buffer() in run_query()\n");
 
    free(query);
    query = NULL;
@@ -194,12 +186,11 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, struct
 
 
 uint64 get_total_found(struct session_data *sdata){
-   MYSQL_RES *res;
    MYSQL_ROW row;
    uint64 total_found=0;
 
    if(mysql_real_query(&(sdata->mysql), "SHOW META LIKE 'total_found'", 28) == 0){
-      res = mysql_store_result(&(sdata->mysql));
+      MYSQL_RES *res = mysql_store_result(&(sdata->mysql));
       if(res != NULL){
          while((row = mysql_fetch_row(res))){
             total_found = strtoull(row[1], NULL, 10);
@@ -212,17 +203,17 @@ uint64 get_total_found(struct session_data *sdata){
 }
 
 
-void export_emails_matching_id_list(struct session_data *sdata, struct session_data *sdata2, struct __data *data, char *where_condition, struct __config *cfg){
+void export_emails_matching_id_list(struct session_data *sdata, struct session_data *sdata2, char *where_condition, struct config *cfg){
    int n;
    uint64 count=0, last_id=0, total_found=0;
 
-   last_id = run_query(sdata, sdata2, data, where_condition, last_id, &n, cfg);
+   last_id = run_query(sdata, sdata2, where_condition, last_id, &n, cfg);
    count += n;
 
    total_found = get_total_found(sdata2);
 
    while(count < total_found){
-      last_id = run_query(sdata, sdata2, data, where_condition, last_id, &n, cfg);
+      last_id = run_query(sdata, sdata2, where_condition, last_id, &n, cfg);
       count += n;
    }
 
@@ -230,102 +221,86 @@ void export_emails_matching_id_list(struct session_data *sdata, struct session_d
 
 
 int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain, int minsize, int maxsize, unsigned long startdate, unsigned long stopdate){
-   int where_condition=1;
    char s[SMALLBUFSIZE];
+   int rc=0;
 
    if(exportall == 1){
-      rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM ");
+      rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM ");
       rc += append_string_to_buffer(&query, SQL_METADATA_TABLE);
       rc += append_string_to_buffer(&query, " WHERE deleted=0 ");
       return rc;
    }
 
-   snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE deleted=0 ", SQL_MESSAGES_VIEW);
+   snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE deleted=0 ", SQL_MESSAGES_VIEW);
 
    rc = append_string_to_buffer(&query, s);
 
    if(from){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`from` IN (");
       rc += append_string_to_buffer(&query, from);
       rc += append_string_to_buffer(&query, ")");
 
       free(from);
-
-      where_condition++;
    }
 
    if(to){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`to` IN (");
       rc += append_string_to_buffer(&query, to);
       rc += append_string_to_buffer(&query, ")");
 
       free(to);
-
-      where_condition++;
    }
 
    if(fromdomain){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`fromdomain` IN (");
       rc += append_string_to_buffer(&query, fromdomain);
       rc += append_string_to_buffer(&query, ")");
 
       free(fromdomain);
-
-      where_condition++;
    }
 
 
    if(todomain){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
 
       rc += append_string_to_buffer(&query, "`todomain` IN (");
       rc += append_string_to_buffer(&query, todomain);
       rc += append_string_to_buffer(&query, ")");
 
       free(todomain);
-
-      where_condition++;
    }
 
    if(minsize > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
       snprintf(s, sizeof(s)-1, " `size` >= %d", minsize);
       rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
    }
 
 
    if(maxsize > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
       snprintf(s, sizeof(s)-1, " `size` <= %d", maxsize);
       rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
    }
 
 
    if(startdate > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
-      snprintf(s, sizeof(s)-1, " `sent` >= %ld", startdate);
+      snprintf(s, sizeof(s)-1, " `sent` >= %lu", startdate);
       rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
    }
 
 
    if(stopdate > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      rc = append_string_to_buffer(&query, " AND ");
-      snprintf(s, sizeof(s)-1, " `sent` <= %ld", stopdate);
+      snprintf(s, sizeof(s)-1, " `sent` <= %lu", stopdate);
       rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
    }
 
 
@@ -334,44 +309,87 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain
    return rc;
 }
 
+#if LIBZIP_VERSION_MAJOR >= 1
+void zip_flush(){
+   zip_close(zip);
 
-int export_emails_matching_to_query(struct session_data *sdata, struct __data *data, char *s, struct __config *cfg){
+   zip = NULL;
+   zip_counter = 0;
+
+   if(!zip_ids) return;
+
+   for(int i=0; i<zip_batch; i++){
+      if(*(zip_ids+i)){
+         char filename[SMALLBUFSIZE];
+         snprintf(filename, sizeof(filename)-1, "%llu.eml", *(zip_ids+i));
+         unlink(filename);
+      }
+   }
+
+   free(zip_ids);
+   zip_ids = NULL;
+}
+#endif
+
+int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg){
    FILE *f;
-   uint64 id, n=0;
+   uint64 id, n=0, dir_counter=0;
    char digest[SMALLBUFSIZE], bodydigest[SMALLBUFSIZE];
    char filename[SMALLBUFSIZE];
-   int rc=0;
+   char export_subdir[SMALLBUFSIZE];
+   struct sql sql;
+   int errorp, rc=0, attachments;
+   unsigned long total_attachments=0;
+
+   if(prepare_sql_statement(sdata, &sql, s) == ERR) return ERR;
 
 
-   if(prepare_sql_statement(sdata, &(data->stmt_generic), s) == ERR) return ERR;
+   p_bind_init(&sql);
 
-
+   if(p_exec_stmt(sdata, &sql) == ERR) goto ENDE;
-   p_bind_init(data);
-
-   if(p_exec_query(sdata, data->stmt_generic, data) == ERR) goto ENDE;
 
 
 
-   p_bind_init(data);
+   p_bind_init(&sql);
 
-   data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+   sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-   data->sql[data->pos] = sdata->ttmpfile; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = RND_STR_LEN; data->pos++;
+   sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++;
-   data->sql[data->pos] = &digest[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(digest)-2; data->pos++;
+   sql.sql[sql.pos] = &digest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(digest)-2; sql.pos++;
-   data->sql[data->pos] = &bodydigest[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(bodydigest)-2; data->pos++;
+   sql.sql[sql.pos] = &bodydigest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(bodydigest)-2; sql.pos++;
+   sql.sql[sql.pos] = (char *)&attachments; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++;
 
-   p_store_results(data->stmt_generic, data);
+   p_store_results(&sql);
 
-   while(p_fetch_results(data->stmt_generic) == OK){
+   while(p_fetch_results(&sql) == OK){
 
       if(id > 0){
 
          if(dryrun == 0){
 
+            if(export_to_stdout){
+               printf("%s", PILEREXPORT_BEGIN_MARK);
+               rc = retrieve_email_from_archive(sdata, stdout, cfg);
+               continue;
+            }
+
+            if(max_files_in_export_dir > 0 && n % max_files_in_export_dir == 0){
+               dir_counter++;
+               snprintf(export_subdir, sizeof(export_subdir)-1, "export-%llu", dir_counter);
+               if(n > 0 && chdir("..")){
+                  p_clean_exit("error chdir(\"..\")", 1);
+               }
+
+               createdir(export_subdir, pwd->pw_uid, pwd->pw_gid, 0700);
+               if(chdir(export_subdir)){
+                  p_clean_exit("error chdir to export-* dir", 1);
+               }
+            }
+
             snprintf(filename, sizeof(filename)-1, "%llu.eml", id);
 
             f = fopen(filename, "w");
             if(f){
-               rc = retrieve_email_from_archive(sdata, data, f, cfg);
+               rc = retrieve_email_from_archive(sdata, f, cfg);
                fclose(f);
 
                n++;
@@ -383,13 +401,48 @@ int export_emails_matching_to_query(struct session_data *sdata, struct __data *d
                if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0){
                   printf("exported: %10llu\r", n); fflush(stdout);
                }
-               else
+               else {
                   printf("verification FAILED. %s\n", filename);
+                  verification_status = 1;
+               }
 
+               if(zipfile){
+               #if LIBZIP_VERSION_MAJOR >= 1
+                  // Open zip file if handler is NULL
+                  if(!zip){
+                     zip = zip_open(zipfile, ZIP_CREATE, &errorp);
+                     if(!zip){
+                        printf("error: error creating zip file=%s, error code=%d\n", zipfile, errorp);
+                        return ERR;
+                     }
+                  }
+
+                  if(!zip_ids) zip_ids = (uint64*) calloc(sizeof(uint64), zip_batch);
+
+                  if(!zip_ids){
+                     printf("calloc error for zip_ids\n");
+                     return ERR;
+                  }
+
+                  zip_source_t *zs = zip_source_file(zip, filename, 0, 0);
+                  if(zs && zip_file_add(zip, filename, zs, ZIP_FL_ENC_UTF_8) >= 0){
+                     *(zip_ids+zip_counter) = id;
+                     zip_counter++;
+                  } else {
+                     printf("error adding file %s: %s\n", filename, zip_strerror(zip));
+                     return ERR;
+                  }
+
+                  if(zip_counter == zip_batch){
+                     zip_flush();
+                  }
+               #endif
+               }
             }
             else printf("cannot open: %s\n", filename);
          }
          else {
+            total_attachments += attachments;
             printf("id:%llu\n", id);
          }
 
@@ -397,11 +450,14 @@ int export_emails_matching_to_query(struct session_data *sdata, struct __data *d
 
    }
 
-   p_free_results(data->stmt_generic);
+   p_free_results(&sql);
 
 ENDE:
-   close_prepared_statement(data->stmt_generic);
+   close_prepared_statement(&sql);
 
+   if(dryrun){
+      printf("attachments: %lu\n", total_attachments);
+   }
 
    printf("\n");
 
@@ -410,14 +466,13 @@ ENDE:
 
 
 int main(int argc, char **argv){
-   int c, minsize=0, maxsize=0;
+   int minsize=0, maxsize=0;
    size_t nmatch=0;
    unsigned long startdate=0, stopdate=0;
    char *configfile=CONFIG_FILE;
    char *to=NULL, *from=NULL, *todomain=NULL, *fromdomain=NULL, *where_condition=NULL;
    struct session_data sdata, sdata2;
-   struct __data data;
+   struct config cfg;
-   struct __config cfg;
 
 
    if(regcomp(&regexp, "^([\\+a-z0-9_\\.@\\-]+)$", REG_ICASE | REG_EXTENDED)){
@@ -436,6 +491,7 @@ int main(int argc, char **argv){
             {"all",          no_argument,        0,  'A' },
             {"dry-run",      no_argument,        0,  'd' },
             {"dryrun",       no_argument,        0,  'd' },
+            {"stdout",       no_argument,        0,  'o' },
             {"help",         no_argument,        0,  'h' },
             {"version",      no_argument,        0,  'v' },
             {"from",         required_argument,  0,  'f' },
@@ -444,7 +500,10 @@ int main(int argc, char **argv){
             {"to-domain",    required_argument,  0,  'R' },
             {"start-date",   required_argument,  0,  'a' },
             {"stop-date",    required_argument,  0,  'b' },
+            {"zip",          required_argument,  0,  'z' },
+            {"zip-batch",    required_argument,  0,  'Z' },
             {"where-condition", required_argument,  0,  'w' },
+            {"max-files",    required_argument,  0,  'D' },
             {"max-matches",  required_argument,  0,  'm' },
             {"index-list",   required_argument,  0,  'i' },
             {0,0,0,0}
@@ -452,9 +511,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:D:oAdhv?", long_options, &option_index);
 #else
-      c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?");
+      int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:D:oAdhv?");
 #endif
 
       if(c == -1) break;
@@ -485,7 +544,10 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&from, optarg);
+                    if(append_email_to_buffer(&from, optarg)){
+                       printf("error: append_email_to_buffer() for from\n");
+                       return 1;
+                    }
 
                     break;
 
@@ -496,7 +558,10 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&to, optarg);
+                    if(append_email_to_buffer(&to, optarg)){
+                       printf("error: append_email_to_buffer() for to\n");
+                       return 1;
+                    }
 
                     break;
 
@@ -507,7 +572,10 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&fromdomain, optarg);
+                    if(append_email_to_buffer(&fromdomain, optarg)){
+                       printf("error: append_email_to_buffer() for fromdomain\n");
+                       return 1;
+                    }
 
                     break;
 
@@ -518,7 +586,10 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&todomain, optarg);
+                    if(append_email_to_buffer(&todomain, optarg)){
+                       printf("error: append_email_to_buffer() for todomain\n");
+                       return 1;
+                    }
 
                     break;
 
@@ -543,6 +614,23 @@ int main(int argc, char **argv){
                     index_list = optarg;
                     break;
 
+         case 'z':  zipfile = optarg;
+                    break;
+
+         case 'Z':  zip_batch = atoi(optarg);
+                    if(zip_batch < 10 || zip_batch > 10000)
+                       zip_batch = 2000;
+                    break;
+
+         case 'D':  max_files_in_export_dir = atoi(optarg);
+                    if(max_files_in_export_dir < 10 || max_files_in_export_dir > 100000)
+                       max_files_in_export_dir = 2000;
+                    break;
+
+         case 'o':
+                    export_to_stdout = 1;
+                    break;
+
          case 'd' :
                     dryrun = 1;
                     break;
@@ -570,6 +658,11 @@ int main(int argc, char **argv){
 
    if(read_key(&cfg)) p_clean_exit(ERR_READING_KEY, 1);
 
+   if(strlen(cfg.username) > 1){
+      pwd = getpwnam(cfg.username);
+      if(!pwd) __fatal(ERR_NON_EXISTENT_USER);
+   }
+
 
    init_session_data(&sdata, &cfg);
 
@@ -591,19 +684,23 @@ int main(int argc, char **argv){
          p_clean_exit("cannot connect to 127.0.0.1:9306", 1);
       }
 
-      export_emails_matching_id_list(&sdata, &sdata2, &data, where_condition, &cfg);
+      export_emails_matching_id_list(&sdata, &sdata2, where_condition, &cfg);
 
       close_database(&sdata2);
    }
    else {
       if(build_query_from_args(from, to, fromdomain, todomain, minsize, maxsize, startdate, stopdate) > 0) p_clean_exit("malloc problem building query", 1);
-      export_emails_matching_to_query(&sdata, &data, query, &cfg);
+      export_emails_matching_to_query(&sdata, query, &cfg);
       free(query);
    }
 
    close_database(&sdata);
 
-   return 0;
+   if(zipfile){
+   #if LIBZIP_VERSION_MAJOR >= 1
+      zip_flush();
+   #endif
+   }
+
+   return verification_status;
 }
-
-

src/pilerget.c

@@ -19,8 +19,7 @@
 int main(int argc, char **argv){
    int readkey=1;
    struct session_data sdata;
-   struct __data data;
+   struct config cfg;
-   struct __config cfg;
 
 
    if(argc < 2){
@@ -48,12 +47,10 @@ int main(int argc, char **argv){
 
    snprintf(sdata.ttmpfile, SMALLBUFSIZE-1, "%s", argv[1]);
    snprintf(sdata.filename, SMALLBUFSIZE-1, "%s", sdata.ttmpfile);
-   retrieve_email_from_archive(&sdata, &data, stdout, &cfg);
+   retrieve_email_from_archive(&sdata, stdout, &cfg);
 
 
    close_database(&sdata);
 
    return 0;
 }
-
-

src/pilerimport.c

@@ -30,9 +30,6 @@
 extern char *optarg;
 extern int optind;
 
-int dryrun=0;
-int import_from_gui=0;
-
 
 void usage(){
    printf("\nusage: pilerimport\n\n");
@@ -55,23 +52,32 @@ void usage(){
    printf("    -s <start position>               Start importing POP3 emails from this position\n");
    printf("    -j <failed folder>                Move failed to import emails to this folder\n");
    printf("    -a <recipient>                    Add recipient to the To:/Cc: list\n");
+   printf("    -T <id>                           Update import table at id=<id>\n");
+   printf("    -Z <ms>                           Delay Z milliseconds in between emails being imported\n");
    printf("    -D                                Dry-run, do not import anything\n");
+   printf("    -y                                Read pilerexport data from stdin\n");
    printf("    -o                                Only download emails for POP3/IMAP import\n");
    printf("    -r                                Remove imported emails\n");
    printf("    -q                                Quiet mode\n");
+   printf("    -A <timestamp>                    Import emails sent after this timestamp\n");
+   printf("    -B <timestamp>                    Import emails sent before this timestamp\n");
 
    exit(0);
 }
 
 
 int main(int argc, char **argv){
-   int i, c, rc=0, n_mbox=0, tot_msgs=0, port=143;
+   int i, n_mbox=0, read_from_pilerexport=0;
-   char *configfile=CONFIG_FILE, *emlfile=NULL, *mboxdir=NULL, *mbox[MBOX_ARGS], *directory=NULL;
+   char *configfile=CONFIG_FILE, *mbox[MBOX_ARGS], *directory=NULL;
-   char *imapserver=NULL, *pop3server=NULL, *username=NULL, *password=NULL, *skiplist=SKIPLIST, *folder=NULL, *folder_imap=NULL;
+   char puf[SMALLBUFSIZE], *imapserver=NULL, *pop3server=NULL;
    struct session_data sdata;
-   struct __config cfg;
+   struct config cfg;
-   struct __data data;
+   struct data data;
    struct import import;
+   struct net net;
+   struct counters counters;
+
+   bzero(&counters, sizeof(counters));
 
    for(i=0; i<MBOX_ARGS; i++) mbox[i] = NULL;
 
@@ -87,10 +93,30 @@ int main(int argc, char **argv){
    import.extra_recipient = import.move_folder = import.failed_folder = NULL;
    import.start_position = 1;
    import.download_only = 0;
-   import.timeout = 30;
+   import.dryrun = 0;
+   import.port = 143;
+   import.server = NULL;
+   import.username = NULL;
+   import.password = NULL;
+   import.database = NULL;
+   import.skiplist = SKIPLIST;
+   import.folder_imap = NULL;
+   memset(import.filename, 0, SMALLBUFSIZE);
+   import.mboxdir = NULL;
+   import.tot_msgs = 0;
+   import.table_id = 0;
+   import.folder = NULL;
+   import.delay = 0;
+   import.after = 0;
+   import.before = 0;
 
    data.import = &import;
 
+   net.socket = -1;
+   net.timeout = 30;
+
+   data.net = &net;
+
    inithash(data.mydomains);
    initrules(data.archiving_rules);
    initrules(data.retention_rules);
@@ -118,13 +144,17 @@ int main(int argc, char **argv){
             {"batch-limit",  required_argument,  0,  'b' },
             {"timeout",      required_argument,  0,  't' },
             {"start-position",required_argument,  0,  's' },
+            {"table-id",     required_argument,  0,  'T' },
+            {"delay",        required_argument,  0,  'Z' },
             {"quiet",        no_argument,        0,  'q' },
-            {"recursive",    required_argument,  0,  'R' },
+            {"recursive",    no_argument,        0,  'R' },
             {"remove-after-import",no_argument,  0,  'r' },
             {"failed-folder",  required_argument,  0,  'j' },
             {"move-folder",  required_argument,  0,  'g' },
+            {"after",        required_argument,  0,  'A' },
+            {"before",       required_argument,  0,  'B' },
             {"only-download",no_argument,        0,  'o' },
-            {"gui-import",   no_argument,        0,  'G' },
+            {"read-from-export",no_argument,     0,  'y' },
             {"dry-run",      no_argument,        0,  'D' },
             {"help",         no_argument,        0,  'h' },
             {0,0,0,0}
@@ -132,9 +162,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:GDRroqh?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:A:B:yDRroqh?", long_options, &option_index);
 #else
-      c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:GDRroqh?");
+      int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:A:B:yDRroqh?");
 #endif
 
       if(c == -1) break;
@@ -146,7 +176,7 @@ int main(int argc, char **argv){
                     break;
 
          case 'e' :
-                    emlfile = optarg;
+                    snprintf(data.import->filename, SMALLBUFSIZE-1, "%s", optarg);
                     break;
 
          case 'd' :
@@ -163,40 +193,42 @@ int main(int argc, char **argv){
                     break;
 
          case 'M' :
-                    mboxdir = optarg;
+                    data.import->mboxdir = optarg;
                     break;
 
          case 'i' :
                     imapserver = optarg;
+                    data.import->server = optarg;
                     break;
 
          case 'K' :
                     pop3server = optarg;
-                    if(port == 143) port = 110;
+                    data.import->server = optarg;
+                    if(data.import->port == 143) data.import->port = 110;
                     break;
 
          case 'u' :
-                    username = optarg;
+                    data.import->username = optarg;
                     break;
 
          case 'p' :
-                    password = optarg;
+                    data.import->password = optarg;
                     break;
 
          case 'P' :
-                    port = atoi(optarg);
+                    data.import->port = atoi(optarg);
                     break;
 
          case 'x' :
-                    skiplist = optarg;
+                    data.import->skiplist = optarg;
                     break;
 
          case 'F' :
-                    folder = optarg;
+                    data.import->folder = optarg;
                     break;
 
          case 'f' :
-                    folder_imap = optarg;
+                    data.import->folder_imap = optarg;
                     break;
 
          case 'R' :
@@ -217,7 +249,7 @@ int main(int argc, char **argv){
 
          case 'o' :
                     data.import->download_only = 1;
-                    dryrun = 1;
+                    data.import->dryrun = 1;
                     break;
 
          case 'b' :
@@ -238,21 +270,46 @@ int main(int argc, char **argv){
                     break;
 
          case 'a' :
-                    data.import->extra_recipient = optarg;
+                    snprintf(puf, sizeof(puf)-1, "%s ", optarg);
+                    data.import->extra_recipient = puf;
                     break;
 
-         case 'G' :
+         case 'T' :
-                    import_from_gui = 1;
+                    if(atoi(optarg) < 1){
+                       printf("invalid import table id: %s\n", optarg);
+                       return -1;
+                    }
+
+                    data.import->table_id = atoi(optarg);
+                    break;
+
+         case 'Z' :
+                    if(atoi(optarg) < 1){
+                       printf("invalid delay value: %s\n", optarg);
+                       return -1;
+                    }
+
+                    data.import->delay = atoi(optarg);
+                    break;
+
+         case 'y' :
+                    read_from_pilerexport = 1;
                     break;
 
          case 'D' :
-                    dryrun = 1;
+                    data.import->dryrun = 1;
                     break;
 
          case 'q' :
                     data.quiet = 1;
                     break;
 
+         case 'A' : data.import->after = atol(optarg);
+                    break;
+
+         case 'B' : data.import->before = atol(optarg);
+                    break;
+
          case 'h' :
          case '?' :
                     usage();
@@ -265,8 +322,7 @@ int main(int argc, char **argv){
    }
 
 
-
+   if(!mbox[0] && !data.import->mboxdir && !data.import->filename[0] && !directory && !imapserver && !pop3server && !read_from_pilerexport) usage();
-   if(!mbox[0] && !mboxdir && !emlfile && !directory && !imapserver && !pop3server && import_from_gui == 0) usage();
 
    if(data.import->failed_folder && !can_i_write_directory(data.import->failed_folder)){
       printf("cannot write failed directory '%s'\n", data.import->failed_folder);
@@ -277,7 +333,9 @@ int main(int argc, char **argv){
 
    cfg = read_config(configfile);
 
-   if((data.recursive_folder_names == 1 || folder) && cfg.enable_folders == 0){
+   memset(cfg.security_header, 0, MAXVAL);
+
+   if((data.recursive_folder_names == 1 || data.import->folder) && cfg.enable_folders == 0){
       printf("please set enable_folders=1 in piler.conf to use the folder options\n");
       return ERR;
    }
@@ -285,6 +343,9 @@ int main(int argc, char **argv){
    /* make sure we don't discard messages without a valid Message-Id when importing manually */
    cfg.archive_emails_not_having_message_id = 1;
 
+   /* The mmap_dedup_test feature is expected to work with the piler daemon only */
+   cfg.mmap_dedup_test = 0;
+
    if(read_key(&cfg)){
       printf("%s\n", ERR_READING_KEY);
       return ERR;
@@ -295,6 +356,7 @@ int main(int argc, char **argv){
 
    if(open_database(&sdata, &cfg) == ERR) return 0;
 
+   if(cfg.rtindex && open_sphx(&sdata, &cfg) == ERR) return 0;
 
    setlocale(LC_CTYPE, cfg.locale);
 
@@ -304,39 +366,39 @@ int main(int argc, char **argv){
    memcached_init(&(data.memc), cfg.memcached_servers, 11211);
 #endif
 
-   if(folder){
+   if(data.import->folder){
-      data.folder = get_folder_id(&sdata, &data, folder, 0);
+      data.folder = get_folder_id(&sdata, data.import->folder, 0);
 
       if(data.folder == ERR_FOLDER){
-         data.folder = add_new_folder(&sdata, &data, folder, 0);
+         data.folder = add_new_folder(&sdata, data.import->folder, 0);
       }
 
       if(data.folder == ERR_FOLDER){
-         printf("error: cannot get/add folder '%s'\n", folder);
+         printf("ERROR: cannot get/add folder '%s'\n", data.import->folder);
          close_database(&sdata);
          return 0;
       }
 
    }
 
-   load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
+   load_rules(&sdata, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
-   load_rules(&sdata, &data, data.retention_rules, SQL_RETENTION_RULE_TABLE);
+   load_rules(&sdata, data.retention_rules, SQL_RETENTION_RULE_TABLE);
-   load_rules(&sdata, &data, data.folder_rules, SQL_FOLDER_RULE_TABLE);
+   load_rules(&sdata, data.folder_rules, SQL_FOLDER_RULE_TABLE);
 
    load_mydomains(&sdata, &data, &cfg);
 
-   if(emlfile) rc = import_message(emlfile, &sdata, &data, &cfg);
+   if(data.import->filename[0] != '\0') import_message(&sdata, &data, &counters, &cfg);
 
    if(mbox[0]){
       for(i=0; i<n_mbox; i++){
-         rc = import_from_mailbox(mbox[i], &sdata, &data, &cfg);
+         import_from_mailbox(mbox[i], &sdata, &data, &counters, &cfg);
       }
    }
-   if(mboxdir) rc = import_mbox_from_dir(mboxdir, &sdata, &data, &tot_msgs, &cfg);
+   if(data.import->mboxdir) import_mbox_from_dir(data.import->mboxdir, &sdata, &data, &counters, &cfg);
-   if(directory) rc = import_from_maildir(directory, &sdata, &data, &tot_msgs, &cfg);
+   if(directory) import_from_maildir(&sdata, &data, directory, &counters, &cfg);
-   if(imapserver && username && password) rc = import_from_imap_server(imapserver, username, password, port, &sdata, &data, folder_imap, skiplist, dryrun, &cfg);
+   if(imapserver) import_from_imap_server(&sdata, &data, &counters, &cfg);
-   if(pop3server && username && password) rc = import_from_pop3_server(pop3server, username, password, port, &sdata, &data, dryrun, &cfg);
+   if(pop3server) import_from_pop3_server(&sdata, &data, &counters, &cfg);
-   if(import_from_gui == 1) rc = read_gui_import_data(&sdata, &data, folder_imap, skiplist, dryrun, &cfg);
+   if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &counters, &cfg);
 
    clearrules(data.archiving_rules);
    clearrules(data.retention_rules);
@@ -344,11 +406,15 @@ int main(int argc, char **argv){
 
    clearhash(data.mydomains);
 
+   update_counters(&sdata, &data, &counters, &cfg);
+
+   syslog(LOG_PRIORITY, "server=%s, user=%s, directory=%s, imported=%lld, duplicated=%lld, discarded=%lld", data.import->server, data.import->username, directory, counters.c_rcvd, counters.c_duplicate, counters.c_ignore);
+
    close_database(&sdata);
 
+   if(cfg.rtindex) close_sphx(&sdata);
+
    if(data.quiet == 0) printf("\n");
 
-   return rc;
+   return 0;
 }
-
-

src/pilerpurge.c

@@ -1,439 +0,0 @@
-/*
- * pilerpurge.c, SJ
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <dirent.h>
-#include <unistd.h>
-#include <time.h>
-#include <locale.h>
-#include <syslog.h>
-#include <piler.h>
-
-
-extern char *optarg;
-extern int optind;
-
-int dryrun = 0;
-unsigned long purged_size=0;
-
-#define NUMBER_OF_ATTACHMENTS_TO_REMOVE_IN_ONE_ROUND 100
-
-#define SQL_STMT_SELECT_PURGE_FROM_OPTION_TABLE "SELECT `value` FROM `" SQL_OPTION_TABLE "` WHERE `key`='enable_purge'"
-#define SQL_STMT_DELETE_FROM_META_TABLE "UPDATE `" SQL_METADATA_TABLE "` SET `deleted`=1 WHERE `id` IN ("
-#define SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID "UPDATE `" SQL_METADATA_TABLE "` SET `deleted`=1 WHERE `piler_id` IN ('"
-#define SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS "SELECT `piler_id`, `attachment_id`, `i` FROM `" SQL_ATTACHMENTS_VIEW "` WHERE `refcount`=0 AND `piler_id` IN ('"
-#define SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE "DELETE FROM `" SQL_ATTACHMENT_TABLE "` WHERE `id` IN ("
-
-
-int is_purge_allowed(struct session_data *sdata, struct __data *data){
-   int rc=0;
-
-   if(prepare_sql_statement(sdata, &(data->stmt_generic), SQL_STMT_SELECT_PURGE_FROM_OPTION_TABLE) == ERR) return rc;
-
-
-   p_bind_init(data);
-
-   if(p_exec_query(sdata, data->stmt_generic, data) == OK){
-
-      p_bind_init(data);
-
-      data->sql[data->pos] = (char *)&rc; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
-
-      p_store_results(data->stmt_generic, data);
-      p_fetch_results(data->stmt_generic);
-      p_free_results(data->stmt_generic);
-   }
-
-   close_prepared_statement(data->stmt_generic);
-
-   return rc;
-}
-
-
-int remove_message_frame_files(char *s, char *update_meta_sql, struct session_data *sdata, struct __config *cfg){
-   char *p, puf[SMALLBUFSIZE], filename[SMALLBUFSIZE];
-   int n=0, result;
-#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
-   struct stat st;
-#endif
-
-   p = s;
-   do {
-      p = split(p, ' ', puf, sizeof(puf)-1, &result);
-
-      if(strlen(puf) == RND_STR_LEN){
-         snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, puf[8], puf[9], puf[10], puf[RND_STR_LEN-4], puf[RND_STR_LEN-3], puf[RND_STR_LEN-2], puf[RND_STR_LEN-1], puf);
-
-      #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
-         if(stat(filename, &st)){
-            snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, puf[RND_STR_LEN-6], puf[RND_STR_LEN-5], puf[RND_STR_LEN-4], puf[RND_STR_LEN-3], puf[RND_STR_LEN-2], puf[RND_STR_LEN-1], puf);
-         }
-      #endif
-
-         if(dryrun == 1){
-            n++;
-            printf("removing messagefile: %s\n", filename);
-         }
-         else {
-            if(unlink(filename) == 0) n++;
-         }
-
-      }
-   } while(p);
-
-
-   update_meta_sql[strlen(update_meta_sql)-1] = ')';
-
-   if(dryrun == 1){
-      printf("update metadata query: *%s*\n\n", update_meta_sql);
-   } else {
-      p_query(sdata, update_meta_sql);
-   }
-
-
-   return n;
-}
-
-
-int remove_attachments(char *in, struct session_data *sdata, struct __data *data, struct __config *cfg){
-   char filename[SMALLBUFSIZE];
-   char *a, buf[NUMBER_OF_ATTACHMENTS_TO_REMOVE_IN_ONE_ROUND*(RND_STR_LEN+1)+10], update_meta_sql[strlen(SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID)+NUMBER_OF_ATTACHMENTS_TO_REMOVE_IN_ONE_ROUND*(RND_STR_LEN+3)+10], delete_attachment_stmt[MAXBUFSIZE];
-   char piler_id[SMALLBUFSIZE], i[BUFLEN];
-   int n=0, m=0, len, attachment_id=0, piler_id_len;
-   unsigned int blen=0, ulen=0, dlen=0;
-#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
-   struct stat st;
-#endif
-
-   if(strlen(in) < 10) return 0;
-
-   len = strlen(SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS) + strlen(in) + 2;
-
-   a = malloc(len);
-   if(!a) return 0;
-
-   memset(a, 0, len);
-
-   in[strlen(in)-2] = ')';
-   in[strlen(in)-1] = '\0';
-   snprintf(a, len-1, "%s%s", SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS, in);
-
-   if(prepare_sql_statement(sdata, &(data->stmt_select_non_referenced_attachments), a) == ERR){ free(a); return n; }
-
-   if(dryrun == 1) printf("attachment select sql: *%s*\n\n", a);
-
-   memset(buf, 0, sizeof(buf));
-   memset(update_meta_sql, 0, sizeof(update_meta_sql));
-   memset(delete_attachment_stmt, 0, sizeof(delete_attachment_stmt));
-
-   snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
-   ulen = strlen(SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
-
-   snprintf(delete_attachment_stmt, sizeof(delete_attachment_stmt)-1, "%s", SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
-   dlen = strlen(SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
-
-   p_bind_init(data);
-   if(p_exec_query(sdata, data->stmt_select_non_referenced_attachments, data) == ERR) goto ENDE;
-
-
-   p_bind_init(data);
-
-   data->sql[data->pos] = &piler_id[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(piler_id)-2; data->pos++;
-   data->sql[data->pos] = (char *)&attachment_id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
-   data->sql[data->pos] = &i[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(i)-2; data->pos++;
-
-   p_store_results(data->stmt_select_non_referenced_attachments, data);
-
-   while(p_fetch_results(data->stmt_select_non_referenced_attachments) == OK){
-
-      piler_id_len = strlen(piler_id);
-
-      if(piler_id_len != RND_STR_LEN || attachment_id <= 0){
-         printf("invalid piler_id: '%s.a%d'\n", piler_id, attachment_id);
-         continue;
-      }
-
-      snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, piler_id[8], piler_id[9], piler_id[10], piler_id[RND_STR_LEN-4], piler_id[RND_STR_LEN-3], piler_id[RND_STR_LEN-2], piler_id[RND_STR_LEN-1], piler_id, attachment_id);
-   #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
-      if(stat(filename, &st)){
-         snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, piler_id[RND_STR_LEN-6], piler_id[RND_STR_LEN-5], piler_id[RND_STR_LEN-4], piler_id[RND_STR_LEN-3], piler_id[RND_STR_LEN-2], piler_id[RND_STR_LEN-1], piler_id, attachment_id);
-      }
-   #endif
-
-      if(dryrun == 1){
-         printf("removing attachment: *%s*\n", filename);
-      } else {
-         unlink(filename);
-      }
-
-
-      if(strlen(i) > 0){
-         if(dlen > sizeof(delete_attachment_stmt) - 200){
-
-            delete_attachment_stmt[dlen-1] = ')';
-            if(dryrun == 1){
-               printf("delete sql: *%s*\n", delete_attachment_stmt);
-            } else {
-               p_query(sdata, delete_attachment_stmt);
-            }
-
-            memset(delete_attachment_stmt, 0, sizeof(delete_attachment_stmt));
-            snprintf(delete_attachment_stmt, sizeof(delete_attachment_stmt)-1, "%s", SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
-            dlen = strlen(SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
-         }
-
-         memcpy(&delete_attachment_stmt[dlen], i, strlen(i)); dlen += strlen(i);
-         memcpy(&delete_attachment_stmt[dlen], ",", 1); dlen++;
-      }
-
-
-      if(attachment_id == 1){
-
-         m++;
-
-         if(m >= NUMBER_OF_ATTACHMENTS_TO_REMOVE_IN_ONE_ROUND){
-            if(ulen > strlen(SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID)+10){
-               update_meta_sql[ulen-2] = ')';
-               update_meta_sql[ulen-1] = '\0';
-            }
-
-            n += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
-
-            memset(buf, 0, sizeof(buf));
-            memset(update_meta_sql, 0, sizeof(update_meta_sql));
-
-            snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
-
-            ulen = strlen(SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
-            blen = 0;
-
-            m = 0;
-         }
-
-         memcpy(&buf[blen], piler_id, piler_id_len); blen += piler_id_len;
-         memcpy(&buf[blen], " ", 1); blen++;
-
-         memcpy(&update_meta_sql[ulen], piler_id, piler_id_len); ulen += piler_id_len;
-         memcpy(&update_meta_sql[ulen], "','", 3); ulen += 3;
-      }
-
-   }
-
-   p_free_results(data->stmt_select_non_referenced_attachments);
-
-   if(ulen > strlen(SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID)+10){
-
-      update_meta_sql[ulen-2] = ')';
-      update_meta_sql[ulen-1] = '\0';
-
-      n += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
-   }
-
-   if(dlen > strlen(SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE)){
-      delete_attachment_stmt[dlen-1] = ')';
-      if(dryrun == 1){
-         printf("delete sql: *%s*\n", delete_attachment_stmt);
-      } else {
-         p_query(sdata, delete_attachment_stmt);
-      }
-
-   }
-
-ENDE:
-   free(a);
-
-   close_prepared_statement(data->stmt_select_non_referenced_attachments);
-
-   return n;
-}
-
-
-int purge_messages_round1(struct session_data *sdata, struct __data *data, char *attachment_condition, struct __config *cfg){
-   int purged=0, size;
-   unsigned int blen=0, ulen=0;
-   char id[BUFLEN], s[SMALLBUFSIZE], buf[MAXBUFSIZE], update_meta_sql[MAXBUFSIZE];
-
-   memset(buf, 0, sizeof(buf));
-   memset(update_meta_sql, 0, sizeof(update_meta_sql));
-
-   snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE);
-   ulen = strlen(SQL_STMT_DELETE_FROM_META_TABLE);
-
-   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `size` FROM `%s` WHERE `deleted`=0 AND `retained` < %ld AND %s AND id NOT IN (SELECT id FROM `%s` WHERE `to` IN (SELECT email FROM `%s`)) AND id NOT IN (SELECT id FROM `%s` WHERE `from` IN (SELECT email FROM `%s`))", SQL_METADATA_TABLE, sdata->now, attachment_condition, SQL_RECIPIENT_TABLE, SQL_LEGAL_HOLD_TABLE, SQL_METADATA_TABLE, SQL_LEGAL_HOLD_TABLE);
-
-   if(dryrun == 1) printf("purge sql: *%s*\n", s);
-
-   if(prepare_sql_statement(sdata, &(data->stmt_select_from_meta_table), s) == ERR) return purged;
-
-   p_bind_init(data);
-
-   if(p_exec_query(sdata, data->stmt_select_from_meta_table, data) == OK){
-
-      p_bind_init(data);
-
-      data->sql[data->pos] = &id[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(id)-2; data->pos++;
-      data->sql[data->pos] = &s[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s)-2; data->pos++;
-      data->sql[data->pos] = (char *)&size; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
-
-      p_store_results(data->stmt_select_from_meta_table, data);
-
-      while(p_fetch_results(data->stmt_select_from_meta_table) == OK){
-
-         memcpy(&update_meta_sql[ulen], id, strlen(id)); ulen += strlen(id);
-         memcpy(&update_meta_sql[ulen], ",", 1); ulen++;
-
-         purged_size += size;
-
-         if(blen >= sizeof(buf)-RND_STR_LEN-2-1){
-
-            purged += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
-
-            memset(buf, 0, sizeof(buf));
-            memset(update_meta_sql, 0, sizeof(update_meta_sql));
-
-            snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE);
-
-            blen = 0;
-            ulen = strlen(SQL_STMT_DELETE_FROM_META_TABLE);
-         }
-
-         memcpy(&buf[blen], s, strlen(s)); blen += strlen(s);
-         memcpy(&buf[blen], " ", 1); blen++;
-
-      }
-
-      p_free_results(data->stmt_select_from_meta_table);
-
-      if(strlen(buf) > 5 && strlen(update_meta_sql) > strlen(SQL_STMT_DELETE_FROM_META_TABLE)+5){
-         purged += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
-      }
-
-   }
-
-   close_prepared_statement(data->stmt_select_from_meta_table);
-
-   return purged;
-}
-
-
-int purge_messages_with_attachments(struct session_data *sdata, struct __data *data, struct __config *cfg){
-   int purged=0, size;
-   unsigned int idlist_len=0;
-   char s[SMALLBUFSIZE], idlist[MAXBUFSIZE];
-
-   memset(idlist, 0, sizeof(idlist));
-
-   snprintf(s, sizeof(s)-1, "SELECT `piler_id`, `size` FROM `%s` WHERE `deleted`=0 AND `retained` < %ld AND attachments > 0", SQL_METADATA_TABLE, sdata->now);
-
-   if(dryrun == 1) printf("purge sql: *%s*\n", s);
-
-   if(prepare_sql_statement(sdata, &(data->stmt_select_from_meta_table), s) == ERR) return purged;
-
-   p_bind_init(data);
-   if(p_exec_query(sdata, data->stmt_select_from_meta_table, data) == OK){
-
-      p_bind_init(data);
-
-      data->sql[data->pos] = &s[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s)-2; data->pos++;
-      data->sql[data->pos] = (char *)&size; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
-
-      p_store_results(data->stmt_select_from_meta_table, data);
-
-      while(p_fetch_results(data->stmt_select_from_meta_table) == OK){
-         memcpy(&idlist[idlist_len], s, strlen(s)); idlist_len += strlen(s);
-         memcpy(&idlist[idlist_len], "','", 3); idlist_len += 3;
-
-         purged_size += size;
-
-         if(idlist_len >= sizeof(idlist)-2*RND_STR_LEN){
-            purged += remove_attachments(idlist, sdata, data, cfg);
-
-            memset(idlist, 0, sizeof(idlist));
-            idlist_len = 0;
-         }
-      }
-
-      p_free_results(data->stmt_select_from_meta_table);
-
-      if(idlist_len > 5){
-         purged += remove_attachments(idlist, sdata, data, cfg);
-      }
-
-   }
-
-   close_prepared_statement(data->stmt_select_from_meta_table);
-
-   return purged;
-}
-
-
-int main(int argc, char **argv){
-   int i, purged=0;
-   char *configfile=CONFIG_FILE, buf[SMALLBUFSIZE];
-   struct session_data sdata;
-   struct __data data;
-   struct __config cfg;
-
-
-   while((i = getopt(argc, argv, "c:dh?")) > 0){
-       switch(i){
-
-         case 'c' :
-                    configfile = optarg;
-                    break;
-
-         case 'd' :
-                    dryrun = 1;
-                    break;
-
-         case 'h' :
-         case '?' :
-         default  : 
-                    break;
-       }
-   }
-
-
-
-   (void) openlog("pilerpurge", LOG_PID, LOG_MAIL);
-
-   cfg = read_config(configfile);
-
-
-   if(open_database(&sdata, &cfg) == ERR) return 0;
-
-
-   setlocale(LC_CTYPE, cfg.locale);
-
-   init_session_data(&sdata, &cfg);
-
-   i = is_purge_allowed(&sdata, &data);
-   if(i == 1){
-      purged += purge_messages_round1(&sdata, &data, "attachments=0", &cfg);
-      purged += purge_messages_with_attachments(&sdata, &data, &cfg);
-      purged += purge_messages_round1(&sdata, &data, "attachments > 0", &cfg);
-
-      syslog(LOG_INFO, "purged %d messages, %ld bytes", purged, purged_size);
-   }
-   else printf("purge is not allowed by configuration, enable_purge=%d\n", i);
-
-   
-   if(purged_size > 100){
-      snprintf(buf, sizeof(buf)-1, "UPDATE `%s` SET size = size - %ld", SQL_COUNTER_TABLE, purged_size);
-
-      if(dryrun == 0) p_query(&sdata, buf);
-   }
-
-
-   close_database(&sdata);
-
-   return 0;
-}
-
-

src/pop3.c

@@ -23,71 +23,26 @@
 #include <piler.h>
 
 
-void update_import_job_stat(struct session_data *sdata, struct __data *data);
+int connect_to_pop3_server(struct data *data){
-
-
-int is_last_complete_pop3_packet(char *s, int len){
-
-   if(*(s+len-5) == '\r' && *(s+len-4) == '\n' && *(s+len-3) == '.' && *(s+len-2) == '\r' && *(s+len-1) == '\n'){
-      return 1;
-   }
-
-   return 0;
-}
-
-
-int connect_to_pop3_server(int sd, char *username, char *password, struct __data *data, int use_ssl){
-   int n;
    char buf[MAXBUFSIZE];
-   X509* server_cert;
-   char *str;
 
-
+   if(data->net->use_ssl == 1){
-   if(use_ssl == 1){
+      init_ssl_to_server(data);
-
-      SSL_library_init();
-      SSL_load_error_strings();
-
-      data->ctx = SSL_CTX_new(TLSv1_client_method());
-      CHK_NULL(data->ctx, "internal SSL error");
-
-      data->ssl = SSL_new(data->ctx);
-      CHK_NULL(data->ssl, "internal ssl error");
-
-      SSL_set_fd(data->ssl, sd);
-      n = SSL_connect(data->ssl);
-      CHK_SSL(n, "internal ssl error");
-
-      printf("Cipher: %s\n", SSL_get_cipher(data->ssl));
-
-      server_cert = SSL_get_peer_certificate(data->ssl);
-      CHK_NULL(server_cert, "server cert error");
-
-      str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0);
-      CHK_NULL(str, "error in server cert");
-      OPENSSL_free(str);
-
-      str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0);
-      CHK_NULL(str, "error in server cert");
-      OPENSSL_free(str);
-
-      X509_free(server_cert);
    }
 
-
+   recvtimeoutssl(data->net, buf, sizeof(buf));
-   recvtimeoutssl(sd, buf, sizeof(buf), data->import->timeout, use_ssl, data->ssl);
 
 
-   snprintf(buf, sizeof(buf)-1, "USER %s\r\n", username);
+   snprintf(buf, sizeof(buf)-1, "USER %s\r\n", data->import->username);
 
-   write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+   write1(data->net, buf, strlen(buf));
-   recvtimeoutssl(sd, buf, sizeof(buf), data->import->timeout, use_ssl, data->ssl);
+   recvtimeoutssl(data->net, buf, sizeof(buf));
 
 
-   snprintf(buf, sizeof(buf)-1, "PASS %s\r\n", password);
+   snprintf(buf, sizeof(buf)-1, "PASS %s\r\n", data->import->password);
 
-   write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+   write1(data->net, buf, strlen(buf));
-   recvtimeoutssl(sd, buf, sizeof(buf), data->import->timeout, use_ssl, data->ssl);
+   recvtimeoutssl(data->net, buf, sizeof(buf));
 
    if(strncmp(buf, "+OK", 3) == 0) return OK;
 
@@ -97,123 +52,154 @@ int connect_to_pop3_server(int sd, char *username, char *password, struct __data
 }
 
 
-int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg){
+void get_number_of_total_messages(struct data *data){
-   int i=0, rc=ERR, n, pos, readlen, fd, lastpos, nreads;
+   char buf[MAXBUFSIZE];
-   char *p, buf[MAXBUFSIZE], filename[SMALLBUFSIZE];
-   char aggrbuf[3*MAXBUFSIZE];
 
-   data->import->processed_messages = 0;
    data->import->total_messages = 0;
 
    snprintf(buf, sizeof(buf)-1, "STAT\r\n");
-   write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+   write1(data->net, buf, strlen(buf));
 
-   recvtimeoutssl(sd, buf, sizeof(buf), data->import->timeout, use_ssl, data->ssl);
+   recvtimeoutssl(data->net, buf, sizeof(buf));
 
    if(strncmp(buf, "+OK ", 4) == 0){
-      p = strchr(&buf[4], ' ');
+      char *p = strchr(&buf[4], ' ');
       if(p){
          *p = '\0';
          data->import->total_messages = atoi(&buf[4]);
       }
    }
-   else return ERR;
+   else {
+      printf("ERROR: '%s'", buf);
+   }
+}
 
 
+int pop3_download_email(struct data *data, int i){
+   char *p, buf[MAXBUFSIZE], savedbuf[MAXBUFSIZE], copybuf[2*MAXBUFSIZE];
+
+   data->import->processed_messages++;
+
+   snprintf(data->import->filename, SMALLBUFSIZE-1, "pop3-tmp-%d-%d.txt", getpid(), i);
+   unlink(data->import->filename);
+
+   int fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
+   if(fd == -1){
+      printf("cannot open: %s\n", data->import->filename);
+      return ERR;
+   }
+
+   memset(savedbuf, 0, sizeof(savedbuf));
+
+   snprintf(buf, sizeof(buf)-1, "RETR %d\r\n", i);
+   write1(data->net, buf, strlen(buf));
+
+   int nlines = 0;
+   int endofmessage = 0;
+   int savedlen = 0;
+   int n = 0;
+
+   while((n = recvtimeoutssl(data->net, buf, sizeof(buf))) > 0){
+      int remaininglen = n;
+
+      if(savedlen){
+         memset(copybuf, 0, sizeof(copybuf));
+         memcpy(copybuf, savedbuf, savedlen);
+         memcpy(&copybuf[savedlen], buf, n);
+
+         remaininglen += savedlen;
+
+         savedlen = 0;
+         memset(savedbuf, 0, sizeof(savedbuf));
+
+         p = &copybuf[0];
+      } else {
+         p = &buf[0];
+      }
+
+      int puflen=0;
+      int rc=OK;
+      int nullbyte=0;
+
+      do {
+         char puf[MAXBUFSIZE];
+
+         puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf)-1, &rc, &nullbyte);
+         remaininglen -= puflen;
+         nlines++;
+
+         if(nlines == 1){
+            if(strncmp(puf, "+OK", 3)){
+               printf("error: %s", puf);
+               return ERR;
+            }
+         } else {
+            if(puf[puflen-1] == '\n'){
+               if(puflen == 3 && puf[0] == '.' && puf[1] == '\r' && puf[2] == '\n'){
+                  endofmessage = 1;
+                  break;
+               }
+
+               int dotstuff = 0;
+               if(puf[0] == '.' && puf[1] != '\r' && puf[1] != '\n') dotstuff = 1;
+
+               if(write(fd, &puf[dotstuff], puflen-dotstuff) == -1) printf("ERROR: writing to fd\n");
+
+            } else if(puflen > 0) {
+               savedlen = puflen;
+               snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
+            }
+         }
+
+         p += puflen;
+
+      } while(puflen > 0);
+
+      if(endofmessage){
+         break;
+      }
+   }
+
+   close(fd);
+
+   return OK;
+}
+
+
+void pop3_delete_message(struct data *data, int i){
+   char buf[SMALLBUFSIZE];
+
+   snprintf(buf, sizeof(buf)-1, "DELE %d\r\n", i);
+   write1(data->net, buf, strlen(buf));
+   recvtimeoutssl(data->net, buf, sizeof(buf));
+}
+
+
+void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   char buf[MAXBUFSIZE];
+
+   data->import->processed_messages = 0;
+
+   get_number_of_total_messages(data);
+
    if(data->quiet == 0) printf("found %d messages\n", data->import->total_messages);
 
-   if(data->import->total_messages <= 0) return OK;
+   if(data->import->total_messages <= 0) return;
 
-   for(i=data->import->start_position; i<=data->import->total_messages; i++){
+   for(int i=data->import->start_position; i<=data->import->total_messages; i++){
-      data->import->processed_messages++;
+      if(pop3_download_email(data, i) == OK){
-      if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/data->import->total_messages); fflush(stdout); }
+         if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/data->import->total_messages); fflush(stdout); }
 
+         if(data->import->dryrun == 0){
+            int rc = import_message(sdata, data, counters, cfg);
 
-      snprintf(buf, sizeof(buf)-1, "RETR %d\r\n", i);
+            if(data->import->remove_after_import == 1 && rc == OK){
-
+               pop3_delete_message(data, i);
-      snprintf(filename, sizeof(filename)-1, "pop3-tmp-%d-%d.txt", getpid(), i);
-      unlink(filename);
-
-      fd = open(filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
-      if(fd == -1){
-         printf("cannot open: %s\n", filename);
-         return rc;
-      }
-
-      write1(sd, buf, strlen(buf), use_ssl, data->ssl);
-
-      readlen = 0;
-      pos = 0;
-      nreads = 0;
-
-      memset(aggrbuf, 0, sizeof(aggrbuf));
-      lastpos = 0;
-
-
-      while((n = recvtimeoutssl(sd, buf, sizeof(buf), data->import->timeout, use_ssl, data->ssl)) > 0){
-         nreads++;
-         readlen += n;
-
-         if(nreads == 1){
-
-            if(strncmp(buf, "+OK", 3) == 0){
-               p = strchr(&buf[3], '\n');
-               if(p){
-                  *p = '\0';
-                  pos = strlen(buf)+1;
-                  *p = '\n';
-               }
-            }
-            else { printf("error: %s", buf); return ERR; }
-
-         }
-
-         if(lastpos + 1 + n < sizeof(aggrbuf)){
-
-            if(nreads == 1){
-               memcpy(aggrbuf+lastpos, buf+pos, n-pos);
-               lastpos += n-pos;
-            }
-            else {
-               memcpy(aggrbuf+lastpos, buf, n);
-               lastpos += n;
             }
          }
-         else {
-            write(fd, aggrbuf, sizeof(buf));
-
-            memmove(aggrbuf, aggrbuf+sizeof(buf), lastpos-sizeof(buf));
-            lastpos -= sizeof(buf);
-
-            memcpy(aggrbuf+lastpos, buf, n);
-            lastpos += n;
-         }
-
-         if(is_last_complete_pop3_packet(aggrbuf, lastpos) == 1){
-            write(fd, aggrbuf, lastpos-3);
-            break;
-         }
-
       }
 
-      close(fd);
+      if(data->import->download_only == 0) unlink(data->import->filename);
-
-      if(dryrun == 0) rc = import_message(filename, sdata, data, cfg);
-      else rc = OK;
-
-      if(dryrun == 0 && rc == OK && data->import->remove_after_import == 1){
-         snprintf(buf, sizeof(buf)-1, "DELE %d\r\n", i);
-         write1(sd, buf, strlen(buf), use_ssl, data->ssl);
-         recvtimeoutssl(sd, buf, sizeof(buf), data->import->timeout, use_ssl, data->ssl);
-      }
-
-      if(i % 100 == 0){
-         time(&(data->import->updated));
-         update_import_job_stat(sdata, data);
-      }
-
-      if(data->import->download_only == 0) unlink(filename);
-
 
       /* whether to quit after processing a batch of messages */
 
@@ -224,16 +210,7 @@ int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data,
 
 
    snprintf(buf, sizeof(buf)-1, "QUIT\r\n");
-   write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+   write1(data->net, buf, strlen(buf));
 
    if(data->quiet == 0) printf("\n");
-
-   time(&(data->import->finished));
-   data->import->status = 2;
-   update_import_job_stat(sdata, data);
-
-
-   return OK;
 }
-
-

src/reindex.c

@@ -42,45 +42,42 @@ void p_clean_exit(char *msg, int rc){
 }
 
 
-uint64 get_max_meta_id(struct session_data *sdata, struct __data *data){
+uint64 get_max_meta_id(struct session_data *sdata){
    char s[SMALLBUFSIZE];
    uint64 id=0;
+   struct sql sql;
 
    snprintf(s, sizeof(s)-1, "SELECT MAX(`id`) FROM %s", SQL_METADATA_TABLE);
 
 
-   if(prepare_sql_statement(sdata, &(data->stmt_generic), s) == ERR) return id;
+   if(prepare_sql_statement(sdata, &sql, s) == ERR) return id;
 
 
-   p_bind_init(data);
+   p_bind_init(&sql);
 
-   if(p_exec_query(sdata, data->stmt_generic, data) == OK){
+   if(p_exec_stmt(sdata, &sql) == OK){
 
-      p_bind_init(data);
+      p_bind_init(&sql);
 
-      data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+      sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
 
-      p_store_results(data->stmt_generic, data);
+      p_store_results(&sql);
-      p_fetch_results(data->stmt_generic);
+      p_fetch_results(&sql);
-      p_free_results(data->stmt_generic);
+      p_free_results(&sql);
    }
 
-   close_prepared_statement(data->stmt_generic);
+   close_prepared_statement(&sql);
 
 
    return id;
 }
 
 
-uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct __data *data, uint64 from_id, uint64 to_id, struct __config *cfg){
+uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *data, uint64 from_id, uint64 to_id, struct config *cfg){
-   FILE *f;
-   char filename[SMALLBUFSIZE];
    char s[SMALLBUFSIZE];
-   int rc=0;
+   uint64 stored_id=0, reindexed=0;
-   uint64 stored_id=0, reindexed=0, delta;
    struct parser_state state;
-
+   struct sql sql;
-   delta = to_id - from_id;
 
    if(cfg->enable_folders == 1)
       snprintf(s, sizeof(s)-1, "SELECT m.`id`, `piler_id`, `arrived`, `sent`, f.folder_id FROM %s m, %s f WHERE m.id=f.id AND (m.id BETWEEN %llu AND %llu) AND `deleted`=0", SQL_METADATA_TABLE, SQL_FOLDER_MESSAGE_TABLE, from_id, to_id);
@@ -88,69 +85,78 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct __data *
       snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `arrived`, `sent` FROM %s WHERE (id BETWEEN %llu AND %llu) AND `deleted`=0", SQL_METADATA_TABLE, from_id, to_id);
 
 
-   if(prepare_sql_statement(sdata, &(data->stmt_generic), s) == ERR) return reindexed;
+   if(prepare_sql_statement(sdata, &sql, s) == ERR) return reindexed;
 
-   p_bind_init(data);
+   p_bind_init(&sql);
 
-   if(p_exec_query(sdata, data->stmt_generic, data) == OK){
+   if(p_exec_stmt(sdata, &sql) == OK){
 
-      p_bind_init(data);
+      p_bind_init(&sql);
 
-      data->sql[data->pos] = (char *)&stored_id; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+      sql.sql[sql.pos] = (char *)&stored_id; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-      data->sql[data->pos] = sdata->ttmpfile; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = RND_STR_LEN+2; data->pos++;
+      sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN+2; sql.pos++;
-      data->sql[data->pos] = (char *)&(sdata->now); data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(unsigned long); data->pos++;
+      sql.sql[sql.pos] = (char *)&(sdata->now); sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(unsigned long); sql.pos++;
-      data->sql[data->pos] = (char *)&(sdata->sent); data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(unsigned long); data->pos++;
+      sql.sql[sql.pos] = (char *)&(sdata->sent); sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(unsigned long); sql.pos++;
       if(cfg->enable_folders == 1){
-         data->sql[data->pos] = (char *)&(data->folder); data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(unsigned long); data->pos++;
+         sql.sql[sql.pos] = (char *)&(data->folder); sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(unsigned long); sql.pos++;
       }
 
-      p_store_results(data->stmt_generic, data);
+      p_store_results(&sql);
 
-      while(p_fetch_results(data->stmt_generic) == OK){
+      while(p_fetch_results(&sql) == OK){
 
-         if(stored_id > 0){
+         char filename[SMALLBUFSIZE];
+         snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id);
 
-            snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id);
+         FILE *f = fopen(filename, "w");
-
+         if(f){
-            f = fopen(filename, "w");
+            int rc = retrieve_email_from_archive(sdata, f, cfg);
-            if(f){
+            fclose(f);
-               rc = retrieve_email_from_archive(sdata, data, f, cfg);
-               fclose(f);
-
-               if(rc){
-                  printf("cannot retrieve: %s\n", filename);
-                  unlink(filename);
-                  continue;
-               }
-
-               snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
-
-               state = parse_message(sdata, 0, data, cfg);
-               post_parse(sdata, &state, cfg);
-
-               rc = store_index_data(sdata, &state, data, stored_id, cfg);
-
-               if(rc == OK) reindexed++;
-               else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename);
 
+            if(rc){
+               printf("cannot retrieve: %s\n", filename);
                unlink(filename);
-
+               continue;
-               if(progressbar){
+            }
-                  printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta));
+
-                  fflush(stdout);
+            snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
-               }
+
-
+            struct stat st;
+            sdata->tot_len = stat(filename, &st) == 0 ? st.st_size : 0;
+
+            sdata->internal_sender = sdata->internal_recipient = sdata->external_recipient = sdata->direction = 0;
+            memset(sdata->attachments, 0, SMALLBUFSIZE);
+
+            state = parse_message(sdata, 1, data, cfg);
+            post_parse(sdata, &state, cfg);
+
+            rc = store_index_data(sdata, &state, data, stored_id, cfg);
+
+            unlink(sdata->tmpframe);
+            remove_stripped_attachments(&state);
+
+            if(rc == OK) reindexed++;
+            else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename);
+
+            unlink(filename);
+
+            if(progressbar){
+               uint64 delta = to_id - from_id + 1;
+
+               printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta));
+               fflush(stdout);
             }
-            else printf("cannot open: %s\n", filename);
 
          }
+         else printf("cannot open: %s\n", filename);
 
       }
 
-      p_free_results(data->stmt_generic);
+
+      p_free_results(&sql);
    }
 
-   close_prepared_statement(data->stmt_generic);
+   close_prepared_statement(&sql);
 
 
    if(progressbar) printf("\n");
@@ -160,16 +166,16 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct __data *
 
 
 int main(int argc, char **argv){
-   int c, all=0;
+   int all=0;
    uint64 from_id=0, to_id=0, n=0;
    char *configfile=CONFIG_FILE, *folder=NULL;
    struct session_data sdata;
-   struct __data data;
+   struct data data;
-   struct __config cfg;
+   struct config cfg;
 
 
    while(1){
-      c = getopt(argc, argv, "c:f:t:F:pahv?");
+      int c = getopt(argc, argv, "c:f:t:F:pahv?");
 
       if(c == -1) break;
 
@@ -209,7 +215,7 @@ int main(int argc, char **argv){
    }
 
 
-   if(all == 0 && (from_id <= 0 || to_id <= 0) ) usage();
+   if(all == 0 && (from_id == 0 || to_id == 0) ) usage();
 
    if(!can_i_write_directory(NULL)) __fatal("cannot write current directory!");
 
@@ -239,10 +245,15 @@ int main(int argc, char **argv){
       p_clean_exit("cannot connect to mysql server", 1);
    }
 
-   load_rules(&sdata, &data, data.folder_rules, SQL_FOLDER_RULE_TABLE);
+   if(cfg.rtindex && open_sphx(&sdata, &cfg) == ERR){
+      p_clean_exit("cannot connect to 127.0.0.1:9306", 1);
+   }
+
+
+   load_rules(&sdata, data.folder_rules, SQL_FOLDER_RULE_TABLE);
 
    if(folder){
-      data.folder = get_folder_id(&sdata, &data, folder, 0);
+      data.folder = get_folder_id(&sdata, folder, 0);
       if(data.folder == 0){
          printf("error: could not get folder id for '%s'\n", folder);
          return 0;
@@ -254,7 +265,7 @@ int main(int argc, char **argv){
 
    if(all == 1){
       from_id = 1;
-      to_id = get_max_meta_id(&sdata, &data);
+      to_id = get_max_meta_id(&sdata);
    }
 
    n = retrieve_email_by_metadata_id(&sdata, &data, from_id, to_id, &cfg);
@@ -266,8 +277,7 @@ int main(int argc, char **argv){
    clearhash(data.mydomains);
 
    close_database(&sdata);
+   if(cfg.rtindex) close_sphx(&sdata);
 
    return 0;
 }
-
-

src/rules.c

@@ -10,75 +10,75 @@
 #include "rules.h"
 
 
-void load_rules(struct session_data *sdata, struct __data *data, struct node *xhash[], char *table){
+void reset_rule_condition(struct rule_cond *rule_cond){
-   char s[SMALLBUFSIZE];
+   memset(rule_cond->domain, 0, SMALLBUFSIZE);
-   struct rule_cond rule_cond;
+   memset(rule_cond->from, 0, SMALLBUFSIZE);
+   memset(rule_cond->to, 0, SMALLBUFSIZE);
+   memset(rule_cond->subject, 0, SMALLBUFSIZE);
+   memset(rule_cond->body, 0, SMALLBUFSIZE);
+   memset(rule_cond->_size, 0, SMALLBUFSIZE);
+   memset(rule_cond->attachment_name, 0, SMALLBUFSIZE);
+   memset(rule_cond->attachment_type, 0, SMALLBUFSIZE);
+   memset(rule_cond->_attachment_size, 0, SMALLBUFSIZE);
 
-   memset(rule_cond.domain, 0, SMALLBUFSIZE);
+   rule_cond->size = 0;
-   memset(rule_cond.from, 0, SMALLBUFSIZE);
+   rule_cond->attachment_size = 0;
-   memset(rule_cond.to, 0, SMALLBUFSIZE);
+   rule_cond->spam = 0;
-   memset(rule_cond.subject, 0, SMALLBUFSIZE);
+   rule_cond->days = 0;
-   memset(rule_cond.body, 0, SMALLBUFSIZE);
+   rule_cond->folder_id = 0;
-   memset(rule_cond._size, 0, SMALLBUFSIZE);
-   memset(rule_cond.attachment_name, 0, SMALLBUFSIZE);
-   memset(rule_cond.attachment_type, 0, SMALLBUFSIZE);
-   memset(rule_cond._attachment_size, 0, SMALLBUFSIZE);
-
-   rule_cond.size = rule_cond.attachment_size = rule_cond.spam = rule_cond.days = rule_cond.folder_id = 0;
-
-   snprintf(s, sizeof(s)-1, "SELECT `domain`, `from`, `to`, `subject`, `body`, `_size`, `size`, `attachment_name`, `attachment_type`, `_attachment_size`, `attachment_size`, `spam`, `days`, `folder_id` FROM `%s`", table);
-
-   if(prepare_sql_statement(sdata, &(data->stmt_generic), s) == ERR) return;
-
-
-   p_bind_init(data);
-
-   if(p_exec_query(sdata, data->stmt_generic, data) == OK){
-
-      p_bind_init(data);
-
-      data->sql[data->pos] = &rule_cond.domain[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
-      data->sql[data->pos] = &rule_cond.from[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
-      data->sql[data->pos] = &rule_cond.to[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
-      data->sql[data->pos] = &rule_cond.subject[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
-      data->sql[data->pos] = &rule_cond.body[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
-      data->sql[data->pos] = &rule_cond._size[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
-      data->sql[data->pos] = (char *)&rule_cond.size; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
-      data->sql[data->pos] = &rule_cond.attachment_name[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
-      data->sql[data->pos] = &rule_cond.attachment_type[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
-      data->sql[data->pos] = &rule_cond._attachment_size[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
-      data->sql[data->pos] = (char *)&rule_cond.attachment_size; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
-      data->sql[data->pos] = (char *)&rule_cond.spam; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
-      data->sql[data->pos] = (char *)&rule_cond.days; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
-      data->sql[data->pos] = (char *)&rule_cond.folder_id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
-
-      p_store_results(data->stmt_generic, data);
-
-      while(p_fetch_results(data->stmt_generic) == OK){
-         append_rule(xhash, &rule_cond, data);
-
-         memset(rule_cond.domain, 0, SMALLBUFSIZE);
-         memset(rule_cond.from, 0, SMALLBUFSIZE);
-         memset(rule_cond.to, 0, SMALLBUFSIZE);
-         memset(rule_cond.subject, 0, SMALLBUFSIZE);
-         memset(rule_cond.body, 0, SMALLBUFSIZE);
-         memset(rule_cond._size, 0, SMALLBUFSIZE);
-         memset(rule_cond.attachment_name, 0, SMALLBUFSIZE);
-         memset(rule_cond.attachment_type, 0, SMALLBUFSIZE);
-         memset(rule_cond._attachment_size, 0, SMALLBUFSIZE);
-
-         rule_cond.size = rule_cond.attachment_size = rule_cond.spam = rule_cond.days = rule_cond.folder_id = 0;
-      }
-
-      p_free_results(data->stmt_generic);
-
-   }
-
-   close_prepared_statement(data->stmt_generic);
 }
 
 
-int append_rule(struct node *xhash[], struct rule_cond *rule_cond, struct __data *data){
+void load_rules(struct session_data *sdata, struct node *xhash[], char *table){
+   char s[SMALLBUFSIZE];
+   struct rule_cond rule_cond;
+   struct sql sql;
+
+   reset_rule_condition(&rule_cond);
+
+   snprintf(s, sizeof(s)-1, "SELECT `domain`, `from`, `to`, `subject`, `body`, `_size`, `size`, `attachment_name`, `attachment_type`, `_attachment_size`, `attachment_size`, `spam`, `days`, `folder_id` FROM `%s`", table);
+
+   if(prepare_sql_statement(sdata, &sql, s) == ERR) return;
+
+
+   p_bind_init(&sql);
+
+   if(p_exec_stmt(sdata, &sql) == OK){
+
+      p_bind_init(&sql);
+
+      sql.sql[sql.pos] = &rule_cond.domain[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond.domain)-2; sql.pos++;
+      sql.sql[sql.pos] = &rule_cond.from[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond.from)-2; sql.pos++;
+      sql.sql[sql.pos] = &rule_cond.to[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond.to)-2; sql.pos++;
+      sql.sql[sql.pos] = &rule_cond.subject[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond.subject)-2; sql.pos++;
+      sql.sql[sql.pos] = &rule_cond.body[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond.body)-2; sql.pos++;
+      sql.sql[sql.pos] = &rule_cond._size[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond._size)-2; sql.pos++;
+      sql.sql[sql.pos] = (char *)&rule_cond.size; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(rule_cond.size); sql.pos++;
+      sql.sql[sql.pos] = &rule_cond.attachment_name[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond.attachment_name)-2; sql.pos++;
+      sql.sql[sql.pos] = &rule_cond.attachment_type[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond.attachment_type)-2; sql.pos++;
+      sql.sql[sql.pos] = &rule_cond._attachment_size[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond._attachment_size)-2; sql.pos++;
+      sql.sql[sql.pos] = (char *)&rule_cond.attachment_size; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(rule_cond.attachment_size); sql.pos++;
+      sql.sql[sql.pos] = (char *)&rule_cond.spam; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(rule_cond.spam); sql.pos++;
+      sql.sql[sql.pos] = (char *)&rule_cond.days; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(rule_cond.days); sql.pos++;
+      sql.sql[sql.pos] = (char *)&rule_cond.folder_id; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(rule_cond.folder_id); sql.pos++;
+
+
+      p_store_results(&sql);
+
+      while(p_fetch_results(&sql) == OK){
+         append_rule(xhash, &rule_cond);
+         reset_rule_condition(&rule_cond);
+      }
+
+      p_free_results(&sql);
+
+   }
+
+   close_prepared_statement(&sql);
+}
+
+
+int append_rule(struct node *xhash[], struct rule_cond *rule_cond){
    struct node *q, *Q=NULL, *node;
    struct rule *rule;
    int rc=0;
@@ -88,7 +88,7 @@ int append_rule(struct node *xhash[], struct rule_cond *rule_cond, struct __data
    memset(node, 0, sizeof(struct node));
    node->r = NULL;
 
-   rule = create_rule_item(rule_cond, data);
+   rule = create_rule_item(rule_cond);
 
    if(rule == NULL){
       free(node);
@@ -116,9 +116,8 @@ int append_rule(struct node *xhash[], struct rule_cond *rule_cond, struct __data
 }
 
 
-struct rule *create_rule_item(struct rule_cond *rule_cond, struct __data *data){
+struct rule *create_rule_item(struct rule_cond *rule_cond){
    struct rule *h=NULL;
-   char empty = '\0';
    int len;
 
    if(rule_cond == NULL) return NULL;
@@ -130,7 +129,7 @@ struct rule *create_rule_item(struct rule_cond *rule_cond, struct __data *data){
    h->compiled = 1;
 
    h->domain = NULL;
-   h->domainlen = data->length[0];
+   h->domainlen = strlen(rule_cond->domain);
 
    if(h->domainlen > 2){
       h->domain = malloc(h->domainlen+2);
@@ -144,16 +143,16 @@ struct rule *create_rule_item(struct rule_cond *rule_cond, struct __data *data){
    h->emptyfrom = h->emptyto = h->emptysubject = h->emptyaname = h->emptyatype = 0;
 
 
-   if(rule_cond->from == NULL || strlen(rule_cond->from) < 1){ rule_cond->from[0] = empty; h->emptyfrom = 1; }
+   if(rule_cond->from[0] == 0){ h->emptyfrom = 1; }
    if(regcomp(&(h->from), rule_cond->from, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->to == NULL || strlen(rule_cond->to) < 1){ rule_cond->to[0] = empty; h->emptyto = 1; }
+   if(rule_cond->to[0] == 0){ h->emptyto = 1; }
    if(regcomp(&(h->to), rule_cond->to, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->subject == NULL || strlen(rule_cond->subject) < 1){ rule_cond->subject[0] = empty; h->emptysubject = 1; }
+   if(rule_cond->subject[0] == 0){ h->emptysubject = 1; }
    if(regcomp(&(h->subject), rule_cond->subject, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->body == NULL || strlen(rule_cond->body) < 1){ rule_cond->body[0] = empty; h->emptybody = 1; }
+   if(rule_cond->body[0] == 0){ h->emptybody = 1; }
    if(regcomp(&(h->body), rule_cond->body, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
    h->spam = rule_cond->spam;
@@ -161,20 +160,16 @@ struct rule *create_rule_item(struct rule_cond *rule_cond, struct __data *data){
    h->folder_id = rule_cond->folder_id;
 
    h->size = rule_cond->size;
-
-   if(rule_cond->_size == NULL) rule_cond->_size[0] = empty;
    snprintf(h->_size, 3, "%s", rule_cond->_size);
 
-   if(rule_cond->attachment_name == NULL || strlen(rule_cond->attachment_name) < 1){ rule_cond->attachment_name[0] = empty; h->emptyaname = 1; }
+   if(rule_cond->attachment_name[0] == 0){ h->emptyaname = 1; }
    if(regcomp(&(h->attachment_name), rule_cond->attachment_name, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->attachment_type == NULL || strlen(rule_cond->attachment_type) < 1){ rule_cond->attachment_type[0] = empty; h->emptyatype = 1; }
+   if(rule_cond->attachment_type[0] == 0){ h->emptyatype = 1; }
    if(regcomp(&(h->attachment_type), rule_cond->attachment_type, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
 
    h->attachment_size = rule_cond->attachment_size;
-
-   if(rule_cond->_attachment_size == NULL) rule_cond->_attachment_size[0] = empty;
    snprintf(h->_attachment_size, 3, "%s", rule_cond->_attachment_size);
 
    len = strlen(rule_cond->domain)+8 + strlen(rule_cond->from)+6 + strlen(rule_cond->to)+4 + strlen(rule_cond->subject)+9 + strlen(rule_cond->body)+6 + strlen(rule_cond->_size)+6 + strlen(rule_cond->attachment_name)+10 + strlen(rule_cond->attachment_type)+10 + strlen(rule_cond->_attachment_size)+10 + 9 + 15 + 15;
@@ -182,7 +177,10 @@ struct rule *create_rule_item(struct rule_cond *rule_cond, struct __data *data){
 
 
 
-   if(h->rulestr) snprintf(h->rulestr, len-1, "domain=%s,from=%s,to=%s,subject=%s,body=%s,size%s%d,att.name=%s,att.type=%s,att.size%s%d,spam=%d", rule_cond->domain, rule_cond->from, rule_cond->to, rule_cond->subject, rule_cond->body, rule_cond->_size, rule_cond->size, rule_cond->attachment_name, rule_cond->attachment_type, rule_cond->_attachment_size, rule_cond->attachment_size, rule_cond->spam);
+   if(h->rulestr){
+      snprintf(h->rulestr, len-1, "domain=%s,from=%s,to=%s,subject=%s,body=%s,size%s%d,att.name=%s,att.type=%s,att.size%s%d,spam=%d", rule_cond->domain, rule_cond->from, rule_cond->to, rule_cond->subject, rule_cond->body, rule_cond->_size, rule_cond->size, rule_cond->attachment_name, rule_cond->attachment_type, rule_cond->_attachment_size, rule_cond->attachment_size, rule_cond->spam);
+      syslog(LOG_INFO, "adding rule: %s", h->rulestr);
+   }
    else h->compiled = 0;
 
    h->r = NULL;
@@ -191,11 +189,44 @@ struct rule *create_rule_item(struct rule_cond *rule_cond, struct __data *data){
 }
 
 
-char *check_againt_ruleset(struct node *xhash[], struct parser_state *state, int size, int spam){
+int count_match(struct rule *p, struct parser_state *state, int size, int spam){
-   size_t nmatch=0;
+   int ismatch=0;
+
+   ismatch += check_spam_rule(spam, p->spam);
+   ismatch += check_size_rule(size, p->size, p->_size);
+   ismatch += check_attachment_rule(state, p);
+
+   if(p->compiled == 1){
+      size_t nmatch=0;
+
+      if(p->emptyfrom == 1){
+         ismatch += RULE_UNDEF;
+      }
+      else if(regexec(&(p->from), state->b_from, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+      if(p->emptyto == 1){
+         ismatch += RULE_UNDEF;
+      }
+      else if(regexec(&(p->to), state->b_to, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+      if(p->emptysubject == 1){
+         ismatch += RULE_UNDEF;
+      }
+      else if(regexec(&(p->subject), state->b_subject, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+      if(p->emptybody == 1){
+         ismatch += RULE_UNDEF;
+      }
+      else if(regexec(&(p->body), state->b_body, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+   }
+
+   return ismatch;
+}
+
+
+char *check_against_ruleset(struct node *xhash[], struct parser_state *state, int size, int spam){
    struct rule *p;
    struct node *q;
-   int ismatch;
 
    q = xhash[0];
 
@@ -204,40 +235,8 @@ char *check_againt_ruleset(struct node *xhash[], struct parser_state *state, int
       if(q->str){
          p = q->str;
 
-         if(p){
+         if(count_match(p, state, size, spam) > 0){
-            ismatch = 0;
+            return p->rulestr;
-
-            ismatch += check_spam_rule(spam, p->spam);
-            ismatch += check_size_rule(size, p->size, p->_size);
-            ismatch += check_attachment_rule(state, p);
-
-            if(p->compiled == 1){
-               if(p->emptyfrom == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->from), state->b_from, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptyto == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->to), state->b_to, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptysubject == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->subject), state->b_subject, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptybody == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->body), state->b_body, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-            }
-
-            if(ismatch > 0){
-               return p->rulestr;
-            }
-
          }
       }
 
@@ -248,11 +247,9 @@ char *check_againt_ruleset(struct node *xhash[], struct parser_state *state, int
 }
 
 
-time_t query_retain_period(struct __data *data, struct parser_state *state, int size, int spam, struct __config *cfg){
+time_t query_retain_period(struct data *data, struct parser_state *state, int size, int spam, struct config *cfg){
-   size_t nmatch=0;
    struct rule *p;
    struct node *q;
-   int ismatch;
 
    q = data->retention_rules[0];
 
@@ -261,47 +258,15 @@ time_t query_retain_period(struct __data *data, struct parser_state *state, int
       if(q->str){
          p = q->str;
 
-         ismatch = 0;
-
          if(p->domainlen > 2){
             if(strcasestr(state->b_to_domain, p->domain) || strcasestr(state->b_from_domain, p->domain)){
                state->retention = p->days;
-               return (time_t)p->days * (time_t)86400;
+               return (time_t)(state->retention) * (time_t)86400;
             }
          }
-         else {
+         else if(count_match(p, state, size, spam) > 0){
-
+            state->retention = p->days;
-            ismatch += check_spam_rule(spam, p->spam);
+            return (time_t)(state->retention) * (time_t)86400;
-            ismatch += check_size_rule(size, p->size, p->_size);
-            ismatch += check_attachment_rule(state, p);
-
-            if(p->compiled == 1){
-               if(p->emptyfrom == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->from), state->b_from, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptyto == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->to), state->b_to, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptysubject == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->subject), state->b_subject, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptybody == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->body), state->b_body, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-            }
-
-            if(ismatch > 0){
-               state->retention = p->days;
-               return (time_t)p->days * (time_t)86400;
-            }
          }
 
       }
@@ -312,15 +277,13 @@ time_t query_retain_period(struct __data *data, struct parser_state *state, int
 
    state->retention = cfg->default_retention_days;
 
-   return (time_t)cfg->default_retention_days * (time_t)86400;
+   return (time_t)(state->retention) * (time_t)86400;
 }
 
 
-int get_folder_id_by_rule(struct __data *data, struct parser_state *state, int size, int spam, struct __config *cfg){
+int get_folder_id_by_rule(struct data *data, struct parser_state *state, int size, int spam, struct config *cfg){
-   size_t nmatch=0;
    struct rule *p;
    struct node *q;
-   int ismatch;
 
    if(cfg->enable_folders == 0) return 0;
 
@@ -331,45 +294,13 @@ int get_folder_id_by_rule(struct __data *data, struct parser_state *state, int s
       if(q->str){
          p = q->str;
 
-         ismatch = 0;
-
          if(p->domainlen > 2){
             if(strcasestr(state->b_to_domain, p->domain) || strcasestr(state->b_from_domain, p->domain)){
                return p->folder_id;
             }
          }
-         else {
+         else if(count_match(p, state, size, spam) > 0){
-
+            return p->folder_id;
-            ismatch += check_spam_rule(spam, p->spam);
-            ismatch += check_size_rule(size, p->size, p->_size);
-            ismatch += check_attachment_rule(state, p);
-
-            if(p->compiled == 1){
-               if(p->emptyfrom == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->from), state->b_from, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptyto == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->to), state->b_to, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptysubject == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->subject), state->b_subject, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-               if(p->emptybody == 1){
-                  ismatch += RULE_UNDEF;
-               }
-               else if(regexec(&(p->body), state->b_body, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-            }
-
-            if(ismatch > 0){
-               return p->folder_id;
-            }
          }
 
       }
@@ -403,14 +334,16 @@ int check_spam_rule(int is_spam, int spam){
 int check_attachment_rule(struct parser_state *state, struct rule *rule){
    int i;
    size_t nmatch=0;
-   int ismatch = 0;
 
-   if(state->n_attachments == 0) return RULE_UNDEF;
+   // If no attachment rule, then return RULE_UNDEF
+   if(rule->emptyaname == 1 && rule->emptyatype == 1 && rule->attachment_size == 0) return RULE_UNDEF;
+
+   // If we have attachments, but no attachment rules, then return RULE_NO_MATCH
+   if(state->n_attachments == 0 && (rule->emptyaname == 0 || rule->emptyatype == 0 || rule->attachment_size)) return RULE_NO_MATCH;
 
-   if(rule->emptyaname == 1 && rule->emptyatype == 1) return RULE_UNDEF;
 
    for(i=1; i<=state->n_attachments; i++){
-      ismatch = 0;
+      int ismatch = 0;
 
       if(rule->emptyaname == 0){
          if(regexec(&(rule->attachment_name), state->attachments[i].filename, nmatch, NULL, 0) == 0)
@@ -441,36 +374,33 @@ void initrules(struct node *xhash[]){
 
 
 void clearrules(struct node *xhash[]){
-   struct node *p, *q;
+   struct node *q;
    struct rule *rule;
 
    q = xhash[0];
 
    while(q != NULL){
-      p = q;
+      struct node *p = q;
       q = q->r;
 
-      if(p){
+      if(p->str){
-         if(p->str){
+         rule = (struct rule*)p->str;
-            rule = (struct rule*)p->str;
 
-            regfree(&(rule->from));
+         regfree(&(rule->from));
-            regfree(&(rule->to));
+         regfree(&(rule->to));
-            regfree(&(rule->subject));
+         regfree(&(rule->subject));
-            regfree(&(rule->body));
+         regfree(&(rule->body));
-            regfree(&(rule->attachment_name));
+         regfree(&(rule->attachment_name));
-            regfree(&(rule->attachment_type));
+         regfree(&(rule->attachment_type));
 
-            free(rule->rulestr);
+         free(rule->rulestr);
 
-            if(rule->domain) free(rule->domain);
+         if(rule->domain) free(rule->domain);
 
-            free(rule);
+         free(rule);
-         }
-         free(p);
       }
+      free(p);
    }
 
    xhash[0] = NULL;
 }
-

src/rules.h

@@ -7,12 +7,12 @@
 
 #include "defs.h"
 
-void load_rules(struct session_data *sdata, struct __data *data, struct node *xhash[], char *table);
+void load_rules(struct session_data *sdata, struct node *xhash[], char *table);
-int append_rule(struct node *xhash[], struct rule_cond *rule_cond, struct __data *data);
+int append_rule(struct node *xhash[], struct rule_cond *rule_cond);
-struct rule *create_rule_item(struct rule_cond *rule_cond, struct __data *data);
+struct rule *create_rule_item(struct rule_cond *rule_cond);
-char *check_againt_ruleset(struct node *xhash[], struct parser_state *state, int size, int spam);
+char *check_against_ruleset(struct node *xhash[], struct parser_state *state, int size, int spam);
-time_t query_retain_period(struct __data *data, struct parser_state *state, int size, int spam, struct __config *cfg);
+time_t query_retain_period(struct data *data, struct parser_state *state, int size, int spam, struct config *cfg);
-int get_folder_id_by_rule(struct __data *data, struct parser_state *state, int size, int spam, struct __config *cfg);
+int get_folder_id_by_rule(struct data *data, struct parser_state *state, int size, int spam, struct config *cfg);
 int check_size_rule(int message_size, int size, char *_size);
 int check_spam_rule(int is_spam, int spam);
 int check_attachment_rule(struct parser_state *state, struct rule *rule);
@@ -21,4 +21,3 @@ void initrules(struct node *xhash[]);
 void clearrules(struct node *xhash[]);
 
 #endif /* _RULES_H */
-

src/screen.c

@@ -0,0 +1,254 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <sys/types.h>
+#include <netdb.h>
+#include <piler.h>
+
+
+void init_smtp_acl(struct smtp_acl *smtp_acl[]){
+   smtp_acl[0] = NULL;
+}
+
+
+void clear_smtp_acl(struct smtp_acl *smtp_acl[]){
+   struct smtp_acl *q;
+
+   q = smtp_acl[0];
+
+   while(q){
+      struct smtp_acl *p = q;
+      q = q->r;
+
+      free(p);
+   }
+
+   smtp_acl[0] = NULL;
+}
+
+
+int add_smtp_acl(struct smtp_acl *smtp_acl[], char *network_str, struct smtp_acl *acl){
+   struct smtp_acl *q, *p=NULL, *node;
+
+   if((node = malloc(sizeof(struct smtp_acl))) == NULL) return 0;
+
+   memset(node, 0, sizeof(struct smtp_acl));
+
+   node->low = acl->low;
+   node->high = acl->high;
+   node->prefix = acl->prefix;
+   node->rejected = acl->rejected;
+   snprintf(node->network_str, sizeof(node->network_str)-1, "%s", network_str);
+
+   node->r = NULL;
+
+   q = smtp_acl[0];
+
+   while(q){
+      p = q;
+      q = q->r;
+   }
+
+   if(!p){
+      smtp_acl[0] = node;
+   } else {
+      p->r = node;
+   }
+
+   return 1;
+}
+
+
+int is_valid_line(char *line){
+   // Currently we support ipv4 stuff only, ie. valid characters are: 0-9./
+   // and a line should look like "1.2.3.4/24 permit" or similar (without quotes)
+
+   if(!strchr(line, '.') || !strchr(line, '/') || (!strchr(line, ' ') && !strchr(line, '\t')) ){
+      return 0;
+   }
+
+   if(!strstr(line, "permit") && !strstr(line, "reject")){
+      return 0;
+   }
+
+   // ascii values:
+   //   46: .
+   //   47: /
+   //   48-57: 0-9
+   //   97-122: a-z
+   //
+   for(; *line; line++){
+      if(isalnum(*line) == 0 && isblank(*line) == 0 && *line != 46 && *line != 47) return 0;
+   }
+
+   return 1;
+}
+
+
+int a_to_hl(char *ipstr, in_addr_t *addr){
+  struct in_addr in;
+
+  if(inet_aton(ipstr, &in) == 1){
+     *addr = ntohl(in.s_addr);
+     return 1;
+  }
+
+  syslog(LOG_PRIORITY, "invalid ipv4 address string: *%s*", ipstr);
+
+  return 0;
+}
+
+
+in_addr_t netmask(int prefix){
+   if(prefix == 0)
+      return( ~((in_addr_t) -1) );
+   else
+      return ~((1 << (32 - prefix)) - 1);
+}
+
+
+in_addr_t network(in_addr_t addr, int prefix){
+   return addr & netmask(prefix);
+}
+
+
+in_addr_t broadcast(in_addr_t addr, int prefix){
+   return addr | ~netmask(prefix);
+}
+
+
+int str_to_net_range(char *network_addr_prefix, struct smtp_acl *smtp_acl){
+   in_addr_t net = 0;
+   int prefix = 0;
+
+   smtp_acl->low = 0;
+   smtp_acl->high = 0;
+
+   // By default we permit unless you specify "reject" (without quotes)
+   // To be on the safer side we permit even if you misspell the word "reject"
+
+   smtp_acl->rejected = 0;
+
+   if(strcasestr(network_addr_prefix, "reject")){
+      smtp_acl->rejected = 1;
+   }
+
+   char *p = strchr(network_addr_prefix, '/');
+   if(!p) return 0;
+
+   if(strlen(network_addr_prefix) > sizeof(smtp_acl->network_str)){
+      syslog(LOG_PRIORITY, "line *%s* is longer than %ld bytes, discarded", network_addr_prefix, sizeof(smtp_acl->network_str));
+      return 0;
+   }
+
+   char buf[SMALLBUFSIZE];
+   snprintf(buf, sizeof(buf)-1, "%s", network_addr_prefix);
+
+   *p = '\0';
+   p++;
+
+   // Even though the remaining part of the acl line continues with some number
+   // then whitespace characters and the permit/reject action atoi() can still
+   // figure out the numeric part properly, that's why I'm lazy here.
+   prefix = atoi(p);
+
+   // The prefix string (p) must start with a digit and the prefix integer must be in 0..32 range
+   if(*p < 48 || *p > 57 || prefix < 0 || prefix > 32){
+      syslog(LOG_PRIORITY, "error: invalid prefix: %s", p);
+      return 0;
+   }
+
+   if(a_to_hl(network_addr_prefix, &net)){
+      smtp_acl->low = network(net, prefix);
+      smtp_acl->high = broadcast(net, prefix);
+      smtp_acl->prefix = prefix;
+
+      syslog(LOG_PRIORITY, "info: parsed acl *%s*", buf);
+
+      return 1;
+   }
+
+   return 0;
+}
+
+
+void load_smtp_acl(struct smtp_acl *smtp_acl[]){
+   int count=0;
+
+   clear_smtp_acl(smtp_acl);
+   init_smtp_acl(smtp_acl);
+
+   FILE *f = fopen(SMTP_ACL_FILE, "r");
+   if(!f){
+      syslog(LOG_PRIORITY, "info: cannot open %s, piler-smtp accepts smtp connections from everywhere", SMTP_ACL_FILE);
+      return;
+   }
+
+   char line[SMALLBUFSIZE];
+   struct smtp_acl acl;
+
+   while(fgets(line, sizeof(line)-1, f)){
+      // Skip comments
+      if(line[0] == ';' || line[0] == '#') continue;
+
+      trimBuffer(line);
+
+      // Skip empty line
+      if(line[0] == 0) continue;
+
+      char line2[SMALLBUFSIZE];
+      int rc = 0;
+      snprintf(line2, sizeof(line2)-1, "%s", line);
+
+      if(is_valid_line(line) == 1 && str_to_net_range(line, &acl) == 1){
+         add_smtp_acl(smtp_acl, line, &acl);
+         count++;
+         rc = 1;
+      }
+
+      if(!rc) syslog(LOG_PRIORITY, "error: failed to parse line: *%s*", line2);
+   }
+
+   fclose(f);
+
+   // If we have entries on the smtp acl list, then add 127.0.0.1/8
+   // to let the GUI health page connect to the piler-smtp daemon
+   if(count){
+      snprintf(line, sizeof(line)-1, "127.0.0.1/8 permit");
+
+      if(str_to_net_range(line, &acl) == 1){
+         add_smtp_acl(smtp_acl, line, &acl);
+      }
+   }
+}
+
+
+int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr){
+   struct smtp_acl *q=smtp_acl[0];
+   in_addr_t addr = 0;
+
+   // Empty acl, let it pass
+   if(!q) return 0;
+
+   if(a_to_hl(ipaddr, &addr) == 0){
+      syslog(LOG_PRIORITY, "error: invalid smtp client address: *%s*", ipaddr);
+      return 1;
+   }
+
+   while(q){
+      if(addr >= q->low && addr <= q->high){
+         if(q->rejected) syslog(LOG_PRIORITY, "denied connection from %s, acl: %s/%d reject", ipaddr, q->network_str, q->prefix);
+         return q->rejected;
+      }
+
+      q = q->r;
+   }
+
+   syslog(LOG_PRIORITY, "denied connection from %s by implicit default deny", ipaddr);
+
+   return 1;
+}

src/screen.h

@@ -0,0 +1,16 @@
+/*
+ * screen.h, SJ
+ */
+
+#ifndef _SCREEN_H
+ #define _SCREEN_H
+
+#include "defs.h"
+
+void init_smtp_acl(struct smtp_acl *smtp_acl[]);
+void clear_smtp_acl(struct smtp_acl *smtp_acl[]);
+int add_smtp_acl(struct smtp_acl *smtp_acl[], char *network_str, struct smtp_acl *acl);
+void load_smtp_acl(struct smtp_acl *smtp_acl[]);
+int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr);
+
+#endif /* _SCREEN_H */

src/session.c

@@ -2,34 +2,45 @@
 #include <stdlib.h>
 #include <string.h>
 #include <netdb.h>
+#include <signal.h>
 #include <piler.h>
 
 
 int get_session_slot(struct smtp_session **sessions, int max_connections);
-void init_smtp_session(struct smtp_session *session, int slot, int sd, struct __config *cfg);
+void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg);
 
+#define GOT_CRLF_DOT_CRLF(p) *p == '\r' && *(p+1) == '\n' && *(p+2) == '.' && *(p+3) == '\r' && *(p+4) == '\n' ? 1 : 0
 
-#ifdef HAVE_LIBWRAP
+uint64 get_sessions_total_memory(struct smtp_session **sessions, int max_connections){
-int is_blocked_by_tcp_wrappers(int sd){
+   uint64 total = 0;
-   struct request_info req;
 
-   request_init(&req, RQ_DAEMON, "piler", RQ_FILE, sd, 0);
+   for(int i=0; i<max_connections; i++){
-
+      if(sessions[i]) total += sessions[i]->bufsize;
-   fromhost(&req);
-
-   if(!hosts_access(&req)){
-      send(sd, SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY, strlen(SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY), 0);
-      syslog(LOG_PRIORITY, "denied connection from %s by tcp_wrappers", eval_client(&req));
-      return 1;
    }
 
-   return 0;
+   return total;
 }
-#endif
 
 
-int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct __config *cfg){
+/*
-   char smtp_banner[SMALLBUFSIZE];
+ * If the sending party sets the email size when it sends the "mail from"
+ * part in the smtp transaction, eg. MAIL FROM:<jajaja@akakak.lo> size=509603
+ * then piler-smtp could know the email size in advance and could do
+ * a better estimate on the allowed number of smtp sessions.
+ */
+
+uint64 get_sessions_total_expected_mail_size(struct smtp_session **sessions, int max_connections){
+   uint64 total = 0;
+
+   for(int i=0; i<max_connections; i++){
+      if(sessions[i]) total += sessions[i]->mail_size;
+   }
+
+   return total;
+}
+
+
+int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg){
    int slot;
 
    /*
@@ -37,25 +48,42 @@ int start_new_session(struct smtp_session **sessions, int socket, int *num_conne
     */
 
    if(*num_connections >= cfg->max_connections){
-      syslog(LOG_PRIORITY, "too many connections (%d), cannot accept socket %d", *num_connections, socket);
+      syslog(LOG_PRIORITY, "ERROR: too many connections (%d), cannot accept socket %d", *num_connections, socket);
       send(socket, SMTP_RESP_421_ERR_ALL_PORTS_ARE_BUSY, strlen(SMTP_RESP_421_ERR_ALL_PORTS_ARE_BUSY), 0);
       close(socket);
       return -1;
    }
 
-#ifdef HAVE_LIBWRAP
+   // Check remote client against the allowed network ranges
-   if(is_blocked_by_tcp_wrappers(socket) == 1){
+   if(cfg->smtp_access_list && is_blocked_by_pilerscreen(smtp_acl, client_addr)){
+      send(socket, SMTP_RESP_550_ERR, strlen(SMTP_RESP_550_ERR), 0);
       close(socket);
       return -1;
    }
-#endif
+
+   /*
+    * We are under the max_smtp_memory threshold
+    */
+
+   uint64 expected_total_mail_size = get_sessions_total_expected_mail_size(sessions, cfg->max_connections);
+   uint64 total_memory = get_sessions_total_memory(sessions, cfg->max_connections);
+
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "DEBUG: total smtp memory allocated: %llu, expected total size: %llu", total_memory, expected_total_mail_size);
+
+   if(total_memory > cfg->max_smtp_memory || expected_total_mail_size > cfg->max_smtp_memory){
+      syslog(LOG_PRIORITY, "ERROR: too much memory consumption: %llu", total_memory);
+      send(socket, SMTP_RESP_451_ERR_TOO_MANY_REQUESTS, strlen(SMTP_RESP_451_ERR_TOO_MANY_REQUESTS), 0);
+      return -1;
+   }
 
    slot = get_session_slot(sessions, cfg->max_connections);
 
    if(slot >= 0 && sessions[slot] == NULL){
       sessions[slot] = malloc(sizeof(struct smtp_session));
       if(sessions[slot]){
-         init_smtp_session(sessions[slot], slot, socket, cfg);
+         init_smtp_session(sessions[slot], slot, socket, client_addr, cfg);
+
+         char smtp_banner[SMALLBUFSIZE];
          snprintf(smtp_banner, sizeof(smtp_banner)-1, SMTP_RESP_220_BANNER, cfg->hostid);
          send(socket, smtp_banner, strlen(smtp_banner), 0);
 
@@ -93,131 +121,217 @@ struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int m
    int i;
 
    for(i=0; i<max_connections; i++){
-      if(sessions[i] && sessions[i]->socket == socket) return sessions[i];
+      if(sessions[i] && sessions[i]->net.socket == socket) return sessions[i];
    }
 
    return NULL;
 }
 
 
-void init_smtp_session(struct smtp_session *session, int slot, int sd, struct __config *cfg){
+void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg){
-   struct sockaddr_in addr;
-   socklen_t addr_size = sizeof(struct sockaddr_in);
-   char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
-
    session->slot = slot;
 
-   session->socket = sd;
-   session->buflen = 0;
    session->protocol_state = SMTP_STATE_INIT;
 
    session->cfg = cfg;
 
-   session->use_ssl = 0;  // use SSL/TLS
+   session->net.socket = sd;
-   session->starttls = 0; // SSL/TLS communication is active (1) or not (0)
+   session->net.use_ssl = 0;  // use SSL/TLS
-   session->ctx = NULL;
+   session->net.starttls = 0; // SSL/TLS communication is active (1) or not (0)
-   session->ssl = NULL;
+   session->net.ctx = NULL;
+   session->net.ssl = NULL;
 
-   memset(session->buf, 0, SMALLBUFSIZE);
+   session->nullbyte = 0;
-   memset(session->remote_host, 0, INET6_ADDRSTRLEN);
 
-   reset_bdat_counters(session);
+   snprintf(session->remote_host, sizeof(session->remote_host)-1, "%s", client_addr);
 
-   time(&(session->lasttime));
+   session->buf = NULL;
+   session->buflen = 0;
+   session->bufsize = 0;
 
-   if(getpeername(sd, (struct sockaddr *)&addr, &addr_size) == 0 &&
+   reset_smtp_session(session);
-      getnameinfo((struct sockaddr *)&addr, addr_size, hbuf, sizeof(hbuf), sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV) == 0){
-         snprintf(session->remote_host, INET6_ADDRSTRLEN-1, "%s", hbuf);
-   }
 }
 
 
 void free_smtp_session(struct smtp_session *session){
-
    if(session){
 
-      if(session->use_ssl == 1){
+      if(session->buf != NULL){
-         SSL_shutdown(session->ssl);
+         free(session->buf);
-         SSL_free(session->ssl);
       }
 
-      if(session->ctx) SSL_CTX_free(session->ctx);
+      if(session->net.use_ssl == 1){
+         SSL_shutdown(session->net.ssl);
+         SSL_free(session->net.ssl);
+      }
+
+      if(session->net.ctx){
+         SSL_CTX_free(session->net.ctx);
+      }
 
       free(session);
    }
 }
 
 
-void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections){
+void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections, char *reason){
-   syslog(LOG_PRIORITY, "disconnected from %s", sessions[slot]->remote_host);
+   if(sessions[slot] == NULL){
+      syslog(LOG_PRIORITY, "session already torn down, slot=%d, reason=%s (%d active connections)", slot, reason, *num_connections);
+      return;
+   }
 
-   close(sessions[slot]->socket);
+   if(*num_connections > 0) (*num_connections)--;
+
+   syslog(LOG_PRIORITY, "disconnected from %s on fd=%d, slot=%d, reason=%s (%d active connections)",
+          sessions[slot]->remote_host, sessions[slot]->net.socket, slot, reason, *num_connections);
+
+   close(sessions[slot]->net.socket);
+
+   if(sessions[slot]->fd != -1){
+      syslog(LOG_PRIORITY, "Removing %s", sessions[slot]->ttmpfile);
+      close(sessions[slot]->fd);
+      unlink(sessions[slot]->ttmpfile);
+      sessions[slot]->fd = -1;
+   }
 
    free_smtp_session(sessions[slot]);
    sessions[slot] = NULL;
-
-   (*num_connections)--;
 }
 
 
-void handle_data(struct smtp_session *session, char *readbuf, int readlen){
+inline int get_last_newline_position(char *buf, int buflen){
-   char *p, puf[MAXBUFSIZE];
+   int i;
-   int result;
 
-   // process BDAT stuff
+   for(i=buflen; i>0; i--){
+      if(*(buf+i) == '\n'){
+         i++;
+         break;
+      }
+   }
+
+   return i;
+}
+
+
+void flush_buffer(struct smtp_session *session){
+   // In the DATA phase skip the 1st character if it's a dot (.)
+   // and there are more characters before the trailing CR-LF
+   //
+   // See https://www.ietf.org/rfc/rfc5321.html#section-4.5.2 for more
+   for(int i=0; i<session->buflen; i++){
+      if(*(session->buf+i) == '\n' && *(session->buf+i+1) == '.' && *(session->buf+i+2) == '.'){
+         int dst = i + 2;
+         int src = dst + 1;
+         int l = session->buflen - src;
+         memmove(session->buf + dst, session->buf + src, l);
+         session->buflen -= 1;
+      }
+   }
+
+   // Exclude the trailing \r\n.\r\n sequence
+
+   session->buflen -= 5;
+
+   if(write(session->fd, session->buf, session->buflen) != session->buflen){
+      session->bad = 1;
+      syslog(LOG_PRIORITY, "ERROR (line: %d) %s: failed to write %d bytes", __LINE__, __func__, session->buflen);
+   }
+
+   session->tot_len = session->buflen;
+}
+
+
+void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg){
+   // Update lasttime if we have something to process
+   time(&(session->lasttime));
 
    if(session->protocol_state == SMTP_STATE_BDAT){
+      process_bdat(session, readbuf, readlen, cfg);
+      return;
+   }
+
+   // realloc memory if the new chunk doesn't fit in
+
+   if(session->buflen + readlen + 10 > session->bufsize){
+      // Handle if the current memory allocation for this email is above the max_message_size threshold
+
+      if(session->buflen > cfg->max_message_size){
+         if(session->too_big == 0) syslog(LOG_PRIORITY, "ERROR: too big email: %d vs %d", session->buflen, cfg->max_message_size);
+         session->bad = 1;
+         session->too_big = 1;
+      }
+
+      if(session->bad == 0){
+         char *q = realloc(session->buf, session->bufsize + SMTPBUFSIZE);
+         if(q){
+            session->buf = q;
+            memset(session->buf+session->bufsize, 0, SMTPBUFSIZE);
+            session->bufsize += SMTPBUFSIZE;
+         } else {
+            syslog(LOG_PRIORITY, "ERROR: realloc %s %s %d", session->ttmpfile, __func__, __LINE__);
+            session->bad = 1;
+         }
+      }
+   }
+
+   // process smtp command
+   if(session->protocol_state != SMTP_STATE_DATA){
+
+      // We got ~2 MB of garbage and no valid smtp command
+      // Terminate the connection
+      if(session->buflen + readlen > SMTPBUFSIZE - 10){
+         session->bad = 1;
+      }
+
+      // We are at the beginning of the smtp transaction
       if(session->bad == 1){
-         // something bad happened in the BDAT processing
+         write1(&(session->net), SMTP_RESP_451_ERR, strlen(SMTP_RESP_451_ERR));
+         syslog(LOG_PRIORITY, "ERROR: sent 451 temp error back to client %s", session->ttmpfile);
          return;
       }
 
-      process_bdat(session, readbuf, readlen);
+      //printf("got %d *%s*\n", readlen, readbuf);
+
+      memcpy(session->buf + session->buflen, readbuf, readlen);
+      session->buflen += readlen;
+
+      int pos = get_last_newline_position(session->buf, session->buflen);
+
+      if(pos < readlen) return; // no complete command
+
+      process_smtp_command(session, cfg);
+
+      memset(session->buf, 0, session->bufsize);
+      session->buflen = 0;
+
+      return;
    }
 
-   // process DATA
+   if(session->bad == 0){
+      memcpy(session->buf + session->buflen, readbuf, readlen);
+      session->buflen += readlen;
 
-   else if(session->protocol_state == SMTP_STATE_DATA){
+      char *p = session->buf + session->buflen - 5;
-      process_data(session, readbuf, readlen);
+      if(session->buflen >= 5 && GOT_CRLF_DOT_CRLF(p)){
-   }
+         flush_buffer(session);
-
+         process_command_period(session);
-   // process other SMTP commands
-
-   else {
-
-      if(session->buflen > 0){
-         snprintf(puf, sizeof(puf)-1, "%s%s", session->buf, readbuf);
-         snprintf(readbuf, BIGBUFSIZE-1, "%s", puf);
-
-         session->buflen = 0;
-         memset(session->buf, 0, SMALLBUFSIZE);
       }
-
+   } else if(strstr(readbuf, "\r\n.\r\n")){
-      p = readbuf;
+      process_command_period(session);
-
-      do {
-         memset(puf, 0, sizeof(puf));
-         p = split(p, '\n', puf, sizeof(puf)-1, &result);
-
-         if(puf[0] == '\0') continue;
-
-         if(result == 1){
-            process_smtp_command(session, puf);
-
-            // if chunking is enabled and we have data after BDAT <len>
-            // then process the rest
-
-            if(session->cfg->enable_chunking == 1 && p && session->protocol_state == SMTP_STATE_BDAT){
-               process_bdat(session, p, strlen(p));
-               break;
-            }
-         }
-         else {
-            snprintf(session->buf, SMALLBUFSIZE-1, "%s", puf);
-            session->buflen = strlen(puf);
-         }
-      } while(p);
-
    }
 
 }
+
+
+void write_envelope_addresses(struct smtp_session *session, struct config *cfg){
+   if(session->fd == -1) return;
+
+   for(int i=0; i<session->num_of_rcpt_to; i++){
+      char *p = strchr(session->rcptto[i], '@');
+      if(p && strncmp(p+1, cfg->hostid, cfg->hostid_len)){
+         char s[SMALLBUFSIZE];
+         snprintf(s, sizeof(s)-1, "X-Piler-Envelope-To: %s\n", session->rcptto[i]);
+         if(write(session->fd, s, strlen(s)) == -1) syslog(LOG_PRIORITY, "ERROR: %s: cannot write envelope to address", session->ttmpfile);
+      }
+   }
+}

src/sig.c

@@ -56,4 +56,3 @@ signal_func *set_signal_handler(int signo, signal_func * func){
 
    return oact.sa_handler;
 }
-

src/smtp.c

@@ -15,53 +15,59 @@
 #include "smtp.h"
 
 
-void process_smtp_command(struct smtp_session *session, char *buf){
+void process_smtp_command(struct smtp_session *session, struct config *cfg){
    char response[SMALLBUFSIZE];
 
-   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "processing command: *%s*", buf);
+   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "got on fd=%d: *%s*", session->net.socket, session->buf);
 
-   if(strncasecmp(buf, SMTP_CMD_HELO, strlen(SMTP_CMD_HELO)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_HELO, strlen(SMTP_CMD_HELO)) == 0){
       process_command_helo(session, response, sizeof(response));
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_EHLO, strlen(SMTP_CMD_EHLO)) == 0 ||
+   if(strncasecmp(session->buf, SMTP_CMD_EHLO, strlen(SMTP_CMD_EHLO)) == 0 ||
-         strncasecmp(buf, LMTP_CMD_LHLO, strlen(LMTP_CMD_LHLO)) == 0){
+         strncasecmp(session->buf, LMTP_CMD_LHLO, strlen(LMTP_CMD_LHLO)) == 0){
-      process_command_ehlo_lhlo(session, response, sizeof(response));
+      process_command_ehlo_lhlo(session, response, sizeof(response), cfg);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_HELP, strlen(SMTP_CMD_HELP)) == 0){
-      process_command_mail_from(session, buf);
+      send_smtp_response(session, SMTP_RESP_221_PILER_SMTP_OK);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){
-      process_command_rcpt_to(session, buf);
+      process_command_mail_from(session);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_DATA, strlen(SMTP_CMD_DATA)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){
-      process_command_data(session);
+      process_command_rcpt_to(session, cfg);
       return;
    }
 
-   if(session->cfg->enable_chunking == 1 && strncasecmp(buf, SMTP_CMD_BDAT, strlen(SMTP_CMD_BDAT)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_DATA, strlen(SMTP_CMD_DATA)) == 0){
-      get_bdat_size_to_read(session, buf);
+      process_command_data(session, cfg);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_QUIT, strlen(SMTP_CMD_QUIT)) == 0){
+   /* Support only BDAT xxxx LAST command */
+   if(session->cfg->enable_chunking == 1 && strncasecmp(session->buf, SMTP_CMD_BDAT, strlen(SMTP_CMD_BDAT)) == 0 && strcasestr(session->buf, "LAST")){
+      get_bdat_size_to_read(session);
+      return;
+   }
+
+   if(strncasecmp(session->buf, SMTP_CMD_QUIT, strlen(SMTP_CMD_QUIT)) == 0){
       process_command_quit(session, response, sizeof(response));
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_RESET, strlen(SMTP_CMD_RESET)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_RESET, strlen(SMTP_CMD_RESET)) == 0){
       process_command_reset(session);
       return;
    }
 
-   if(session->cfg->tls_enable == 1 && strncasecmp(buf, SMTP_CMD_STARTTLS, strlen(SMTP_CMD_STARTTLS)) == 0 && session->use_ssl == 0){
+   if(session->cfg->tls_enable == 1 && strncasecmp(session->buf, SMTP_CMD_STARTTLS, strlen(SMTP_CMD_STARTTLS)) == 0 && session->net.use_ssl == 0){
       process_command_starttls(session);
       return;
    }
@@ -70,54 +76,12 @@ void process_smtp_command(struct smtp_session *session, char *buf){
 }
 
 
-void process_data(struct smtp_session *session, char *readbuf, int readlen){
-   char puf[SMALLBUFSIZE+BIGBUFSIZE];
-   int n, pos, len;
-
-   memset(puf, 0, sizeof(puf));
-
-   if(session->buflen > 0){
-      memcpy(&puf[0], session->buf, session->buflen);
-      memcpy(&puf[session->buflen], readbuf, readlen);
-      len = session->buflen + readlen;
-   }
-   else {
-      memcpy(&puf[0], readbuf, readlen);
-      len = readlen;
-   }
-
-   pos = searchStringInBuffer(&puf[0], len, SMTP_CMD_PERIOD, 5);
-
-   if(pos > 0){
-      write(session->fd, puf, pos);
-      session->tot_len += pos;
-      process_command_period(session);
-   }
-   else {
-      n = search_char_backward(&puf[0], len, '\r');
-
-      if(n == -1 || len - n > 4){
-         write(session->fd, puf, len);
-         session->tot_len += len;
-      }
-      else {
-         write(session->fd, puf, n);
-         session->tot_len += n;
-
-         snprintf(session->buf, SMALLBUFSIZE-1, "%s", &puf[n]);
-         session->buflen = len - n;
-      }
-   }
-}
-
-
 void wait_for_ssl_accept(struct smtp_session *session){
    int rc;
-   char ssl_error[SMALLBUFSIZE];
 
    if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "waiting for ssl handshake");
 
-   rc = SSL_accept(session->ssl);
+   rc = SSL_accept(session->net.ssl);
 
    // Since we use non-blocking IO, SSL_accept() is likely to return with -1
    // "In this case a call to SSL_get_error() with the return value of SSL_accept()
@@ -125,33 +89,35 @@ void wait_for_ssl_accept(struct smtp_session *session){
    //
    // In this case we may proceed.
 
-   if(rc == 1 || SSL_get_error(session->ssl, rc) == SSL_ERROR_WANT_READ){
+   if(rc == 1 || SSL_get_error(session->net.ssl, rc) == SSL_ERROR_WANT_READ){
-      session->use_ssl = 1;
+      session->net.use_ssl = 1;
    }
 
-   if(session->cfg->verbosity >= _LOG_DEBUG || session->use_ssl == 0){
+   if(session->cfg->verbosity >= _LOG_DEBUG || session->net.use_ssl == 0){
+      char ssl_error[SMALLBUFSIZE];
+
       ERR_error_string_n(ERR_get_error(), ssl_error, SMALLBUFSIZE);
       syslog(LOG_PRIORITY, "SSL_accept() result, rc=%d, errorcode: %d, error text: %s",
-             rc, SSL_get_error(session->ssl, rc), ssl_error);
+             rc, SSL_get_error(session->net.ssl, rc), ssl_error);
    }
 }
 
 
 void send_smtp_response(struct smtp_session *session, char *buf){
-   write1(session->socket, buf, strlen(buf), session->use_ssl, session->ssl);
+   write1(&(session->net), buf, strlen(buf));
-   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "sent: %s", buf);
+   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "sent on fd=%d: %s", session->net.socket, buf);
 }
 
 
 void process_command_helo(struct smtp_session *session, char *buf, int buflen){
    if(session->protocol_state == SMTP_STATE_INIT) session->protocol_state = SMTP_STATE_HELO;
 
-   snprintf(buf, buflen-1, "220 %s ESMTP\r\n", session->cfg->hostid);
+   snprintf(buf, buflen-1, "250 %s\r\n", session->cfg->hostid);
    send_smtp_response(session, buf);
 }
 
 
-void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen){
+void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen, struct config *cfg){
    char extensions[SMALLBUFSIZE];
 
    memset(extensions, 0, sizeof(extensions));
@@ -159,34 +125,49 @@ void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int bufl
    if(session->protocol_state == SMTP_STATE_INIT) session->protocol_state = SMTP_STATE_HELO;
 
    // if tls is not started, but it's enabled in the config
-   if(session->use_ssl == 0 && session->cfg->tls_enable == 1) snprintf(extensions, sizeof(extensions)-1, "%s", SMTP_EXTENSION_STARTTLS);
+   if(session->net.use_ssl == 0 && session->cfg->tls_enable == 1) snprintf(extensions, sizeof(extensions)-1, "%s", SMTP_EXTENSION_STARTTLS);
    if(session->cfg->enable_chunking == 1) strncat(extensions, SMTP_EXTENSION_CHUNKING, sizeof(extensions)-strlen(extensions)-2);
 
-   snprintf(buf, buflen-1, SMTP_RESP_250_EXTENSIONS, session->cfg->hostid, extensions);
+   //#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n%s250-SIZE %d\r\n250 8BITMIME\r\n"
+
+   snprintf(buf, buflen-1, SMTP_RESP_250_EXTENSIONS, session->cfg->hostid, cfg->max_message_size, extensions);
 
    send_smtp_response(session, buf);
 }
 
 
 int init_ssl(struct smtp_session *session){
-   session->ctx = SSL_CTX_new(TLSv1_server_method());
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+   session->net.ctx = SSL_CTX_new(TLSv1_server_method());
+#else
+   session->net.ctx = SSL_CTX_new(TLS_server_method());
+#endif
 
-   if(session->ctx == NULL){
+   if(session->net.ctx == NULL){
       syslog(LOG_PRIORITY, "SSL ctx is null");
       return 0;
    }
 
-   if(SSL_CTX_set_cipher_list(session->ctx, session->cfg->cipher_list) == 0){
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+   SSL_CTX_set_options(session->net.ctx, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3|SSL_OP_NO_TLSv1|SSL_OP_NO_TLSv1_1);
+#else
+   if(SSL_CTX_set_min_proto_version(session->net.ctx, session->cfg->tls_min_version_number) == 0){
+      syslog(LOG_PRIORITY, "failed SSL_CTX_set_min_proto_version() to %s/%d", session->cfg->tls_min_version, session->cfg->tls_min_version_number);
+      return 0;
+   }
+#endif
+
+   if(SSL_CTX_set_cipher_list(session->net.ctx, session->cfg->cipher_list) == 0){
       syslog(LOG_PRIORITY, "failed to set cipher list: '%s'", session->cfg->cipher_list);
       return 0;
    }
 
-   if(SSL_CTX_use_PrivateKey_file(session->ctx, session->cfg->pemfile, SSL_FILETYPE_PEM) != 1){
+   if(SSL_CTX_use_PrivateKey_file(session->net.ctx, session->cfg->pemfile, SSL_FILETYPE_PEM) != 1){
       syslog(LOG_PRIORITY, "cannot load private key from %s", session->cfg->pemfile);
       return 0;
    }
 
-   if(SSL_CTX_use_certificate_file(session->ctx, session->cfg->pemfile, SSL_FILETYPE_PEM) != 1){
+   if(SSL_CTX_use_certificate_file(session->net.ctx, session->cfg->pemfile, SSL_FILETYPE_PEM) != 1){
       syslog(LOG_PRIORITY, "cannot load certificate from %s", session->cfg->pemfile);
       return 0;
    }
@@ -200,56 +181,69 @@ void process_command_starttls(struct smtp_session *session){
 
    if(init_ssl(session) == 1){
 
-      session->ssl = SSL_new(session->ctx);
+      session->net.ssl = SSL_new(session->net.ctx);
-      if(session->ssl){
+      if(session->net.ssl){
 
-         SSL_set_options(session->ssl, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3);
+         if(SSL_set_fd(session->net.ssl, session->net.socket) == 1){
-
+            session->net.starttls = 1;
-         if(SSL_set_fd(session->ssl, session->socket) == 1){
-            session->starttls = 1;
             send_smtp_response(session, SMTP_RESP_220_READY_TO_START_TLS);
             session->protocol_state = SMTP_STATE_INIT;
 
-            if(session->starttls == 1 && session->use_ssl == 0)
+            if(session->net.starttls == 1 && session->net.use_ssl == 0)
                wait_for_ssl_accept(session);
 
             return;
-         } syslog(LOG_PRIORITY, "%s: SSL_set_fd() failed", session->ttmpfile);
+         } syslog(LOG_PRIORITY, "ERROR: %s: SSL_set_fd() failed", session->ttmpfile);
-      } syslog(LOG_PRIORITY, "%s: SSL_new() failed", session->ttmpfile);
+      } syslog(LOG_PRIORITY, "ERROR: %s: SSL_new() failed", session->ttmpfile);
-   } syslog(LOG_PRIORITY, "SSL ctx is null!");
+   } syslog(LOG_PRIORITY, "ERROR: init_ssl()");
 
    send_smtp_response(session, SMTP_RESP_454_ERR_TLS_TEMP_ERROR);
 }
 
 
-void process_command_mail_from(struct smtp_session *session, char *buf){
+void process_command_mail_from(struct smtp_session *session){
    memset(session->mailfrom, 0, SMALLBUFSIZE);
 
    if(session->protocol_state != SMTP_STATE_HELO && session->protocol_state != SMTP_STATE_PERIOD && session->protocol_state != SMTP_STATE_BDAT){
-      send(session->socket, SMTP_RESP_503_ERR, strlen(SMTP_RESP_503_ERR), 0);
+      send(session->net.socket, SMTP_RESP_503_ERR, strlen(SMTP_RESP_503_ERR), 0);
    }
    else {
-      memset(&(session->ttmpfile[0]), 0, SMALLBUFSIZE);
-      make_random_string(&(session->ttmpfile[0]), QUEUE_ID_LEN);
       session->protocol_state = SMTP_STATE_MAIL_FROM;
 
-      extractEmail(buf, session->mailfrom);
+      extractEmail(session->buf, session->mailfrom);
 
-      reset_bdat_counters(session);
+      int mailsize = get_size_from_smtp_mail_from(session->buf);
-      session->tot_len = 0;
+
+      reset_smtp_session(session);
+
+      session->mail_size = mailsize;
 
       send_smtp_response(session, SMTP_RESP_250_OK);
    }
 }
 
 
-void process_command_rcpt_to(struct smtp_session *session, char *buf){
+void process_command_rcpt_to(struct smtp_session *session, struct config *cfg){
 
    if(session->protocol_state == SMTP_STATE_MAIL_FROM || session->protocol_state == SMTP_STATE_RCPT_TO){
 
       // For now, we are not interested in the envelope recipients
 
       session->protocol_state = SMTP_STATE_RCPT_TO;
+
+      if(session->num_of_rcpt_to < MAX_RCPT_TO){
+         extractEmail(session->buf, session->rcptto[session->num_of_rcpt_to]);
+
+         // Check if we should accept archive_address only
+         if(cfg->archive_address[0] && !strstr(cfg->archive_address, session->rcptto[session->num_of_rcpt_to])){
+            syslog(LOG_PRIORITY, "ERROR: Invalid recipient: *%s*", session->rcptto[session->num_of_rcpt_to]);
+            send_smtp_response(session, SMTP_RESP_550_ERR_INVALID_RECIPIENT);
+            return;
+         }
+
+         session->num_of_rcpt_to++;
+      }
+
       send_smtp_response(session, SMTP_RESP_250_OK);
    }
    else {
@@ -258,7 +252,7 @@ void process_command_rcpt_to(struct smtp_session *session, char *buf){
 }
 
 
-void process_command_data(struct smtp_session *session){
+void process_command_data(struct smtp_session *session, struct config *cfg){
    session->tot_len = 0;
 
    if(session->protocol_state != SMTP_STATE_RCPT_TO){
@@ -273,6 +267,8 @@ void process_command_data(struct smtp_session *session){
       else {
          session->protocol_state = SMTP_STATE_DATA;
          send_smtp_response(session, SMTP_RESP_354_DATA_OK);
+
+         if(cfg->process_rcpt_to_addresses == 1) write_envelope_addresses(session, cfg);
       }
    }
 
@@ -280,24 +276,39 @@ void process_command_data(struct smtp_session *session){
 
 void process_command_period(struct smtp_session *session){
    char buf[SMALLBUFSIZE];
+   struct timezone tz;
+   struct timeval tv1, tv2;
 
    session->protocol_state = SMTP_STATE_PERIOD;
 
    // TODO: add some error handling
 
+   gettimeofday(&tv1, &tz);
    fsync(session->fd);
    close(session->fd);
+   gettimeofday(&tv2, &tz);
 
    session->fd = -1;
 
-   syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d", session->ttmpfile, session->mailfrom, session->tot_len);
+   syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d, client=%s, fd=%d, fsync=%ld", session->ttmpfile, session->mailfrom, session->tot_len, session->remote_host, session->net.socket, tvdiff(tv2, tv1));
 
-   move_email(session);
+   if(session->bad == 1 || session->too_big == 1){
+      snprintf(buf, sizeof(buf)-1, SMTP_RESP_451_ERR);
+      unlink(session->ttmpfile);
+      syslog(LOG_PRIORITY, "ERROR: problem in processing, removing %s", session->ttmpfile);
+      if(session->too_big)
+         snprintf(buf, sizeof(buf)-1, "%s", SMTP_RESP_552_ERR_TOO_BIG_EMAIL);
+      else
+         snprintf(buf, sizeof(buf)-1, "%s", SMTP_RESP_451_ERR);
+   } else {
+      move_email(session);
+      snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
+   }
 
-   snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
+   if(session->buf){
-
+      memset(session->buf, 0, session->bufsize);
-   session->buflen = 0;
+      session->buflen = 0;
-   memset(session->buf, 0, SMALLBUFSIZE);
+   }
 
    send_smtp_response(session, buf);
 }
@@ -313,14 +324,32 @@ void process_command_quit(struct smtp_session *session, char *buf, int buflen){
 
 
 void process_command_reset(struct smtp_session *session){
+   reset_smtp_session(session);
    send_smtp_response(session, SMTP_RESP_250_OK);
-
-   session->tot_len = 0;
-   session->fd = -1;
    session->protocol_state = SMTP_STATE_HELO;
+}
+
+
+void reset_smtp_session(struct smtp_session *session){
+   session->tot_len = 0;
+   session->mail_size = 0;
+   session->too_big = 0;
+   session->bad = 0;
+
+   session->fd = -1;
+
+   session->num_of_rcpt_to = 0;
+   for(int i=0; i<MAX_RCPT_TO; i++) memset(session->rcptto[i], 0, SMALLBUFSIZE);
 
    reset_bdat_counters(session);
 
+   time(&(session->lasttime));
+
    memset(&(session->ttmpfile[0]), 0, SMALLBUFSIZE);
-   make_random_string(&(session->ttmpfile[0]), QUEUE_ID_LEN);
+   make_random_string((unsigned char*)&(session->ttmpfile[0]), QUEUE_ID_LEN);
+
+   if(session->buf){
+      memset(session->buf, 0, session->bufsize);
+   }
+   session->buflen = 0;
 }

src/smtp.h

@@ -7,22 +7,24 @@
 
 #include <piler.h>
 
-void process_smtp_command(struct smtp_session *session, char *buf);
+void process_smtp_command(struct smtp_session *session, struct config *cfg);
-void process_data(struct smtp_session *session, char *readbuf, int readlen);
+void process_data(struct smtp_session *session, char *buf, int buflen);
 
 void send_smtp_response(struct smtp_session *session, char *buf);
 void process_command_helo(struct smtp_session *session, char *buf, int buflen);
-void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen);
+void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen, struct config *cfg);
 void process_command_quit(struct smtp_session *session, char *buf, int buflen);
 void process_command_reset(struct smtp_session *session);
-void process_command_mail_from(struct smtp_session *session, char *buf);
+void process_command_mail_from(struct smtp_session *session);
-void process_command_rcpt_to(struct smtp_session *session, char *buf);
+void process_command_rcpt_to(struct smtp_session *session, struct config *cfg);
-void process_command_data(struct smtp_session *session);
+void process_command_data(struct smtp_session *session, struct config *cfg);
 void process_command_period(struct smtp_session *session);
 void process_command_starttls(struct smtp_session *session);
 
 void reset_bdat_counters(struct smtp_session *session);
-void get_bdat_size_to_read(struct smtp_session *session, char *buf);
+void get_bdat_size_to_read(struct smtp_session *session);
-void process_bdat(struct smtp_session *session, char *readbuf, int readlen);
+void process_bdat(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg);
+
+void reset_smtp_session(struct smtp_session *session);
 
 #endif

src/smtpcodes.h

@@ -18,6 +18,7 @@
 
 #define SMTP_CMD_HELO "HELO"
 #define SMTP_CMD_EHLO "EHLO"
+#define SMTP_CMD_HELP "HELP"
 #define SMTP_CMD_MAIL_FROM "MAIL FROM:"
 #define SMTP_CMD_RCPT_TO "RCPT TO:"
 #define SMTP_CMD_DATA "DATA"
@@ -32,11 +33,12 @@
 
 // SMTP responses
 
+#define SMTP_RESP_221_PILER_SMTP_OK "221 piler-smtp is OK\r\n"
 #define SMTP_RESP_220_BANNER "220 %s ESMTP\r\n"
 #define SMTP_RESP_220_READY_TO_START_TLS "220 Ready to start TLS\r\n"
 #define SMTP_RESP_221_GOODBYE "221 %s Goodbye\r\n"
 #define SMTP_RESP_250_OK "250 Ok\r\n"
-#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n250-PIPELINING\r\n%s250-SIZE\r\n250 8BITMIME\r\n"
+#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n250-SIZE %d\r\n%s250 8BITMIME\r\n"
 
 #define SMTP_EXTENSION_STARTTLS "250-STARTTLS\r\n"
 #define SMTP_EXTENSION_CHUNKING "250-CHUNKING\r\n"
@@ -49,18 +51,16 @@
 #define SMTP_RESP_421_ERR_TMP "421 %s service not available\r\n"
 #define SMTP_RESP_421_ERR_WRITE_FAILED "421 writing queue file failed\r\n"
 #define SMTP_RESP_421_ERR_ALL_PORTS_ARE_BUSY "421 All server ports are busy\r\n"
-#define SMTP_RESP_450_ERR_CMD_NOT_IMPLEMENTED "450 command not implemented\r\n"
+#define SMTP_RESP_451_ERR_TOO_MANY_REQUESTS "451 Too many requests, try again later\r\n"
 #define SMTP_RESP_451_ERR "451 Error in processing, try again later\r\n"
 #define SMTP_RESP_454_ERR_TLS_TEMP_ERROR "454 TLS not available currently\r\n"
 
 #define SMTP_RESP_502_ERR "502 Command not implemented\r\n"
 #define SMTP_RESP_503_ERR "503 Bad command sequence\r\n"
-#define SMTP_RESP_530_ERR_MUST_ISSUE_STARTTLS_FIRST "530 MUST issue STARTTLS command first\r\n"
+#define SMTP_RESP_550_ERR_INVALID_RECIPIENT "550 Invalid recipient\r\n"
-#define SMTP_RESP_550_ERR "550 Access denied.\r\n"
-#define SMTP_RESP_550_INVALID_RECIPIENT "550 Unknown recipient\r\n"
-#define SMTP_RESP_550_ERR_TOO_LONG_RCPT_TO "550 too long recipient\r\n"
 #define SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY "550 You are banned by local policy\r\n"
-#define SMTP_RESP_552_ERR "552 Too many recipients\r\n"
+#define SMTP_RESP_550_ERR "550 Service currently unavailable\r\n"
+#define SMTP_RESP_552_ERR_TOO_BIG_EMAIL "552 Too big email\r\n"
 
 
 // LMTP commands
@@ -70,4 +70,3 @@
 // LMTP responses
 
 #define LMTP_RESP_220_BANNER "220 %s LMTP\r\n"
-