Bumped up version number to 1.4.5

Signed-off-by: Janos SUTO <sj@acts.hu>

.gitignore

@@ -0,0 +1,26 @@
+Makefile
+*.o
+*.so
+*.a
+*.deb
+src/libpiler.so.0*
+src/piler
+src/pilerconf
+src/piler-smtp
+src/pileraget
+src/pilerexport
+src/pilerget
+src/pilerimport
+src/pilerstats
+src/pilertest
+src/reindex
+unit_tests/check_attachments
+unit_tests/check_decoder
+unit_tests/check_digest
+unit_tests/check_hash
+unit_tests/check_misc
+unit_tests/check_mydomains
+unit_tests/check_parser
+unit_tests/check_parser_utils
+unit_tests/check_rules
+unit_tests/smtp

CREDITS

@@ -4,4 +4,5 @@ The FSF.hu Foundation (http://fsf.hu/) supported and donated piler within the
 Nemeth Adam reviewed the web interface, and gave lots of useful hints and insights to
 improve the web ui of piler.
 
-Remi Smith improved the restricted auditor feature for a better multitenancy.
+Remi Smith improved the restricted auditor feature for a better multitenancy,
+and invented the default theme.

G

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+read -r version < VERSION
+
+tarball="piler-${version}.tar.gz"
+
+git archive --prefix="piler-${version}/" HEAD | gzip -c > "$tarball"
+sha256sum "$tarball"

LICENSE

@@ -1,6 +1,6 @@
 piler, an enterprise level email archiving application
 
-Copyright (C) 2012-2013, Janos SUTO <sj@acts.hu>
+Copyright (C) 2012-2023, Janos SUTO <sj@acts.hu>
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
@@ -13,4 +13,3 @@ GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License
 along with this program. If not, see <http://www.gnu.org/licenses/>.
-

Makefile.in

@@ -16,9 +16,9 @@ localstatedir = @localstatedir@
 CC = @CC@
 CFLAGS = @CFLAGS@ @CPPFLAGS@
 DEFS = @defs@
-INCDIR = -I. @INCDIR@ @mysql_includes@
+INCDIR = -I. @INCDIR@ @sql_includes@
 LIBDIR = -L. @LIBDIR@ @LDFLAGS@
-LIBS = @LIBS@ @mysql_libs@
+LIBS = @LIBS@ @sql_libs@
 RUNNING_USER = @RUNNING_USER@
 RUNNING_GROUP = `@id_bin@ -gn $(RUNNING_USER)`
 
@@ -33,7 +33,7 @@ INSTALL = @INSTALL@
 am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \
 	piler-config.h stamp.h stamp-h1 params.h
 
-all: all-recursive
+all: all-recursive config-php
 install: installdirs install-recursive
 
 
@@ -63,20 +63,36 @@ $(RECURSIVE_TARGETS):
 	fi; test -z "$$fail"
 
 
+config-php:
+	sed -e "s%SYSCONFDIR%$(sysconfdir)%g" -e "s%SBINDIR%$(sbindir)%g" config.php.in > webui/config.php
+	sed -e "s%BINDIR%$(bindir)%g" -i webui/config.php
+
 
 installdirs: mkinstalldirs
 	$(srcdir)/mkinstalldirs \
-		$(DESTDIR)$(bindir) $(DESTDIR)$(sbindir) $(DESTDIR)$(libdir) $(DESTDIR)$(libexecdir)/piler $(DESTDIR)$(sysconfdir) \
+		$(DESTDIR)$(bindir) \
+		$(DESTDIR)$(sbindir) \
+		$(DESTDIR)$(libdir) \
+		$(DESTDIR)$(libexecdir)/piler \
+		$(DESTDIR)$(datarootdir)/piler \
+		$(DESTDIR)$(sysconfdir) \
+		$(DESTDIR)$(sysconfdir)/piler \
+		$(DESTDIR)/etc/init.d \
 		$(DESTDIR)$(localstatedir)/piler/store \
-		$(DESTDIR)$(localstatedir)/piler/stat $(DESTDIR)$(localstatedir)/piler/tmp \
-		$(DESTDIR)$(localstatedir)/piler/sphinx
+		$(DESTDIR)$(localstatedir)/piler/stat \
+		$(DESTDIR)$(localstatedir)/piler/tmp \
+		$(DESTDIR)$(localstatedir)/piler/error \
+		$(DESTDIR)$(localstatedir)/piler/export \
+		$(DESTDIR)$(localstatedir)/piler/manticore
 
 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/run/piler
 	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/store
 	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/imap
 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/stat
 	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/tmp
-	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/sphinx
+	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/error
+	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/export
+	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/manticore
 
 
 install-am:
@@ -110,7 +126,7 @@ key:
 	dd if=/dev/urandom bs=56 count=1 of=piler.key
 
 postinstall:
-	@sh util/postinstall.sh $(RUNNING_USER) $(RUNNING_GROUP) $(sysconfdir) $(localstatedir)
+	@bash util/postinstall.sh $(RUNNING_USER) $(RUNNING_GROUP) $(sysconfdir) $(localstatedir) $(libexecdir)
 
 
 

README

@@ -24,9 +24,9 @@ Features:
 
 - AD / LDAP authentication
 
-- IMAP authentication
+- IMAP, POP3 authentication
 
-- single sign-on
+- single sign-on (SSO)
 
 - Google Apps support
 
@@ -34,6 +34,8 @@ Features:
 
 - STARTTLS support
 
+- Google Authenticator support for 2-factor authentication
+
 - i18n
 - customisable theme
 

RELEASE_NOTES

@@ -0,0 +1,209 @@
+1.4.5:
+------
+
+- Introduced 2 new variables in /etc/piler/piler.conf affecting piler-smtp
+
+  ; max message size in bytes
+  ; piler-smtp will reject any message that's bigger than this number
+  max_message_size=50000000
+
+  ; max memory in bytes piler-smtp uses for buffering messages
+  ; when this limit is exceeded, no new emails will be accepted
+  ; until the used memory for all in progress emails decreases
+  ; below this level
+  max_smtp_memory=500000000
+
+  Be sure to adjust these values to your environment!
+
+- Added read-only connection support for manticore
+  If using sphinx, add the following to config-site.php:
+  $config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9306';
+
+- pilerimport supports Zimbra IMAP impersonation
+
+  Generate the following base64 encoded string:
+  (Be sure to use the actual usernames and password):
+
+  pw="$( printf '%s\0%s\0%s' 'username' 'zimbra_admin_username' 'zimbra_admin_password' | base64 )"
+
+  Then specify -u ZIMBRA -p "$pw" for pilerimport, eg.
+
+  pilerimport -i imap.server -u ZIMBRA -p "$pw" ...
+
+  Note that "ZIMBRA" is a special username, it indicates for pilerimport
+  to actually use the imap impersonation for Zimbra.
+
+
+1.4.4:
+------
+
+- Renamed "group" table to "usergroup"
+  Be sure to run util/db-upgrade.sql on the mysql piler database
+
+
+1.4.3:
+------
+
+- Improved real-time index support (no need to use rtindex.py any more)
+  See https://www.mailpiler.org/wiki/current:manticore for more
+
+
+1.4.2:
+------
+
+- Fixed docker image
+- Fixed php socket path in piler-nginx.conf
+- Fixed manticore index data settings
+
+
+1.4.1:
+------
+
+- Replaced sphinx with manticore 5.0.2, see https://www.mailpiler.org/wiki/current:manticore
+- Replaced obsoleted libchart library with chart.js
+- Added real-time index support
+- Added oauth2 support to imapfetch.py
+- Added support for pilerexport to spread files among several directories. See the -D option
+
+
+1.3.12:
+-------
+
+- Introduced new piler.conf variable: tls_min_version
+
+  It sets the minimum TLS protocol version the piler-smtp daemon supports.
+
+  Possible values:
+  - TLSv1   (not recommended)
+  - TLSv1.1 (not recommended)
+  - TLSv1.2 (default)
+  - TLSv1.3
+
+- Introduced the archive_address feature, see etc/example.conf for the details
+- Introduced the raw: search label
+- Added Italian translation. Credits: Stefano Gatto
+- timestamp signing sorts by 'id' column
+- timestamp hash value defaults to sha256
+- Minor fixes
+- Added support for php 8.1
+- Fixed handling long email addresses
+
+Be sure to apply util/db-upgrade.sql
+
+
+1.3.11:
+-------
+
+- [BUGFIX] Refactored the smtp timeout check
+- Obsoleted the LDAP port parameter. Specify the ldap host in the form
+  of protocol://hostname:port, eg. ldaps://ldap.example.com:636
+
+1.3.10:
+-------
+
+- Added security header feature
+- Introduced the smtp acl list, and obsoleted the tcp_wrappers check
+- Switched from Blowfish encryption to AES-256
+
+1.3.9:
+------
+
+- Added a separator to searching for attachment names
+- [BUGFIX] Render multiple mail parts in mail view instead of only the last part
+- Use TLS v1.2 with openssl 1.0.x for connecting remote pop3/imap servers
+- Instant search results to the gui when the search page loads
+- Support sphinx-3.3.1, introduced sphinx strict mode variable
+- GUI domain fixes
+- gcc 9 fixes
+- Fix permission on sphinx data dir to 700
+- pilerpurge.py should honor the mysqlhost value
+- Password change enabled by default
+- Health page fixes
+- GUI mime parser fixes
+- Start/stop script fix
+- Optimized search page for mobile devices, set $config['ENABLE_MOBILE_PREVIEW'] = 1; in config-site.php to enable it
+
+
+1.3.0:
+-----
+
+- Switched from utf8 to utf8mb4 encoding.
+
+- Switched from preforking model to epoll
+
+- Removed PDF support
+
+- GUI fixes
+
+- Added HTML purifier support
+
+- Rewritten pilerpurge in Python
+
+1.2.0:
+------
+
+Heavy SQL scheme changes throughout the development of 1.2.0!
+
+2016.10.06.
+
+Removed deb and rpm packaging metadata.
+
+2016.09.21.
+
+Improved logging for the GUI. See the new config option: $config['LOG_LEVEL'] = NORMAL;
+
+2016.09.18.
+
+Moved all piler configuration from ${sysconfdir} to ${sysconfdir}/piler, eg.
+
+root@01b74b787c26:/# ls -la /usr/local/etc/piler/
+total 36
+drwxr-xr-x 1 root root    24 Oct  6 20:49 .
+drwxr-xr-x 1 root root    74 Oct  6 20:49 ..
+-rw-r--r-- 4 root root  1008 Oct  6 20:39 piler.conf
+-rw-r----- 5 root piler 1066 Oct  6 20:39 piler.conf.dist
+-rw-r----- 5 root piler   56 Oct  6 20:40 piler.key
+-rw------- 5 root root  5256 Oct  6 20:40 piler.pem
+-rw-r--r-- 1 root root  6270 Oct  6 20:49 sphinx.conf
+-rw-r--r-- 5 root piler 6299 Oct  6 20:39 sphinx.conf.dist
+
+Make sure to update the piler shipped shell scripts
+(rc.searchd, indexer.*.sh, ...) as well!
+
+2016.08.31.
+
+Added chunking / BDAT support for the piler daemon.
+You may turn it on by setting enable_chunking=1 in piler.conf
+
+2016.06.19.
+
+Improved attachment table indexing.
+
+2016.05.07.
+
+Obsoleted the --enable-starttls configure option.
+
+2016.03.02.
+
+Discard messages shorther than 100 bytes.
+
+2016.02.10.
+
+Added the private feature (for auditors only).
+
+
+2015.11.26.
+
+Removed the pilergetd stuff.
+
+2015.07.08.
+
+Delete option for auditors.
+
+2015.05.07.
+
+Added milter script to contrib directory.
+
+2015.04.22.
+
+Added timestamp support.

SECURITY.md

@@ -0,0 +1,20 @@
+Disclosure policy
+
+If you find a security issue, please contact the project owner at sj@acts.hu
+with the details (ie. piler version, details of the setup, how to exploit the
+vulnerability, etc).
+
+Please provide 30 days for verifying the vulnerability, fixing the issue, and
+notifying the piler users.
+
+Security update policy
+
+If a security vulnerability has found, the details, possible mitigations,
+workarounds, etc. will be shared on the piler mailing list (piler-user@list.acts.hu)
+and on the wiki: https://www.mailpiler.org/
+
+Security configurations
+
+ - Use https for the GUI
+ - Reset the default passwords for admin and auditor
+ - Use the smtp acl feature to restrict SMTP access to the archive, see https://mailpiler.com/smtp-acl-list/

VERSION

@@ -0,0 +1 @@
+1.4.5

aclocal.m4

@@ -13,8 +13,8 @@
 
 m4_ifndef([AC_AUTOCONF_VERSION],
   [m4_copy([m4_PACKAGE_VERSION], [AC_AUTOCONF_VERSION])])dnl
-m4_if(m4_defn([AC_AUTOCONF_VERSION]), [2.68],,
-[m4_warning([this file was generated for autoconf 2.68.
+m4_if(m4_defn([AC_AUTOCONF_VERSION]), [2.69],,
+[m4_warning([this file was generated for autoconf 2.69.
 You have another version of autoconf.  It may work, but is not guaranteed to.
 If you have problems, you may need to regenerate the build system entirely.
 To do so, use the procedure documented by the package, typically `autoreconf'.])])

bitbucket-pipelines.yml

@@ -0,0 +1,22 @@
+# This is a sample build configuration for all languages.
+# Check our guides at https://confluence.atlassian.com/x/VYk8Lw for more examples.
+# Only use spaces to indent your .yml configuration.
+# -----
+# You can specify a custom docker image from Docker Hub as your build environment.
+
+image: sutoj/builder:bionic
+clone:
+  depth: 5
+
+pipelines:
+  default:
+    - step:
+        script:
+          - service mysql start
+          - ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --with-database=mariadb
+          - make clean all install
+          - mysql -u piler -ppiler123 piler1 < /usr/share/piler/db-mysql.sql
+          - cd unit_tests
+          - ./run.sh
+          - cd ../webui
+          - phpunit

config.php.in

@@ -0,0 +1,562 @@
+<?php
+
+ini_set('session.cookie_httponly', true);
+ini_set('session.use_strict_mode', 1);
+ini_set('session.use_only_cookies', 1);
+
+
+define('NORMAL', 1);
+define('DEBUG', 5);
+
+$config = [];
+
+
+/*
+ * you can override any of these values by putting the
+ * variable to be overridden in config-site.php
+ */
+
+$config['ENABLE_MOBILE_PREVIEW'] = 0;
+
+$config['SITE_LOGO_LG'] = '/view/theme/default/assets/images/archive-logo-lg.png';
+$config['SITE_LOGO_SM'] = '/view/theme/default/assets/images/archive-logo-sm.png';
+$config['COMPATIBILITY'] = 'Which browsers are supported, etc';
+
+$config['BRANDING_TEXT'] = '';
+$config['BRANDING_URL'] = '';
+$config['BRANDING_LOGO'] = $config['SITE_LOGO_SM'];
+$config['BRANDING_BACKGROUND_COLOUR'] = '';
+$config['BRANDING_TEXT_COLOUR'] = '';
+$config['BRANDING_FAVICON'] = '/view/theme/default/assets/ico/favicon.png';
+$config['SUPPORT_LINK'] = '';
+$config['TITLE_PREFIX'] = '';
+
+$config['CUSTOM_PRE_AUTH_FUNCTION'] = '';
+$config['CUSTOM_EMAIL_QUERY_FUNCTION'] = '';
+
+$config['DOMAIN_REGEX'] = '/^[a-zA-Z0-9]+[a-zA-Z0-9-_\.]{0,}\.[a-zA-Z0-9]{2,20}$/';
+
+$config['BOOTSTRAP_THEME'] = '-cosmo';
+
+$config['DEFAULT_LANG'] = 'en';
+
+$config['THEME'] = 'default';
+
+$config['SITE_NAME'] = 'piler.example.com';
+$config['SITE_URL'] = 'http://piler.example.com/';
+
+$config['EXTERNAL_DASHBOARD_URL'] = '';
+
+$config['SESSION_EXPIRY'] = 3600;
+$config['DELTA_INDEXER_PERIOD'] = 1800;
+
+$config['ENABLE_SAAS'] = 0;
+$config['CAPTCHA_FAILED_LOGIN_COUNT'] = 0;
+
+$config['ENABLE_REFERENCES'] = 1;
+$config['ENABLE_DOWNLOADING_ALL_SEARCH_HITS'] = 0;
+$config['EML_NAME_BASED_ON_SUBJECT'] = 1;
+
+$config['ENABLE_TABLE_RESIZE'] = 0;
+$config['DEMO_MODE'] = 0;
+
+$config['SHOW_MENU_FOR_OUTLOOK'] = 0;
+
+$config['SEARCH_QUERY_QUOTING'] = 0;
+
+$config['LOCALIZE_MESSAGE_HEADERS_IN_PREVIEW'] = 1;
+
+$config['TIMEZONE'] = 'Europe/Budapest';
+
+$config['PROVIDED_BY'] = 'www.mailpiler.org';
+$config['SITE_KEYWORDS'] = 'piler email archiver';
+$config['SITE_DESCRIPTION'] = 'piler email archiver';
+
+$config['INDEXER_BEACON'] = '/var/piler/stat/indexer';
+$config['PURGE_BEACON'] = '/var/piler/stat/purge';
+
+$config['ENABLE_PDF_DOWNLOAD'] = 0;
+// You may need to run "Xvfb :1 -screen 0 1024x768x16"
+// In this case specify WKHTMLTOPDF_COMMAND = "DISPLAY=:1.0 wkhtmltopdf";
+$config['WKHTMLTOPDF_COMMAND'] = "wkhtmltopdf";
+
+// authentication against an ldap directory (disabled by default)
+
+$config['ENABLE_LDAP_AUTH'] = 0;
+$config['LDAP_HOST'] = 'ldap.example.com';
+$config['LDAP_HELPER_DN'] = 'cn=....';
+$config['LDAP_HELPER_PASSWORD'] = 'xxxxxxx';
+$config['LDAP_MAIL_ATTR'] = 'mail';
+$config['LDAP_AUDITOR_MEMBER_DN'] = '';
+$config['LDAP_ADMIN_MEMBER_DN'] = '';
+$config['LDAP_BASE_DN'] = '';
+$config['LDAP_USE_START_TLS'] = 0;
+
+// AD specific settings
+//
+$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'user';
+$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'group';
+$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'member';
+$config['LDAP_MAIL_ATTR'] = 'proxyAddresses';
+
+// zimbra specific settings
+//$config['LDAP_HELPER_DN'] = 'uid=zimbra,cn=admins,cn=zimbra';
+//$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'zimbraAccount';
+//$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'zimbraDistributionList';
+//$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'zimbraMailForwardingAddress';
+//$config['LDAP_MAIL_ATTR'] = 'mail';
+
+// Lotus Notes specific settings for ldap authentication
+//
+//$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'dominoPerson';
+//$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'dominoGroup');
+//$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mail';
+//$config['LDAP_MAIL_ATTR'] = 'mail';
+
+// iredmail specific settings
+//$config['LDAP_HELPER_DN'] = 'cn=vmailadmin,dc=example,dc=com';
+//$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'mailUser';
+//$config['LDAP_BASE_DN'] = 'o=domains,dc=example,dc=com';
+//$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'mailList';
+//$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'memberOfGroup';
+//$config['LDAP_MAIL_ATTR'] = 'mail';
+
+// Uninvention specific settings
+//$config['LDAP_MAIL_ATTR'] = 'mailPrimaryAddress';
+//$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'person';
+//$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'person';
+//$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mailAlternativeAddress';
+
+
+// enable single sign-on (disabled by default)
+$config['ENABLE_SSO_LOGIN'] = 0;
+
+$config['STRIP_DOMAIN_NAME_FROM_USERNAME'] = 0;
+
+// enable authentication against an imap server (disabled by default)
+
+$config['ENABLE_IMAP_AUTH'] = 0;
+$config['RESTORE_OVER_IMAP'] = 0;
+$config['IMAP_RESTORE_FOLDER_INBOX'] = 'INBOX';
+$config['IMAP_RESTORE_FOLDER_SENT'] = 'Sent';
+$config['IMAP_HOST'] = 'mail.example.com';
+$config['IMAP_PORT'] =  993;
+$config['IMAP_SSL'] = true;
+
+// enable authentication against a pop3 server (disabled by default)
+
+$config['ENABLE_POP3_AUTH'] = 0;
+$config['POP3_HOST'] = 'mail.example.com';
+$config['POP3_PORT'] = 995;
+$config['POP3_SSL'] = true;
+
+$config['ENABLE_GOOGLE_AUTHENTICATOR'] = 1;
+
+// enable authentication against google (disabled by default)
+// see http://www.mailpiler.org/en/google-apps-free.html for details
+
+$config['ENABLE_GOOGLE_LOGIN'] = 0;
+$config['GOOGLE_CLIENT_ID'] = 'xxxxxxxxxxx';
+$config['GOOGLE_CLIENT_SECRET'] = 'xxxxxxxxxxxxx';
+$config['GOOGLE_DEVELOPER_KEY'] = 'xxxxxxxxxxxx';
+$config['GOOGLE_APPLICATION_NAME'] = 'piler enterprise email archiver';
+$config['GOOGLE_ALL_MAIL'] = '[Gmail]/All Mail';
+
+
+$config['ENABLE_AUDIT'] = 1;
+$config['MEMCACHED_ENABLED'] = 0;
+$config['PASSWORD_CHANGE_ENABLED'] = 1;
+$config['ENABLE_STATISTICS'] = 1;
+$config['ENABLE_HISTORY'] = 1;
+$config['ENABLE_DELETE'] = 0;
+$config['NEED_TO_APPROVE_DELETE'] = 0;
+$config['ENABLE_REMOTE_IMAGES'] = '0';
+$config['ENABLE_ON_THE_FLY_VERIFICATION'] = 0;
+$config['ENABLE_LDAP_IMPORT_FEATURE'] = 0;
+$config['ENABLE_FOLDER_RESTRICTIONS'] = 0;
+
+// When enabled the search page auto loads the search results
+// ie. as if the user had clicked on the Search button
+$config['ENABLE_INSTANT_SEARCH'] = 0;
+$config['SEARCH_RESULT_CHECKBOX_CHECKED'] = 0;
+$config['HELPER_URL'] = '';
+$config['LOG_LEVEL'] = NORMAL;
+$config['REWRITE_MESSAGE_ID'] = 0;
+$config['RESTORE_EMAILS_AS_ATTACHMENT'] = 0;
+$config['RESTRICTED_AUDITOR'] = 0;
+
+$config['SHOW_ENVELOPE_JOURNAL'] = 0;
+$config['BULK_DOWNLOAD_FOR_USERS'] = 1;
+$config['MAX_DOWNLOAD_PER_HOUR'] = 0;
+$config['MAX_RESTORE_PER_HOUR'] = 0;
+
+$config['MAX_CGI_FROM_SUBJ_LEN'] = 48;
+$config['PAGE_LEN'] = 20;
+$config['MAX_NUMBER_OF_FROM_ITEMS'] = 5;
+$config['MAX_SEARCH_HITS'] = 1000;
+
+$config['SPHINX_MAIN_INDEX_THRESHOLD'] = 2000000000;
+
+$config['DEFAULT_RETENTION'] = 0;
+
+$config['LOCALHOST'] = '127.0.0.1';
+$config['PILER_HOST'] = '0.0.0.0';
+$config['PILER_PORT'] = 25;
+$config['SMARTHOST'] = '';
+$config['SMARTHOST_PORT'] = 25;
+$config['SMARTHOST_USER'] = '';
+$config['SMARTHOST_PASSWORD'] = '';
+$config['SMTP_DOMAIN'] = 'example.com';
+$config['SMTP_FROMADDR'] = 'no-reply@example.com';
+$config['ADMIN_EMAIL'] = 'admin@example.com';
+
+$config['PILER_HEADER_FIELD'] = 'X-piler-id: ';
+
+$config['DEFAULT_POLICY'] = 'default_policy';
+
+$config['DIR_BASE'] = dirname(__FILE__) . '/';
+
+$config['DIR_SPHINX'] = '/var/piler/manticore/';
+$config['DIR_STAT'] = '/var/piler/stat';
+$config['DIR_IMAP'] = '/var/piler/imap';
+$config['DIR_TMP'] = '/var/piler/tmp';
+
+$config['DECRYPT_BINARY'] = 'BINDIR/pilerget';
+$config['DECRYPT_ATTACHMENT_BINARY'] = 'BINDIR/pileraget';
+$config['DECRYPT_BUFFER_LENGTH'] = 65536;
+
+$config['OPENSSL_BINARY'] = '/usr/bin/openssl';
+$config['TSA_URL'] = '';
+$config['TSA_PUBLIC_KEY_FILE'] = '';
+$config['TSA_START_ID'] = 1;
+$config['TSA_STAMP_REQUEST_UNIT_SIZE'] = 10000;
+$config['TSA_VERIFY_CERTIFICATE'] = true;
+$config['TSA_RELAXED_CHECK'] = false;
+$config['TSA_AUTH_USER'] = '';
+$config['TSA_AUTH_PASSWORD'] = '';
+$config['TSA_AUTH_CERT_FILE'] = '';
+$config['TSA_AUTH_KEY_FILE'] = '';
+$config['TSA_AUTH_KEY_PASSWORD'] = '';
+
+$config['DB_DRIVER'] = 'mysql';
+$config['DB_PREFIX'] = '';
+$config['DB_HOSTNAME'] = 'localhost';
+$config['DB_USERNAME'] = 'piler';
+$config['DB_PASSWORD'] = 'piler';
+$config['DB_DATABASE'] = 'piler';
+$config['DB_CHARSET'] = 'utf8mb4';
+
+$config['SPHINX_DRIVER'] = 'sphinx';
+$config['SPHINX_DATABASE'] = 'sphinx';
+$config['SPHINX_HOSTNAME'] = '127.0.0.1:9306';
+$config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9307';
+$config['SPHINX_MAIN_INDEX'] = 'main1,dailydelta1,delta1';
+$config['SPHINX_ATTACHMENT_INDEX'] = 'att1';
+$config['SPHINX_TAG_INDEX'] = 'tag1';
+$config['SPHINX_NOTE_INDEX'] = 'note1';
+$config['SPHINX_STRICT_SCHEMA'] = 1;
+$config['RT'] = 0;
+$config['MAX_EMAIL_LEN'] = 41;
+
+$config['RELOAD_COMMAND'] = 'sudo -n /etc/init.d/rc.piler reload';
+$config['PILERIMPORT_IMAP_COMMAND'] = '/usr/local/bin/pilerimport -d /var/piler/imap -q -r';
+$config['CPU_USAGE_COMMAND'] = "LC_ALL=C mpstat | tail -1 | rev | awk '{ print $1 }' | rev";
+$config['PILER_BINARY'] = "SBINDIR/piler";
+
+$config['LDAP_IMPORT_CONFIG_FILE'] = '/usr/local/etc/ldap-import.cfg';
+
+$config['DN_MAX_LEN'] = 255;
+$config['USE_EMAIL_AS_USERNAME'] = 1;
+$config['LDAP_IMPORT_MINIMUM_NUMBER_OF_USERS_TO_HEALTH_OK'] = 100;
+
+$config['HEADER_LINE_TO_HIDE'] = 'X-Envelope-To:';
+
+$config['PILER_LOGIN_HELPER_PLACEHOLDER'] = 'PILER_COMMENT_FOR_PROPER_LOGIN_SO_THIS_CAN_BE_ANYTHING_JUST_BE_IT_SOMETHING_LIKE_A_UNIQUE_VALUE';
+
+
+$config['SIZE_X'] = 430;
+$config['SIZE_Y'] = 250;
+
+$config['DATE_TEMPLATE'] = 'Y.m.d';
+$config['DATE_FORMAT'] = 'YYYY-MM-DD';
+$config['JQUERY_DATE_FORMAT'] = 'yy-mm-dd';
+$config['DECIMAL_SEPARATOR'] = "."; // See https://www.php.net/manual/en/function.number-format
+$config['THOUSANDS_SEPARATOR'] = ","; // for the format options
+
+$config['ENABLE_GB2312_FIX'] = 1;
+
+$config['FROM_LENGTH_TO_SHOW'] = 28;
+
+$config['HISTORY_REFRESH'] = 60;
+$config['HEALTH_REFRESH'] = 300;
+$config['HEALTH_RATIO'] = 80;
+
+$config['MAX_AUDIT_HITS'] = 1000;
+
+$config['MIN_PASSWORD_LENGTH'] = 6;
+
+$config['MIN_PREFIX_LEN'] = 5;
+
+$config['ADMIN_CAN_POWER_SEARCH'] = 0;
+$config['FOUR_EYES_LOGIN_FOR_AUDITOR'] = 0;
+
+$config['MEMCACHED_PREFIX'] = '_piler:';
+$config['MEMCACHED_TTL'] = 900;
+
+$config['JS_CODE'] = '
+    <script src="//code.jquery.com/jquery-2.2.4.min.js" integrity="sha256-BbhdlvQf/xTY9gja0Dq3HiwQF8LaCRTXxZKRutelT44=" crossorigin="anonymous"></script>
+    <script src="//code.jquery.com/ui/1.12.1/jquery-ui.min.js" integrity="sha256-VazP97ZCwtekAsvgPBSUwPFKdrwD3unUfSGVYrahUqU=" crossorigin="anonymous"></script>
+    <script src="//stackpath.bootstrapcdn.com/bootstrap/3.4.1/js/bootstrap.min.js" integrity="sha384-aJ21OjlMXNL5UyIl/XNwTMqvzeRMZH2w8c5cRVpzpU8Y5bApTppSuUkhZXN0VxHd" crossorigin="anonymous"></script>
+
+    <script type="text/javascript" src="/view/javascript/rc-splitter.js"></script>
+    <script type="text/javascript" src="/view/javascript/piler.js"></script>
+';
+
+$config['CSS_CODE'] = '
+    <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css" integrity="sha384-HSMxcRTRxnN+Bdg0JdbxYKrThecOKuH5zCYotlSAcp1+c8xmyTe9GYg1l9a69psu" crossorigin="anonymous">
+   <link rel="stylesheet" href="/view/theme/default/assets/css/piler.css" />
+';
+
+$SUPPRESS_RECIPIENTS = array();
+
+
+$memcached_servers = array(
+      array('127.0.0.1', 11211)
+                          );
+
+$partitions_to_monitor = array('/', '/home', '/var', '/var/piler', '/tmp');
+$config['DATA_PARTITION'] = '/var';
+
+$config['DELIMITER'] = "\t";
+
+$config['TRACKING_CODE'] = '';
+
+$mailattrs = ["mail", "mailalternateaddress", "proxyaddresses", "zimbraMailForwardingAddress", "member", "memberOfGroup", "othermailbox", "mailprimaryaddress", "mailalternativeaddress"];
+
+$langs = array(
+                'cz',
+                'de',
+                'en',
+                'es',
+                'fr',
+                'hu',
+                'it',
+                'pl',
+                'pt',
+                'ru',
+                'tr',
+                'tw'
+               );
+
+
+$themes = array(
+                'default',
+               );
+
+
+$paging = array(
+                 10,
+                 20,
+                 30,
+                 50
+               );
+
+$automated_search_recipients = array();
+
+
+$letters = array('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z');
+
+
+define('NOW', time());
+
+
+/*
+ * normally you don't have to change anything below
+ */
+
+if(file_exists('SYSCONFDIR/piler/config-site.php')) { require_once 'SYSCONFDIR/piler/config-site.php'; }
+
+ini_set('session.cookie_lifetime', $config['SESSION_EXPIRY']);
+
+require($config['DIR_BASE'] . "/system/registry.php");
+require($config['DIR_BASE'] . "/system/request.php");
+
+$session = new Session();
+Registry::set("session", $session);
+
+
+if($session->get("theme") && preg_match("/^([a-zA-Z0-9\-\_]+)$/", $session->get("theme"))) { $config['THEME'] = $session->get("theme"); }
+
+include("system/helper/detectmobilebrowser.php");
+
+// make sure auditors are restricted in a saas environment
+if($config['ENABLE_SAAS'] == 1) { $config['RESTRICTED_AUDITOR'] = 1; }
+if($session->get("username") == 'auditor@local' || isset($_SERVER['argv'][2]) ) { $config['RESTRICTED_AUDITOR'] = 0; }
+
+
+
+foreach ($config as $k => $v) {
+   define($k, $v);
+}
+
+
+define('TABLE_USER', 'user');
+define('TABLE_GROUP', 'usergroup');
+define('TABLE_GROUP_USER', 'group_user');
+define('TABLE_GROUP_EMAIL', 'group_email');
+define('TABLE_FOLDER', 'folder');
+define('TABLE_FOLDER_USER', 'folder_user');
+define('TABLE_FOLDER_EXTRA', 'folder_extra');
+define('TABLE_FOLDER_MESSAGE', 'folder_message');
+define('TABLE_EMAIL', 'email');
+define('TABLE_META', 'metadata');
+define('TABLE_RCPT', 'rcpt');
+define('TABLE_ATTACHMENT', 'attachment');
+define('TABLE_SEARCH', 'search');
+define('TABLE_EMAIL_LIST', 'email_groups');
+define('TABLE_TAG', 'tag');
+define('TABLE_NOTE', '`note`');
+define('TABLE_USER_SETTINGS', 'user_settings');
+define('TABLE_REMOTE', 'remote');
+define('TABLE_DOMAIN', 'domain');
+define('TABLE_DOMAIN_USER', 'domain_user');
+define('TABLE_COUNTER', 'counter');
+define('TABLE_COUNTER_MSG', 'counter_messages');
+define('TABLE_STAT_COUNTER', 'counter_stats');
+define('TABLE_AUDIT', 'audit');
+define('TABLE_ARCHIVING_RULE', 'archiving_rule');
+define('TABLE_FOLDER_RULE', 'folder_rule');
+define('TABLE_RETENTION_RULE', 'retention_rule');
+define('TABLE_OPTION', 'option');
+define('TABLE_LDAP', 'ldap');
+define('TABLE_CUSTOMER_SETTINGS', 'customer_settings');
+define('TABLE_ONLINE', 'online');
+define('TABLE_IMPORT', 'import');
+define('TABLE_GOOGLE', 'google');
+define('TABLE_GOOGLE_IMAP', 'google_imap');
+define('TABLE_AUTOSEARCH', 'autosearch');
+define('TABLE_LEGAL_HOLD', 'legal_hold');
+define('TABLE_TIMESTAMP', 'timestamp');
+define('TABLE_PRIVATE', 'private');
+define('TABLE_DELETED', 'deleted');
+define('VIEW_MESSAGES', 'v_messages');
+
+define('EOL', "\r\n");
+
+define('DIR_SYSTEM', DIR_BASE . 'system/');
+define('DIR_MODEL', DIR_BASE . 'model/');
+define('DIR_DATABASE', DIR_BASE . 'system/database/');
+define('DIR_IMAGE', DIR_BASE . 'image/');
+define('DIR_LANGUAGE', DIR_BASE . 'language/');
+define('DIR_APPLICATION', DIR_BASE . 'controller/');
+define('DIR_THEME', DIR_BASE . 'view/theme/');
+define('DIR_REPORT', DIR_BASE . 'reports/');
+define('DIR_LOG', DIR_BASE . 'log/');
+
+define('REMOTE_IMAGE_REPLACEMENT', '/view/theme/' . THEME . '/images/remote.gif');
+define('ICON_ARROW_UP', '/view/theme/' . THEME . '/images/arrowup.gif');
+define('ICON_ARROW_DOWN', '/view/theme/' . THEME . '/images/arrowdown.gif');
+define('ICON_ATTACHMENT', '/view/theme/' . THEME . '/images/attachment_icon.png');
+define('ICON_TAG', '/view/theme/' . THEME . '/images/tag_blue.png');
+define('ICON_GREEN_OK', '/view/theme/' . THEME . '/images/green_ok.png');
+define('ICON_RED_X', '/view/theme/' . THEME . '/images/red_x.png');
+define('ICON_DOWNLOAD', '/view/theme/' . THEME . '/images/download_icon.jpg');
+define('ICON_NOTES', '/view/theme/' . THEME . '/images/notes.png');
+define('ICON_PLUS', '/view/theme/' . THEME . '/images/plus.gif');
+define('ICON_MINUS', '/view/theme/' . THEME . '/images/minus.gif');
+define('ICON_EMPTY', '/view/theme/' . THEME . '/images/1x1.gif');
+
+define('QSHAPE_ACTIVE_INCOMING', DIR_STAT . '/active+incoming');
+define('QSHAPE_ACTIVE_INCOMING_SENDER', DIR_STAT . '/active+incoming-sender');
+define('QSHAPE_DEFERRED', DIR_STAT . '/deferred');
+define('QSHAPE_DEFERRED_SENDER', DIR_STAT . '/deferred-sender');
+
+define('CPUSTAT', DIR_STAT . '/cpu.stat');
+define('AD_SYNC_STAT', DIR_STAT . '/adsync.stat');
+define('ARCHIVE_SIZE', DIR_STAT . '/archive.size');
+define('SPHINX_CURRENT_MAIN_INDEX_SIZE', DIR_STAT . '/current_main_index_size');
+define('SPHINX_TOTAL_INDEX_SIZE', DIR_STAT . '/total_index_size');
+define('LOCK_FILE', DIR_LOG . 'lock');
+
+define('SEARCH_HELPER_URL', SITE_URL . 'search-helper.php');
+define('AUDIT_HELPER_URL', SITE_URL . 'audit-helper.php');
+define('BULK_RESTORE_URL', SITE_URL . 'bulkrestore.php');
+define('SAVE_SEARCH_URL', SITE_URL . 'index.php?route=search/save');
+define('LOAD_SAVED_SEARCH_URL', SITE_URL . 'index.php?route=search/load');
+define('SEARCH_TAG_URL', SITE_URL . 'index.php?route=search/tag');
+define('MESSAGE_NOTE_URL', SITE_URL . 'index.php?route=message/note');
+define('GOOGLE_REDIRECT_URL', SITE_URL . 'google.php');
+define('HEALTH_URL', SITE_URL . 'index.php?route=health/health');
+define('HEALTH_WORKER_URL', SITE_URL . 'index.php?route=health/worker');
+
+define('LDAP_TYPE_GENERIC', 'generic_ldap');
+
+define('ATTACHMENT_DUMP_CHECKPOINT', 'attachment_dump_checkpoint');
+
+define('ACTION_ALL', 0);
+define('ACTION_UNKNOWN', 1);
+define('ACTION_LOGIN', 2);
+define('ACTION_LOGIN_FAILED', 3);
+define('ACTION_LOGOUT', 4);
+define('ACTION_VIEW_MESSAGE', 5);
+define('ACTION_VIEW_HEADER', 6);
+define('ACTION_UNAUTHORIZED_VIEW_MESSAGE', 7);
+define('ACTION_RESTORE_MESSAGE', 8);
+define('ACTION_DOWNLOAD_MESSAGE', 9);
+define('ACTION_SEARCH', 10);
+define('ACTION_SAVE_SEARCH', 11);
+define('ACTION_CHANGE_USER_SETTINGS', 12);
+define('ACTION_REMOVE_MESSAGE', 13);
+define('ACTION_UNAUTHORIZED_REMOVE_MESSAGE', 14);
+define('ACTION_DOWNLOAD_ATTACHMENT', 15);
+define('ACTION_UNAUTHORIZED_DOWNLOAD_ATTACHMENT', 16);
+define('ACTION_VIEW_JOURNAL', 17);
+define('ACTION_NOT_SPAM', 18);
+define('ACTION_MARK_AS_PRIVATE', 19);
+define('ACTION_MARK_MESSAGE_FOR_REMOVAL', 20);
+define('ACTION_REJECT_REMOVAL', 21);
+
+$actions = array(
+                  'unknown' => 1,
+                  'login' => 2,
+                  'loginfailed' => 3,
+                  'logout' => 4,
+                  'view' => 5,
+                  'view_header' => 6,
+                  'restore' => 8,
+                  'download' => 9,
+                  'search' => 10,
+                  'save_search' => 11,
+                  'download_attachment' => 15,
+                  'journal' => 17,
+                  'private' => 19,
+                  'marked_for_removal', 20,
+                  'reject_removal', 21
+                );
+
+
+$import_status = array(
+                         0 => 'PENDING',
+                         1 => 'RUNNING',
+                         2 => 'FINISHED',
+                         3 => 'ERROR'
+                 );
+
+
+$counters = array(MEMCACHED_PREFIX . 'rcvd', MEMCACHED_PREFIX . 'virus', MEMCACHED_PREFIX . 'duplicate', MEMCACHED_PREFIX . 'ignore', MEMCACHED_PREFIX . 'counters_last_update');
+
+if(!isset($health_smtp_servers)) {
+   $health_smtp_servers = array( array(PILER_HOST, PILER_PORT, "piler"), array(SMARTHOST, SMARTHOST_PORT, "smarthost") );
+}
+
+if(SPHINX_STRICT_SCHEMA) {
+   define('FROM_TOKEN', '@sender');
+   define('FROMDOMAIN_TOKEN', '@senderdomain');
+   define('TO_TOKEN', '@rcpt');
+   define('TODOMAIN_TOKEN', '@rcptdomain');
+} else {
+   define('FROM_TOKEN', '@from');
+   define('FROMDOMAIN_TOKEN', '@fromdomain');
+   define('TO_TOKEN', '@to');
+   define('TODOMAIN_TOKEN', '@todomain');
+}

configure.in

@@ -24,9 +24,9 @@ AC_SUBST(OBJS)
 AC_SUBST(RUNNING_USER)
 AC_SUBST(SUBDIRS)
 AC_SUBST(MAKE)
-AC_SUBST(mysql_includes)
-AC_SUBST(mysql_libs)
-AC_SUBST(mysql_obj)
+AC_SUBST(sql_includes)
+AC_SUBST(sql_libs)
+AC_SUBST(sql_obj)
 AC_SUBST(libclamav_extra_libs)
 AC_SUBST(id_bin)
 AC_SUBST(iv)
@@ -37,10 +37,10 @@ have_clamd="no"
 
 have_antivirus="no"
 have_mysql="no"
+have_psql="no"
 have_tre="no"
 have_zip="no"
 have_zlib="no"
-have_starttls="no"
 have_tcpwrappers="no"
 have_tweak_sent_time="no"
 
@@ -50,15 +50,18 @@ catppt="no"
 ppthtml="no"
 xls2csv="no"
 unrtf="no"
+tnef="no"
 
+timeout_binary=""
 
 have_static_build="no"
+have_compat_storage_layout="no"
 
 antispam_libs="-lz -lm -ldl -lcrypto -lssl"
 defs=""
 objs=""
 user_obj=""
-mysql_obj=""
+sql_obj=""
 os=`uname -s`
 
 id_bin="id"
@@ -77,12 +80,12 @@ if test -f `which gcc`; then gcc -v 2> aa; GCC_VERSION=`tail -1 aa`; rm -f aa; f
 if test "$os" = "SunOS"; then
    echo "#define CONFIGURE_PARAMS \"Build Date: "`date`'\\n" \\' > $CONFIGURE_PARAMS_FILE
 else
-   echo "#define CONFIGURE_PARAMS \"Build Date: "`date`"\\nldd version: $LDD_VERSION\\ngcc version: $GCC_VERSION\n"\" \\ > $CONFIGURE_PARAMS_FILE
+   echo "#define CONFIGURE_PARAMS \"Build Date: "`date`"\\nldd version: $LDD_VERSION\\ngcc version: $GCC_VERSION\\nOS: "`uname -a`"\n"\" \\ > $CONFIGURE_PARAMS_FILE
 fi
 echo "\"Configure command: ./configure $PARAMS\"" >> $CONFIGURE_PARAMS_FILE
 
 
-SUBDIRS="src etc util init.d test"
+SUBDIRS="src etc util init.d systemd unit_tests webui"
 
 
 dnl static build
@@ -91,6 +94,12 @@ AC_ARG_ENABLE(static-build,
           [  --enable-static-build   build statically linked executables (default: dynamically linked)], have_static_build=$enableval, have_static_build="no")
 
 
+dnl
+
+AC_ARG_ENABLE(compat-layout,
+          [  --enable-compat-layout  support for older storage layout (default: no)], have_compat_storage_layout=$enableval, have_compat_storage_layout="no")
+
+
 dnl clamd
 
 
@@ -106,22 +115,13 @@ AC_ARG_ENABLE(memcached,
         [  --enable-memcached      build memcached support], want_memcached=$enableval, want_memcached="no")
 
 
-AC_ARG_ENABLE(starttls,
-	[  --enable-starttls       build starttls support], have_starttls=$enableval, have_starttls="no")
+dnl tcpwrappers library
 
+AC_CHECK_HEADERS(tcpd.h, have_tcpwrappers=yes, echo "tcpd.h is not found")
+AC_CHECK_LIB([wrap],[hosts_access],[AC_CHECK_LIB(wrap, hosts_access, have_tcpwrappers=yes, echo "libwrap is not found"; have_tcpwrappers=no)],[],[])ac_cv_lib_wrap=ac_cv_lib_wrap_main
 
-AC_ARG_ENABLE(tcpwrappers,
-	[  --enable-tcpwrappers    build tcpwrappers support], want_tcpwrappers=$enableval, want_tcpwrappers="no")
-
-
-if test "$want_tcpwrappers" = "yes"; then
-   AC_CHECK_HEADERS(tcpd.h, have_tcpwrappers=yes, have_tcpwrappers=no)
-   AC_CHECK_LIB([wrap],[hosts_access],[AC_CHECK_LIB(wrap, hosts_access, have_tcpwrappers=yes, echo "libwrap is not found"; have_tcpwrappers=no)],[],[])ac_cv_lib_wrap=ac_cv_lib_wrap_main
-
-   if test "$have_tcpwrappers" = "no"; then
-      echo "can't find either tcpd.h or libwrap";
-      exit 1;
-   fi
+if test "$have_tcpwrappers" = "no"; then
+   echo "can't find either tcpd.h or libwrap. Tcpwrappers support is disabled";
 fi
 
 
@@ -176,24 +176,90 @@ if test "$have_zlib" = "no"; then
 fi
 
 
-AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, yes)
+AC_ARG_WITH(database,
+[  --with-database[[=mysql]]  select the used database, currently mysql only (default: none)],[
+   if test "$withval" != "no" -a "$withval" != "yes"; then
+      DATABASE=$withval
+   fi
 
-if test x$MYSQL_CONFIG = xyes; then
-   have_mysql="yes"
+
+   if test "$withval" = "mysql"; then
+
+      AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, yes)
+
+      if test x$MYSQL_CONFIG = xyes; then
+         have_mysql="yes"
+      fi
+
+      if test "$have_mysql" = "yes"; then
+
+         sql_includes=`mysql_config --cflags`
+         sql_libs=`mysql_config --libs_r`
+         sql_obj="mysql.o"
+
+         AC_CHECK_LIB([guide],[main],[AC_CHECK_LIB(guide, _intel_fast_memset, have_icc_guide=yes, have_icc_guide=no)],[],[])ac_cv_lib_guide=ac_cv_lib_guide_main
+
+      else
+         echo "MySQL support is not found"
+         exit 1;
+      fi
+   fi
+
+
+   if test "$withval" = "mariadb"; then
+
+      AC_CHECK_PROG(MYSQL_CONFIG, mariadb_config, yes)
+
+      if test x$MYSQL_CONFIG = xyes; then
+         have_mysql="yes"
+      fi
+
+      if test "$have_mysql" = "yes"; then
+
+         sql_includes=`mariadb_config --cflags`
+         sql_libs=`mariadb_config --libs_r`
+         sql_obj="mysql.o"
+
+      else
+         echo "mariadb_config is not found, please install \"MariaDB Client Library for C 1.0.0 Stable\", see https://downloads.mariadb.org/client-native/1.0.0/"
+         exit 1;
+      fi
+   fi
+
+
+   if test "$withval" = "psql"; then
+
+      AC_CHECK_PROG(PSQL_CONFIG, pg_config, yes)
+
+      if test x$PSQL_CONFIG = xyes; then
+         have_psql="yes"
+      fi
+
+      if test "$have_psql" = "yes"; then
+
+         sql_includes="-I`pg_config --includedir`"
+         sql_libs="-L`pg_config --libdir`"
+         sql_obj="psql.o"
+
+      else
+         echo "PSQL support is not found"
+         exit 1;
+      fi
+   fi
+
+])
+
+
+dnl timeout binary
+
+if test z`which timeout 2>/dev/null` != "z"; then
+   timeout_binary=`which timeout`
+   AC_DEFINE_UNQUOTED(TIMEOUT_BINARY, "$timeout_binary", [timeout binary])
 fi
 
-if test "$have_mysql" = "yes"; then
-
-   mysql_includes=`mysql_config --cflags`
-   mysql_libs=`mysql_config --libs_r`
-
-   AC_CHECK_LIB([guide],[main],[AC_CHECK_LIB(guide, _intel_fast_memset, have_icc_guide=yes, have_icc_guide=no)],[],[])ac_cv_lib_guide=ac_cv_lib_guide_main
-
-else
-   echo "MySQL support is not found"
-   exit 1;
-fi
 
+read -r version < VERSION
+AC_DEFINE_UNQUOTED(VERSION, "$version")
 
 
 dnl user running piler
@@ -247,16 +313,63 @@ DATADIR=$data_dir
 AC_SUBST(DATADIR)
 AC_DEFINE_UNQUOTED(DATADIR,"$data_dir",[where to look for the data files])
 
-AC_DEFINE_UNQUOTED(VIRUS_TEMPLATE, "$my_prefix/share/clapf/template.virus", [where the virus template is])
-AC_DEFINE_UNQUOTED(ZOMBIE_NET_REGEX, "$my_prefix/share/clapf/zombienets.regex", [where the virus template is])
+
+dnl configure libexec directory
+
+libexec_dir=`echo $libexecdir | grep prefix`
+
+if test -n "$libexec_dir"; then
+    if test "$prefix" = "NONE"
+    then
+        libexec_dir="$ac_default_prefix/libexec"
+    else
+        libexec_dir="$prefix/libexec"
+    fi
+else
+    libexec_dir="$libexecdir"
+fi
+
+LIBEXECDIR=$libexec_dir
+AC_SUBST(LIBEXECDIR)
+AC_DEFINE_UNQUOTED(LIBEXECDIR,"$libexec_dir",[where to look for the piler helpers])
 
 
+dnl configure dataroot directory
+
+dataroot_dir=`echo $datarootdir | grep prefix`
+
+if test -n "$dataroot_dir"; then
+    if test "$prefix" = "NONE"
+    then
+        dataroot_dir="$ac_default_prefix/share"
+    else
+        dataroot_dir="$prefix/share"
+    fi
+else
+    dataroot_dir="$datarootdir"
+fi
+
+
+DATAROOTDIR=$dataroot_dir
+AC_SUBST(DATAROOTDIR)
+AC_DEFINE_UNQUOTED(DATAROOTDIR,"$dataroot_dir",[where to look for the share data files])
+
+
+
+if test "$have_mysql" = "no" && test "$have_psql" = "no"; then
+   echo
+   echo "please specify the used database with --with-database=..."
+   echo
+   exit 1
+fi
 
 
 dnl let us know if we are building on FreeBSD
 
 if test "$os" = "FreeBSD"; then
    defs="$defs -DFREEBSD"
+   antispam_libs="-lz -lm -lcrypto -lssl -liconv"
+   MAKE="gmake"
 fi
 
 if test "$os" = "Linux"; then
@@ -270,11 +383,6 @@ if test "$os" = "SunOS"; then
    if test -x /usr/xpg4/bin/id; then id_bin="/usr/xpg4/bin/id"; fi
 fi
 
-if test `echo $os | grep -c CYGWIN` -eq 1; then
-   defs="$defs -DCYGWIN"
-   AC_DEFINE_UNQUOTED(ZOMBIE_NET_REGEX, "zombienets.regex", [where the virus template is])
-fi
-
 dnl whether we have antivirus support
 
 if test "$have_clamd" = "yes" ; then
@@ -296,6 +404,21 @@ fi
 
 echo
 
+if test "$have_compat_storage_layout" = "yes"; then
+   echo "support for older storage layout: yes"
+   AC_DEFINE_UNQUOTED(HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT, 1, [compat storage layout support])
+fi
+
+
+if test "$have_mysql" = "yes"; then
+   echo "database: mysql"
+fi
+
+if test "$have_psql" = "yes"; then
+   echo "database: psql"
+fi
+
+
 if test "$have_tre" = "yes"; then
    echo "tre library: yes"
    defs="$defs -DHAVE_TRE"
@@ -306,19 +429,8 @@ if test "$have_zip" = "yes"; then
    echo "zip library: yes"
    AC_DEFINE_UNQUOTED(HAVE_ZIP, 1, [libzip support])
    antispam_libs="$antispam_libs -lzip"
-fi
-
-if test "$have_starttls" = "yes"; then
-   echo "starttls support: yes"
-   AC_DEFINE_UNQUOTED(HAVE_STARTTLS, 1, [starttls support])
-fi
-
-if test "$have_tcpwrappers" = "yes"; then
-   echo "tcpwrappers support: yes"
-   AC_DEFINE_UNQUOTED(HAVE_LIBWRAP, 1, [tcpwrappers support])
-   antispam_libs="$antispam_libs -lwrap -lnsl"
 else
-   echo "tcpwrappers support: no"
+   echo "zip library: no"
 fi
 
 
@@ -341,16 +453,18 @@ if test "$have_mysql" = "yes"; then
 fi
 
 if test "$have_icc_guide" = "yes" && test "$have_mysql" = "yes"; then
-   mysql_libs="$mysql_libs -lguide"
+   sql_libs="$sql_libs -lguide"
 fi
 
+if test "$have_psql" = "yes"; then
+   defs="$defs -DNEED_PSQL"
+fi
 
 if test z`which pdftotext 2>/dev/null` != "z"; then
    pdftotext=`which pdftotext`
    AC_DEFINE_UNQUOTED(HAVE_PDFTOTEXT, "$pdftotext", [path to pdftotext])
 fi
 
-
 if test z`which catdoc 2>/dev/null` != "z"; then
    catdoc=`which catdoc`
    AC_DEFINE_UNQUOTED(HAVE_CATDOC, "$catdoc", [path to catdoc])
@@ -381,6 +495,12 @@ if test z`which unrtf 2>/dev/null` != "z"; then
 fi
 
 
+if test z`which tnef 2>/dev/null` != "z"; then
+   tnef=`which tnef`
+   AC_DEFINE_UNQUOTED(HAVE_TNEF, "$tnef", [path to tnef])
+fi
+
+
 if test "$have_tweak_sent_time" = "yes"; then
    AC_DEFINE_UNQUOTED(HAVE_TWEAK_SENT_TIME, 1, [tweak sent time])
 fi
@@ -394,25 +514,35 @@ echo "catppt: $catppt"
 echo "ppthtml: $ppthtml"
 echo "xls2csv: $xls2csv"
 echo "unrtf: $unrtf"
+echo "tnef: $tnef"
 
 
 id -u $RUNNING_USER 2>/dev/null 1>/dev/null
 if test $? -eq 1; then echo "the user \"$RUNNING_USER\" does not exists, please create it, first with adduser..."; exit 1; fi
 
-iv=`util/gen-iv.pl`
-
 echo; echo
 
-CFLAGS="$static -O2 -Wall -g"
-LIBS="$antispam_libs $sunos_libs "
-OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o list.o parser.o parser_utils.o rules.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o imap.o pop3.o extract.o mydomains.o mysql.o $objs"
+gcc_version="$(gcc -dumpversion)"
+extra_cflags=""
+if [[ "${gcc_version:0:1}" -gt 6 ]]; then
+   extra_cflags="-Wimplicit-fallthrough=2"
+fi
 
-AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile test/Makefile contrib/imap/Makefile])
+CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra $extra_cflags -Wuninitialized -Wimplicit-fallthrough=2 -Wno-format-truncation -g"
+LIBS="$antispam_libs $sunos_libs "
+OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o screen.o $objs"
+
+AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile])
 AC_OUTPUT
 
+echo
+echo
+echo "IMPORTANT! If you upgrade, be sure to read https://www.mailpiler.org/upgrade-instructions/"
+echo
+echo
 
 echo
-echo
-echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/en/upgrade.html"
+echo "Did you know that piler has an enterprise edition as well?"
+echo "Check out what it can do for you at https://mailpiler.com/#features"
 echo
 echo

contrib/export-attachments/export-attachments.php

@@ -0,0 +1,103 @@
+#!/usr/bin/php
+<?php
+
+define('EXPORT_LOCK_FILE', '/var/piler/tmp/export-attachments.lock');
+
+$webuidir = "";
+
+$outdir = '';
+
+
+$opts = 'hw:d:';
+$lopts = [
+   'webui:',
+   'dir:',
+   'help'
+];
+
+
+if($options = getopt($opts, $lopts)) {
+   if(isset($options['webui'])) { $webuidir = $options['webui']; }
+   if(isset($options['dir'])) { $outdir = $options['dir']; }
+   if(isset($options['help'])) { usage(); }
+}
+
+
+if($webuidir == '' || $outdir == '') { usage(); }
+
+
+
+require_once("$webuidir/config.php");
+
+require(DIR_SYSTEM . "/startup.php");
+
+$request = new Request();
+Registry::set("request", $request);
+
+
+$start = NULL;
+
+$loader = new Loader();
+Registry::set('load', $loader);
+
+$loader->load->model('domain/domain');
+$loader->load->model('search/search');
+$loader->load->model('search/message');
+$loader->load->model('message/attachment');
+
+
+$db = new DB(DB_DRIVER, DB_HOSTNAME, DB_USERNAME, DB_PASSWORD, DB_DATABASE, DB_PREFIX);
+Registry::set('db', $db);
+
+Registry::set('auditor_user', 1);
+
+openlog("export-attachments", LOG_PID, LOG_MAIL);
+
+$fp = fopen(EXPORT_LOCK_FILE, "w");
+if(!flock($fp, LOCK_EX)) {
+   syslog(LOG_INFO, "WARN: couldn't get a lock on " . EXPORT_LOCK_FILE);
+   exit;
+}
+
+
+$domain = new ModelDomainDomain();
+$attachment = new ModelMessageAttachment();
+$message = new ModelSearchMessage();
+
+$domains = $domain->get_mapped_domains();
+
+$last_id = $attachment->get_last_attachment_id();
+$start_id = $attachment->get_checkpoint();
+
+syslog(LOG_INFO, "start: $start_id, limit: $last_id");
+
+
+for($i=$start_id; $i<$last_id; $i++) {
+   $a = $attachment->get_attachment_by_id($i);
+   $m = $message->get_message_addresses_by_piler_id($a['piler_id'], $domains);
+
+   $attachment->dump_attachment($outdir, "out", $m['sender'], $i, $a);
+
+   foreach($m['rcpt'] as $rcpt) {
+      $attachment->dump_attachment($outdir, "in", $rcpt, $i, $a);
+   }
+
+   if($i % 100 == 0) { $attachment->update_checkpoint($i); }
+}
+
+$attachment->update_checkpoint($i);
+
+// Release lock
+flock($fp, LOCK_UN);
+fclose($fp);
+
+
+
+function usage() {
+   print "\nUsage: " . __FILE__ . "\n\n";
+   print "\t--webui <path to webui directory>\n";
+   print "\t--dir <basedir to write attachments>\n";
+   print "\t--help\n\n";
+
+   exit;
+}

contrib/installer/focal.sh

@@ -0,0 +1,257 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+
+MY_IP="1.2.3.4"
+PILER_HOSTNAME="${PILER_HOSTNAME:-archive.example.com}"
+MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
+MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
+SERVER_ID="${SERVER_ID:-0}"
+USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
+SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
+PHP_FPM_SOCKET="/var/run/php/php7.4-fpm.sock"
+
+MYSQL_HOSTNAME="localhost"
+MYSQL_DATABASE="piler"
+MYSQL_USERNAME="piler"
+
+SPHINX_TARGZ="sphinx-3.3.1-bin.tar.gz"
+DOWNLOAD_URL="https://download.mailpiler.com"
+PILER_DEB="piler_1.3.12-focal-eb2b22b_amd64.deb"
+PILER_USER="piler"
+CONFIG_SITE_PHP="/etc/piler/config-site.php"
+
+export DEBIAN_FRONTEND=noninteractive
+
+install_prerequisites() {
+   apt-get update
+   apt-get -y --no-install-recommends install \
+      wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \
+      php7.4-gd php7.4-curl php7.4-xml ca-certificates zip catdoc unrtf poppler-utils nginx tnef libzip5 \
+      libtre5 libwrap0 cron libmariadb-dev python3 python3-mysqldb libmariadb-dev mariadb-client-core-10.3 \
+      mariadb-server-10.3
+
+   wget -q -O "/tmp/${SPHINX_TARGZ}" "${DOWNLOAD_URL}/generic-local/${SPHINX_TARGZ}"
+   tar -C / -zxvf "/tmp/${SPHINX_TARGZ}"
+   wget -O /usr/local/bin/traefik "${DOWNLOAD_URL}/generic-local/traefik"
+   chmod +x /usr/local/bin/traefik
+   setcap cap_net_bind_service+ep /usr/local/bin/traefik
+}
+
+
+fix_mysql_settings() {
+   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
+[mysqld]
+
+innodb_buffer_pool_size=512M
+innodb_flush_log_at_trx_commit=1
+innodb_log_buffer_size=64M
+innodb_log_file_size=64M
+innodb_read_io_threads=4
+innodb_write_io_threads=4
+innodb_log_files_in_group=2
+
+innodb_file_per_table
+PILER_CNF
+}
+
+start_mysql() {
+   fix_mysql_settings
+
+   service mysql restart
+   mysqladmin -u root password "${MYSQL_ROOT_PASSWORD}"
+}
+
+
+install_piler() {
+   wget "${DOWNLOAD_URL}/piler/${PILER_DEB}"
+   dpkg -i "$PILER_DEB"
+   rm -f "$PILER_DEB"
+
+   crontab -u "$PILER_USER" /usr/share/piler/piler.cron
+
+   rm -f "$PILER_DEB" /etc/nginx/sites-enabled/default
+   ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf
+
+   touch /var/piler/.bash_history
+   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
+}
+
+
+create_my_cnf() {
+   local user=$1
+   local password=$2
+   local my_cnf=$3
+
+   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
+   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
+   chown $PILER_USER:$PILER_USER "$my_cnf"
+   chmod 600 "$my_cnf"
+}
+
+
+fix_config_site_php() {
+   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"
+
+   {
+      echo "\$config['SERVER_ID'] = $SERVER_ID;"
+      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
+      echo "\$config['SPHINX_VERSION'] = 331;"
+   } >> "$CONFIG_SITE_PHP"
+
+   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
+      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
+   fi
+
+   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
+}
+
+
+init_db() {
+   sed -e "s%MYSQL_HOSTNAME%$MYSQL_HOSTNAME%g" -e "s%MYSQL_DATABASE%$MYSQL_DATABASE%g" -e "s%MYSQL_USERNAME%$MYSQL_USERNAME%g" -e "s%MYSQL_PASSWORD%$MYSQL_PILER_PASSWORD%g" \
+      /usr/share/piler/db-mysql-root.sql.in | mysql --defaults-file=/etc/piler/.my.cnf-root -h $MYSQL_HOSTNAME
+   mysql --defaults-file=/etc/piler/.my.cnf -h $MYSQL_HOSTNAME $MYSQL_DATABASE < /usr/share/piler/db-mysql.sql
+}
+
+
+add_systemd_services() {
+   pushd /etc/systemd/system
+
+   ln -sf /usr/libexec/piler/piler.service .
+   ln -sf /usr/libexec/piler/piler-smtp.service .
+   ln -sf /usr/libexec/piler/pilersearch.service .
+
+   popd
+
+   systemctl daemon-reload
+
+   systemctl enable piler
+   systemctl enable piler-smtp
+   systemctl enable pilersearch
+   systemctl enable traefik
+}
+
+
+fix_configs() {
+   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
+      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
+      nginx -t
+      nginx -s reload
+
+      sed -i 's/server {/server {\n\tlisten 127.0.0.1:80;/' /etc/piler/piler-nginx.conf
+      systemctl stop nginx
+      sleep 5
+      systemctl start nginx
+   fi
+
+   if [[ ! -f /etc/piler/piler.conf ]]; then
+      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.example.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > /etc/piler/piler.conf
+      chmod 600 /etc/piler/piler.conf
+      chown $PILER_USER:$PILER_USER /etc/piler/piler.conf
+   fi
+
+   sed -i 's/tls_enable=0/tls_enable=1/g' /etc/piler/piler.conf
+
+   sed -i -e "s/MYSQL_HOSTNAME/${MYSQL_HOSTNAME}/g" \
+          -e "s/MYSQL_DATABASE/${MYSQL_DATABASE}/g" \
+          -e "s/MYSQL_USERNAME/${MYSQL_USERNAME}/g" \
+          -e "s/MYSQL_PASSWORD/${MYSQL_PILER_PASSWORD}/g" \
+          /etc/piler/sphinx.conf
+}
+
+setup_traefik() {
+   wget -O /etc/systemd/system/traefik.service "${DOWNLOAD_URL}/generic-local/traefik.service"
+
+   mkdir -p /usr/local/etc/traefik
+   touch /usr/local/etc/traefik/acme.json
+   chmod 600 /usr/local/etc/traefik/acme.json
+   chown www-data:www-data /usr/local/etc/traefik/acme.json
+
+   cat > /usr/local/etc/traefik/traefik.yaml << TRAEFIK
+log:
+  level: INFO
+
+entryPoints:
+  web:
+    address: "$MY_IP:80"
+    http:
+      redirections:
+        entryPoint:
+          to: websecure
+          scheme: https
+          permanent: true
+  websecure:
+    address: "$MY_IP:443"
+
+providers:
+  file:
+    filename: "/usr/local/etc/traefik/traefik.yaml"
+
+certificatesResolvers:
+  le:
+    acme:
+      storage: "/usr/local/etc/traefik/acme.json"
+      email: admin@$PILER_HOSTNAME
+      httpChallenge:
+        entryPoint: web
+
+tls:
+  options:
+    default:
+      minVersion: VersionTLS13
+
+http:
+  middlewares:
+    piler_headers:
+      headers:
+        customResponseHeaders:
+          Server: ""
+          Strict-Transport-Security: "max-age=31536000"
+          X-Content-Type-Optionsi: "nosniff"
+          Referrer-Policy: "same-origin"
+  routers:
+    master:
+      rule: "Host(\`$PILER_HOSTNAME\`)"
+      service: www
+      middlewares:
+        - "piler_headers"
+      tls:
+        certResolver: le
+  services:
+    www:
+      loadBalancer:
+        servers:
+        - url: "http://127.0.0.1:80/"
+TRAEFIK
+}
+
+
+install_prerequisites
+
+start_mysql
+
+install_piler
+
+create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
+create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf
+
+fix_config_site_php
+
+setup_traefik
+
+add_systemd_services
+
+fix_configs
+init_db
+
+su -c "indexer --all -c /etc/piler/sphinx.conf" $PILER_USER
+
+[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler
+
+systemctl start pilersearch
+systemctl start piler
+systemctl start piler-smtp
+systemctl start traefik

contrib/installer/jammy.sh

@@ -0,0 +1,258 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+
+MY_IP="1.2.3.4"
+PILER_HOSTNAME="${PILER_HOSTNAME:-archive.example.com}"
+MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
+MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
+SERVER_ID="${SERVER_ID:-0}"
+USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
+SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
+PHP_FPM_SOCKET="/var/run/php/php8.1-fpm.sock"
+
+MYSQL_HOSTNAME="localhost"
+MYSQL_DATABASE="piler"
+MYSQL_USERNAME="piler"
+
+DOWNLOAD_URL="https://download.mailpiler.com"
+PILER_DEB="piler_1.4.4-jammy-0573c6da_amd64.deb"
+PILER_USER="piler"
+PILER_CONF="/etc/piler/piler.conf"
+CONFIG_SITE_PHP="/etc/piler/config-site.php"
+SEARCHCFG="/etc/piler/manticore.conf"
+
+TRAEFIK_VERSION="v2.11.0"
+ARCH="amd64"
+
+export DEBIAN_FRONTEND=noninteractive
+
+install_prerequisites() {
+   apt-get update
+   apt-get -y --no-install-recommends install \
+      wget rsyslog openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql \
+      php8.1-fpm php8.1-zip php8.1-ldap php8.1-gd php8.1-curl php8.1-xml \
+      php8.1-mbstring ca-certificates zip catdoc unrtf poppler-utils \
+      nginx tnef libzip4 libtre5 libwrap0 cron libmariadb-dev python3 \
+      python3-mysqldb libmariadb-dev mariadb-client-core-10.6 \
+      mariadb-server-10.6
+
+   wget "https://github.com/traefik/traefik/releases/download/${TRAEFIK_VERSION}/traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz"
+   tar zxvf "traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz" traefik
+   chown root:root traefik
+   chmod 755 traefik
+   mv traefik /usr/local/bin
+   setcap cap_net_bind_service+ep /usr/local/bin/traefik
+
+   wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
+   dpkg -i manticore-repo.noarch.deb && \
+   apt-get update && \
+   apt-get install -y manticore manticore-columnar-lib manticore-extra && \
+   rm -f manticore-repo.noarch.deb
+   systemctl stop manticore
+}
+
+
+fix_mysql_settings() {
+   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
+[mysqld]
+
+innodb_buffer_pool_size=512M
+innodb_flush_log_at_trx_commit=1
+innodb_log_buffer_size=64M
+innodb_log_file_size=64M
+innodb_read_io_threads=4
+innodb_write_io_threads=4
+innodb_log_files_in_group=2
+
+innodb_file_per_table
+PILER_CNF
+}
+
+start_mysql() {
+   fix_mysql_settings
+
+   service mysql restart
+   mysqladmin -u root password "${MYSQL_ROOT_PASSWORD}"
+}
+
+
+install_piler() {
+   wget "https://bitbucket.org/jsuto/piler/downloads/${PILER_DEB}"
+   dpkg -i "$PILER_DEB"
+   rm -f "$PILER_DEB"
+
+   sed -i 's/.*indexer.*/###/g' /usr/share/piler/piler.cron
+   crontab -u "$PILER_USER" /usr/share/piler/piler.cron
+
+   rm -f "$PILER_DEB" /etc/nginx/sites-enabled/default
+   ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf
+
+   touch /var/piler/.bash_history
+   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
+}
+
+
+create_my_cnf() {
+   local user=$1
+   local password=$2
+   local my_cnf=$3
+
+   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
+   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
+   chown $PILER_USER:$PILER_USER "$my_cnf"
+   chmod 600 "$my_cnf"
+}
+
+
+fix_config_site_php() {
+   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"
+
+   {
+      echo "\$config['SERVER_ID'] = $SERVER_ID;"
+      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
+   } >> "$CONFIG_SITE_PHP"
+
+   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
+      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
+   fi
+
+   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
+   echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
+}
+
+
+init_db() {
+   sed -e "s%MYSQL_HOSTNAME%$MYSQL_HOSTNAME%g" -e "s%MYSQL_DATABASE%$MYSQL_DATABASE%g" -e "s%MYSQL_USERNAME%$MYSQL_USERNAME%g" -e "s%MYSQL_PASSWORD%$MYSQL_PILER_PASSWORD%g" \
+      /usr/share/piler/db-mysql-root.sql.in | mysql --defaults-file=/etc/piler/.my.cnf-root -h $MYSQL_HOSTNAME
+   mysql --defaults-file=/etc/piler/.my.cnf -h $MYSQL_HOSTNAME $MYSQL_DATABASE < /usr/share/piler/db-mysql.sql
+}
+
+
+add_systemd_services() {
+   pushd /etc/systemd/system
+
+   ln -sf /usr/libexec/piler/piler.service .
+   ln -sf /usr/libexec/piler/piler-smtp.service .
+   ln -sf /usr/libexec/piler/pilersearch.service .
+
+   popd
+
+   systemctl daemon-reload
+
+   systemctl enable piler
+   systemctl enable piler-smtp
+   systemctl enable pilersearch
+   systemctl enable traefik
+}
+
+
+fix_configs() {
+   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
+      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
+      nginx -t
+      nginx -s reload
+
+      sed -i 's/server {/server {\n\tlisten 127.0.0.1:80;/' /etc/piler/piler-nginx.conf
+      systemctl stop nginx
+      sleep 5
+      systemctl start nginx
+   fi
+
+   if [[ ! -f "$PILER_CONF" ]]; then
+      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.example.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > "$PILER_CONF"
+      chmod 600 "$PILER_CONF"
+      chown $PILER_USER:$PILER_USER "$PILER_CONF"
+   fi
+
+   sed -i 's/tls_enable=0/tls_enable=1/g' "$PILER_CONF"
+   sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
+
+   sed -i "s/define('RT.*/define('RT', 1);/" "$SEARCHCFG"
+}
+
+setup_traefik() {
+   wget -O /etc/systemd/system/traefik.service "${DOWNLOAD_URL}/generic-local/traefik.service"
+
+   mkdir -p /usr/local/etc/traefik
+   touch /usr/local/etc/traefik/acme.json
+   chmod 600 /usr/local/etc/traefik/acme.json
+   chown www-data:www-data /usr/local/etc/traefik/acme.json
+
+   cat > /usr/local/etc/traefik/traefik.yaml << TRAEFIK
+log:
+  level: INFO
+
+entryPoints:
+  websecure:
+    address: "$MY_IP:443"
+
+providers:
+  file:
+    filename: "/usr/local/etc/traefik/traefik.yaml"
+
+certificatesResolvers:
+  le:
+    acme:
+      storage: "/usr/local/etc/traefik/acme.json"
+      email: admin@$PILER_HOSTNAME
+      tlsChallenge: {}
+
+tls:
+  options:
+    default:
+      minVersion: VersionTLS13
+
+http:
+  middlewares:
+    piler_headers:
+      headers:
+        customResponseHeaders:
+          Server: ""
+          Strict-Transport-Security: "max-age=31536000"
+          X-Content-Type-Optionsi: "nosniff"
+          Referrer-Policy: "same-origin"
+  routers:
+    master:
+      rule: "Host(\`$PILER_HOSTNAME\`)"
+      service: www
+      middlewares:
+        - "piler_headers"
+      tls:
+        certResolver: le
+  services:
+    www:
+      loadBalancer:
+        servers:
+        - url: "http://127.0.0.1:80/"
+TRAEFIK
+}
+
+
+install_prerequisites
+
+start_mysql
+
+install_piler
+
+create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
+create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf
+
+fix_config_site_php
+
+setup_traefik
+
+add_systemd_services
+
+fix_configs
+init_db
+
+[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler
+
+systemctl start pilersearch
+systemctl start piler
+systemctl start piler-smtp
+systemctl start traefik

contrib/milter/pilter.pl

@@ -0,0 +1,133 @@
+#!/usr/bin/perl
+
+use strict;
+
+use Carp qw(verbose);
+use Sendmail::PMilter qw(:all);
+use Data::Dumper;
+use Sys::Syslog;
+use Sys::Syslog qw(:DEFAULT setlogsock);
+use Time::HiRes qw(gettimeofday);
+use Getopt::Long;
+
+my $savedir = "/var/piler/imap";
+my $username = 'piler';
+my $conn = 'inet:33333@127.0.0.1';
+
+my $miltername = 'pilter';
+my $priority = "mail|info";
+my $fname = '';
+my $msglen = 0;
+my $newdir = "new";
+my $curdir = "cur";
+my $newname;
+my $curname;
+my $messageid;
+my $help = 0;
+
+my $opts = GetOptions(
+                     "conn=s" => \$conn,
+                     "username=s" => \$username,
+                     "savedir=s" => \$savedir,
+                     "h" => \$help,
+                     "help" => \$help
+                    );
+
+if($help == 1) { die("usage: $0 --conn inet:33333\@127.0.0.1 --username piler --savedir /var/piler/imap"); }
+
+chdir $savedir || die("cannot chdir to $savedir");
+
+my %cbs;
+for my $cb (qw(close connect helo abort envfrom envrcpt header eoh body eom)) {
+	$cbs{$cb} = sub {
+		my $ctx = shift;
+                my ($seconds, $microseconds);
+
+                ###if($cb eq "connect") {
+                if($cb eq "envrcpt") {
+                   ($seconds, $microseconds) = gettimeofday;
+                   $fname = $seconds . "-" . $microseconds . "-" . $$ . "-" . &get_random_name;
+
+                   ###syslog $priority, "fname=$fname";
+
+                   if(! -d $newdir) { mkdir $newdir, 0700; }
+
+                   $newname = $newdir . "/" . $fname;
+                   $curname = $curdir . "/" . $fname;
+
+                   $msglen = 0;
+                   $messageid = '';
+
+                   if(!open(F, ">$newname")) { return SMFIS_TEMPFAIL; }
+                }
+
+                elsif($cb eq "header") {
+                   $msglen += length ( @_[0] . ": " . @_[1] . "\n" );
+                   print F @_[0] . ": " . @_[1] . "\n";
+
+                   if(@_[0] =~ /^message-id$/i) { $messageid = @_[1]; }
+                }
+
+                elsif($cb eq "eoh") {
+                   print F "\n";
+                }
+
+                elsif($cb eq "body") {
+                   $msglen += @_[1];
+                   print F @_[0];
+                }
+
+                elsif($cb eq "eom") {
+                   close F;
+
+                   if(! -d $curdir) { mkdir $curdir, 0755; }
+
+                   rename $newname, $curname;
+
+                   syslog $priority, "message-id=$messageid, fname=$fname, size=$msglen";
+
+                }
+
+                #if ($cb =~ /^(connect|help|envfrom|envrcpt)$/) {
+                #       print Dumper($ctx->{symbols})."\n";
+                #}
+
+		SMFIS_CONTINUE;
+	}
+}
+
+openlog($miltername, 'pid', 'mail');
+
+$< = $> = getpwnam $username;
+
+syslog $priority, "$miltername starting";
+
+
+my $milter = new Sendmail::PMilter;
+$milter->setconn($conn);
+
+$milter->register($miltername, \%cbs, SMFI_CURR_ACTS);
+
+my $dispatcher = Sendmail::PMilter::prefork_dispatcher(
+	max_children => 5,
+	max_requests_per_child => 100,
+);
+
+$milter->set_dispatcher($dispatcher);
+
+$milter->main();
+
+
+
+sub get_random_name {
+   my @chars = ('A'..'Z', 0..9);
+   my $s = '';
+
+   srand;
+
+   $s .= $chars[rand @chars] for 1..8;
+
+   return $s;
+}
+
+

contrib/pop3/batch-import-without-removing.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+counter_file="pop3-position"
+COUNTER=0
+STEPS=10
+
+export PATH=$PATH:/usr/bin:/usr/local/bin
+
+if [ ! -f $counter_file ]; then COUNTER=1; else COUNTER=`cat $counter_file`; fi
+
+
+pilerimport -K pop3.youromain.com -u username -p password -s `cat $counter_file` -b $STEPS
+
+
+COUNTER=`expr $COUNTER + $STEPS`
+
+printf "%d" "$COUNTER" > $counter_file

contrib/proxmox-lxc/README

@@ -0,0 +1,9 @@
+You nedd proxmox with lxc
+
+Fill out conf file first and provide public key
+
+It automatically
+- downloads the latest debian template
+- provides a container
+- installs piler until the post installation step
+- gives a self signed certificate

contrib/proxmox-lxc/create_lxc.sh

@@ -0,0 +1,195 @@
+#!/bin/bash
+
+# This script wil create and fire up a standard debian buster lxc container on your proxmox pve.
+# The Script will look for the next free lxc number and take the next free and use it. So take
+# care that behind your last number is place for it. 
+
+#### SOME VARIABLES TO ADJUST ####
+
+# Storage with templates
+LXC_TMP="local"
+
+# Size and pool of rootfs / in GB
+SIZ_ROT="100"
+S_ROT_P="local-zfs"
+
+# Size and pool of Filestorage in GB will mounted to /share
+SIZ_FIL="100"
+S_FIL_P="local-zfs"
+
+#Weather or not (1 and 0) the container will createt as unpriviliged LXC
+LXC_UNP="1"
+
+# Size of the RAM assigned to the LXC
+LXC_MEM="1024"
+
+# Size of the SWAP assigned to the LXC
+LXC_SWA="1024"
+
+# The hostname (eq. zamba1 or mailpiler1)
+LXC_HOST="zamba"
+
+# The domainname (searchdomain /etc/resolf.conf & hosts)
+LXC_SDN="zmb.local"
+
+# IP-address and subnet
+LXC_IP="10.10.80.20/24"
+
+# Gateway
+LXC_GW="10.10.80.10"
+
+# DNS-server and here shoud be your AD-DC
+LXC_DNS="10.10.80.10"
+
+# Networkbridge for this machine
+LXC_BRD="vmbr80"
+
+# root password - take care to delete from this file
+LXC_PWD="MYPASSWD"
+
+LXC_KEY="ssh-rsa xxxxxxxx"
+
+############### Zamba-Server-Section ###############
+
+# Domain Entries to samba/smb.conf. Will be also uses for samba domain-provisioning when zmb-pdc will choosen.
+ZMB_REA="ZMB.LOCAL"
+ZMB_DOM="ZMB"
+
+# THE Domain-Admin and passwd for zamba-install
+ZMB_ADA="Administrator"
+ZMB_APW="MYPASSWORD"
+
+############### Mailpiler-Section ###############
+
+# The FQDN vor the Hostname. This must be exactly the same like the LXC_HOST / LXC_SDN at section above.
+PILER_DOM="piler.zmb.rocks"
+SMARTHOST="10.10.80.20"
+PILER_VER="1.3.10"
+SPHINX_VER="3.3.1"
+PHP_VER="7.4"
+
+############### Matrix-Section ###############
+
+# The FQDN vor the Hostname. This should be the same like the LXC_HOST / LXC_SDN at section above.
+MRX_DOM="matrix.zmb.rocks"
+ELE_DOM="element.zmb.rocks"
+ELE_VER="v1.7.21"
+JIT_DOM="meet.zmb.rocks"
+
+#################################
+
+# CHeck is the newest template available, else download it.
+
+DEB_LOC=$(pveam list $LXC_TMP | grep debian-10-standard | cut -d'_' -f2)
+
+DEB_REP=$(pveam available --section system | grep debian-10-standard | cut -d'_' -f2)
+
+if [[ $DEB_LOC == $DEB_REP ]];
+then
+  echo "Newest Version of Debian 10 Standard $DEP_REP exists.";
+else
+  echo "Will now download newest Debian 10 Standard $DEP_REP.";
+  pveam download $LXC_TMP debian-10-standard_$DEB_REP\_amd64.tar.gz
+fi
+
+# Get next free LXC-number
+LXC_LST=$( lxc-ls | egrep -o '.{1,5}$' )
+LXC_CHK=$((LXC_LST+1));
+
+if  [ $LXC_CHK -lt 100 ] || [ -f /etc/pve/qemu-server/$LXC_CHK.conf ]; then
+  LXC_NBR=$(pvesh get /cluster/nextid);
+else
+  LXC_NBR=$LXC_CHK;
+fi
+
+echo "Will now create LXC Container $LXC_NBR!";
+
+# Create the container
+pct create $LXC_NBR -unprivileged $LXC_UNP $LXC_TMP:vztmpl/debian-10-standard_$DEB_REP\_amd64.tar.gz -rootfs $S_ROT_P:$SIZ_ROT;
+sleep 2;
+
+pct set $LXC_NBR -memory $LXC_MEM -swap $LXC_SWA -hostname $LXC_HOST \-nameserver $LXC_DNS -searchdomain $LXC_SDN -onboot 1 -timezone Europe/Berlin -net0 name=eth0,bridge=$LXC_BRD,firewall=1,gw=$LXC_GW,ip=$LXC_IP,type=veth;
+sleep 2;
+
+PS3="Select the Server-Function: "
+
+select opt in just_lxc zmb-standalone zmb-member zmb-pdc mailpiler matrix quit; do
+  case $opt in
+    just_lxc)
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      echo "Should be ready!"
+      break
+      ;;
+    zmb-standalone)
+      break
+      ;;
+    zmb-member)
+      echo "Make some additions to LXC for AD-Member-Server!"
+      pct set $LXC_NBR -mp0 $S_FIL_P:$SIZ_FIL,mp=/tank
+      sleep 2;
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      cp /root/zmb_mem.orig /root/zmb_mem.sh
+      sed -i "s|#ZMB_VAR|#ZMB_VAR\nZMB_REA='$ZMB_REA'\nZMB_DOM='$ZMB_DOM'\nZMB_ADA='$ZMB_ADA'\nZMB_APW='$ZMB_APW'|" /root/zmb_mem.sh
+      pct push $LXC_NBR /root/zmb_mem.sh /root/zmb_mem.sh
+      echo "Install zamba as AD-Member-Server!"
+      lxc-attach -n$LXC_NBR bash /root/zmb_mem.sh
+      break
+      ;;
+    zmb-pdc)
+      break
+      ;;
+    mailpiler)
+      echo "Make some additions to LXC for Mailpiler!"
+      pct set $LXC_NBR -features nesting=1
+      sleep 2;
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      cp /root/mailpiler.orig /root/mailpiler.sh
+      sed -i "s|#PILER_VAR|#PILER_VAR\nPILER_DOM='$PILER_DOM'\nSMARTHOST='$SMARTHOST'\nPILER_VER='$PILER_VER'\nSPHINX_VER='$SPHINX_VER'\nPHP_VER='$PHP_VER'|" /root/mailpiler.sh
+      pct push $LXC_NBR /root/mailpiler.sh /root/mailpiler.sh
+      echo "Install Mailpiler mailarchiv!"
+      lxc-attach -n$LXC_NBR bash mailpiler.sh
+      break
+      ;;
+    matrix)
+      echo "Make some additions to LXC for Matrix!"
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      cp /root/matrix.orig /root/matrix.sh
+      sed -i "s|#MATRIX_VAR|#Matrix_VAR\nMRX_DOM='$MRX_DOM'\nELE_DOM='$ELE_DOM'\nELE_VER='$ELE_VER'\nJIT_DOM='$JIT_DOM'|" /root/matrix.sh
+      pct push $LXC_NBR /root/matrix.sh /root/matrix.sh
+      echo "Install Matrix Chatserver!"
+      lxc-attach -n$LXC_NBR bash matrix.sh
+      break
+      ;;
+    quit)
+      break
+      ;;
+    *)
+      echo "Invalid option!"
+      ;;
+    esac
+done
+

contrib/proxmox-lxc/mailpiler.orig

@@ -0,0 +1,179 @@
+#!/bin/bash
+
+#Variables will be filled in from the mainscript:
+
+#PILER_VAR
+
+
+HOSTNAME=$(hostname -f)
+
+echo "Ensure your Hostname is set to your Piler FQDN!"
+
+echo $HOSTNAME
+
+if 
+    [ "$HOSTNAME" != "$PILER_DOM" ]
+then
+        echo "Hostname doesn't match Piler_Domain! Check install.sh, /etc/hosts, /etc/hostname." && exit
+else
+        echo "Hostname matches PILER_DOMAIN, so starting installation."
+fi
+
+apt install -y gpg apt-transport-https lsb-release
+
+wget -q https://packages.sury.org/php/apt.gpg -O- | apt-key add -
+echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" | tee /etc/apt/sources.list.d/php.list
+
+apt update && apt full-upgrade -y
+
+apt install -y mc sysstat build-essential libwrap0-dev libpst-dev tnef libytnef0-dev unrtf catdoc libtre-dev tre-agrep poppler-utils libzip-dev unixodbc libpq5 software-properties-common libpoppler-dev openssl libssl-dev memcached telnet nginx mariadb-server default-libmysqlclient-dev python-mysqldb gcc libwrap0 libzip4 latex2rtf latex2html catdoc tnef libpq5 zipcmp zipmerge ziptool libsodium23
+
+apt update && apt install -y php$PHP_VER-{fpm,common,ldap,mysql,cli,opcache,phpdbg,gd,memcache,json,readline,zip}
+
+apt purge -y postfix
+
+cat > /etc/mysql/conf.d/mailpiler.conf <<EOF
+innodb_buffer_pool_size=256M
+innodb_flush_log_at_trx_commit=1
+innodb_log_buffer_size=64M
+innodb_log_file_size=16M
+query_cache_size=0
+query_cache_type=0
+query_cache_limit=2M
+EOF
+
+systemctl restart mariadb
+
+cd /tmp
+wget https://download.mailpiler.com/generic-local/sphinx-$SPHINX_VER-bin.tar.gz
+tar -xvzf sphinx-$SPHINX_VER-bin.tar.gz -C /
+
+groupadd piler
+useradd -g piler -m -s /bin/bash -d /var/piler piler
+usermod -L piler
+chmod 755 /var/piler
+
+wget https://bitbucket.org/jsuto/piler/downloads/piler-$PILER_VER.tar.gz
+tar -xvzf piler-$PILER_VER.tar.gz
+cd piler-$PILER_VER/
+./configure --localstatedir=/var --with-database=mysql --enable-tcpwrappers --enable-memcached
+make
+make install
+ldconfig
+
+cp util/postinstall.sh util/postinstall.sh.bak
+sed -i "s/   SMARTHOST=.*/   SMARTHOST="\"$SMARTHOST\""/" util/postinstall.sh
+sed -i 's/   WWWGROUP=.*/   WWWGROUP="www-data"/' util/postinstall.sh
+
+make postinstall
+
+cp /usr/local/etc/piler/piler.conf /usr/local/etc/piler/piler.conf.bak
+sed -i "s/hostid=.*/hostid=$PILER_DOM/" /usr/local/etc/piler/piler.conf
+sed -i "s/update_counters_to_memcached=.*/update_counters_to_memcached=1/" /usr/local/etc/piler/piler.conf
+
+su piler -c "indexer --all --config /usr/local/etc/piler/sphinx.conf"
+
+/etc/init.d/rc.piler start
+/etc/init.d/rc.searchd start
+
+update-rc.d rc.piler defaults
+update-rc.d rc.searchd defaults
+
+mkdir -p /etc/nginx/ssl
+openssl req -x509 -nodes -days 3650 -newkey rsa:4096 -keyout /etc/nginx/ssl/piler.key -out /etc/nginx/ssl/piler.crt -subj "/CN=$PILER_DOM" -addext "subjectAltName=DNS:$PILER_DOM"
+
+cd /etc/nginx/sites-available
+cp /tmp/piler-$PILER_VER/contrib/webserver/piler-nginx.conf /etc/nginx/sites-available/
+ln -s /etc/nginx/sites-available/piler-nginx.conf /etc/nginx/sites-enabled/piler-nginx.conf
+
+sed -i "s|PILER_HOST|$PILER_DOM|g" /etc/nginx/sites-available/piler-nginx.conf
+sed -i "s|/var/run/php/php7.4-fpm.sock|/var/run/php/php$PHP_VER-fpm.sock|g" /etc/nginx/sites-available/piler-nginx.conf
+
+sed -i "/server_name.*/a \\
+        listen 443 ssl http2;\n\n\
+        ssl_certificate /etc/nginx/ssl/piler.crt;\n\
+        ssl_certificate_key /etc/nginx/ssl/piler.key;\n\n\
+        ssl_session_timeout 1d;\n\
+        ssl_session_cache shared:SSL:15m;\n\
+        ssl_session_tickets off;\n\n\
+        # modern configuration of Mozilla SSL configurator. Tweak to your needs.\n\
+        ssl_protocols TLSv1.2 TLSv1.3;\n\
+        ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;\n\
+        ssl_prefer_server_ciphers off;\n\n\
+        add_header X-Frame-Options SAMEORIGIN;\n\
+        add_header X-Content-Type-Options nosniff;" /etc/nginx/sites-available/piler-nginx.conf
+
+sed -i "/^server {.*/i\
+server {\n\
+        listen 80;\n\
+        server_name $PILER_DOM;\n\
+        server_tokens off;\n\
+        # HTTP to HTTPS redirect.\n\
+        return 301 https://$PILER_DOM;\n\
+}" /etc/nginx/sites-available/piler-nginx.conf
+
+cp /usr/local/etc/piler/config-site.php /usr/local/etc/piler/config-site.php.bak
+sed -i "s|\$config\['SITE_URL'\] = .*|\$config\['SITE_URL'\] = 'https://$PILER_DOM/';|" /usr/local/etc/piler/config-site.php
+cat >> /usr/local/etc/piler/config-site.php <<EOF
+
+// CUSTOM
+\$config['PROVIDED_BY'] = '$PILER_DOM';
+\$config['SUPPORT_LINK'] = 'https://$PILER_DOM';
+\$config['COMPATIBILITY'] = '';
+
+// fancy features.
+\$config['ENABLE_INSTANT_SEARCH'] = 1;
+\$config['ENABLE_TABLE_RESIZE'] = 1;
+
+\$config['ENABLE_DELETE'] = 1;
+\$config['ENABLE_ON_THE_FLY_VERIFICATION'] = 1;
+
+// general settings.
+\$config['TIMEZONE'] = 'Europe/Berlin';
+
+// authentication
+// Enable authentication against an imap server
+//\$config['ENABLE_IMAP_AUTH'] = 1;
+//\$config['RESTORE_OVER_IMAP'] = 1;
+//\$config['IMAP_RESTORE_FOLDER_INBOX'] = 'INBOX';
+//\$config['IMAP_RESTORE_FOLDER_SENT'] = 'Sent';
+//\$config['IMAP_HOST'] = '$SMARTHOST';
+//\$config['IMAP_PORT'] =  993;
+//\$config['IMAP_SSL'] = true;
+
+// authentication against an ldap directory (disabled by default)
+//\$config['ENABLE_LDAP_AUTH'] = 1;
+//\$config['LDAP_HOST'] = '$SMARTHOST';
+//\$config['LDAP_PORT'] = 389;
+//\$config['LDAP_HELPER_DN'] = 'cn=administrator,cn=users,dc=mydomain,dc=local';
+//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
+//\$config['LDAP_MAIL_ATTR'] = 'mail';
+//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
+//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
+//\$config['LDAP_BASE_DN'] = 'ou=Benutzer,dc=krs,dc=local';
+
+// authentication against an Uninvention based ldap directory 
+//\$config['ENABLE_LDAP_AUTH'] = 1;
+//\$config['LDAP_HOST'] = '$SMARTHOST';
+//\$config['LDAP_PORT'] = 7389;
+//\$config['LDAP_HELPER_DN'] = 'uid=ldap-search-user,cn=users,dc=mydomain,dc=local';
+//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
+//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
+//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
+//\$config['LDAP_BASE_DN'] = 'cn=users,dc=mydomain,dc=local';
+//\$config['LDAP_MAIL_ATTR'] = 'mailPrimaryAddress';
+//\$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'person';
+//\$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'person';
+//\$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mailAlternativeAddress';
+
+// special settings.
+\$config['MEMCACHED_ENABLED'] = 1;
+\$config['SPHINX_STRICT_SCHEMA'] = 1; // required for Sphinx $SPHINX_VER, see https://bitbucket.org/jsuto/piler/issues/1085/sphinx-331.
+EOF
+
+nginx -t && systemctl restart nginx
+
+apt autoremove -y
+apt clean -y
+
+

contrib/proxmox-lxc/matrix.orig

@@ -0,0 +1,150 @@
+#!/bin/bash
+
+#MATRIX_VAR
+
+
+MRX_PKE=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+
+ELE_DBNAME="synapse_db"
+ELE_DBUSER="synapse_user"
+ELE_DBPASS=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+
+apt update && apt full-upgrade -y
+
+apt install -y lsb-release apt-transport-https curl gpg software-properties-common net-tools nginx mc postgresql python3-psycopg2
+
+wget wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/matrix-org.list
+apt update && apt install -y matrix-synapse-py3
+systemctl enable matrix-synapse
+
+ss -tulpen
+
+mkdir /etc/nginx/ssl
+openssl req -x509 -nodes -days 3650 -newkey rsa:4096 -keyout /etc/nginx/ssl/matrix.key -out /etc/nginx/ssl/matrix.crt -subj "/CN=$MRX_DOM" -addext "subjectAltName=DNS:$MRX_DOM"
+
+cat > /etc/nginx/sites-available/$MRX_DOM <<EOF
+# Virtual Host configuration for example.com
+#
+# You can move that to a different file under sites-available/ and symlink that
+# to sites-enabled/ to enable it.
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name $MRX_DOM;
+
+    return 301 https://$MRX_DOM;
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    server_name $MRX_DOM;
+
+    ssl on;
+    ssl_certificate /etc/nginx/ssl/matrix.crt;
+    ssl_certificate_key /etc/nginx/ssl/matrix.key;
+
+    location / {
+      proxy_pass http://127.0.0.1:8008;
+      proxy_set_header X-Forwarded-For \$remote_addr;
+    }
+}
+
+server {
+    listen 8448 ssl;
+    listen [::]:8448 ssl;
+    server_name $MRX_DOM;
+
+    ssl on;
+    ssl_certificate /etc/nginx/ssl/matrix.crt;
+    ssl_certificate_key /etc/nginx/ssl/matrix.key;
+
+    # If you don't wanna serve a site, comment this out
+    root /var/www/$MRX_DOM;
+    index index.html index.htm;
+
+    location / {
+        proxy_pass http://127.0.0.1:8008;
+        proxy_set_header X-Forwarded-For \$remote_addr;
+    }
+}
+
+EOF
+ln -s /etc/nginx/sites-available/$MRX_DOM /etc/nginx/sites-enabled/$MRX_DOM
+
+cat > /etc/nginx/sites-available/$ELE_DOM <<EOF
+# Virtual Host configuration for example.com
+#
+# You can move that to a different file under sites-available/ and symlink that
+# to sites-enabled/ to enable it.
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name $ELE_DOM;
+    return 301 https://$ELE_DOM;
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    server_name $ELE_DOM;
+
+    ssl on;
+    ssl_certificate /etc/nginx/ssl/matrix.crt;
+    ssl_certificate_key /etc/nginx/ssl/matrix.key;
+
+    # If you don't wanna serve a site, comment this out
+    root /var/www/$ELE_DOM/element;
+    index index.html index.htm;
+} 
+
+EOF
+
+ln -s /etc/nginx/sites-available/$ELE_DOM /etc/nginx/sites-enabled/$ELE_DOM
+
+systemctl restart nginx
+
+mkdir /var/www/$ELE_DOM
+cd /var/www/$ELE_DOM
+wget https://packages.riot.im/element-release-key.asc
+gpg --import element-release-key.asc
+
+wget https://github.com/vector-im/element-web/releases/download/$ELE_VER/element-$ELE_VER.tar.gz
+wget https://github.com/vector-im/element-web/releases/download/$ELE_VER/element-$ELE_VER.tar.gz.asc
+gpg --verify element-$ELE_VER.tar.gz.asc
+
+tar -xzvf element-$ELE_VER.tar.gz
+ln -s element-$ELE_VER element
+chown www-data:www-data -R element
+cp ./element/config.sample.json ./element/config.json
+sed -i "s|https://matrix-client.matrix.org|https://$MRX_DOM|" ./element/config.json
+sed -i "s|\"server_name\": \"matrix.org\"|\"server_name\": \"$MRX_DOM\"|" ./element/config.json
+
+su postgres <<EOF
+psql -c "CREATE USER $ELE_DBUSER WITH PASSWORD '$ELE_DBPASS';"
+psql -c "CREATE DATABASE $ELE_DBNAME ENCODING 'UTF8' LC_COLLATE='C' LC_CTYPE='C' template=template0 OWNER $ELE_DBUSER;"
+echo "Postgres User '$ELE_DBUSER' and database '$ELE_DBNAME' created."
+EOF
+
+cd /
+sed -i "s|#registration_shared_secret: <PRIVATE STRING>|registration_shared_secret: \"$MRX_PKE\"|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|#public_baseurl: https://example.com/|public_baseurl: https://$MRX_DOM/|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|#enable_registration: false|enable_registration: true|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|name: sqlite3|name: psycopg2|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|database: /var/lib/matrix-synapse/homeserver.db|database: $ELE_DBNAME\n    user: $ELE_DBUSER\n    password: $ELE_DBPASS\n    host: 127.0.0.1\n    cp_min: 5\n    cp_max: 10|" /etc/matrix-synapse/homeserver.yaml
+
+systemctl restart matrix-synapse
+
+register_new_matrix_user -c /etc/matrix-synapse/homeserver.yaml http://127.0.0.1:8008
+
+#curl https://download.jitsi.org/jitsi-key.gpg.key | sh -c 'gpg --dearmor > /usr/share/keyrings/jitsi-keyring.gpg'
+#echo 'deb [signed-by=/usr/share/keyrings/jitsi-keyring.gpg] https://download.jitsi.org stable/' | tee /etc/apt/sources.list.d/jitsi-stable.list > /dev/null
+
+#apt update
+#apt install -y jitsi-meet
+
+
+

contrib/proxmox-lxc/zmb_mem.orig

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+#ZMB_VAR
+
+
+
+apt update && apt full-upgrade -y
+echo -ne '\n' | apt install -y acl dnsutils mc samba winbind libpam-winbind libnss-winbind krb5-user krb5-config samba-dsdb-modules samba-vfs-modules 
+
+mv /etc/krb5.conf /etc/krb5.conf.bak
+cat > /etc/krb5.conf <<EOF
+[libdefaults]
+	default_realm = $ZMB_REA
+    ticket_lifetime = 600
+	dns_lookup_realm = true
+	dns_lookup_kdc = true
+	renew_lifetime = 7d
+EOF
+
+echo -e "$ZMB_APW" | kinit -V $ZMB_ADA
+klist
+
+mv /etc/samba/smb.conf /etc/samba/smb.conf.bak
+cat > /etc/samba/smb.conf <<EOF
+[global]
+	workgroup = $ZMB_DOM
+	security = ADS
+	realm = $ZMB_REA
+	server string = %h server
+
+	vfs objects = acl_xattr shadow_copy2
+    map acl inherit = Yes
+    store dos attributes = Yes
+	idmap config *:backend = tdb
+	idmap config *:range = 3000000-4000000
+	idmap config *:schema_mode = rfc2307
+
+	winbind refresh tickets = Yes
+	winbind use default domain = Yes
+	winbind separator = /
+	winbind nested groups = yes
+	winbind nss info = rfc2307
+
+	pam password change = Yes
+	passwd chat = *Enter\snew\s*\spassword:* %n\n *Retype\snew\s*\spassword:* %n\n *password\supdated\ssuccessfully* .
+	passwd program = /usr/bin/passwd %u
+
+	template homedir = /home/%U
+	template shell = /bin/bash
+	bind interfaces only = Yes
+	interfaces = lo eth0
+	log file = /var/log/samba/log.%m
+	logging = syslog
+	max log size = 1000
+	panic action = /usr/share/samba/panic-action %d
+
+	load printers = No
+	printcap name = /dev/null
+	printing = bsd
+	disable spoolss = Yes
+
+	allow trusted domains = No
+	dns proxy = No
+	shadow: snapdir = .zfs/snapshot
+	shadow: sort = desc
+	shadow: format = -%Y-%m-%d-%H%M
+	shadow: snapprefix = ^zfs-auto-snap_\(frequent\)\{0,1\}\(hourly\)\{0,1\}\(daily\)\{0,1\}\(monthly\)\{0,1\}
+	shadow: delimiter = -20
+
+[share]
+	comment = Main Share
+	path = /tank/share
+	read only = No
+	create mask = 0660
+	directory mask = 0770
+	inherit acls = Yes
+
+
+
+EOF
+
+systemctl restart smbd
+
+echo -e "$ZMB_APW" | net ads join -U $ZMB_ADA createcomputer=Computers
+sed -i "s|files systemd|files systemd winbind|g" /etc/nsswitch.conf
+sed -i "s|#WINBINDD_OPTS=|WINBINDD_OPTS=|" /etc/default/winbind
+echo -e "session optional        pam_mkhomedir.so skel=/etc/skel umask=077" >> /etc/pam.d/common-session
+
+systemctl restart winbind nmbd
+wbinfo -u
+wbinfo -g
+
+mkdir /tank/share
+chown 'administrator':'domain users' /tank/share
+
+setfacl -Rm u:administrator:rwx,g::-,o::- /tank/share
+setfacl -Rdm u:administrator:rwx,g::-,o::- /tank/share
+
+systemctl restart smbd nmbd winbind
+

contrib/selinux/README

@@ -0,0 +1,13 @@
+setsebool -P allow_httpd_mod_auth_ntlm_winbind on
+setsebool -P httpd_can_network_connect on
+setsebool -P httpd_ssi_exec on
+
+setsebool -P httpd_use_nfs 1 (in case of nfs)
+
+
+checkmodule -M -m -o piler.mod piler.te 
+semodule_package -o piler.pp -m piler.mod 
+semodule -i piler.pp 
+
+chcon -R --type=httpd_sys_rw_content_t /var/piler/www/tmp
+

contrib/selinux/piler.te

@@ -0,0 +1,38 @@
+module piler 1.2;
+
+require {
+        type devlog_t;
+        type httpd_sys_script_t;
+        type httpd_t;
+        type initrc_t;
+        type initrc_var_run_t;
+        type kernel_t;
+        type public_content_t;
+        type var_t;
+
+        class capability { kill setuid setgid sys_resource };
+        class dir search;
+        class file { read execute open getattr };
+        class netlink_audit_socket create;
+        class process { setrlimit signal };
+        class sock_file write;
+        class unix_dgram_socket sendto;
+
+}
+
+#============= httpd_sys_script_t ==============
+
+allow httpd_sys_script_t var_t:file { getattr open read };
+allow httpd_sys_script_t devlog_t:sock_file write;
+allow httpd_sys_script_t httpd_t:file { getattr read open };
+allow httpd_sys_script_t httpd_t:dir search;
+allow httpd_sys_script_t initrc_t:process signal;
+allow httpd_sys_script_t initrc_var_run_t:file { getattr open read };
+allow httpd_sys_script_t kernel_t:unix_dgram_socket sendto;
+allow httpd_sys_script_t self:capability { kill setuid setgid sys_resource };
+allow httpd_sys_script_t self:netlink_audit_socket create;
+allow httpd_sys_script_t self:process setrlimit;
+
+allow httpd_t var_t:file getattr;
+
+

contrib/smtp/smtp-source.py

@@ -0,0 +1,171 @@
+#!/usr/bin/python
+
+import argparse
+import smtplib
+import random
+import string
+import quopri
+import time
+import sys
+import os
+import email.utils
+
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+
+"""
+check out http://blog.magiksys.net/generate-and-send-mail-with-python-tutorial
+for a real professional solution
+"""
+
+eol = "\r\n"
+emails = []
+dictionary = '/'.join(__file__.split('/')[:-1]) + "/dictionary.txt"
+checkpoint_for_newline = 300
+
+
+def create_random_string(len=60):
+    return ''.join(random.choice(string.uppercase + string.digits) for i in range(len))
+
+
+def create_msg(words, size=10000):
+    msg_text = ""
+    msg_size = 0
+    new_line_checkpoint = checkpoint_for_newline
+
+    while msg_size < size:
+        choice = random.SystemRandom().choice
+        s = str(choice(words)).replace('\n', ' ')
+        msg_text += s
+        msg_size += len(s)
+        if msg_size > new_line_checkpoint:
+            new_line_checkpoint += checkpoint_for_newline
+            msg_text += eol + eol
+
+    return msg_text
+
+
+def create_text_mime_part(text, charset, encoding, boundary):
+    part = ""
+
+    part += "--" + boundary + eol
+    part += "Content-Type: text/plain; charset=\"" + charset + "\"" + eol
+    part += "Content-Transfer-Encoding: " + encoding + eol
+    part += eol
+    part += text + eol + eol
+
+    return part
+ 
+
+def create_headers(args, boundary):
+    hdr = ""
+
+    hdr += "Content-Type: multipart/alternative;" + eol
+    hdr += "   boundary=\"" + boundary + "\"" + eol
+    hdr += "MIME-Version: 1.0" + eol
+    hdr += "Subject: " + args.subject + eol
+    hdr += "From: " + args.sender + eol
+    hdr += "To: " + ', ' . join(args.rcpt) + eol
+    hdr += "Date: " + email.utils.formatdate(time.time(), localtime=True) + eol
+    hdr += "Message-ID: " + "<" + create_random_string(40) + "@" + args.helo + ">" + eol
+    hdr += eol + eol
+
+    return hdr
+
+
+def create_message(words, args):
+    msg = ""
+
+    boundary = '_' + create_random_string() + '_'
+
+    msg += create_headers(args, boundary)
+
+    the_text = create_msg(words, args.msglen)
+    the_text = quopri.encodestring(the_text)
+
+    msg += create_text_mime_part(the_text, "iso-8859-2", "quoted-printable", boundary)
+    msg += "--" + boundary + eol
+
+    return msg
+
+
+def read_emails_from_dir(directory):
+    for dir, dirs, files in os.walk(directory):
+        for file in files:
+            emails.append(os.path.join(dir, file))
+
+    args.count = len(emails)
+
+
+def get_next_email_text():
+    if emails == []: return ''
+
+    filename = emails.pop()
+    with open(filename) as f:
+        return f.read()
+
+
+parser = argparse.ArgumentParser()
+
+parser.add_argument("--helo", type=str, help="ehlo hostname", default="myhost.aaa.fu")
+parser.add_argument("-f", "--sender", type=str, help="sender email", default="sender@aaa.fu")
+parser.add_argument("-r", "--rcpt", type=str, nargs='+', help="recipient email(s)", default=["archive@aaa.fu"])
+parser.add_argument("-c", "--count", type=int, help="number of emails to send", default=1)
+parser.add_argument("-d", "--debug", type=int, help="debug level", default=0)
+parser.add_argument("--session", type=int, help="number of emails to send in one smtp transaction", default=1)
+parser.add_argument("-s", "--server", type=str, help="smtp server", required=True)
+parser.add_argument("-p", "--port", type=int, help="smtp port", default=25)
+parser.add_argument("--subject", type=str, help="subject", default="This is test subject")
+parser.add_argument("-l", "--msglen", type=int, help="message length (approx.)", default=20000)
+parser.add_argument("--starttls", help="use STARTTLS", action="store_true")
+parser.add_argument("--pem", type=str, help="pem file for starttls", default="")
+parser.add_argument("--dir", type=str, help="directory to send emails (must be eml files) from", default="")
+
+args = parser.parse_args()
+
+
+if args.starttls and args.pem == "":
+    sys.exit("make a pem file for starttls")
+
+
+i = 0
+total_count = 0
+
+if args.dir:
+    read_emails_from_dir(args.dir)
+else:
+    with open(dictionary) as f:
+        words = f.readlines()
+
+
+while i < args.count:
+    server = smtplib.SMTP(args.server, args.port, args.helo, 10)
+
+    if args.starttls:
+        server.starttls(args.pem, args.pem)
+
+    k = 0
+
+    while i < args.count and k < args.session:
+
+        if args.dir:
+            message = get_next_email_text()
+        else:
+            message = create_message(words, args)
+
+        server.set_debuglevel(args.debug)
+        server.sendmail(args.sender, args.rcpt, message)
+
+        k += 1
+        total_count += 1
+ 
+        if args.debug == 0:
+            sys.stdout.write('%s\r' % str(total_count))
+            sys.stdout.flush()
+
+    server.quit()
+    i += 1
+
+
+if args.debug == 0:
+    print('')

contrib/webserver/piler-apache-2.x.conf

@@ -0,0 +1,15 @@
+<VirtualHost *:80>
+    ServerName HOSTNAME
+
+    DocumentRoot "/var/piler/www"
+
+    <Directory /var/piler/www>
+       Require all granted
+
+       AllowOverride all
+    </Directory>
+
+    ErrorLog "/var/log/apache2/HOSTNAME-error_log"
+    CustomLog "/var/log/apache2/HOSTNAME-access_log" common
+</VirtualHost>
+

contrib/webserver/piler-nginx.conf

@@ -0,0 +1,69 @@
+
+server {
+        server_name PILER_HOST;
+
+        root /var/piler/www;
+
+        server_tokens off;
+
+        add_header X-XSS-Protection "1; mode=block";
+        add_header X-Content-Type-Options "nosniff";
+        add_header Referrer-Policy "same-origin";
+
+        gzip on;
+        gzip_types text/plain application/xml text/css;
+        gzip_vary on;
+
+        location / {
+            index  index.php index.html;
+            try_files $uri $uri/ /index.php;
+        }
+
+        #error_page  404              /404.html;
+
+        error_page   500 502 503 504  /50x.html;
+        location = /50x.html {
+            root   html;
+        }
+
+        location ~ [^/]\.php(/|$) {
+            fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
+
+            fastcgi_split_path_info ^(.+?\.php)(/.*)$;
+            if (!-f $document_root$fastcgi_script_name) {
+               return 404;
+            }
+
+            fastcgi_pass unix:/var/run/php/php8.1-fpm.sock;
+            fastcgi_index index.php;
+            include fastcgi_params;
+        }
+
+        location ~* \.(ico|css|js|gif|jpe?g|png)$ {
+            expires 2w;
+        }
+
+        rewrite /search.php /index.php?route=search/search&type=simple;
+        rewrite /advanced.php /index.php?route=search/search&type=advanced;
+        rewrite /expert.php /index.php?route=search/search&type=expert;
+        rewrite /search-helper.php /index.php?route=search/helper;
+        rewrite /audit-helper.php /index.php?route=audit/helper;
+        rewrite /message.php /index.php?route=message/view;
+        rewrite /bulkrestore.php /index.php?route=message/bulkrestore;
+        rewrite /bulkremove.php /index.php?route=message/bulkremove;
+        rewrite /rejectremove.php /index.php?route=message/rejectremove;
+        rewrite /bulkpdf.php /index.php?route=message/bulkpdf;
+        rewrite /folders.php /index.php?route=folder/list&;
+        rewrite /settings.php /index.php?route=user/settings;
+        rewrite /login.php /index.php?route=login/login;
+        rewrite /logout.php /index.php?route=login/logout;
+        rewrite /google.php /index.php?route=login/google;
+        rewrite /domain.php /index.php?route=domain/domain;
+        rewrite /ldap.php /index.php?route=ldap/list;
+        rewrite /customer.php /index.php?route=customer/list;
+        rewrite /retention.php /index.php?route=policy/retention;
+        rewrite /archiving.php /index.php?route=policy/archiving;
+        rewrite /legalhold.php /index.php?route=policy/legalhold;
+        rewrite /view/javascript/piler.js /js.php;
+}
+

cppcheck.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+cppcheck -DHAVE_PDFTOTEXT -DHAVE_PPTHTML -DHAVE_TNEF -DHAVE_UNRTF -DHAVE_XLS2CSV -DHAVE_CATPPT -DHAVE_CATDOC -DHAVE_ZIP -D_GNU_SOURCE -DHAVE_DAEMON -DHAVE_TRE -DHAVE_CLAMD -DHAVE_LIBCLAMAV -DNEED_MYSQL --error-exitcode=1 --enable=all --suppressions-list=suppressions.txt --force src/ unit_tests/

docker/Dockerfile

@@ -0,0 +1,43 @@
+FROM ubuntu:22.04
+
+ARG PACKAGE
+
+LABEL description="piler ubuntu jammy image" \
+      maintainer="Janos SUTO, sj@acts.hu" \
+      package="${PACKAGE}"
+
+ENV DEBIAN_FRONTEND="noninteractive" \
+    DISTRO="jammy" \
+    PILER_USER="piler" \
+    MYSQL_DATABASE="piler"
+
+COPY ${PACKAGE} /
+
+RUN apt-get update && \
+    apt-get -y --no-install-recommends install \
+       wget openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql php8.1-fpm php8.1-zip php8.1-ldap \
+       php8.1-gd php8.1-curl php8.1-xml php8.1-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip4 \
+       libtre5 cron libmariadb-dev mariadb-client-core-10.6 python3 python3-mysqldb ca-certificates curl rsyslog && \
+    wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
+    dpkg -i manticore-repo.noarch.deb && \
+    rm -f manticore-repo.noarch.deb && \
+    apt-get update && \
+    apt-get install -y manticore manticore-columnar-lib && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    sed -i '/session    required     pam_loginuid.so/c\#session    required     pam_loginuid.so' /etc/pam.d/cron && \
+    dpkg -i ${PACKAGE} && \
+    touch /etc/piler/MANTICORE && \
+    ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled && \
+    rm -f ${PACKAGE} /etc/nginx/sites-enabled/default /etc/piler/piler.key /etc/piler/piler.pem /etc/piler/config-site.php && \
+    crontab -u $PILER_USER /usr/share/piler/piler.cron
+
+VOLUME ["/etc/piler"]
+VOLUME ["/var/piler/store"]
+VOLUME ["/var/piler/manticore"]
+
+EXPOSE 25 80 443
+
+COPY start.sh /start.sh
+
+CMD ["/start.sh"]

docker/README.md

@@ -0,0 +1,16 @@
+## How to run piler
+
+Edit the variables in docker-compose.yaml, then run
+
+```
+docker-compose up -d
+```
+
+## How to build the image for yourself
+
+Pick the latest deb package from Bitbucket download page (https://bitbucket.org/jsuto/piler/downloads/)
+and use it as the PACKAGE build argument, eg.
+
+```
+docker build --build-arg PACKAGE=piler_1.4.1-jammy-c860ca67_amd64.deb -t piler:1.4.1 .
+```

docker/build.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+IMAGE_NAME="sutoj/piler:1.4.4"
+
+if [[ $# -ne 1 ]]; then echo "ERROR: missing package name" 1>&2; exit 1; fi
+
+docker build --pull --build-arg PACKAGE="$1" -t "$IMAGE_NAME" .

docker/docker-compose.yaml

@@ -0,0 +1,59 @@
+version: "3"
+services:
+  mysql:
+    image: mariadb:11.1.2
+    container_name: mysql
+    restart: unless-stopped
+    cap_drop:
+      - ALL
+    cap_add:
+      - dac_override
+      - setuid
+      - setgid
+    environment:
+      - MYSQL_DATABASE=piler
+      - MYSQL_USER=piler
+      - MYSQL_PASSWORD=piler123
+      - MYSQL_RANDOM_ROOT_PASSWORD=yes
+    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
+    volumes:
+      - db_data:/var/lib/mysql
+  piler:
+    image: sutoj/piler:1.4.4
+    container_name: piler
+    init: true
+    environment:
+      - MYSQL_HOSTNAME=mysql
+      - MYSQL_DATABASE=piler
+      - MYSQL_USER=piler
+      - MYSQL_PASSWORD=piler123
+      - PILER_HOSTNAME=cust1.acts.hu
+      - RT=1
+    ports:
+      - "25:25"
+      - "80:80"
+    volumes:
+      - piler_etc:/etc/piler
+      - piler_manticore:/var/piler/manticore
+      - piler_store:/var/piler/store
+    healthcheck:
+      test: curl -s smtp://localhost/
+      interval: "60s"
+      timeout: "3s"
+      start_period: "15s"
+      retries: 3
+    deploy:
+      resources:
+        reservations:
+          memory: 512M
+        limits:
+          memory: 512M
+
+    depends_on:
+      - "mysql"
+
+volumes:
+  db_data: {}
+  piler_etc: {}
+  piler_manticore: {}
+  piler_store: {}

docker/start.sh

@@ -0,0 +1,216 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+CONFIG_DIR="/etc/piler"
+VOLUME_DIR="/var/piler"
+PILER_CONF="${CONFIG_DIR}/piler.conf"
+PILER_KEY="${CONFIG_DIR}/piler.key"
+PILER_PEM="${CONFIG_DIR}/piler.pem"
+PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf"
+SPHINX_CONF="${CONFIG_DIR}/manticore.conf"
+CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php"
+PILER_MY_CNF="${CONFIG_DIR}/.my.cnf"
+RT="${RT:-0}"
+
+error() {
+   echo "ERROR:" "$*" 1>&2
+   exit 1
+}
+
+
+log() {
+   echo "DEBUG:" "$*"
+}
+
+
+pre_flight_check() {
+   [[ -v PILER_HOSTNAME ]] || error "Missing PILER_HOSTNAME env variable"
+   [[ -v MYSQL_HOSTNAME ]] || error "Missing MYSQL_HOSTNAME env variable"
+   [[ -v MYSQL_DATABASE ]] || error "Missing MYSQL_DATABASE env variable"
+   [[ -v MYSQL_USER ]]     || error "Missing MYSQL_USER env variable"
+   [[ -v MYSQL_PASSWORD ]] || error "Missing MYSQL_PASSWORD env variable"
+}
+
+
+give_it_to_piler() {
+   local f="$1"
+
+   [[ -f "$f" ]] || error "${f} does not exist, aborting"
+
+   chown "${PILER_USER}:${PILER_USER}" "$f"
+   chmod 600 "$f"
+}
+
+
+make_certificate() {
+   local f="$1"
+   local crt="/tmp/1.cert"
+   local SSL_CERT_DATA="/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com"
+
+   log "Making an ssl certificate"
+
+   openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 -subj "$SSL_CERT_DATA" -keyout "$f" -out "$crt" -sha1 2>/dev/null
+   cat "$crt" >> "$f"
+   rm -f "$crt"
+
+   give_it_to_piler "$f"
+}
+
+
+make_piler_key() {
+   local f="$1"
+
+   log "Generating piler.key"
+
+   dd if=/dev/urandom bs=56 count=1 of="$f" 2>/dev/null
+   [[ $(stat -c '%s' "$f") -eq 56 ]] || error "could not read 56 bytes from /dev/urandom to ${f}"
+
+   give_it_to_piler "$f"
+}
+
+
+fix_configs() {
+   [[ -f "$PILER_KEY" ]] || make_piler_key "$PILER_KEY"
+   [[ -f "$PILER_PEM" ]] || make_certificate "$PILER_PEM"
+
+   if [[ ! -f "$PILER_NGINX_CONF" ]]; then
+      log "Writing ${PILER_NGINX_CONF}"
+
+      cp "${PILER_NGINX_CONF}.dist" "$PILER_NGINX_CONF"
+      sed -i "s%PILER_HOST%${PILER_HOSTNAME}%" "$PILER_NGINX_CONF"
+   fi
+
+   if [[ ! -f "$PILER_CONF" ]]; then
+      log "Writing ${PILER_CONF}"
+
+      sed \
+         -e "s/mysqluser=.*/mysqluser=${MYSQL_USER}/g" \
+         -e "s/mysqldb=.*/mysqldb=${MYSQL_DATABASE}/g" \
+         -e "s/verystrongpassword/${MYSQL_PASSWORD}/g" \
+         -e "s/hostid=.*/hostid=${PILER_HOSTNAME}/g" \
+         -e "s/tls_enable=.*/tls_enable=1/g" \
+         -e "s/mysqlsocket=.*/mysqlsocket=/g" "${PILER_CONF}.dist" > "$PILER_CONF"
+
+      {
+         echo "mysqlhost=${MYSQL_HOSTNAME}"
+      } >> "$PILER_CONF"
+
+      give_it_to_piler "$PILER_CONF"
+   fi
+
+   if [[ ! -f "$CONFIG_SITE_PHP" ]]; then
+      log "Writing ${CONFIG_SITE_PHP}"
+
+      cp "${CONFIG_DIR}/config-site.dist.php" "$CONFIG_SITE_PHP"
+
+      sed -i "s%HOSTNAME%${PILER_HOSTNAME}%" "$CONFIG_SITE_PHP"
+
+      {
+         echo "\$config['DECRYPT_BINARY'] = '/usr/bin/pilerget';"
+         echo "\$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/bin/pileraget';"
+         echo "\$config['PILER_BINARY'] = '/usr/sbin/piler';"
+         echo "\$config['DB_HOSTNAME'] = '$MYSQL_HOSTNAME';"
+         echo "\$config['DB_DATABASE'] = '$MYSQL_DATABASE';"
+         echo "\$config['DB_USERNAME'] = '$MYSQL_USER';"
+         echo "\$config['DB_PASSWORD'] = '$MYSQL_PASSWORD';"
+         echo "\$config['ENABLE_MEMCACHED'] = 1;"
+         echo "\$memcached_server = ['memcached', 11211];"
+      } >> "$CONFIG_SITE_PHP"
+   fi
+
+   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \
+       -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \
+       -e "s%MYSQL_USERNAME%${MYSQL_USER}%" \
+       -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" \
+       -i "$SPHINX_CONF"
+
+   # Fixes for RT index
+
+   if [[ $RT -eq 1 ]]; then
+      sed -i "s/define('RT', 0)/define('RT', 1)/" "$SPHINX_CONF"
+      if ! grep "'RT'" "$CONFIG_SITE_PHP"; then
+         echo "\$config['RT'] = 1;" >> "$CONFIG_SITE_PHP"
+      fi
+
+      if ! grep "'SPHINX_MAIN_INDEX'" "$CONFIG_SITE_PHP"; then
+         echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
+      fi
+
+      sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
+   fi
+}
+
+
+wait_until_mysql_server_is_ready() {
+   while true; do if mysql "--defaults-file=${PILER_MY_CNF}" <<< "show databases"; then break; fi; log "${MYSQL_HOSTNAME} is not ready"; sleep 5; done
+
+   log "${MYSQL_HOSTNAME} is ready"
+}
+
+
+init_database() {
+   local table
+   local has_metadata_table=0
+
+   wait_until_mysql_server_is_ready
+
+   while read -r table; do
+      if [[ "$table" == metadata ]]; then has_metadata_table=1; fi
+   done < <(mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< 'show tables')
+
+   if [[ $has_metadata_table -eq 0 ]]; then
+      log "no metadata table, creating tables"
+
+      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" < /usr/share/piler/db-mysql.sql
+   else
+      log "metadata table exists"
+   fi
+
+   if [[ -v ADMIN_USER_PASSWORD_HASH ]]; then
+      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< "update user set password='${ADMIN_USER_PASSWORD_HASH}' where uid=0"
+   fi
+}
+
+
+create_my_cnf_files() {
+   printf "[client]\nhost = %s\nuser = %s\npassword = %s\n[mysqldump]\nhost = %s\nuser = %s\npassword = %s\n" \
+      "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" \
+      > "$PILER_MY_CNF"
+
+   give_it_to_piler "$PILER_MY_CNF"
+}
+
+
+start_services() {
+   service cron start
+   service php8.1-fpm start
+   service nginx start
+   rsyslogd
+}
+
+
+start_piler() {
+   if [[ $RT -eq 0 && ! -f "${VOLUME_DIR}/manticore/main1.spp" ]]; then
+      log "main1.spp does not exist, creating index files"
+      su -c "indexer --all --config ${SPHINX_CONF}" "$PILER_USER"
+   fi
+
+   # No pid file should exist for piler
+   rm -f /var/run/piler/*pid
+
+   /etc/init.d/rc.searchd start
+   /etc/init.d/rc.piler start
+}
+
+
+pre_flight_check
+fix_configs
+create_my_cnf_files
+init_database
+start_services
+start_piler
+
+sleep infinity

etc/Makefile.in

@@ -16,9 +16,9 @@ localstatedir = @localstatedir@
 CC = @CC@
 CFLAGS = @CFLAGS@ @CPPFLAGS@
 DEFS = @defs@
-INCDIR = -I. -I../.. -I../../src @INCDIR@ @mysql_includes@
+INCDIR = -I. -I../.. -I../../src @INCDIR@ @sql_includes@
 LIBDIR = -L. @LIBDIR@ @LDFLAGS@ -L../../src
-LIBS = @LIBS@ @mysql_libs@
+LIBS = @LIBS@ @sql_libs@
 RUNNING_USER = @RUNNING_USER@
 RUNNING_GROUP = `@id_bin@ -gn $(RUNNING_USER)`
 
@@ -26,14 +26,24 @@ INSTALL = @INSTALL@
 
 all:
 	sed -e 's%pidfile=.*%pidfile=$(localstatedir)/run/piler/piler.pid%g' \
-	-e 's%iv=.*%iv=@iv@%g' \
-	-e 's%workdir=.*%workdir=$(localstatedir)/piler/tmp%g' < $(srcdir)/example.conf > $(srcdir)/piler.conf
+	-e 's%pemfile=%pemfile=$(sysconfdir)/piler/piler.pem%g' \
+	-e 's%workdir=.*%workdir=$(localstatedir)/piler/tmp%g' < $(srcdir)/example.conf | grep -v ^\; | grep '=' | sort > $(srcdir)/piler.conf
 
 install:
-	if [ ! -f "$(DESTDIR)$(sysconfdir)/piler.conf" ]; then $(INSTALL) -m 0640 -g $(RUNNING_GROUP) $(srcdir)/piler.conf $(DESTDIR)$(sysconfdir)/piler.conf; fi
+	$(INSTALL) -m 0640 -g $(RUNNING_GROUP) $(srcdir)/piler.conf $(DESTDIR)$(sysconfdir)/piler/piler.conf.dist
+	sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/sphinx.conf.in > sphinx.conf.dist
+	sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/manticore.conf.in > manticore.conf.dist
+	$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/sphinx.conf.dist $(DESTDIR)$(sysconfdir)/piler/sphinx.conf.dist
+	$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/manticore.conf.dist $(DESTDIR)$(sysconfdir)/piler/manticore.conf.dist
+	$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/config-site.dist.php $(DESTDIR)$(sysconfdir)/piler/config-site.dist.php
+	$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/cron.jobs.in $(DESTDIR)$(datarootdir)/piler/piler.cron
+	sed -i -e 's%LOCALSTATEDIR%$(localstatedir)%g' $(DESTDIR)$(datarootdir)/piler/piler.cron
+	sed -i -e 's%LIBEXECDIR%$(libexecdir)%g' $(DESTDIR)$(datarootdir)/piler/piler.cron
+	sed -i -e 's%BINDIR%$(bindir)%g' $(DESTDIR)$(datarootdir)/piler/piler.cron
+	sed -i -e 's%SYSCONFDIR%$(sysconfdir)%g' $(DESTDIR)$(datarootdir)/piler/piler.cron
 
 clean:
-	rm -f piler.conf cron.jobs
+	rm -f piler.conf cron.jobs sphinx.conf.dist
 
 distclean: clean
 	rm -f Makefile

etc/config-site.dist.php

@@ -0,0 +1,13 @@
+<?php
+
+define('SITE_NAME_CONST', 'SITE_NAME');
+
+$config[SITE_NAME_CONST] = 'HOSTNAME';
+$config['SITE_URL'] = 'http://' . $config[SITE_NAME_CONST] . '/';
+
+$config['SMTP_DOMAIN'] = $config[SITE_NAME_CONST];
+$config['SMTP_FROMADDR'] = 'no-reply@' . $config[SITE_NAME_CONST];
+$config['ADMIN_EMAIL'] = 'admin@' . $config[SITE_NAME_CONST];
+
+$config['DB_PASSWORD'] = 'MYSQL_PASSWORD';
+

etc/cron.jobs.in

@@ -1,36 +1,14 @@
-#########################################################
-###     all the cron jobs you may need for piler      ###
-### be sure to review it and adjust it for your needs ###
-#########################################################
-
-root's crontab:
-
-### optional: query postfix queue statistics
-*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape > LOCALSTATEDIR/piler/stat/active+incoming
-*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape -s > LOCALSTATEDIR/piler/stat/active+incoming-sender
-*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape deferred > LOCALSTATEDIR/piler/stat/deferred
-*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape -s deferred > LOCALSTATEDIR/piler/stat/deferred-sender
-
-
-piler's crontab:
-
-### mandatory
-
-*/5 * * * * LC_ALL=C mpstat | tail -1 | awk '{print $11}' > /var/piler/stat/cpu.stat
-*/15 * * * * /usr/local/bin/indexer --quiet delta1 --rotate && sleep 2 && /usr/local/bin/indexer --quiet --merge main1 delta1 --merge-dst-range deleted 0 0 --rotate
-*/15 * * * * /usr/local/bin/indexer --quiet tag1 --rotate
-*/15 * * * * /usr/local/bin/indexer --quiet note1 --rotate
-
-### optional: the same report you can see on the health page
-30 7 * * * /usr/local/libexec/piler/daily-report.php /srv/www/webui.yourdomain.com
-
-### optional: populate accouting data
-30 6 * * * /usr/local/libexec/piler/generate_stats.php /srv/www/webui.yourdomain.com
-
-### optional: regular AD sync
-0 8 * * * /usr/bin/php LIBEXECDIR/piler/ldap_sync.php /srv/www/webui.yourdomain.com > LOCALSTATEDIR/piler/stat/adsync.stat
-
-### optional: purge aged emails
-2 0 * * * /usr/local/bin/pilerpurge
+### PILERSTART
+5,35 * * * * LIBEXECDIR/piler/indexer.delta.sh
+30   2 * * * LIBEXECDIR/piler/indexer.main.sh
+40 3 * * * LIBEXECDIR/piler/purge.sh
+*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet tag1 --rotate
+*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet note1 --rotate
+*/5 * * * *  /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \;
+*/5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error
 
+### optional
+###30 6 * * * /usr/bin/php LIBEXECDIR/piler/generate_stats.php --webui LOCALSTATEDIR/piler/www >/dev/null
+###*/5 * * * * LIBEXECDIR/piler/import.sh
 
+### PILEREND

etc/example.conf

@@ -22,31 +22,61 @@ username=piler
 ; The default is 7 years + 2 days (=7*365+2=2557 days)
 default_retention_days=2557
 
-; this is a 16 character long vector
-iv=****************
+; The initialization vector for encryption.
+; By now it has become obsolete. Don't use it for
+; new installations. However, if you used it before
+; then you must keep it as it is.
+;iv=
 
 ; whether to encrypt messages (1) or not (0).
+; Make sure to set this value to your needs right after installing piler,
+; and don't change it after you got the first email. Otherwise
+; you'll have half the archive encrypted, the other half unencrypted
+; which will cause problems.
 encrypt_messages=1
 
 ; number of worker processes, ie. the number of simultaneous smtp connections to piler.
-number_of_worker_processes=10
+; This value should be the number of cpus + 1, ie. 2 for a single cpu host
+number_of_worker_processes=2
+
+; max. number of parallel connections piler-smtp can handle.
+; Important! If you want to change this value, then you must first
+; stop piler-smtp, change the value, then start piler-smtp.
+; I don't suggest to go under 10 or above 1000.
+max_connections=64
 
 ; number of processed emails per each piler process
-max_requests_per_child=1000
+max_requests_per_child=10000
 
 ; SMTP HELO identification string
-hostid=mailarchiver.localhost
+; this should be the FQDN part of the email address
+; where you copy emails, eg. archive@piler.example.com -> piler.example.com
+hostid=piler.example.com
+
+; whether to process rcpt to addresses and add them to rcpt table (1) or not (0)
+process_rcpt_to_addresses=0
 
 ; write pid file
 pidfile=/var/run/piler/piler.pid
 
 ; piler will listen here
+; if you have postfix, exim, ... installed on localhost
+; then make sure to set the listen_addr parameter for piler
 listen_addr=0.0.0.0
 listen_port=25
 
 clamd_socket=/tmp/clamd
 
-session_timeout=420
+; check for client timeout interval. Default: 20 sec
+check_for_client_timeout_interval=20
+
+; smtp timeout. Default: 60 sec
+smtp_timeout=60
+
+helper_timeout=20
+
+; whether to run external attachment extractors (1) or not (0)
+extract_attachments=1
 
 ; the 2nd parameter of the listen() system call. Please note that this is set
 ; when piler starts up and you should restart piler if you change this variable.
@@ -55,6 +85,12 @@ backlog=20
 
 workdir=/var/piler/tmp
 
+; whether to enable writing folder_message table (1) or not (0)
+enable_folders=0
+
+; discard a message if it's shorter than this value (in bytes)
+min_message_size=100
+
 ;
 ; starttls stuff
 ;
@@ -68,11 +104,18 @@ tls_enable=0
 pemfile=
 
 ; cipher list to use, see 'man SSL_CTX_set_cipher_list' for more details
-cipher_list=HIGH:MEDIUM
+cipher_list=ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS
+
+
+; set the minimum TLS protocol version for piler-smtp daemon
+;
+; Valid values: TLSv1, TLSv1.1, TLSv1.2, TLSv1.3
+; TLSv1 and TLSv1.1 are not recommended for security reasons
+tls_min_version=TLSv1.2
 
 
 ; piler's own header to indicate previously archived messages
-piler_header_field=X-piler: piler already archived this email
+piler_header_field=X-piler-id:
 
 ; extra header field to treat as To:
 ;
@@ -95,9 +138,30 @@ extra_to_field=X-Envelope-To:
 ; to messages without message-id.
 archive_emails_not_having_message_id=0
 
+; whether to archive each and every single email received (0) or
+; only those on the mydomains list (1). The default is to archive
+; everything
+archive_only_mydomains=0
+
+; whether to syslog the recipients of the email in the following format:
+; 400000.....xxxxx: rcpt=recipient1@domain.com
+; 400000.....xxxxx: rcpt=recipient2@domain.com
+syslog_recipients=0
+
 ; minimum word length in mail body to index
 min_word_len=1
 
+; whether to enable CJK (=Chinese, Japanese, and Korean) "characters".
+; the text piler can see with CJK languages may have extremely long
+; sequences without any whitespace. To prevent the parser to drop
+; these very long sequences, enable (1) this feature. By default it's
+; disabled (0).
+enable_cjk=0
+
+; whether to enable the CHUNKING / BDAT feature (1) or not (0)
+; You may read about the feature at https://tools.ietf.org/html/rfc3030)
+enable_chunking=0
+
 ; if piler detects this line in the mail header, then it will assume
 ; the message is a spam. You should include your own antispam solution's
 ; specific line.
@@ -133,7 +197,7 @@ update_counters_to_memcached=0
 ; hint: if you are using a mysql replicated environment and you do _not_ want
 ; piler to write to the replicated database (because you do sync it some other
 ; way to the master database or you are not interested in keeping the counters
-; persistantly at all), then specify a big number here, that fits to the 
+; persistantly at all), then specify a big number here, that fits to the
 ; "long int" size, eg. 2147483647
 memcached_to_db_interval=900
 
@@ -142,18 +206,69 @@ memcached_to_db_interval=900
 ; mysql stuff
 ;
 
+// this can be either utf8 or utf8mb4. Make sure to match the value
+// to the charset of the piler database! Also, make sure to set this
+// value in sphinx.conf
+mysqlcharset=utf8mb4
 ;mysqlhost=127.0.0.1
 ;mysqlport=3306
-mysqlsocket=/tmp/mysql.sock
+mysqlsocket=/var/run/mysqld/mysqld.sock
 mysqluser=piler
 mysqlpwd=verystrongpassword
 mysqldb=piler
 mysql_connect_timeout=2
 
 
+;
+; manticore stuff
+;
+sphxhost=127.0.0.1
+sphxport=9306
+; whether to enable (1) or not (0) the rt index. If so, then piler
+; writes the index data directly to manticore's sql API
+rtindex=0
 
 ; if you want to change the 'sent' time as you archive the message
 ; set this in seconds. This can be a postive or negative value.
 ; By default this feature is not enabled, use --tweak-sent-time
 ; configure option to enable it.
 tweak_sent_time_offset=0
+
+; whether to enable (1) or not (0) the extra mmap dedup test feature
+; if you change it, be sure to stop, then start piler
+mmap_dedup_test=0
+
+; security header that must be present in the first mail header of
+; the message. If the security_header value is not empty, then the
+; parser checks for this header line. Unless it's found it will discard
+; the given email. Note that this feature is supposed to be a security
+; mechanism against unwanted email in the archive if limiting smtp
+; clients via an IP-address list is not feasible.
+security_header=
+
+; By default the archive accepts any envelope recipient addresses.
+; If your archive's port 25 is wide open to the Internet (which it
+; shouldn't be, then spammers may find it, and fill it with spam.
+;
+; By setting this variable you may restrict the envelope address
+; to a single email address, eg. some-random-address-12345@archive.example.com
+; Then the archive will reject any other envelope recipients
+archive_address=
+
+; whether to enable (1) or not (0) an smtp access list similar to
+; postfix's postscreen. Valid actions in the acl file are "permit"
+; and "reject" (without quotes). See smtp.acl.example for more.
+;
+; Important! There's an implicit default deny at the end of the
+; rules. In other words if you decide to use the acl file, then
+; everyone is not explicitly permitted is denied.
+smtp_access_list=0
+
+; max message size in bytes
+; piler-smtp will reject any message that's bigger than this number
+max_message_size=50000000
+
+; max memory in bytes piler-smtp uses for buffering messages
+; when this limit is exceeded, no new emails will be accepted
+; until the used memory decreases below this level
+max_smtp_memory=500000000

etc/manticore.conf.in

@@ -0,0 +1,268 @@
+#!/usr/bin/php
+<?php
+
+define('LOCALSTATEDIR', '/var');
+define('NGRAM_CONFIG', "        #ngram_len               = 1\n        #ngram_chars             = U+3000..U+2FA1F\n");
+
+# See http://sphinxsearch.com/wiki/doku.php?id=charset_tables for more on the charset_table settings
+# The following settings contains English and some Latin extras
+define('CHARSET_TABLE', "0..9, english, _, \
+                                  U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
+                                  U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
+                                  U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
+                                  U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171\n");
+
+define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
+
+define('RT', 0);
+
+?>
+
+#
+# minimal manticore configuration suited to piler
+#
+
+<?php if(RT == 0) { ?>
+
+source base
+{
+   type = mysql
+   sql_host = MYSQL_HOSTNAME
+   sql_db = MYSQL_DATABASE
+   sql_user = MYSQL_USERNAME
+   sql_pass = MYSQL_PASSWORD
+
+   sql_attr_uint = size
+   sql_attr_uint = sent
+   sql_attr_uint = attachments
+}
+
+source delta : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query_pre  = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index
+   sql_query_post_index  = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
+
+   sql_query_killlist = SELECT `id` FROM `metadata` WHERE `deleted`=1
+}
+
+source main1 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source main2 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source main3 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source main4 : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+source dailydelta : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+
+index main1
+{
+        source                  = main1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index main2
+{
+        source                  = main2
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main2
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index main3
+{
+        source                  = main3
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main3
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index main4
+{
+        source                  = main4
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main4
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index dailydelta1
+{
+        source                  = dailydelta
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/dailydelta1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+        killlist_target         = main1:kl, main2:kl, main3:kl, main4:kl, dailydelta1:kl
+<?php print NGRAM_CONFIG; ?>
+}
+
+index delta1
+{
+        source                  = delta
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/delta1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        stored_fields           =
+        charset_table           = <?php print CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+source tag : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query  = SELECT `_id`, `id` AS iid, `uid`, `tag` FROM `tag`
+
+   sql_attr_uint = iid
+   sql_attr_uint = uid
+}
+
+source note : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query  = SELECT `_id`, `id` AS iid, `uid`, `note` FROM `note`
+
+   sql_attr_uint = iid
+   sql_attr_uint = uid
+}
+
+index tag1
+{
+        source                  = tag
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/tag1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+        stored_fields           =
+<?php print NGRAM_CONFIG; ?>
+}
+
+
+index note1
+{
+        source                  = note
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/note1
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print CHARSET_TABLE; ?>
+        stored_fields           =
+<?php print NGRAM_CONFIG; ?>
+}
+
+<?php } else { ?>
+
+index piler1
+{
+    type = rt
+    path = /var/piler/manticore/piler1
+    rt_mem_limit = 512M
+    stored_fields =
+    min_word_len = 1
+    min_prefix_len = 5
+    charset_table  = <?php print CHARSET_TABLE; ?>
+    # See https://manual.manticoresearch.com/Creating_an_index/Data_types#Row-wise-and-columnar-attribute-storages
+    # if you want to enable columnar storage
+    # columnar_attrs = *
+    rt_field = sender
+    rt_field = rcpt
+    rt_field = senderdomain
+    rt_field = rcptdomain
+    rt_field = subject
+    rt_field = body
+    rt_field = attachment_types
+    rt_attr_bigint = arrived
+    rt_attr_bigint = sent
+    rt_attr_uint = size
+    rt_attr_uint = direction
+    rt_attr_uint = folder
+    rt_attr_uint = attachments
+}
+
+index tag1
+{
+    type = rt
+    path = /var/piler/manticore/tag1
+    rt_mem_limit = 16M
+    stored_fields = tag
+    min_word_len = 2
+    min_prefix_len = 5
+    charset_table  = <?php print CHARSET_TABLE; ?>
+    rt_field = tag
+    rt_attr_bigint = mid
+    rt_attr_uint = uid
+}
+
+index note1
+{
+    type = rt
+    path = /var/piler/manticore/note1
+    rt_mem_limit = 16M
+    stored_fields = note
+    min_word_len = 2
+    min_prefix_len = 5
+    charset_table  = <?php print CHARSET_TABLE; ?>
+    rt_field = note
+    rt_attr_bigint = mid
+    rt_attr_uint = uid
+}
+
+<?php } ?>
+
+
+searchd
+{
+        listen                  = 127.0.0.1:9312
+        listen                  = 127.0.0.1:9306:mysql
+        listen                  = 127.0.0.1:9307:mysql_readonly
+        log                     = /var/piler/manticore/manticore.log
+        binlog_max_log_size     = 256M
+        binlog_path             = /var/piler/manticore
+        binlog_flush            = 2
+        query_log               = /var/piler/manticore/query.log
+        network_timeout         = 5
+        pid_file                = /var/run/piler/searchd.pid
+        seamless_rotate         = 1
+        preopen_tables          = 1
+        unlink_old              = 1
+        thread_stack            = 512k
+<?php if(RT) { ?>
+        rt_flush_period         = 300
+<?php } ?>
+
+}

etc/smtp.acl.example

@@ -0,0 +1,9 @@
+# Allow office365 servers. See the below URI for more
+# https://docs.microsoft.com/en-us/microsoft-365/enterprise/urls-and-ip-address-ranges?view=o365-worldwide
+#
+# Anything is not listed below will be rejected
+#
+40.92.0.0/15 permit
+40.107.0.0/16 permit
+52.100.0.0/14 permit
+104.47.0.0/17 permit

etc/sphinx.conf.in

@@ -1,5 +1,36 @@
+#!/usr/bin/php
+<?php
+
+define('SPHINX_VERSION', 331); // If you have sphinx-3.3.1, then set SPHINX_VERSION to 331
+define('LOCALSTATEDIR', '@LOCALSTATEDIR@');
+define('NGRAM_CONFIG', "        #ngram_len               = 1\n        #ngram_chars             = U+3000..U+2FA1F\n");
+
+# See http://sphinxsearch.com/wiki/doku.php?id=charset_tables for more on the charset_table settings
+# The following settings contains English and some Latin extras
+define('SPHINX_CHARSET_TABLE', "0..9, english, _, \
+                                  U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
+                                  U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
+                                  U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
+                                  U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171\n");
+
+// Sphinx 3.3.1 introduced some strict rules for fulltext search column names
+// In order to comply with it you must enable SPHINX_STRICT_SCHEMA variable
+// Be sure to check out http://www.mailpiler.org/wiki/current:sphinx3 for more
+// NB: The SPHINX_STRICT_SCHEMA in sphinx.conf MUST BE THE SAME as in config.php (or in config-site.php)
+//
+define('SPHINX_STRICT_SCHEMA', 1);
+define('RT', 0);
+
+if(SPHINX_STRICT_SCHEMA) {
+   define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
+} else {
+   define('SELECT_FIELDS', 'id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
+}
+
+?>
+
 #
-# Minimal Sphinx configuration sample (clean, simple, functional)
+# minimal sphinx configuration suited to piler
 #
 
 source base
@@ -11,172 +42,242 @@ source base
    sql_pass = MYSQL_PASSWORD
 
    sql_attr_uint = size
-   sql_attr_uint = arrived
    sql_attr_uint = sent
-   sql_attr_uint = direction
-   sql_attr_uint = folder
    sql_attr_uint = attachments
 }
 
+<?php if(!RT) { ?>
+
 source delta : base
 {
-   sql_query_pre = SET NAMES utf8
-   sql_query_pre  = REPLACE INTO sph_counter SELECT 1, MAX(id) FROM sph_index
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query_pre  = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index
    sql_query_post_index  = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index \
-        WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
 
+<?php if(SPHINX_VERSION < 300) { ?>
    sql_query_killlist = SELECT `id` FROM `metadata` WHERE `deleted`=1
+<?php } else { ?>
+   sql_query_kbatch = SELECT `id` FROM `metadata` WHERE `deleted`=1
+<?php } ?>
 }
 
 source main1 : base
 {
-   sql_query_pre = SET NAMES utf8
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1;
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
 }
 
 source main2 : base
 {
-   sql_query_pre = SET NAMES utf8
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1;
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
 }
 
 source main3 : base
 {
-   sql_query_pre = SET NAMES utf8
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1;
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
 }
 
 source main4 : base
 {
-   sql_query_pre = SET NAMES utf8
-   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1;
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
 }
 
+source dailydelta : base
+{
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+}
+<?php } ?>
+
 source tag : base
 {
-   sql_query_pre = SET NAMES utf8
-   sql_query  = SELECT `_id`, `id`, `uid`, `tag` FROM `tag`
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query  = SELECT `_id`, `id` AS iid, `uid`, `tag` FROM `tag`
 
-   sql_attr_uint = id
+   sql_attr_uint = iid
    sql_attr_uint = uid
 
 }
 
-
 source note : base
 {
-   sql_query_pre = SET NAMES utf8
-   sql_query  = SELECT `_id`, `id`, `uid`, `note` FROM `note`
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query  = SELECT `_id`, `id` AS iid, `uid`, `note` FROM `note`
 
-   sql_attr_uint = id
+   sql_attr_uint = iid
    sql_attr_uint = uid
 
 }
 
-
+<?php if(!RT) { ?>
 index main1
 {
-	source			= main1
-	path			= /var/piler/sphinx/main1
-	docinfo			= extern
-	charset_type		= utf-8
-        enable_star             = 1
-        min_prefix_len          = 6
+        source                  = main1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main1
+<?php if(SPHINX_VERSION < 300) { ?>
+        docinfo                 = extern
+        dict                    = keywords
+<?php } ?>
+        min_prefix_len          = 5
         min_word_len            = 1
-
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
 }
 
 index main2
 {
         source                  = main2
-        path                    = /var/piler/sphinx/main2
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main2
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-        charset_type            = utf-8
-        enable_star             = 1
-        min_prefix_len          = 6
+        dict                    = keywords
+<?php } ?>
+        min_prefix_len          = 5
         min_word_len            = 1
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
 }
 
 index main3
 {
         source                  = main3
-        path                    = /var/piler/sphinx/main3
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main3
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-        charset_type            = utf-8
-        enable_star             = 1
-        min_prefix_len          = 6
+        dict                    = keywords
+<?php } ?>
+        min_prefix_len          = 5
         min_word_len            = 1
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
 }
 
 index main4
 {
         source                  = main4
-        path                    = /var/piler/sphinx/main4
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main4
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-        charset_type            = utf-8
-        enable_star             = 1
-        min_prefix_len          = 6
+        dict                    = keywords
+<?php } ?>
+        min_prefix_len          = 5
         min_word_len            = 1
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+}
+
+index dailydelta1
+{
+        source                  = dailydelta
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/dailydelta1
+<?php if(SPHINX_VERSION < 300) { ?>
+        docinfo                 = extern
+        dict                    = keywords
+<?php } ?>
+        min_prefix_len          = 5
+        min_word_len            = 1
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
 }
 
 index delta1
 {
         source                  = delta
-        path                    = /var/piler/sphinx/delta1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/delta1
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-        charset_type            = utf-8
-        enable_star             = 1
-        min_prefix_len          = 6
+        dict                    = keywords
+<?php } ?>
+        min_prefix_len          = 5
         min_word_len            = 1
-
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
+<?php if(SPHINX_VERSION >= 310) { ?>
+        kbatch = main1, main2, main3, main4, dailydelta1
+<?php } ?>
 }
 
+<?php } ?>
 
 index tag1
 {
         source                  = tag
-        path                    = /var/piler/sphinx/tag1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/tag1
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-        charset_type            = utf-8
-        enable_star             = 1
-        min_prefix_len          = 6
+        dict                    = keywords
+<?php } ?>
+        min_prefix_len          = 5
         min_word_len            = 1
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
 }
 
 
 index note1
 {
         source                  = note
-        path                    = /var/piler/sphinx/note1
+        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/note1
+<?php if(SPHINX_VERSION < 300) { ?>
         docinfo                 = extern
-        charset_type            = utf-8
-        enable_star             = 1
-        min_prefix_len          = 6
+        dict                    = keywords
+<?php } ?>
+        min_prefix_len          = 5
         min_word_len            = 1
+        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
+<?php print NGRAM_CONFIG; ?>
 }
 
+<?php if(RT) { ?>
+index piler1
+{
+        type = rt
+        path = /var/piler/sphinx/piler1
+        rt_mem_limit = 128M
+        rt_field = sender
+        rt_field = rcpt
+        rt_field = senderdomain
+        rt_field = rcptdomain
+        rt_field = subject
+        rt_field = body
+        rt_field = attachment_types
+        rt_attr_bigint = arrived
+        rt_attr_bigint = sent
+        rt_attr_uint = size
+        rt_attr_uint = direction
+        rt_attr_uint = folder
+        rt_attr_uint = attachments
+}
+<?php } ?>
 
 indexer
 {
-	mem_limit		= 64M
+        mem_limit               = 256M
 }
 
 
 searchd
 {
-	listen			= 127.0.0.1:9312
-	listen			= 127.0.0.1:9306:mysql41
-	log			= /dev/null
-	binlog_path             = 
-	##query_log		=
-	read_timeout		= 5
-	max_children		= 30
-	pid_file		= /var/run/piler/searchd.pid
-	max_matches		= 1000
-	seamless_rotate		= 1
-	preopen_indexes		= 1
-	unlink_old		= 1
-	workers			= threads # for RT to work
-	compat_sphinxql_magics  = 0
+        listen                  = 127.0.0.1:9312
+        listen                  = 127.0.0.1:9306:mysql41
+        log                     = /dev/null
+        binlog_path             = /var/piler/sphinx
+        binlog_max_log_size     = 16M
+        binlog_flush            = 2
+<?php if(RT) { ?>
+        rt_flush_period         = 300
+<?php } ?>
+        ##query_log             =
+        read_timeout            = 5
+        max_children            = 30
+        pid_file                = /var/run/piler/searchd.pid
+        seamless_rotate         = 1
+        preopen_indexes         = 1
+        unlink_old              = 1
+        thread_stack            = 512k
+        workers                 = thread_pool
 }

init.d/Makefile.in

@@ -28,6 +28,8 @@ all:
 
 
 install:
+	$(INSTALL) -m 0755 $(srcdir)/rc.piler $(DESTDIR)/etc/init.d/rc.piler
+	$(INSTALL) -m 0755 $(srcdir)/rc.searchd $(DESTDIR)/etc/init.d/rc.searchd
 
 clean:
 	rm -f rc.piler rc.searchd

init.d/rc.piler.in

@@ -2,49 +2,80 @@
 ##
 ##
 
+### BEGIN INIT INFO
+# Provides:          piler
+# Required-Start:    $remote_fs $syslog $named $network $time mysql
+# Required-Stop:     $remote_fs $syslog $named $network mysql
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: piler email archiver
+# Description:       piler email archiver
+### END INIT INFO
+
+
 NAME=piler
-PID_FILE=`SBINDIR/pilerconf -q pidfile | cut -f2 -d=`
-PID_NUMBER=`test -f ${PID_FILE} && cat ${PID_FILE}`
+OPTIONS=""
+PID_FILE="$(SBINDIR/pilerconf $OPTIONS -q pidfile | cut -f2 -d=)"
+PID_NUMBER="$(test -f "$PID_FILE" && cat "$PID_FILE")"
+PILER_SMTP_PID="$(pgrep piler-smtp)"
 
 start() {
-	echo "starting piler . . ."
-	SBINDIR/piler -d
+   echo "starting piler-smtp . . . "
+   SBINDIR/piler-smtp -d
+
+   echo "starting $NAME . . ."
+   SBINDIR/piler -d $OPTIONS
 }
 
 stop() {
-	echo "stopping piler"
-	kill ${PID_NUMBER}
+   if [ "$PID_NUMBER" != "" ]; then echo "stopping piler"; kill "$PID_NUMBER"; fi
+
+   if [ "$PILER_SMTP_PID" != "" ]; then echo "stopping piler-smtp"; kill "$PILER_SMTP_PID"; fi
 }
 
 check_status(){
-        test -f /proc/${PID_NUMBER}/status
+        if [ -f "/proc/${PID_NUMBER}/status" ]; then
+           echo "piler is running, pid: ${PID_NUMBER}";
+        else
+           echo "piler is NOT running";
+        fi
+
+        if [ "${PILER_SMTP_PID}" != '' ]; then
+           echo "piler-smtp is running, pid: ${PILER_SMTP_PID}";
+        else
+           echo "piler-smtp is NOT running";
+        fi
 }
 
 case "$1" in
-	start)
-		start;
-   	;;
+        start)
+                start;
+        ;;
 
-	stop)
-		stop;
-   	;;
+        stop)
+                stop;
+        ;;
 
         status)
-                if check_status;
-                 then
-                echo "${NAME} is running."
-                 else
-                echo "${NAME} is not running."
+                if check_status; then
+                   exit 0
+                else
+                   exit 1
                 fi
         ;;
 
-	restart)
-		stop;
-		sleep 1;
-		start;
-   	;;
+        restart)
+                stop;
+                sleep 1;
+                start;
+        ;;
 
-	*)
-		echo "Usage: $0 start|stop|restart|status"
+        reload)
+                kill -HUP "$PID_NUMBER"
+                echo "reloaded"
+        ;;
+
+        *)
+                echo "Usage: $0 start|stop|restart|reload|status"
+        ;;
 esac
-

init.d/rc.searchd.in

@@ -2,18 +2,46 @@
 ##
 ##
 
+### BEGIN INIT INFO
+# Provides:          pilersearch
+# Required-Start:    $remote_fs $syslog $named $network $time
+# Required-Stop:     $remote_fs $syslog $named $network
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: sphinxsearch
+# Description:       sphinxsearch
+### END INIT INFO
+
+export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
+
 NAME=searchd
 PID_FILE=/var/run/piler/searchd.pid
-PID_NUMBER=`test -f ${PID_FILE} && cat ${PID_FILE}`
+PID_NUMBER=$(test -f ${PID_FILE} && cat ${PID_FILE})
+CONFIG_FILE=sphinx.conf
+
+if [ -f SYSCONFDIR/piler/MANTICORE ]; then
+   CONFIG_FILE=manticore.conf
+fi
+
 
 start() {
 	echo "starting searchd . . ."
-	su piler -c 'BINDIR/searchd'
+
+	if [ ! -d /var/run/piler ]; then
+		mkdir -p /var/run/piler
+		chown piler:piler /var/run/piler
+	fi
+
+        if [ $(id -u) -eq 0 ]; then
+		su piler -c "searchd --config SYSCONFDIR/piler/${CONFIG_FILE}"
+	else
+		searchd
+	fi
 }
 
 stop() {
 	echo "stopping searchd"
-	kill ${PID_NUMBER}
+	kill "$PID_NUMBER"
 }
 
 check_status(){
@@ -32,9 +60,11 @@ case "$1" in
         status)
                 if check_status;
                  then
-                echo "${NAME} is running."
+                    echo "${NAME} is running."
+                    exit 0
                  else
-                echo "${NAME} is not running."
+                    echo "${NAME} is not running."
+                    exit 1
                 fi
         ;;
 

piler-config.h.in

@@ -2,22 +2,30 @@
  * piler-config.h.in, SJ
  */
 
+#define VERSION "x.x.x"
+
 #define CONFDIR "/usr/local/etc"
 #define DATADIR "/usr/local/var"
+#define DATAROOTDIR "/usr/local/share"
 
-#define KEYFILE CONFDIR "/piler.key"
+#define KEYFILE CONFDIR "/piler/piler.key"
+#define LICENCE_SIGNATURE_FILE CONFDIR "/piler/piler.lic"
+
+#define MESSAGE_ID_DEDUP_FILE DATAROOTDIR "/piler/deduphelper"
 
 #define HAVE_DAEMON 1
 
+#undef TIMEOUT_BINARY
+
 #undef HAVE_PDFTOTEXT
 #undef HAVE_CATDOC
 #undef HAVE_CATPPT
 #undef HAVE_XLS2CSV
 #undef HAVE_PPTHTML
 #undef HAVE_UNRTF
+#undef HAVE_TNEF
 #undef HAVE_ZIP
 
-#undef HAVE_STARTTLS
-#undef HAVE_LIBWRAP
-
 #undef HAVE_TWEAK_SENT_TIME
+
+#undef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT

src/Makefile.in

@@ -16,11 +16,11 @@ localstatedir = @localstatedir@
 CC = @CC@
 CFLAGS = @CFLAGS@ @CPPFLAGS@
 DEFS = @defs@
-INCDIR = -I. -I.. @INCDIR@ @mysql_includes@
+INCDIR = -I. -I.. @INCDIR@ @sql_includes@
 LIBDIR = -L. @LIBDIR@ @LDFLAGS@
-LIBS = @LIBS@ @mysql_libs@
+LIBS = @LIBS@ @sql_libs@
 OBJS = @OBJS@
-MYSQL_OBJS = @mysql_obj@
+SQL_OBJS = @sql_obj@
 RUNNING_USER = @RUNNING_USER@
 RUNNING_GROUP = `@id_bin@ -gn $(RUNNING_USER)`
 
@@ -33,20 +33,22 @@ MAKE = `which make`
 
 INSTALL = @INSTALL@
 
-all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge reindex test
+all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport reindex test stats piler-smtp
 install: install-piler
 
 
 piler: piler.c libpiler.a
 	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR) @LDFLAGS@ @libclamav_extra_libs@
 
-libpiler.a: $(OBJS) $(MYSQL_OBJS)
-	ar cr libpiler.a $(OBJS) $(MYSQL_OBJS)
+libpiler.a: $(OBJS) $(SQL_OBJS)
+	ar cr libpiler.a $(OBJS) $(SQL_OBJS)
 	ranlib libpiler.a
-	$(CC) -shared -o libpiler.so.$(LIBPILER_VERSION) $(OBJS) $(MYSQL_OBJS)
+	$(CC) -shared -Wl,-soname,libpiler.so.$(LIBPILER_VERSION) -o libpiler.so.$(LIBPILER_VERSION) $(OBJS) $(SQL_OBJS)
 	ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so
 	ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so.$(PILER_VERSION)
 
+piler-smtp: piler-smtp.c libpiler.a
+	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< cfg.o misc.o tai.o smtp.o session.o dirs.o sig.o bdat.o screen.o $(LIBS) $(LIBDIR)
 
 pilerget: pilerget.c libpiler.a
 	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
@@ -60,9 +62,6 @@ pilerimport: pilerimport.c libpiler.a
 pilerexport: pilerexport.c libpiler.a
 	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
 
-pilerpurge: pilerpurge.c libpiler.a
-	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
-
 pilerconf: pilerconf.c libpiler.a
 	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
 
@@ -72,8 +71,11 @@ reindex: reindex.c libpiler.a
 test: test.c libpiler.a
 	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o pilertest $< -lpiler $(LIBS) $(LIBDIR) @LDFLAGS@
 
+stats: stats.c libpiler.a
+	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o pilerstats $< -lpiler $(LIBS) $(LIBDIR) @LDFLAGS@
+
 %.o: $(srcdir)/%.c
-	$(CC) $(CFLAGS) -fPIC $(INCDIR) $(DEFS) -c $< -o $@
+	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -c $< -o $@
 
 
 install-piler:
@@ -83,16 +85,18 @@ install-piler:
 	(cd $(DESTDIR)$(libdir) && ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so.$(PILER_VERSION))
 
 	$(INSTALL) -m 0755 piler $(DESTDIR)$(sbindir)
+	$(INSTALL) -m 0755 piler-smtp $(DESTDIR)$(sbindir)
 	$(INSTALL) -m 0755 pilerconf $(DESTDIR)$(sbindir)
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerget $(DESTDIR)$(bindir)
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pileraget $(DESTDIR)$(bindir)
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerimport $(DESTDIR)$(bindir)
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerexport $(DESTDIR)$(bindir)
-	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerpurge $(DESTDIR)$(bindir)
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) reindex $(DESTDIR)$(bindir)
+	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilertest $(DESTDIR)$(bindir)
+	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerstats $(DESTDIR)$(bindir)
 
 clean:
-	rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge pilertest reindex
+	rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilertest pilerstats reindex piler-smtp
 
 distclean: clean
 	rm -f Makefile

src/archive.c

@@ -16,6 +16,9 @@
 #include <syslog.h>
 #include <openssl/blowfish.h>
 #include <openssl/evp.h>
+#if OPENSSL_VERSION_MAJOR >= 3
+   #include <openssl/provider.h>
+#endif
 #include <zlib.h>
 #include <assert.h>
 #include <piler.h>
@@ -54,11 +57,14 @@ void zerr(int ret){
 
 int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
    int ret, pos=0;
-   unsigned have;
    z_stream strm;
    char *new_ptr;
    unsigned char out[REALLYBIGBUFSIZE];
 
+   /* expecting deflate with 32k window size (0x78) */
+   if(len > 0 && in[0] != 0x78)
+      return Z_DATA_ERROR;
+
    /* allocate inflate state */
 
    strm.zalloc = Z_NULL;
@@ -96,7 +102,7 @@ int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
                 return ret;
       }
 
-      have = REALLYBIGBUFSIZE - strm.avail_out;
+      unsigned have = REALLYBIGBUFSIZE - strm.avail_out;
 
       /*
        * write the uncompressed result either to stdout
@@ -131,11 +137,16 @@ int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
 }
 
 
-int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct __config *cfg){
+int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct config *cfg){
    int rc=0, n, olen, tlen, len, fd=-1;
    unsigned char *s=NULL, *addr=NULL, inbuf[REALLYBIGBUFSIZE];
    struct stat st;
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
    EVP_CIPHER_CTX ctx;
+#else
+   EVP_CIPHER_CTX *ctx=NULL;
+#endif
+   int blocklen;
 
 
    if(filename == NULL) return 1;
@@ -154,12 +165,47 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
       return 1;
    }
 
+   // The new encryption scheme uses piler id starting with 5000....
 
    if(cfg->encrypt_messages == 1){
+   #if OPENSSL_VERSION_NUMBER < 0x10100000L
       EVP_CIPHER_CTX_init(&ctx);
-      EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      if(strstr(filename, "/5000")){
+         rc = EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+      } else {
+         rc = EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      }
 
-      len = st.st_size+EVP_MAX_BLOCK_LENGTH;
+      if(!rc){
+         syslog(LOG_PRIORITY, "ERROR: EVP_DecryptInit_ex()");
+         goto CLEANUP;
+      }
+
+      blocklen = EVP_CIPHER_CTX_block_size(&ctx);
+   #else
+      ctx = EVP_CIPHER_CTX_new();
+      if(!ctx) goto CLEANUP;
+
+      EVP_CIPHER_CTX_init(ctx);
+      if(strstr(filename, "/5000")){
+         rc = EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+      } else {
+      #if OPENSSL_VERSION_MAJOR >= 3
+         OSSL_PROVIDER_load(NULL, "legacy");
+         OSSL_PROVIDER_load(NULL, "default");
+      #endif
+         rc = EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      }
+
+      if(!rc){
+         syslog(LOG_PRIORITY, "ERROR: EVP_DecryptInit_ex()");
+         goto CLEANUP;
+      }
+
+      blocklen = EVP_CIPHER_CTX_block_size(ctx);
+   #endif
+
+      len = st.st_size+blocklen;
 
       s = malloc(len);
 
@@ -172,7 +218,11 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
 
       while((n = read(fd, inbuf, sizeof(inbuf)))){
 
+      #if OPENSSL_VERSION_NUMBER < 0x10100000L
          if(!EVP_DecryptUpdate(&ctx, s+tlen, &olen, inbuf, n)){
+      #else
+         if(!EVP_DecryptUpdate(ctx, s+tlen, &olen, inbuf, n)){
+      #endif
             syslog(LOG_PRIORITY, "%s: EVP_DecryptUpdate()", filename);
             goto CLEANUP;
          }
@@ -181,14 +231,24 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
       }
 
 
+   #if OPENSSL_VERSION_NUMBER < 0x10100000L
       if(EVP_DecryptFinal(&ctx, s + tlen, &olen) != 1){
+   #else
+      if(EVP_DecryptFinal(ctx, s + tlen, &olen) != 1){
+   #endif
          syslog(LOG_PRIORITY, "%s: EVP_DecryptFinal()", filename);
          goto CLEANUP;
       }
 
 
       tlen += olen;
+
+      // old fileformat with static IV
       rc = inf(s, tlen, mode, buffer, dest);
+      // new fileformat, starting with blocklen bytes of garbage
+      if(rc != Z_OK && tlen >= blocklen){
+         rc = inf(s+blocklen, tlen-blocklen, mode, buffer, dest);
+      }
    }
    else {
       addr = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
@@ -201,37 +261,45 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
 
 
 CLEANUP:
-   if(fd != -1) close(fd);
+   if(fd != -1) close(fd); //-V547
    if(s) free(s);
-   if(cfg->encrypt_messages == 1) EVP_CIPHER_CTX_cleanup(&ctx);
+   if(cfg->encrypt_messages == 1)
+   #if OPENSSL_VERSION_NUMBER < 0x10100000L
+      EVP_CIPHER_CTX_cleanup(&ctx);
+   #else
+      if(ctx) EVP_CIPHER_CTX_free(ctx);
+   #endif
 
    return 0;
 }
 
 
-int retrieve_email_from_archive(struct session_data *sdata, struct __data *data, FILE *dest, struct __config *cfg){
-   int i, attachments;
-   char *buffer=NULL, *saved_buffer, *p, filename[SMALLBUFSIZE], pointer[SMALLBUFSIZE];
+int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct config *cfg){
+   int attachments;
+   char *buffer=NULL, *saved_buffer, *p, filename[SMALLBUFSIZE];
    struct ptr_array ptr_arr[MAX_ATTACHMENTS];
+#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
    struct stat st;
-
+#endif
 
    if(strlen(sdata->ttmpfile) != RND_STR_LEN){
       printf("invalid piler-id: %s\n", sdata->ttmpfile);
       return 1;
    }
 
-   attachments = query_attachments(sdata, data, &ptr_arr[0], cfg);
+   attachments = query_attachments(sdata, &ptr_arr[0]);
 
    if(attachments == -1){
       printf("problem querying the attachment of %s\n", sdata->ttmpfile);
       return 1;
    }
 
-   snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
+   snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, *(sdata->ttmpfile+24), *(sdata->ttmpfile+25), *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
+#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
    if(stat(filename, &st)){
       snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+RND_STR_LEN-6), *(sdata->ttmpfile+RND_STR_LEN-5), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
    }
+#endif
 
    if(attachments == 0){
       retrieve_file_from_archive(filename, WRITE_TO_STDOUT, &buffer, dest, cfg);
@@ -242,7 +310,8 @@ int retrieve_email_from_archive(struct session_data *sdata, struct __data *data,
       if(buffer){
          saved_buffer = buffer;
 
-         for(i=1; i<=attachments; i++){
+         for(int i=1; i<=attachments; i++){
+            char pointer[SMALLBUFSIZE];
             snprintf(pointer, sizeof(pointer)-1, "ATTACHMENT_POINTER_%s.a%d_XXX_PILER", sdata->ttmpfile, i);
 
             p = strstr(buffer, pointer);
@@ -252,11 +321,13 @@ int retrieve_email_from_archive(struct session_data *sdata, struct __data *data,
                buffer = p + strlen(pointer);
 
                if(strlen(ptr_arr[i].piler_id) == RND_STR_LEN){
-                  snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);
+                  snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, ptr_arr[i].piler_id[24], ptr_arr[i].piler_id[25], ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);
 
+               #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
                   if(stat(filename, &st)){
                      snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, ptr_arr[i].piler_id[RND_STR_LEN-6], ptr_arr[i].piler_id[RND_STR_LEN-5], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);
                   }
+               #endif
 
                   retrieve_file_from_archive(filename, WRITE_TO_STDOUT, NULL, dest, cfg);
                }
@@ -275,5 +346,3 @@ int retrieve_email_from_archive(struct session_data *sdata, struct __data *data,
 
    return 0;
 }
-
-

src/attachment.c

@@ -16,121 +16,58 @@
 #include <piler.h>
 
 
-int store_attachments(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg){
-   uint64 id=0;
+int store_attachments(struct session_data *sdata, struct parser_state *state, struct config *cfg){
    int i, rc=1, found, affected_rows;
-   MYSQL_BIND bind[7];
-   unsigned long len[7];
+   struct sql sql, sql2;
 
-
-   if(prepare_a_mysql_statement(sdata, &(data->stmt_insert_into_attachment_table), SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE) == ERR) return rc;
-   if(prepare_a_mysql_statement(sdata, &(data->stmt_get_attachment_id_by_signature), SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE) == ERR) return rc;
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE) == ERR) return rc;
+   if(prepare_sql_statement(sdata, &sql2, SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE) == ERR) return rc;
 
 
    for(i=1; i<=state->n_attachments; i++){
       found = 0;
-      id = 0;
+      uint64 id = 0;
 
       if(state->attachments[i].size > 0){
 
-         memset(bind, 0, sizeof(bind));
+         p_bind_init(&sql2);
 
-         bind[0].buffer_type = MYSQL_TYPE_STRING;
-         bind[0].buffer = state->attachments[i].digest;
-         bind[0].is_null = 0;
-         len[0] = strlen(state->attachments[i].digest); bind[0].length = &len[0];
+         sql2.sql[sql2.pos] = state->attachments[i].digest; sql2.type[sql2.pos] = TYPE_STRING; sql2.pos++;
+         sql2.sql[sql2.pos] = (char *)&(state->attachments[i].size); sql2.type[sql2.pos] = TYPE_LONG; sql2.pos++;
 
-         if(mysql_stmt_bind_param(data->stmt_get_attachment_id_by_signature, bind)){
-            syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error for get attachment id: %s", sdata->ttmpfile, SQL_ATTACHMENT_TABLE, mysql_stmt_error(data->stmt_get_attachment_id_by_signature));
-            goto NOT_FOUND;
+         if(p_exec_stmt(sdata, &sql2) == OK){
+
+            p_bind_init(&sql2);
+
+            sql2.sql[sql2.pos] = (char *)&id; sql2.type[sql2.pos] = TYPE_LONGLONG; sql2.len[sql2.pos] = sizeof(uint64); sql2.pos++;
+
+            p_store_results(&sql2);
+            if(p_fetch_results(&sql2) == OK) found = 1;
+            p_free_results(&sql2);
          }
 
-         if(mysql_stmt_execute(data->stmt_get_attachment_id_by_signature)){
-            syslog(LOG_PRIORITY, "%s get attachment id execute error: *%s*", sdata->ttmpfile, mysql_error(&(sdata->mysql)));
-            goto NOT_FOUND;
-         }
-
-
-         memset(bind, 0, sizeof(bind));
-
-         bind[0].buffer_type = MYSQL_TYPE_LONGLONG;
-         bind[0].buffer = (char *)&id;
-         bind[0].is_null = 0;
-         bind[0].length = 0;
-
-         if(mysql_stmt_bind_result(data->stmt_get_attachment_id_by_signature, bind)){
-            syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_result() error: %s", sdata->ttmpfile, SQL_ATTACHMENT_TABLE, mysql_stmt_error(data->stmt_get_attachment_id_by_signature));
-            goto NOT_FOUND;
-         }
-
-         if(mysql_stmt_store_result(data->stmt_get_attachment_id_by_signature)){
-            goto NOT_FOUND;
-         }
-
-         if(!mysql_stmt_fetch(data->stmt_get_attachment_id_by_signature)){
-            found = 1;
-         }
-
-NOT_FOUND:
-
          if(found == 0){
-            if(store_file(sdata, state->attachments[i].internalname, 0, 0, cfg) == 0){
+            if(store_file(sdata, state->attachments[i].internalname, 0, cfg) == 0){
                syslog(LOG_PRIORITY, "%s: error storing attachment: %s", sdata->ttmpfile, state->attachments[i].internalname);
                goto CLOSE;
             }
          }
 
 
-         memset(bind, 0, sizeof(bind));
+         p_bind_init(&sql);
 
-         bind[0].buffer_type = MYSQL_TYPE_STRING;
-         bind[0].buffer = sdata->ttmpfile;
-         bind[0].is_null = 0;
-         len[0] = strlen(sdata->ttmpfile); bind[0].length = &len[0];
+         sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+         sql.sql[sql.pos] = (char *)&i; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+         sql.sql[sql.pos] = state->attachments[i].digest; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+         sql.sql[sql.pos] = state->attachments[i].filename; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+         sql.sql[sql.pos] = state->attachments[i].type; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+         sql.sql[sql.pos] = (char *)&(state->attachments[i].size); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+         sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
 
-         bind[1].buffer_type = MYSQL_TYPE_LONG;
-         bind[1].buffer = (char *)&i;
-         bind[1].is_null = 0;
-         bind[1].length = 0;
-
-         bind[2].buffer_type = MYSQL_TYPE_STRING;
-         bind[2].buffer = state->attachments[i].digest;
-         bind[2].is_null = 0;
-         len[2] = strlen(state->attachments[i].digest); bind[2].length = &len[2];
-
-         bind[3].buffer_type = MYSQL_TYPE_STRING;
-         bind[3].buffer = state->attachments[i].filename;
-         bind[3].is_null = 0;
-         len[3] = strlen(state->attachments[i].filename); bind[3].length = &len[3];
-
-         bind[4].buffer_type = MYSQL_TYPE_STRING;
-         bind[4].buffer = state->attachments[i].type;
-         bind[4].is_null = 0;
-         len[4] = strlen(state->attachments[i].type); bind[4].length = &len[4];
-
-         bind[5].buffer_type = MYSQL_TYPE_LONG;
-         bind[5].buffer = (char *)&(state->attachments[i].size);
-         bind[5].is_null = 0;
-         bind[5].length = 0;
-
-         bind[6].buffer_type = MYSQL_TYPE_LONGLONG;
-         bind[6].buffer = (char *)&id;
-         bind[6].is_null = 0;
-         bind[6].length = 0;
+         if(p_exec_stmt(sdata, &sql) == ERR) goto CLOSE;
 
 
-         if(mysql_stmt_bind_param(data->stmt_insert_into_attachment_table, bind)){
-            syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_ATTACHMENT_TABLE, mysql_stmt_error(data->stmt_insert_into_attachment_table));
-            goto CLOSE;
-         }
-
-
-         if(mysql_stmt_execute(data->stmt_insert_into_attachment_table)){
-            syslog(LOG_PRIORITY, "%s attachment sql error: *%s*", sdata->ttmpfile, mysql_error(&(sdata->mysql)));
-            goto CLOSE;
-         }
-
-         affected_rows = mysql_stmt_affected_rows(data->stmt_insert_into_attachment_table);
+         affected_rows = p_get_affected_rows(&sql);
          if(affected_rows != 1){
             syslog(LOG_PRIORITY, "%s attachment sql error: affected rows: %d", sdata->ttmpfile, affected_rows);
             goto CLOSE;
@@ -146,139 +83,70 @@ NOT_FOUND:
    rc = 0;
 
 CLOSE:
-   mysql_stmt_close(data->stmt_insert_into_attachment_table);
-   mysql_stmt_close(data->stmt_get_attachment_id_by_signature);
+   close_prepared_statement(&sql);
+   close_prepared_statement(&sql2);
 
    return rc;
 }
 
 
-int query_attachment_pointers(struct session_data *sdata, struct __data *data, uint64 ptr, char *piler_id, int *id, struct __config *cfg){
+int query_attachment_pointers(struct session_data *sdata, uint64 ptr, char *piler_id, int *id){
    int rc=0;
-   MYSQL_BIND bind[2];
-   my_bool is_null[2];
-   unsigned long len=0;
+   struct sql sql;
 
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER) == ERR) return rc;
 
-   if(prepare_a_mysql_statement(sdata, &(data->stmt_get_attachment_pointer), SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER) == ERR) goto ENDE;
+   p_bind_init(&sql);
 
-   memset(bind, 0, sizeof(bind));
+   sql.sql[sql.pos] = (char *)&ptr; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
 
-   bind[0].buffer_type = MYSQL_TYPE_LONGLONG;
-   bind[0].buffer = (char *)&ptr;
-   bind[0].is_null = 0;
-   len = sizeof(uint64); bind[0].length = &len;
+   if(p_exec_stmt(sdata, &sql) == OK){
 
+      p_bind_init(&sql);
 
-   if(mysql_stmt_bind_param(data->stmt_get_attachment_pointer, bind)){
-      goto CLOSE;
+      sql.sql[sql.pos] = piler_id; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++;
+      sql.sql[sql.pos] = (char *)id; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++;
+
+      p_store_results(&sql);
+
+      if(p_fetch_results(&sql) == OK) rc = 1;
+      p_free_results(&sql);
    }
 
-
-   if(mysql_stmt_execute(data->stmt_get_attachment_pointer)){
-      goto CLOSE;
-   }
-
-
-   memset(bind, 0, sizeof(bind));
-
-   bind[0].buffer_type = MYSQL_TYPE_STRING;
-   bind[0].buffer = piler_id;
-   bind[0].buffer_length = RND_STR_LEN;
-   bind[0].is_null = &is_null[0];
-   bind[0].length = &len;
-
-   bind[1].buffer_type = MYSQL_TYPE_LONG;
-   bind[1].buffer = (char *)id;
-   bind[1].is_null = 0;
-   bind[1].length = 0;
-
-
-   if(mysql_stmt_bind_result(data->stmt_get_attachment_pointer, bind)){
-      goto CLOSE;
-   }
-
-
-   if(mysql_stmt_store_result(data->stmt_get_attachment_pointer)){
-      goto CLOSE;
-   }
-
-   if(!mysql_stmt_fetch(data->stmt_get_attachment_pointer)){
-      if(is_null[0] == 0){
-         rc = 1;
-      }
-   }
-
-CLOSE:
-   mysql_stmt_close(data->stmt_get_attachment_pointer);
-
-ENDE:
+   close_prepared_statement(&sql);
 
    return rc;
 }
 
 
-int query_attachments(struct session_data *sdata, struct __data *data, struct ptr_array *ptr_arr, struct __config *cfg){
+int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr){
    int i, rc, id, attachments=0;
    uint64 ptr;
-   MYSQL_BIND bind[2];
-   my_bool is_null[2];
-   unsigned long len=0;
-
+   struct sql sql;
 
    for(i=0; i<MAX_ATTACHMENTS; i++) memset((char*)&ptr_arr[i], 0, sizeof(struct ptr_array));
 
-   if(prepare_a_mysql_statement(sdata, &(data->stmt_query_attachment), SQL_PREPARED_STMT_QUERY_ATTACHMENT) == ERR) goto ENDE;
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_QUERY_ATTACHMENT) == ERR) return attachments;
 
+   p_bind_init(&sql);
 
-   memset(bind, 0, sizeof(bind));
+   sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
 
-   bind[0].buffer_type = MYSQL_TYPE_STRING;
-   bind[0].buffer = sdata->ttmpfile;
-   bind[0].is_null = 0;
-   len = strlen(sdata->ttmpfile); bind[0].length = &len;
+   if(p_exec_stmt(sdata, &sql) == ERR) goto CLOSE;
 
-   if(mysql_stmt_bind_param(data->stmt_query_attachment, bind)){
-      goto CLOSE;
-   }
+   p_bind_init(&sql);
 
+   sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++;
+   sql.sql[sql.pos] = (char *)&ptr; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
 
-   if(mysql_stmt_execute(data->stmt_query_attachment)){
-      goto CLOSE;
-   }
+   p_store_results(&sql);
 
-
-   memset(bind, 0, sizeof(bind));
-
-   bind[0].buffer_type = MYSQL_TYPE_LONG;
-   bind[0].buffer = (char *)&id;
-   bind[0].is_null = &is_null[0];
-   bind[0].length = 0;
-
-   bind[1].buffer_type = MYSQL_TYPE_LONGLONG;
-   bind[1].buffer = (char *)&ptr;
-   bind[1].is_null = &is_null[1];
-   bind[1].length = 0;
-
-
-
-   if(mysql_stmt_bind_result(data->stmt_query_attachment, bind)){
-      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_result() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(data->stmt_query_attachment));
-      goto CLOSE;
-   }
-
-
-   if(mysql_stmt_store_result(data->stmt_query_attachment)){
-      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_store_result() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(data->stmt_query_attachment));
-      goto CLOSE;
-   }
-
-   while(!mysql_stmt_fetch(data->stmt_query_attachment)){
+   while(p_fetch_results(&sql) == OK){
 
       if(id > 0 && id < MAX_ATTACHMENTS){
          if(ptr > 0){
             ptr_arr[id].ptr = ptr;
-            rc = query_attachment_pointers(sdata, data, ptr, &(ptr_arr[id].piler_id[0]), &(ptr_arr[id].attachment_id), cfg);
+            rc = query_attachment_pointers(sdata, ptr, &(ptr_arr[id].piler_id[0]), &(ptr_arr[id].attachment_id));
             if(!rc){
                attachments = -1;
                goto CLOSE;
@@ -293,12 +161,10 @@ int query_attachments(struct session_data *sdata, struct __data *data, struct pt
       }
    }
 
-CLOSE:
-   mysql_stmt_close(data->stmt_query_attachment);
+   p_free_results(&sql);
 
-ENDE:
+CLOSE:
+   close_prepared_statement(&sql);
 
    return attachments;
 }
-
-

src/av.h

@@ -20,60 +20,6 @@
 #define CLAMD_RESP_INFECTED "FOUND"
 #define CLAMD_RESP_ERROR "ERROR"
 
-int clamd_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg);
-int clamd_net_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg);
-
-// Dr.Web stuff
-
-#define DRWEB_RESP_VIRUS 0x20
-#define DRWEB_VIRUS_HAS_FOUND_MESSAGE "Virus has been found in message. See drwebd.log for details"
-
-int drweb_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg);
-
-
-// avast! stuff
-
-#define AVAST_READY "220"
-#define AVAST_CMD_QUIT "QUIT\r\n"
-
-#define AVAST_RESP_OK "200"
-#define AVAST_RESP_ENGINE_ERROR "451"
-#define AVAST_RESP_SYNTAX_ERROR "501"
-
-#define AVAST_RESP_CLEAN "[+]"
-#define AVAST_RESP_INFECTED "[L]"
-
-int avast_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg);
-int avast_cmd_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg);
-
-// Kaspersky stuff
-
-#define KAV_CMD_QUIT "QUIT\r\n"
-
-#define KAV_READY "201 "
-#define KAV_RESP_CLEAN "220 File is clean"
-#define KAV_RESP_INFECTED "230 File is infected"
-#define KAV_RESP_INFECTED_NAME "322-"
-#define KAV_RESP_NOT_FOUND "525 File not found"
-
-int kav_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg);
-
-// avg stuff
-
-#define AVG_READY "220"
-#define AVG_CMD_QUIT "QUIT\r\n"
-
-#define AVG_RESP_OK "200"
-#define AVG_RESP_VIRUS "403"
-#define AVG_RESP_NOT_FOUND "404"
-#define AVG_RESP_ERROR "501"
-
-#define AVG_NOT_FOUND 404
-
-int avg_scan(char *tmpdir, char *tmpfile, char *engine, char *avinfo, struct __config *cfg);
-
-
-int moveMessageToQuarantine(struct session_data *sdata, struct __config *cfg);
-void sendNotificationToPostmaster(struct session_data *sdata, char *rcpttoemail, char *fromemail, char *virusinfo, char *avengine, struct __config *cfg);
+int clamd_scan(char *tmpfile, struct config *cfg);
 
 #endif /* _AV_H */

src/avir.c

@@ -10,46 +10,12 @@
 #include <piler.h>
 
 
-int do_av_check(struct session_data *sdata, char *rcpttoemail, char *virusinfo, struct __data *data, struct __config *cfg){
+int do_av_check(char *filename, struct config *cfg){
    int rav = AVIR_OK;
-   char avengine[SMALLBUFSIZE];
 
-   if(sdata->need_scan == 0) return rav;
+   if(clamd_scan(filename, cfg) == AV_VIRUS) rav = AVIR_VIRUS;
 
-   memset(avengine, 0, SMALLBUFSIZE);
-
-#ifdef HAVE_LIBCLAMAV
-   const char *virname;
-   unsigned int options=0;
-
-   options = CL_SCAN_STDOPT | CL_SCAN_ARCHIVE | CL_SCAN_MAIL | CL_SCAN_OLE2;
-
-   if(cfg->use_libclamav_block_max_feature == 1) options |= CL_SCAN_BLOCKMAX;
-
-   if(cfg->clamav_block_encrypted_archives == 1) options |= CL_SCAN_BLOCKENCRYPTED;
-
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to pass to libclamav", sdata->ttmpfile);
-
-   if(cl_scanfile(sdata->ttmpfile, &virname, NULL, data->engine, options) == CL_VIRUS){
-      memset(virusinfo, 0, SMALLBUFSIZE);
-      strncpy(virusinfo, virname, SMALLBUFSIZE-1);
-      rav = AVIR_VIRUS;
-      snprintf(avengine, SMALLBUFSIZE-1, "libClamAV");
-   }
-
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: virus info: '%s'", sdata->ttmpfile, virname);
-#endif
-
-#ifdef HAVE_CLAMD
-   if(strlen(cfg->clamd_addr) > 3 && cfg->clamd_port > 0){
-      if(clamd_net_scan(sdata->ttmpfile, avengine, virusinfo, cfg) == AV_VIRUS) rav = AVIR_VIRUS;
-   } else {
-      if(clamd_scan(sdata->ttmpfile, avengine, virusinfo, cfg) == AV_VIRUS) rav = AVIR_VIRUS;
-   }
-#endif
-
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: done virus scanning", sdata->ttmpfile);
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: done virus scanning", filename);
 
    return rav;
 }
-

src/base64.c

@@ -1,79 +0,0 @@
-/*
- * base64.c, SJ
- */
-
-#include <stdio.h>
-#include <string.h>
-
-
-char base64_value(char c){
-   static const char *base64_table = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-
-   if((int)c > 63) return '=';
-
-   return base64_table[(int)c];
-}
-
-
-void base64_encode_block(unsigned char *in, int inlen, char *out){
-   char a, b, c, d, fragment;
-
-   sprintf(out, "====");
-
-   if(inlen <= 0) return;
-
-   fragment = *in & 0x3;
-
-   a = *in >> 2;
-
-   out[0] = base64_value(a);
-
-   b = fragment << 4;
-
-   if(inlen > 1)
-      b += *(in+1) >> 4;
-
-   out[1] = base64_value(b);
-
-   if(inlen == 1) return;
-
-
-   c = *(in+1) & 0xf;
-   c = c << 2;
-
-   if(inlen > 2){
-      fragment = *(in+2) & 0xfc;
-      c += fragment >> 6;
-
-      d = *(in+2) & 0x3f;
-      out[3] = base64_value(d);
-   }
-
-   out[2] = base64_value(c);
-}
-
-
-void base64_encode(unsigned char *in, int inlen, char *out, int outlen){
-   int i=0, j, pos=0;
-   unsigned char buf[3];
-
-   memset(buf, 0, 3);
-   memset(out, 0, outlen);
-
-   for(j=0; j<inlen; j++){
-
-      if(i == 3){
-         base64_encode_block(buf, 3, &out[pos]); pos += 4;
-
-         memset(buf, 0, 3);
-
-         i = 0;
-      }
-
-      buf[i] = *(in+j);
-      i++;       
-   }
-
-   base64_encode_block(buf, i, &out[pos]);
-}
-

src/bdat.c

@@ -0,0 +1,117 @@
+/*
+ * bdat.c, SJ
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <signal.h>
+#include <syslog.h>
+#include <time.h>
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+#include <piler.h>
+#include <smtp.h>
+
+
+void reset_bdat_counters(struct smtp_session *session){
+   session->bdat_bytes_to_read = 0;
+   session->bad = 0;
+}
+
+
+void get_bdat_size_to_read(struct smtp_session *session){
+   char *p;
+
+   session->bdat_bytes_to_read = 0;
+
+   session->protocol_state = SMTP_STATE_BDAT;
+
+   p = strcasestr(session->buf, " LAST");
+   if(p){
+      *p = '\0';
+   }
+
+   // determine the size to be read
+
+   p = strchr(session->buf, ' ');
+   if(p){
+      session->bdat_bytes_to_read = atoi(p);
+      if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "fd=%d: BDAT len=%d", session->net.socket, session->bdat_bytes_to_read);
+   }
+
+   if(!p || session->bdat_bytes_to_read <= 0){
+      session->bdat_bytes_to_read = 0;
+      syslog(LOG_INFO, "%s: ERROR: malformed BDAT command", session->ttmpfile);
+   }
+}
+
+
+void process_bdat(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg){
+   if(readlen <= 0) return;
+
+   if(session->fd == -1){
+      session->fd = open(session->ttmpfile, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR|S_IRGRP);
+      if(session->fd == -1){
+         syslog(LOG_PRIORITY, "%s: %s", ERR_OPEN_TMP_FILE, session->ttmpfile);
+      }
+
+      if(cfg->process_rcpt_to_addresses == 1) write_envelope_addresses(session, cfg);
+   }
+
+   session->bdat_bytes_to_read -= readlen;
+
+   if(session->fd != -1){
+      if(write(session->fd, readbuf, readlen) != -1){
+         session->tot_len += readlen;
+
+         if(session->cfg->verbosity >= _LOG_EXTREME) syslog(LOG_INFO, "%s: wrote %d bytes, %d bytes to go", session->ttmpfile, readlen, session->bdat_bytes_to_read);
+      }
+      else syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
+   }
+
+
+   if(session->bdat_bytes_to_read < 0){
+      // malformed data from client: we got more data then had been told in BDAT argument
+      syslog(LOG_PRIORITY, "ERROR: invalid BDAT data. Expected %d, got %d bytes", session->bdat_bytes_to_read + readlen, readlen);
+      session->bad = 1;
+
+      close(session->fd);
+      unlink(session->ttmpfile);
+
+      session->fd = -1;
+   }
+
+
+   // If there's nothing more to read, then send response to smtp client
+
+   if(session->bdat_bytes_to_read <= 0){
+
+      if(session->fd == -1){
+         send_smtp_response(session, SMTP_RESP_421_ERR_WRITE_FAILED);
+      }
+      else {
+         fsync(session->fd);
+         close(session->fd);
+
+         session->fd = -1;
+
+         move_email(session);
+
+         char buf[SMALLBUFSIZE];
+         snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
+         send_smtp_response(session, buf);
+         syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d, client=%s, fd=%d", session->ttmpfile, session->mailfrom, session->tot_len, session->remote_host, session->net.socket);
+      }
+
+      // technically we are not in the PERIOD state, but it's good enough
+      // to quit the BDAT processing state
+      session->protocol_state = SMTP_STATE_PERIOD;
+   }
+}

src/cfg.c

@@ -17,30 +17,16 @@ int string_parser(char *src, char *target, int limit){
    return 0;
 };
 
-int multi_line_string_parser(char *src, char *target, int limit){
-   if(strlen(src) > 0 && strlen(target) + strlen(src) + 3 < limit){
-      strncat(target, src, limit);
-      strncat(target, "\r\n", limit);
-
-      return 0;
-   }
-
-   return 1;
-};
-
-int int_parser(char *src, int *target, int limit){
+int int_parser(char *src, int *target){
    *target = strtol(src, (char **) NULL, 10);
 
    return 0;
 };
 
-
-int float_parser(char *src, float *target, int limit){
-   *target = strtof(src, (char **) NULL);
-
+int uint64_parser(char *src, uint64 *target){
+   *target = strtoull(src, (char**)NULL, 10);
    return 0;
-};
-
+}
 
 struct _parse_rule {
    char *name;
@@ -58,51 +44,73 @@ struct _parse_rule {
 
 struct _parse_rule config_parse_rules[] =
 {
+   { "archive_address", "string", (void*) string_parser, offsetof(struct config, archive_address), "", MAXVAL-1},
+   { "archive_emails_not_having_message_id", "integer", (void*) int_parser, offsetof(struct config, archive_emails_not_having_message_id), "0", sizeof(int)},
+   { "archive_only_mydomains", "integer", (void*) int_parser, offsetof(struct config, archive_only_mydomains), "0", sizeof(int)},
+   { "backlog", "integer", (void*) int_parser, offsetof(struct config, backlog), "20", sizeof(int)},
+   { "check_for_client_timeout_interval", "integer", (void*) int_parser, offsetof(struct config, check_for_client_timeout_interval), "20", sizeof(int)},
+   { "cipher_list", "string", (void*) string_parser, offsetof(struct config, cipher_list), "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS", MAXVAL-1},
+   { "clamd_addr", "string", (void*) string_parser, offsetof(struct config, clamd_addr), "", MAXVAL-1},
+   { "clamd_port", "integer", (void*) int_parser, offsetof(struct config, clamd_port), "0", sizeof(int)},
+   { "clamd_socket", "string", (void*) string_parser, offsetof(struct config, clamd_socket), CLAMD_SOCKET, MAXVAL-1},
+   { "debug", "integer", (void*) int_parser, offsetof(struct config, debug), "0", sizeof(int)},
+   { "default_retention_days", "integer", (void*) int_parser, offsetof(struct config, default_retention_days), "2557", sizeof(int)},
+   { "enable_chunking", "integer", (void*) int_parser, offsetof(struct config, enable_chunking), "0", sizeof(int)},
+   { "enable_cjk", "integer", (void*) int_parser, offsetof(struct config, enable_cjk), "0", sizeof(int)},
+   { "enable_folders", "integer", (void*) int_parser, offsetof(struct config, enable_folders), "0", sizeof(int)},
+   { "encrypt_messages", "integer", (void*) int_parser, offsetof(struct config, encrypt_messages), "1", sizeof(int)},
+   { "extra_to_field", "string", (void*) string_parser, offsetof(struct config, extra_to_field), "", MAXVAL-1},
+   { "extract_attachments", "integer", (void*) int_parser, offsetof(struct config, extract_attachments), "1", sizeof(int)},
+   { "helper_timeout", "integer", (void*) int_parser, offsetof(struct config, helper_timeout), "20", sizeof(int)},
+   { "hostid", "string", (void*) string_parser, offsetof(struct config, hostid), HOSTID, MAXVAL-1},
+   { "iv", "string", (void*) string_parser, offsetof(struct config, iv), "", MAXVAL-1},
+   { "listen_addr", "string", (void*) string_parser, offsetof(struct config, listen_addr), "0.0.0.0", MAXVAL-1},
+   { "listen_port", "integer", (void*) int_parser, offsetof(struct config, listen_port), "25", sizeof(int)},
+   { "locale", "string", (void*) string_parser, offsetof(struct config, locale), "", MAXVAL-1},
+   { "max_connections", "integer", (void*) int_parser, offsetof(struct config, max_connections), "64", sizeof(int)},
+   { "max_message_size", "integer", (void*) int_parser, offsetof(struct config, max_message_size), "50000000", sizeof(int)},
+   { "max_requests_per_child", "integer", (void*) int_parser, offsetof(struct config, max_requests_per_child), "10000", sizeof(int)},
+   { "max_smtp_memory", "uint64", (void*) uint64_parser, offsetof(struct config, max_smtp_memory), "500000000", sizeof(uint64)},
+   { "memcached_servers", "string", (void*) string_parser, offsetof(struct config, memcached_servers), "127.0.0.1", MAXVAL-1},
+   { "memcached_to_db_interval", "integer", (void*) int_parser, offsetof(struct config, memcached_to_db_interval), "900", sizeof(int)},
+   { "memcached_ttl", "integer", (void*) int_parser, offsetof(struct config, memcached_ttl), "86400", sizeof(int)},
+   { "min_message_size", "integer", (void*) int_parser, offsetof(struct config, min_message_size), "100", sizeof(int)},
+   { "min_word_len", "integer", (void*) int_parser, offsetof(struct config, min_word_len), "1", sizeof(int)},
+   { "mmap_dedup_test", "integer", (void*) int_parser, offsetof(struct config, mmap_dedup_test), "0", sizeof(int)},
+   { "mysqlcharset", "string", (void*) string_parser, offsetof(struct config, mysqlcharset), "utf8mb4", MAXVAL-1},
+   { "mysqlhost", "string", (void*) string_parser, offsetof(struct config, mysqlhost), "", MAXVAL-1},
+   { "mysqlport", "integer", (void*) int_parser, offsetof(struct config, mysqlport), "", sizeof(int)},
+   { "mysqlsocket", "string", (void*) string_parser, offsetof(struct config, mysqlsocket), "/tmp/mysql.sock", MAXVAL-1},
+   { "mysqluser", "string", (void*) string_parser, offsetof(struct config, mysqluser), "piler", MAXVAL-1},
+   { "mysqlpwd", "string", (void*) string_parser, offsetof(struct config, mysqlpwd), "", MAXVAL-1},
+   { "mysqldb", "string", (void*) string_parser, offsetof(struct config, mysqldb), "piler", MAXVAL-1},
+   { "mysql_connect_timeout", "integer", (void*) int_parser, offsetof(struct config, mysql_connect_timeout), "2", sizeof(int)},
+   { "number_of_worker_processes", "integer", (void*) int_parser, offsetof(struct config, number_of_worker_processes), "2", sizeof(int)},
+   { "pemfile", "string", (void*) string_parser, offsetof(struct config, pemfile), "", MAXVAL-1},
+   { "pidfile", "string", (void*) string_parser, offsetof(struct config, pidfile), PIDFILE, MAXVAL-1},
+   { "piler_header_field", "string", (void*) string_parser, offsetof(struct config, piler_header_field), "X-piler-id:", MAXVAL-1},
+   { "process_rcpt_to_addresses", "integer", (void*) int_parser, offsetof(struct config, process_rcpt_to_addresses), "0", sizeof(int)},
+   { "queuedir", "string", (void*) string_parser, offsetof(struct config, queuedir), QUEUE_DIR, MAXVAL-1},
+   { "rtindex", "integer", (void*) int_parser, offsetof(struct config, rtindex), "0", sizeof(int)},
+   { "security_header", "string", (void*) string_parser, offsetof(struct config, security_header), "", MAXVAL-1},
+   { "server_id", "integer", (void*) int_parser, offsetof(struct config, server_id), "0", sizeof(int)},
+   { "sphxdb", "string", (void*) string_parser, offsetof(struct config, sphxdb), "piler1", MAXVAL-1},
+   { "sphxhost", "string", (void*) string_parser, offsetof(struct config, sphxhost), "127.0.0.1", MAXVAL-1},
+   { "sphxport", "integer", (void*) int_parser, offsetof(struct config, sphxport), "9306", sizeof(int)},
+   { "smtp_access_list", "integer", (void*) int_parser, offsetof(struct config, smtp_access_list), "0", sizeof(int)},
+   { "smtp_timeout", "integer", (void*) int_parser, offsetof(struct config, smtp_timeout), "60", sizeof(int)},
+   { "spam_header_line", "string", (void*) string_parser, offsetof(struct config, spam_header_line), "", MAXVAL-1},
+   { "syslog_recipients", "integer", (void*) int_parser, offsetof(struct config, syslog_recipients), "0", sizeof(int)},
+   { "tls_enable", "integer", (void*) int_parser, offsetof(struct config, tls_enable), "0", sizeof(int)},
+   { "tls_min_version", "string", (void*) string_parser, offsetof(struct config, tls_min_version), "TLSv1.2", MAXVAL-1},
+   { "tweak_sent_time_offset", "integer", (void*) int_parser, offsetof(struct config, tweak_sent_time_offset), "0", sizeof(int)},
+   { "update_counters_to_memcached", "integer", (void*) int_parser, offsetof(struct config, update_counters_to_memcached), "0", sizeof(int)},
+   { "username", "string", (void*) string_parser, offsetof(struct config, username), "piler", MAXVAL-1},
+   { "use_antivirus", "integer", (void*) int_parser, offsetof(struct config, use_antivirus), "1", sizeof(int)},
+   { "verbosity", "integer", (void*) int_parser, offsetof(struct config, verbosity), "1", sizeof(int)},
+   { "workdir", "string", (void*) string_parser, offsetof(struct config, workdir), WORK_DIR, MAXVAL-1},
 
-   { "archive_emails_not_having_message_id", "integer", (void*) int_parser, offsetof(struct __config, archive_emails_not_having_message_id), "0", sizeof(int)},
-   { "backlog", "integer", (void*) int_parser, offsetof(struct __config, backlog), "20", sizeof(int)},
-   { "cipher_list", "string", (void*) string_parser, offsetof(struct __config, cipher_list), "HIGH:MEDIUM", MAXVAL-1},
-   { "clamd_addr", "string", (void*) string_parser, offsetof(struct __config, clamd_addr), "", MAXVAL-1},
-   { "clamd_port", "integer", (void*) int_parser, offsetof(struct __config, clamd_port), "0", sizeof(int)},
-   { "clamd_socket", "string", (void*) string_parser, offsetof(struct __config, clamd_socket), CLAMD_SOCKET, MAXVAL-1},
-   { "debug", "integer", (void*) int_parser, offsetof(struct __config, debug), "0", sizeof(int)},
-   { "default_retention_days", "integer", (void*) int_parser, offsetof(struct __config, default_retention_days), "2557", sizeof(int)},
-   { "encrypt_messages", "integer", (void*) int_parser, offsetof(struct __config, encrypt_messages), "1", sizeof(int)},
-   { "extra_to_field", "string", (void*) string_parser, offsetof(struct __config, extra_to_field), "", MAXVAL-1},
-   { "hostid", "string", (void*) string_parser, offsetof(struct __config, hostid), HOSTID, MAXVAL-1},
-   { "iv", "string", (void*) string_parser, offsetof(struct __config, iv), "", MAXVAL-1},
-   { "listen_addr", "string", (void*) string_parser, offsetof(struct __config, listen_addr), "127.0.0.1", MAXVAL-1},
-   { "listen_port", "integer", (void*) int_parser, offsetof(struct __config, listen_port), "10025", sizeof(int)},
-   { "locale", "string", (void*) string_parser, offsetof(struct __config, locale), "", MAXVAL-1},
-   { "max_requests_per_child", "integer", (void*) int_parser, offsetof(struct __config, max_requests_per_child), "1000", sizeof(int)},
-   { "memcached_servers", "string", (void*) string_parser, offsetof(struct __config, memcached_servers), "127.0.0.1", MAXVAL-1},
-   { "memcached_to_db_interval", "integer", (void*) int_parser, offsetof(struct __config, memcached_to_db_interval), "900", sizeof(int)},
-   { "memcached_ttl", "integer", (void*) int_parser, offsetof(struct __config, memcached_ttl), "86400", sizeof(int)},
-   { "min_word_len", "integer", (void*) int_parser, offsetof(struct __config, min_word_len), "1", sizeof(int)},
-   { "mysqlhost", "string", (void*) string_parser, offsetof(struct __config, mysqlhost), "", MAXVAL-1},
-   { "mysqlport", "integer", (void*) int_parser, offsetof(struct __config, mysqlport), "", sizeof(int)},
-   { "mysqlsocket", "string", (void*) string_parser, offsetof(struct __config, mysqlsocket), "/tmp/mysql.sock", MAXVAL-1},
-   { "mysqluser", "string", (void*) string_parser, offsetof(struct __config, mysqluser), "piler", MAXVAL-1},
-   { "mysqlpwd", "string", (void*) string_parser, offsetof(struct __config, mysqlpwd), "", MAXVAL-1},
-   { "mysqldb", "string", (void*) string_parser, offsetof(struct __config, mysqldb), "piler", MAXVAL-1},
-   { "mysql_connect_timeout", "integer", (void*) int_parser, offsetof(struct __config, mysql_connect_timeout), "2", sizeof(int)},
-   { "number_of_worker_processes", "integer", (void*) int_parser, offsetof(struct __config, number_of_worker_processes), "10", sizeof(int)},
-   { "pemfile", "string", (void*) string_parser, offsetof(struct __config, pemfile), "", MAXVAL-1},
-   { "pidfile", "string", (void*) string_parser, offsetof(struct __config, pidfile), PIDFILE, MAXVAL-1},
-   { "piler_header_field", "string", (void*) string_parser, offsetof(struct __config, piler_header_field), "", MAXVAL-1},
-   { "queuedir", "string", (void*) string_parser, offsetof(struct __config, queuedir), QUEUE_DIR, MAXVAL-1},
-   { "server_id", "integer", (void*) int_parser, offsetof(struct __config, server_id), "0", sizeof(int)},
-   { "session_timeout", "integer", (void*) int_parser, offsetof(struct __config, session_timeout), "420", sizeof(int)},
-   { "spam_header_line", "string", (void*) string_parser, offsetof(struct __config, spam_header_line), "", MAXVAL-1},
-   { "tls_enable", "integer", (void*) int_parser, offsetof(struct __config, tls_enable), "0", sizeof(int)},
-   { "tweak_sent_time_offset", "integer", (void*) int_parser, offsetof(struct __config, tweak_sent_time_offset), "0", sizeof(int)},
-   { "update_counters_to_memcached", "integer", (void*) int_parser, offsetof(struct __config, update_counters_to_memcached), "0", sizeof(int)},
-   { "username", "string", (void*) string_parser, offsetof(struct __config, username), "piler", MAXVAL-1},
-   { "use_antivirus", "integer", (void*) int_parser, offsetof(struct __config, use_antivirus), "1", sizeof(int)},
-   { "verbosity", "integer", (void*) int_parser, offsetof(struct __config, verbosity), "1", sizeof(int)},
-   { "workdir", "string", (void*) string_parser, offsetof(struct __config, workdir), WORK_DIR, MAXVAL-1},
-
-   {NULL, NULL, NULL, 0, 0}
+   {NULL, NULL, NULL, 0, 0, 0}
 };
 
 
@@ -110,7 +118,7 @@ struct _parse_rule config_parse_rules[] =
  * parse configfile
  */
 
-int parse_config_file(char *configfile, struct __config *target_cfg, struct _parse_rule *rules){
+int parse_config_file(char *configfile, struct config *target_cfg, struct _parse_rule *rules){
    char line[MAXVAL], *chpos;
    FILE *f;
 
@@ -135,7 +143,7 @@ int parse_config_file(char *configfile, struct __config *target_cfg, struct _par
                   printf("failed to parse %s: \"%s\"\n", line, chpos+1);
                }
                break;
-            }				
+            }
 
             i++;
          }
@@ -150,7 +158,27 @@ int parse_config_file(char *configfile, struct __config *target_cfg, struct _par
 }
 
 
-int load_default_config(struct __config *cfg, struct _parse_rule *rules){
+int get_tls_protocol_number(char *protocol){
+   struct tls_protocol tls_protocols[] = {
+      { "TLSv1", TLS1_VERSION },
+      { "TLSv1.1", TLS1_1_VERSION },
+      { "TLSv1.2", TLS1_2_VERSION },
+   #ifdef TLS1_3_VERSION
+      { "TLSv1.3", TLS1_3_VERSION },
+   #endif
+   };
+
+   for(unsigned int i=0; i<sizeof(tls_protocols)/sizeof(struct tls_protocol); i++){
+      if(!strcmp(protocol, tls_protocols[i].proto)) {
+         return tls_protocols[i].version;
+      }
+   }
+
+   return 0;
+}
+
+
+int load_default_config(struct config *cfg, struct _parse_rule *rules){
    int i=0;
 
    while(rules[i].name){
@@ -166,12 +194,12 @@ int load_default_config(struct __config *cfg, struct _parse_rule *rules){
  * read configuration file variables
  */
 
-struct __config read_config(char *configfile){
-   struct __config cfg;
+struct config read_config(char *configfile){
+   struct config cfg;
 
    /* reset config structure and fill it with defaults */
 
-   memset((char *)&cfg, 0, sizeof(struct __config));
+   memset((char *)&cfg, 0, sizeof(struct config));
 
    load_default_config(&cfg, config_parse_rules);
 
@@ -180,6 +208,11 @@ struct __config read_config(char *configfile){
 
    if(parse_config_file(configfile, &cfg, config_parse_rules) == -1) printf("error parsing the configfile: %s\n", configfile);
 
+   cfg.hostid_len = strlen(cfg.hostid);
+
+   // Get the TLS protocol constant from string, ie. TLSv1.3 -> 772
+   cfg.tls_min_version_number = get_tls_protocol_number(cfg.tls_min_version);
+
    return cfg;
 }
 
@@ -188,9 +221,10 @@ struct __config read_config(char *configfile){
  * print a single configuration item as key=value
  */
 
-void print_config_item(struct __config *cfg, struct _parse_rule *rules, int i){
+void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
    int j;
    float f;
+   uint64 u;
    char *p, buf[MAXVAL];
 
    p = (char*)cfg + rules[i].offset;
@@ -199,6 +233,10 @@ void print_config_item(struct __config *cfg, struct _parse_rule *rules, int i){
       memcpy((char*)&j, p, sizeof(int));
       printf("%s=%d\n", rules[i].name, j);
    }
+   else if(strcmp(rules[i].type, "uint64") == 0){
+      memcpy((char*)&u, p, sizeof(uint64));
+      printf("%s=%llu\n", rules[i].name, u);
+   }
    else if(strcmp(rules[i].type, "float") == 0){
       memcpy((char*)&f, p, sizeof(float));
       printf("%s=%.4f\n", rules[i].name, f);
@@ -215,7 +253,7 @@ void print_config_item(struct __config *cfg, struct _parse_rule *rules, int i){
       trimBuffer(p);
       printf("%s=%s\n", rules[i].name, p);
    }
- 
+
 }
 
 
@@ -223,7 +261,7 @@ void print_config_item(struct __config *cfg, struct _parse_rule *rules, int i){
  * print all known configuration items
  */
 
-void print_config_all(struct __config *cfg, char *key){
+void print_config_all(struct config *cfg, char *key){
    int i=0;
    struct _parse_rule *rules;
 
@@ -247,7 +285,7 @@ void print_config_all(struct __config *cfg, char *key){
  * print all configuration items found in configfile
  */
 
-void print_config(char *configfile, struct __config *cfg){
+void print_config(char *configfile, struct config *cfg){
    FILE *f;
    char line[MAXVAL], *chpos, previtem[MAXVAL];
    struct _parse_rule *rules;
@@ -282,12 +320,10 @@ void print_config(char *configfile, struct __config *cfg){
 
             i++;
          }
- 
+
          if(!rules[i].name) printf("unknown key: \"%s\" \n", line);
       }
    }
 
    fclose(f);
 }
-
-

src/cfg.h

@@ -7,11 +7,13 @@
 
 #include "config.h"
 
-struct __config {
+struct config {
    int server_id;
    char username[MAXVAL];
 
    char hostid[MAXVAL];
+   int hostid_len;
+
    char pidfile[MAXVAL];
 
    char listen_addr[MAXVAL];
@@ -23,27 +25,36 @@ struct __config {
 
    int encrypt_messages;
 
+   int enable_chunking;
    int tls_enable;
    char pemfile[MAXVAL];
    char cipher_list[MAXVAL];
+   char tls_min_version[MAXVAL];
+   int tls_min_version_number;
 
    int use_antivirus;
 
    char memcached_servers[MAXVAL];
    int memcached_ttl;
 
+   int max_connections;
    int number_of_worker_processes;
    int max_requests_per_child;
 
    int backlog;
 
+   int process_rcpt_to_addresses;
+
    char workdir[MAXVAL];
    char queuedir[MAXVAL];
 
    int verbosity;
    char locale[MAXVAL];
 
-   int session_timeout;
+   int check_for_client_timeout_interval;
+   int smtp_timeout;
+   int helper_timeout;
+   int extract_attachments;
 
    char piler_header_field[MAXVAL];
    char extra_to_field[MAXVAL];
@@ -55,8 +66,12 @@ struct __config {
 
    int default_retention_days;
 
+   char security_header[MAXVAL];
+   char archive_address[MAXVAL];
+
    // mysql stuff
 
+   char mysqlcharset[MAXVAL];
    char mysqlhost[MAXVAL];
    int mysqlport;
    char mysqlsocket[MAXVAL];
@@ -65,16 +80,39 @@ struct __config {
    char mysqldb[MAXVAL];
    int mysql_connect_timeout;
 
+   // manticore stuff
+   char sphxhost[MAXVAL];
+   int sphxport;
+   char sphxdb[MAXVAL];
+   int rtindex;
+
    int update_counters_to_memcached;
    int memcached_to_db_interval;
 
    int archive_emails_not_having_message_id;
 
+   int archive_only_mydomains;
+
    int min_word_len;
 
+   int min_message_size;
+
    int tweak_sent_time_offset;
 
+   int enable_cjk;
+
+   int syslog_recipients;
+
+   int mmap_dedup_test;
+
+   int enable_folders;
+
    int debug;
+
+   int smtp_access_list;
+
+   int max_message_size;
+   uint64 max_smtp_memory;
 };
 
 

src/clamd.c

@@ -18,20 +18,18 @@
 #include <piler.h>
 
 
-int clamd_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg){
+int clamd_scan(char *tmpfile, struct config *cfg){
    int s, n;
-   char *p, *q, buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE];
+   char buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE];
    struct sockaddr_un server;
 
-   memset(avinfo, 0, SMALLBUFSIZE);
-
    chmod(tmpfile, 0644);
 
    strcpy(server.sun_path, cfg->clamd_socket);
    server.sun_family = AF_UNIX;
 
    if((s = socket(AF_UNIX, SOCK_STREAM, 0)) == -1){
-      syslog(LOG_PRIORITY, "ERR: create socket");
+      syslog(LOG_PRIORITY, "ERR: create socket to %s", cfg->clamd_socket);
       return AV_ERROR;
    }
 
@@ -43,7 +41,7 @@ int clamd_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg){
 
    /* issue the SCAN command with full path to the temporary directory */
 
-   
+
    memset(scan_cmd, 0, SMALLBUFSIZE);
    snprintf(scan_cmd, SMALLBUFSIZE-1, "SCAN %s/%s\r\n", cfg->workdir, tmpfile);
 
@@ -60,13 +58,13 @@ int clamd_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg){
    if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: CLAMD DEBUG: %d %s", tmpfile, n, buf);
 
    if(strcasestr(buf, CLAMD_RESP_INFECTED)){
-      p = strchr(buf, ' ');
+      char *p = strchr(buf, ' ');
       if(p){
-         q = strrchr(p, ' ');
+         char *q = strrchr(p, ' ');
          if(q){
             *q = '\0';
             p++;
-            strncpy(avinfo, p, SMALLBUFSIZE-1);
+            syslog(LOG_PRIORITY, "%s: VIRUS <%s> found, status=%s", tmpfile, p, S_STATUS_DISCARDED);
          }
       }
 
@@ -75,74 +73,3 @@ int clamd_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg){
 
    return AV_OK;
 }
-
-
-int clamd_net_scan(char *tmpfile, char *engine, char *avinfo, struct __config *cfg){
-   int n, psd, rc, ret=AV_OK;
-   char *p, *q, buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE];
-   char port_string[6];
-   struct addrinfo hints, *res;
-
-   memset(avinfo, 0, SMALLBUFSIZE);
-
-   chmod(tmpfile, 0644);
-
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to pass to clamd", tmpfile);
-
-   snprintf(port_string, sizeof(port_string)-1, "%d", cfg->clamd_port);
-
-   memset(&hints, 0, sizeof(hints));
-   hints.ai_family = AF_UNSPEC;
-   hints.ai_socktype = SOCK_STREAM;
-
-   if((rc = getaddrinfo(cfg->clamd_addr, port_string, &hints, &res)) != 0){
-      syslog(LOG_PRIORITY, "%s: getaddrinfo for '%s': %s\n", tmpfile, cfg->clamd_addr, gai_strerror(rc));
-      return AV_ERROR;
-   }
-
-   if((psd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
-      syslog(LOG_PRIORITY, "%s: ERR: create socket", tmpfile);
-      ret = AV_ERROR;
-      goto ENDE_CLAMD;
-   }
-
-   if(connect(psd, res->ai_addr, res->ai_addrlen) == -1){
-      syslog(LOG_PRIORITY, "%s: CLAMD ERR: connect to %s %d", tmpfile, cfg->clamd_addr, cfg->clamd_port);
-      ret = AV_ERROR;
-      goto ENDE_CLAMD;
-   }
-
-
-   memset(scan_cmd, 0, SMALLBUFSIZE);
-   snprintf(scan_cmd, SMALLBUFSIZE-1, "SCAN %s/%s\r\n", cfg->workdir, tmpfile);
-
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: CLAMD CMD: %s", tmpfile, scan_cmd);
-
-   send(psd, scan_cmd, strlen(scan_cmd), 0);
-
-   n = recvtimeout(psd, buf, MAXBUFSIZE, TIMEOUT);
-   close(psd);
-
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: CLAMD DEBUG: %d %s", tmpfile, n, buf);
-
-   if(strcasestr(buf, CLAMD_RESP_INFECTED)){
-      p = strchr(buf, ' ');
-      if(p){
-         q = strrchr(p, ' ');
-         if(q){
-            *q = '\0';
-            p++;
-            strncpy(avinfo, p, SMALLBUFSIZE-1);
-         }
-      }
-
-      ret = AV_VIRUS;
-   }
-
-
-ENDE_CLAMD:
-   freeaddrinfo(res);
-
-   return ret;
-}
-

src/config.h

@@ -9,23 +9,21 @@
 #include "piler-config.h"
 #include "params.h"
 
-#define PROGNAME "piler"
-#define PILERGETD_PROGNAME "pilergetd"
+typedef unsigned long long uint64;
 
-#define VERSION "0.1.23"
-
-#define BUILD 789
+#define BUILD 1001
 
 #define HOSTID "mailarchiver"
 
-#define CONFIG_FILE CONFDIR "/piler.conf"
+#define CONFIG_FILE CONFDIR "/piler/piler.conf"
+#define SMTP_ACL_FILE CONFDIR "/piler/smtp.acl"
 #define WORK_DIR DATADIR "/piler/tmp"
 #define QUEUE_DIR DATADIR "/piler/store"
+#define ERROR_DIR DATADIR "/piler/error"
 
 #define CLAMD_SOCKET "/tmp/clamd"
 
 #define PIDFILE "/var/run/piler/piler.pid"
-#define PILERGETD_PIDFILE "/var/run/piler/pilergetd.pid"
 #define QUARANTINELEN 255
 #define TIMEOUT 60
 #define TIMEOUT_USEC 500000
@@ -34,13 +32,30 @@
 #define SMALLBUFSIZE 512
 #define BIGBUFSIZE 131072
 #define REALLYBIGBUFSIZE 524288
+#define SMTPBUFSIZE 2048000
 #define TINYBUFSIZE 128
 #define MAXVAL 256
 #define RANDOM_POOL "/dev/urandom"
 #define RND_STR_LEN 36
 #define BUFLEN 32
+#define QUEUE_ID_LEN 16
 #define IPLEN 16+1
 #define KEYLEN 56
+#define MIN_EMAIL_ADDRESS_LEN 9
+// Sphinx 3.x has an issue with tokens longer than 41 characters.
+//
+// When a regular user executes a query, then his default email address filter
+// causes the query to fail with the below error message, even when the query
+// itself is correct:
+//
+// SELECT id FROM main1,dailydelta1,delta1 WHERE MATCH(' ( (@sender thisisanextremelylongemailaddressyesareallylongoneyeahitolyouXaddressXcom ) | (@rcpt thisisanextremelylongemailaddressyesareallylongoneyeahitolyouXaddressXcom) ) ') ORDER BY `sent` DESC LIMIT 0,20 OPTION max_matches=1000'
+//
+// ERROR 1064 (42000): index dailydelta1,delta1,main1: syntax error, unexpected $end near ' '
+//
+// Note that we use 42, because the parser adds a trailing space to the tokens
+// See https://www.mailpiler.org/wiki/current:sphinx3 and
+// https://bitbucket.org/jsuto/piler/issues/1082/no-sphinx-results-with-long-email for more
+#define MAX_EMAIL_ADDRESS_SPHINX_LEN 42
 
 #define CRLF "\n"
 
@@ -57,12 +72,16 @@
 #define MEMCACHED_MSGS_DUPLICATE MEMCACHED_CLAPF_PREFIX "duplicate"
 #define MEMCACHED_MSGS_IGNORE MEMCACHED_CLAPF_PREFIX "ignore"
 #define MEMCACHED_MSGS_SIZE MEMCACHED_CLAPF_PREFIX "size"
+#define MEMCACHED_MSGS_STORED_SIZE MEMCACHED_CLAPF_PREFIX "stored_size"
 
 
+#define PILEREXPORT_BEGIN_MARK "x-exported-by-pilerexport: start\n"
+
 #define LOG_PRIORITY LOG_INFO
 
 #define _LOG_INFO 3
 #define _LOG_DEBUG 5
+#define _LOG_EXTREME 100
 
 #define MAX_RCPT_TO 128
 
@@ -83,30 +102,42 @@
 #define SQL_RECIPIENT_TABLE "rcpt"
 #define SQL_ARCHIVING_RULE_TABLE "archiving_rule"
 #define SQL_RETENTION_RULE_TABLE "retention_rule"
+#define SQL_FOLDER_RULE_TABLE "folder_rule"
 #define SQL_COUNTER_TABLE "counter"
 #define SQL_OPTION_TABLE "option"
 #define SQL_DOMAIN_TABLE "domain"
+#define SQL_CUSTOMER_TABLE "customer"
+#define SQL_IMPORT_TABLE "import"
+#define SQL_LEGAL_HOLD_TABLE "legal_hold"
+#define SQL_FOLDER_MESSAGE_TABLE "folder_message"
 #define SQL_MESSAGES_VIEW "v_messages"
 #define SQL_ATTACHMENTS_VIEW "v_attachment"
 
-#define SQL_PREPARED_STMT_GET_META_ID_BY_MESSAGE_ID  "SELECT id FROM " SQL_METADATA_TABLE " WHERE message_id=?"
+#define SQL_PREPARED_STMT_GET_DOMAINS                "SELECT `domain` FROM `" SQL_DOMAIN_TABLE "`"
+#define SQL_PREPARED_STMT_GET_META_ID_BY_MESSAGE_ID  "SELECT id, piler_id FROM " SQL_METADATA_TABLE " WHERE message_id=?"
 #define SQL_PREPARED_STMT_INSERT_INTO_RCPT_TABLE     "INSERT INTO " SQL_RECIPIENT_TABLE " (`id`,`to`,`todomain`) VALUES(?,?,?)"
 #define SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE   "INSERT INTO " SQL_SPHINX_TABLE " (`id`, `from`, `to`, `fromdomain`, `todomain`, `subject`, `body`, `arrived`, `sent`, `size`, `direction`, `folder`, `attachments`, `attachment_types`) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
 #define SQL_PREPARED_STMT_INSERT_INTO_META_TABLE     "INSERT INTO " SQL_METADATA_TABLE " (`from`,`fromdomain`,`subject`,`spam`,`arrived`,`sent`,`retained`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`reference`,`digest`,`bodydigest`,`vcode`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
 #define SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE     "INSERT INTO " SQL_ATTACHMENT_TABLE " (`piler_id`,`attachment_id`,`sig`,`name`,`type`,`size`,`ptr`) VALUES(?,?,?,?,?,?,?)"
-#define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE   "SELECT `id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=?"
+#define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE   "SELECT `id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=? AND `ptr`=0 AND `size`=?"
 #define SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER     "SELECT `piler_id`, `attachment_id` FROM " SQL_ATTACHMENT_TABLE " WHERE id=?"
 #define SQL_PREPARED_STMT_QUERY_ATTACHMENT           "SELECT `attachment_id`, `ptr` FROM " SQL_ATTACHMENT_TABLE " WHERE piler_id=? ORDER BY attachment_id ASC"
 #define SQL_PREPARED_STMT_GET_FOLDER_ID              "SELECT `id` FROM " SQL_FOLDER_TABLE " WHERE `name`=? AND `parent_id`=?"
 #define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE   "INSERT INTO `" SQL_FOLDER_TABLE "` (`name`, `parent_id`) VALUES(?,?)"
 #define SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE  "UPDATE " SQL_METADATA_TABLE " SET reference=? WHERE message_id=? AND reference=''"
-
+#define SQL_PREPARED_STMT_GET_METADATA_REFERENCE     "SELECT COUNT(*) AS count FROM " SQL_METADATA_TABLE " WHERE reference=?"
+#define SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS        "SELECT id, type, username, password, server FROM " SQL_IMPORT_TABLE " WHERE started=0 ORDER BY id LIMIT 0,1"
+#define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE      "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `id`) VALUES(?,?)"
+#define SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE        "UPDATE " SQL_IMPORT_TABLE " SET started=?, status=?, imported=? WHERE id=?"
 
 /* Error codes */
 
 #define OK 0
 #define ERR 1
 #define ERR_EXISTS 2
+#define ERR_DISCARDED 3
+#define ERR_MYDOMAINS 4
+#define ERR_FOLDER -1
 
 #define AVIR_OK 0
 #define AVIR_VIRUS 1
@@ -120,5 +151,10 @@
 #define WRITE_TO_STDOUT 0
 #define WRITE_TO_BUFFER 1
 
-#endif /* _CONFIG_H */
+#define S_STATUS_UNDEF "undef"
+#define S_STATUS_STORED "stored"
+#define S_STATUS_DUPLICATE "duplicate"
+#define S_STATUS_DISCARDED "discarded"
+#define S_STATUS_ERROR "error"
 
+#endif /* _CONFIG_H */

src/counters.c

@@ -11,119 +11,120 @@
 #include <piler.h>
 
 
-struct __counters loadCounters(struct session_data *sdata, struct __config *cfg){
+struct counters load_counters(struct session_data *sdata){
    char buf[SMALLBUFSIZE];
-   struct __counters counters;
+   struct counters counters;
+   struct sql sql;
 
    bzero(&counters, sizeof(counters));
 
-   snprintf(buf, SMALLBUFSIZE-1, "SELECT `rcvd`, `virus`, `duplicate`, `ignore`, `size` FROM `%s`", SQL_COUNTER_TABLE);
+   snprintf(buf, SMALLBUFSIZE-1, "SELECT `rcvd`, `virus`, `duplicate`, `ignore`, `size`, `stored_size` FROM `%s`", SQL_COUNTER_TABLE);
 
-#ifdef NEED_MYSQL
-   MYSQL_RES *res;
-   MYSQL_ROW row;
 
-   if(mysql_real_query(&(sdata->mysql), buf, strlen(buf)) == 0){
-      res = mysql_store_result(&(sdata->mysql));
-      if(res != NULL){
-         row = mysql_fetch_row(res);
-         if(row){
-            counters.c_rcvd = strtoull(row[0], NULL, 10);
-            counters.c_virus = strtoull(row[1], NULL, 10);
-            counters.c_duplicate = strtoull(row[2], NULL, 10);
-            counters.c_ignore = strtoull(row[3], NULL, 10);
-            counters.c_size = strtoull(row[4], NULL, 10);
-         }
-         mysql_free_result(res);
-      }
+   if(prepare_sql_statement(sdata, &sql, buf) == ERR) return counters;
+
+
+   p_bind_init(&sql);
+
+   if(p_exec_stmt(sdata, &sql) == OK){
+
+      p_bind_init(&sql);
+
+      sql.sql[sql.pos] = (char *)&counters.c_rcvd; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+      sql.sql[sql.pos] = (char *)&counters.c_virus; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+      sql.sql[sql.pos] = (char *)&counters.c_duplicate; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+      sql.sql[sql.pos] = (char *)&counters.c_ignore; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+      sql.sql[sql.pos] = (char *)&counters.c_size; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+      sql.sql[sql.pos] = (char *)&counters.c_stored_size; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+
+      p_store_results(&sql);
+      p_fetch_results(&sql);
+      p_free_results(&sql);
    }
 
-#endif
+   close_prepared_statement(&sql);
 
    return counters;
 }
 
 
-void update_counters(struct session_data *sdata, struct __data *data, struct __counters *counters, struct __config *cfg){
+void update_counters(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
    char buf[MAXBUFSIZE];
 #ifdef HAVE_MEMCACHED
-   unsigned long long mc, rcvd;
-   struct __counters c;
-   char key[MAX_MEMCACHED_KEY_LEN];
+   unsigned long long mc;
+   struct counters c;
    unsigned int flags=0;
+#endif
 
+   if(counters->c_virus + counters->c_duplicate + counters->c_ignore + counters->c_size + counters->c_stored_size == 0) return;
+
+#ifdef HAVE_MEMCACHED
    if(cfg->update_counters_to_memcached == 1){
 
       /* increment counters to memcached */
 
-      if(memcached_increment(&(data->memc), MEMCACHED_MSGS_RCVD, strlen(MEMCACHED_MSGS_RCVD), counters->c_rcvd, &mc) == MEMCACHED_SUCCESS){
-         rcvd = mc;
+      if(memcached_increment(&(data->memc), MEMCACHED_MSGS_RCVD, counters->c_rcvd, &mc) == MEMCACHED_SUCCESS){
+         unsigned long long rcvd = mc;
 
-         if(counters->c_virus > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), counters->c_virus, &mc);
-         if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), counters->c_duplicate, &mc);
-         if(counters->c_ignore > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), counters->c_ignore, &mc);
-         if(counters->c_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_SIZE, strlen(MEMCACHED_MSGS_SIZE), counters->c_size, &mc);
+         if(counters->c_virus > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_VIRUS, counters->c_virus, &mc);
+         if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_DUPLICATE, counters->c_duplicate, &mc);
+         if(counters->c_ignore > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_IGNORE, counters->c_ignore, &mc);
+         if(counters->c_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_SIZE, counters->c_size, &mc);
+         if(counters->c_stored_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_STORED_SIZE, counters->c_stored_size, &mc);
 
 
-         bzero(&c, sizeof(c)); 
+         bzero(&c, sizeof(c));
 
-         snprintf(buf, MAXBUFSIZE-1, "%s %s %s %s %s %s", MEMCACHED_MSGS_RCVD, MEMCACHED_MSGS_VIRUS, MEMCACHED_MSGS_DUPLICATE, MEMCACHED_MSGS_IGNORE, MEMCACHED_MSGS_SIZE, MEMCACHED_COUNTERS_LAST_UPDATE);
+         snprintf(buf, MAXBUFSIZE-1, "%s %s %s %s %s %s %s", MEMCACHED_MSGS_RCVD, MEMCACHED_MSGS_VIRUS, MEMCACHED_MSGS_DUPLICATE, MEMCACHED_MSGS_IGNORE, MEMCACHED_MSGS_SIZE, MEMCACHED_MSGS_STORED_SIZE, MEMCACHED_COUNTERS_LAST_UPDATE);
 
          if(memcached_mget(&(data->memc), buf) == MEMCACHED_SUCCESS){
+            char key[MAX_MEMCACHED_KEY_LEN];
+
             while((memcached_fetch_result(&(data->memc), &key[0], &buf[0], &flags))){
                if(!strcmp(key, MEMCACHED_MSGS_RCVD)) c.c_rcvd = strtoull(buf, NULL, 10);
                else if(!strcmp(key, MEMCACHED_MSGS_VIRUS)) c.c_virus = strtoull(buf, NULL, 10);
                else if(!strcmp(key, MEMCACHED_MSGS_DUPLICATE)) c.c_duplicate = strtoull(buf, NULL, 10);
                else if(!strcmp(key, MEMCACHED_MSGS_IGNORE)) c.c_ignore = strtoull(buf, NULL, 10);
                else if(!strcmp(key, MEMCACHED_MSGS_SIZE)) c.c_size = strtoull(buf, NULL, 10);
+               else if(!strcmp(key, MEMCACHED_MSGS_STORED_SIZE)) c.c_stored_size = strtoull(buf, NULL, 10);
                else if(!strcmp(key, MEMCACHED_COUNTERS_LAST_UPDATE)) mc = strtoull(buf, NULL, 10);
             }
 
 
-            if(sdata->now - mc > cfg->memcached_to_db_interval && c.c_rcvd > 0 && c.c_rcvd >= rcvd){
-               snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_set(&(data->memc), MEMCACHED_COUNTERS_LAST_UPDATE, strlen(MEMCACHED_COUNTERS_LAST_UPDATE), buf, strlen(buf), 0, 0);
+            if(sdata->now - mc > (unsigned long long)cfg->memcached_to_db_interval && c.c_rcvd > 0 && c.c_rcvd >= rcvd){
+               snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), "set", MEMCACHED_COUNTERS_LAST_UPDATE, buf, strlen(buf), 0, 0);
+
+               snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET `rcvd`=%llu, `virus`=%llu, `duplicate`=%llu, `ignore`=%llu, `size`=%llu, `stored_size`=%llu", SQL_COUNTER_TABLE, c.c_rcvd, c.c_virus, c.c_duplicate, c.c_ignore, c.c_size, c.c_stored_size);
 
-               snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET `rcvd`=%llu, `virus`=%llu, `duplicate`=%llu, `ignore`=%llu, `size`=%llu", SQL_COUNTER_TABLE, c.c_rcvd, c.c_virus, c.c_duplicate, c.c_ignore, c.c_size);
- 
                //if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: update counters: %s", sdata->ttmpfile, buf);
 
-               goto EXEC_SQL;
+               p_query(sdata, buf);
             }
          }
 
       }
       else {
 
-         c = loadCounters(sdata, cfg);
+         c = load_counters(sdata);
 
-         snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), MEMCACHED_COUNTERS_LAST_UPDATE, strlen(MEMCACHED_COUNTERS_LAST_UPDATE), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), "add", MEMCACHED_COUNTERS_LAST_UPDATE, buf, strlen(buf), 0, 0);
 
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_virus + counters->c_virus); memcached_add(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_rcvd + counters->c_rcvd); memcached_add(&(data->memc), MEMCACHED_MSGS_RCVD, strlen(MEMCACHED_MSGS_RCVD), buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_duplicate + counters->c_duplicate); memcached_add(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_ignore + counters->c_ignore); memcached_add(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_size + counters->c_size); memcached_add(&(data->memc), MEMCACHED_MSGS_SIZE, strlen(MEMCACHED_MSGS_SIZE), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_virus + counters->c_virus); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_VIRUS, buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_rcvd + counters->c_rcvd); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_RCVD, buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_duplicate + counters->c_duplicate); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_DUPLICATE, buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_ignore + counters->c_ignore); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_IGNORE, buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_size + counters->c_size); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_SIZE, buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_stored_size + counters->c_stored_size); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_STORED_SIZE, buf, strlen(buf), 0, 0);
       }
 
    }
    else {
 #endif
-      snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET `rcvd`=`rcvd`+%llu, `virus`=`virus`+%llu, `duplicate`=`duplicate`+%llu, `ignore`=`ignore`+%llu, `size`=`size`+%llu", SQL_COUNTER_TABLE, counters->c_rcvd, counters->c_virus, counters->c_duplicate, counters->c_ignore, counters->c_size);
-
-      //if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: update counters: %s", sdata->ttmpfile, buf);
-
-#ifdef HAVE_MEMCACHED
-EXEC_SQL:
-#endif
-
-   #ifdef NEED_MYSQL
-      mysql_real_query(&(sdata->mysql), buf, strlen(buf));
-   #endif
+      snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET `rcvd`=`rcvd`+%llu, `virus`=`virus`+%llu, `duplicate`=`duplicate`+%llu, `ignore`=`ignore`+%llu, `size`=`size`+%llu, `stored_size`=`stored_size`+%llu", SQL_COUNTER_TABLE, counters->c_rcvd, counters->c_virus, counters->c_duplicate, counters->c_ignore, counters->c_size, counters->c_stored_size);
+      p_query(sdata, buf);
 
 #ifdef HAVE_MEMCACHED
    }
 #endif
 
 }
-
-

src/decoder.c

@@ -6,6 +6,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <ctype.h>
+#include <iconv.h>
 #include "decoder.h"
 #include "htmlentities.h"
 #include "config.h"
@@ -66,21 +67,38 @@ static int compmi(const void *m1, const void *m2){
 }
 
 
-void sanitiseBase64(char *s){
-   char *p1;
+inline void utf8_encode_char(unsigned char c, unsigned char *buf, int buflen, int *len){
+   int count=0;
 
-   if(s == NULL) return;
+   memset(buf, 0, buflen);
 
-   for(; *s; s++){
-      if(b64[(unsigned int)(*s & 0xFF)] == 255){
-         for(p1 = s; p1[0] != '\0'; p1++)
-            p1[0] = p1[1];
+      /*
+       * Code point          1st byte    2nd byte    3rd byte    4th byte
+       * ----------          --------    --------    --------    --------
+       * U+0000..U+007F      00..7F
+       * U+0080..U+07FF      C2..DF      80..BF
+       * U+0800..U+0FFF      E0          A0..BF      80..BF
+       *
+       * FIXME: See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#G7404 for valid sequences
+       */
+
+      if(c <= 0x7F){
+         *(buf+count) = c;
+         count++;
       }
-   }
+
+      else {
+         *(buf+count) = ( 0xC0 | (c >> 6) );
+         count++;
+         *(buf+count) = ( 0x80 | (c & 0x3F) );
+         count++;
+      }
+
+   *len = count;
 }
 
 
-inline void pack_4_into_3(char *s, char *s2){
+inline static void pack_4_into_3(char *s, char *s2){
    int j, n[4], k1, k2;
 
    memset(s2, 0, 3);
@@ -111,32 +129,12 @@ inline void pack_4_into_3(char *s, char *s2){
 
 
 int decodeBase64(char *p){
-   int i, len=0;
-   char s[5], s2[3], puf[MAXBUFSIZE];
+   int len=0;
+   unsigned char puf[MAXBUFSIZE];
 
-   if(strlen(p) < 4 || strlen(p) > MAXBUFSIZE/2)
-      return 0;
+   memset(puf, 0, sizeof(puf));
 
-   for(i=0; i<strlen(p); i++){
-      memcpy(s, p+i, 4);
-      s[4] = '\0';
-
-      i += 3;
-
-      /* safety check against abnormally long lines */
-
-      if(len + 3 > sizeof(puf)-1) break;
-
-      if(strlen(s) == 4){
-         pack_4_into_3(s, s2);
-         memcpy(puf+len, s2, 3);
-
-         len += 3;
-      }
-
-   }
-
-   *(puf+len) = '\0';
+   len = decode_base64_to_buffer(p, strlen(p), &puf[0], sizeof(puf)-1);
 
    snprintf(p, MAXBUFSIZE-1, "%s", puf);
 
@@ -151,21 +149,23 @@ int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
    if(plen < 4 || plen > blen)
       return 0;
 
-   for(i=0; i<plen; i++){
+   for(i=0; i<plen; i+=4){
       memcpy(s, p+i, 4);
       s[4] = '\0';
-
-      i += 3;
+      int decodedlen = 3;
 
       /* safety check against abnormally long lines */
 
-      if(len + 3 > blen-1) break;
+      if(len + decodedlen > blen-1) break;
 
       if(strlen(s) == 4){
          pack_4_into_3(s, s2);
-         memcpy(b+len, s2, 3);
+         if(s[3] == '=') decodedlen = 2;
+         if(s[2] == '=') decodedlen = 1;
 
-         len += 3;
+         memcpy(b+len, s2, decodedlen);
+
+         len += decodedlen;
       }
 
    }
@@ -175,13 +175,13 @@ int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
 
 
 void decodeQP(char *p){
-   int i, k=0, a, b;
-   char c;
+   unsigned int i;
+   int k=0, a, b;
 
    if(p == NULL) return;
 
    for(i=0; i<strlen((char*)p); i++){
-      c = p[i];
+      char c = p[i];
 
       if(p[i] == '=' && isxdigit(p[i+1]) && isxdigit(p[i+2])){
          a = p[i+1];
@@ -191,16 +191,20 @@ void decodeQP(char *p){
 
          i += 2;
       }
+      else if(p[i] == '_'){
+         c = ' ';
+      }
 
       p[k] = c;
       k++;
+
    }
 
    p[k] = '\0';
 }
 
 
-void decodeHTML(char *p){
+void decodeHTML(char *p, int utf8){
    unsigned char buf[MAXBUFSIZE], __u[8];
    char *s, *q;
    int count=0, len, c;
@@ -230,9 +234,16 @@ void decodeHTML(char *p){
                res = bsearch(&key, htmlentities, NUM_OF_HTML_ENTITIES, sizeof(struct mi), compmi);
 
                if(res && res->val <= 255){
-                  utf8_encode_char(res->val, &__u[0], sizeof(__u), &len);
-                  memcpy(&buf[count], &__u[0], len);
-                  count += len;
+
+                  if(utf8 == 1){
+                     utf8_encode_char(res->val, &__u[0], sizeof(__u), &len);
+                     memcpy(&buf[count], &__u[0], len);
+                     count += len;
+                  }
+                  else {
+                     buf[count] = res->val;
+                     count++;
+                  }
                }
                else {
                   buf[count] = 'q';
@@ -257,7 +268,8 @@ void decodeHTML(char *p){
 
 
 void decodeURL(char *p){
-   int i, c, k=0, a, b;
+   unsigned int i;
+   int c, k=0, a, b;
 
    if(p == NULL) return;
 
@@ -295,76 +307,37 @@ void decodeURL(char *p){
 }
 
 
-inline void utf8_encode_char(unsigned char c, unsigned char *buf, int buflen, int *len){
-   int count=0;
+int utf8_encode(char *inbuf, int inbuflen, char *outbuf, int outbuflen, char *encoding){
+   iconv_t cd;
+   size_t inbytesleft, outbytesleft;
+   int ret = ERR;
 
-   memset(buf, 0, buflen);
+   memset(outbuf, 0, outbuflen);
 
-      /*
-       * Code point          1st byte    2nd byte    3rd byte    4th byte
-       * ----------          --------    --------    --------    --------
-       * U+0000..U+007F      00..7F
-       * U+0080..U+07FF      C2..DF      80..BF
-       * U+0800..U+0FFF      E0          A0..BF      80..BF
-       */
+   // Iconv sometimes produces an invalid utf8 sequence for gb2312.
+   // The fix is to use cp936, instead of gb2312 encoding.
+   //
+   // If there will be more similar exceptions, then we have to use
+   // a more efficient lookup method
 
-      if(c <= 0x7F){
-         *(buf+count) = c;
-         count++;
-      }
+   if(strcasecmp(encoding, "gb2312") == 0)
+      cd = iconv_open("utf-8", "cp936");
+   else if(strcasecmp(encoding, "ks_c_5601-1987") == 0)
+      cd = iconv_open("utf-8", "EUC-KR");
+   else
+      cd = iconv_open("utf-8", encoding);
 
-      else if(c <= 0x7FF){
-         *(buf+count) = ( 0xC0 | (c >> 6) );
-         count++;
-         *(buf+count) = ( 0x80 | (c & 0x3F) );
-         count++;
-      }
+   if(cd != (iconv_t)-1){
+      inbytesleft = inbuflen;
+      outbytesleft = outbuflen-1;
 
+      if(iconv(cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft) == (size_t) -1)
+         ret = ERR;
+      else
+         ret = OK;
 
-      else if (c <= 0xFFFF){
-         *(buf+count) = ( 0xE0 | (c >> 12) );
-         count++;
-         *(buf+count) = ( 0x80 | ((c >> 6) & 0x3F) );
-         count++;
-         *(buf+count) = ( 0x80 | (c & 0x3F) );
-         count++;
-      }
-
-   *len = count;
-}
-
-
-void utf8_encode(unsigned char *p){
-   int count=0, len;
-   unsigned char *u, *s, utf8[MAXBUFSIZE], __u[8];
-
-   if(p == NULL || strlen((char *)p) == 0) return;
-
-   memset(utf8, 0, MAXBUFSIZE);
-   u = &utf8[0];
-   s = p;
-
-   for(; *s; s++){
-
-      utf8_encode_char(*s, &__u[0], sizeof(__u), &len);
-
-      /*
-       * this condition should never happen, as according to the RFCs:
-       *
-       * "Each line of characters MUST be no more than 998 characters, and
-       * SHOULD be no more than 78 characters, excluding the CRLF."
-       *
-       */
-
-      if(count+len > sizeof(utf8)-1) break;
-
-      //printf("%s", __u);
-      memcpy(u+count, &__u[0], len);
-
-      count += len;
+      iconv_close(cd);
    }
 
-   *(u+count) = '\0'; count++;
-   memcpy(p, u, count);
+   return ret;
 }
-

src/decoder.h

@@ -5,15 +5,11 @@
 #ifndef _DECODER_H
  #define _DECODER_H
 
-void base64_encode(unsigned char *in, int inlen, char *out, int outlen);
-
-void sanitiseBase64(char *s);
 int decodeBase64(char *p);
 int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen);
 void decodeQP(char *p);
-void decodeHTML(char *p);
+void decodeHTML(char *p, int utf8);
 void decodeURL(char *p);
-inline void utf8_encode_char(unsigned char c, unsigned char *buf, int buflen, int *len);
-void utf8_encode(unsigned char *p);
+int utf8_encode(char *inbuf, int inbuflen, char *outbuf, int outbuflen, char *encoding);
 
 #endif /* _DECODER_H */

src/defs.h

@@ -7,27 +7,17 @@
 
 #ifdef NEED_MYSQL
   #include <mysql.h>
-#endif
-#ifdef NEED_SQLITE3
-  #include <sqlite3.h>
-
-   /* for older versions of sqlite3 do not have the sqlite3_prepare_v2() function, 2009.12.30, SJ */
-
-  #if SQLITE_VERSION_NUMBER < 3006000
-    #define sqlite3_prepare_v2 sqlite3_prepare
-  #endif
-
+  #include <mysqld_error.h>
 #endif
 #ifdef HAVE_TRE
    #include <tre/tre.h>
    #include <tre/regex.h>
 #endif
-#ifdef HAVE_LIBWRAP
-   #include <tcpd.h>
-#endif
 
+#include <openssl/md5.h>
 #include <openssl/sha.h>
 #include <openssl/ssl.h>
+#include <netinet/in.h>
 #include "tai.h"
 #include "config.h"
 
@@ -35,35 +25,47 @@
 #define MSG_BODY 0
 #define MSG_RECEIVED 1
 #define MSG_FROM 2
-#define MSG_TO 3
-#define MSG_CC 4
-#define MSG_SUBJECT 5
-#define MSG_CONTENT_TYPE 6
-#define MSG_CONTENT_TRANSFER_ENCODING 7
-#define MSG_CONTENT_DISPOSITION 8
-#define MSG_MESSAGE_ID 9
-#define MSG_REFERENCES 10
-#define MSG_RECIPIENT 11
+#define MSG_SENDER 3
+#define MSG_TO 4
+#define MSG_CC 5
+#define MSG_SUBJECT 6
+#define MSG_CONTENT_TYPE 7
+#define MSG_CONTENT_TRANSFER_ENCODING 8
+#define MSG_CONTENT_DISPOSITION 9
+#define MSG_MESSAGE_ID 10
+#define MSG_REFERENCES 11
+#define MSG_RECIPIENT 12
+#define MSG_ENVELOPE_TO 13
 
-#define MAXHASH 8171
+#define MAXHASH 277
 
 #define BASE64_RATIO 1.33333333
 
-#define DIGEST_LENGTH SHA256_DIGEST_LENGTH
+#define DIGEST_LENGTH EVP_MAX_MD_SIZE
+#define DIGEST_HEX_LENGTH 2*DIGEST_LENGTH+1
 
 #define UNDEF 0
 #define READY 1
 #define BUSY 2
 
+#define MAX_SQL_VARS 20
+
+#define TYPE_UNDEF 0
+#define TYPE_SHORT 1
+#define TYPE_LONG 2
+#define TYPE_LONGLONG 3
+#define TYPE_STRING 4
 
 #define MAXCHILDREN 64
 
-
-typedef void signal_func (int);
+#define RULE_UNDEF 0
+#define RULE_MATCH 1
+#define RULE_NO_MATCH -100
 
 
 struct child {
    pid_t pid;
+   int serial;
    int messages;
    int status;
 };
@@ -74,7 +76,7 @@ struct attachment {
    char type[TINYBUFSIZE];
    char shorttype[TINYBUFSIZE];
    char aname[TINYBUFSIZE];
-   char filename[TINYBUFSIZE];
+   char filename[SMALLBUFSIZE];
    char internalname[TINYBUFSIZE];
    char digest[2*DIGEST_LENGTH+1];
    char dumped;
@@ -88,9 +90,29 @@ struct ptr_array {
 };
 
 
-struct list {
-   char s[SMALLBUFSIZE];
-   struct list *r;
+struct node {
+   void *str;
+   unsigned int key;
+   struct node *r;
+};
+
+
+struct smtp_acl {
+   char network_str[BUFLEN];
+   in_addr_t low, high;
+   int prefix;
+   int rejected;
+   struct smtp_acl *r;
+};
+
+
+struct net {
+   int socket;
+   int use_ssl;
+   int starttls;
+   int timeout;
+   SSL_CTX *ctx;
+   SSL *ssl;
 };
 
 
@@ -99,6 +121,8 @@ struct rule {
    regex_t from;
    regex_t to;
    regex_t subject;
+   regex_t body;
+   regex_t attachment_name;
    regex_t attachment_type;
 #endif
    int spam;
@@ -107,7 +131,13 @@ struct rule {
    int attachment_size;
    char _attachment_size[4];
 
+   char *domain;
+   int domainlen;
+
    int days;
+   int folder_id;
+
+   char emptyfrom, emptyto, emptysubject, emptybody, emptyaname, emptyatype;
 
    char *rulestr;
    char compiled;
@@ -116,7 +146,21 @@ struct rule {
 };
 
 
-struct _state {
+struct rule_cond {
+   char domain[SMALLBUFSIZE];
+   char from[SMALLBUFSIZE];
+   char to[SMALLBUFSIZE];
+   char subject[SMALLBUFSIZE];
+   char body[SMALLBUFSIZE];
+   char _size[SMALLBUFSIZE];
+   char attachment_name[SMALLBUFSIZE];
+   char attachment_type[SMALLBUFSIZE];
+   char _attachment_size[SMALLBUFSIZE];
+   int size, attachment_size, spam, days, folder_id;
+};
+
+
+struct parser_state {
    int line_num;
    int message_state;
    int is_header;
@@ -129,8 +173,10 @@ struct _state {
    int qp;
    int htmltag;
    int style;
+   int meta_content_type;
    int skip_html;
    int has_to_dump;
+   int has_to_dump_whole_body;
    int fd;
    int b64fd;
    int mfd;
@@ -139,36 +185,47 @@ struct _state {
    int content_type_is_set;
    int pushed_pointer;
    int saved_size;
-   int writebufpos;
-   int abufpos;
+   unsigned int writebufpos;
+   unsigned int abufpos;
+   unsigned int received_header;
    char attachedfile[RND_STR_LEN+SMALLBUFSIZE];
    char message_id[SMALLBUFSIZE];
+   char message_id_hash[2*DIGEST_LENGTH+1];
    char miscbuf[MAX_TOKEN_LEN];
    char qpbuf[MAX_TOKEN_LEN];
+   char receivedbuf[SMALLBUFSIZE];
    unsigned long n_token;
    unsigned long n_subject_token;
    unsigned long n_body_token;
    unsigned long n_chain_token;
 
-   char filename[TINYBUFSIZE];
    char type[TINYBUFSIZE];
+   char charset[TINYBUFSIZE];
 
-   struct list *boundaries;
-   struct list *rcpt;
-   struct list *rcpt_domain;
-   struct list *journal_recipient;
+   char attachment_name_buf[SMALLBUFSIZE];
+   int anamepos;
+
+   struct node *boundaries[MAXHASH];
+   struct node *rcpt[MAXHASH];
+   struct node *rcpt_domain[MAXHASH];
+   struct node *journal_recipient[MAXHASH];
 
    int n_attachments;
    struct attachment attachments[MAX_ATTACHMENTS];
 
    char reference[SMALLBUFSIZE];
 
-   char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE];
+   char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_sender[SMALLBUFSIZE], b_sender_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE];
    char b_journal_to[MAXBUFSIZE];
 
-   int bodylen;
-   int tolen;
-   int journaltolen;
+   unsigned int bodylen;
+   unsigned int tolen;
+   unsigned int todomainlen;
+   unsigned int found_security_header;
+
+   long unsigned int journaltolen;
+
+   int retention;
 };
 
 
@@ -180,22 +237,23 @@ struct session_data {
    char acceptbuf[SMALLBUFSIZE];
    char attachments[SMALLBUFSIZE];
    char internal_sender, internal_recipient, external_recipient;
+   uint64 duplicate_id;
+   short int customer_id;
    int direction;
    int tls;
    int spam_message;
-   int fd, hdr_len, tot_len, num_of_rcpt_to, rav;
+   int fd, hdr_len, tot_len, stored_len, num_of_rcpt_to, rav;
    int need_scan;
    float __acquire, __parsed, __av, __store, __compress, __encrypt;
    char bodydigest[2*DIGEST_LENGTH+1];
    char digest[2*DIGEST_LENGTH+1];
    time_t now, sent, delivered, retained;
    char ms_journal;
+   char import;
    int journal_envelope_length, journal_bottom_length;
+   unsigned int sql_errno;
 #ifdef NEED_MYSQL
-   MYSQL mysql;
-#endif
-#ifdef NEED_SQLITE3
-   sqlite3 *db;
+   MYSQL mysql, sphx;
 #endif
 };
 
@@ -240,47 +298,134 @@ struct memcached_server {
 #endif
 
 
-struct __data {
-   int folder;
+struct import {
+   char *extra_recipient;
+   char *move_folder;
+   char *failed_folder;
+   int status;
+   int total_messages;
+   int processed_messages;
+   int batch_processing_limit;
+   int start_position;
+   int import_job_id;
+   int remove_after_import;
+   int download_only;
+   int keep_eml;
+   int timeout;
+   int cap_uidplus;
+   int fd;
+   long total_size;
+   int dryrun;
+   int tot_msgs;
+   int port;
+   int seq;
+   int table_id;
+   int delay;
+   char *server;
+   char *username;
+   char *password;
+   char *database;
+   char *skiplist;
+   char *folder_imap;
+   char *folder_name;
+   char *mboxdir;
+   char *folder;
+   char filename[SMALLBUFSIZE];
+   time_t started, updated, finished, after, before;
+};
+
+
+struct licence {
+   char customer[TINYBUFSIZE];
+   char hostname[TINYBUFSIZE];
+   char ip[TINYBUFSIZE];
+   time_t expiry;
+   int licenced_users;
+};
+
+
+struct data {
+   int folder, quiet;
    char recursive_folder_names;
    char starttls[TINYBUFSIZE];
-   char mydomains[MAXBUFSIZE];
-
-#ifdef NEED_MYSQL
-   MYSQL_STMT *stmt_get_meta_id_by_message_id;
-   MYSQL_STMT *stmt_insert_into_rcpt_table;
-   MYSQL_STMT *stmt_insert_into_sphinx_table;
-   MYSQL_STMT *stmt_insert_into_meta_table;
-   MYSQL_STMT *stmt_insert_into_attachment_table;
-   MYSQL_STMT *stmt_get_attachment_id_by_signature;
-   MYSQL_STMT *stmt_get_attachment_pointer;
-   MYSQL_STMT *stmt_query_attachment;
-   MYSQL_STMT *stmt_get_folder_id;
-   MYSQL_STMT *stmt_insert_into_folder_table;
-   MYSQL_STMT *stmt_update_metadata_reference;
-#endif
+   struct node *mydomains[MAXHASH];
+   struct node *imapfolders[MAXHASH];
+   struct import *import;
+   struct licence licence;
+   char *dedup;
+   int child_serial;
+   int pos;
 
 #ifdef HAVE_TRE
-   struct rule *archiving_rules;
-   struct rule *retention_rules;
+   struct node *archiving_rules[1];
+   struct node *retention_rules[1];
+   struct node *folder_rules[1];
 #endif
 
 #ifdef HAVE_MEMCACHED
    struct memcached_server memc;
 #endif
 
-   SSL_CTX *ctx;
-   SSL *ssl;
+   struct net *net;
 };
 
 
-struct __counters {
+#if !defined(MARIADB_BASE_VERSION) && !defined(MARIADB_VERSION_ID) && \
+  MYSQL_VERSION_ID >= 80001 && MYSQL_VERSION_ID != 80002
+typedef bool my_bool;
+#endif
+
+struct sql {
+#ifdef NEED_MYSQL
+   MYSQL_STMT *stmt;
+#endif
+   char *sql[MAX_SQL_VARS];
+   int type[MAX_SQL_VARS];
+   int len[MAX_SQL_VARS];
+   unsigned long length[MAX_SQL_VARS];
+   my_bool is_null[MAX_SQL_VARS];
+   my_bool error[MAX_SQL_VARS];
+   int pos;
+};
+
+
+struct counters {
    unsigned long long c_rcvd;
    unsigned long long c_virus;
    unsigned long long c_duplicate;
    unsigned long long c_ignore;
    unsigned long long c_size;
+   unsigned long long c_stored_size;
+};
+
+
+struct smtp_session {
+   char ttmpfile[SMALLBUFSIZE];
+   char mailfrom[SMALLBUFSIZE];
+   char rcptto[MAX_RCPT_TO][SMALLBUFSIZE];
+   char remote_host[INET6_ADDRSTRLEN+1];
+   char nullbyte;
+   time_t lasttime;
+   int protocol_state;
+   int slot;
+   int fd;
+   int bad;
+   int tot_len;
+   int bdat_bytes_to_read;
+   int num_of_rcpt_to;
+   struct config *cfg;
+   struct net net;
+   int max_message_size;
+   char *buf;
+   int buflen;
+   int bufsize;
+   int too_big;
+   int mail_size;
+};
+
+struct tls_protocol {
+   char *proto;
+   int version;
 };
 
 #endif /* _DEFS_H */
-

src/digest.c

@@ -19,6 +19,7 @@
 
 int search_header_end(char *p, int n){
    int hdr_len=0;
+   char *q = p;
 
    if(n < 5) return hdr_len;
 
@@ -28,30 +29,43 @@ int search_header_end(char *p, int n){
       hdr_len++;
    }
 
-   return 0;
+   if(*(p-1) == '\n' && strcasestr(q, "Message-ID:")) return n; else return 0;
 }
 
 
-int make_digests(struct session_data *sdata, struct __config *cfg){
-   int i=0, n, fd, offset=3, hdr_len=0, len=0;
+int make_digests(struct session_data *sdata, struct config *cfg){
+   int n, fd, offset=3, hdr_len=0;
    char *body=NULL;
-   unsigned char buf[BIGBUFSIZE], md[DIGEST_LENGTH], md2[DIGEST_LENGTH];
-   SHA256_CTX context, context2;
+   unsigned char buf[BIGBUFSIZE];
+
+   EVP_MD_CTX *ctx, *ctx2;
+   const EVP_MD *md, *md2;
+   unsigned int i=0, md_len, md_len2;
+   unsigned char md_value[EVP_MAX_MD_SIZE], md_value2[EVP_MAX_MD_SIZE];
 
    memset(sdata->bodydigest, 0, 2*DIGEST_LENGTH+1);
    memset(sdata->digest, 0, 2*DIGEST_LENGTH+1);
-   SHA256_Init(&context);
-   SHA256_Init(&context2);
 
+   md = EVP_get_digestbyname("sha256");
+   md2 = EVP_get_digestbyname("sha256");
+   if(md == NULL || md2 == NULL){
+      syslog(LOG_PRIORITY, "ERROR: unknown message digest: sha256 in %s:%d", __func__, __LINE__);
+      return 1;
+   }
+
+   ctx = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx, md, NULL);
+
+   ctx2 = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx2, md2, NULL);
 
    fd = open(sdata->filename, O_RDONLY);
    if(fd == -1) return -1;
 
+   memset(buf, 0, sizeof(buf));
 
    while((n = read(fd, buf, sizeof(buf))) > 0){
-      len += n;
-
-      SHA256_Update(&context2, buf, n);
+      EVP_DigestUpdate(ctx2, buf, n);
 
       body = (char *)&buf[0];
 
@@ -68,7 +82,7 @@ int make_digests(struct session_data *sdata, struct __config *cfg){
       }
 
 
-      SHA256_Update(&context, body, n);
+      EVP_DigestUpdate(ctx, body, n);
 
       i++;
    }
@@ -77,59 +91,93 @@ int make_digests(struct session_data *sdata, struct __config *cfg){
 
    sdata->hdr_len = hdr_len;
 
-   SHA256_Final(md, &context);
-   SHA256_Final(md2, &context2);
+   EVP_DigestFinal_ex(ctx, md_value, &md_len);
+   EVP_MD_CTX_free(ctx);
+   EVP_DigestFinal_ex(ctx2, md_value2, &md_len2);
+   EVP_MD_CTX_free(ctx2);
 
-   for(i=0;i<DIGEST_LENGTH;i++){
-      snprintf(sdata->bodydigest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
-      snprintf(sdata->digest + i*2, 2*DIGEST_LENGTH, "%02x", md2[i]);
+   for(i=0;i<md_len;i++){
+      snprintf(sdata->bodydigest + i*2, 3, "%02x", md_value[i]);
+   }
+
+   for(i=0;i<md_len2;i++){
+      snprintf(sdata->digest + i*2, 3, "%02x", md_value2[i]);
    }
 
    return 0;
 }
 
 
-void digest_file(char *filename, char *digest){
-   int fd, i, n;
-   unsigned char buf[MAXBUFSIZE], md[DIGEST_LENGTH];
-   SHA256_CTX context;
+void raw_digest_file(char *digestname, char *filename, unsigned char *md_value){
+   int fd, n;
+   unsigned char buf[MAXBUFSIZE];
+   EVP_MD_CTX *ctx;
+   const EVP_MD *md;
+   unsigned int md_len;
 
-   memset(digest, 0, 2*DIGEST_LENGTH+1);
+   md = EVP_get_digestbyname(digestname);
+   if(md == NULL){
+      syslog(LOG_PRIORITY, "ERROR: unknown message digest: '%s' in %s:%d", digestname, __func__, __LINE__);
+      return;
+   }
 
    fd = open(filename, O_RDONLY);
    if(fd == -1) return;
 
-   SHA256_Init(&context);
+   ctx = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx, md, NULL);
 
    while((n = read(fd, buf, sizeof(buf))) > 0){
-      SHA256_Update(&context, buf, n);
+      EVP_DigestUpdate(ctx, buf, n);
    }
 
    close(fd);
 
-   SHA256_Final(md, &context);
-
-   for(i=0;i<DIGEST_LENGTH;i++)
-      snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
-
+   EVP_DigestFinal_ex(ctx, md_value, &md_len);
+   EVP_MD_CTX_free(ctx);
 }
 
 
-void digest_string(char *s, char *digest){
-   int i;
+void digest_file(char *filename, char *digest){
    unsigned char md[DIGEST_LENGTH];
-   SHA256_CTX context;
+
+   raw_digest_file("sha256", filename, &md[0]);
 
    memset(digest, 0, 2*DIGEST_LENGTH+1);
 
-   SHA256_Init(&context);
-
-   SHA256_Update(&context, s, strlen(s));
-
-   SHA256_Final(md, &context);
-
-   for(i=0;i<DIGEST_LENGTH;i++)
+   for(int i=0;i<SHA256_DIGEST_LENGTH;i++){
       snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
-
+   }
 }
 
+
+void digest_string(char *digestname, char *s, char *digest){
+   EVP_MD_CTX *ctx;
+   const EVP_MD *md;
+   unsigned int i, md_len;
+   unsigned char md_value[DIGEST_LENGTH];
+
+   memset(digest, 0, 2*DIGEST_LENGTH+2);
+
+   md = EVP_get_digestbyname(digestname);
+   if(md == NULL){
+      syslog(LOG_PRIORITY, "ERROR: unknown message digest: '%s' in %s:%d", digestname, __func__, __LINE__);
+      return;
+   }
+
+   ctx = EVP_MD_CTX_new();
+   EVP_DigestInit_ex(ctx, md, NULL);
+   EVP_DigestUpdate(ctx, s, strlen(s));
+   EVP_DigestFinal_ex(ctx, md_value, &md_len);
+   EVP_MD_CTX_free(ctx);
+
+   for(i=0;i<md_len;i++){
+      snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md_value[i]);
+   }
+}
+
+
+void create_md5_from_email_address(char *puf, char *md5buf){
+   digest_string("md5", puf, md5buf);
+   md5buf[2*MD5_DIGEST_LENGTH] = ' ';
+}

src/dirs.c

@@ -13,11 +13,27 @@
 #include <piler.h>
 
 
-void createdir(char *path, uid_t uid, gid_t gid, mode_t mode);
+void createdir(char *path, uid_t uid, gid_t gid, mode_t mode){
+   struct stat st;
+
+   if(strlen(path) > 2){
+      if(path[strlen(path)-1] == '/') path[strlen(path)-1] = '\0';
+
+      if(stat(path, &st)){
+         if(mkdir(path, mode) == 0){
+            if(chown(path, uid, gid))
+               syslog(LOG_PRIORITY, "ERROR: createdir(): chown() failed on %s", path);
+            syslog(LOG_PRIORITY, "created directory: *%s*", path);
+         }
+         else syslog(LOG_PRIORITY, "ERROR: could not create directory: *%s*", path);
+      }
+   }
+}
 
 
-void check_and_create_directories(struct __config *cfg, uid_t uid, gid_t gid){
+void check_and_create_directories(struct config *cfg, uid_t uid, gid_t gid){
    char *p, s[SMALLBUFSIZE];
+   int i;
 
    p = strrchr(cfg->workdir, '/');
    if(p){
@@ -45,24 +61,13 @@ void check_and_create_directories(struct __config *cfg, uid_t uid, gid_t gid){
       *p = '/';
    }
 
-}
-
-
-void createdir(char *path, uid_t uid, gid_t gid, mode_t mode){
-   struct stat st;
-
-   if(strlen(path) > 2){
-      if(path[strlen(path)-1] == '/') path[strlen(path)-1] = '\0';
-
-      if(stat(path, &st)){
-         if(mkdir(path, mode) == 0){
-            chown(path, uid, gid);
-            syslog(LOG_PRIORITY, "created directory: *%s*", path);
-         }
-         else syslog(LOG_PRIORITY, "could not create directory: *%s*", path);
-      }
-
+   for(i=0; i<cfg->number_of_worker_processes; i++){
+      snprintf(s, sizeof(s)-1, "%s/%d", cfg->workdir, i);
+   #ifdef HAVE_ANTIVIRUS
+      createdir(s, uid, gid, 0711);
+   #else
+      createdir(s, uid, gid, 0700);
+   #endif
    }
+
 }
-
-

src/errmsg.h

@@ -15,9 +15,12 @@
 #define ERR_SETGID "ERR: setgid()"
 #define ERR_SELECT "ERR: select()"
 #define ERR_CHDIR "ERR: chdir() to working directory failed"
+#define ERR_DAEMON "ERR: daemon()"
 #define ERR_OPEN_TMP_FILE "ERR: opening a tempfile"
 #define ERR_TIMED_OUT "ERR: timed out"
 #define ERR_FORK_FAILED "ERR: cannot fork()"
+#define ERR_OPEN_DEDUP_FILE "ERR: cannot open dedup file"
+#define ERR_PID_FILE_EXISTS "ERR: pidfile exists. If piler daemon is not running, and the pidfile exists (unclean shutdown?) then remove it, and start piler again"
 
 #define ERR_MYSQL_CONNECT "Cannot connect to mysql server"
 #define ERR_PSQL_CONNECT "Cannot connect to PSql server"
@@ -28,4 +31,7 @@
 
 #define ERR_READING_KEY "ERR: reading cipher key"
 
+#define ERR_LICENCE "ERR: licence validation error"
+#define ERR_LICENCE_IP "ERR: licenced IP-address doesn't match with listen_addr parameter"
+
 #endif /* _ERRMSG_H */

src/extract.c

@@ -4,70 +4,77 @@
 #include <unistd.h>
 #include <sys/types.h>
 #include <sys/socket.h>
+#include <sys/wait.h>
 #include <sys/stat.h>
+#include <dirent.h>
 #include <fcntl.h>
+#include <ctype.h>
+#include <signal.h>
 #include <piler.h>
 
 #ifdef HAVE_ZIP
    #include <zip.h>
 #endif
 
+#define die(e) do { syslog(LOG_INFO, "error: helper: %s", e); exit(EXIT_FAILURE); } while (0);
 
-void remove_xml(char *buf, int *html){
+
+int remove_xml(char *src, char *dest, int destlen, int *html){
    int i=0;
-   char *p;
 
-   p = buf;
+   memset(dest, 0, destlen);
 
-   for(; *p; p++){
-      if(*p == '<'){ *html = 1; }
-
-      if(*html == 0){
-         *(buf+i) = *p;
-         i++;
+   for(; *src; src++){
+      if(*src == '<'){
+         *html = 1;
       }
-
-      if(*p == '>'){
+      else if(*src == '>'){
          *html = 0;
-
-         if(i > 2 && *(buf+i-1) != ' '){
-            *(buf+i) = ' '; i++;
-         }
-
+         // make sure there's a whitespace between tag contents
+         if(i < destlen && i > 0 && !isspace(dest[i-1]))
+            dest[i++] = ' ';
+      }
+      else{
+         if(*html == 0){
+            if(i < destlen) *(dest+i) = *src;
+            i++;
+         }
       }
-
    }
 
-   *(buf+i) = '\0';
+   return i;
 }
 
 
 #ifdef HAVE_ZIP
-int extract_opendocument(struct session_data *sdata, struct _state *state, char *filename, char *prefix){
-   int errorp, i=0, len=0, html=0;
-   int len2;
-   char buf[MAXBUFSIZE];
+int extract_opendocument(struct session_data *sdata, struct parser_state *state, char *filename, char *prefix){
+   int errorp, i=0, html=0;
+   unsigned int len2;
+   char buf[4*MAXBUFSIZE], puf[4*MAXBUFSIZE];
    struct zip *z;
    struct zip_stat sb;
    struct zip_file *zf;
 
-   z = zip_open(filename, 0, &errorp);
-   if(!z) return 1;
+   z = zip_open(filename, ZIP_CHECKCONS, &errorp);
+   if(!z){
+      syslog(LOG_INFO, "%s: error: corrupt zip file=%s, error code=%d", sdata->ttmpfile, filename, errorp);
+      return 1;
+   }
 
    memset(buf, 0, sizeof(buf));
 
    while(zip_stat_index(z, i, 0, &sb) == 0){
-      if(strncmp(sb.name, prefix, strlen(prefix)) == 0 && (int)sb.size > 0){
+      if(ZIP_EM_NONE == sb.encryption_method && strncmp(sb.name, prefix, strlen(prefix)) == 0 && (int)sb.size > 0){
 
          zf = zip_fopen_index(z, i, 0);
          if(zf){
+            int len;
             while((len = zip_fread(zf, buf, sizeof(buf)-2)) > 0){
 
-               remove_xml(buf, &html);
-               len2 = strlen(buf);
+               len2 = remove_xml(buf, puf, sizeof(puf), &html);
 
                if(len2 > 0 && state->bodylen < BIGBUFSIZE-len2-1){
-                  memcpy(&(state->b_body[state->bodylen]), buf, len2);
+                  memcpy(&(state->b_body[state->bodylen]), puf, len2);
                   state->bodylen += len2;
                }
 
@@ -90,8 +97,8 @@ int extract_opendocument(struct session_data *sdata, struct _state *state, char
 }
 
 
-int unzip_file(struct session_data *sdata, struct _state *state, char *filename, int *rec){
-   int errorp, i=0, len=0, fd;
+int unzip_file(struct session_data *sdata, struct parser_state *state, char *filename, int *rec, struct config *cfg){
+   int errorp, i=0, fd;
    char *p, extracted_filename[SMALLBUFSIZE], buf[MAXBUFSIZE];
    struct zip *z;
    struct zip_stat sb;
@@ -99,41 +106,53 @@ int unzip_file(struct session_data *sdata, struct _state *state, char *filename,
 
    (*rec)++;
 
-   z = zip_open(filename, 0, &errorp);
-   if(!z) return 1;
+   z = zip_open(filename, ZIP_CHECKCONS, &errorp);
+   if(!z){
+      syslog(LOG_INFO, "%s: error: corrupt zip file=%s, error code=%d", sdata->ttmpfile, filename, errorp);
+      return 1;
+   }
 
    while(zip_stat_index(z, i, 0, &sb) == 0){
       //printf("processing file inside the zip: %s, index: %d, size: %d\n", sb.name, sb.index, (int)sb.size);
 
-      p = strrchr(sb.name, '.');
+      if(ZIP_EM_NONE == sb.encryption_method) {
 
-      if((int)sb.size > 0 && p && strcmp(get_attachment_extractor_by_filename((char*)sb.name), "other")){
+         p = strrchr(sb.name, '.');
 
-         snprintf(extracted_filename, sizeof(extracted_filename)-1, "%s-%d-%d%s", sdata->ttmpfile, *rec, i, p);
+         if((int)sb.size > 0 && p && strcmp(get_attachment_extractor_by_filename((char*)sb.name), "other")){
 
-         fd = open(extracted_filename, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR);
-         if(fd != -1){
-            zf = zip_fopen_index(z, i, 0);
-            if(zf){
-               while((len = zip_fread(zf, buf, sizeof(buf))) > 0){
-                  write(fd, buf, len);
+            snprintf(extracted_filename, sizeof(extracted_filename)-1, "%s-%d-%d%s", sdata->ttmpfile, *rec, i, p);
+
+            if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: writing zip content to '%s'", sdata->ttmpfile, extracted_filename);
+
+            fd = open(extracted_filename, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
+            if(fd != -1){
+               zf = zip_fopen_index(z, i, 0);
+               if(zf){
+                  int len;
+                  while((len = zip_fread(zf, buf, sizeof(buf))) > 0){
+                     if(write(fd, buf, len) == -1) syslog(LOG_PRIORITY, "ERROR: error writing to fd in %s", __func__);
+                  }
+                  zip_fclose(zf);
                }
-               zip_fclose(zf);
+               else syslog(LOG_PRIORITY, "%s: cannot extract '%s' from '%s'", sdata->ttmpfile, sb.name, extracted_filename);
+
+               close(fd);
+
+               extract_attachment_content(sdata, state, extracted_filename, get_attachment_extractor_by_filename(extracted_filename), rec, cfg);
+
+               unlink(extracted_filename);
+
+            }
+            else {
+               syslog(LOG_PRIORITY, "%s: cannot open '%s'", sdata->ttmpfile, extracted_filename);
             }
-            else syslog(LOG_PRIORITY, "%s: cannot extract '%s' from '%s'", sdata->ttmpfile, sb.name, extracted_filename);
-
-            close(fd);
-
-            extract_attachment_content(sdata, state, extracted_filename, get_attachment_extractor_by_filename(extracted_filename), rec);
-
-            unlink(extracted_filename);
-
-         }
-         else {
-            syslog(LOG_PRIORITY, "%s: cannot open '%s'", sdata->ttmpfile, extracted_filename);
          }
 
       }
+      else {
+         syslog(LOG_PRIORITY, "error: attachment ('%s') is in encrypted zip file", sb.name);
+      }
 
       i++;
    }
@@ -146,66 +165,61 @@ int unzip_file(struct session_data *sdata, struct _state *state, char *filename,
 
 #endif
 
-void read_content_with_popen(struct session_data *sdata, struct _state *state, char *cmd){
-   int len;
-   char buf[MAXBUFSIZE];
-   FILE *f;
 
-   f = popen(cmd, "r");
-   if(f){
-      while(fgets(buf, sizeof(buf)-1, f)){
-         len = strlen(buf);
+#ifdef HAVE_TNEF
 
-         if(state->bodylen < BIGBUFSIZE-len-1){
-            memcpy(&(state->b_body[state->bodylen]), buf, len);
-            state->bodylen += len;
+int extract_tnef(struct session_data *sdata, struct parser_state *state, char *filename, struct config *cfg){
+   int rc=0, n, rec=1;
+   char tmpdir[BUFLEN], buf[SMALLBUFSIZE];
+   struct dirent **namelist;
+
+   memset(tmpdir, 0, sizeof(tmpdir));
+   make_random_string((unsigned char *)&tmpdir[0], sizeof(tmpdir)-3);
+
+   memcpy(&tmpdir[sizeof(tmpdir)-3], ".d", 2);
+
+   if(mkdir(tmpdir, 0700)) return rc;
+
+   snprintf(buf, sizeof(buf)-1, "%s --unix-paths -C %s %s", HAVE_TNEF, tmpdir, filename);
+
+   if(system(buf) == -1) syslog(LOG_INFO, "error: running %s", buf);
+
+   n = scandir(tmpdir, &namelist, NULL, alphasort);
+   if(n < 0) syslog(LOG_INFO, "error: reading %s", tmpdir);
+   else {
+      while(n--){
+         if(strcmp(namelist[n]->d_name, ".") && strcmp(namelist[n]->d_name, "..")){
+
+            snprintf(buf, sizeof(buf)-1, "%s/%s", tmpdir, namelist[n]->d_name);
+            extract_attachment_content(sdata, state, buf, get_attachment_extractor_by_filename(buf), &rec, cfg);
+
+            unlink(buf);
          }
-         else break;
+
+         free(namelist[n]);
       }
-
-      fclose(f);
+      free(namelist);
    }
-   else syslog(LOG_PRIORITY, "%s: popen(): %s", sdata->ttmpfile, buf);
 
+   rmdir(tmpdir);
+
+   return rc;
+}
+
+#endif
+
+
+void kill_helper(){
+   syslog(LOG_PRIORITY, "error: helper is killed by alarm");
+   die("timeout for helper!");
 }
 
 
-void extract_attachment_content(struct session_data *sdata, struct _state *state, char *filename, char *type, int *rec){
-   char cmd[SMALLBUFSIZE];
-
-   if(strcmp(type, "other") == 0) return;
-
-   memset(cmd, 0, sizeof(cmd));
-
-#ifdef HAVE_PDFTOTEXT
-   if(strcmp(type, "pdf") == 0) snprintf(cmd, sizeof(cmd)-1, "%s -enc UTF-8 %s -", HAVE_PDFTOTEXT, filename);
-#endif
-
-#ifdef HAVE_CATDOC
-   if(strcmp(type, "doc") == 0) snprintf(cmd, sizeof(cmd)-1, "%s -d utf-8 %s", HAVE_CATDOC, filename);
-#endif
-
-#ifdef HAVE_CATPPT
-   if(strcmp(type, "ppt") == 0) snprintf(cmd, sizeof(cmd)-1, "%s -d utf-8 %s", HAVE_CATPPT, filename);
-#endif
-
-#ifdef HAVE_XLS2CSV
-   if(strcmp(type, "xls") == 0) snprintf(cmd, sizeof(cmd)-1, "%s -d utf-8 %s", HAVE_XLS2CSV, filename);
-#endif
-
-#ifdef HAVE_PPTHTML
-   if(strcmp(type, "ppt") == 0) snprintf(cmd, sizeof(cmd)-1, "%s %s", HAVE_PPTHTML, filename);
-#endif
-
-#ifdef HAVE_UNRTF
-   if(strcmp(type, "rtf") == 0) snprintf(cmd, sizeof(cmd)-1, "%s --text %s", HAVE_UNRTF, filename);
-#endif
-
-   if(strlen(cmd) > 12){
-      read_content_with_popen(sdata, state, cmd);
-      return;
-   }
+void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct config *cfg){
+   int link[2];
+   pid_t pid;
 
+   if(strcmp(type, "other") == 0 || strcmp(type, "text") == 0) return;
 
 #ifdef HAVE_ZIP
    if(strcmp(type, "odf") == 0){
@@ -230,7 +244,7 @@ void extract_attachment_content(struct session_data *sdata, struct _state *state
 
    if(strcmp(type, "zip") == 0){
       if(*rec < MAX_ZIP_RECURSION_LEVEL){
-         unzip_file(sdata, state, filename, rec);
+         unzip_file(sdata, state, filename, rec, cfg);
       }
       else {
          syslog(LOG_PRIORITY, "%s: multiple recursion level zip attachment, skipping %s", sdata->ttmpfile, filename);
@@ -238,6 +252,79 @@ void extract_attachment_content(struct session_data *sdata, struct _state *state
    }
 #endif
 
+#ifdef HAVE_TNEF
+   if(strcmp(type, "tnef") == 0){
+      extract_tnef(sdata, state, filename, cfg);
+      return;
+   }
+#endif
+
+
+   /*
+    * http://stackoverflow.com/questions/7292642/grabbing-output-from-exec
+    */
+
+   if(pipe(link) == -1){
+      syslog(LOG_PRIORITY, "%s: cannot open link", sdata->ttmpfile);
+      return;
+   }
+
+   if((pid = fork()) == -1){
+      syslog(LOG_PRIORITY, "%s: cannot fork", sdata->ttmpfile);
+      close(link[0]);
+      close(link[1]);
+      return;
+   }
+
+   if(pid == 0){
+      dup2(link[1], STDOUT_FILENO);
+      close(link[0]);
+      close(link[1]);
+
+      alarm(cfg->helper_timeout);
+      sig_catch(SIGALRM, kill_helper);
+
+   #ifdef HAVE_PDFTOTEXT
+      if(strcmp(type, "pdf") == 0) execl(HAVE_PDFTOTEXT, HAVE_PDFTOTEXT, "-enc", "UTF-8", filename, "-", (char *) 0);
+   #endif
+
+   #ifdef HAVE_CATDOC
+      if(strcmp(type, "doc") == 0) execl(HAVE_CATDOC, HAVE_CATDOC, "-d", "utf-8", filename, (char *) 0);
+   #endif
+
+   #ifdef HAVE_CATPPT
+      if(strcmp(type, "ppt") == 0) execl(HAVE_CATPPT, HAVE_CATPPT, "-d", "utf-8", filename, (char *) 0);
+   #endif
+
+   #ifdef HAVE_XLS2CSV
+      if(strcmp(type, "xls") == 0) execl(HAVE_XLS2CSV, HAVE_XLS2CSV, "-d", "utf-8", filename, (char *) 0);
+   #endif
+
+   #ifdef HAVE_PPTHTML
+       if(strcmp(type, "ppt") == 0) execl(HAVE_PPTHTML, HAVE_PPTHTML, filename, (char *) 0);
+   #endif
+
+   #ifdef HAVE_UNRTF
+      if(strcmp(type, "rtf") == 0) execl(HAVE_UNRTF, HAVE_UNRTF, "--text", filename, (char *) 0);
+   #endif
+
+      die("execl");
+   }
+   else {
+      close(link[1]);
+      ssize_t n;
+      char outbuf[MAXBUFSIZE];
+
+      while((n = read(link[0], outbuf, sizeof(outbuf))) > 0){
+         if(state->bodylen < BIGBUFSIZE-n-1){
+            memcpy(&(state->b_body[state->bodylen]), outbuf, n);
+            state->bodylen += n;
+         }
+      }
+      close(link[0]);
+      wait(NULL);
+      return;
+   }
+
+
 }
-
-

src/hash.c

@@ -0,0 +1,163 @@
+/*
+ * hash.c, SJ
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <piler.h>
+
+
+inline int hash(unsigned int key){
+   return key % MAXHASH;
+}
+
+
+void inithash(struct node *xhash[]){
+   int i;
+
+   for(i=0;i<MAXHASH;i++){
+      xhash[i] = NULL;
+   }
+}
+
+
+void clearhash(struct node *xhash[]){
+   int i;
+   struct node *p, *q;
+
+   for(i=0;i<MAXHASH;i++){
+      q = xhash[i];
+      while(q != NULL){
+         p = q;
+
+         q = q->r;
+         if(p->str){
+            free(p->str);
+         }
+         free(p);
+      }
+      xhash[i] = NULL;
+   }
+}
+
+
+struct node *makenewnode(char *s){
+   struct node *h;
+   int len;
+
+   if(s == NULL) return NULL;
+
+   len = strlen(s);
+
+   if((h = malloc(sizeof(struct node))) == NULL) return NULL;
+
+   memset(h, 0, sizeof(struct node));
+
+   h->str = malloc(len+2);
+
+   if(h->str == NULL){
+      free(h);
+      return NULL;
+   }
+
+   memset(h->str, 0, len+2);
+
+   snprintf(h->str, len+1, "%s", s);
+
+   h->key = DJBHash(s, len);
+   h->r = NULL;
+
+   return h;
+}
+
+
+int addnode(struct node *xhash[], char *s){
+   struct node *p=NULL, *q;
+   unsigned int key = 0;
+   int len;
+
+   if(s == NULL) return 0;
+
+   len = strlen(s);
+
+   key = DJBHash(s, len);
+
+   if(xhash[hash(key)] == NULL){
+      xhash[hash(key)] = makenewnode(s);
+   }
+   else {
+      q = xhash[hash(key)];
+      while(q != NULL){
+         p = q;
+         if(p->key == key){
+            return 0;
+         }
+         else {
+            q = q->r;
+         }
+      }
+      if(p) p->r = makenewnode(s);
+   }
+
+   return 1;
+}
+
+
+struct node *findnode(struct node *xhash[], char *s){
+   struct node *q;
+   unsigned int key;
+   int len;
+
+   if(s == NULL) return NULL;
+
+   len = strlen(s);
+
+   key = DJBHash(s, len);
+
+   q = xhash[hash(key)];
+
+   if(q == NULL) return NULL;
+
+   while(q != NULL){
+
+      if(strcmp(q->str, s) == 0){
+         return q;
+      }
+      else {
+         q = q->r;
+      }
+   }
+
+   return NULL;
+}
+
+
+int is_substr_in_hash(struct node *xhash[], char *s){
+   int i;
+   struct node *q;
+
+   for(i=0;i<MAXHASH;i++){
+      q = xhash[i];
+      while(q != NULL){
+         if(q->str && strstr(s, q->str)) return 1;
+
+         q = q->r;
+      }
+   }
+
+   return 0;
+}
+
+
+unsigned int DJBHash(char* str, unsigned int len){
+   unsigned int hashval = 5381;
+   unsigned int i    = 0;
+
+   for(i=0; i < len; str++, i++){
+      hashval = ((hashval << 5) + hashval) + (*str);
+   }
+
+   return hashval;
+}

src/hash.h

@@ -0,0 +1,20 @@
+/*
+ * hash.h, SJ
+ */
+
+#ifndef _HASH_H
+ #define _HASH_H
+
+#include "cfg.h"
+#include "defs.h"
+
+
+void inithash(struct node *xhash[]);
+void clearhash(struct node *xhash[]);
+struct node *makenewnode(char *s);
+int addnode(struct node *xhash[], char *s);
+struct node *findnode(struct node *xhash[], char *s);
+int is_substr_in_hash(struct node *xhash[], char *s);
+unsigned int DJBHash(char* str, unsigned int len);
+
+#endif /* _HASH_H */

src/html.h

@@ -1,38 +0,0 @@
-
-struct html_tag {
-   unsigned char length;
-   char *entity;
-};
-
-#define NUM_OF_SKIP_TAGS2 10
-
-struct html_tag skip_html_tags2[] = {
-   { 4, "html" },
-   { 5, "/html" },
-   { 5, "/body" },
-   { 4, "meta" },
-   { 4, "head" },
-   { 5, "/head" },
-   { 5, "style" },
-   { 6, "/style" },
-   { 3, "div" },
-   { 4, "/div" }
-};
-
-
-#define NUM_OF_SKIP_TAGS 11
-
-struct html_tag skip_html_tags[] = {
-   { 5, "style" },
-   { 4, "dir=" },
-   { 8, "content=" },
-   { 5, "name=" },
-   { 3, "id=" },
-   { 2, "v:" },
-   { 6, "class=" },
-   { 5, "xmlns" },
-   { 10, "http-equiv" },
-   { 7, "spidmax" },
-   { 5, "data=" }
-};
-

src/htmlentities.h

@@ -285,4 +285,3 @@ struct mi htmlentities[] = {
 
 
 #endif /* _HTMLENTITIES_H */
-

src/imap.c

@@ -23,12 +23,11 @@
 #include <piler.h>
 
 
-int get_message_length_from_imap_answer(char *s, int *_1st_line_bytes){
+int get_message_length_from_imap_answer(char *s){
    char *p, *q;
    int len=0;
 
-
-   p = strstr(s, "\r\n");
+   p = strstr(s, "\r");
    if(!p){
       printf("invalid reply: %s", s);
       return len;
@@ -36,9 +35,6 @@ int get_message_length_from_imap_answer(char *s, int *_1st_line_bytes){
 
    *p = '\0';
 
-   //*_1st_line_bytes = strlen(s)+2;
-   *_1st_line_bytes = p-s+2;
-
    if(*(p-1) == '}') *(p-1) = '\0';
 
 
@@ -55,39 +51,107 @@ int get_message_length_from_imap_answer(char *s, int *_1st_line_bytes){
 }
 
 
-int is_last_complete_packet(char *s, int len, char *tagok, char *tagbad, int *pos){
-   char *p;
+int read_response(char *buf, int buflen, struct data *data){
+   int i=0, n, len=0, rc=0;
+   char puf[MAXBUFSIZE], tagok[SMALLBUFSIZE], tagno[SMALLBUFSIZE], tagbad[SMALLBUFSIZE];
 
-   *pos = 0;
+   snprintf(tagok, sizeof(tagok)-1, "A%d OK", data->import->seq);
+   snprintf(tagno, sizeof(tagno)-1, "A%d NO", data->import->seq);
+   snprintf(tagbad, sizeof(tagbad)-1, "A%d BAD", data->import->seq);
 
-   if(*(s+len-2) == '\r' && *(s+len-1) == '\n'){
-      if((p = strstr(s, tagok))){
-         *pos = p - s;
-         if(*pos > 3) *pos -= 2;
-         return 1;
-      }
-      if(strstr(s, tagbad)) return 1;
+   memset(buf, 0, buflen);
+
+   while(!strstr(buf, tagok)){
+      n = recvtimeoutssl(data->net, puf, sizeof(puf));
+
+      if(n + len < buflen) strncat(buf, puf, n);
+      else goto END;
+
+      /*
+       * possible error message from the imap server:
+       *
+       *  * BYE Temporary problem, please try again later\r\n
+       */
+
+      if(i == 0 && (strstr(puf, tagno) || strstr(puf, tagbad) || strstr(puf, "* BYE ")) ) goto END;
+
+
+      len += n;
+      i++;
    }
 
-   return 0;
+   rc = 1;
+
+END:
+
+   (data->import->seq)++;
+
+   return rc;
 }
 
 
-int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sdata, struct __data *data, int use_ssl, struct __config *cfg){
-   int rc=ERR, i, n, pos, endpos, messages=0, len, readlen, fd, lastpos, nreads, processed_messages=0;
-   char *p, tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], tagbad[SMALLBUFSIZE], buf[MAXBUFSIZE], filename[SMALLBUFSIZE];
-   char aggrbuf[3*MAXBUFSIZE];
+int connect_to_imap_server(struct data *data){
+   char buf[MAXBUFSIZE];
 
-   snprintf(tag, sizeof(tag)-1, "A%d", *seq); snprintf(tagok, sizeof(tagok)-1, "\r\nA%d OK", (*seq)++);
-   snprintf(buf, sizeof(buf)-1, "%s SELECT \"%s\"\r\n", tag, folder);
+   data->import->cap_uidplus = 0;
 
-   n = write1(sd, buf, use_ssl, data->ssl);
-   n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
+   if(data->net->use_ssl == 1){
+      init_ssl_to_server(data);
+   }
 
-   if(!strstr(buf, tagok)){
+
+   recvtimeoutssl(data->net, buf, sizeof(buf));
+
+
+   /* imap cmd: LOGIN */
+
+   if(strcmp(data->import->username, "ZIMBRA") == 0){
+      snprintf(buf, sizeof(buf)-1, "A%d AUTHENTICATE PLAIN %s\r\n", data->import->seq, data->import->password);
+   }
+   else {
+      snprintf(buf, sizeof(buf)-1, "A%d LOGIN %s \"%s\"\r\n", data->import->seq, data->import->username, data->import->password);
+   }
+
+   write1(data->net, buf, strlen(buf));
+   if(read_response(buf, sizeof(buf), data) == 0){
+      printf("login failed, server reponse: %s\n", buf);
+      return ERR;
+   }
+
+   if(strstr(buf, "UIDPLUS")){
+      data->import->cap_uidplus = 1;
+   }
+   else {
+
+      /* run the CAPABILITY command if the reply doesn't contain the UIDPLUS capability */
+
+      snprintf(buf, sizeof(buf)-1, "A%d CAPABILITY\r\n", data->import->seq);
+
+      write1(data->net, buf, strlen(buf));
+      read_response(buf, sizeof(buf), data);
+
+      if(strstr(buf, "UIDPLUS")) data->import->cap_uidplus = 1;
+   }
+
+
+   return OK;
+}
+
+
+int imap_select_cmd_on_folder(char *folder, struct data *data){
+   int messages=0;
+   char *p, buf[MAXBUFSIZE];
+
+   if(strchr(folder, '"'))
+      snprintf(buf, sizeof(buf)-1, "A%d SELECT %s\r\n", data->import->seq, folder);
+   else
+      snprintf(buf, sizeof(buf)-1, "A%d SELECT \"%s\"\r\n", data->import->seq, folder);
+
+   write1(data->net, buf, strlen(buf));
+   if(read_response(buf, sizeof(buf), data) == 0){
       trimBuffer(buf);
-      printf("error: %s\n", buf);
-      return rc;
+      printf("ERROR: select cmd error: %s\n", buf);
+      return messages;
    }
 
    p = strstr(buf, " EXISTS");
@@ -100,229 +164,343 @@ int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sda
       }
    }
 
-   printf("found %d messages\n", messages);
+   if(data->quiet == 0){printf("found %d messages\n", messages); }
 
-   if(messages <= 0) return rc;
+   data->import->total_messages += messages;
 
-   for(i=1; i<=messages; i++){
-      processed_messages++;
-      printf("processed: %7d\r", processed_messages); fflush(stdout);
-
-      snprintf(tag, sizeof(tag)-1, "A%d", *seq);
-      snprintf(tagok, sizeof(tagok)-1, "\r\nA%d OK", (*seq)++);
-      snprintf(tagbad, sizeof(tagbad)-1, "\r\n%s BAD", tag);
-
-      snprintf(buf, sizeof(buf)-1, "%s FETCH %d (BODY.PEEK[])\r\n", tag, i);
-
-      snprintf(filename, sizeof(filename)-1, "%s-%d.txt", folder, i);
-      unlink(filename);
-
-      fd = open(filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
-      if(fd == -1){
-         printf("cannot open: %s\n", filename);
-         return rc;
-      }
-
-
-      n = write1(sd, buf, use_ssl, data->ssl);
-
-      readlen = 0;
-      pos = 0;
-      len = 0;
-      nreads = 0;
-      endpos = 0;
-
-      memset(aggrbuf, 0, sizeof(aggrbuf));
-      lastpos = 0;
-
-
-      while((n = recvtimeoutssl(sd, buf, sizeof(buf), 15, use_ssl, data->ssl)) > 0){
-         nreads++;
-         readlen += n;
-
-         if(nreads == 1){
-            len = get_message_length_from_imap_answer(buf, &pos);
-
-            if(len < 10){
-               printf("%d: too short message! %s\n", i, buf);
-               break;
-            }
-         }
-
-         if(lastpos + 1 + n < sizeof(aggrbuf)){
-
-            if(nreads == 1){
-               memcpy(aggrbuf+lastpos, buf+pos, n-pos);
-               lastpos += n-pos;
-            }
-            else {
-               memcpy(aggrbuf+lastpos, buf, n);
-               lastpos += n;
-            }
-         }
-         else {
-            write(fd, aggrbuf, sizeof(buf));
-
-            memmove(aggrbuf, aggrbuf+sizeof(buf), lastpos-sizeof(buf));
-            lastpos -= sizeof(buf);
-
-            memcpy(aggrbuf+lastpos, buf, n);
-            lastpos += n;
-         }
-
-         if(is_last_complete_packet(aggrbuf, lastpos, tagok, tagbad, &endpos) == 1){
-            write(fd, aggrbuf, lastpos-(lastpos-endpos));
-            break;
-         }
-
-      } 
-
-      close(fd);
-
-      rc = import_message(filename, sdata, data, cfg);
-
-      if(rc == ERR) printf("error importing '%s'\n", filename);
-      else unlink(filename);
-   }
-
-   printf("\n");
-
-   return OK;
+   return messages;
 }
 
 
-int connect_to_imap_server(int sd, int *seq, char *username, char *password, int port, struct __data *data, int use_ssl){
-   int n;
-   char tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], buf[MAXBUFSIZE];
-   X509* server_cert;
-   char *str;
+int imap_download_email(struct data *data, int i){
+   int fd, len, result, tagoklen, tagbadlen;
+   int n, readlen=0, nreads=0, readpos=0, finished=0, msglen=0, msg_written_len=0;
+   char *p, buf[MAXBUFSIZE], puf[MAXBUFSIZE], tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], tagbad[SMALLBUFSIZE];
 
+   data->import->processed_messages++;
 
-   if(use_ssl == 1){
+   snprintf(data->import->filename, SMALLBUFSIZE-1, "%d-imap-%d.txt", getpid(), data->import->processed_messages);
 
-      SSL_library_init();
-      SSL_load_error_strings();
+   unlink(data->import->filename);
 
-      data->ctx = SSL_CTX_new(SSLv3_client_method());
-      CHK_NULL(data->ctx, "internal SSL error");
-
-      data->ssl = SSL_new(data->ctx);
-      CHK_NULL(data->ssl, "internal ssl error");
-
-      SSL_set_fd(data->ssl, sd);
-      n = SSL_connect(data->ssl);
-      CHK_SSL(n, "internal ssl error");
-
-      //printf("Cipher: %s\n", SSL_get_cipher(data->ssl));
-
-      server_cert = SSL_get_peer_certificate(data->ssl);
-      CHK_NULL(server_cert, "server cert error");
-
-      //if(verbose){
-         str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0);
-         CHK_NULL(str, "error in server cert");
-         printf("server cert:\n\t subject: %s\n", str);
-         OPENSSL_free(str);
-
-         str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0);
-         CHK_NULL(str, "error in server cert");
-         printf("\t issuer: %s\n\n", str);
-         OPENSSL_free(str);
-      //}
-
-      X509_free(server_cert);
-   }
-
-
-   n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
-
-
-   snprintf(tag, sizeof(tag)-1, "A%d", *seq); snprintf(tagok, sizeof(tagok)-1, "A%d OK", (*seq)++);
-   snprintf(buf, sizeof(buf)-1, "%s CAPABILITY\r\n", tag);
-
-   write1(sd, buf, use_ssl, data->ssl);
-   n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
-
-
-   snprintf(tag, sizeof(tag)-1, "A%d", *seq); snprintf(tagok, sizeof(tagok)-1, "A%d OK", (*seq)++);
-   snprintf(buf, sizeof(buf)-1, "%s LOGIN %s \"%s\"\r\n", tag, username, password);
-
-   write1(sd, buf, use_ssl, data->ssl);
-   n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
-
-   if(strncmp(buf, tagok, strlen(tagok))){
-      printf("login failed, server reponse: %s\n", buf);
+   fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
+   if(fd == -1){
+      printf("cannot open: %s\n", data->import->filename);
       return ERR;
    }
 
+   snprintf(tag, sizeof(tag)-1, "A%d", data->import->seq);
+   snprintf(tagok, sizeof(tagok)-1, "A%d OK", (data->import->seq)++);
+   snprintf(tagbad, sizeof(tagbad)-1, "%s BAD", tag);
+
+   tagoklen = strlen(tagok);
+   tagbadlen = strlen(tagbad);
+
+   snprintf(buf, sizeof(buf)-1, "%s FETCH %d (BODY.PEEK[])\r\n", tag, i);
+   write1(data->net, buf, strlen(buf));
+
+   while((n = recvtimeoutssl(data->net, &buf[readpos], sizeof(buf)-readpos)) > 0){
+
+      readlen += n;
+
+      if(strchr(buf, '\n')){
+         readpos = 0;
+         p = &buf[0];
+         do {
+            nreads++;
+            memset(puf, 0, sizeof(puf));
+            p = split(p, '\n', puf, sizeof(puf)-1, &result);
+            len = strlen(puf);
+
+            if(result == 1){
+               // process a complete line
+
+               if(nreads == 1){
+
+                  if(strcasestr(puf, " FETCH ")){
+                     msglen = get_message_length_from_imap_answer(puf);
+
+                     if(msglen == 0){
+                        finished = 1;
+                        break;
+                     }
+                     continue;
+                  }
+
+                  if(strcasestr(puf, " BYE")){
+                     printf("imap server sent BYE response: '%s'\n", puf);
+                     close(fd);
+                     unlink(data->import->filename);
+                     return ERR;
+                  }
+
+               }
+
+               if(len > 0 && msg_written_len < msglen){
+                  if(write(fd, puf, len) == -1) printf("ERROR: writing to fd\n");
+                  if(write(fd, "\n", 1) == -1) printf("ERROR: writing to fd\n");
+                  msg_written_len += len + 1;
+               }
+
+               if(strncmp(puf, tagok, tagoklen) == 0){
+                  finished = 1;
+                  break;
+               }
+
+               if(strncmp(puf, tagbad, tagbadlen) == 0){
+                  printf("ERROR happened reading the message!\n");
+                  finished = 1;
+                  break;
+               }
+
+            }
+            else {
+               // prepend the last incomplete line back to 'buf'
+
+               snprintf(buf, sizeof(buf)-2, "%s", puf);
+               readpos = len;
+               break;
+            }
+
+         } while(p);
+
+
+      }
+      else {
+         readpos += n;
+      }
+
+      if(finished == 1) break;
+   }
+
+   close(fd);
+
+   if(msglen > 10) return OK;
+
+   return ERR;
+}
+
+
+void imap_delete_message(struct data *data, int i){
+   char buf[SMALLBUFSIZE];
+
+   snprintf(buf, sizeof(buf)-1, "A%d STORE %d +FLAGS.SILENT (\\Deleted)\r\n", data->import->seq, i);
+   write1(data->net, buf, strlen(buf));
+   read_response(buf, sizeof(buf), data);
+}
+
+
+void imap_move_message_to_folder(struct data *data, int i){
+   int tagoklen;
+   char buf[SMALLBUFSIZE], tagok[SMALLBUFSIZE];
+
+   snprintf(tagok, sizeof(tagok)-1, "A%d OK", data->import->seq);
+   tagoklen = strlen(tagok);
+
+   snprintf(buf, sizeof(buf)-1, "A%d COPY %d %s\r\n", data->import->seq, i, data->import->move_folder);
+   write1(data->net, buf, strlen(buf));
+   read_response(buf, sizeof(buf), data);
+
+   if(strncmp(buf, tagok, tagoklen) == 0){
+      snprintf(buf, sizeof(buf)-1, "A%d STORE %d +FLAGS.SILENT (\\Deleted)\r\n", data->import->seq, i);
+      write1(data->net, buf, strlen(buf));
+      read_response(buf, sizeof(buf), data);
+   }
+}
+
+
+void imap_expunge_message(struct data *data){
+   char buf[SMALLBUFSIZE];
+
+   snprintf(buf, sizeof(buf)-1, "A%d EXPUNGE\r\n", data->import->seq);
+   write1(data->net, buf, strlen(buf));
+   read_response(buf, sizeof(buf), data);
+}
+
+
+int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   int i, messages=0;
+
+   messages = imap_select_cmd_on_folder(folder, data);
+
+   if(messages <= 0) return OK;
+
+   if(data->recursive_folder_names == 1){
+      data->folder = get_folder_id(sdata, folder, 0);
+      if(data->folder == ERR_FOLDER) data->folder = add_new_folder(sdata, folder, 0);
+   }
+
+   for(i=data->import->start_position; i<=messages; i++){
+      if(imap_download_email(data, i) == OK){
+         if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/messages); fflush(stdout); }
+
+         if(data->import->dryrun == 0){
+            int rc = import_message(sdata, data, counters, cfg);
+
+            if(data->import->remove_after_import == 1 && rc == OK){
+               imap_delete_message(data, i);
+            }
+
+            if(data->import->move_folder && data->import->cap_uidplus == 1){
+               imap_move_message_to_folder(data, i);
+            }
+         }
+
+         if(data->import->download_only == 0) unlink(data->import->filename);
+      }
+
+      /* whether to quit after processing a batch of messages */
+
+      if(data->import->batch_processing_limit > 0 && data->import->processed_messages >= data->import->batch_processing_limit){
+         break;
+      }
+   }
+
+   if((data->import->remove_after_import == 1 || data->import->move_folder) && data->import->dryrun == 0){
+      imap_expunge_message(data);
+   }
+
+
+   if(data->quiet == 0){printf("\n"); }
+
    return OK;
 }
 
 
-void close_connection(int sd, struct __data *data, int use_ssl){
-   close(sd);
- 
-   if(use_ssl == 1){
-      SSL_shutdown(data->ssl);
-      SSL_free(data->ssl);
-      SSL_CTX_free(data->ctx);
-      ERR_free_strings();
-   }
+void send_imap_close(struct data *data){
+   char puf[SMALLBUFSIZE];
+   snprintf(puf, sizeof(puf)-1, "A%d CLOSE\r\n", data->import->seq);
+
+   write1(data->net, puf, strlen(puf));
 }
 
 
-int list_folders(int sd, int *seq, char *folders, int foldersize, int use_ssl, struct __data *data){
-   char *p, *q, tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], buf[3*MAXBUFSIZE+3], puf[MAXBUFSIZE];
-   int len=0, n;
+int list_folders(struct data *data){
+   char *p, *q, *r, *buf, *ruf, tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], puf[MAXBUFSIZE];
+   char attrs[SMALLBUFSIZE], folder[SMALLBUFSIZE];
+   int len=MAXBUFSIZE+3, pos=0, n, rc=ERR, fldrlen=0, result;
 
-   memset(buf, 0, sizeof(buf));
+   if(data->quiet == 0){printf("List of IMAP folders:\n"); }
 
-   snprintf(tag, sizeof(tag)-1, "A%d", *seq); snprintf(tagok, sizeof(tagok)-1, "A%d OK", (*seq)++);
-   //snprintf(puf, sizeof(puf)-1, "%s LIST \"\" %%\r\n", tag);
-   snprintf(puf, sizeof(puf)-1, "%s LIST \"\" \"*\"\r\n", tag);
+   buf = malloc(len);
+   if(!buf) return rc;
 
-   write1(sd, puf, use_ssl, data->ssl);
+   memset(buf, 0, len);
+
+   snprintf(tag, sizeof(tag)-1, "A%d", data->import->seq); snprintf(tagok, sizeof(tagok)-1, "A%d OK", (data->import->seq)++);
+   if(data->import->folder_imap == NULL)
+      snprintf(puf, sizeof(puf)-1, "%s LIST \"\" \"*\"\r\n", tag);
+   else
+      snprintf(puf, sizeof(puf)-1, "%s LIST \"%s\" \"*\"\r\n", tag, data->import->folder_imap);
+
+   write1(data->net, puf, strlen(puf));
+
+   p = NULL;
 
    while(1){
-      n = recvtimeoutssl(sd, puf, sizeof(puf), 10, use_ssl, data->ssl);
-      if(len + n < sizeof(buf)){
-         memcpy(&buf[len], puf, n);
-         len += n;
-      }
-      else break;
+      n = recvtimeoutssl(data->net, puf, sizeof(puf));
+      if(n < 0) return ERR;
 
-      if(strstr(buf, tagok)) break;
+      if(pos + n >= len){
+         q = realloc(buf, len+MAXBUFSIZE+1);
+         if(!q){
+            printf("realloc failure: %d bytes\n", pos+MAXBUFSIZE+1);
+            goto ENDE_FOLDERS;
+         }
+
+         buf = q;
+         memset(buf+pos, 0, MAXBUFSIZE+1);
+         len += MAXBUFSIZE+1;
+      }
+
+      memcpy(buf + pos, puf, n);
+      pos += n;
+
+      p = strstr(buf, tagok);
+      if(p) break;
    }
- 
-   p = &buf[0];
+
+   // trim the "A3 OK LIST completed" trailer off
+   if(p) *p = '\0'; //-V547
+
+   memset(attrs, 0, sizeof(attrs));
+
+   p = buf;
    do {
       memset(puf, 0, sizeof(puf));
-      p = split(p, '\n', puf, sizeof(puf)-1);
+      p = split(p, '\n', puf, sizeof(puf)-1, &result);
       trimBuffer(puf);
 
-      if(strncmp(puf, "* LIST ", 7) == 0){
+      if(strncmp(puf, "* LIST ", 7) == 0 || fldrlen){
 
-         q = strstr(puf, ") \"");
+         if (fldrlen)
+            q = puf;
+         else
+            q = strstr(puf, ") \"");
          if(q){
-            q += 5;
+            if (!fldrlen) {
+               *q = '\0';
+               snprintf(attrs, sizeof(attrs)-1, "%s", &puf[8]);
 
-            if(*q == ' ') q++;
-            if(*q == '"') q++;
+               q += 3;
+               while(*q != '"') q++;
+               q++;
+               if(*q == ' ') q++;
+            }
 
-            if(q[strlen(q)-1] == '"') q[strlen(q)-1] = '\0';
+            if(!fldrlen && *q == '{' && q[strlen(q)-1] == '}') {
+               q++;
+               fldrlen = strtol(q, NULL, 10);
+            } else {
 
-            strncat(folders, "\n", foldersize-1);
-            strncat(folders, q, foldersize-1);
+               if(fldrlen) {
+                  int ruflen = strlen(q) * 2;
+                  ruf = malloc(ruflen + 1);
+                  if(ruf){
+                     snprintf(ruf, ruflen, "%s", q);
+                     r = ruf;
+                     while(*r != '\0') {
+                        if(*r == '\\') {
+                           memmove(r + 1, r, strlen(r));
+                           r++;
+                        }
+                        r++;
+                     }
+
+                     snprintf(folder, sizeof(folder)-1, "%s", ruf);
+
+                     free(ruf);
+                  }
+                  else {
+                     printf("error: ruf = malloc()\n");
+                  }
+
+                  fldrlen = 0;
+               } else {
+                  snprintf(folder, sizeof(folder)-1, "%s", q);
+               }
+
+               if(!strstr(attrs, "\\Noselect")){
+                  addnode(data->imapfolders, folder);
+               }
+               else if(data->quiet == 0){printf("skipping "); }
+
+               if(data->quiet == 0){printf("=> '%s [%s]'\n", folder, attrs); }
+
+               memset(attrs, 0, sizeof(attrs));
+            }
 
          }
       }
-      else {
-         if(strncmp(puf, tagok, strlen(tagok)) == 0) {}
-      }
 
    } while(p);
 
-   return 0;
+
+   rc = OK;
+
+ENDE_FOLDERS:
+
+   free(buf);
+
+   return rc;
 }
-
-

src/import.c

@@ -18,22 +18,29 @@
 #include <piler.h>
 
 
-int import_message(char *filename, struct session_data *sdata, struct __data *data, struct __config *cfg){
-   int rc=ERR, fd;
+int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   int rc=ERR;
    char *rule;
    struct stat st;
-   struct _state state;
-   struct __counters counters;
+   struct parser_state state;
 
+   if(data->import->delay > 0){
+      struct timespec req;
+
+      req.tv_sec = 0;
+      req.tv_nsec = 1000000 * data->import->delay;
+
+      nanosleep(&req, NULL);
+   }
 
    init_session_data(sdata, cfg);
 
-   if(cfg->verbosity > 1) printf("processing: %s\n", filename);
+   if(cfg->verbosity > 1) printf("processing: %s\n", data->import->filename);
 
-   if(strcmp(filename, "-") == 0){
+   if(strcmp(data->import->filename, "-") == 0){
 
       if(read_from_stdin(sdata) == ERR){
-         printf("error reading from stdin\n");
+         printf("ERROR: error reading from stdin\n");
          return rc;
       }
 
@@ -42,179 +49,189 @@ int import_message(char *filename, struct session_data *sdata, struct __data *da
    }
    else {
 
-      if(stat(filename, &st) != 0){
-         printf("cannot stat() %s\n", filename);
+      if(stat(data->import->filename, &st) != 0){
+         printf("ERROR: cannot stat() %s\n", data->import->filename);
          return rc;
       }
 
       if(S_ISREG(st.st_mode) == 0){
-         printf("%s is not a file\n", filename);
+         printf("ERROR: %s is not a file\n", data->import->filename);
          return rc;
       }
 
-      fd = open(filename, O_RDONLY);
-      if(fd == -1){
-         printf("cannot open %s\n", filename);
-         return rc;
-      }
-      close(fd);
-
-      snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
+      snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", data->import->filename);
 
       sdata->tot_len = st.st_size;
    }
 
 
-   
-   sdata->sent = 0;
+   if(sdata->tot_len < cfg->min_message_size){
+      printf("%s is too short: %d bytes\n", sdata->filename, sdata->tot_len);
+      return rc;
+   }
+
+   data->import->total_size += sdata->tot_len;
+
+
    sdata->delivered = 0;
 
+   sdata->import = 1;
+
    state = parse_message(sdata, 1, data, cfg);
    post_parse(sdata, &state, cfg);
-
-   if(sdata->sent <= 0 && sdata->delivered > 0) sdata->sent = sdata->delivered;
-
-   if(sdata->sent > sdata->now) sdata->sent = sdata->now;
-   if(sdata->sent == -1) sdata->sent = 0;
-
-   /* fat chances that you won't import emails before 1990.01.01 */
-
-   if(sdata->sent > 631148400) sdata->retained = sdata->sent;
-
-   rule = check_againt_ruleset(data->archiving_rules, &state, sdata->tot_len, sdata->spam_message);
+   rule = check_against_ruleset(data->archiving_rules, &state, sdata->tot_len, sdata->spam_message);
 
    if(rule){
-      printf("discarding %s by archiving policy: %s\n", filename, rule);
+      if(data->quiet == 0) printf("discarding %s by archiving policy: %s\n", data->import->filename, rule);
       rc = OK;
-      goto ENDE;
+   }
+   else {
+      make_digests(sdata, cfg);
+
+      if(sdata->hdr_len < 10){
+         printf("%s: invalid message, hdr_len: %d\n", data->import->filename, sdata->hdr_len);
+         rc = ERR;
+      }
+      else if(data->import->after > 0 && sdata->sent < data->import->after){
+         if(cfg->verbosity > 1) printf("discarding older email: %s (%ld/%ld)\n", sdata->filename, sdata->sent, data->import->after);
+         rc = ERR_DISCARDED;
+      }
+      else if(data->import->before > 0 && sdata->sent > data->import->before){
+         if(cfg->verbosity > 1) printf("discarding newer email: %s (%ld/%ld)\n", sdata->filename, sdata->sent, data->import->before);
+         rc = ERR_DISCARDED;
+      }
+      else {
+         // When importing emails, we should add the retention value (later) to the original sent value
+         sdata->retained = sdata->sent;
+
+         // backup original value of data->folder
+         int folder = data->folder;
+         rc = process_message(sdata, &state, data, cfg);
+         data->folder = folder;
+         unlink(state.message_id_hash);
+      }
    }
 
-   make_digests(sdata, cfg);
-
-   rc = process_message(sdata, &state, data, cfg);
-
-ENDE:
    unlink(sdata->tmpframe);
 
-   if(strcmp(filename, "-") == 0) unlink(sdata->ttmpfile);
+   remove_stripped_attachments(&state);
+
+   if(strcmp(data->import->filename, "-") == 0) unlink(sdata->ttmpfile);
 
 
    switch(rc) {
       case OK:
-                        bzero(&counters, sizeof(counters));
-                        counters.c_size += sdata->tot_len;
-                        update_counters(sdata, data, &counters, cfg);
+                        counters->c_rcvd++;
+                        counters->c_size += sdata->tot_len;
+                        counters->c_stored_size += sdata->stored_len;
 
                         break;
 
       case ERR_EXISTS:
                         rc = OK;
-                        printf("duplicate: %s (id: %s)\n", filename, sdata->ttmpfile);
+
+                        counters->c_duplicate++;
+
+                        if(data->quiet == 0) printf("duplicate: %s (duplicate id: %llu)\n", data->import->filename, sdata->duplicate_id);
+                        break;
+
+      case ERR_DISCARDED:
+                        rc = OK;
+
+                        counters->c_ignore++;
+
                         break;
 
       default:
-                        printf("failed to import: %s (id: %s)\n", filename, sdata->ttmpfile);
+                        printf("failed to import: %s (id: %s)\n", data->import->filename, sdata->ttmpfile);
                         break;
-   } 
+   }
+
+   if(rc != OK && data->import->failed_folder){
+      char *p = strrchr(data->import->filename, '/');
+      if(p)
+         p++;
+      else
+         p = data->import->filename;
+
+      char newpath[SMALLBUFSIZE];
+      snprintf(newpath, sizeof(newpath)-2, "%s/%s", data->import->failed_folder, p);
+
+      if(rename(data->import->filename, newpath))
+         printf("cannot move %s to %s\n", data->import->filename, newpath);
+   }
 
    return rc;
 }
 
 
-unsigned long get_folder_id(struct session_data *sdata, struct __data *data, char *foldername, int parent_id){
-   unsigned long id=0;
-   MYSQL_BIND bind[2];
-   unsigned long len[2];
+int update_import_table(struct session_data *sdata, struct data *data) {
+   int ret=ERR, status=2, started=1;
+   struct sql sql;
 
-   if(prepare_a_mysql_statement(sdata, &(data->stmt_get_folder_id), SQL_PREPARED_STMT_GET_FOLDER_ID) == ERR) goto ENDE;
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE) == ERR) return ret;
 
-   memset(bind, 0, sizeof(bind));
+   p_bind_init(&sql);
 
-   bind[0].buffer_type = MYSQL_TYPE_STRING;
-   bind[0].buffer = foldername;
-   bind[0].is_null = 0;
-   len[0] = strlen(foldername); bind[0].length = &len[0];
+   sql.sql[sql.pos] = (char *)&(started); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&(status); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&(data->import->tot_msgs); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&(data->import->table_id); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
 
-   bind[1].buffer_type = MYSQL_TYPE_LONG;
-   bind[1].buffer = (char *)&parent_id;
-   bind[1].is_null = 0;
-   bind[1].length = 0;
+   if(p_exec_stmt(sdata, &sql) == OK) ret = OK;
 
-   if(mysql_stmt_bind_param(data->stmt_get_folder_id, bind)){
-      goto CLOSE;
+   close_prepared_statement(&sql);
+
+   return ret;
+}
+
+
+int get_folder_id(struct session_data *sdata, char *foldername, int parent_id){
+   int id=ERR_FOLDER;
+   struct sql sql;
+
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_FOLDER_ID) == ERR) return id;
+
+   p_bind_init(&sql);
+
+   sql.sql[sql.pos] = foldername; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = (char *)&parent_id; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+
+   if(p_exec_stmt(sdata, &sql) == OK){
+
+      p_bind_init(&sql);
+
+      sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(unsigned long); sql.pos++;
+
+      p_store_results(&sql);
+      p_fetch_results(&sql);
+      p_free_results(&sql);
    }
 
-
-   if(mysql_stmt_execute(data->stmt_get_folder_id)){
-      goto CLOSE;
-   }
-
-   memset(bind, 0, sizeof(bind));
-
-   bind[0].buffer_type = MYSQL_TYPE_LONG;
-   bind[0].buffer = (char *)&id;
-   bind[0].is_null = 0;
-   bind[0].length = 0;
-
-   if(mysql_stmt_bind_result(data->stmt_get_folder_id, bind)){
-      goto CLOSE;
-   }
-
-
-   if(mysql_stmt_store_result(data->stmt_get_folder_id)){
-      goto CLOSE;
-   }
-
-   mysql_stmt_fetch(data->stmt_get_folder_id);
-
-CLOSE:
-   mysql_stmt_close(data->stmt_get_folder_id);
-
-ENDE:
+   close_prepared_statement(&sql);
 
    return id;
 }
 
 
-unsigned long add_new_folder(struct session_data *sdata, struct __data *data, char *foldername, int parent_id){
-   unsigned long id=0;
-   MYSQL_BIND bind[2];
-   unsigned long len[2];
+int add_new_folder(struct session_data *sdata, char *foldername, int parent_id){
+   int id=ERR_FOLDER;
+   struct sql sql;
 
+   if(foldername == NULL) return id;
 
-   if(prepare_a_mysql_statement(sdata, &(data->stmt_insert_into_folder_table), SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE) == ERR) goto ENDE;
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE) == ERR) return id;
 
-   memset(bind, 0, sizeof(bind));
+   p_bind_init(&sql);
 
-   bind[0].buffer_type = MYSQL_TYPE_STRING;
-   bind[0].buffer = foldername;
-   bind[0].is_null = 0;
-   len[0] = strlen(foldername); bind[0].length = &len[0];
+   sql.sql[sql.pos] = foldername; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = (char *)&parent_id; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
 
-   bind[1].buffer_type = MYSQL_TYPE_LONG;
-   bind[1].buffer = (char *)&parent_id;
-   bind[1].is_null = 0;
-   bind[1].length = 0;
-
-   if(mysql_stmt_bind_param(data->stmt_insert_into_folder_table, bind)){
-      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_FOLDER_TABLE, mysql_stmt_error(data->stmt_insert_into_folder_table));
-      goto CLOSE;
+   if(p_exec_stmt(sdata, &sql) == OK){
+      id = p_get_insert_id(&sql);
    }
 
-   if(mysql_stmt_execute(data->stmt_insert_into_folder_table)){
-      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute error: *%s*", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_error(&(sdata->mysql)));
-      goto CLOSE;
-   }
-
-
-   id = mysql_stmt_insert_id(data->stmt_insert_into_folder_table);
-
-CLOSE:
-   mysql_stmt_close(data->stmt_insert_into_folder_table);
-
-ENDE:
+   close_prepared_statement(&sql);
 
    return id;
 }
-
-

src/import.h

@@ -0,0 +1,28 @@
+/*
+ * import.h, SJ
+ */
+
+#ifndef _IMPORT_H
+   #define _IMPORT_H
+
+
+int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+int update_import_table(struct session_data *sdata, struct data *data);
+
+int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg);
+int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+
+int connect_to_pop3_server(struct data *data);
+void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+
+int connect_to_imap_server(struct data *data);
+int list_folders(struct data *data);
+int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+void send_imap_close(struct data *data);
+
+void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+
+#endif /* _IMPORT_H */

src/import_imap.c

@@ -0,0 +1,113 @@
+/*
+ * import_imap.c
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <getopt.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   int i, rc=ERR, ret=OK, skipmatch;
+   char port_string[8], puf[SMALLBUFSIZE];
+   struct addrinfo hints, *res;
+   struct node *q;
+
+   data->net->use_ssl = 0;
+   data->import->seq = 1;
+
+   inithash(data->imapfolders);
+
+   snprintf(port_string, sizeof(port_string)-1, "%d", data->import->port);
+
+   memset(&hints, 0, sizeof(hints));
+   hints.ai_family = AF_UNSPEC;
+   hints.ai_socktype = SOCK_STREAM;
+
+   if((rc = getaddrinfo(data->import->server, port_string, &hints, &res)) != 0){
+      printf("getaddrinfo for '%s': %s\n", data->import->server, gai_strerror(rc));
+      return ERR;
+   }
+
+   if(data->import->port == 993) data->net->use_ssl = 1;
+
+
+   if((data->net->socket = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
+      printf("cannot create socket\n");
+      ret = ERR;
+      goto ENDE_IMAP;
+   }
+
+   if(connect(data->net->socket, res->ai_addr, res->ai_addrlen) == -1){
+      printf("connect()\n");
+      ret = ERR;
+      goto ENDE_IMAP;
+   }
+
+   if(connect_to_imap_server(data) == ERR){
+      close(data->net->socket);
+      ret = ERR;
+      goto ENDE_IMAP;
+   }
+
+
+   if(list_folders(data) == ERR) goto ENDE_IMAP;
+
+
+   for(i=0;i<MAXHASH;i++){
+      q = data->imapfolders[i];
+      while(q != NULL){
+
+         if(q->str && strlen(q->str) > 1){
+
+            skipmatch = 0;
+
+            if(data->import->skiplist && strlen(data->import->skiplist) > 0){
+               snprintf(puf, sizeof(puf)-1, "%s,", (char *)q->str);
+               if(strstr(data->import->skiplist, puf)) skipmatch = 1;
+            }
+
+            if(skipmatch == 1){
+               if(data->quiet == 0) printf("SKIPPING FOLDER: %s\n", (char *)q->str);
+            }
+            else {
+               if(data->quiet == 0) printf("processing folder: %s... ", (char *)q->str);
+
+               if(process_imap_folder(q->str, sdata, data, counters, cfg) == ERR) ret = ERR;
+            }
+
+         }
+
+         q = q->r;
+
+      }
+   }
+
+   send_imap_close(data);
+
+   close_connection(data->net);
+
+ENDE_IMAP:
+   freeaddrinfo(res);
+
+   clearhash(data->imapfolders);
+
+   data->import->status = 2;
+
+   return ret;
+}

src/import_mailbox.c

@@ -0,0 +1,151 @@
+/*
+ * import_mailbox.c
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <getopt.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   FILE *F, *f=NULL;
+   int rc=ERR, tot_msgs=0, ret=OK;
+   char buf[MAXBUFSIZE];
+   time_t t;
+
+
+   F = fopen(mailbox, "r");
+   if(!F){
+      printf("cannot open mailbox: %s\n", mailbox);
+      return rc;
+   }
+
+   t = time(NULL);
+
+   while(fgets(buf, sizeof(buf)-1, F)){
+
+      if(buf[0] == 'F' && buf[1] == 'r' && buf[2] == 'o' && buf[3] == 'm' && buf[4] == ' '){
+         tot_msgs++;
+         if(f){
+            fclose(f);
+            f = NULL;
+            rc = import_message(sdata, data, counters, cfg);
+            if(rc == ERR){
+               printf("error importing: '%s'\n", data->import->filename);
+               ret = ERR;
+            }
+            else unlink(data->import->filename);
+
+            if(data->quiet == 0){ printf("processed: %7d\r", data->import->tot_msgs); fflush(stdout); }
+         }
+
+         snprintf(data->import->filename, sizeof(data->import->filename)-1, "%ld-%d", t, data->import->tot_msgs);
+         f = fopen(data->import->filename, "w+");
+         continue;
+      }
+
+      if(f) fprintf(f, "%s", buf);
+   }
+
+   if(f){
+      fclose(f);
+      rc = import_message(sdata, data, counters, cfg);
+      if(rc == ERR){
+         printf("ERROR: error importing: '%s'\n", data->import->filename);
+         ret = ERR;
+      }
+      else unlink(data->import->filename);
+
+      if(data->quiet == 0){ printf("processed: %7d\r", data->import->tot_msgs); fflush(stdout); }
+   }
+
+   fclose(F);
+
+   return ret;
+}
+
+
+int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   DIR *dir;
+   struct dirent *de;
+   int rc=ERR, ret=OK, i=0;
+   int folder;
+   char fname[SMALLBUFSIZE];
+   struct stat st;
+
+   dir = opendir(directory);
+   if(!dir){
+      printf("cannot open directory: %s\n", directory);
+      return ERR;
+   }
+
+
+   while((de = readdir(dir))){
+      if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
+
+      snprintf(fname, sizeof(fname)-1, "%s/%s", directory, de->d_name);
+
+      if(stat(fname, &st) == 0){
+         if(S_ISDIR(st.st_mode)){
+            folder = data->folder;
+            rc = import_mbox_from_dir(fname, sdata, data, counters, cfg);
+            data->folder = folder;
+            if(rc == ERR) ret = ERR;
+         }
+         else {
+
+            if(S_ISREG(st.st_mode)){
+               if(i == 0 && data->recursive_folder_names == 1){
+                  folder = get_folder_id(sdata, fname, data->folder);
+                  if(folder == ERR_FOLDER){
+                     folder = add_new_folder(sdata, fname, data->folder);
+
+                     if(folder == ERR_FOLDER){
+                        printf("error: cannot get/add folder '%s' to parent id: %d\n", fname, data->folder);
+                        return ERR;
+                     }
+                     else {
+                        data->folder = folder;
+                     }
+                  }
+
+               }
+
+               rc = import_from_mailbox(fname, sdata, data, counters, cfg);
+               if(rc == OK) (data->import->tot_msgs)++;
+               else ret = ERR;
+
+               i++;
+            }
+            else {
+               printf("%s is not a file\n", fname);
+            }
+
+         }
+      }
+      else {
+         printf("cannot stat() %s\n", fname);
+      }
+
+   }
+   closedir(dir);
+
+   return ret;
+}
+
+

src/import_maildir.c

@@ -0,0 +1,111 @@
+/*
+ * import_maildir.c
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <getopt.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg){
+   DIR *dir;
+   struct dirent *de;
+   int rc=ERR, ret=OK, i=0;
+   int folder;
+   char *p, subdir[SMALLBUFSIZE];
+   struct stat st;
+
+   dir = opendir(directory);
+   if(!dir){
+      printf("cannot open directory: %s\n", directory);
+      return ERR;
+   }
+
+
+   while((de = readdir(dir))){
+      if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
+
+      snprintf(data->import->filename, SMALLBUFSIZE-1, "%s/%s", directory, de->d_name);
+
+      if(stat(data->import->filename, &st) == 0){
+         if(S_ISDIR(st.st_mode)){
+            folder = data->folder;
+            snprintf(subdir, sizeof(subdir)-1, "%s", data->import->filename);
+            rc = import_from_maildir(sdata, data, subdir, counters, cfg);
+            data->folder = folder;
+            if(rc == ERR) ret = ERR;
+         }
+         else {
+
+            if(S_ISREG(st.st_mode)){
+               if(i == 0 && data->recursive_folder_names == 1){
+                  p = strrchr(directory, '/');
+                  if(p) p++;
+                  else {
+                     printf("ERROR: invalid directory name: '%s'\n", directory);
+                     return ERR;
+                  }
+
+                  folder = get_folder_id(sdata, p, data->folder);
+                  if(folder == ERR_FOLDER){
+                     folder = add_new_folder(sdata, p, data->folder);
+
+                     if(folder == ERR_FOLDER){
+                        printf("error: cannot get/add folder '%s' to parent id: %d\n", p, data->folder);
+                        return ERR;
+                     }
+                     else {
+                        data->folder = folder;
+                     }
+                  }
+
+               }
+
+               rc = import_message(sdata, data, counters, cfg);
+
+               if(rc == OK) (data->import->tot_msgs)++;
+               else if(rc == ERR){
+                  printf("ERROR: error importing: '%s'\n", data->import->filename);
+                  ret = ERR;
+               }
+
+               if(data->import->remove_after_import == 1 && rc != ERR) unlink(data->import->filename);
+
+               i++;
+
+               if(data->quiet == 0){ printf("processed: %7d\r", data->import->tot_msgs); fflush(stdout); }
+            }
+            else {
+               printf("%s is not a file\n", data->import->filename);
+            }
+
+         }
+      }
+      else {
+         printf("cannot stat() %s\n", data->import->filename);
+      }
+
+   }
+   closedir(dir);
+
+   if(data->import->table_id > 0){
+      update_import_table(sdata, data);
+   }
+
+   return ret;
+}

src/import_pilerexport.c

@@ -0,0 +1,114 @@
+/*
+ * import_pop3.c
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <getopt.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+void import_the_file(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   close(data->import->fd);
+   data->import->fd = -1;
+
+   if(import_message(sdata, data, counters, cfg) != ERR){
+      unlink(data->import->filename);
+   }
+}
+
+
+void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+
+   if(!strcmp(buf, PILEREXPORT_BEGIN_MARK)){
+      if((*count) > 0){
+         import_the_file(sdata, data, counters, cfg);
+      }
+
+      (*count)++;
+
+      snprintf(data->import->filename, SMALLBUFSIZE-1, "import-%llu", *count);
+
+      data->import->fd = open(data->import->filename, O_CREAT|O_WRONLY|O_TRUNC, S_IRUSR|S_IWUSR);
+      if(data->import->fd == -1){
+         // Do some error handling
+         printf("error: cannot open %s\n", data->import->filename);
+      }
+
+   } else {
+      if(write(data->import->fd, buf, buflen) != buflen){
+         printf("error: didnt write %d bytes\n", buflen);
+      }
+   }
+}
+
+
+void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   int n, rc, nullbyte, savedlen=0, puflen;
+   uint64 count=0;
+   char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE];
+
+   memset(savedbuf, 0, sizeof(savedbuf));
+
+   data->import->fd = -1;
+
+   do {
+      memset(buf, 0, sizeof(buf));
+      n = fread(buf, 1, sizeof(buf)-1, stdin);
+
+      int remaininglen = n;
+
+      if(savedlen > 0){
+         memset(copybuf, 0, sizeof(copybuf));
+
+         memcpy(copybuf, savedbuf, savedlen);
+         memcpy(&copybuf[savedlen], buf, n);
+
+         remaininglen += savedlen;
+
+         savedlen = 0;
+         memset(savedbuf, 0, sizeof(savedbuf));
+
+         p = &copybuf[0];
+      }
+      else {
+         p = &buf[0];
+      }
+
+      do {
+         puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf), &rc, &nullbyte);
+         p += puflen;
+         remaininglen -= puflen;
+
+         if(puflen > 0){
+            if(rc == OK){
+               process_buffer(puf, puflen, &count, sdata, data, counters, cfg);
+            }
+            else {
+               snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
+               savedlen = puflen;
+            }
+         }
+
+      } while(puflen > 0);
+
+   } while(n > 0);
+
+   if(data->import->fd != -1){
+      import_the_file(sdata, data, counters, cfg);
+   }
+}

src/import_pop3.c

@@ -0,0 +1,67 @@
+/*
+ * import_pop3.c
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <getopt.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   int rc;
+   char port_string[8];
+   struct addrinfo hints, *res;
+
+   data->net->use_ssl = 0;
+
+   snprintf(port_string, sizeof(port_string)-1, "%d", data->import->port);
+
+   memset(&hints, 0, sizeof(hints));
+   hints.ai_family = AF_UNSPEC;
+   hints.ai_socktype = SOCK_STREAM;
+
+   if((rc = getaddrinfo(data->import->server, port_string, &hints, &res)) != 0){
+      printf("getaddrinfo for '%s': %s\n", data->import->server, gai_strerror(rc));
+      return;
+   }
+
+   if(data->import->port == 995) data->net->use_ssl = 1;
+
+   if((data->net->socket = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
+      printf("cannot create socket\n");
+      goto ENDE_POP3;
+   }
+
+   if(connect(data->net->socket, res->ai_addr, res->ai_addrlen) == -1){
+      printf("connect()\n");
+      goto ENDE_POP3;
+   }
+
+
+   if(connect_to_pop3_server(data) == ERR){
+      close(data->net->socket);
+      goto ENDE_POP3;
+   }
+
+   process_pop3_emails(sdata, data, counters, cfg);
+
+   close_connection(data->net);
+
+ENDE_POP3:
+   freeaddrinfo(res);
+}

src/list.c

@@ -1,97 +0,0 @@
-/*
- * list.c, SJ
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include "list.h"
-#include "config.h"
-
-
-int append_list(struct list **list, char *p){
-   struct list *q, *t, *u=NULL;
-
-   q = *list;
-
-   while(q){
-      if(strcmp(q->s, p) == 0)
-         return 0;
-
-      u = q;
-      q = q->r;
-   }
-
-   t = create_list_item(p);
-   if(t){
-      if(*list == NULL)
-         *list = t;
-      else if(u)
-         u->r = t;
-
-      return 1;
-   }
-
-   return -1;
-}
-
-
-struct list *create_list_item(char *s){
-   struct list *h=NULL;
-
-   if((h = malloc(sizeof(struct list))) == NULL)
-      return NULL;
-
-   snprintf(h->s, SMALLBUFSIZE-1, "%s", s);
-   h->r = NULL;
-
-   return h;
-}
-
-
-int is_string_on_list(struct list *list, char *s){
-   struct list *p;
-
-   p = list;
-
-   while(p != NULL){
-      if(strcmp(p->s, s) == 0) return 1;
-      p = p->r;
-   }
-
-   return 0;
-}
-
-
-int is_item_on_string(struct list *list, char *s){
-   struct list *p;
-
-   p = list;
-
-   while(p != NULL){
-      if(strstr(s, p->s)) return 1;
-      p = p->r;
-   }
-
-   return 0;
-}
-
-
-void free_list(struct list *list){
-   struct list *p, *q;
-
-   p = list;
-
-   while(p != NULL){
-      q = p->r;
-
-      if(p)
-         free(p);
-
-      p = q;
-   }
-}
-
-
-

src/list.h

@@ -1,17 +0,0 @@
-/*
- * list.h, SJ
- */
-
-#ifndef _LIST_H
- #define _LIST_H
-
-#include "defs.h"
-
-int append_list(struct list **list, char *p);
-struct list *create_list_item(char *s);
-int is_string_on_list(struct list *list, char *s);
-int is_item_on_string(struct list *list, char *s);
-void free_list(struct list *list);
-
-#endif /* _LIST_H */
-

src/memc.c

@@ -59,7 +59,7 @@ void memcached_init(struct memcached_server *ptr, char *server_ip, int server_po
 
 
 int set_socket_options(struct memcached_server *ptr){
-   int error, flag=1, flags, rval;
+   int error, flag=1, flags;
    struct timeval waittime;
    struct linger linger;
 
@@ -128,8 +128,7 @@ int set_socket_options(struct memcached_server *ptr){
 
 
    if((flags & O_NONBLOCK) == 0){
-      rval = fcntl(ptr->fd, F_SETFL, flags | O_NONBLOCK);
-      if(rval == -1) return MEMCACHED_FAILURE;
+      if(fcntl(ptr->fd, F_SETFL, flags | O_NONBLOCK) == -1) return MEMCACHED_FAILURE;
    }
 
    return MEMCACHED_SUCCESS;
@@ -197,7 +196,7 @@ int memcached_connect(struct memcached_server *ptr){
         {
           (void)close(ptr->fd);
           ptr->fd= -1;
-          
+
         }
 
       return MEMCACHED_FAILURE;
@@ -217,16 +216,17 @@ int memcached_shutdown(struct memcached_server *ptr){
 }
 
 
-int memcached_add(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry){
+int memcached_add(struct memcached_server *ptr, char *cmd, char *key, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry){
    int len=0;
 
    if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
 
-   snprintf(ptr->buf, MAXBUFSIZE-1, "add %s %d %ld %d \r\n", key, flags, expiry, valuelen);
+   // cmd could be either 'add' or 'set'
+   snprintf(ptr->buf, MAXBUFSIZE-1, "%s %s %u %lu %u \r\n", cmd, key, flags, expiry, valuelen);
    len = strlen(ptr->buf);
 
-   strncat(ptr->buf, value, MAXBUFSIZE-1);
-   strncat(ptr->buf, "\r\n", MAXBUFSIZE-1);
+   strncat(ptr->buf, value, MAXBUFSIZE-strlen(ptr->buf)-1);
+   strncat(ptr->buf, "\r\n", MAXBUFSIZE-strlen(ptr->buf)-1);
 
    len += valuelen + 2;
 
@@ -240,30 +240,7 @@ int memcached_add(struct memcached_server *ptr, char *key, unsigned int keylen,
 }
 
 
-int memcached_set(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry){
-   int len=0;
-
-   if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
-
-   snprintf(ptr->buf, MAXBUFSIZE-1, "set %s %d %ld %d \r\n", key, flags, expiry, valuelen);
-   len = strlen(ptr->buf);
-
-   strncat(ptr->buf, value, MAXBUFSIZE-1);
-   strncat(ptr->buf, "\r\n", MAXBUFSIZE-1);
-
-   len += valuelen + 2;
-
-   send(ptr->fd, ptr->buf, len, 0);
-
-   ptr->last_read_bytes = __recvtimeout(ptr->fd, ptr->buf, MAXBUFSIZE, ptr->rcv_timeout);
-
-   if(strcmp("STORED\r\n", ptr->buf)) return MEMCACHED_FAILURE;
-
-   return MEMCACHED_SUCCESS;
-}
-
-
-int memcached_increment(struct memcached_server *ptr, char *key, unsigned int keylen, unsigned long long value, unsigned long long *result){
+int memcached_increment(struct memcached_server *ptr, char *key, unsigned long long value, unsigned long long *result){
    char *p;
 
    if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
@@ -285,52 +262,6 @@ int memcached_increment(struct memcached_server *ptr, char *key, unsigned int ke
 }
 
 
-char *memcached_get(struct memcached_server *ptr, char *key, unsigned int *len, unsigned int *flags){
-   char *p;
-
-   if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return NULL;
-
-   snprintf(ptr->buf, MAXBUFSIZE, "get %s \r\n", key);
-   send(ptr->fd, ptr->buf, strlen(ptr->buf), 0);
-
-   ptr->last_read_bytes = __recvtimeout(ptr->fd, ptr->buf, MAXBUFSIZE, ptr->rcv_timeout);
-
-   if(ptr->last_read_bytes <= 0){
-      memcached_shutdown(ptr);
-      return NULL;
-   }
-
-
-   if(ptr->last_read_bytes < 10) return NULL;
-
-   if(strncmp("VALUE ", ptr->buf, 6)) return NULL;
-
-   p = strchr(ptr->buf, '\r');
-   if(!p) return NULL;
-   *p = '\0';
-   ptr->result = p + 2;
-
-   p = strrchr(ptr->buf + 6, ' ');
-   if(!p) return NULL;
-   *len = atoi(p+1);
-   *p = '\0';
-
-
-   p = strrchr(ptr->buf + 6, ' ');
-   if(!p) return NULL;
-   *flags = atoi(p+1);
-   *p = '\0';
-
- 
-   p = strchr(ptr->result, '\r');
-   if(!p) return NULL;
-
-   *p = '\0';
-
-   return ptr->result;
-}
-
-
 int memcached_mget(struct memcached_server *ptr, char *key){
 
    if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
@@ -394,5 +325,3 @@ char *memcached_fetch_result(struct memcached_server *ptr, char *key, char *valu
 
    return p;
 }
-
-

src/memc.h

@@ -9,9 +9,8 @@ void memcached_init(struct memcached_server *ptr, char *server_ip, int server_po
 int set_socket_options(struct memcached_server *ptr);
 int memcached_connect(struct memcached_server *ptr);
 int memcached_shutdown(struct memcached_server *ptr);
-int memcached_add(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry);
-int memcached_set(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry);
-int memcached_increment(struct memcached_server *ptr, char *key, unsigned int keylen, unsigned long long value, unsigned long long *result);
+int memcached_add(struct memcached_server *ptr, char *cmd, char *key, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry);
+int memcached_increment(struct memcached_server *ptr, char *key, unsigned long long value, unsigned long long *result);
 char *memcached_get(struct memcached_server *ptr, char *key, unsigned int *len, unsigned int *flags);
 int memcached_mget(struct memcached_server *ptr, char *key);
 char *memcached_fetch_result(struct memcached_server *ptr, char *key, char *value, unsigned int *flags);

src/message.c

@@ -10,6 +10,7 @@
 #include <sys/mman.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
+#include <errno.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <syslog.h>
@@ -17,172 +18,154 @@
 #include <zlib.h>
 
 
-int store_index_data(struct session_data *sdata, struct _state *state, struct __data *data, uint64 id, struct __config *cfg){
+int store_index_data(struct session_data *sdata, struct parser_state *state, struct data *data, uint64 id, struct config *cfg){
    int rc=ERR;
-   char *subj;
+   char *subj, *sender=state->b_from, *sender_domain=state->b_from_domain;
+   struct sql sql;
 
-   MYSQL_BIND bind[14];
-   unsigned long len[14];
+   if(data->folder == 0){
+      data->folder = get_folder_id_by_rule(data, state, sdata->tot_len, sdata->spam_message, cfg);
+   }
 
    subj = state->b_subject;
    if(*subj == ' ') subj++;
 
-
-   if(prepare_a_mysql_statement(sdata, &(data->stmt_insert_into_sphinx_table), SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc;
-
-
    fix_email_address_for_sphinx(state->b_from);
+   fix_email_address_for_sphinx(state->b_sender);
    fix_email_address_for_sphinx(state->b_to);
    fix_email_address_for_sphinx(state->b_from_domain);
+   fix_email_address_for_sphinx(state->b_sender_domain);
    fix_email_address_for_sphinx(state->b_to_domain);
 
-
-   memset(bind, 0, sizeof(bind));
-
-   bind[0].buffer_type = MYSQL_TYPE_LONGLONG;
-   bind[0].buffer = (char *)&id;
-   bind[0].is_null = 0;
-   bind[0].length = 0;
-
-   bind[1].buffer_type = MYSQL_TYPE_STRING;
-   bind[1].buffer = state->b_from;
-   bind[1].is_null = 0;
-   len[1] = strlen(state->b_from); bind[1].length = &len[1];
-
-   bind[2].buffer_type = MYSQL_TYPE_STRING;
-   bind[2].buffer = state->b_to;
-   bind[2].is_null = 0;
-   len[2] = strlen(state->b_to); bind[2].length = &len[2];
-
-   bind[3].buffer_type = MYSQL_TYPE_STRING;
-   bind[3].buffer = state->b_from_domain;
-   bind[3].is_null = 0;
-   len[3] = strlen(state->b_from_domain); bind[3].length = &len[3];
-
-   bind[4].buffer_type = MYSQL_TYPE_STRING;
-   bind[4].buffer = state->b_to_domain;
-   bind[4].is_null = 0;
-   len[4] = strlen(state->b_to_domain); bind[4].length = &len[4];
-
-   bind[5].buffer_type = MYSQL_TYPE_STRING;
-   bind[5].buffer = subj;
-   bind[5].is_null = 0;
-   len[5] = strlen(subj); bind[5].length = &len[5];
-
-   bind[6].buffer_type = MYSQL_TYPE_STRING;
-   bind[6].buffer = state->b_body;
-   bind[6].is_null = 0;
-   len[6] = strlen(state->b_body); bind[6].length = &len[6];
-
-   bind[7].buffer_type = MYSQL_TYPE_LONG;
-   bind[7].buffer = (char *)&sdata->now;
-   bind[7].is_null = 0;
-   bind[7].length = 0;
-
-   bind[8].buffer_type = MYSQL_TYPE_LONG;
-   bind[8].buffer = (char *)&sdata->sent;
-   bind[8].is_null = 0;
-   bind[8].length = 0;
-
-   bind[9].buffer_type = MYSQL_TYPE_LONG;
-   bind[9].buffer = (char *)&sdata->tot_len;
-   bind[9].is_null = 0;
-   bind[9].length = 0;
-
-   bind[10].buffer_type = MYSQL_TYPE_LONG;
-   bind[10].buffer = (char *)&sdata->direction;
-   bind[10].is_null = 0;
-   bind[10].length = 0;
-
-   bind[11].buffer_type = MYSQL_TYPE_LONG;
-   bind[11].buffer = (char *)&data->folder;
-   bind[11].is_null = 0;
-   bind[11].length = 0;
-
-   bind[12].buffer_type = MYSQL_TYPE_LONG;
-   bind[12].buffer = (char *)&state->n_attachments;
-   bind[12].is_null = 0;
-   bind[12].length = 0;
-
-   bind[13].buffer_type = MYSQL_TYPE_STRING;
-   bind[13].buffer = sdata->attachments;
-   bind[13].is_null = 0;
-   len[13] = strlen(sdata->attachments); bind[13].length = &len[13];
-
-
-   if(mysql_stmt_bind_param(data->stmt_insert_into_sphinx_table, bind)){
-      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_stmt_error(data->stmt_insert_into_sphinx_table));
-      goto CLOSE;
+   if(state->b_sender_domain[0]){
+      sender = state->b_sender;
+      sender_domain = state->b_sender_domain;
    }
 
+   if(cfg->rtindex){
+      // Manticore doesn't support prepared statements using sphinxQL
+      // so we have to go through a painful query assembly escaping
+      // the untrusted input
+      //
+      char a[4*MAXBUFSIZE+4*SMALLBUFSIZE];
+      char *query=NULL;
 
-   if(mysql_stmt_execute(data->stmt_insert_into_sphinx_table)){
-      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute error: *%s*", sdata->ttmpfile, SQL_SPHINX_TABLE, mysql_error(&(sdata->mysql)));
-      goto CLOSE;
+      snprintf(a, sizeof(a)-1, "REPLACE INTO %s (id, arrived, sent, size, direction, folder, attachments, attachment_types, sender, rcpt, senderdomain, rcptdomain, subject, body) VALUES (%llu,%ld,%ld,%d,%d,%d,%d,'%s','", cfg->sphxdb, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, data->folder, state->n_attachments, sdata->attachments);
+
+      int ret = append_string_to_buffer(&query, a);
+
+      unsigned long len = strlen(sender);
+      char *s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, sender, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(state->b_to);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, state->b_to, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(sender_domain);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, sender_domain, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(state->b_to_domain);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, state->b_to_domain, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(subj);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, subj, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "','");
+
+      len = strlen(state->b_body);
+      s = malloc(2*len+1);
+      mysql_real_escape_string(&(sdata->sphx), s, state->b_body, len);
+      ret += append_string_to_buffer(&query, s);
+      free(s);
+      ret += append_string_to_buffer(&query, "')");
+
+      if(ret == 0 && mysql_real_query(&(sdata->sphx), query, strlen(query)) == OK) rc = OK;
+      else syslog(LOG_PRIORITY, "ERROR: %s failed to store index data for id=%llu, errno=%d, append ret=%d", sdata->ttmpfile, id, mysql_errno(&(sdata->sphx)), ret);
+
+      free(query);
    }
+   else {
+      if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc;
 
-   rc = OK;
+      p_bind_init(&sql);
 
-CLOSE:
-   mysql_stmt_close(data->stmt_insert_into_sphinx_table);
+      sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
+      sql.sql[sql.pos] = sender; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = state->b_to; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = state->b_to_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = state->b_body; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+      sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = (char *)&sdata->sent; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = (char *)&sdata->tot_len; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = (char *)&sdata->direction; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = (char *)&data->folder; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = (char *)&state->n_attachments; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+      sql.sql[sql.pos] = sdata->attachments; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
 
+      if(p_exec_stmt(sdata, &sql) == OK) rc = OK;
+      else syslog(LOG_PRIORITY, "ERROR: %s failed to store index data for id=%llu, sql_errno=%d", sdata->ttmpfile, id, sdata->sql_errno);
+
+      close_prepared_statement(&sql);
+   }
 
    return rc;
 }
 
 
-uint64 get_metaid_by_messageid(struct session_data *sdata, struct __data *data, char *message_id, struct __config *cfg){
-   unsigned long len=0;
+uint64 get_metaid_by_messageid(struct session_data *sdata, char *message_id, char *piler_id){
    uint64 id=0;
-   MYSQL_BIND bind[1];
+   struct sql sql;
 
-   memset(bind, 0, sizeof(bind));
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_META_ID_BY_MESSAGE_ID) == ERR) return id;
 
-   bind[0].buffer_type = MYSQL_TYPE_STRING;
-   bind[0].buffer = message_id;
-   bind[0].is_null = 0;
-   len = strlen(message_id); bind[0].length = &len;
+   p_bind_init(&sql);
+   sql.sql[sql.pos] = message_id; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
 
-   if(mysql_stmt_bind_param(data->stmt_get_meta_id_by_message_id, bind)){
-      goto CLOSE;
+   if(p_exec_stmt(sdata, &sql) == OK){
+
+      p_bind_init(&sql);
+
+      sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+      sql.sql[sql.pos] = piler_id; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++;
+
+      p_store_results(&sql);
+
+      p_fetch_results(&sql);
+
+      p_free_results(&sql);
    }
 
-   if(mysql_stmt_execute(data->stmt_get_meta_id_by_message_id)){
-      goto CLOSE;
-   }
-
-   memset(bind, 0, sizeof(bind));
-
-   bind[0].buffer_type = MYSQL_TYPE_LONGLONG;
-   bind[0].buffer = (char *)&id;
-   bind[0].is_null = 0;
-   bind[0].length = 0;
-
-
-   if(mysql_stmt_bind_result(data->stmt_get_meta_id_by_message_id, bind)){
-      goto CLOSE;
-   }
-
-
-   if(mysql_stmt_store_result(data->stmt_get_meta_id_by_message_id)){
-      goto CLOSE;
-   }
-
-   mysql_stmt_fetch(data->stmt_get_meta_id_by_message_id);
-
-CLOSE:
+   close_prepared_statement(&sql);
 
    return id;
 }
 
 
-int store_recipients(struct session_data *sdata, struct __data *data, char *to, uint64 id, int log_errors, struct __config *cfg){
-   int ret=OK, n=0;
+int store_recipients(struct session_data *sdata, char *to, uint64 id, struct config *cfg){
+   int rc=OK, n=0;
    char *p, *q, puf[SMALLBUFSIZE];
+   struct sql sql;
 
-   MYSQL_BIND bind[3];
-   unsigned long len[3];
-
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_RCPT_TABLE) == ERR) return ERR;
 
    p = to;
    do {
@@ -193,306 +176,289 @@ int store_recipients(struct session_data *sdata, struct __data *data, char *to,
       if(q && strlen(q) > 3 && does_it_seem_like_an_email_address(puf) == 1){
          q++;
 
-         memset(bind, 0, sizeof(bind));
+         p_bind_init(&sql);
 
-         bind[0].buffer_type = MYSQL_TYPE_LONGLONG;
-         bind[0].buffer = (char *)&id;
-         bind[0].is_null = 0;
-         bind[0].length = 0;
+         sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
+         sql.sql[sql.pos] = &puf[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+         sql.sql[sql.pos] = q; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
 
-         bind[1].buffer_type = MYSQL_TYPE_STRING;
-         bind[1].buffer = &puf[0];
-         bind[1].is_null = 0;
-         len[1] = strlen(puf); bind[1].length = &len[1];
-
-         bind[2].buffer_type = MYSQL_TYPE_STRING;
-         bind[2].buffer = q;
-         bind[2].is_null = 0;
-         len[2] = strlen(q); bind[2].length = &len[2];
-
-         if(mysql_stmt_bind_param(data->stmt_insert_into_rcpt_table, bind)){
-            syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_stmt_error(data->stmt_insert_into_rcpt_table));
-            ret = ERR;
-            goto CLOSE;
-         }
-
-
-         if(mysql_stmt_execute(data->stmt_insert_into_rcpt_table) && log_errors == 1){
-            syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute error: *%s*", sdata->ttmpfile, SQL_RECIPIENT_TABLE, mysql_error(&(sdata->mysql)));
-            ret = ERR;
+         if(p_exec_stmt(sdata, &sql) == ERR){
+            if(sdata->sql_errno != ER_DUP_ENTRY){
+               syslog(LOG_PRIORITY, "ERROR: %s: failed to add '%s' for id=%llu to rcpt table, sql_errno=%d", sdata->ttmpfile, puf, id, sdata->sql_errno);
+               rc = ERR;
+            }
          }
          else n++;
       }
 
    } while(p);
 
+   close_prepared_statement(&sql);
 
-CLOSE:
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: added %d recipients", sdata->ttmpfile, n);
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored %d recipients, rc=%d", sdata->ttmpfile, n, rc);
+
+   return rc;
+}
+
+
+int store_folder_id(struct session_data *sdata, struct data *data, uint64 id, struct config *cfg){
+   int rc=ERR;
+   struct sql sql;
+
+   if(data->folder == ERR_FOLDER) return rc;
+
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE) == ERR) return rc;
+
+   p_bind_init(&sql);
+
+   sql.sql[sql.pos] = (char *)&data->folder; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
+
+   if(p_exec_stmt(sdata, &sql) == OK) rc = OK;
+   close_prepared_statement(&sql);
+
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored folderdata, rc=%d", sdata->ttmpfile, rc);
+
+   return rc;
+}
+
+
+int update_metadata_reference(struct session_data *sdata, struct parser_state *state, char *ref, struct config *cfg){
+   int ret = ERR;
+   struct sql sql;
+
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE) == ERR) return ret;
+
+   p_bind_init(&sql);
+
+   sql.sql[sql.pos] = ref; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = state->reference; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+
+   if(p_exec_stmt(sdata, &sql) == OK) ret = OK;
+
+   close_prepared_statement(&sql);
+
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: updated meta reference for '%s', rc=%d", sdata->ttmpfile, state->reference, ret);
 
    return ret;
 }
 
 
-int update_metadata_reference(struct session_data *sdata, struct _state *state, struct __data *data, char *ref, struct __config *cfg){
-   int rc, ret = ERR;
-   MYSQL_BIND bind[2];
-   unsigned long len[2];
-
-   memset(bind, 0, sizeof(bind));
-
-   bind[0].buffer_type = MYSQL_TYPE_STRING;
-   bind[0].buffer = ref;
-   bind[0].is_null = 0;
-   len[0] = strlen(ref); bind[0].length = &len[0];
-
-   bind[1].buffer_type = MYSQL_TYPE_STRING;
-   bind[1].buffer = state->reference;
-   bind[1].is_null = 0;
-   len[1] = strlen(state->reference); bind[1].length = &len[1];
-
-   if(mysql_stmt_bind_param(data->stmt_update_metadata_reference, bind)){
-      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(data->stmt_update_metadata_reference));
-      goto CLOSE;
-   }
-
-   rc = mysql_stmt_execute(data->stmt_update_metadata_reference);
-   if(rc == 0) ret = OK;
-
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: updated meta reference for '%s', rc=%d", sdata->ttmpfile, state->reference, rc);
-
-CLOSE:
-   return ret;
-}
-
-
-int store_meta_data(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg){
-   int rc, ret=ERR;
-   char *subj, *p, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1];
-
-   MYSQL_BIND bind[17];
-   unsigned long len[17];
-
-   my_ulonglong id=0;
-
+int store_meta_data(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
+   int rc=ERR;
+   char *subj, *sender, *sender_domain, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1];
+   uint64 id=0;
+   struct sql sql;
 
    subj = state->b_subject;
    if(*subj == ' ') subj++;
 
-   snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, state->b_from, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
+   if(state->b_sender_domain[0]){
+      sender = state->b_sender;
+      sender_domain = state->b_sender_domain;
+      get_first_email_address_from_string(state->b_sender, s2, sizeof(s2));
+   } else {
+      sender = state->b_from;
+      sender_domain = state->b_from_domain;
+      get_first_email_address_from_string(state->b_from, s2, sizeof(s2));
+   }
 
-   digest_string(s, &vcode[0]);
+   snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, sender, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
+
+   digest_string("sha256", s, &vcode[0]);
 
    memset(ref, 0, sizeof(ref));
    if(strlen(state->reference) > 10){
-      digest_string(state->reference, &ref[0]);
-      update_metadata_reference(sdata, state, data, &ref[0], cfg);
+      digest_string("sha256", state->reference, &ref[0]);
+      update_metadata_reference(sdata, state, &ref[0], cfg);
+   }
+   else if(state->reference[0] == 0){
+      // during import, the order of messages is often random
+      // check if this is a message which is already referenced
+      uint64 count=0;
+
+      digest_string("sha256", state->message_id, &ref[0]);
+      if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_METADATA_REFERENCE) != ERR){
+         p_bind_init(&sql);
+
+         sql.sql[sql.pos] = &ref[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+
+         if(p_exec_stmt(sdata, &sql) == OK){
+            p_bind_init(&sql);
+
+            sql.sql[sql.pos] = (char *)&count; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+            p_store_results(&sql);
+            p_fetch_results(&sql);
+            p_free_results(&sql);
+         }
+      }
+
+      close_prepared_statement(&sql);
+
+      // no reference yet
+      if(count <= 0)
+         ref[0] = 0;
    }
 
 
-   if(prepare_a_mysql_statement(sdata, &(data->stmt_insert_into_meta_table), SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR;
-
-   memset(s2, 0, sizeof(s2));
-
-   p = state->b_from;
-   do {
-      memset(s2, 0, sizeof(s2));
-      p = split(p, ' ', s2, sizeof(s2)-1);
-
-      if(s2[0] == '\0') continue;
-
-      if(does_it_seem_like_an_email_address(s2) == 1){ break; }
-   } while(p);
-
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR;
 
    if(strlen(state->b_to) < 5){
       snprintf(state->b_to, SMALLBUFSIZE-1, "undisclosed-recipients@no.domain");
    }
 
-   memset(bind, 0, sizeof(bind));
 
-   bind[0].buffer_type = MYSQL_TYPE_STRING;
-   bind[0].buffer = &s2[0];
-   bind[0].is_null = 0;
-   len[0] = strlen(s2); bind[0].length = &len[0];
+   p_bind_init(&sql);
 
-   bind[1].buffer_type = MYSQL_TYPE_STRING;
-   bind[1].buffer = state->b_from_domain;
-   bind[1].is_null = 0;
-   len[1] = strlen(state->b_from_domain); bind[1].length = &len[1];
+   sql.sql[sql.pos] = &s2[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = (char *)&sdata->spam_message; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&sdata->sent; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&sdata->retained; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&sdata->tot_len; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&sdata->hdr_len; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&sdata->direction; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = (char *)&state->n_attachments; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = state->message_id; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = &ref[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = sdata->digest; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = sdata->bodydigest; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = &vcode[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
 
-   bind[2].buffer_type = MYSQL_TYPE_STRING;
-   bind[2].buffer = subj;
-   bind[2].is_null = 0;
-   len[2] = strlen(subj); bind[2].length = &len[2];
+   if(p_exec_stmt(sdata, &sql) == OK){
+      id = p_get_insert_id(&sql);
 
-   bind[3].buffer_type = MYSQL_TYPE_LONG;
-   bind[3].buffer = (char *)&sdata->spam_message;
-   bind[3].is_null = 0;
-   bind[3].length = 0;
+      if(store_recipients(sdata, state->b_to, id, cfg) == OK){
 
-   bind[4].buffer_type = MYSQL_TYPE_LONG;
-   bind[4].buffer = (char *)&sdata->now;
-   bind[4].is_null = 0;
-   bind[4].length = 0;
+         if(store_index_data(sdata, state, data, id, cfg) == OK) rc = OK;
 
-   bind[5].buffer_type = MYSQL_TYPE_LONG;
-   bind[5].buffer = (char *)&sdata->sent;
-   bind[5].is_null = 0;
-   bind[5].length = 0;
+         if(cfg->enable_folders == 1){
+            rc = store_folder_id(sdata, data, id, cfg);
+         }
+      }
 
-   bind[6].buffer_type = MYSQL_TYPE_LONG;
-   bind[6].buffer = (char *)&sdata->retained;
-   bind[6].is_null = 0;
-   bind[6].length = 0;
+      // There were some sql errors, so we should rollback everything
+      if(rc == ERR){
+         rollback(sdata, state, id, cfg);
+      }
 
-   bind[7].buffer_type = MYSQL_TYPE_LONG;
-   bind[7].buffer = (char *)&sdata->tot_len;
-   bind[7].is_null = 0;
-   bind[7].length = 0;
-
-   bind[8].buffer_type = MYSQL_TYPE_LONG;
-   bind[8].buffer = (char *)&sdata->hdr_len;
-   bind[8].is_null = 0;
-   bind[8].length = 0;
-
-   bind[9].buffer_type = MYSQL_TYPE_LONG;
-   bind[9].buffer = (char *)&sdata->direction;
-   bind[9].is_null = 0;
-   bind[9].length = 0;
-
-   bind[10].buffer_type = MYSQL_TYPE_LONG;
-   bind[10].buffer = (char *)&state->n_attachments;
-   bind[10].is_null = 0;
-   bind[10].length = 0;
-
-   bind[11].buffer_type = MYSQL_TYPE_STRING;
-   bind[11].buffer = sdata->ttmpfile;
-   bind[11].is_null = 0;
-   len[11] = strlen(sdata->ttmpfile); bind[11].length = &len[11];
-
-   bind[12].buffer_type = MYSQL_TYPE_STRING;
-   bind[12].buffer = state->message_id;
-   bind[12].is_null = 0;
-   len[12] = strlen(state->message_id); bind[12].length = &len[12];
-
-   bind[13].buffer_type = MYSQL_TYPE_STRING;
-   bind[13].buffer = &ref[0];
-   bind[13].is_null = 0;
-   len[13] = strlen(ref); bind[13].length = &len[13];
-
-   bind[14].buffer_type = MYSQL_TYPE_STRING;
-   bind[14].buffer = sdata->digest;
-   bind[14].is_null = 0;
-   len[14] = strlen(sdata->digest); bind[14].length = &len[14];
-
-   bind[15].buffer_type = MYSQL_TYPE_STRING;
-   bind[15].buffer = sdata->bodydigest;
-   bind[15].is_null = 0;
-   len[15] = strlen(sdata->bodydigest); bind[15].length = &len[15];
-
-   bind[16].buffer_type = MYSQL_TYPE_STRING;
-   bind[16].buffer = &vcode[0];
-   bind[16].is_null = 0;
-   len[16] = strlen(vcode); bind[16].length = &len[16];
-
-
-
-   if(mysql_stmt_bind_param(data->stmt_insert_into_meta_table, bind)){
-      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_bind_param() error: %s", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_stmt_error(data->stmt_insert_into_meta_table));
-      goto CLOSE;
-   }
-
-
-   rc = mysql_stmt_execute(data->stmt_insert_into_meta_table);
-
-   if(rc){
-      syslog(LOG_PRIORITY, "%s: %s.mysql_stmt_execute() error: *%s*", sdata->ttmpfile, SQL_METADATA_TABLE, mysql_error(&(sdata->mysql)));
-      ret = ERR_EXISTS;
    }
    else {
-      id = mysql_stmt_insert_id(data->stmt_insert_into_meta_table);
-
-      rc = store_recipients(sdata, data, state->b_to, id, 1, cfg);
-
-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored recipients, rc=%d", sdata->ttmpfile, rc);
-
-      if(rc == OK){
-
-         rc = store_index_data(sdata, state, data, id, cfg);
-
-         if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored indexdata, rc=%d", sdata->ttmpfile, rc);
-
-         if(rc == OK)
-            ret = OK;
-      }
+      syslog(LOG_PRIORITY, "ERROR: %s storing metadata, sql_errno=%d", sdata->ttmpfile, sdata->sql_errno);
    }
 
-CLOSE:
-   mysql_stmt_close(data->stmt_insert_into_meta_table);
+   close_prepared_statement(&sql);
 
-   return ret;
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored metadata, rc=%d",  sdata->ttmpfile, rc);
+
+   return rc;
 }
 
 
-void remove_stripped_attachments(struct _state *state){
+void rollback(struct session_data *sdata, struct parser_state *state, uint64 id, struct config *cfg){
+   char buf[SMALLBUFSIZE];
+
+   snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE id=%llu", SQL_SPHINX_TABLE, id);
+   p_query(sdata, buf);
+   syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
+
+   snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE id=%llu", SQL_RECIPIENT_TABLE, id);
+   p_query(sdata, buf);
+   syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
+
+   snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE id=%llu", SQL_METADATA_TABLE, id);
+   p_query(sdata, buf);
+   syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
+
+   snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE piler_id='%s'", SQL_ATTACHMENT_TABLE, sdata->ttmpfile);
+   p_query(sdata, buf);
+   syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
+
+   if(cfg->enable_folders == 1){
+      snprintf(buf, sizeof(buf)-1, "DELETE FROM " SQL_FOLDER_MESSAGE_TABLE " WHERE id=%llu", id);
+      p_query(sdata, buf);
+      syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
+   }
+
+   remove_stored_message_files(sdata, state, cfg);
+}
+
+
+void remove_stripped_attachments(struct parser_state *state){
    int i;
 
    for(i=1; i<=state->n_attachments; i++) unlink(state->attachments[i].internalname);
 }
 
 
-int process_message(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg){
-   int i, rc;
-   uint64 id=0;
+int is_duplicated_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
+   int fd;
+   char piler_id[SMALLBUFSIZE];
 
    /* discard if existing message_id */
 
-   id = get_metaid_by_messageid(sdata, data, state->message_id, cfg);
+   sdata->duplicate_id = get_metaid_by_messageid(sdata, state->message_id, piler_id);
 
-   if(id > 0){
+   if(sdata->duplicate_id > 0){
       remove_stripped_attachments(state);
 
-      if(strlen(state->b_journal_to) > 0){
-         if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to add journal rcpt (%s) to id=%llu for message-id: '%s'", sdata->ttmpfile, state->b_journal_to, id, state->message_id);
-         store_recipients(sdata, data, state->b_journal_to, id, 0, cfg);
+      if(sdata->restored_copy == 0 && strlen(state->b_journal_to) > 0){
+         if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to add journal rcpt (%s) to id=%llu for message-id: '%s'", sdata->ttmpfile, state->b_journal_to, sdata->duplicate_id, state->message_id);
+         store_recipients(sdata, state->b_journal_to, sdata->duplicate_id, cfg);
       }
 
       return ERR_EXISTS;
    }
 
+   fd = open(state->message_id_hash, O_CREAT|O_EXCL, S_IRUSR|S_IWUSR);
+   if(fd == -1){
+      remove_stripped_attachments(state);
+      syslog(LOG_PRIORITY, "%s: touch %s FAILED (%s), error: %s", sdata->ttmpfile, state->message_id_hash, state->message_id, strerror(errno));
+      return ERR_EXISTS;
+   }
+   close(fd);
 
-   /* store base64 encoded file attachments */
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: touch %s OK (%s)", sdata->ttmpfile, state->message_id_hash, state->message_id);
 
-   if(state->n_attachments > 0){
-      rc = store_attachments(sdata, state, data, cfg);
 
-      for(i=1; i<=state->n_attachments; i++){
-         unlink(state->attachments[i].internalname);
+   if(cfg->mmap_dedup_test == 1 && data->dedup != MAP_FAILED && data->child_serial >= 0 && data->child_serial < MAXCHILDREN){
+
+      if(strstr(data->dedup, state->message_id_hash)){
+         if(cfg->verbosity >= _LOG_DEBUG){
+            syslog(LOG_INFO, "%s: dedup string: %s", sdata->ttmpfile, data->dedup);
+            syslog(LOG_INFO, "%s: message-id-hash=%s, serial=%d", sdata->ttmpfile, state->message_id_hash, data->child_serial);
+         }
+
+         remove_stripped_attachments(state);
+         return ERR_EXISTS;
       }
 
-      if(rc) return ERR;
-   }
-
-
-   rc = store_file(sdata, sdata->tmpframe, 0, 0, cfg);
-   if(rc == 0){
-      syslog(LOG_PRIORITY, "%s: error storing message: %s", sdata->ttmpfile, sdata->tmpframe);
-      return ERR;
-   }
-
-
-   sdata->retained += query_retain_period(data->retention_rules, state, sdata->tot_len, sdata->spam_message, cfg);
-
-   rc = store_meta_data(sdata, state, data, cfg);
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored metadata, rc=%d",  sdata->ttmpfile, rc);
-   if(rc == ERR_EXISTS){
-
-      remove_stored_message_files(sdata, state, cfg);
-
-      return ERR_EXISTS;
+      memcpy(data->dedup + data->child_serial*DIGEST_LENGTH*2, state->message_id_hash, DIGEST_LENGTH*2);
    }
 
    return OK;
 }
 
+
+int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
+
+   if(is_duplicated_message(sdata, state, data, cfg) == ERR_EXISTS)
+      return ERR_EXISTS;
+
+   sdata->retained += query_retain_period(data, state, sdata->tot_len, sdata->spam_message, cfg);
+
+
+   if(state->n_attachments > 0 && store_attachments(sdata, state, cfg) == ERR) return ERR;
+
+
+   if(store_file(sdata, sdata->tmpframe, 0, cfg) == 0){
+      syslog(LOG_PRIORITY, "ERROR: %s: failed to store message: %s", sdata->ttmpfile, sdata->tmpframe);
+      return ERR;
+   }
+
+
+   return store_meta_data(sdata, state, data, cfg);
+}

src/misc.c

@@ -19,6 +19,7 @@
 #include <unistd.h>
 #include <ctype.h>
 #include <openssl/ssl.h>
+#include <openssl/err.h>
 #include "misc.h"
 #include "smtpcodes.h"
 #include "errmsg.h"
@@ -31,13 +32,52 @@ int get_build(){
 }
 
 
+void get_extractor_list(){
+   printf("Extractors: ");
+
+   #ifdef HAVE_PDFTOTEXT
+      printf("%s ", HAVE_PDFTOTEXT);
+   #endif
+
+   #ifdef HAVE_CATDOC
+      printf("%s ", HAVE_CATDOC);
+   #endif
+
+   #ifdef HAVE_CATPPT
+      printf("%s ", HAVE_CATPPT);
+   #endif
+
+   #ifdef HAVE_XLS2CSV
+      printf("%s ", HAVE_XLS2CSV);
+   #endif
+
+   #ifdef HAVE_PPTHTML
+      printf("%s ", HAVE_PPTHTML);
+   #endif
+
+   #ifdef HAVE_UNRTF
+      printf("%s ", HAVE_UNRTF);
+   #endif
+
+   #ifdef HAVE_TNEF
+      printf("%s ", HAVE_TNEF);
+   #endif
+
+   #ifdef HAVE_ZIP
+      printf("libzip ");
+   #endif
+
+   printf("\n\n");
+}
+
+
 void __fatal(char *s){
    fprintf(stderr, "%s\n", s);
    exit(1);
 }
 
 /*
- * calculate the difference betwwen two timevals in [usec]
+ * calculate the difference between two timevals in [usec]
  */
 
 long tvdiff(struct timeval a, struct timeval b){
@@ -48,87 +88,31 @@ long tvdiff(struct timeval a, struct timeval b){
 }
 
 
-/*
- * search something in a buffer
- */
-
-int searchStringInBuffer(char *s, int len1, char *what, int len2){
-   int i, k, r;
-
-   for(i=0; i<len1; i++){
-      r = 0;
-
-      for(k=0; k<len2; k++){
-         if(*(s+i+k) == *(what+k))
-            r++;
-      }
-
-      if(r == len2)
-         return i;
-   }
-
-   return 0;
-}
-
-
-/*
- * count a character in buffer
- */
-
-int countCharacterInBuffer(char *p, char c){
-   int i=0;
-
-   for(; *p; p++){
-      if(*p == c)
-         i++;
-   }
-
-   return i;
-}
-
-
-void replaceCharacterInBuffer(char *p, char from, char to){
-   int i, k=0;
-
-   for(i=0; i<strlen(p); i++){
-      if(p[i] == from){
-         if(to > 0){
-            p[k] = to;
-            k++;
-         }
-      }
-      else {
-         p[k] = p[i];
-         k++;
-      }
-
-   }
-
-   p[k] = '\0';
-}
-
-
 /*
  * split a string by a character as delimiter
  */
 
-char *split(char *row, int ch, char *s, int size){
-   char *r;
+char *split(char *str, int ch, char *buf, int buflen, int *result){
+   char *p;
 
-   if(row == NULL || s == NULL)
-      return NULL;
+   *result = 0;
 
-   r = strchr(row, ch);
-   if(r) *r = '\0';
+   if(str == NULL || buf == NULL || buflen < 2) return NULL;
 
-   snprintf(s, size, "%s", row);
-
-   if(r){
-      *r = ch;
-      r++;
+   p = strchr(str, ch);
+   if(p){
+      *p = '\0';
    }
 
-   return r;
+   snprintf(buf, buflen, "%s", str);
+
+   if(p){
+      *p = ch;
+      *result = 1;
+      p++;
+   }
+
+   return p;
 }
 
 
@@ -159,10 +143,8 @@ char *split_str(char *row, char *what, char *s, int size){
       r += strlen(what);
    }
 
-   if(s != NULL){
-      strncpy(s, row, len);
-      s[len] = '\0';
-   }
+   strncpy(s, row, len);
+   s[len] = '\0';
 
    return r;
 }
@@ -176,13 +158,13 @@ int trimBuffer(char *s){
    int n=0;
    char *p;
 
-   p = strrchr(s, '\n');
+   p = strchr(s, '\n');
    if(p){
       *p = '\0';
       n++;
    }
 
-   p = strrchr(s, '\r');
+   p = strchr(s, '\r');
    if(p){
       *p = '\0';
       n++;
@@ -192,6 +174,36 @@ int trimBuffer(char *s){
 }
 
 
+int extract_verp_address(char *email){
+   char *p1;
+
+   // a VERP address is like archive+user=domain.com@myarchive.local
+
+   if(!email) return 0;
+
+   if(strlen(email) < 5) return 0;
+
+   p1 = strchr(email, '+');
+   if(p1){
+      char *p2 = strchr(p1, '@');
+      if(p2 && p2 > p1 + 2){
+         if(strchr(p1+1, '=')){
+            char puf[SMALLBUFSIZE];
+            memset(puf, 0, sizeof(puf));
+
+            memcpy(&puf[0], p1+1, p2-p1-1);
+            char *p = strchr(puf, '=');
+            if(p) *p = '@';
+            strcpy(email, puf);
+         }
+      }
+
+   }
+
+   return 0;
+}
+
+
 int extractEmail(char *rawmail, char *email){
    char *p;
 
@@ -203,6 +215,7 @@ int extractEmail(char *rawmail, char *email){
       p = strchr(email, '>');
       if(p){
          *p = '\0';
+         extract_verp_address(email);
          return 1;
       }
    }
@@ -211,6 +224,36 @@ int extractEmail(char *rawmail, char *email){
 }
 
 
+/*
+ * Generate a random string from /dev/urandom or
+ * using the rand() function if not possible
+ */
+
+void make_random_string(unsigned char *buf, int buflen){
+   const char alphanum[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+   int i, fd, urandom=0, len = sizeof(alphanum)-1;
+
+   fd = open(RANDOM_POOL, O_RDONLY);
+   if(fd != -1){
+      if(readFromEntropyPool(fd, buf, buflen) == buflen){
+         for(i=0; i<buflen; i++){
+            *(buf+i) = alphanum[*(buf+i) % len];
+         }
+
+         urandom = 1;
+      }
+      close(fd);
+   }
+
+   if(urandom == 1) return;
+
+   for(i=0; i<buflen; i++){
+      *(buf+i) = alphanum[rand() % len];
+   }
+
+}
+
+
 void create_id(char *id, unsigned char server_id){
    int i;
    unsigned char buf[RND_STR_LEN/2];
@@ -219,11 +262,13 @@ void create_id(char *id, unsigned char server_id){
 
    get_random_bytes(buf, RND_STR_LEN/2, server_id);
 
+   // New encryption scheme using AES-256
+   buf[0] = 0x50;
+
    for(i=0; i < RND_STR_LEN/2; i++){
       sprintf(id, "%02x", buf[i]);
       id += 2;
    }
-
 }
 
 
@@ -241,7 +286,7 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
    taia_now(&now);
    taia_pack(nowpack, &now);
 
-   memcpy(buf, nowpack, 12);
+   memcpy(buf, nowpack, 12); //-V512
 
    fd = open(RANDOM_POOL, O_RDONLY);
    if(fd == -1) return ret;
@@ -251,7 +296,7 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
    if(readFromEntropyPool(fd, buf+12+1, len-12-1) != len-12-1){
       syslog(LOG_PRIORITY, "%s: %s", ERR_CANNOT_READ_FROM_POOL, RANDOM_POOL);
    }
-   
+
    close(fd);
    return ret;
 }
@@ -261,12 +306,12 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
  * read random data from entropy pool
  */
 
-int readFromEntropyPool(int fd, void *_s, size_t n){
+int readFromEntropyPool(int fd, void *_s, ssize_t n){
    char *s = _s;
-   ssize_t res, pos = 0;
+   ssize_t pos = 0;
 
    while(n > pos){
-      res = read(fd, s + pos, n - pos);
+      ssize_t res = read(fd, s + pos, n - pos);
       switch(res){
          case  -1: continue;
          case   0: return res;
@@ -302,13 +347,13 @@ int recvtimeout(int s, char *buf, int len, int timeout){
 }
 
 
-int write1(int sd, char *buf, int use_ssl, SSL *ssl){
+int write1(struct net *net, void *buf, int buflen){
    int n;
 
-   if(use_ssl == 1)
-      n = SSL_write(ssl, buf, strlen(buf));
+   if(net->use_ssl == 1)
+      n = SSL_write(net->ssl, buf, buflen);
    else
-      n = send(sd, buf, strlen(buf), 0);
+      n = send(net->socket, buf, buflen, 0);
 
    return n;
 }
@@ -324,28 +369,28 @@ int ssl_want_retry(SSL *ssl, int ret, int timeout){
    i = SSL_get_error(ssl, ret);
    if(i == SSL_ERROR_NONE)
       return 1;
- 
+
    tv.tv_sec = timeout/1000;
    tv.tv_usec = 0;
    FD_ZERO(&fds);
- 
+
    switch(i){
       case SSL_ERROR_WANT_READ: // pause until the socket is readable
           sock = SSL_get_rfd(ssl);
           FD_SET(sock, &fds);
           i = select(sock+1, &fds, 0, 0, &tv);
           break;
- 
+
       case SSL_ERROR_WANT_WRITE: // pause until the socket is writeable
          sock = SSL_get_wfd(ssl);
          FD_SET(sock, &fds);
          i = select(sock+1, 0, &fds, 0, &tv);
          break;
- 
+
       case SSL_ERROR_ZERO_RETURN: // the sock closed, just return quietly
          i = 0;
          break;
- 
+
       default: // ERROR - unexpected error code
          i = -1;
          break;
@@ -369,19 +414,31 @@ int ssl_read_timeout(SSL *ssl, void *buf, int len, int timeout){
 }
 
 
-int recvtimeoutssl(int s, char *buf, int len, int timeout, int use_ssl, SSL *ssl){
+int recvtimeoutssl(struct net *net, char *buf, int len){
 
     memset(buf, 0, len);
 
-    if(use_ssl == 1){
-       return ssl_read_timeout(ssl, buf, len-1, timeout);
+    if(net->use_ssl == 1){
+       return ssl_read_timeout(net->ssl, buf, len-1, net->timeout);
     }
     else {
-       return recvtimeout(s, buf, len-1, timeout);
+       return recvtimeout(net->socket, buf, len-1, net->timeout);
     }
 }
 
 
+void close_connection(struct net *net){
+   close(net->socket);
+
+   if(net->use_ssl == 1){
+      SSL_shutdown(net->ssl);
+      SSL_free(net->ssl);
+      SSL_CTX_free(net->ctx);
+      ERR_free_strings();
+   }
+}
+
+
 void write_pid_file(char *pidfile){
    FILE *f;
 
@@ -412,31 +469,7 @@ int drop_privileges(struct passwd *pwd){
 }
 
 
-int is_email_address_on_my_domains(char *email, struct __data *data){
-   int rc=0;
-   char *p, *q=NULL;
-
-   if(email == NULL || data->mydomains[0] == '\0') return rc;
-
-   p = strchr(email, '@');
-
-   if(!p) return rc;
-
-   if(strlen(p) < 3) return rc;
-
-   q = strrchr(p+1, ' ');
-
-   if(q) *q = '\0';
-
-   if(strcasestr(data->mydomains, p+1)) rc = 1;
-
-   if(q) *q = ' ';
-
-   return rc;
-}
-
-
-void init_session_data(struct session_data *sdata, struct __config *cfg){
+void init_session_data(struct session_data *sdata, struct config *cfg){
    int i;
 
 
@@ -457,6 +490,8 @@ void init_session_data(struct session_data *sdata, struct __config *cfg){
 
    memset(sdata->fromemail, 0, SMALLBUFSIZE);
 
+   sdata->duplicate_id = 0;
+
    sdata->restored_copy = 0;
 
    sdata->internal_sender = sdata->internal_recipient = sdata->external_recipient = 0;
@@ -464,6 +499,7 @@ void init_session_data(struct session_data *sdata, struct __config *cfg){
 
    sdata->hdr_len = 0;
    sdata->tot_len = 0;
+   sdata->stored_len = 0;
    sdata->num_of_rcpt_to = 0;
 
    sdata->ms_journal = 0;
@@ -476,13 +512,19 @@ void init_session_data(struct session_data *sdata, struct __config *cfg){
 
    sdata->spam_message = 0;
 
+   sdata->customer_id = 0;
+
    sdata->__acquire = sdata->__parsed = sdata->__av = sdata->__store = sdata->__compress = sdata->__encrypt = 0;
 
+   sdata->import = 0;
 
    for(i=0; i<MAX_RCPT_TO; i++) memset(sdata->rcptto[i], 0, SMALLBUFSIZE);
 
    time(&(sdata->now));
-   sdata->sent = sdata->delivered = sdata->retained = sdata->now;
+   sdata->delivered = sdata->retained = sdata->now;
+   sdata->sent = 0;
+
+   sdata->sql_errno = 0;
 
 #ifdef HAVE_TWEAK_SENT_TIME
    sdata->sent += cfg->tweak_sent_time_offset;
@@ -516,7 +558,180 @@ int read_from_stdin(struct session_data *sdata){
 
 
 void strtolower(char *s){
-   for(; *s; s++) *s = tolower(*s);
+   for(; *s; s++){
+      if(*s >= 65 && *s <= 90) *s = tolower(*s);
+   }
+}
+
+
+int make_socket_non_blocking(int fd){
+   int flags, s;
+
+   flags = fcntl(fd, F_GETFL, 0);
+   if(flags == -1){
+      return -1;
+   }
+
+   flags |= O_NONBLOCK;
+
+   s = fcntl(fd, F_SETFL, flags);
+   if(s == -1){
+      return -1;
+   }
+
+   return 0;
+}
+
+
+int create_and_bind(char *listen_addr, int listen_port){
+   struct addrinfo hints;
+   struct addrinfo *result, *rp;
+   char port_string[8];
+   int rc, fd;
+
+   memset(&hints, 0, sizeof (struct addrinfo));
+   hints.ai_family = AF_UNSPEC;
+   hints.ai_socktype = SOCK_STREAM;
+   hints.ai_flags = AI_PASSIVE;
+
+   snprintf(port_string, sizeof(port_string)-1, "%d", listen_port);
+
+   rc = getaddrinfo(listen_addr, port_string, &hints, &result);
+   if(rc != 0){
+      syslog(LOG_PRIORITY, "getaddrinfo for '%s': %s", listen_addr, gai_strerror(rc));
+      return -1;
+   }
+
+   for(rp = result; rp != NULL; rp = rp->ai_next){
+      fd = socket(rp->ai_family, rp->ai_socktype, rp->ai_protocol);
+      if(fd == -1)
+         continue;
+
+      rc = bind(fd, rp->ai_addr, rp->ai_addrlen);
+      if(rc == 0){
+          break;
+      }
+
+      close(fd);
+   }
+
+   if(rp == NULL){
+      syslog(LOG_PRIORITY, "cannot bind to port: %s:%d", listen_addr, listen_port);
+      return -1;
+   }
+
+   freeaddrinfo(result);
+
+   return fd;
+}
+
+
+int can_i_write_directory(char *dir){
+   int fd;
+   char filename[SMALLBUFSIZE];
+
+   if(dir)
+      snprintf(filename, sizeof(filename)-1, "%s/__piler_%d", dir, getpid());
+   else
+      snprintf(filename, sizeof(filename)-1, "__piler_%d", getpid());
+
+   fd = open(filename, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR|S_IRGRP);
+   if(fd == -1){
+      return 0;
+   }
+
+   close(fd);
+   unlink(filename);
+
+   return 1;
+}
+
+
+void move_email(struct smtp_session *session){
+   char buf[SMALLBUFSIZE];
+
+   snprintf(buf, sizeof(buf)-1, "%d/%s", session->ttmpfile[0] % session->cfg->number_of_worker_processes, session->ttmpfile);
+
+   if(session->nullbyte){
+      snprintf(buf, sizeof(buf)-1, "%s/%s", ERROR_DIR, session->ttmpfile);
+      syslog(LOG_PRIORITY, "ERROR: %s contains an invalid NUL-byte, moving it to %s", session->ttmpfile, ERROR_DIR);
+   }
+
+   if(rename(session->ttmpfile, buf)){
+      syslog(LOG_PRIORITY, "ERROR: couldn't rename %s to %s (reason: %s)", session->ttmpfile, buf, strerror(errno));
+   }
+}
+
+
+int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte){
+   int i=0;
+
+   *rc = ERR;
+   *nullbyte = 0;
+
+   memset(buf, 0, buflen);
+
+   for(int j=0; j<slen; j++){
+      if(i<buflen-2){
+         if(*(s+j) == 0){
+            *nullbyte = 1;
+         }
+
+         buf[i] = *(s+j);
+         i++;
+
+         if(*(s+j) == c){
+            *rc = OK;
+            break;
+         }
+      }
+      else {
+         break;
+      }
+   }
+
+   return i;
+}
+
+
+int init_ssl_to_server(struct data *data){
+   int n;
+   X509* server_cert;
+   char *str;
+
+   SSL_library_init();
+   SSL_load_error_strings();
+
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+   data->net->ctx = SSL_CTX_new(TLSv1_2_client_method());
+#else
+   data->net->ctx = SSL_CTX_new(TLS_client_method());
+#endif
+   CHK_NULL(data->net->ctx, "internal SSL error");
+
+   data->net->ssl = SSL_new(data->net->ctx);
+   CHK_NULL(data->net->ssl, "internal ssl error");
+
+   SSL_set_fd(data->net->ssl, data->net->socket);
+   n = SSL_connect(data->net->ssl);
+   CHK_SSL(n, "internal ssl error");
+
+   //printf("Cipher: %s\n", SSL_get_cipher(data->net->ssl));
+
+   server_cert = SSL_get_peer_certificate(data->net->ssl);
+   CHK_NULL(server_cert, "server cert error");
+
+   str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0);
+   CHK_NULL(str, "error in server cert");
+   OPENSSL_free(str);
+
+   str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0);
+   CHK_NULL(str, "error in server cert");
+   OPENSSL_free(str);
+
+   X509_free(server_cert);
+
+   return OK;
 }
 
 
@@ -541,3 +756,54 @@ char *strcasestr(const char *s, const char *find){
 }
 #endif
 
+
+int append_string_to_buffer(char **buffer, char *str){
+   int arglen;
+   char *s=NULL;
+
+   arglen = strlen(str);
+
+   if(!*buffer){
+      *buffer = malloc(arglen+1);
+      memset(*buffer, 0, arglen+1);
+      memcpy(*buffer, str, arglen);
+   }
+   else {
+      int len = strlen(*buffer);
+      s = realloc(*buffer, len + arglen+1);
+      if(!s) return 1;
+
+      *buffer = s;
+
+      memset(*buffer+len, 0, arglen+1);
+      memcpy(*buffer+len, str, arglen);
+   }
+
+   return 0;
+}
+
+
+int get_size_from_smtp_mail_from(char *s){
+   int size=0;
+   char *p;
+
+   p = strcasestr(s, "SIZE=");
+   if(p){
+      p += strlen("SIZE=");
+      char *q = p;
+      for(; *q; q++){
+         if(isspace(*q)) break;
+      }
+
+      // We extract max. 9 characters, which is just under 1GB
+      // and not overflowing an int variable
+      if(q - p <= 9){
+         char c = *q;
+         *q = '\0';
+         size = atoi(p);
+         *q = c;
+      }
+   }
+
+   return size;
+}

src/misc.h

@@ -5,6 +5,7 @@
 #ifndef _MISC_H
  #define _MISC_H
 
+#include <sys/socket.h>
 #include <openssl/ssl.h>
 #include <sys/time.h>
 #include <pwd.h>
@@ -15,32 +16,45 @@
 #define CHK_SSL(err, msg) if ((err)==-1) { printf("ssl error: %s\n", msg); return ERR; }
 
 int get_build();
+void get_extractor_list();
 void __fatal(char *s);
 long tvdiff(struct timeval a, struct timeval b);
-int searchStringInBuffer(char *s, int len1, char *what, int len2);
-int countCharacterInBuffer(char *p, char c);
-void replaceCharacterInBuffer(char *p, char from, char to);
-char *split(char *row, int ch, char *s, int size);
+char *split(char *str, int ch, char *buf, int buflen, int *result);
 char *split_str(char *row, char *what, char *s, int size);
 int trimBuffer(char *s);
 int extractEmail(char *rawmail, char *email);
+int extract_verp_address(char *email);
+void make_random_string(unsigned char *buf, int buflen);
 void create_id(char *id, unsigned char server_id);
 int get_random_bytes(unsigned char *buf, int len, unsigned char server_id);
-int readFromEntropyPool(int fd, void *_s, size_t n);
+int readFromEntropyPool(int fd, void *_s, ssize_t n);
 int recvtimeout(int s, char *buf, int len, int timeout);
-int write1(int sd, char *buf, int use_ssl, SSL *ssl);
-int recvtimeoutssl(int s, char *buf, int len, int timeout, int use_ssl, SSL *ssl);
+int write1(struct net *net, void *buf, int buflen);
+int recvtimeoutssl(struct net *net, char *buf, int len);
+void close_connection(struct net *net);
 
 void write_pid_file(char *pidfile);
 int drop_privileges(struct passwd *pwd);
 
-int is_email_address_on_my_domains(char *email, struct __data *data);
-void init_session_data(struct session_data *sdata, struct __config *cfg);
+void init_session_data(struct session_data *sdata, struct config *cfg);
 int read_from_stdin(struct session_data *sdata);
 void strtolower(char *s);
 
+int make_socket_non_blocking(int fd);
+int create_and_bind(char *listen_addr, int listen_port);
+
+int can_i_write_directory(char *dir);
+
+void move_email(struct smtp_session *session);
+int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte);
+
+int init_ssl_to_server(struct data *data);
+
 #ifndef _GNU_SOURCE
    char *strcasestr(const char *s, const char *find);
 #endif
 
+int append_string_to_buffer(char **buffer, char *str);
+int get_size_from_smtp_mail_from(char *s);
+
 #endif /* _MISC_H */

src/mydomains.c

@@ -10,35 +10,56 @@
 #include <piler.h>
 
 
-void load_mydomains(struct session_data *sdata, struct __data *data, struct __config *cfg){
-   int clen=0, len=0, size=sizeof(data->mydomains);
+void load_mydomains(struct session_data *sdata, struct data *data, struct config *cfg){
    char s[SMALLBUFSIZE];
-   MYSQL_RES *res;
-   MYSQL_ROW row;
+   struct sql sql;
 
-   memset(data->mydomains, 0, size);
+   memset(s, 0, sizeof(s));
 
-   snprintf(s, sizeof(s)-1, "SELECT `domain` FROM `%s`", SQL_DOMAIN_TABLE);
 
-   if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){
-      res = mysql_store_result(&(sdata->mysql));
-      if(res != NULL){
-         while((row = mysql_fetch_row(res))){
-            snprintf(s, sizeof(s)-1, "%s,", (char*)row[0]);
-            len = strlen(s);
+   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_DOMAINS) == ERR) return;
 
-            if(clen + len + 1 < size){
-               memcpy(data->mydomains+clen, s, len);
-               clen += len;
-            }
-            else break;
-         }
 
-         mysql_free_result(res);
+   p_bind_init(&sql);
+
+   if(p_exec_stmt(sdata, &sql) == OK){
+
+      p_bind_init(&sql);
+
+      sql.sql[sql.pos] = &s[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(s)-2; sql.pos++;
+
+      p_store_results(&sql);
+
+      while(p_fetch_results(&sql) == OK){
+         if(addnode(data->mydomains, s) == 0) syslog(LOG_PRIORITY, "failed to append mydomain: '%s'", s);
+         else if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "added mydomain: '%s'", s);
+
+         memset(s, 0, sizeof(s));
       }
 
+      p_free_results(&sql);
    }
 
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "mydomains: '%s'", data->mydomains);
+   close_prepared_statement(&sql);
 }
 
+
+int is_email_address_on_my_domains(char *email, struct data *data){
+   int rc=0;
+   char *q, *s;
+
+   if(email == NULL) return rc;
+
+   q = strchr(email, '@');
+   if(!q || strlen(q) < 3) return rc;
+
+   s = strrchr(q+1, ' ');
+
+   if(s) *s = '\0';
+
+   if(findnode(data->mydomains, q+1)) rc = 1;
+
+   if(s) *s = ' ';
+
+   return rc;
+}

src/mysql.c

@@ -9,20 +9,253 @@
 #include <piler.h>
 
 
-int prepare_a_mysql_statement(struct session_data *sdata, MYSQL_STMT **stmt, char *s){
+int open_database(struct session_data *sdata, struct config *cfg){
+   int rc=1;
+   char buf[BUFLEN];
 
-   *stmt = mysql_stmt_init(&(sdata->mysql));
-   if(!*stmt){
-      syslog(LOG_PRIORITY, "%s: mysql_stmt_init() error", sdata->ttmpfile);
+   mysql_init(&(sdata->mysql));
+
+   mysql_options(&(sdata->mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg->mysql_connect_timeout);
+   mysql_options(&(sdata->mysql), MYSQL_OPT_RECONNECT, (const char*)&rc);
+
+   if(mysql_real_connect(&(sdata->mysql), cfg->mysqlhost, cfg->mysqluser, cfg->mysqlpwd, cfg->mysqldb, cfg->mysqlport, cfg->mysqlsocket, 0) == 0){
+      syslog(LOG_PRIORITY, "cant connect to mysql server");
       return ERR;
    }
 
-   if(mysql_stmt_prepare(*stmt, s, strlen(s))){
-      syslog(LOG_PRIORITY, "%s: mysql_stmt_prepare() error: %s => sql: %s", sdata->ttmpfile, mysql_stmt_error(*stmt), s);
+   snprintf(buf, sizeof(buf)-2, "SET NAMES %s", cfg->mysqlcharset);
+   mysql_real_query(&(sdata->mysql), buf, strlen(buf));
+
+   snprintf(buf, sizeof(buf)-2, "SET CHARACTER SET %s", cfg->mysqlcharset);
+   mysql_real_query(&(sdata->mysql), buf, strlen(buf));
+
+   return OK;
+}
+
+
+int open_sphx(struct session_data *sdata, struct config *cfg){
+   int rc=1;
+   char buf[BUFLEN];
+
+   mysql_init(&(sdata->sphx));
+
+   mysql_options(&(sdata->sphx), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg->mysql_connect_timeout);
+   mysql_options(&(sdata->sphx), MYSQL_OPT_RECONNECT, (const char*)&rc);
+
+   if(mysql_real_connect(&(sdata->sphx), cfg->sphxhost, "", "", cfg->sphxdb, cfg->sphxport, "", 0) == 0){
+      syslog(LOG_PRIORITY, "cant connect to %s:%d", cfg->sphxhost, cfg->sphxport);
       return ERR;
    }
 
-   return OK; 
+   snprintf(buf, sizeof(buf)-2, "SET NAMES %s", cfg->mysqlcharset);
+   mysql_real_query(&(sdata->sphx), buf, strlen(buf));
+
+   snprintf(buf, sizeof(buf)-2, "SET CHARACTER SET %s", cfg->mysqlcharset);
+   mysql_real_query(&(sdata->sphx), buf, strlen(buf));
+
+   return OK;
+}
+
+
+void close_database(struct session_data *sdata){
+   mysql_close(&(sdata->mysql));
+}
+
+
+void close_sphx(struct session_data *sdata){
+   mysql_close(&(sdata->sphx));
+}
+
+
+void p_bind_init(struct sql *sql){
+   int i;
+
+   sql->pos = 0;
+
+   for(i=0; i<MAX_SQL_VARS; i++){
+      sql->sql[i] = NULL;
+      sql->type[i] = TYPE_UNDEF;
+      sql->len[i] = 0;
+   }
+}
+
+
+void p_query(struct session_data *sdata, char *s){
+   mysql_real_query(&(sdata->mysql), s, strlen(s));
+}
+
+
+int p_exec_stmt(struct session_data *sdata, struct sql *sql){
+   MYSQL_BIND bind[MAX_SQL_VARS];
+   unsigned long length[MAX_SQL_VARS];
+   int i, ret=ERR;
+
+   sdata->sql_errno = 0;
+   memset(bind, 0, sizeof(bind));
+
+   for(i=0; i<MAX_SQL_VARS; i++){
+      if(sql->type[i] > TYPE_UNDEF){
+
+
+         switch(sql->type[i]) {
+             case TYPE_SHORT:
+                                  bind[i].buffer_type = MYSQL_TYPE_SHORT;
+                                  bind[i].length = 0;
+                                  break;
+
+             case TYPE_LONG:
+                                  bind[i].buffer_type = MYSQL_TYPE_LONG;
+                                  bind[i].length = 0;
+                                  break;
+
+
+             case TYPE_LONGLONG:
+                                  bind[i].buffer_type = MYSQL_TYPE_LONGLONG;
+                                  bind[i].length = 0;
+                                  break;
+
+
+             case TYPE_STRING:
+                                  bind[i].buffer_type = MYSQL_TYPE_STRING;
+                                  length[i] = strlen(sql->sql[i]);
+                                  bind[i].length = &length[i];
+                                  break;
+
+
+             default:
+                                  bind[i].buffer_type = 0;
+                                  bind[i].length = 0;
+                                  break;
+
+         };
+
+
+         bind[i].buffer = sql->sql[i];
+         bind[i].is_null = 0;
+
+      }
+      else { break; }
+   }
+
+   if(mysql_stmt_bind_param(sql->stmt, bind)){
+      sdata->sql_errno = mysql_stmt_errno(sql->stmt);
+      syslog(LOG_PRIORITY, "ERROR: %s: mysql_stmt_bind_param() '%s' (errno: %d)", sdata->ttmpfile, mysql_stmt_error(sql->stmt), sdata->sql_errno);
+      return ret;
+   }
+
+   if(mysql_stmt_execute(sql->stmt)){
+      sdata->sql_errno = mysql_stmt_errno(sql->stmt);
+      syslog(LOG_PRIORITY, "ERROR: %s: mysql_stmt_execute() '%s' (errno: %d)", sdata->ttmpfile, mysql_error(&(sdata->mysql)), sdata->sql_errno);
+      return ret;
+   }
+
+   ret = OK;
+
+   return ret;
+}
+
+
+int p_store_results(struct sql *sql){
+   MYSQL_BIND bind[MAX_SQL_VARS];
+   int i, ret=ERR;
+
+   memset(bind, 0, sizeof(bind));
+
+   for(i=0; i<MAX_SQL_VARS; i++){
+      if(sql->type[i] > TYPE_UNDEF){
+
+         switch(sql->type[i]) {
+             case TYPE_SHORT:     bind[i].buffer_type = MYSQL_TYPE_SHORT;
+                                  break;
+
+
+             case TYPE_LONG:      bind[i].buffer_type = MYSQL_TYPE_LONG;
+                                  break;
+
+
+             case TYPE_LONGLONG:
+                                  bind[i].buffer_type = MYSQL_TYPE_LONGLONG;
+                                  break;
+
+
+             case TYPE_STRING:
+                                  bind[i].buffer_type = MYSQL_TYPE_STRING;
+                                  bind[i].buffer_length = sql->len[i];
+                                  break;
+
+             default:
+                                  bind[i].buffer_type = 0;
+                                  break;
+
+         };
+
+
+         bind[i].buffer = (char *)sql->sql[i];
+         bind[i].is_null = &(sql->is_null[i]);
+         bind[i].length = &(sql->length[i]);
+         bind[i].error = &(sql->error[i]);
+
+      }
+      else { break; }
+   }
+
+   if(mysql_stmt_bind_result(sql->stmt, bind)){
+      return ret;
+   }
+
+
+   if(mysql_stmt_store_result(sql->stmt)){
+      return ret;
+   }
+
+   ret = OK;
+
+   return ret;
+}
+
+
+int p_fetch_results(struct sql *sql){
+
+   if(mysql_stmt_fetch(sql->stmt) == 0) return OK;
+
+   return ERR;
+}
+
+
+void p_free_results(struct sql *sql){
+   mysql_stmt_free_result(sql->stmt);
+}
+
+
+uint64 p_get_insert_id(struct sql *sql){
+   return mysql_stmt_insert_id(sql->stmt);
+}
+
+
+int p_get_affected_rows(struct sql *sql){
+   return mysql_stmt_affected_rows(sql->stmt);
+}
+
+
+int prepare_sql_statement(struct session_data *sdata, struct sql *sql, char *s){
+
+   sql->stmt = mysql_stmt_init(&(sdata->mysql));
+   if(!(sql->stmt)){
+      syslog(LOG_PRIORITY, "%s: error: mysql_stmt_init()", sdata->ttmpfile);
+      return ERR;
+   }
+
+   if(mysql_stmt_prepare(sql->stmt, s, strlen(s))){
+      syslog(LOG_PRIORITY, "%s: error: mysql_stmt_prepare() %s => sql: %s", sdata->ttmpfile, mysql_stmt_error(sql->stmt), s);
+      return ERR;
+   }
+
+   return OK;
+}
+
+
+void close_prepared_statement(struct sql *sql){
+   if(sql->stmt) mysql_stmt_close(sql->stmt);
 }
 
 
@@ -34,26 +267,3 @@ void insert_offset(struct session_data *sdata, int server_id){
 
    mysql_real_query(&(sdata->mysql), s, strlen(s));
 }
-
-
-int create_prepared_statements(struct session_data *sdata, struct __data *data){
-   int rc = OK;
-
-   data->stmt_get_meta_id_by_message_id = NULL;
-   data->stmt_insert_into_rcpt_table = NULL;
-   data->stmt_update_metadata_reference = NULL;
-
-   if(prepare_a_mysql_statement(sdata, &(data->stmt_get_meta_id_by_message_id), SQL_PREPARED_STMT_GET_META_ID_BY_MESSAGE_ID) == ERR) rc = ERR;
-   if(prepare_a_mysql_statement(sdata, &(data->stmt_insert_into_rcpt_table), SQL_PREPARED_STMT_INSERT_INTO_RCPT_TABLE) == ERR) rc = ERR;
-   if(prepare_a_mysql_statement(sdata, &(data->stmt_update_metadata_reference), SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE) == ERR) rc = ERR;
-
-   return rc;
-}
-
-
-void close_prepared_statements(struct __data *data){
-   if(data->stmt_get_meta_id_by_message_id) mysql_stmt_close(data->stmt_get_meta_id_by_message_id);
-   if(data->stmt_insert_into_rcpt_table) mysql_stmt_close(data->stmt_insert_into_rcpt_table);
-   if(data->stmt_update_metadata_reference) mysql_stmt_close(data->stmt_update_metadata_reference);
-}
-

src/parser.c

@@ -16,12 +16,11 @@
 #include <piler.h>
 
 
-struct _state parse_message(struct session_data *sdata, int take_into_pieces, struct __data *data, struct __config *cfg){
+struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct data *data, struct config *cfg){
    FILE *f;
-   int i, len;
-   char *p, buf[MAXBUFSIZE], puf[SMALLBUFSIZE];
+   char buf[MAXBUFSIZE];
    char writebuffer[MAXBUFSIZE], abuffer[MAXBUFSIZE];
-   struct _state state;
+   struct parser_state state;
 
    init_state(&state);
 
@@ -31,45 +30,6 @@ struct _state parse_message(struct session_data *sdata, int take_into_pieces, st
       return state;
    }
 
-
-   if(sdata->num_of_rcpt_to > 0){
-      for(i=0; i<sdata->num_of_rcpt_to; i++){
-         extractEmail(sdata->rcptto[i], puf);
-
-         if(strlen(puf) > 5){
-            p = strstr(puf, cfg->hostid);
-            if(p && *(p-1) == '.'){
-
-               *(p-1) = ' ';
-               *p = '\0';
-
-               len = strlen(puf);
-
-               if(does_it_seem_like_an_email_address(puf) == 1){
-
-                  if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: processing rcpt to address: *%s*", sdata->ttmpfile, puf);
-
-                  if(state.tolen < MAXBUFSIZE-len-1){
-                     if(is_string_on_list(state.rcpt, puf) == 0){
-                        append_list(&(state.rcpt), puf);
-                        memcpy(&(state.b_to[state.tolen]), puf, len);
-                        state.tolen += len;
-
-                        if(state.tolen < MAXBUFSIZE-len-1){
-                           split_email_address(puf);
-                           memcpy(&(state.b_to[state.tolen]), puf, len);
-                           state.tolen += len;
-                        }
-                     }
-                  }
-
-               }
-            }
-         }
-
-      }
-   }
-
    if(take_into_pieces == 1){
       state.mfd = open(sdata->tmpframe, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR);
       if(state.mfd == -1){
@@ -84,33 +44,70 @@ struct _state parse_message(struct session_data *sdata, int take_into_pieces, st
    }
 
    if(take_into_pieces == 1 && state.writebufpos > 0){
-      len = write(state.mfd, writebuffer, state.writebufpos);
-      memset(writebuffer, 0, sizeof(writebuffer));
+      if(write(state.mfd, writebuffer, state.writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: %s: write(), %s, %d, %s", sdata->ttmpfile, __func__, __LINE__, __FILE__);
+      memset(writebuffer, 0, sizeof(writebuffer)); //-V597
       state.writebufpos = 0;
    }
 
    if(take_into_pieces == 1){
       close(state.mfd); state.mfd = 0;
+
+      if(state.has_to_dump_whole_body == 1){
+         if(state.abufpos > 0){
+            flush_attachment_buffer(&state, &abuffer[0], sizeof(abuffer));
+         }
+         if(state.fd != -1) close(state.fd);
+         if(state.b64fd != -1) close(state.b64fd);
+      }
+
    }
 
    fclose(f);
 
+   if(data->import && data->import->extra_recipient){
+      char tmpbuf[SMALLBUFSIZE];
+      snprintf(tmpbuf, sizeof(tmpbuf)-1, "%s", data->import->extra_recipient);
+      add_recipient(tmpbuf, strlen(tmpbuf), sdata, &state, data, cfg);
+   }
+
+   // If both Sender: and From: headers exist, and they are different, then append
+   // the From: address to recipients list to give him access to this email as well
+
+   if(state.b_sender_domain[0]){
+      char frombuf[SMALLBUFSIZE];
+      char senderbuf[SMALLBUFSIZE];
+      get_first_email_address_from_string(state.b_from, frombuf, sizeof(frombuf));
+      get_first_email_address_from_string(state.b_sender, senderbuf, sizeof(senderbuf));
+      if(strcmp(frombuf, senderbuf)){
+         frombuf[strlen(frombuf)] = ' ';
+         add_recipient(frombuf, strlen(frombuf), sdata, &state, data, cfg);
+      }
+   }
+
    return state;
 }
 
 
-void post_parse(struct session_data *sdata, struct _state *state, struct __config *cfg){
-   int i, len, rec=0;
-   char *p;
+void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg){
+   int i;
 
-   free_list(state->boundaries);
-   free_list(state->rcpt);
-   free_list(state->rcpt_domain);
-   free_list(state->journal_recipient);
+   clearhash(state->boundaries);
+   clearhash(state->rcpt);
+   clearhash(state->rcpt_domain);
+   clearhash(state->journal_recipient);
 
+   // Fix From: and Sender: lines if they are too long
+   if(strlen(state->b_from) > 255) state->b_from[255] = '\0';
+   if(strlen(state->b_from_domain) > 255) state->b_from_domain[255] = '\0';
+
+   if(strlen(state->b_sender) > 255) state->b_sender[255] = '\0';
+   if(strlen(state->b_sender_domain) > 255) state->b_sender_domain[255] = '\0';
+
+   // Truncate the message_id if it's >255 characters
+   if(strlen(state->message_id) > 255) state->message_id[255] = '\0';
+
+   fixupEncodedHeaderLine(state->b_subject, MAXBUFSIZE);
    trimBuffer(state->b_subject);
-   fixupEncodedHeaderLine(state->b_subject);
-
 
    if(sdata->internal_sender == 0) sdata->direction = DIRECTION_INCOMING;
    else {
@@ -119,20 +116,50 @@ void post_parse(struct session_data *sdata, struct _state *state, struct __confi
       if(sdata->internal_recipient == 1 && sdata->external_recipient == 1) sdata->direction = DIRECTION_INTERNAL_AND_OUTGOING;
    }
 
+   char *q = strrchr(state->receivedbuf, ';');
+   // I've seen some odd 1st Received: lines where the date
+   // wasn't the last item after a semicolon, eg.
+   //
+   // Received: from some.server.local [172.16.102.18]
+   //   by some.server.com with POP3 (Message Archiver)
+   //   for <nobody@127.0.0.1> (single-drop); Fri, 22 Nov 2019 11:24:49 -0500 (EST)
+   //   smtp.auth=3Daaaaaaaa@yourdomain.com|Received:=20by=20yourdomain
+   //   .com=20with=20ESMTPA=20id=20md50124862414.msg=3B=20Fri,=2022=20
+   //   ...
+   //
+   // Such line messes with the parse_date_header() function
+   // so the workaround is to discard such Received: line
+   if(q && strlen(q+1) < 45){
+      time_t received_timestamp = parse_date_header(q+1);
+      if(received_timestamp > 10000000){
+         // If the calculated date based on Date: header line differs more than 1 week
+         // then we'll override it with the data parsed from the first Received: line
+         if(labs(received_timestamp - sdata->sent) > 604800) sdata->sent = received_timestamp;
+      }
+   }
 
    for(i=1; i<=state->n_attachments; i++){
+      char puf[SMALLBUFSIZE];
+      snprintf(puf, sizeof(puf)-1, "%s ", state->attachments[i].filename);
+
+      unsigned int len = strlen(puf);
+      if(state->bodylen < BIGBUFSIZE-len-1){
+         memcpy(&(state->b_body[state->bodylen]), puf, len);
+         state->bodylen += len;
+      }
+
       digest_file(state->attachments[i].internalname, &(state->attachments[i].digest[0]));
 
-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest);
+      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s, dumped: %d", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest, state->attachments[i].dumped);
 
-      p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type);
+      char *p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type);
       len = strlen(p);
       if(strlen(sdata->attachments) < SMALLBUFSIZE-len-1 && !strstr(sdata->attachments, p)) memcpy(&(sdata->attachments[strlen(sdata->attachments)]), p, len);
 
       if(state->attachments[i].dumped == 1){
-         rec = 0;
-         if(state->bodylen < BIGBUFSIZE-1024) extract_attachment_content(sdata, state, state->attachments[i].aname, get_attachment_extractor_by_filename(state->attachments[i].filename), &rec);
-      
+         int rec = 0;
+         if(cfg->extract_attachments == 1 && state->bodylen < BIGBUFSIZE-1024) extract_attachment_content(sdata, state, state->attachments[i].aname, get_attachment_extractor_by_filename(state->attachments[i].filename), &rec, cfg);
+
          unlink(state->attachments[i].aname);
       }
 
@@ -145,10 +172,14 @@ void post_parse(struct session_data *sdata, struct _state *state, struct __confi
       else snprintf(state->message_id, SMALLBUFSIZE-1, "null");
    }
 
+
+   digest_string("sha256", state->message_id, &(state->message_id_hash[0]));
+
+   if(sdata->sent == 0) sdata->sent = sdata->now;
 }
 
 
-void storno_attachment(struct _state *state){
+void storno_attachment(struct parser_state *state){
    state->has_to_dump = 0;
 
    if(state->n_attachments <= 0) return;
@@ -159,7 +190,7 @@ void storno_attachment(struct _state *state){
    memset(state->attachments[state->n_attachments].type, 0, TINYBUFSIZE);
    memset(state->attachments[state->n_attachments].shorttype, 0, TINYBUFSIZE);
    memset(state->attachments[state->n_attachments].aname, 0, TINYBUFSIZE);
-   memset(state->attachments[state->n_attachments].filename, 0, TINYBUFSIZE);
+   memset(state->attachments[state->n_attachments].filename, 0, SMALLBUFSIZE);
    memset(state->attachments[state->n_attachments].internalname, 0, TINYBUFSIZE);
    memset(state->attachments[state->n_attachments].digest, 0, 2*DIGEST_LENGTH+1);
 
@@ -168,30 +199,70 @@ void storno_attachment(struct _state *state){
 }
 
 
-int parse_line(char *buf, struct _state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, int writebuffersize, char *abuffer, int abuffersize, struct __data *data, struct __config *cfg){
-   char *p, *q, puf[SMALLBUFSIZE];
-   unsigned char b64buffer[MAXBUFSIZE];
-   int n64, len, writelen, boundary_line=0;
+void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned int abuffersize){
+   if(write(state->fd, abuffer, state->abufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
+
+   if(state->b64fd != -1){
+      abuffer[state->abufpos] = '\0';
+      if(state->base64 == 1){
+         unsigned char b64buffer[MAXBUFSIZE];
+         int n64 = base64_decode_attachment_buffer(abuffer, &b64buffer[0], sizeof(b64buffer));
+         if(write(state->b64fd, b64buffer, n64) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
+      }
+      else if(write(state->b64fd, abuffer, state->abufpos) == -1){
+         syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
+      }
+   }
+
+   state->abufpos = 0;
+   memset(abuffer, 0, abuffersize);
+}
+
+
+int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg){
+   char *p;
+   int boundary_line=0;
+   unsigned int len;
 
    if(cfg->debug == 1) printf("line: %s", buf);
 
    state->line_num++;
    len = strlen(buf);
 
-   if(state->is_1st_header == 1 && (strncmp(buf, "Received: by piler", strlen("Received: by piler")) == 0 || strncmp(buf, "X-piler-id: ", strlen("X-piler-id: ")) == 0) ){
-      sdata->restored_copy = 1;
-   }
+   /*
+    * check a few things in the 1st header
+    */
 
-   if(state->is_1st_header == 1 && *(cfg->spam_header_line) != '\0' && strncmp(buf, cfg->spam_header_line, strlen(cfg->spam_header_line)) == 0){
-      sdata->spam_message = 1;
-   }
+   if(state->is_1st_header == 1){
 
-   if(state->is_1st_header == 1 && sdata->ms_journal == 0 && strncmp(buf, "X-MS-Journal-Report:", strlen("X-MS-Journal-Report:")) == 0){
-      sdata->ms_journal = 1;
-      memset(state->message_id, 0, SMALLBUFSIZE);
+      if(strncmp(buf, "Received: by piler", strlen("Received: by piler")) == 0){
+         sdata->restored_copy = 1;
+      }
+
+      if(cfg->security_header[0] && state->found_security_header == 0 && strstr(buf, cfg->security_header)){
+         state->found_security_header = 1;
+      }
+
+      if(*(cfg->piler_header_field) != 0 && strncmp(buf, cfg->piler_header_field, strlen(cfg->piler_header_field)) == 0){
+         sdata->restored_copy = 1;
+      }
+
+      if(sdata->ms_journal == 0 && ( strncmp(buf, "X-MS-Journal-Report:", strlen("X-MS-Journal-Report:")) == 0 || strncmp(buf, "X-WM-Journal-Report: journal", strlen("X-WM-Journal-Report: journal")) == 0) ){
+
+         memset(state->b_to, 0, MAXBUFSIZE);
+         state->tolen = 0;
+         memset(state->b_to_domain, 0, SMALLBUFSIZE);
+         state->todomainlen = 0;
+
+         clearhash(state->rcpt);
+         clearhash(state->rcpt_domain);
+
+         //if(sdata->import == 0){
+            sdata->ms_journal = 1;
+            memset(state->message_id, 0, SMALLBUFSIZE);
+         //}
+      }
 
-      memset(state->b_from, 0, SMALLBUFSIZE); 
-      memset(state->b_from_domain, 0, SMALLBUFSIZE);
    }
 
 
@@ -202,76 +273,55 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
       state->is_1st_header = 0;
    }
 
-   if(sdata->ms_journal == 1 && strncasecmp(buf, "Received:", strlen("Received:")) == 0){
-      state->is_1st_header = 1;
-      state->is_header = 1;
-      memset(state->b_body, 0, BIGBUFSIZE);
-      state->bodylen = 0;
-      memset(state->b_subject, 0, MAXBUFSIZE);
-   }
 
    if(take_into_pieces == 1){
-      if(state->message_state == MSG_BODY && state->fd != -1 && is_item_on_string(state->boundaries, buf) == 0){
-         //n = write(state->fd, buf, len); // WRITE
+      if(state->message_state == MSG_BODY && state->fd != -1 && (state->has_to_dump_whole_body == 1 || is_substr_in_hash(state->boundaries, buf) == 0) ){
          if(len + state->abufpos > abuffersize-1){
-            write(state->fd, abuffer, state->abufpos); 
-
-            if(state->b64fd != -1){
-               abuffer[state->abufpos] = '\0';
-               if(state->base64 == 1){
-                  n64 = base64_decode_attachment_buffer(abuffer, state->abufpos, &b64buffer[0], sizeof(b64buffer));
-                  n64 = write(state->b64fd, b64buffer, n64);
-               }
-               else {
-                  n64 = write(state->b64fd, abuffer, state->abufpos);
-               }
-            }
-
-            state->abufpos = 0; memset(abuffer, 0, abuffersize);
+            flush_attachment_buffer(state, abuffer, abuffersize);
          }
          memcpy(abuffer+state->abufpos, buf, len); state->abufpos += len;
 
          state->attachments[state->n_attachments].size += len;
+
+         // When processing the body and writing to an attachment file, then we finish here
+         return 0;
       }
       else {
          state->saved_size += len;
-         //n = write(state->mfd, buf, len); // WRITE
          if(len + state->writebufpos > writebuffersize-1){
-            write(state->mfd, writebuffer, state->writebufpos); state->writebufpos = 0; memset(writebuffer, 0, writebuffersize);
+            if(write(state->mfd, writebuffer, state->writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
+            state->writebufpos = 0;
+            memset(writebuffer, 0, writebuffersize);
          }
          memcpy(writebuffer+state->writebufpos, buf, len); state->writebufpos += len;
       }
    }
 
 
-   if(state->message_state == MSG_BODY && state->has_to_dump == 1 &&  state->pushed_pointer == 0){
-      //printf("####name: %s, type: %s, base64: %d\n", state->filename, state->type, state->base64);
-
+   if(state->message_state == MSG_BODY && state->has_to_dump == 1 && state->pushed_pointer == 0){
       state->pushed_pointer = 1;
 
 
       // this is a real attachment to dump, it doesn't have to be base64 encoded!
-      if(strlen(state->filename) > 4 && strlen(state->type) > 3 && state->n_attachments < MAX_ATTACHMENTS-1){
+      if(state->attachment_name_buf[0] != 0 && strcasestr(state->attachment_name_buf, "name") && strlen(state->type) > 3 && state->n_attachments < MAX_ATTACHMENTS-1){
          state->n_attachments++;
 
-         snprintf(state->attachments[state->n_attachments].filename, TINYBUFSIZE-1, "%s", state->filename);
+         extractNameFromHeaderLine(state->attachment_name_buf, "name", state->attachments[state->n_attachments].filename, SMALLBUFSIZE);
          snprintf(state->attachments[state->n_attachments].type, TINYBUFSIZE-1, "%s", state->type);
          snprintf(state->attachments[state->n_attachments].internalname, TINYBUFSIZE-1, "%s.a%d", sdata->ttmpfile, state->n_attachments);
          snprintf(state->attachments[state->n_attachments].aname, TINYBUFSIZE-1, "%s.a%d.bin", sdata->ttmpfile, state->n_attachments);
 
-         //printf("DUMP FILE: %s\n", state->attachments[state->n_attachments].internalname);
-
          if(take_into_pieces == 1){
             state->fd = open(state->attachments[state->n_attachments].internalname, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR);
 
-            fixupEncodedHeaderLine(state->attachments[state->n_attachments].filename);
+            fixupEncodedHeaderLine(state->attachments[state->n_attachments].filename, SMALLBUFSIZE);
 
             p = get_attachment_extractor_by_filename(state->attachments[state->n_attachments].filename);
 
             snprintf(state->attachments[state->n_attachments].shorttype, TINYBUFSIZE-1, "%s", p);
- 
+
             if(strcmp("other", p)){
-               state->b64fd = open(state->attachments[state->n_attachments].aname, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR);
+               state->b64fd = open(state->attachments[state->n_attachments].aname, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
                state->attachments[state->n_attachments].dumped = 1;
             }
 
@@ -282,13 +332,16 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
                syslog(LOG_PRIORITY, "%s: error opening %s", sdata->ttmpfile, state->attachments[state->n_attachments].internalname);
             }
             else {
+               char puf[SMALLBUFSIZE];
                snprintf(puf, sizeof(puf)-1, "ATTACHMENT_POINTER_%s.a%d_XXX_PILER", sdata->ttmpfile, state->n_attachments);
-               //n = write(state->mfd, puf, strlen(puf)); // WRITE
-               writelen = strlen(puf);
+               int writelen = strlen(puf);
                if(writelen + state->writebufpos > writebuffersize-1){
-                  write(state->mfd, writebuffer, state->writebufpos); state->writebufpos = 0; memset(writebuffer, 0, writebuffersize);
+                  if(write(state->mfd, writebuffer, state->writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
+                  state->writebufpos = 0;
+                  memset(writebuffer, 0, writebuffersize);
                }
-               memcpy(writebuffer+state->writebufpos, puf, writelen); state->writebufpos += writelen;
+               memcpy(writebuffer+state->writebufpos, puf, writelen);
+               state->writebufpos += writelen;
             }
          }
 
@@ -321,34 +374,123 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
    if(state->is_header == 0 && buf[0] != ' ' && buf[0] != '\t') state->message_state = MSG_BODY;
 
 
+   // journal fix
+
+   if(state->message_state == MSG_BODY && sdata->ms_journal == 1){
+      state->is_header = 1;
+      state->is_1st_header = 1;
+   }
+
+
    /* header checks */
 
    if(state->is_header == 1){
 
-      if(strncasecmp(buf, "From:", strlen("From:")) == 0) state->message_state = MSG_FROM;
+      if(*(cfg->spam_header_line) != '\0' && strncmp(buf, cfg->spam_header_line, strlen(cfg->spam_header_line)) == 0){
+         sdata->spam_message = 1;
+      }
 
-      else if(strncasecmp(buf, "Content-Type:", strlen("Content-Type:")) == 0) state->message_state = MSG_CONTENT_TYPE;
-      else if(strncasecmp(buf, "Content-Transfer-Encoding:", strlen("Content-Transfer-Encoding:")) == 0) state->message_state = MSG_CONTENT_TRANSFER_ENCODING;
-      else if(strncasecmp(buf, "Content-Disposition:", strlen("Content-Disposition:")) == 0) state->message_state = MSG_CONTENT_DISPOSITION;
+      if(strncasecmp(buf, "X-Piler-Envelope-To:", strlen("X-Piler-Envelope-To:")) == 0){
+         state->message_state = MSG_ENVELOPE_TO;
+         buf += strlen("X-Piler-Envelope-To:");
+      }
+      else if(strncasecmp(buf, "From:", strlen("From:")) == 0){
+         state->message_state = MSG_FROM;
+         buf += strlen("From:");
+      }
+      else if(strncasecmp(buf, "Sender:", strlen("Sender:")) == 0){
+         state->message_state = MSG_SENDER;
+         buf += strlen("Sender:");
+      }
+      else if(strncasecmp(buf, "Content-Type:", strlen("Content-Type:")) == 0){
+         state->message_state = MSG_CONTENT_TYPE;
+      }
+      else if(strncasecmp(buf, "Content-Transfer-Encoding:", strlen("Content-Transfer-Encoding:")) == 0){
+         state->message_state = MSG_CONTENT_TRANSFER_ENCODING;
+         if(state->is_1st_header == 1 && strcasestr(buf, "base64")){
+            state->has_to_dump = 1;
+            state->has_to_dump_whole_body = 1;
+         }
+      }
 
-      else if(strncasecmp(buf, "To:", 3) == 0) state->message_state = MSG_TO;
-      else if(strncasecmp(buf, "Cc:", 3) == 0) state->message_state = MSG_CC;
-      else if(strncasecmp(buf, "Bcc:", 4) == 0) state->message_state = MSG_CC;
-      else if(strncasecmp(buf, "Message-Id:", 11) == 0) state->message_state = MSG_MESSAGE_ID;
+      /*
+       * We only enter MSG_CONTENT_DISPOSITION state if we couldn't find
+       * the filename in MSG_CONTENT_TYPE state. We also assume that
+       * Content-Type: comes first, then Content-Disposition:
+       */
+      else if(strncasecmp(buf, "Content-Disposition:", strlen("Content-Disposition:")) == 0 && strcasestr(state->attachment_name_buf, "name") == NULL){
+         state->message_state = MSG_CONTENT_DISPOSITION;
+      }
+      else if(strncasecmp(buf, "To:", 3) == 0){
+         state->message_state = MSG_TO;
+         buf += strlen("To:");
+      }
+      else if(strncasecmp(buf, "Cc:", 3) == 0){
+         state->message_state = MSG_CC;
+         buf += strlen("Cc:");
+      }
+      else if(strncasecmp(buf, "Bcc:", 4) == 0){
+         state->message_state = MSG_CC;
+         buf += strlen("Bcc:");
+      }
+      else if(strncasecmp(buf, "Message-Id:", 11) == 0){
+         state->message_state = MSG_MESSAGE_ID;
+         buf += strlen("Message-Id:");
+      }
       else if(strncasecmp(buf, "References:", 11) == 0) state->message_state = MSG_REFERENCES;
-      else if(strncasecmp(buf, "Subject:", strlen("Subject:")) == 0) state->message_state = MSG_SUBJECT;
-      else if(strncasecmp(buf, "Recipient:", strlen("Recipient:")) == 0) state->message_state = MSG_RECIPIENT;
-      else if(strncasecmp(buf, "Date:", strlen("Date:")) == 0 && sdata->sent == 0) sdata->sent = parse_date_header(buf, cfg);
-      else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf, cfg);
-      else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0) state->message_state = MSG_RECEIVED;
-      else if(cfg->extra_to_field[0] != '\0' && strncasecmp(buf, cfg->extra_to_field, strlen(cfg->extra_to_field)) == 0) state->message_state = MSG_TO;
+      else if(strncasecmp(buf, "Subject:", strlen("Subject:")) == 0){
+         state->message_state = MSG_SUBJECT;
+         buf += strlen("Subject:");
+      }
+      else if(strncasecmp(buf, "Recipient:", strlen("Recipient:")) == 0){
+         state->message_state = MSG_RECIPIENT;
+         buf += strlen("Recipient:");
+      }
+      if(sdata->ms_journal == 1 && (state->message_state == MSG_TO || state->message_state == MSG_RECIPIENT) ){
+         p = strstr(buf, "Expanded:");
+         if(p) *p = '\0';
+      }
+
+
+      /*
+       * by default sdata->sent = 0, and let the parser extract value from the Date: header
+       */
+
+      else if(strncasecmp(buf, "Date:", strlen("Date:")) == 0 && state->is_1st_header == 1 && sdata->sent == 0){
+
+         if(strstr(buf, "=?") && strstr(buf, "?=")) fixupEncodedHeaderLine(buf, MAXBUFSIZE);
+
+         sdata->sent = parse_date_header(buf+5);
+
+         /* allow +2 days drift in the parsed Date: value */
+
+         if(sdata->sent - sdata->now > 2*86400) sdata->sent = sdata->now;
+      }
+
+      else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf+14);
+      else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0){
+         state->message_state = MSG_RECEIVED;
+         state->received_header++;
+      }
+      else if(cfg->extra_to_field[0] != '\0' && strncasecmp(buf, cfg->extra_to_field, strlen(cfg->extra_to_field)) == 0){
+         state->message_state = MSG_RECIPIENT;
+         buf += strlen(cfg->extra_to_field);
+      }
 
       if(state->message_state == MSG_MESSAGE_ID && state->message_id[0] == 0){
-         p = strchr(buf+11, ' ');
-         if(p) p = buf + 12;
-         else p = buf + 11;
+         while(isspace(*buf)){
+            buf++;
+         }
 
-         snprintf(state->message_id, SMALLBUFSIZE-1, "%s", p);
+         snprintf(state->message_id, SMALLBUFSIZE-1, "%s", buf);
+      }
+
+      if(state->message_state == MSG_CONTENT_TYPE || state->message_state == MSG_CONTENT_DISPOSITION){
+         fill_attachment_name_buf(state, buf);
+      }
+
+      if(state->received_header == 1 && state->message_state == MSG_RECEIVED && strlen(state->receivedbuf) + len < sizeof(state->receivedbuf)){
+         memcpy(&(state->receivedbuf[strlen(state->receivedbuf)]), buf, len);
       }
 
       /* we are interested in only From:, To:, Subject:, Received:, Content-*: header lines */
@@ -363,23 +505,35 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
    }
 
 
-   if(state->message_state == MSG_BODY && sdata->ms_journal == 1 && strncasecmp(buf, "Recipient:", strlen("Recipient:")) == 0){
-      state->is_header = 1;
-      state->is_1st_header = 1;
-      state->message_state = MSG_RECIPIENT;
-   }
-
-   if(state->message_state == MSG_BODY && sdata->ms_journal == 1 && strncasecmp(buf, "Bcc:", 4) == 0){
-      state->is_header = 1;
-      state->is_1st_header = 1;
-      state->message_state = MSG_CC;
-   }
+   /*
+    * A normal journal looks like this:
+    *
+    *   Sender: sender@domain
+    *   Subject: Test normal
+    *   Message-Id: ...
+    *   Recipient: user1@domain
+    *   Recipient: user2@domain, Forwarded: user1@domain
+    *
+    * However if outlook forwards an email, then the journal is somewhat changed:
+    *
+    *   Sender: sender@domain
+    *   Subject: Test through outlook
+    *   Message-Id: ...
+    *   To: user1@domain
+    *   To: user2@domain, Forwarded: user1@domain
+    *
+    *
+    * Outlook.com has the following scheme, when expanded from a distribution list:
+    *
+    *   Sender: sender@domain
+    *   Subject: Test Email
+    *   Message-Id: ...
+    *   To: user1@domain, Expanded: listaddress@domain
+    *   To: user2@domain, Expanded: listaddress@domain
+    *
+    */
 
 
-   if(state->message_state == MSG_RECIPIENT){
-      p = strstr(buf, "Expanded:");
-      if(p) *p = '\0';
-   }
 
 
    if(state->is_1st_header == 1 && state->message_state == MSG_REFERENCES){
@@ -388,29 +542,13 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
    }
 
 
-   if(state->is_1st_header == 1 && state->message_state == MSG_SUBJECT && strlen(state->b_subject) + strlen(buf) < MAXBUFSIZE-1){
-
-      if(state->b_subject[0] == '\0'){
-         p = &buf[0];
-         if(strncmp(buf, "Subject:", strlen("Subject:")) == 0) p += strlen("Subject:");
-         if(*p == ' ') p++;
-
-         strncat(state->b_subject, p, MAXBUFSIZE-1);
-      }
-      else {
-
-         p = strrchr(state->b_subject, ' ');
-         if(p && ( strcasestr(p+1, "?Q?") || strcasestr(p+1, "?B?") ) ){
-            strncat(state->b_subject, buf+1, MAXBUFSIZE-1);
-         }
-         else strncat(state->b_subject, buf, MAXBUFSIZE-1);
-
-      }
-
-   }
-
    if(state->is_1st_header == 1){
-      fixupEncodedHeaderLine(buf);
+
+      if(state->message_state == MSG_SUBJECT && strlen(state->b_subject) + strlen(buf) < MAXBUFSIZE-1){
+         // buffer the subject lines, and decode it later
+         strncat(state->b_subject, buf, MAXBUFSIZE-strlen(state->b_subject)-1);
+      }
+      else { fixupEncodedHeaderLine(buf, MAXBUFSIZE); }
    }
 
 
@@ -438,8 +576,7 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
          strcasestr(buf, "multipart/report") ||
          strcasestr(buf, "message/delivery-status") ||
          strcasestr(buf, "text/rfc822-headers") ||
-         strcasestr(buf, "message/rfc822") ||
-         strcasestr(buf, "application/ms-tnef")
+         strcasestr(buf, "message/rfc822")
       ){
          state->textplain = 1;
       }
@@ -452,15 +589,29 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
       if(strcasestr(buf, "message/rfc822")){
          state->message_rfc822 = 1;
          state->is_header = 1;
+
+         state->has_to_dump = 0;
+
+         if(sdata->ms_journal == 1){
+            state->is_1st_header = 1;
+
+            // reset all headers, except To:
+
+            memset(state->b_subject, 0, MAXBUFSIZE);
+            memset(state->b_body, 0, BIGBUFSIZE);
+            memset(state->b_from, 0, SMALLBUFSIZE);
+            memset(state->b_from_domain, 0, SMALLBUFSIZE);
+            memset(state->b_sender, 0, SMALLBUFSIZE);
+            memset(state->b_sender_domain, 0, SMALLBUFSIZE);
+            memset(state->message_id, 0, SMALLBUFSIZE);
+
+            sdata->ms_journal = 0;
+         }
       }
 
 
-      if(strcasestr(buf, "charset") && strcasestr(buf, "UTF-8")) state->utf8 = 1;
-   }
-
-
-   if((state->message_state == MSG_CONTENT_TYPE || state->message_state == MSG_CONTENT_DISPOSITION) && strlen(state->filename) < 5){
-      extractNameFromHeaderLine(buf, "name", state->filename);
+      if(strcasestr(buf, "charset")) extractNameFromHeaderLine(buf, "charset", state->charset, TINYBUFSIZE);
+      if(strcasestr(state->charset, "UTF-8")) state->utf8 = 1;
    }
 
 
@@ -473,7 +624,8 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
 
    /* boundary check, and reset variables */
 
-   boundary_line = is_item_on_string(state->boundaries, buf);
+   boundary_line = is_substr_in_hash(state->boundaries, buf);
+
 
    if(!strstr(buf, "boundary=") && !strstr(buf, "boundary =") && boundary_line == 1){
       state->is_header = 1;
@@ -483,20 +635,7 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
       if(state->has_to_dump == 1){
          if(take_into_pieces == 1 && state->fd != -1){
             if(state->abufpos > 0){
-               write(state->fd, abuffer, state->abufpos);
-
-               if(state->b64fd != -1){
-                  abuffer[state->abufpos] = '\0';
-                  if(state->base64 == 1){
-                     n64 = base64_decode_attachment_buffer(abuffer, state->abufpos, &b64buffer[0], sizeof(b64buffer));
-                     n64 = write(state->b64fd, b64buffer, n64);
-                  }
-                  else {
-                     n64 = write(state->b64fd, abuffer, state->abufpos);
-                  }
-               }
-
-               state->abufpos = 0; memset(abuffer, 0, abuffersize); 
+               flush_attachment_buffer(state, abuffer, abuffersize);
             }
             close(state->fd);
             close(state->b64fd);
@@ -517,12 +656,15 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
 
       state->pushed_pointer = 0;
 
-      memset(state->filename, 0, TINYBUFSIZE);
       memset(state->type, 0, TINYBUFSIZE);
+      memset(state->charset, 0, TINYBUFSIZE);
+
+      memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
+      state->anamepos = 0;
 
       state->message_state = MSG_UNDEF;
 
-      return 0;      
+      return 0;
    }
 
    if(boundary_line == 1){ return 0; }
@@ -532,7 +674,7 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
 
 
    /* skip irrelevant headers */
-   if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT) return 0;
+   if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_SENDER && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT && state->message_state != MSG_ENVELOPE_TO) return 0;
 
 
    /* don't process body if it's not a text or html part */
@@ -545,8 +687,6 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
    }
 
 
-   /* remove all HTML tags */
-   if(state->texthtml == 1 && state->message_state == MSG_BODY) markHTML(buf, state);
 
    if(state->message_state == MSG_BODY && state->qp == 1){
       fixupSoftBreakInQuotedPritableLine(buf, state); // 2011.12.07
@@ -556,108 +696,31 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
    /* I believe that we can live without this function call */
    //decodeURL(buf);
 
-   if(state->texthtml == 1) decodeHTML(buf);
+   /* remove all HTML tags */
+   if(state->texthtml == 1 && state->message_state == MSG_BODY) markHTML(buf, state);
+
+
+   if(state->texthtml == 1){
+      size_t buflen = strlen(buf);
+      decodeHTML(buf, state->utf8);
+      /* decodeHTML converted some entities to iso-8859-1 */
+      if(state->utf8 != 1 && strlen(buf) != buflen){
+        /* no charset or us-ascii: switch to iso-8859-1 */
+        if (state->charset[0] == 0 || strcasecmp(state->charset, "us-ascii") == 0){
+          syslog(LOG_PRIORITY, "%s: assuming iso-8859-1 encoding for HTML (was '%s')", sdata->ttmpfile, state->charset);
+          snprintf(state->charset, TINYBUFSIZE-1, "ISO8859-1");
+        }
+      }
+   }
 
    /* encode the body if it's not utf-8 encoded */
-   if(state->message_state == MSG_BODY && state->utf8 != 1) utf8_encode((unsigned char*)buf);
+   if(state->message_state == MSG_BODY && state->utf8 != 1){
+      char tmpbuf[MAXBUFSIZE];
+      int result = utf8_encode(buf, strlen(buf), &tmpbuf[0], sizeof(tmpbuf), state->charset);
+      if(result == OK) snprintf(buf, MAXBUFSIZE-1, "%s", tmpbuf);
+   }
 
-
-   translateLine((unsigned char*)buf, state);
-
-   reassembleToken(buf);
-
-
-   if(state->is_header == 1) p = strchr(buf, ' ');
-   else p = buf;
-
-   //printf("a: %d/%d/%d/%d %s\n", state->is_1st_header, state->is_header, state->message_rfc822, state->message_state, buf);
-
-   do {
-      memset(puf, 0, sizeof(puf));
-      p = split(p, ' ', puf, sizeof(puf)-1);
-
-      if(puf[0] == '\0') continue;
-
-      degenerateToken((unsigned char*)puf);
-
-      if(puf[0] == '\0') continue;
-
-      strncat(puf, " ", sizeof(puf)-1);
-
-      if(strncasecmp(puf, "http://", 7) == 0 || strncasecmp(puf, "https://", 8) == 0) fixURL(puf);
-
-      if(state->is_header == 0 && strncmp(puf, "__URL__", 7) && (puf[0] == ' ' || strlen(puf) > MAX_WORD_LEN || isHexNumber(puf)) ) continue;
-
-
-      len = strlen(puf);
-
-
-      if(state->message_state == MSG_FROM && state->is_1st_header == 1 && strlen(state->b_from) < SMALLBUFSIZE-len-1){
-         memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
-
-         if(does_it_seem_like_an_email_address(puf) == 1 && state->b_from_domain[0] == '\0' && len > 5){
-            q = strchr(puf, '@');
-            if(q && strlen(q) > 5){
-               memcpy(&(state->b_from_domain), q+1, strlen(q+1)-1);
-               if(strstr(sdata->mailfrom, "<>")){
-                  snprintf(sdata->fromemail, SMALLBUFSIZE-1, "%s", puf);
-                  sdata->fromemail[len-1] = '\0';
-               }
-            }
-
-            if(is_email_address_on_my_domains(puf, data) == 1) sdata->internal_sender = 1;
-
-            if(strlen(state->b_from) < SMALLBUFSIZE-len-1){
-               split_email_address(puf);
-               memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
-            }
-         }
-      }
-      else if((state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && state->is_1st_header == 1 && state->tolen < MAXBUFSIZE-len-1){
-         strtolower(puf);
-
-         if(state->message_state == MSG_RECIPIENT && is_string_on_list(state->journal_recipient, puf) == 0){
-            append_list(&(state->journal_recipient), puf);
-            memcpy(&(state->b_journal_to[state->journaltolen]), puf, len);
-            memcpy(&(state->b_journal_to[state->journaltolen]), puf, len);
-            if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: journal rcpt: '%s'", sdata->ttmpfile, puf);
-         }
-
-
-         if(is_string_on_list(state->rcpt, puf) == 0){
-            append_list(&(state->rcpt), puf);
-            memcpy(&(state->b_to[state->tolen]), puf, len);
-            state->tolen += len;
-
-            if(does_it_seem_like_an_email_address(puf) == 1){
-               if(is_email_address_on_my_domains(puf, data) == 1) sdata->internal_recipient = 1;
-               else sdata->external_recipient = 1;
-
-               q = strchr(puf, '@');
-               if(q){
-                  if(is_string_on_list(state->rcpt_domain, q+1) == 0){
-                     append_list(&(state->rcpt_domain), q+1);
-                     memcpy(&(state->b_to_domain[strlen(state->b_to_domain)]), q+1, strlen(q+1));
-                  }
-               }
-
-               if(state->tolen < MAXBUFSIZE-len-1){
-                  split_email_address(puf);
-                  memcpy(&(state->b_to[state->tolen]), puf, len);
-                  state->tolen += len;
-               }
-
-            }
-         }
-
-      }
-      else if(state->message_state == MSG_BODY && len >= cfg->min_word_len && state->bodylen < BIGBUFSIZE-len-1){
-         memcpy(&(state->b_body[state->bodylen]), puf, len);
-         state->bodylen += len;
-      }
-
-   } while(p);
+   tokenize(buf, state, sdata, data, cfg);
 
    return 0;
 }
-

src/parser.h

@@ -9,30 +9,35 @@
 #include "config.h"
 #include "defs.h"
 
-struct _state parse_message(struct session_data *sdata, int take_into_pieces, struct __data *data, struct __config *cfg);
-void post_parse(struct session_data *sdata, struct _state *state, struct __config *cfg);
-int parse_line(char *buf, struct _state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, int writebuffersize, char *abuffer, int abuffersize, struct __data *data, struct __config *cfg);
+struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct data *data, struct config *cfg);
+void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg);
+int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg);
 
-void init_state(struct _state *state);
-unsigned long parse_date_header(char *s, struct __config *cfg);
-int isHexNumber(char *p);
-int extract_boundary(char *p, struct _state *state);
-void fixupEncodedHeaderLine(char *buf);
-void fixupSoftBreakInQuotedPritableLine(char *buf, struct _state *state);
-void fixupBase64EncodedLine(char *buf, struct _state *state);
-void markHTML(char *buf, struct _state *state);
-int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct _state *state);
-void translateLine(unsigned char *p, struct _state *state);
+void init_state(struct parser_state *state);
+time_t parse_date_header(char *s);
+int extract_boundary(char *p, struct parser_state *state);
+void fixupEncodedHeaderLine(char *buf, int buflen);
+void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state);
+void fixupBase64EncodedLine(char *buf, struct parser_state *state);
+void markHTML(char *buf, struct parser_state *state);
+void setStateHTML(char *htmlbuf, int pos, struct parser_state *state);
+void translateLine(unsigned char *p, struct parser_state *state);
 void fix_email_address_for_sphinx(char *s);
 void split_email_address(char *s);
 int does_it_seem_like_an_email_address(char *email);
+void add_recipient(char *email, unsigned int len, struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
 void reassembleToken(char *p);
 void degenerateToken(unsigned char *p);
-void fixURL(char *url);
-int extractNameFromHeaderLine(char *s, char *name, char *resultbuf);
+void fixURL(char *buf, int buflen);
+void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultbuflen);
 char *determine_attachment_type(char *filename, char *type);
 char *get_attachment_extractor_by_filename(char *filename);
-void parse_reference(struct _state *state, char *s);
-int base64_decode_attachment_buffer(char *p, int plen, unsigned char *b, int blen);
+void parse_reference(struct parser_state *state, char *s);
+int base64_decode_attachment_buffer(char *p, unsigned char *b, int blen);
+void fix_plus_sign_in_email_address(char *puf, char **at_sign, unsigned int *len);
+void tokenize(char *buf, struct parser_state *state, struct session_data *sdata, struct data *data, struct config *cfg);
+void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned int abuffersize);
+void fill_attachment_name_buf(struct parser_state *state, char *buf);
+int get_first_email_address_from_string(char *str, char *buf, int buflen);
 
 #endif /* _PARSER_H */

src/piler-smtp.c

@@ -0,0 +1,353 @@
+/*
+ * piler-smtp.c
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <pwd.h>
+#include <sys/ioctl.h>
+#include <signal.h>
+#include <sys/epoll.h>
+#include <netinet/in.h>
+#include <locale.h>
+#include <errno.h>
+#include <sys/time.h>
+#include <time.h>
+#include <syslog.h>
+#include <openssl/ssl.h>
+#include <openssl/err.h>
+#include <piler.h>
+
+#define PROGNAME "piler-smtp"
+
+extern char *optarg;
+extern int optind;
+
+struct epoll_event event, *events=NULL;
+int num_connections = 0;
+int listenerfd = -1;
+int loglevel = 1;
+
+char *configfile = CONFIG_FILE;
+struct config cfg;
+struct passwd *pwd;
+struct smtp_session *session, **sessions=NULL;
+struct smtp_acl *smtp_acl[MAXHASH];
+
+time_t prev_timeout_check = 0;
+
+void usage(){
+   printf("\nusage: piler\n\n");
+   printf("    -c <config file>                  Config file to use if not the default\n");
+   printf("    -d                                Fork to the background\n");
+   printf("    -v                                Return the version and build number\n");
+   printf("    -V                                Return the version and some build parameters\n");
+   printf("    -L <log level>                    Set the log level: 1-5\n");
+
+   exit(0);
+}
+
+
+void p_clean_exit(int sig){
+   if(sig > 0) syslog(LOG_PRIORITY, "got signal: %d, %s", sig, strsignal(sig));
+
+   if(listenerfd != -1) close(listenerfd);
+
+   if(sessions){
+      for(int i=0; i<cfg.max_connections; i++){
+         if(sessions[i]) free_smtp_session(sessions[i]);
+      }
+
+      free(sessions);
+   }
+
+   if(events) free(events);
+
+   clear_smtp_acl(smtp_acl);
+
+   syslog(LOG_PRIORITY, "%s has been terminated", PROGNAME);
+
+   ERR_free_strings();
+
+   exit(1);
+}
+
+
+void fatal(char *s){
+   syslog(LOG_PRIORITY, "%s", s);
+   p_clean_exit(0);
+}
+
+
+void check_for_client_timeout(){
+   time_t now;
+
+   time(&now);
+
+   if(cfg.verbosity >= LOG_DEBUG) syslog(LOG_PRIORITY, "%s @%ld", __func__, now);
+
+   if(now - prev_timeout_check < cfg.smtp_timeout) return;
+
+   if(num_connections > 0){
+      for(int i=0; i<cfg.max_connections; i++){
+         if(sessions[i] && now - sessions[i]->lasttime >= cfg.smtp_timeout){
+            syslog(LOG_PRIORITY, "client %s timeout, lasttime: %ld", sessions[i]->remote_host, sessions[i]->lasttime);
+            tear_down_session(sessions, sessions[i]->slot, &num_connections, "timeout");
+         }
+      }
+   }
+
+   time(&prev_timeout_check);
+}
+
+
+void initialise_configuration(){
+   cfg = read_config(configfile);
+
+   if(strlen(cfg.username) > 1){
+      pwd = getpwnam(cfg.username);
+      if(!pwd) fatal(ERR_NON_EXISTENT_USER);
+   }
+
+   if(getuid() == 0 && pwd){
+      check_and_create_directories(&cfg, pwd->pw_uid, pwd->pw_gid);
+   }
+
+   if(chdir(cfg.workdir)){
+      syslog(LOG_PRIORITY, "workdir: *%s*", cfg.workdir);
+      fatal(ERR_CHDIR);
+   }
+
+   setlocale(LC_MESSAGES, cfg.locale);
+   setlocale(LC_CTYPE, cfg.locale);
+
+   load_smtp_acl(smtp_acl);
+
+   syslog(LOG_PRIORITY, "reloaded config: %s", configfile);
+}
+
+
+int main(int argc, char **argv){
+   int client_sockfd;
+   int i, daemonise=0;
+   int client_len = sizeof(struct sockaddr_storage);
+   ssize_t readlen;
+   struct sockaddr_storage client_address;
+   char readbuf[REALLYBIGBUFSIZE];
+   int efd;
+
+   while((i = getopt(argc, argv, "c:L:dvVh")) > 0){
+      switch(i){
+
+        case 'c' :
+                   configfile = optarg;
+                   break;
+
+        case 'L':
+                   loglevel = atoi(optarg);
+                   break;
+
+        case 'd' :
+                   daemonise = 1;
+                   break;
+
+        case 'v' :
+        case 'V' :
+                   printf("%s build %d\n", VERSION, get_build());
+                   return 0;
+
+        case 'h' :
+        default  :
+                   usage();
+      }
+   }
+
+   (void) openlog(PROGNAME, LOG_PID, LOG_MAIL);
+
+   initialise_configuration();
+
+   listenerfd = create_and_bind(cfg.listen_addr, cfg.listen_port);
+   if(listenerfd == -1){
+      exit(1);
+   }
+
+   if(make_socket_non_blocking(listenerfd) == -1){
+      fatal("make_socket_non_blocking()");
+   }
+
+   if(listen(listenerfd, cfg.backlog) == -1){
+      fatal("ERROR: listen()");
+   }
+
+   if(drop_privileges(pwd)) fatal(ERR_SETUID);
+
+   efd = epoll_create1(0);
+   if(efd == -1){
+      fatal("ERROR: epoll_create()");
+   }
+
+   event.data.fd = listenerfd;
+   event.events = EPOLLIN | EPOLLET;
+   if(epoll_ctl(efd, EPOLL_CTL_ADD, listenerfd, &event) == -1){
+      fatal("ERROR: epoll_ctl() on efd");
+   }
+
+   set_signal_handler(SIGINT, p_clean_exit);
+   set_signal_handler(SIGTERM, p_clean_exit);
+   set_signal_handler(SIGKILL, p_clean_exit);
+   set_signal_handler(SIGSEGV, p_clean_exit);
+
+   set_signal_handler(SIGPIPE, SIG_IGN);
+   set_signal_handler(SIGALRM, SIG_IGN);
+
+   set_signal_handler(SIGHUP, initialise_configuration);
+
+   // calloc() initialitizes the allocated memory
+
+   sessions = calloc(cfg.max_connections, sizeof(struct smtp_session));
+   events = calloc(cfg.max_connections, sizeof(struct epoll_event));
+
+   if(!sessions || !events) fatal("ERROR: calloc()");
+
+   SSL_library_init();
+   SSL_load_error_strings();
+
+   srand(getpid());
+
+   syslog(LOG_PRIORITY, "%s %s, build %d starting", PROGNAME, VERSION, get_build());
+
+#if HAVE_DAEMON == 1
+   if(daemonise == 1 && daemon(1, 0) == -1) fatal(ERR_DAEMON);
+#endif
+
+   for(;;){
+      int n = epoll_wait(efd, events, cfg.max_connections, 1000);
+      for(i=0; i<n; i++){
+
+         // Office365 sometimes behaves oddly: when it receives the 250 OK
+         // message after sending the email, it doesn't send the QUIT command
+         // rather it aborts the connection
+
+         if((events[i].events & EPOLLERR) || (events[i].events & EPOLLHUP) || (!(events[i].events & EPOLLIN))){
+            if(cfg.verbosity >= _LOG_EXTREME) syslog(LOG_PRIORITY, "ERROR: the remote end hung up without sending QUIT");
+            session = get_session_by_socket(sessions, cfg.max_connections, events[i].data.fd);
+            if(session)
+               tear_down_session(sessions, session->slot, &num_connections, "hungup");
+            else
+               close(events[i].data.fd);
+            continue;
+         }
+
+         // We have 1 or more incoming connections to process
+
+         else if(listenerfd == events[i].data.fd){
+
+            while(1){
+
+               client_sockfd = accept(listenerfd, (struct sockaddr *)&client_address, (socklen_t *)&client_len);
+               if(client_sockfd == -1){
+                  if((errno == EAGAIN) || (errno == EWOULDBLOCK)){
+                     // We have processed all incoming connections
+                     break;
+                  }
+                  else {
+                     syslog(LOG_PRIORITY, "ERROR: accept(): '%s'", strerror(errno));
+                     break;
+                  }
+               }
+
+               char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
+               memset(hbuf, 0, sizeof(hbuf));
+               memset(sbuf, 0, sizeof(sbuf));
+
+               if(getnameinfo((struct sockaddr *)&client_address, client_len, hbuf, sizeof(hbuf), sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV) == 0){
+                  // Strictly speaking it's not correct to log num_connections+1 connections
+                  // but it still gives a good clue how many connections we have at the moment
+                  syslog(LOG_PRIORITY, "connected from %s:%s on fd=%d (active connections: %d)", hbuf, sbuf, client_sockfd, num_connections + 1);
+               }
+
+               if(make_socket_non_blocking(client_sockfd) == -1){
+                  syslog(LOG_PRIORITY, "ERROR: cannot make the socket non blocking");
+                  break;
+               }
+
+               event.data.fd = client_sockfd;
+               event.events = EPOLLIN | EPOLLET;
+               if(epoll_ctl(efd, EPOLL_CTL_ADD, client_sockfd, &event) == -1){
+                  syslog(LOG_PRIORITY, "ERROR: epoll_ctl() on client_sockfd");
+                  break;
+               }
+
+               start_new_session(sessions, client_sockfd, &num_connections, smtp_acl, hbuf, &cfg);
+            }
+
+            continue;
+         }
+
+
+         // handle data from an existing connection
+
+         else {
+            int done = 0;
+
+            session = get_session_by_socket(sessions, cfg.max_connections, events[i].data.fd);
+            if(session == NULL){
+               syslog(LOG_PRIORITY, "ERROR: cannot find session for this socket: %d", events[i].data.fd);
+               close(events[i].data.fd);
+               continue;
+            }
+
+            time(&(session->lasttime));
+
+            while(1){
+               if(session->net.use_ssl == 1)
+                  readlen = SSL_read(session->net.ssl, (char*)&readbuf[0], sizeof(readbuf)-1);
+               else
+                  readlen = read(events[i].data.fd, (char*)&readbuf[0], sizeof(readbuf)-1);
+
+               if(cfg.verbosity >= _LOG_EXTREME && readlen > 0) syslog(LOG_PRIORITY, "got %ld bytes to read", readlen);
+
+               if(readlen == -1){
+                  /* If errno == EAGAIN, that means we have read all data. So go back to the main loop. */
+                  if(errno != EAGAIN){
+                     done = 1;
+                  }
+                  break;
+               }
+               else if(readlen == 0){
+                  /* End of file. The remote has closed the connection. */
+                  done = 1;
+                  break;
+               }
+
+               readbuf[readlen] = '\0';
+               handle_data(session, &readbuf[0], readlen, &cfg);
+
+               if(session->protocol_state == SMTP_STATE_BDAT && session->bad == 1){
+                  done = 1;
+                  break;
+               }
+            }
+
+            /* Don't wait until the remote client closes the connection after he sent the QUIT command */
+
+            if(done || session->protocol_state == SMTP_STATE_FINISHED){
+               tear_down_session(sessions, session->slot, &num_connections, "done");
+            }
+         }
+
+
+      }
+
+      check_for_client_timeout();
+   }
+
+   return 0;
+}

src/piler.c

@@ -8,6 +8,7 @@
 #include <strings.h>
 #include <sys/types.h>
 #include <sys/socket.h>
+#include <sys/mman.h>
 #include <netdb.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
@@ -20,59 +21,41 @@
 #include <syslog.h>
 #include <time.h>
 #include <unistd.h>
+#include <dirent.h>
 #include <locale.h>
 #include <errno.h>
-#include <openssl/ssl.h>
-#include <openssl/err.h>
 #include <piler.h>
 
+#define PROGNAME "piler"
 
 extern char *optarg;
 extern int optind;
 
-int sd;
 int quit = 0;
 int received_sighup = 0;
 char *configfile = CONFIG_FILE;
-struct __config cfg;
-struct __data data;
+struct config cfg;
+struct data data;
 struct passwd *pwd;
 
 struct child children[MAXCHILDREN];
 
 
-signal_func *set_signal_handler(int signo, signal_func * func);
-static void takesig(int sig);
-static void child_sighup_handler(int sig);
-static void child_main(struct child *ptr);
-static pid_t child_make(struct child *ptr);
-int search_slot_by_pid(pid_t pid);
-void kill_children(int sig);
 void p_clean_exit();
-void fatal(char *s);
-void initialise_configuration();
 
 
+void usage(){
+   printf("\nusage: piler\n\n");
+   printf("    -c <config file>                  Config file to use if not the default\n");
+   printf("    -d                                Fork to the background\n");
+   printf("    -v                                Return the version and build number\n");
+   printf("    -V                                Return the version and some build parameters\n");
 
-
-/*
- * most of the preforking code was taken from the tinyproxy project
- */
-
-signal_func *set_signal_handler(int signo, signal_func * func){
-   struct sigaction act, oact;
-
-   act.sa_handler = func;
-   sigemptyset (&act.sa_mask);
-   act.sa_flags = 0;
-
-   if(sigaction(signo, &act, &oact) < 0) return SIG_ERR;
-
-   return oact.sa_handler;
+   exit(0);
 }
 
 
-static void takesig(int sig){
+void takesig(int sig){
    int i, status;
    pid_t pid;
 
@@ -84,7 +67,6 @@ static void takesig(int sig){
                 break;
 
         case SIGTERM:
-        case SIGKILL:
                 quit = 1;
                 p_clean_exit();
                 break;
@@ -97,6 +79,7 @@ static void takesig(int sig){
                    if(quit == 0){
                       i = search_slot_by_pid(pid);
                       if(i >= 0){
+                         children[i].serial = i;
                          children[i].status = READY;
                          children[i].pid = child_make(&children[i]);
                       }
@@ -111,28 +94,190 @@ static void takesig(int sig){
 }
 
 
-static void child_sighup_handler(int sig){
+void child_sighup_handler(int sig){
    if(sig == SIGHUP){
       received_sighup = 1;
    }
 }
 
 
-void *get_in_addr(struct sockaddr *sa){
-   if(sa->sa_family == AF_INET) return &(((struct sockaddr_in*)sa)->sin_addr);
-   return &(((struct sockaddr_in6*)sa)->sin6_addr);
+int perform_checks(char *filename, struct session_data *sdata, struct data *data, struct parser_state *parser_state, struct config *cfg){
+
+   if(cfg->security_header[0] && parser_state->found_security_header == 0){
+      syslog(LOG_PRIORITY, "%s: discarding: missing security header", filename);
+      return ERR_DISCARDED;
+   }
+
+   char *arule = check_against_ruleset(data->archiving_rules, parser_state, sdata->tot_len, sdata->spam_message);
+
+   if(arule){
+      syslog(LOG_PRIORITY, "%s: discarding: archiving policy: *%s*", filename, arule);
+      return ERR_DISCARDED;
+   }
+
+
+   if(cfg->archive_only_mydomains == 1 && sdata->internal_sender == 0 && sdata->internal_recipient == 0){
+      syslog(LOG_PRIORITY, "%s: discarding: not on mydomains", filename);
+      return ERR_DISCARDED;
+   }
+
+   make_digests(sdata, cfg);
+
+   // A normal header is much bigger than 10 bytes. We get here for header-only
+   // messages without a Message-ID: line. I believe that no such message is valid, and
+   // it's a reasonable to discard it, and not allowing it to fill up the error directory.
+
+   if(sdata->hdr_len < 10){
+      syslog(LOG_PRIORITY, "%s: discarding: a header-only message without a Message-ID line", filename);
+      return ERR_DISCARDED;
+   }
+
+   int rc = process_message(sdata, parser_state, data, cfg);
+   unlink(parser_state->message_id_hash);
+
+   return rc;
 }
 
 
-static void child_main(struct child *ptr){
-   int new_sd;
-   char s[INET6_ADDRSTRLEN];
-   struct sockaddr_storage client_addr;
-   socklen_t addr_size;
+int process_email(char *filename, struct session_data *sdata, struct data *data, int size, struct config *cfg){
+   char tmpbuf[SMALLBUFSIZE];
+   char *status=S_STATUS_UNDEF;
+   char *p;
+   struct timezone tz;
+   struct timeval tv1, tv2;
+   struct parser_state parser_state;
+   struct counters counters;
+
+   gettimeofday(&tv1, &tz);
+
+   bzero(&counters, sizeof(counters));
+
+#ifdef HAVE_ANTIVIRUS
+   if(do_av_check(filename, cfg) == AVIR_VIRUS){
+      unlink(filename);
+      return OK;
+   }
+#endif
+
+   init_session_data(sdata, cfg);
+
+   sdata->tot_len = size;
+
+   snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
+
+   parser_state = parse_message(sdata, 1, data, cfg);
+
+   post_parse(sdata, &parser_state, cfg);
+
+   if(cfg->syslog_recipients == 1){
+      char *rcpt = parser_state.b_to;
+      do {
+         rcpt = split_str(rcpt, " ", tmpbuf, sizeof(tmpbuf)-1);
+
+         if(does_it_seem_like_an_email_address(tmpbuf) == 1){
+            syslog(LOG_PRIORITY, "%s: rcpt=%s", sdata->ttmpfile, tmpbuf);
+         }
+      } while(rcpt);
+   }
+
+   int rc = perform_checks(filename, sdata, data, &parser_state, cfg);
+   unlink(sdata->tmpframe);
+
+   remove_stripped_attachments(&parser_state);
+
+   if(rc == OK){
+      status = S_STATUS_STORED;
+      counters.c_rcvd = 1;
+      counters.c_size += sdata->tot_len;
+      counters.c_stored_size = sdata->stored_len;
+   }
+   else if(rc == ERR_EXISTS){
+      status = S_STATUS_DUPLICATE;
+      counters.c_duplicate = 1;
+      syslog(LOG_PRIORITY, "%s: discarding: duplicate message, id: %llu, message-id: %s", filename, sdata->duplicate_id, parser_state.message_id);
+   }
+   else if(rc == ERR_DISCARDED){
+      status = S_STATUS_DISCARDED;
+      counters.c_ignore = 1;
+   }
+   else {
+      status = S_STATUS_ERROR;
+
+      // move the file from piler/tmp/[0-xxx] dir to piler/error directory
+      p = strchr(filename, '/');
+      if(p)
+         p++;
+      else
+         p = filename;
+
+      snprintf(tmpbuf, sizeof(tmpbuf)-1, "%s/%s", ERROR_DIR, p);
+      if(rename(filename, tmpbuf) == 0)
+         syslog(LOG_PRIORITY, "%s: moved to %s", filename, tmpbuf);
+      else
+         syslog(LOG_PRIORITY, "%s: failed to moved to %s", filename, tmpbuf);
+   }
+
+   if(rc != ERR) unlink(filename);
+
+   update_counters(sdata, data, &counters, cfg);
+
+   gettimeofday(&tv2, &tz);
+
+   syslog(LOG_PRIORITY, "%s: %s, size=%d/%d, attachments=%d, reference=%s, "
+                        "message-id=%s, retention=%d, folder=%d, delay=%.4f, status=%s",
+                             filename, sdata->ttmpfile, sdata->tot_len, sdata->stored_len,
+                             parser_state.n_attachments, parser_state.reference, parser_state.message_id,
+                             parser_state.retention, data->folder, tvdiff(tv2,tv1)/1000000.0, status);
+
+   return rc;
+}
+
+
+int process_dir(char *directory, struct session_data *sdata, struct data *data, struct config *cfg){
+   DIR *dir;
+   struct dirent *de;
+   int tot_msgs=0;
+   char fname[SMALLBUFSIZE];
+   struct stat st;
+
+   dir = opendir(directory);
+   if(!dir){
+      syslog(LOG_PRIORITY, "cannot open directory: %s", directory);
+      return tot_msgs;
+   }
+
+   while((de = readdir(dir))){
+      if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
+
+      snprintf(fname, sizeof(fname)-1, "%s/%s", directory, de->d_name);
+
+      if(stat(fname, &st) == 0){
+         if(S_ISREG(st.st_mode) && process_email(fname, sdata, data, st.st_size, cfg) != ERR){
+            tot_msgs++;
+         }
+      }
+      else {
+         syslog(LOG_PRIORITY, "ERROR: cannot stat: %s", fname);
+      }
+   }
+
+   closedir(dir);
+
+   return tot_msgs;
+}
+
+
+void child_main(struct child *ptr){
+   struct session_data sdata;
+   char dir[TINYBUFSIZE];
+
+   /* open directory, then process its files, then sleep 1 sec, and repeat */
 
    ptr->messages = 0;
 
-   if(cfg.verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "child (pid: %d) started main()", getpid());
+   snprintf(dir, sizeof(dir)-1, "%d", ptr->serial);
+
+   if(cfg.verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "child (pid: %d, serial: %d) started main() working on '%s'", getpid(), ptr->serial, dir);
 
    while(1){
       if(received_sighup == 1){
@@ -140,34 +285,38 @@ static void child_main(struct child *ptr){
          break;
       }
 
-      ptr->status = READY;
-
-      addr_size = sizeof(client_addr);
-      new_sd = accept(sd, (struct sockaddr *)&client_addr, &addr_size);
-
-      if(new_sd == -1) continue;
-
-      ptr->status = BUSY;
-
-      inet_ntop(client_addr.ss_family, get_in_addr((struct sockaddr *)&client_addr), s, sizeof(s));
-
-      syslog(LOG_PRIORITY, "connection from %s", s);
-
-
       sig_block(SIGHUP);
-      ptr->messages += handle_smtp_session(new_sd, &data, &cfg);
-      sig_unblock(SIGHUP);
 
-      close(new_sd);
-
-      if(cfg.max_requests_per_child > 0 && ptr->messages >= cfg.max_requests_per_child){
-         if(cfg.verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "child (pid: %d) served enough: %d", getpid(), ptr->messages);
-         break;
+      int sphxopen = 0;
+      if(cfg.rtindex && open_sphx(&sdata, &cfg) == OK){
+         sphxopen = 1;
       }
 
-   }
+      if((cfg.rtindex == 0 || sphxopen == 1) && open_database(&sdata, &cfg) == OK){
+         ptr->messages += process_dir(dir, &sdata, &data, &cfg);
+         close_database(&sdata);
 
-   ptr->status = UNDEF;
+         if(cfg.rtindex){
+            close_sphx(&sdata);
+         }
+
+         sleep(1);
+      }
+      else {
+         syslog(LOG_PRIORITY, "ERROR: cannot open database");
+         sleep(10);
+      }
+
+      sig_unblock(SIGHUP);
+
+      // Let the child quit after processing max_requests_per_child messages
+
+      if(cfg.max_requests_per_child > 0 && ptr->messages >= cfg.max_requests_per_child){
+         if(cfg.verbosity >= _LOG_DEBUG)
+            syslog(LOG_PRIORITY, "child (pid: %d, serial: %d) served enough: %d", getpid(), ptr->messages, ptr->serial);
+         break;
+      }
+   }
 
 #ifdef HAVE_MEMCACHED
    memcached_shutdown(&(data.memc));
@@ -179,7 +328,7 @@ static void child_main(struct child *ptr){
 }
 
 
-static pid_t child_make(struct child *ptr){
+pid_t child_make(struct child *ptr){
    pid_t pid;
 
    if((pid = fork()) > 0) return pid;
@@ -208,10 +357,12 @@ int child_pool_create(){
       children[i].pid = 0;
       children[i].messages = 0;
       children[i].status = UNDEF;
+      children[i].serial = -1;
    }
 
    for(i=0; i<cfg.number_of_worker_processes; i++){
       children[i].status = READY;
+      children[i].serial = i;
       children[i].pid = child_make(&children[i]);
 
       if(children[i].pid == -1){
@@ -248,23 +399,19 @@ void kill_children(int sig){
 
 
 void p_clean_exit(){
-   if(sd != -1) close(sd);
-
    kill_children(SIGTERM);
 
-   free_rule(data.archiving_rules);
-   free_rule(data.retention_rules);
+   clearrules(data.archiving_rules);
+   clearrules(data.retention_rules);
+   clearrules(data.folder_rules);
+
+   clearhash(data.mydomains);
 
    syslog(LOG_PRIORITY, "%s has been terminated", PROGNAME);
 
    unlink(cfg.pidfile);
 
-#ifdef HAVE_STARTTLS
-   if(data.ctx){
-      SSL_CTX_free(data.ctx);
-      ERR_free_strings();
-   }
-#endif
+   if(data.dedup != MAP_FAILED) munmap(data.dedup, MAXCHILDREN*DIGEST_LENGTH*2);
 
    exit(1);
 }
@@ -276,33 +423,12 @@ void fatal(char *s){
 }
 
 
-#ifdef HAVE_STARTTLS
-int init_ssl(){
-
-   SSL_library_init();
-   SSL_load_error_strings();
-
-   data.ctx = SSL_CTX_new(SSLv23_server_method());
-
-   if(data.ctx == NULL){ syslog(LOG_PRIORITY, "SSL_CTX_new() failed"); return ERR; }
-
-   if(SSL_CTX_set_cipher_list(data.ctx, cfg.cipher_list) == 0){ syslog(LOG_PRIORITY, "failed to set cipher list: '%s'", cfg.cipher_list); return ERR; }
-
-   if(SSL_CTX_use_PrivateKey_file(data.ctx, cfg.pemfile, SSL_FILETYPE_PEM) != 1){ syslog(LOG_PRIORITY, "cannot load private key from %s", cfg.pemfile); return ERR; }
-
-   if(SSL_CTX_use_certificate_file(data.ctx, cfg.pemfile, SSL_FILETYPE_PEM) != 1){ syslog(LOG_PRIORITY, "cannot load certificate from %s", cfg.pemfile); return ERR; }
-
-   return OK;
-}
-#endif
-
-
 void initialise_configuration(){
    struct session_data sdata;
 
    cfg = read_config(configfile);
 
-   if(cfg.number_of_worker_processes < 5) cfg.number_of_worker_processes = 5;
+   if(cfg.number_of_worker_processes < 2) cfg.number_of_worker_processes = 2;
    if(cfg.number_of_worker_processes > MAXCHILDREN) cfg.number_of_worker_processes = MAXCHILDREN;
 
    if(strlen(cfg.username) > 1){
@@ -325,37 +451,34 @@ void initialise_configuration(){
    setlocale(LC_CTYPE, cfg.locale);
 
 
-   free_rule(data.archiving_rules);
-   free_rule(data.retention_rules);
+   clearrules(data.archiving_rules);
+   clearrules(data.retention_rules);
+   clearrules(data.folder_rules);
+
+   clearhash(data.mydomains);
 
    data.folder = 0;
    data.recursive_folder_names = 0;
-   data.archiving_rules = NULL;
-   data.retention_rules = NULL;
 
-   memset(data.starttls, 0, sizeof(data.starttls));
+   inithash(data.mydomains);
+   initrules(data.archiving_rules);
+   initrules(data.retention_rules);
+   initrules(data.folder_rules);
 
-#ifdef HAVE_STARTTLS
-   if(cfg.tls_enable > 0 && data.ctx == NULL && init_ssl() == OK){
-      snprintf(data.starttls, sizeof(data.starttls)-1, "250-STARTTLS\r\n");
-   }
-#endif
-
-   mysql_init(&(sdata.mysql));
-   mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout);
-   if(mysql_real_connect(&(sdata.mysql), cfg.mysqlhost, cfg.mysqluser, cfg.mysqlpwd, cfg.mysqldb, cfg.mysqlport, cfg.mysqlsocket, 0) == 0){
+   if(open_database(&sdata, &cfg) == ERR){
       syslog(LOG_PRIORITY, "cannot connect to mysql server");
       return;
    }
 
-   load_rules(&sdata, &(data.archiving_rules), SQL_ARCHIVING_RULE_TABLE);
-   load_rules(&sdata, &(data.retention_rules), SQL_RETENTION_RULE_TABLE);
+   load_rules(&sdata, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
+   load_rules(&sdata, data.retention_rules, SQL_RETENTION_RULE_TABLE);
+   load_rules(&sdata, data.folder_rules, SQL_FOLDER_RULE_TABLE);
 
    load_mydomains(&sdata, &data, &cfg);
 
    if(cfg.server_id > 0) insert_offset(&sdata, cfg.server_id);
 
-   mysql_close(&(sdata.mysql));
+   close_database(&sdata);
 
 
    syslog(LOG_PRIORITY, "reloaded config: %s", configfile);
@@ -367,9 +490,8 @@ void initialise_configuration(){
 
 
 int main(int argc, char **argv){
-   int i, rc, yes=1, daemonise=0;
-   char port_string[6];
-   struct addrinfo hints, *res;
+   int i, daemonise=0;
+   struct stat st;
 
 
    while((i = getopt(argc, argv, "c:dvVh")) > 0){
@@ -384,13 +506,17 @@ int main(int argc, char **argv){
                    break;
 
         case 'v' :
+                   printf("%s build %d\n", VERSION, get_build());
+                   return 0;
+
         case 'V' :
-                   printf("%s %s, build %d, Janos SUTO <sj@acts.hu>\n\n%s\n\n", PROGNAME, VERSION, get_build(), CONFIGURE_PARAMS);
+                   printf("%s %s, build %d, Janos SUTO <sj@acts.hu>\n\n%s\nMySQL client library version: %s\n", PROGNAME, VERSION, get_build(), CONFIGURE_PARAMS, mysql_get_client_info());
+                   get_extractor_list();
                    return 0;
 
         case 'h' :
         default  : 
-                   __fatal("usage: ...");
+                   usage();
       }
    }
 
@@ -398,11 +524,12 @@ int main(int argc, char **argv){
 
    data.folder = 0;
    data.recursive_folder_names = 0;
-   data.archiving_rules = NULL;
-   data.retention_rules = NULL;
-   data.ctx = NULL;
-   data.ssl = NULL;
-
+   inithash(data.mydomains);
+   initrules(data.archiving_rules);
+   initrules(data.retention_rules);
+   initrules(data.folder_rules);
+   data.dedup = MAP_FAILED;
+   data.import = NULL;
 
    initialise_configuration();
 
@@ -412,59 +539,37 @@ int main(int argc, char **argv){
    if(read_key(&cfg)) fatal(ERR_READING_KEY);
 
 
-   memset(&hints, 0, sizeof(hints));
-   hints.ai_family = AF_UNSPEC;
-   hints.ai_socktype = SOCK_STREAM;
-
-   snprintf(port_string, sizeof(port_string)-1, "%d", cfg.listen_port);
-
-   if((rc = getaddrinfo(cfg.listen_addr, port_string, &hints, &res)) != 0){
-      fprintf(stderr, "getaddrinfo for '%s': %s\n", cfg.listen_addr, gai_strerror(rc));
-      return 1;
-   }
-
-
-   if((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1)
-      fatal(ERR_OPEN_SOCKET);
-
-   if(setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)) == -1)
-      fatal(ERR_SET_SOCK_OPT);
-
-   if(bind(sd, res->ai_addr, res->ai_addrlen) == -1)
-      fatal(ERR_BIND_TO_PORT);
-
-   if(listen(sd, cfg.backlog) == -1)
-      fatal(ERR_LISTEN);
-
-
-   freeaddrinfo(res);
-
-
    if(drop_privileges(pwd)) fatal(ERR_SETUID);
 
+   if(stat(cfg.pidfile, &st) == 0) fatal(ERR_PID_FILE_EXISTS);
+
+   if(cfg.mmap_dedup_test == 1){
+      int dedupfd = open(MESSAGE_ID_DEDUP_FILE, O_RDWR);
+      if(dedupfd == -1) fatal(ERR_OPEN_DEDUP_FILE);
+
+      data.dedup = mmap(NULL, MAXCHILDREN*DIGEST_LENGTH*2, PROT_READ|PROT_WRITE, MAP_SHARED, dedupfd, 0);
+      close(dedupfd);
+
+      if(data.dedup == MAP_FAILED) syslog(LOG_INFO, "cannot mmap() %s, errno=%d", MESSAGE_ID_DEDUP_FILE, errno);
+   }
 
    syslog(LOG_PRIORITY, "%s %s, build %d starting", PROGNAME, VERSION, get_build());
 
-
 #if HAVE_DAEMON == 1
-   if(daemonise == 1) i = daemon(1, 0);
+   if(daemonise == 1 && daemon(1, 0) == -1) fatal(ERR_DAEMON);
 #endif
 
    write_pid_file(cfg.pidfile);
 
-
    child_pool_create();
 
    set_signal_handler(SIGCHLD, takesig);
    set_signal_handler(SIGTERM, takesig);
-   set_signal_handler(SIGKILL, takesig);
    set_signal_handler(SIGHUP, takesig);
 
-
    for(;;){ sleep(1); }
 
    p_clean_exit();
 
    return 0;
 }
-

src/piler.h

@@ -6,18 +6,21 @@
  #define _PILER_H
 
 #include <misc.h>
-#include <list.h>
 #include <parser.h>
 #include <errmsg.h>
 #include <smtpcodes.h>
 #include <decoder.h>
-#include <list.h>
+#include <hash.h>
 #include <rules.h>
 #include <defs.h>
 #include <tai.h>
 #include <sig.h>
 #include <av.h>
 #include <rules.h>
+#include <screen.h>
+#include <sql.h>
+#include <import.h>
+#include <smtp.h>
 #include <config.h>
 #include <unistd.h>
 
@@ -25,48 +28,62 @@
    #include "memc.h"
 #endif
 
-int read_key(struct __config *cfg);
+int read_key(struct config *cfg);
 void insert_offset(struct session_data *sdata, int server_id);
 
-int do_av_check(struct session_data *sdata, char *rcpttoemail, char *virusinfo, struct __data *data, struct __config *cfg);
+void tear_down_client(int n);
 
-int make_digests(struct session_data *sdata, struct __config *cfg);
+int do_av_check(char *filename, struct config *cfg);
+
+int make_digests(struct session_data *sdata, struct config *cfg);
 void digest_file(char *filename, char *digest);
-void digest_string(char *s, char *digest);
+void digest_string(char *digestname, char *s, char *digest);
+void create_md5_from_email_address(char *puf, char *md5buf);
 
-int handle_smtp_session(int new_sd, struct __data *data, struct __config *cfg);
+void remove_stripped_attachments(struct parser_state *state);
+int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
+void rollback(struct session_data *sdata, struct parser_state *state, uint64 id, struct config *cfg);
+int reimport_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
+int store_file(struct session_data *sdata, char *filename, int len, struct config *cfg);
+int remove_stored_message_files(struct session_data *sdata, struct parser_state *state, struct config *cfg);
+int store_attachments(struct session_data *sdata, struct parser_state *state, struct config *cfg);
+int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr);
 
-void remove_stripped_attachments(struct _state *state);
-int process_message(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg);
-int store_file(struct session_data *sdata, char *filename, int startpos, int len, struct __config *cfg);
-int remove_stored_message_files(struct session_data *sdata, struct _state *state, struct __config *cfg);
-int store_attachments(struct session_data *sdata, struct _state *state, struct __data *data, struct __config *cfg);
-int query_attachments(struct session_data *sdata, struct __data *data, struct ptr_array *ptr_arr, struct __config *cfg);
+struct config read_config(char *configfile);
 
-struct __config read_config(char *configfile);
+void createdir(char *path, uid_t uid, gid_t gid, mode_t mode);
+void check_and_create_directories(struct config *cfg, uid_t uid, gid_t gid);
 
-void check_and_create_directories(struct __config *cfg, uid_t uid, gid_t gid);
+void update_counters(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
 
-void update_counters(struct session_data *sdata, struct __data *data, struct __counters *counters, struct __config *cfg);
+int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct config *cfg);
+int file_from_archive_to_network(char *filename, int sd, int tls_enable, struct data *data, struct config *cfg);
 
-int retrieve_email_from_archive(struct session_data *sdata, struct __data *data, FILE *dest, struct __config *cfg);
+int get_folder_id(struct session_data *sdata, char *foldername, int parent_id);
+int add_new_folder(struct session_data *sdata, char *foldername, int parent_id);
 
-int prepare_a_mysql_statement(struct session_data *sdata, MYSQL_STMT **stmt, char *s);
+int store_index_data(struct session_data *sdata, struct parser_state *state, struct data *data, uint64 id, struct config *cfg);
 
-int import_message(char *filename, struct session_data *sdata, struct __data *data, struct __config *cfg);
-unsigned long get_folder_id(struct session_data *sdata, struct __data *data, char *foldername, int parent_id);
-unsigned long add_new_folder(struct session_data *sdata, struct __data *data, char *foldername, int parent_id);
+void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct config *cfg);
 
-int store_index_data(struct session_data *sdata, struct _state *state, struct __data *data, uint64 id, struct __config *cfg);
+int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct config *cfg);
 
-void extract_attachment_content(struct session_data *sdata, struct _state *state, char *filename, char *type, int *rec);
+void load_mydomains(struct session_data *sdata, struct data *data, struct config *cfg);
+int is_email_address_on_my_domains(char *email, struct data *data);
 
-int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct __config *cfg);
+int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg);
+void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections, char *reason);
+struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int max_connections, int socket);
+void write_envelope_addresses(struct smtp_session *session, struct config *cfg);
+void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg);
+void free_smtp_session(struct smtp_session *session);
 
-void load_mydomains(struct session_data *sdata, struct __data *data, struct __config *cfg);
-
-int create_prepared_statements(struct session_data *sdata, struct __data *data);
-void close_prepared_statements(struct __data *data);
+void child_sighup_handler(int sig);
+void child_main(struct child *ptr);
+pid_t child_make(struct child *ptr);
+int search_slot_by_pid(pid_t pid);
+void kill_children(int sig);
+void fatal(char *s);
+void initialise_configuration();
 
 #endif /* _PILER_H */
-

src/pileraget.c

@@ -19,8 +19,10 @@
 int main(int argc, char **argv){
    int readkey=1;
    char filename[SMALLBUFSIZE];
+#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
    struct stat st;
-   struct __config cfg;
+#endif
+   struct config cfg;
 
 
    if(argc < 3){
@@ -44,14 +46,14 @@ int main(int argc, char **argv){
       return 1;
    }
 
-   snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
+   snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, argv[1][24], argv[1][25], argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
+#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
    if(stat(filename, &st)){
       snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][RND_STR_LEN-6], argv[1][RND_STR_LEN-5], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
    }
+#endif
 
    retrieve_file_from_archive(filename, WRITE_TO_STDOUT, NULL, stdout, &cfg);
 
    return 0;
 }
-
-

src/pilerconf.c

@@ -12,13 +12,13 @@ extern char *optarg;
 extern int optind;
 
 
-void print_config_all(struct __config *cfg, char *key);
-void print_config(char *configfile, struct __config *cfg);
+void print_config_all(struct config *cfg, char *key);
+void print_config(char *configfile, struct config *cfg);
 
 int main(int argc, char **argv){
    int i, print_from_file=0;
    char *configfile=CONFIG_FILE, *query=NULL;
-   struct __config cfg;
+   struct config cfg;
 
    while((i = getopt(argc, argv, "c:q:nh?")) > 0){
        switch(i){
@@ -41,7 +41,7 @@ int main(int argc, char **argv){
                     break;
 
 
-         default  : 
+         default  :
                     break;
        }
    }

src/pilerexport.c

@@ -8,11 +8,13 @@
 #include <sys/time.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/mman.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <time.h>
 #include <locale.h>
 #include <syslog.h>
+#include <zip.h>
 #include <getopt.h>
 #include <piler.h>
 
@@ -21,19 +23,48 @@ extern char *optarg;
 extern int optind;
 
 int dryrun = 0;
+int exportall = 0;
+int verification_status = 0;
+int export_to_stdout = 0;
 char *query=NULL;
+int verbosity = 0;
+int max_matches = 1000;
+char *index_list = "main1,dailydelta1,delta1";
+struct passwd *pwd;
 regex_t regexp;
+char *zipfile = NULL;
+struct zip *zip = NULL;
+uint64 *zip_ids = NULL;
+int zip_counter = 0;
+int zip_batch = 2000;
+int max_files_in_export_dir = 0;
 
+int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg);
 
 
 void usage(){
-   printf("\nusage: pilerexport \n\n");
-   printf("    [-c|--config <config file>] \n");
-   printf("    -a|--start-date <YYYY.MM.DD> -b|--stop-date <YYYY.MM.DD> \n");
-   printf("    -f|--from <email@address> -r|--to <email@address>\n");
-   printf("    -s|--minsize <number> -S|--maxsize <number>\n");
-   printf("    -A|--all  -d|--dryrun \n");
-   printf("\n    use -A if you don't want to specify the start/stop time nor any from/to address\n\n");
+   printf("\nusage: pilerexport\n\n");
+
+   printf("    [-c <config file>]                Config file to use if not the default\n");
+   printf("    -a <start date>                   Start date in YYYY.MM.DD format\n");
+   printf("    -b <stop date>                    Stop date in YYYY.MM.DD format\n");
+   printf("    -f <email@address.com>            From address\n");
+   printf("    -r <email@address.com>            Recipient address\n");
+   printf("    -F <domain.com>                   From domain\n");
+   printf("    -R <domain.com>                   Recipient domain\n");
+   printf("    -s <size>                         Min. size\n");
+   printf("    -S <size>                         Max. size\n");
+   printf("    -w <where condition>              Where condition to pass to sphinx, eg. \"match('@subject: piler')\"\n");
+   printf("    -m <max. matches>                 Max. matches to apply to sphinx query (default: %d)\n", max_matches);
+   printf("    -i <index list>                   Sphinx indices to use  (default: %s)\n", index_list);
+#if LIBZIP_VERSION_MAJOR >= 1
+   printf("    -z <zip file>                     Write exported EML files to a zip file\n");
+   printf("    -Z <batch size>                   Zip batch size. Valid range: 10-10000, default: 2000\n");
+#endif
+   printf("    -A                                Export all emails from archive\n");
+   printf("    -D <max files>                    Max. number of files to put in a single directory, default: 2000\n");
+   printf("    -o                                Export emails to stdout\n");
+   printf("    -d                                Dry run\n");
 
    regfree(&regexp);
 
@@ -71,15 +102,12 @@ unsigned long convert_time(char *yyyymmdd, int h, int m, int s){
 
    tm.tm_mday = atoi(yyyymmdd);
 
-
-   tm.tm_isdst = -1;
-
    return mktime(&tm);
 }
 
 
 int append_email_to_buffer(char **buffer, char *email){
-   int len, arglen;
+   int arglen;
    char *s=NULL, emailaddress[SMALLBUFSIZE];
 
    snprintf(emailaddress, sizeof(emailaddress)-1, "'%s'", email);
@@ -91,7 +119,7 @@ int append_email_to_buffer(char **buffer, char *email){
       memcpy(*buffer, emailaddress, arglen);
    }
    else {
-      len = strlen(*buffer);
+      int len = strlen(*buffer);
       s = realloc(*buffer, len + arglen+2);
       if(!s){
          printf("malloc problem!\n");
@@ -109,88 +137,326 @@ int append_email_to_buffer(char **buffer, char *email){
 }
 
 
-int append_string_to_buffer(char **buffer, char *str){
-   int len, arglen;
-   char *s=NULL;
+uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *where_condition, uint64 last_id, int *num, struct config *cfg){
+   MYSQL_ROW row;
+   uint64 id=0;
+   char s[MAXBUFSIZE];
+   int rc=0;
 
-   arglen = strlen(str);
+   *num = 0;
 
-   if(!*buffer){
-      *buffer = malloc(arglen+1);
-      memset(*buffer, 0, arglen+1);
-      memcpy(*buffer, str, arglen);
-   }
-   else {
-      len = strlen(*buffer);
-      s = realloc(*buffer, len + arglen+1);
-      if(!s) return 1;
+   if(!where_condition) return id;
 
-      *buffer = s;
+   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE id IN (", SQL_METADATA_TABLE);
+   rc += append_string_to_buffer(&query, s);
 
-      memset(*buffer+len, 0, arglen+1);
-      memcpy(*buffer+len, str, arglen);
+   snprintf(s, sizeof(s)-1, "SELECT id FROM %s WHERE %s AND id > %llu ORDER BY id ASC LIMIT 0,%d", index_list, where_condition, last_id, max_matches);
+
+   if(dryrun){
+      printf("sphinx query: %s\n", s);
    }
 
-   return 0;
+   syslog(LOG_PRIORITY, "sphinx query: %s", s);
+
+   if(mysql_real_query(&(sdata2->mysql), s, strlen(s)) == 0){
+      MYSQL_RES *res = mysql_store_result(&(sdata2->mysql));
+      if(res != NULL){
+         while((row = mysql_fetch_row(res))){
+            id = strtoull(row[0], NULL, 10);
+            (*num)++;
+            rc += append_string_to_buffer(&query, row[0]);
+            rc += append_string_to_buffer(&query, ",");
+
+         }
+         mysql_free_result(res);
+
+         rc += append_string_to_buffer(&query, "-1)");
+      }
+   }
+
+   if(!rc) export_emails_matching_to_query(sdata, query, cfg);
+   else printf("error: append_string_to_buffer() in run_query()\n");
+
+   free(query);
+   query = NULL;
+
+
+   return id;
 }
 
 
-int export_emails_matching_to_query(struct session_data *sdata, struct __data *data, char *s, struct __config *cfg){
-   MYSQL_RES *res;
+uint64 get_total_found(struct session_data *sdata){
    MYSQL_ROW row;
-   FILE *f;
-   uint64 id, n=0;
-   char *digest=NULL, *bodydigest=NULL;
-   char filename[SMALLBUFSIZE];
-   int rc=0;
+   uint64 total_found=0;
 
-
-   rc = mysql_real_query(&(sdata->mysql), s, strlen(s));
-
-   if(rc == 0){
-      res = mysql_store_result(&(sdata->mysql));
-      if(res){
+   if(mysql_real_query(&(sdata->mysql), "SHOW META LIKE 'total_found'", 28) == 0){
+      MYSQL_RES *res = mysql_store_result(&(sdata->mysql));
+      if(res != NULL){
          while((row = mysql_fetch_row(res))){
-
-            id = strtoull(row[0], NULL, 10);
-            if(id > 0){
-               snprintf(sdata->ttmpfile, SMALLBUFSIZE-1, "%s", (char*)row[1]);
-               digest = (char*)row[2];
-               bodydigest = (char*)row[3];
-
-               if(dryrun == 0){
-
-                  snprintf(filename, sizeof(filename)-1, "%llu.eml", id);
-
-                  f = fopen(filename, "w");
-                  if(f){
-                     rc = retrieve_email_from_archive(sdata, data, f, cfg);
-                     fclose(f);
-
-                     n++;
-
-                     snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
-
-                     make_digests(sdata, cfg);
-
-                     if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0){
-                        printf("exported: %10llu\r", n); fflush(stdout);
-                     }
-                     else
-                        printf("verification FAILED. %s\n", filename);
-
-                  }
-                  else printf("cannot open: %s\n", filename);
-               }
-               else {
-                  printf("id:%llu\n", id);
-               }
-
-            }
+            total_found = strtoull(row[1], NULL, 10);
          }
          mysql_free_result(res);
       }
-      else rc = 1;
+   }
+
+   return total_found;
+}
+
+
+void export_emails_matching_id_list(struct session_data *sdata, struct session_data *sdata2, char *where_condition, struct config *cfg){
+   int n;
+   uint64 count=0, last_id=0, total_found=0;
+
+   last_id = run_query(sdata, sdata2, where_condition, last_id, &n, cfg);
+   count += n;
+
+   total_found = get_total_found(sdata2);
+
+   while(count < total_found){
+      last_id = run_query(sdata, sdata2, where_condition, last_id, &n, cfg);
+      count += n;
+   }
+
+}
+
+
+int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain, int minsize, int maxsize, unsigned long startdate, unsigned long stopdate){
+   char s[SMALLBUFSIZE];
+   int rc=0;
+
+   if(exportall == 1){
+      rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM ");
+      rc += append_string_to_buffer(&query, SQL_METADATA_TABLE);
+      rc += append_string_to_buffer(&query, " WHERE deleted=0 ");
+      return rc;
+   }
+
+   snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE deleted=0 ", SQL_MESSAGES_VIEW);
+
+   rc = append_string_to_buffer(&query, s);
+
+   if(from){
+      rc = append_string_to_buffer(&query, " AND ");
+
+      rc += append_string_to_buffer(&query, "`from` IN (");
+      rc += append_string_to_buffer(&query, from);
+      rc += append_string_to_buffer(&query, ")");
+
+      free(from);
+   }
+
+   if(to){
+      rc = append_string_to_buffer(&query, " AND ");
+
+      rc += append_string_to_buffer(&query, "`to` IN (");
+      rc += append_string_to_buffer(&query, to);
+      rc += append_string_to_buffer(&query, ")");
+
+      free(to);
+   }
+
+   if(fromdomain){
+      rc = append_string_to_buffer(&query, " AND ");
+
+      rc += append_string_to_buffer(&query, "`fromdomain` IN (");
+      rc += append_string_to_buffer(&query, fromdomain);
+      rc += append_string_to_buffer(&query, ")");
+
+      free(fromdomain);
+   }
+
+
+   if(todomain){
+      rc = append_string_to_buffer(&query, " AND ");
+
+      rc += append_string_to_buffer(&query, "`todomain` IN (");
+      rc += append_string_to_buffer(&query, todomain);
+      rc += append_string_to_buffer(&query, ")");
+
+      free(todomain);
+   }
+
+   if(minsize > 0){
+      rc = append_string_to_buffer(&query, " AND ");
+      snprintf(s, sizeof(s)-1, " `size` >= %d", minsize);
+      rc += append_string_to_buffer(&query, s);
+   }
+
+
+   if(maxsize > 0){
+      rc = append_string_to_buffer(&query, " AND ");
+      snprintf(s, sizeof(s)-1, " `size` <= %d", maxsize);
+      rc += append_string_to_buffer(&query, s);
+   }
+
+
+   if(startdate > 0){
+      rc = append_string_to_buffer(&query, " AND ");
+      snprintf(s, sizeof(s)-1, " `sent` >= %lu", startdate);
+      rc += append_string_to_buffer(&query, s);
+   }
+
+
+   if(stopdate > 0){
+      rc = append_string_to_buffer(&query, " AND ");
+      snprintf(s, sizeof(s)-1, " `sent` <= %lu", stopdate);
+      rc += append_string_to_buffer(&query, s);
+   }
+
+
+   rc += append_string_to_buffer(&query, " ORDER BY id ASC");
+
+   return rc;
+}
+
+#if LIBZIP_VERSION_MAJOR >= 1
+void zip_flush(){
+   zip_close(zip);
+
+   zip = NULL;
+   zip_counter = 0;
+
+   if(!zip_ids) return;
+
+   for(int i=0; i<zip_batch; i++){
+      if(*(zip_ids+i)){
+         char filename[SMALLBUFSIZE];
+         snprintf(filename, sizeof(filename)-1, "%llu.eml", *(zip_ids+i));
+         unlink(filename);
+      }
+   }
+
+   free(zip_ids);
+   zip_ids = NULL;
+}
+#endif
+
+int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg){
+   FILE *f;
+   uint64 id, n=0, dir_counter=0;
+   char digest[SMALLBUFSIZE], bodydigest[SMALLBUFSIZE];
+   char filename[SMALLBUFSIZE];
+   char export_subdir[SMALLBUFSIZE];
+   struct sql sql;
+   int errorp, rc=0, attachments;
+   unsigned long total_attachments=0;
+
+   if(prepare_sql_statement(sdata, &sql, s) == ERR) return ERR;
+
+
+   p_bind_init(&sql);
+
+   if(p_exec_stmt(sdata, &sql) == ERR) goto ENDE;
+
+
+
+   p_bind_init(&sql);
+
+   sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+   sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++;
+   sql.sql[sql.pos] = &digest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(digest)-2; sql.pos++;
+   sql.sql[sql.pos] = &bodydigest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(bodydigest)-2; sql.pos++;
+   sql.sql[sql.pos] = (char *)&attachments; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++;
+
+   p_store_results(&sql);
+
+   while(p_fetch_results(&sql) == OK){
+
+      if(id > 0){
+
+         if(dryrun == 0){
+
+            if(export_to_stdout){
+               printf("%s", PILEREXPORT_BEGIN_MARK);
+               rc = retrieve_email_from_archive(sdata, stdout, cfg);
+               continue;
+            }
+
+            if(max_files_in_export_dir > 0 && n % max_files_in_export_dir == 0){
+               dir_counter++;
+               snprintf(export_subdir, sizeof(export_subdir)-1, "export-%llu", dir_counter);
+               if(n > 0 && chdir("..")){
+                  p_clean_exit("error chdir(\"..\")", 1);
+               }
+
+               createdir(export_subdir, pwd->pw_uid, pwd->pw_gid, 0700);
+               if(chdir(export_subdir)){
+                  p_clean_exit("error chdir to export-* dir", 1);
+               }
+            }
+
+            snprintf(filename, sizeof(filename)-1, "%llu.eml", id);
+
+            f = fopen(filename, "w");
+            if(f){
+               rc = retrieve_email_from_archive(sdata, f, cfg);
+               fclose(f);
+
+               n++;
+
+               snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
+
+               make_digests(sdata, cfg);
+
+               if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0){
+                  printf("exported: %10llu\r", n); fflush(stdout);
+               }
+               else {
+                  printf("verification FAILED. %s\n", filename);
+                  verification_status = 1;
+               }
+
+               if(zipfile){
+               #if LIBZIP_VERSION_MAJOR >= 1
+                  // Open zip file if handler is NULL
+                  if(!zip){
+                     zip = zip_open(zipfile, ZIP_CREATE, &errorp);
+                     if(!zip){
+                        printf("error: error creating zip file=%s, error code=%d\n", zipfile, errorp);
+                        return ERR;
+                     }
+                  }
+
+                  if(!zip_ids) zip_ids = (uint64*) calloc(sizeof(uint64), zip_batch);
+
+                  if(!zip_ids){
+                     printf("calloc error for zip_ids\n");
+                     return ERR;
+                  }
+
+                  zip_source_t *zs = zip_source_file(zip, filename, 0, 0);
+                  if(zs && zip_file_add(zip, filename, zs, ZIP_FL_ENC_UTF_8) >= 0){
+                     *(zip_ids+zip_counter) = id;
+                     zip_counter++;
+                  } else {
+                     printf("error adding file %s: %s\n", filename, zip_strerror(zip));
+                     return ERR;
+                  }
+
+                  if(zip_counter == zip_batch){
+                     zip_flush();
+                  }
+               #endif
+               }
+            }
+            else printf("cannot open: %s\n", filename);
+         }
+         else {
+            total_attachments += attachments;
+            printf("id:%llu\n", id);
+         }
+
+      }
+
+   }
+
+   p_free_results(&sql);
+
+ENDE:
+   close_prepared_statement(&sql);
+
+   if(dryrun){
+      printf("attachments: %lu\n", total_attachments);
    }
 
    printf("\n");
@@ -200,16 +466,13 @@ int export_emails_matching_to_query(struct session_data *sdata, struct __data *d
 
 
 int main(int argc, char **argv){
-   int c, rc, exportall=0, minsize=0, maxsize=0;
-   int where_condition=0;
+   int minsize=0, maxsize=0;
    size_t nmatch=0;
    unsigned long startdate=0, stopdate=0;
    char *configfile=CONFIG_FILE;
-   char *to=NULL, *from=NULL;
-   char s[SMALLBUFSIZE];
-   struct session_data sdata;
-   struct __data data;
-   struct __config cfg;
+   char *to=NULL, *from=NULL, *todomain=NULL, *fromdomain=NULL, *where_condition=NULL;
+   struct session_data sdata, sdata2;
+   struct config cfg;
 
 
    if(regcomp(&regexp, "^([\\+a-z0-9_\\.@\\-]+)$", REG_ICASE | REG_EXTENDED)){
@@ -228,21 +491,29 @@ int main(int argc, char **argv){
             {"all",          no_argument,        0,  'A' },
             {"dry-run",      no_argument,        0,  'd' },
             {"dryrun",       no_argument,        0,  'd' },
+            {"stdout",       no_argument,        0,  'o' },
             {"help",         no_argument,        0,  'h' },
             {"version",      no_argument,        0,  'v' },
             {"from",         required_argument,  0,  'f' },
             {"to",           required_argument,  0,  'r' },
+            {"from-domain",  required_argument,  0,  'F' },
+            {"to-domain",    required_argument,  0,  'R' },
             {"start-date",   required_argument,  0,  'a' },
             {"stop-date",    required_argument,  0,  'b' },
-            {"id",           required_argument,  0,  'i' },
+            {"zip",          required_argument,  0,  'z' },
+            {"zip-batch",    required_argument,  0,  'Z' },
+            {"where-condition", required_argument,  0,  'w' },
+            {"max-files",    required_argument,  0,  'D' },
+            {"max-matches",  required_argument,  0,  'm' },
+            {"index-list",   required_argument,  0,  'i' },
             {0,0,0,0}
          };
 
       int option_index = 0;
 
-      c = getopt_long(argc, argv, "c:s:S:f:r:a:b:i:Adhv?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:D:oAdhv?", long_options, &option_index);
 #else
-      c = getopt(argc, argv, "c:s:S:f:r:a:b:i:Adhv?");
+      int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:D:oAdhv?");
 #endif
 
       if(c == -1) break;
@@ -273,7 +544,10 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&from, optarg);
+                    if(append_email_to_buffer(&from, optarg)){
+                       printf("error: append_email_to_buffer() for from\n");
+                       return 1;
+                    }
 
                     break;
 
@@ -284,10 +558,40 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&to, optarg);
+                    if(append_email_to_buffer(&to, optarg)){
+                       printf("error: append_email_to_buffer() for to\n");
+                       return 1;
+                    }
 
                     break;
 
+         case 'F' :
+
+                    if(regexec(&regexp, optarg, nmatch, NULL, 0)){
+                       printf("%s is not a valid domain name\n", optarg);
+                       break;
+                    }
+
+                    if(append_email_to_buffer(&fromdomain, optarg)){
+                       printf("error: append_email_to_buffer() for fromdomain\n");
+                       return 1;
+                    }
+
+                    break;
+
+         case 'R' :
+
+                    if(regexec(&regexp, optarg, nmatch, NULL, 0)){
+                       printf("%s is not a valid domain name\n", optarg);
+                       break;
+                    }
+
+                    if(append_email_to_buffer(&todomain, optarg)){
+                       printf("error: append_email_to_buffer() for todomain\n");
+                       return 1;
+                    }
+
+                    break;
 
          case 'a' :
                     startdate = convert_time(optarg, 0, 0, 0);
@@ -298,12 +602,39 @@ int main(int argc, char **argv){
                     stopdate = convert_time(optarg, 23, 59, 59);
                     break;
 
+         case 'w' :
+                    where_condition = optarg;
+                    break;
+
+         case 'm' :
+                    max_matches = atoi(optarg);
+                    break;
+
+         case 'i' :
+                    index_list = optarg;
+                    break;
+
+         case 'z':  zipfile = optarg;
+                    break;
+
+         case 'Z':  zip_batch = atoi(optarg);
+                    if(zip_batch < 10 || zip_batch > 10000)
+                       zip_batch = 2000;
+                    break;
+
+         case 'D':  max_files_in_export_dir = atoi(optarg);
+                    if(max_files_in_export_dir < 10 || max_files_in_export_dir > 100000)
+                       max_files_in_export_dir = 2000;
+                    break;
+
+         case 'o':
+                    export_to_stdout = 1;
+                    break;
 
          case 'd' :
                     dryrun = 1;
                     break;
 
-
          default  :
                     usage();
                     break;
@@ -312,117 +643,64 @@ int main(int argc, char **argv){
    }
 
 
-   if(from == NULL && to == NULL && startdate == 0 && stopdate == 0 && exportall == 0) usage();
+   if(from == NULL && to == NULL && fromdomain == NULL && todomain == NULL && where_condition == NULL && startdate == 0 && stopdate == 0 && exportall == 0) usage();
 
 
    regfree(&regexp);
 
+   if(!can_i_write_directory(NULL)) __fatal("cannot write current directory!");
+
    (void) openlog("pilerexport", LOG_PID, LOG_MAIL);
 
-   if(exportall == 1){
-      rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM ");
-      rc += append_string_to_buffer(&query, SQL_METADATA_TABLE);
-      goto GO;
-   }
-
-   snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE ", SQL_MESSAGES_VIEW);
-
-   rc = append_string_to_buffer(&query, s);
-
-   if(from){
-      rc += append_string_to_buffer(&query, "`from` IN (");
-      rc += append_string_to_buffer(&query, from);
-      rc += append_string_to_buffer(&query, ")"); 
-
-      free(from);
-
-      where_condition++;
-   }
-
-
-   if(to){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
-
-      rc += append_string_to_buffer(&query, "`to` IN (");
-      rc += append_string_to_buffer(&query, to);
-      rc += append_string_to_buffer(&query, ")");
-
-      free(to);
-
-      where_condition++;
-   }
-
-
-   if(minsize > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
-      snprintf(s, sizeof(s)-1, " `size` >= %d", minsize);
-      rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
-   }
-
-
-   if(maxsize > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
-      snprintf(s, sizeof(s)-1, " `size` <= %d", maxsize);
-      rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
-   }
-
-
-   if(startdate > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
-      snprintf(s, sizeof(s)-1, " `sent` >= %ld", startdate);
-      rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
-   }
-
-
-   if(stopdate > 0){
-      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
-      snprintf(s, sizeof(s)-1, " `sent` <= %ld", stopdate);
-      rc += append_string_to_buffer(&query, s);
-
-      where_condition++;
-   }
-
-
-   rc += append_string_to_buffer(&query, " ORDER BY id ASC");
-
-
-GO:
-   if(rc) p_clean_exit("malloc problem building query", 1);
-
 
    cfg = read_config(configfile);
 
 
    if(read_key(&cfg)) p_clean_exit(ERR_READING_KEY, 1);
 
+   if(strlen(cfg.username) > 1){
+      pwd = getpwnam(cfg.username);
+      if(!pwd) __fatal(ERR_NON_EXISTENT_USER);
+   }
+
 
    init_session_data(&sdata, &cfg);
 
 
-   mysql_init(&(sdata.mysql));
-   mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout);
-   if(mysql_real_connect(&(sdata.mysql), cfg.mysqlhost, cfg.mysqluser, cfg.mysqlpwd, cfg.mysqldb, cfg.mysqlport, cfg.mysqlsocket, 0) == 0){
+   if(open_database(&sdata, &cfg) == ERR){
       p_clean_exit("cannot connect to mysql server", 1);
    }
 
-   mysql_real_query(&(sdata.mysql), "SET NAMES utf8", strlen("SET NAMES utf8"));
-   mysql_real_query(&(sdata.mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8"));
 
+   if(where_condition){
 
-   rc = export_emails_matching_to_query(&sdata, &data, query, &cfg);
+      init_session_data(&sdata2, &cfg);
 
-   free(query);
+      strcpy(cfg.mysqlhost, "127.0.0.1");
+      cfg.mysqlport = 9306;
+      cfg.mysqlsocket[0] = '\0';
 
+      if(open_database(&sdata2, &cfg) == ERR){
+         p_clean_exit("cannot connect to 127.0.0.1:9306", 1);
+      }
 
-   mysql_close(&(sdata.mysql));
+      export_emails_matching_id_list(&sdata, &sdata2, where_condition, &cfg);
 
-   return 0;
+      close_database(&sdata2);
+   }
+   else {
+      if(build_query_from_args(from, to, fromdomain, todomain, minsize, maxsize, startdate, stopdate) > 0) p_clean_exit("malloc problem building query", 1);
+      export_emails_matching_to_query(&sdata, query, &cfg);
+      free(query);
+   }
+
+   close_database(&sdata);
+
+   if(zipfile){
+   #if LIBZIP_VERSION_MAJOR >= 1
+      zip_flush();
+   #endif
+   }
+
+   return verification_status;
 }
-
-

src/pilerget.c

@@ -19,8 +19,7 @@
 int main(int argc, char **argv){
    int readkey=1;
    struct session_data sdata;
-   struct __data data;
-   struct __config cfg;
+   struct config cfg;
 
 
    if(argc < 2){
@@ -32,7 +31,10 @@ int main(int argc, char **argv){
 
    cfg = read_config(CONFIG_FILE);
 
-   if(argc >= 3) readkey = 0;
+   if(argc >= 3){
+      readkey = 0;
+      cfg.encrypt_messages = 0;
+   }
 
    if(readkey == 1 && read_key(&cfg)){
       printf("%s\n", ERR_READING_KEY);
@@ -40,25 +42,15 @@ int main(int argc, char **argv){
    }
 
 
-   mysql_init(&(sdata.mysql));
-   mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout);
-   if(mysql_real_connect(&(sdata.mysql), cfg.mysqlhost, cfg.mysqluser, cfg.mysqlpwd, cfg.mysqldb, cfg.mysqlport, cfg.mysqlsocket, 0) == 0){
-      printf("cannot connect to mysql server\n");
-      return 0;
-   }
-
-   mysql_real_query(&(sdata.mysql), "SET NAMES utf8", strlen("SET NAMES utf8"));
-   mysql_real_query(&(sdata.mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8"));
+   if(open_database(&sdata, &cfg) == ERR) return 0;
 
 
    snprintf(sdata.ttmpfile, SMALLBUFSIZE-1, "%s", argv[1]);
    snprintf(sdata.filename, SMALLBUFSIZE-1, "%s", sdata.ttmpfile);
-   retrieve_email_from_archive(&sdata, &data, stdout, &cfg);
+   retrieve_email_from_archive(&sdata, stdout, &cfg);
 
 
-   mysql_close(&(sdata.mysql));
+   close_database(&sdata);
 
    return 0;
 }
-
-

src/pilerimport.c

@@ -30,375 +30,97 @@
 extern char *optarg;
 extern int optind;
 
-int quiet=0;
-int remove_after_successful_import = 0;
-
-
-int connect_to_imap_server(int sd, int *seq, char *username, char *password, int port, struct __data *data, int use_ssl);
-int list_folders(int sd, int *seq, char *folders, int foldersize, int use_ssl, struct __data *data);
-int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sdata, struct __data *data, int use_ssl, struct __config *cfg);
-int connect_to_pop3_server(int sd, char *username, char *password, int port, struct __data *data, int use_ssl);
-int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data, int use_ssl, struct __config *cfg);
-void close_connection(int sd, struct __data *data, int use_ssl);
-
-
-int import_from_mailbox(char *mailbox, struct session_data *sdata, struct __data *data, struct __config *cfg){
-   FILE *F, *f=NULL;
-   int rc=ERR, tot_msgs=0, ret=OK;
-   char buf[MAXBUFSIZE], fname[SMALLBUFSIZE];
-   time_t t;
-
-
-   F = fopen(mailbox, "r");
-   if(!F){
-      printf("cannot open mailbox: %s\n", mailbox);
-      return rc;
-   }
-
-   t = time(NULL);
-
-   while(fgets(buf, sizeof(buf)-1, F)){
-
-      if(buf[0] == 'F' && buf[1] == 'r' && buf[2] == 'o' && buf[3] == 'm' && buf[4] == ' '){
-         tot_msgs++;
-         if(f){
-            fclose(f);
-            f = NULL;
-            rc = import_message(fname, sdata, data, cfg);
-            if(rc == ERR){
-               printf("error importing: '%s'\n", fname);
-               ret = ERR;
-            }
-            else unlink(fname);
-
-            if(quiet == 0) printf("processed: %7d\r", tot_msgs); fflush(stdout);
-         }
-
-         snprintf(fname, sizeof(fname)-1, "%ld-%d", t, tot_msgs);
-         f = fopen(fname, "w+");
-         continue;
-      }
-
-      if(f) fprintf(f, "%s", buf);
-   }
-
-   if(f){
-      fclose(f);
-      rc = import_message(fname, sdata, data, cfg);
-      if(rc == ERR){
-         printf("error importing: '%s'\n", fname);
-         ret = ERR;
-      }
-      else unlink(fname);
-
-      if(quiet == 0) printf("processed: %7d\r", tot_msgs); fflush(stdout);
-   }
-
-   fclose(F);
-
-   return ret;
-}
-
-
-int import_mbox_from_dir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg){
-   DIR *dir;
-   struct dirent *de;
-   int rc=ERR, ret=OK, i=0;
-   unsigned long folder;
-   char fname[SMALLBUFSIZE];
-   struct stat st;
-
-   dir = opendir(directory);
-   if(!dir){
-      printf("cannot open directory: %s\n", directory);
-      return ERR;
-   }
-
-
-   while((de = readdir(dir))){
-      if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
-
-      snprintf(fname, sizeof(fname)-1, "%s/%s", directory, de->d_name);
-
-      if(stat(fname, &st) == 0){
-         if(S_ISDIR(st.st_mode)){
-            folder = data->folder;
-            rc = import_mbox_from_dir(fname, sdata, data, tot_msgs, cfg);
-            data->folder = folder;
-            if(rc == ERR) ret = ERR;
-         }
-         else {
-
-            if(S_ISREG(st.st_mode)){
-               if(i == 0 && data->recursive_folder_names == 1){
-                  folder = get_folder_id(sdata, data, fname, data->folder);
-                  if(folder == 0){
-                     folder = add_new_folder(sdata, data, fname, data->folder);
-
-                     if(folder == 0){
-                        printf("error: cannot get/add folder '%s' to parent id: %d\n", fname, data->folder);
-                        return ERR;
-                     }
-                     else {
-                        data->folder = folder;
-                     }
-                  }
-
-               }
-
-               rc = import_from_mailbox(fname, sdata, data, cfg);
-               if(rc == OK) (*tot_msgs)++;
-               else ret = ERR;
-
-               i++;
-            }
-            else {
-               printf("%s is not a file\n", fname);
-            }
-
-         }
-      }
-      else {
-         printf("cannot stat() %s\n", fname);
-      }
-
-   }
-   closedir(dir);
-
-   return ret;
-}
-
-
-int import_from_maildir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg){
-   DIR *dir;
-   struct dirent *de;
-   int rc=ERR, ret=OK, i=0;
-   unsigned long folder;
-   char *p, fname[SMALLBUFSIZE];
-   struct stat st;
-
-   dir = opendir(directory);
-   if(!dir){
-      printf("cannot open directory: %s\n", directory);
-      return ERR;
-   }
-
-
-   while((de = readdir(dir))){
-      if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
-
-      snprintf(fname, sizeof(fname)-1, "%s/%s", directory, de->d_name);
-
-      if(stat(fname, &st) == 0){
-         if(S_ISDIR(st.st_mode)){
-            folder = data->folder;
-            rc = import_from_maildir(fname, sdata, data, tot_msgs, cfg);
-            data->folder = folder;
-            if(rc == ERR) ret = ERR;
-         }
-         else {
-
-            if(S_ISREG(st.st_mode)){
-               if(i == 0 && data->recursive_folder_names == 1){
-                  p = strrchr(directory, '/');
-                  if(p) p++;
-                  else {
-                     printf("invalid directory name: '%s'\n", directory);
-                     return ERR;
-                  }
-
-                  folder = get_folder_id(sdata, data, p, data->folder);
-                  if(folder == 0){
-                     folder = add_new_folder(sdata, data, p, data->folder);
-
-                     if(folder == 0){
-                        printf("error: cannot get/add folder '%s' to parent id: %d\n", p, data->folder);
-                        return ERR;
-                     }
-                     else {
-                        data->folder = folder;
-                     }
-                  }
-
-               }
-
-               rc = import_message(fname, sdata, data, cfg);
-               if(rc == OK) (*tot_msgs)++;
-               else {
-                  printf("error importing: '%s'\n", fname);
-                  ret = ERR;
-               }
- 
-               if(remove_after_successful_import == 1 && ret != ERR) unlink(fname);
-
-               i++;
-
-               if(quiet == 0) printf("processed: %7d\r", *tot_msgs); fflush(stdout);
-            }
-            else {
-               printf("%s is not a file\n", fname);
-            }
-
-         }
-      }
-      else {
-         printf("cannot stat() %s\n", fname);
-      }
-
-   }
-   closedir(dir);
-
-   return ret;
-}
-
-
-int import_from_imap_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, char *skiplist, struct __config *cfg){
-   int rc=ERR, ret=OK, sd, seq=1, skipmatch, use_ssl=0;
-   char *p, puf[SMALLBUFSIZE];
-   char muf[SMALLBUFSIZE];
-   char folders[MAXBUFSIZE];
-   char port_string[6];
-   struct addrinfo hints, *res;
-
-   snprintf(port_string, sizeof(port_string)-1, "%d", port);
-
-   memset(&hints, 0, sizeof(hints));
-   hints.ai_family = AF_UNSPEC;
-   hints.ai_socktype = SOCK_STREAM;
-
-   if((rc = getaddrinfo(server, port_string, &hints, &res)) != 0){
-      printf("getaddrinfo for '%s': %s\n", server, gai_strerror(rc));
-      return ERR;
-   }
-
-   if(port == 993) use_ssl = 1;
-
-
-   if((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
-      printf("cannot create socket\n");
-      ret = ERR;
-      goto ENDE_IMAP;
-   }
-
-   if(connect(sd, res->ai_addr, res->ai_addrlen) == -1){
-      printf("connect()\n");
-      ret = ERR;
-      goto ENDE_IMAP;
-   }
-
-   if(connect_to_imap_server(sd, &seq, username, password, port, data, use_ssl) == ERR){
-      close(sd);
-      ret = ERR;
-      goto ENDE_IMAP;
-   }
-
-
-   list_folders(sd, &seq, &folders[0], sizeof(folders), use_ssl, data);
-
-
-   p = &folders[0];
-   do {
-      memset(puf, 0, sizeof(puf));
-      p = split(p, '\n', puf, sizeof(puf)-1);
-
-      if(strlen(puf) < 1) continue;
-
-      skipmatch = 0;
-
-      if(skiplist && strlen(skiplist) > 0){
-         snprintf(muf, sizeof(muf)-1, "%s,", puf);
-         if(strstr(skiplist, muf)) skipmatch = 1;
-      }
-
-      if(skipmatch == 1){
-         if(quiet == 0) printf("SKIPPING FOLDER: %s\n", puf);
-         continue;
-      }
-
-      if(quiet == 0) printf("processing folder: %s... ", puf);
-
-      if(process_imap_folder(sd, &seq, puf, sdata, data, use_ssl, cfg) == ERR) ret = ERR;
-
-   } while(p);
-
-
-   close_connection(sd, data, use_ssl);
-
-ENDE_IMAP:
-   freeaddrinfo(res);
-
-   return ret;
-}
-
-
-int import_from_pop3_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, struct __config *cfg){
-   int rc, ret=OK, sd, use_ssl=0;
-   char port_string[6];
-   struct addrinfo hints, *res;
-
-   snprintf(port_string, sizeof(port_string)-1, "%d", port);
-
-   memset(&hints, 0, sizeof(hints));
-   hints.ai_family = AF_UNSPEC;
-   hints.ai_socktype = SOCK_STREAM;
-
-   if((rc = getaddrinfo(server, port_string, &hints, &res)) != 0){
-      printf("getaddrinfo for '%s': %s\n", server, gai_strerror(rc));
-      return ERR;
-   }
-
-   if(port == 995) use_ssl = 1;
-
-   if((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
-      printf("cannot create socket\n");
-      ret = ERR;
-      goto ENDE_POP3;
-   }
-
-   if(connect(sd, res->ai_addr, res->ai_addrlen) == -1){
-      printf("connect()\n");
-      ret = ERR;
-      goto ENDE_POP3;
-   }
-
-
-   if(connect_to_pop3_server(sd, username, password, port, data, use_ssl) == ERR){
-      close(sd);
-      ret = ERR;
-      goto ENDE_POP3;
-   }
-
-   if(process_pop3_emails(sd, sdata, data, use_ssl, cfg) == ERR) ret = ERR;
-
-   close_connection(sd, data, use_ssl);
-
-ENDE_POP3:
-   freeaddrinfo(res);
-
-   return ret;
-}
-
 
 void usage(){
-   printf("usage: pilerimport [-c <config file>] -e <eml file> | -m <mailbox file> | -d <directory> | -i <imap server> | -K <pop3 server> | -u <imap username> -p <imap password> -P <imap port> [-F <foldername>] [-R] [-r] [-q]\n");
+   printf("\nusage: pilerimport\n\n");
+   printf("    [-c <config file>]                Config file to use if not the default\n");
+   printf("    -e <eml file>                     EML file to import\n");
+   printf("    -m <mailbox file>                 Mbox file to import\n");
+   printf("    -d <dir>                          Directory with EML files to import\n");
+   printf("    -i <imap server>                  IMAP server to connect\n");
+   printf("    -K <pop3 server>                  POP3 server to connect\n");
+   printf("    -u <username>                     Username for imap/pop3 import\n");
+   printf("    -p <password>                     Password for imap/pop3 import\n");
+   printf("    -P <port>                         Port for imap/pop3 import (default: 143/110\n");
+   printf("    -t <timeout>                      Timeout in sec for imap/pop3 import\n");
+   printf("    -x <folder1,folder2,....folderN,> Comma separated list of imap folders to skip. Add the trailing comma!\n");
+   printf("    -f <imap folder>                  IMAP folder name to import\n");
+   printf("    -g <imap folder>                  Move email after import to this IMAP folder\n");
+   printf("    -F <folder>                       Piler folder name to assign to this import\n");
+   printf("    -R                                Assign IMAP folder names as Piler folder names\n");
+   printf("    -b <batch limit>                  Import only this many emails\n");
+   printf("    -s <start position>               Start importing POP3 emails from this position\n");
+   printf("    -j <failed folder>                Move failed to import emails to this folder\n");
+   printf("    -a <recipient>                    Add recipient to the To:/Cc: list\n");
+   printf("    -T <id>                           Update import table at id=<id>\n");
+   printf("    -Z <ms>                           Delay Z milliseconds in between emails being imported\n");
+   printf("    -D                                Dry-run, do not import anything\n");
+   printf("    -y                                Read pilerexport data from stdin\n");
+   printf("    -o                                Only download emails for POP3/IMAP import\n");
+   printf("    -r                                Remove imported emails\n");
+   printf("    -q                                Quiet mode\n");
+   printf("    -A <timestamp>                    Import emails sent after this timestamp\n");
+   printf("    -B <timestamp>                    Import emails sent before this timestamp\n");
+
    exit(0);
 }
 
 
 int main(int argc, char **argv){
-   int i, c, rc=0, n_mbox=0, tot_msgs=0, port=143;
-   char *configfile=CONFIG_FILE, *emlfile=NULL, *mboxdir=NULL, *mbox[MBOX_ARGS], *directory=NULL;
-   char *imapserver=NULL, *pop3server=NULL, *username=NULL, *password=NULL, *skiplist=SKIPLIST, *folder=NULL;
+   int i, n_mbox=0, read_from_pilerexport=0;
+   char *configfile=CONFIG_FILE, *mbox[MBOX_ARGS], *directory=NULL;
+   char puf[SMALLBUFSIZE], *imapserver=NULL, *pop3server=NULL;
    struct session_data sdata;
-   struct __config cfg;
-   struct __data data;
+   struct config cfg;
+   struct data data;
+   struct import import;
+   struct net net;
+   struct counters counters;
+
+   bzero(&counters, sizeof(counters));
 
    for(i=0; i<MBOX_ARGS; i++) mbox[i] = NULL;
 
+   srand(getpid());
+
    data.folder = 0;
    data.recursive_folder_names = 0;
-   data.archiving_rules = NULL;
-   data.retention_rules = NULL;
+   data.quiet = 0;
+
+
+   import.import_job_id = import.total_messages = import.total_size = import.processed_messages = import.batch_processing_limit = 0;
+   import.started = import.updated = import.finished = import.remove_after_import = 0;
+   import.extra_recipient = import.move_folder = import.failed_folder = NULL;
+   import.start_position = 1;
+   import.download_only = 0;
+   import.dryrun = 0;
+   import.port = 143;
+   import.server = NULL;
+   import.username = NULL;
+   import.password = NULL;
+   import.database = NULL;
+   import.skiplist = SKIPLIST;
+   import.folder_imap = NULL;
+   memset(import.filename, 0, SMALLBUFSIZE);
+   import.mboxdir = NULL;
+   import.tot_msgs = 0;
+   import.table_id = 0;
+   import.folder = NULL;
+   import.delay = 0;
+   import.after = 0;
+   import.before = 0;
+
+   data.import = &import;
+
+   net.socket = -1;
+   net.timeout = 30;
+
+   data.net = &net;
+
+   inithash(data.mydomains);
+   initrules(data.archiving_rules);
+   initrules(data.retention_rules);
+   initrules(data.folder_rules);
 
    while(1){
 
@@ -417,18 +139,32 @@ int main(int argc, char **argv){
             {"port",         required_argument,  0,  'P' },
             {"skiplist",     required_argument,  0,  'x' },
             {"folder",       required_argument,  0,  'F' },
-            {"quiet",        required_argument,  0,  'q' },
-            {"recursive",    required_argument,  0,  'R' },
-            {"remove-after-import",    required_argument,  0,  'r' },
+            {"folder_imap",  required_argument,  0,  'f' },
+            {"add-recipient",required_argument,  0,  'a' },
+            {"batch-limit",  required_argument,  0,  'b' },
+            {"timeout",      required_argument,  0,  't' },
+            {"start-position",required_argument,  0,  's' },
+            {"table-id",     required_argument,  0,  'T' },
+            {"delay",        required_argument,  0,  'Z' },
+            {"quiet",        no_argument,        0,  'q' },
+            {"recursive",    no_argument,        0,  'R' },
+            {"remove-after-import",no_argument,  0,  'r' },
+            {"failed-folder",  required_argument,  0,  'j' },
+            {"move-folder",  required_argument,  0,  'g' },
+            {"after",        required_argument,  0,  'A' },
+            {"before",       required_argument,  0,  'B' },
+            {"only-download",no_argument,        0,  'o' },
+            {"read-from-export",no_argument,     0,  'y' },
+            {"dry-run",      no_argument,        0,  'D' },
             {"help",         no_argument,        0,  'h' },
             {0,0,0,0}
          };
 
       int option_index = 0;
 
-      c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:Rrqh?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:A:B:yDRroqh?", long_options, &option_index);
 #else
-      c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:Rrqh?");
+      int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:A:B:yDRroqh?");
 #endif
 
       if(c == -1) break;
@@ -440,7 +176,7 @@ int main(int argc, char **argv){
                     break;
 
          case 'e' :
-                    emlfile = optarg;
+                    snprintf(data.import->filename, SMALLBUFSIZE-1, "%s", optarg);
                     break;
 
          case 'd' :
@@ -457,36 +193,42 @@ int main(int argc, char **argv){
                     break;
 
          case 'M' :
-                    mboxdir = optarg;
+                    data.import->mboxdir = optarg;
                     break;
 
          case 'i' :
                     imapserver = optarg;
+                    data.import->server = optarg;
                     break;
 
          case 'K' :
                     pop3server = optarg;
-                    if(port == 143) port = 110;
+                    data.import->server = optarg;
+                    if(data.import->port == 143) data.import->port = 110;
                     break;
 
          case 'u' :
-                    username = optarg;
+                    data.import->username = optarg;
                     break;
 
          case 'p' :
-                    password = optarg;
+                    data.import->password = optarg;
                     break;
 
          case 'P' :
-                    port = atoi(optarg);
+                    data.import->port = atoi(optarg);
                     break;
 
          case 'x' :
-                    skiplist = optarg;
+                    data.import->skiplist = optarg;
                     break;
 
          case 'F' :
-                    folder = optarg;
+                    data.import->folder = optarg;
+                    break;
+
+         case 'f' :
+                    data.import->folder_imap = optarg;
                     break;
 
          case 'R' :
@@ -494,11 +236,78 @@ int main(int argc, char **argv){
                     break;
 
          case 'r' :
-                    remove_after_successful_import = 1;
+                    data.import->remove_after_import = 1;
+                    break;
+
+         case 'g' :
+                    data.import->move_folder = optarg;
+                    break;
+
+         case 'j' :
+                    data.import->failed_folder = optarg;
+                    break;
+
+         case 'o' :
+                    data.import->download_only = 1;
+                    data.import->dryrun = 1;
+                    break;
+
+         case 'b' :
+                    data.import->batch_processing_limit = atoi(optarg);
+                    break;
+
+         case 't' :
+                    data.import->timeout = atoi(optarg);
+                    break;
+
+         case 's' :
+                    if(atoi(optarg) < 1){
+                       printf("invalid start position: %s\n", optarg);
+                       return -1;
+                    }
+
+                    data.import->start_position = atoi(optarg);
+                    break;
+
+         case 'a' :
+                    snprintf(puf, sizeof(puf)-1, "%s ", optarg);
+                    data.import->extra_recipient = puf;
+                    break;
+
+         case 'T' :
+                    if(atoi(optarg) < 1){
+                       printf("invalid import table id: %s\n", optarg);
+                       return -1;
+                    }
+
+                    data.import->table_id = atoi(optarg);
+                    break;
+
+         case 'Z' :
+                    if(atoi(optarg) < 1){
+                       printf("invalid delay value: %s\n", optarg);
+                       return -1;
+                    }
+
+                    data.import->delay = atoi(optarg);
+                    break;
+
+         case 'y' :
+                    read_from_pilerexport = 1;
+                    break;
+
+         case 'D' :
+                    data.import->dryrun = 1;
                     break;
 
          case 'q' :
-                    quiet = 1;
+                    data.quiet = 1;
+                    break;
+
+         case 'A' : data.import->after = atol(optarg);
+                    break;
+
+         case 'B' : data.import->before = atol(optarg);
                     break;
 
          case 'h' :
@@ -507,85 +316,105 @@ int main(int argc, char **argv){
                     break;
 
 
-         default  : 
+         default  :
                     break;
        }
    }
 
 
+   if(!mbox[0] && !data.import->mboxdir && !data.import->filename[0] && !directory && !imapserver && !pop3server && !read_from_pilerexport) usage();
 
-   if(!mbox[0] && !mboxdir && !emlfile && !directory && !imapserver && !pop3server) usage();
+   if(data.import->failed_folder && !can_i_write_directory(data.import->failed_folder)){
+      printf("cannot write failed directory '%s'\n", data.import->failed_folder);
+      return ERR;
+   }
+
+   if(!can_i_write_directory(NULL)) __fatal("cannot write current directory!");
 
    cfg = read_config(configfile);
 
+   memset(cfg.security_header, 0, MAXVAL);
+
+   if((data.recursive_folder_names == 1 || data.import->folder) && cfg.enable_folders == 0){
+      printf("please set enable_folders=1 in piler.conf to use the folder options\n");
+      return ERR;
+   }
+
+   /* make sure we don't discard messages without a valid Message-Id when importing manually */
+   cfg.archive_emails_not_having_message_id = 1;
+
+   /* The mmap_dedup_test feature is expected to work with the piler daemon only */
+   cfg.mmap_dedup_test = 0;
+
    if(read_key(&cfg)){
       printf("%s\n", ERR_READING_KEY);
       return ERR;
    }
 
-   mysql_init(&(sdata.mysql));
-   mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout);
-   if(mysql_real_connect(&(sdata.mysql), cfg.mysqlhost, cfg.mysqluser, cfg.mysqlpwd, cfg.mysqldb, cfg.mysqlport, cfg.mysqlsocket, 0) == 0){
-      printf("error: cant connect to mysql server\n");
-      return ERR;
-   }
+   /* enable using the extra email address */
+   if(data.import->extra_recipient) cfg.process_rcpt_to_addresses = 1;
 
-   mysql_real_query(&(sdata.mysql), "SET NAMES utf8", strlen("SET NAMES utf8"));
-   mysql_real_query(&(sdata.mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8"));
+   if(open_database(&sdata, &cfg) == ERR) return 0;
 
-   if(create_prepared_statements(&sdata, &data) == ERR){
-      rc = ERR;
-      goto ENDE;
-   }
+   if(cfg.rtindex && open_sphx(&sdata, &cfg) == ERR) return 0;
 
    setlocale(LC_CTYPE, cfg.locale);
 
    (void) openlog("pilerimport", LOG_PID, LOG_MAIL);
 
-   if(folder){
-      data.folder = get_folder_id(&sdata, &data, folder, 0);
+#ifdef HAVE_MEMCACHED
+   memcached_init(&(data.memc), cfg.memcached_servers, 11211);
+#endif
 
-      if(data.folder == 0){
-         data.folder = add_new_folder(&sdata, &data, folder, 0);
+   if(data.import->folder){
+      data.folder = get_folder_id(&sdata, data.import->folder, 0);
+
+      if(data.folder == ERR_FOLDER){
+         data.folder = add_new_folder(&sdata, data.import->folder, 0);
       }
 
-      if(data.folder == 0){
-         printf("error: cannot get/add folder '%s'\n", folder);
-         mysql_close(&(sdata.mysql));
+      if(data.folder == ERR_FOLDER){
+         printf("ERROR: cannot get/add folder '%s'\n", data.import->folder);
+         close_database(&sdata);
          return 0;
       }
 
    }
 
-   load_rules(&sdata, &(data.archiving_rules), SQL_ARCHIVING_RULE_TABLE);
-   load_rules(&sdata, &(data.retention_rules), SQL_RETENTION_RULE_TABLE);
+   load_rules(&sdata, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
+   load_rules(&sdata, data.retention_rules, SQL_RETENTION_RULE_TABLE);
+   load_rules(&sdata, data.folder_rules, SQL_FOLDER_RULE_TABLE);
 
    load_mydomains(&sdata, &data, &cfg);
 
-   if(emlfile) rc = import_message(emlfile, &sdata, &data, &cfg);
+   if(data.import->filename[0] != '\0') import_message(&sdata, &data, &counters, &cfg);
 
    if(mbox[0]){
       for(i=0; i<n_mbox; i++){
-         rc = import_from_mailbox(mbox[i], &sdata, &data, &cfg);
+         import_from_mailbox(mbox[i], &sdata, &data, &counters, &cfg);
       }
    }
-   if(mboxdir) rc = import_mbox_from_dir(mboxdir, &sdata, &data, &tot_msgs, &cfg);
-   if(directory) rc = import_from_maildir(directory, &sdata, &data, &tot_msgs, &cfg);
-   if(imapserver && username && password) rc = import_from_imap_server(imapserver, username, password, port, &sdata, &data, skiplist, &cfg);
-   if(pop3server && username && password) rc = import_from_pop3_server(pop3server, username, password, port, &sdata, &data, &cfg);
+   if(data.import->mboxdir) import_mbox_from_dir(data.import->mboxdir, &sdata, &data, &counters, &cfg);
+   if(directory) import_from_maildir(&sdata, &data, directory, &counters, &cfg);
+   if(imapserver) import_from_imap_server(&sdata, &data, &counters, &cfg);
+   if(pop3server) import_from_pop3_server(&sdata, &data, &counters, &cfg);
+   if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &counters, &cfg);
 
+   clearrules(data.archiving_rules);
+   clearrules(data.retention_rules);
+   clearrules(data.folder_rules);
 
-   free_rule(data.archiving_rules);
-   free_rule(data.retention_rules);
+   clearhash(data.mydomains);
 
-ENDE:
-   close_prepared_statements(&data);
+   update_counters(&sdata, &data, &counters, &cfg);
 
-   mysql_close(&(sdata.mysql));
+   syslog(LOG_PRIORITY, "server=%s, user=%s, directory=%s, imported=%lld, duplicated=%lld, discarded=%lld", data.import->server, data.import->username, directory, counters.c_rcvd, counters.c_duplicate, counters.c_ignore);
 
-   if(quiet == 0) printf("\n");
+   close_database(&sdata);
 
-   return rc;
+   if(cfg.rtindex) close_sphx(&sdata);
+
+   if(data.quiet == 0) printf("\n");
+
+   return 0;
 }
-
-

src/pilerpurge.c

@@ -1,367 +0,0 @@
-/*
- * pilerpurge.c, SJ
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <dirent.h>
-#include <unistd.h>
-#include <time.h>
-#include <locale.h>
-#include <syslog.h>
-#include <piler.h>
-
-
-extern char *optarg;
-extern int optind;
-
-int dryrun = 0;
-
-
-#define SQL_STMT_SELECT_PURGE_FROM_OPTION_TABLE "SELECT `value` FROM `" SQL_OPTION_TABLE "` WHERE `key`='enable_purge'"
-#define SQL_STMT_DELETE_FROM_META_TABLE "UPDATE `" SQL_METADATA_TABLE "` SET `deleted`=1 WHERE `id` IN ("
-#define SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID "UPDATE `" SQL_METADATA_TABLE "` SET `deleted`=1 WHERE `piler_id` IN ('"
-#define SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS "SELECT `piler_id`, `attachment_id`, `i` FROM `" SQL_ATTACHMENTS_VIEW "` WHERE `refcount`=0 AND `piler_id` IN ('"
-#define SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE "DELETE FROM `" SQL_ATTACHMENT_TABLE "` WHERE `id` IN ("
-
-
-int is_purge_allowed(struct session_data *sdata, struct __config *cfg){
-   int rc=0;
-   MYSQL_RES *res;
-   MYSQL_ROW row;
-
-   if(mysql_real_query(&(sdata->mysql), SQL_STMT_SELECT_PURGE_FROM_OPTION_TABLE, strlen(SQL_STMT_SELECT_PURGE_FROM_OPTION_TABLE)) == 0){
-      res = mysql_store_result(&(sdata->mysql));
-      if(res){
-         row = mysql_fetch_row(res);
-         if(row[0]){
-            rc = atoi(row[0]);
-         }
-
-         mysql_free_result(res);
-      }
-   }
-
-   return rc;
-}
-
-
-int remove_message_frame_files(char *s, char *update_meta_sql, struct session_data *sdata, struct __config *cfg){
-   char *p, puf[SMALLBUFSIZE], filename[SMALLBUFSIZE];
-   int n=0;
-   struct stat st;
-
-   p = s;
-   do {
-      p = split(p, ' ', puf, sizeof(puf)-1);
-
-      if(strlen(puf) == RND_STR_LEN){
-         snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, puf[8], puf[9], puf[10], puf[RND_STR_LEN-4], puf[RND_STR_LEN-3], puf[RND_STR_LEN-2], puf[RND_STR_LEN-1], puf);
-
-         if(stat(filename, &st)){
-            snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, puf[RND_STR_LEN-6], puf[RND_STR_LEN-5], puf[RND_STR_LEN-4], puf[RND_STR_LEN-3], puf[RND_STR_LEN-2], puf[RND_STR_LEN-1], puf);
-         }
-
-         if(dryrun == 1){
-            n++;
-            printf("removing messagefile: %s\n", filename);
-         }
-         else {
-            if(unlink(filename) == 0) n++;
-         }
-
-      }
-   } while(p);
-
-
-   update_meta_sql[strlen(update_meta_sql)-1] = ')';
-
-   if(dryrun == 1){
-      printf("running sql query: *%s*\n\n", update_meta_sql);
-   } else {
-      mysql_real_query(&(sdata->mysql), update_meta_sql, strlen(update_meta_sql));
-   }
-
-
-   return n;
-}
-
-
-int remove_attachments(char *in, struct session_data *sdata, struct __config *cfg){
-   char filename[SMALLBUFSIZE];
-   char *a, buf[BIGBUFSIZE-300], update_meta_sql[BIGBUFSIZE], delete_attachment_stmt[BIGBUFSIZE];
-   int n=0, len;
-   MYSQL_RES *res;
-   MYSQL_ROW row;
-   struct stat st;
-
-
-   if(strlen(in) < 10) return 0;
-
-   len = strlen(SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS) + strlen(in) + 2;
-
-   a = malloc(len);
-   if(!a) return 0;
-
-   memset(a, 0, len);
-
-   in[strlen(in)-2] = ')';
-   in[strlen(in)-1] = '\0';
-   snprintf(a, len-1, "%s%s", SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS, in);
-
-   if(dryrun == 1) printf("running sql query: *%s*\n\n", a);
-
-
-   memset(buf, 0, sizeof(buf));
-   memset(update_meta_sql, 0, sizeof(update_meta_sql));
-   memset(delete_attachment_stmt, 0, sizeof(delete_attachment_stmt));
-
-   snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
-   snprintf(delete_attachment_stmt, sizeof(delete_attachment_stmt)-1, "%s", SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
-
-   if(mysql_real_query(&(sdata->mysql), a, strlen(a)) == 0){
-      res = mysql_store_result(&(sdata->mysql));
-      if(res){
-         while((row = mysql_fetch_row(res))){
-            if(!row[0]) continue;
-
-            snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, row[0][8], row[0][9], row[0][10], row[0][RND_STR_LEN-4], row[0][RND_STR_LEN-3], row[0][RND_STR_LEN-2], row[0][RND_STR_LEN-1], row[0], atoi(row[1]));
-            if(stat(filename, &st)){
-               snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, row[0][RND_STR_LEN-6], row[0][RND_STR_LEN-5], row[0][RND_STR_LEN-4], row[0][RND_STR_LEN-3], row[0][RND_STR_LEN-2], row[0][RND_STR_LEN-1], row[0], atoi(row[1]));
-            }
-
-            if(dryrun == 1){
-               printf("removing attachment: *%s*\n", filename);
-            } else {
-               unlink(filename);
-            }
-
-
-            if(row[2]){
-               memcpy(&delete_attachment_stmt[strlen(delete_attachment_stmt)], row[2], strlen(row[2]));
-               memcpy(&delete_attachment_stmt[strlen(delete_attachment_stmt)], ",", 1);
-            }
-
-            if(atoi(row[1]) == 1){
-
-               memcpy(&update_meta_sql[strlen(update_meta_sql)], row[0], strlen(row[0]));
-               memcpy(&update_meta_sql[strlen(update_meta_sql)], "','", 3);
-
-               if(strlen(buf) >= sizeof(buf)-RND_STR_LEN-2-1){
-                  if(strlen(update_meta_sql) > 10){
-                     update_meta_sql[strlen(update_meta_sql)-2] = ')';
-                     update_meta_sql[strlen(update_meta_sql)-1] = '\0';
-                  }
-
-                  n += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
-
-                  if(strlen(delete_attachment_stmt) > strlen(SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE)){
-                     delete_attachment_stmt[strlen(delete_attachment_stmt)-1] = ')';
-                     if(dryrun == 1){
-                        printf("delete sql: *%s*\n", delete_attachment_stmt);
-                     } else {
-                        mysql_real_query(&(sdata->mysql), delete_attachment_stmt, strlen(delete_attachment_stmt));
-                     }
-                  }
-
-                  memset(buf, 0, sizeof(buf));
-                  memset(update_meta_sql, 0, sizeof(update_meta_sql));
-                  memset(delete_attachment_stmt, 0, sizeof(delete_attachment_stmt));
-
-                  snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
-                  snprintf(delete_attachment_stmt, sizeof(delete_attachment_stmt)-1, "%s", SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
-               }
-
-               memcpy(&buf[strlen(buf)], row[0], strlen(row[0]));
-               memcpy(&buf[strlen(buf)], " ", 1);
-
-            }
-         }
-
-         mysql_free_result(res);
-      }
-   }
-
-   free(a);
-
-
-   if(strlen(buf) > 5 && strlen(update_meta_sql) > strlen(SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID)+10){
-      update_meta_sql[strlen(update_meta_sql)-2] = ')';
-      update_meta_sql[strlen(update_meta_sql)-1] = '\0';
-
-      n += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
-   }
-
-   if(strlen(delete_attachment_stmt) > strlen(SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE)){
-      delete_attachment_stmt[strlen(delete_attachment_stmt)-1] = ')';
-      if(dryrun == 1){
-         printf("delete sql: *%s*\n", delete_attachment_stmt);
-      } else {
-         mysql_real_query(&(sdata->mysql), delete_attachment_stmt, strlen(delete_attachment_stmt));
-      }
-
-   }
-
-
-   return n;
-}
-
-
-int purge_messages_without_attachment(struct session_data *sdata, struct __config *cfg){
-   int purged=0;
-   char s[SMALLBUFSIZE], buf[BIGBUFSIZE-300], update_meta_sql[BIGBUFSIZE];
-   MYSQL_RES *res;
-   MYSQL_ROW row;
-
-   memset(buf, 0, sizeof(buf));
-   memset(update_meta_sql, 0, sizeof(update_meta_sql));
-
-   snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE);
-
-   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id` FROM `%s` WHERE `deleted`=0 AND `retained` < %ld AND attachments=0", SQL_METADATA_TABLE, sdata->now);
-
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "purge sql: *%s*", s);
-
-   if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){
-      res = mysql_store_result(&(sdata->mysql));
-      if(res){
-         while((row = mysql_fetch_row(res))){
-
-            if((char *)row[0] && (char *)row[1]){
-
-               memcpy(&update_meta_sql[strlen(update_meta_sql)], row[0], strlen(row[0]));
-               memcpy(&update_meta_sql[strlen(update_meta_sql)], ",", 1);
-
-               if(strlen(buf) >= sizeof(buf)-RND_STR_LEN-2-1){
-
-                  purged += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
-
-                  memset(buf, 0, sizeof(buf));
-                  memset(update_meta_sql, 0, sizeof(update_meta_sql));
-
-                  snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE);
-               }
-
-               memcpy(&buf[strlen(buf)], row[1], strlen(row[1]));
-               memcpy(&buf[strlen(buf)], " ", 1);
-
-            }
-
-         }
-
-         mysql_free_result(res);
-      }
-   }
-
-   if(strlen(buf) > 5 && strlen(update_meta_sql) > strlen(SQL_STMT_DELETE_FROM_META_TABLE)+5){
-      purged += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
-   }
-
-   return purged;
-}
-
-
-int purge_messages_with_attachments(struct session_data *sdata, struct __config *cfg){
-   int purged=0;
-   char s[SMALLBUFSIZE], idlist[BIGBUFSIZE];
-   MYSQL_RES *res;
-   MYSQL_ROW row;
-
-   memset(idlist, 0, sizeof(idlist));
-
-   snprintf(s, sizeof(s)-1, "SELECT `piler_id` FROM `%s` WHERE `deleted`=0 AND `retained` < %ld AND attachments > 0", SQL_METADATA_TABLE, sdata->now);
-
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "purge sql: *%s*", s);
-
-   if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){
-      res = mysql_store_result(&(sdata->mysql));
-      if(res){
-         while((row = mysql_fetch_row(res))){
-            if((char *)row[0]){
-               memcpy(&idlist[strlen(idlist)], row[0], strlen(row[0]));
-               memcpy(&idlist[strlen(idlist)], "','", 3);
-            }
-
-            if(strlen(idlist) >= sizeof(idlist)-2*RND_STR_LEN){
-               purged += remove_attachments(idlist, sdata, cfg);
-               memset(idlist, 0, sizeof(idlist));
-            }
-         }
-
-         mysql_free_result(res);
-      }
-   }
-
-   if(strlen(idlist) > 5){
-      purged += remove_attachments(idlist, sdata, cfg);
-   }
-
-   return purged;
-}
-
-
-int main(int argc, char **argv){
-   int i, purged=0;
-   char *configfile=CONFIG_FILE;
-   struct session_data sdata;
-   struct __config cfg;
-
-
-   while((i = getopt(argc, argv, "c:dh?")) > 0){
-       switch(i){
-
-         case 'c' :
-                    configfile = optarg;
-                    break;
-
-         case 'd' :
-                    dryrun = 1;
-                    break;
-
-         case 'h' :
-         case '?' :
-         default  : 
-                    break;
-       }
-   }
-
-
-
-   (void) openlog("pilerpurge", LOG_PID, LOG_MAIL);
-
-   cfg = read_config(configfile);
-
-
-   mysql_init(&(sdata.mysql));
-   mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout);
-   if(mysql_real_connect(&(sdata.mysql), cfg.mysqlhost, cfg.mysqluser, cfg.mysqlpwd, cfg.mysqldb, cfg.mysqlport, cfg.mysqlsocket, 0) == 0){
-      printf("cannot connect to mysql server\n");
-      return 0;
-   }
-
-   mysql_real_query(&(sdata.mysql), "SET NAMES utf8", strlen("SET NAMES utf8"));
-   mysql_real_query(&(sdata.mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8"));
-
-   setlocale(LC_CTYPE, cfg.locale);
-
-   init_session_data(&sdata, &cfg);
-
-   i = is_purge_allowed(&sdata, &cfg);
-   if(i == 1){
-      purged += purge_messages_without_attachment(&sdata, &cfg);
-      purged += purge_messages_with_attachments(&sdata, &cfg);
-
-      printf("purged: %d\n", purged);
-   }
-   else printf("purge is not allowed by configuration, enable_purge=%d\n", i);
-
-   mysql_close(&(sdata.mysql));
-
-   return 0;
-}
-
-

src/pop3.c

@@ -23,183 +23,194 @@
 #include <piler.h>
 
 
-int is_last_complete_pop3_packet(char *s, int len){
+int connect_to_pop3_server(struct data *data){
+   char buf[MAXBUFSIZE];
 
-   if(*(s+len-5) == '\r' && *(s+len-4) == '\n' && *(s+len-3) == '.' && *(s+len-2) == '\r' && *(s+len-1) == '\n'){
-      return 1;
+   if(data->net->use_ssl == 1){
+      init_ssl_to_server(data);
    }
 
-   return 0;
+   recvtimeoutssl(data->net, buf, sizeof(buf));
+
+
+   snprintf(buf, sizeof(buf)-1, "USER %s\r\n", data->import->username);
+
+   write1(data->net, buf, strlen(buf));
+   recvtimeoutssl(data->net, buf, sizeof(buf));
+
+
+   snprintf(buf, sizeof(buf)-1, "PASS %s\r\n", data->import->password);
+
+   write1(data->net, buf, strlen(buf));
+   recvtimeoutssl(data->net, buf, sizeof(buf));
+
+   if(strncmp(buf, "+OK", 3) == 0) return OK;
+
+   printf("error: %s", buf);
+
+   return ERR;
 }
 
 
-int connect_to_pop3_server(int sd, char *username, char *password, int port, struct __data *data, int use_ssl){
-   int n;
+void get_number_of_total_messages(struct data *data){
    char buf[MAXBUFSIZE];
-   X509* server_cert;
-   char *str;
+
+   data->import->total_messages = 0;
+
+   snprintf(buf, sizeof(buf)-1, "STAT\r\n");
+   write1(data->net, buf, strlen(buf));
+
+   recvtimeoutssl(data->net, buf, sizeof(buf));
+
+   if(strncmp(buf, "+OK ", 4) == 0){
+      char *p = strchr(&buf[4], ' ');
+      if(p){
+         *p = '\0';
+         data->import->total_messages = atoi(&buf[4]);
+      }
+   }
+   else {
+      printf("ERROR: '%s'", buf);
+   }
+}
 
 
-   if(use_ssl == 1){
+int pop3_download_email(struct data *data, int i){
+   char *p, buf[MAXBUFSIZE], savedbuf[MAXBUFSIZE], copybuf[2*MAXBUFSIZE];
 
-      SSL_library_init();
-      SSL_load_error_strings();
+   data->import->processed_messages++;
 
-      data->ctx = SSL_CTX_new(SSLv3_client_method());
-      CHK_NULL(data->ctx, "internal SSL error");
+   snprintf(data->import->filename, SMALLBUFSIZE-1, "pop3-tmp-%d-%d.txt", getpid(), i);
+   unlink(data->import->filename);
 
-      data->ssl = SSL_new(data->ctx);
-      CHK_NULL(data->ssl, "internal ssl error");
-
-      SSL_set_fd(data->ssl, sd);
-      n = SSL_connect(data->ssl);
-      CHK_SSL(n, "internal ssl error");
-
-      server_cert = SSL_get_peer_certificate(data->ssl);
-      CHK_NULL(server_cert, "server cert error");
-
-      str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0);
-      CHK_NULL(str, "error in server cert");
-      OPENSSL_free(str);
-
-      str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0);
-      CHK_NULL(str, "error in server cert");
-      OPENSSL_free(str);
-
-      X509_free(server_cert);
+   int fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
+   if(fd == -1){
+      printf("cannot open: %s\n", data->import->filename);
+      return ERR;
    }
 
+   memset(savedbuf, 0, sizeof(savedbuf));
 
-   n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
+   snprintf(buf, sizeof(buf)-1, "RETR %d\r\n", i);
+   write1(data->net, buf, strlen(buf));
 
+   int nlines = 0;
+   int endofmessage = 0;
+   int savedlen = 0;
+   int n = 0;
 
-   snprintf(buf, sizeof(buf)-1, "USER %s\r\n", username);
+   while((n = recvtimeoutssl(data->net, buf, sizeof(buf))) > 0){
+      int remaininglen = n;
 
-   write1(sd, buf, use_ssl, data->ssl);
-   n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
+      if(savedlen){
+         memset(copybuf, 0, sizeof(copybuf));
+         memcpy(copybuf, savedbuf, savedlen);
+         memcpy(&copybuf[savedlen], buf, n);
 
+         remaininglen += savedlen;
 
-   snprintf(buf, sizeof(buf)-1, "PASS %s\r\n", password);
+         savedlen = 0;
+         memset(savedbuf, 0, sizeof(savedbuf));
 
-   write1(sd, buf, use_ssl, data->ssl);
-   n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
+         p = &copybuf[0];
+      } else {
+         p = &buf[0];
+      }
 
-   if(strncmp(buf, "+OK", 3) == 0) return OK;
+      int puflen=0;
+      int rc=OK;
+      int nullbyte=0;
+
+      do {
+         char puf[MAXBUFSIZE];
+
+         puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf)-1, &rc, &nullbyte);
+         remaininglen -= puflen;
+         nlines++;
+
+         if(nlines == 1){
+            if(strncmp(puf, "+OK", 3)){
+               printf("error: %s", puf);
+               return ERR;
+            }
+         } else {
+            if(puf[puflen-1] == '\n'){
+               if(puflen == 3 && puf[0] == '.' && puf[1] == '\r' && puf[2] == '\n'){
+                  endofmessage = 1;
+                  break;
+               }
+
+               int dotstuff = 0;
+               if(puf[0] == '.' && puf[1] != '\r' && puf[1] != '\n') dotstuff = 1;
+
+               if(write(fd, &puf[dotstuff], puflen-dotstuff) == -1) printf("ERROR: writing to fd\n");
+
+            } else if(puflen > 0) {
+               savedlen = puflen;
+               snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
+            }
+         }
+
+         p += puflen;
+
+      } while(puflen > 0);
+
+      if(endofmessage){
+         break;
+      }
+   }
+
+   close(fd);
 
    return OK;
 }
 
 
-int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data, int use_ssl, struct __config *cfg){
-   int i, rc=ERR, n, messages=0, processed_messages=0, pos, readlen, fd, lastpos, nreads;
-   char *p, buf[MAXBUFSIZE], filename[SMALLBUFSIZE];
-   char aggrbuf[3*MAXBUFSIZE];
+void pop3_delete_message(struct data *data, int i){
+   char buf[SMALLBUFSIZE];
 
-   snprintf(buf, sizeof(buf)-1, "STAT\r\n");
-   n = write1(sd, buf, use_ssl, data->ssl);
-
-   n = recvtimeoutssl(sd, buf, sizeof(buf), 10, use_ssl, data->ssl);
-
-   if(strncmp(buf, "+OK ", 4) == 0){
-      p = strchr(&buf[4], ' ');
-      if(p){
-         *p = '\0';
-         messages = atoi(&buf[4]);
-      }
-   }
-   else return ERR;
+   snprintf(buf, sizeof(buf)-1, "DELE %d\r\n", i);
+   write1(data->net, buf, strlen(buf));
+   recvtimeoutssl(data->net, buf, sizeof(buf));
+}
 
 
-   printf("found %d messages\n", messages);
+void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+   char buf[MAXBUFSIZE];
 
-   if(messages <= 0) return rc;
+   data->import->processed_messages = 0;
 
-   for(i=1; i<=messages; i++){
-      processed_messages++;
-      printf("processed: %7d\r", processed_messages); fflush(stdout);
+   get_number_of_total_messages(data);
 
+   if(data->quiet == 0) printf("found %d messages\n", data->import->total_messages);
 
-      snprintf(buf, sizeof(buf)-1, "RETR %d\r\n", i);
+   if(data->import->total_messages <= 0) return;
 
-      snprintf(filename, sizeof(filename)-1, "pop3-tmp-%d.txt", i);
-      unlink(filename);
+   for(int i=data->import->start_position; i<=data->import->total_messages; i++){
+      if(pop3_download_email(data, i) == OK){
+         if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/data->import->total_messages); fflush(stdout); }
 
-      fd = open(filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
-      if(fd == -1){
-         printf("cannot open: %s\n", filename);
-         return rc;
+         if(data->import->dryrun == 0){
+            int rc = import_message(sdata, data, counters, cfg);
+
+            if(data->import->remove_after_import == 1 && rc == OK){
+               pop3_delete_message(data, i);
+            }
+         }
       }
 
-      n = write1(sd, buf, use_ssl, data->ssl);
+      if(data->import->download_only == 0) unlink(data->import->filename);
 
-      readlen = 0;
-      pos = 0;
-      nreads = 0;
+      /* whether to quit after processing a batch of messages */
 
-      memset(aggrbuf, 0, sizeof(aggrbuf));
-      lastpos = 0;
-
-
-      while((n = recvtimeoutssl(sd, buf, sizeof(buf), 15, use_ssl, data->ssl)) > 0){
-         nreads++;
-         readlen += n;
-
-         if(nreads == 1){
-
-            if(strncmp(buf, "+OK", 3) == 0){
-               p = strchr(&buf[3], '\n');
-               if(p){
-                  *p = '\0';
-                  pos = strlen(buf)+1;
-                  *p = '\n';
-               }
-            }
-            else { printf("error: %s", buf); return ERR; }
-
-         }
-
-         if(lastpos + 1 + n < sizeof(aggrbuf)){
-
-            if(nreads == 1){
-               memcpy(aggrbuf+lastpos, buf+pos, n-pos);
-               lastpos += n-pos;
-            }
-            else {
-               memcpy(aggrbuf+lastpos, buf, n);
-               lastpos += n;
-            }
-         }
-         else {
-            write(fd, aggrbuf, sizeof(buf));
-
-            memmove(aggrbuf, aggrbuf+sizeof(buf), lastpos-sizeof(buf));
-            lastpos -= sizeof(buf);
-
-            memcpy(aggrbuf+lastpos, buf, n);
-            lastpos += n;
-         }
-
-         if(is_last_complete_pop3_packet(aggrbuf, lastpos) == 1){
-            write(fd, aggrbuf, lastpos-3);
-            break;
-         }
-
-      } 
-
-      close(fd);
-
-      rc = import_message(filename, sdata, data, cfg);
-
-      unlink(filename);
+      if(data->import->batch_processing_limit > 0 && data->import->processed_messages >= data->import->batch_processing_limit){
+         break;
+      }
    }
 
 
    snprintf(buf, sizeof(buf)-1, "QUIT\r\n");
-   n = write1(sd, buf, use_ssl, data->ssl);
+   write1(data->net, buf, strlen(buf));
 
-   printf("\n");
-
-   return OK;
+   if(data->quiet == 0) printf("\n");
 }
-
-