2025-04-06 03:10:33 +02:00
729 changed files with 104166 additions and 50743 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,26 +0,0 @@
-Makefile
-*.o
-*.so
-*.a
-*.deb
-src/libpiler.so.0*
-src/piler
-src/pilerconf
-src/piler-smtp
-src/pileraget
-src/pilerexport
-src/pilerget
-src/pilerimport
-src/pilerstats
-src/pilertest
-src/reindex
-unit_tests/check_attachments
-unit_tests/check_decoder
-unit_tests/check_digest
-unit_tests/check_hash
-unit_tests/check_misc
-unit_tests/check_mydomains
-unit_tests/check_parser
-unit_tests/check_parser_utils
-unit_tests/check_rules
-unit_tests/smtp
--- a/12
+++ b/12
@ -1,12 +0,0 @@
-#!/bin/bash
-
-set -o nounset
-set -o errexit
-set -o pipefail
-
-read -r version < VERSION
-
-tarball="piler-${version}.tar.gz"
-
-git archive --prefix="piler-${version}/" HEAD | gzip -c > "$tarball"
-sha256sum "$tarball"
--- a/2
+++ b/2
@ -1,6 +1,6 @@
 piler, an enterprise level email archiving application

-Copyright (C) 2012-2023, Janos SUTO <sj@acts.hu>
+Copyright (C) 2012-2016, Janos SUTO <sj@acts.hu>

 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by
--- a/Makefile.in
+++ b/Makefile.in
@ -33,7 +33,7 @@ INSTALL = @INSTALL@
 am__CONFIG_DISTCLEAN_FILES = config.status config.cache config.log \
 	piler-config.h stamp.h stamp-h1 params.h

-all: all-recursive config-php
+all: all-recursive
 install: installdirs install-recursive


@ -63,10 +63,6 @@ $(RECURSIVE_TARGETS):
 	fi; test -z "$$fail"


-config-php:
-	sed -e "s%SYSCONFDIR%$(sysconfdir)%g" -e "s%SBINDIR%$(sbindir)%g" config.php.in > webui/config.php
-	sed -e "s%BINDIR%$(bindir)%g" -i webui/config.php
-

 installdirs: mkinstalldirs
 	$(srcdir)/mkinstalldirs \
@ -79,20 +75,15 @@ installdirs: mkinstalldirs
 		$(DESTDIR)$(sysconfdir)/piler \
 		$(DESTDIR)/etc/init.d \
 		$(DESTDIR)$(localstatedir)/piler/store \
-		$(DESTDIR)$(localstatedir)/piler/stat \
-		$(DESTDIR)$(localstatedir)/piler/tmp \
-		$(DESTDIR)$(localstatedir)/piler/error \
-		$(DESTDIR)$(localstatedir)/piler/export \
-		$(DESTDIR)$(localstatedir)/piler/manticore
+		$(DESTDIR)$(localstatedir)/piler/stat $(DESTDIR)$(localstatedir)/piler/tmp \
+		$(DESTDIR)$(localstatedir)/piler/sphinx

 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/run/piler
 	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/store
 	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/imap
 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/stat
 	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/tmp
-	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/error
-	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/export
-	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/manticore
+	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/sphinx


 install-am:
@ -126,7 +117,7 @@ key:
 	dd if=/dev/urandom bs=56 count=1 of=piler.key

 postinstall:
-	@bash util/postinstall.sh $(RUNNING_USER) $(RUNNING_GROUP) $(sysconfdir) $(localstatedir) $(libexecdir)
+	@sh util/postinstall.sh $(RUNNING_USER) $(RUNNING_GROUP) $(sysconfdir) $(localstatedir) $(libexecdir)



--- a/126
+++ b/126
@ -1,129 +1,3 @@
-1.4.5:
------
-
- Introduced 2 new variables in /etc/piler/piler.conf affecting piler-smtp
-
-  ; max message size in bytes
-  ; piler-smtp will reject any message that's bigger than this number
-  max_message_size=50000000
-
-  ; max memory in bytes piler-smtp uses for buffering messages
-  ; when this limit is exceeded, no new emails will be accepted
-  ; until the used memory for all in progress emails decreases
-  ; below this level
-  max_smtp_memory=500000000
-
-  Be sure to adjust these values to your environment!
-
- Added read-only connection support for manticore
-  If using sphinx, add the following to config-site.php:
-  $config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9306';
-
- pilerimport supports Zimbra IMAP impersonation
-
-  Generate the following base64 encoded string:
-  (Be sure to use the actual usernames and password):
-
-  pw="$( printf '%s\0%s\0%s' 'username' 'zimbra_admin_username' 'zimbra_admin_password' | base64 )"
-
-  Then specify -u ZIMBRA -p "$pw" for pilerimport, eg.
-
-  pilerimport -i imap.server -u ZIMBRA -p "$pw" ...
-
-  Note that "ZIMBRA" is a special username, it indicates for pilerimport
-  to actually use the imap impersonation for Zimbra.
-
-
-1.4.4:
------
-
- Renamed "group" table to "usergroup"
-  Be sure to run util/db-upgrade.sql on the mysql piler database
-
-
-1.4.3:
------
-
- Improved real-time index support (no need to use rtindex.py any more)
-  See https://www.mailpiler.org/wiki/current:manticore for more
-
-
-1.4.2:
------
-
- Fixed docker image
- Fixed php socket path in piler-nginx.conf
- Fixed manticore index data settings
-
-
-1.4.1:
------
-
- Replaced sphinx with manticore 5.0.2, see https://www.mailpiler.org/wiki/current:manticore
- Replaced obsoleted libchart library with chart.js
- Added real-time index support
- Added oauth2 support to imapfetch.py
- Added support for pilerexport to spread files among several directories. See the -D option
-
-
-1.3.12:
-------
-
- Introduced new piler.conf variable: tls_min_version
-
-  It sets the minimum TLS protocol version the piler-smtp daemon supports.
-
-  Possible values:
-  - TLSv1   (not recommended)
-  - TLSv1.1 (not recommended)
-  - TLSv1.2 (default)
-  - TLSv1.3
-
- Introduced the archive_address feature, see etc/example.conf for the details
- Introduced the raw: search label
- Added Italian translation. Credits: Stefano Gatto
- timestamp signing sorts by 'id' column
- timestamp hash value defaults to sha256
- Minor fixes
- Added support for php 8.1
- Fixed handling long email addresses
-
-Be sure to apply util/db-upgrade.sql
-
-
-1.3.11:
-------
-
- [BUGFIX] Refactored the smtp timeout check
- Obsoleted the LDAP port parameter. Specify the ldap host in the form
-  of protocol://hostname:port, eg. ldaps://ldap.example.com:636
-
-1.3.10:
-------
-
- Added security header feature
- Introduced the smtp acl list, and obsoleted the tcp_wrappers check
- Switched from Blowfish encryption to AES-256
-
-1.3.9:
------
-
- Added a separator to searching for attachment names
- [BUGFIX] Render multiple mail parts in mail view instead of only the last part
- Use TLS v1.2 with openssl 1.0.x for connecting remote pop3/imap servers
- Instant search results to the gui when the search page loads
- Support sphinx-3.3.1, introduced sphinx strict mode variable
- GUI domain fixes
- gcc 9 fixes
- Fix permission on sphinx data dir to 700
- pilerpurge.py should honor the mysqlhost value
- Password change enabled by default
- Health page fixes
- GUI mime parser fixes
- Start/stop script fix
- Optimized search page for mobile devices, set $config['ENABLE_MOBILE_PREVIEW'] = 1; in config-site.php to enable it
-
-
 1.3.0:
 -----

--- a/SECURITY.md
+++ b/SECURITY.md
@ -1,20 +0,0 @@
-Disclosure policy
-
-If you find a security issue, please contact the project owner at sj@acts.hu
-with the details (ie. piler version, details of the setup, how to exploit the
-vulnerability, etc).
-
-Please provide 30 days for verifying the vulnerability, fixing the issue, and
-notifying the piler users.
-
-Security update policy
-
-If a security vulnerability has found, the details, possible mitigations,
-workarounds, etc. will be shared on the piler mailing list (piler-user@list.acts.hu)
-and on the wiki: https://www.mailpiler.org/
-
-Security configurations
-
- - Use https for the GUI
- - Reset the default passwords for admin and auditor
- - Use the smtp acl feature to restrict SMTP access to the archive, see https://mailpiler.com/smtp-acl-list/
--- a/2
+++ b/2
@ -1 +1 @@
-1.4.5
+1.3.0
--- a/bitbucket-pipelines.yml
+++ b/bitbucket-pipelines.yml
@ -3,20 +3,10 @@
 # Only use spaces to indent your .yml configuration.
 # -----
 # You can specify a custom docker image from Docker Hub as your build environment.
-
-image: sutoj/builder:bionic
-clone:
-  depth: 5
+# image: docker-image:tag

 pipelines:
  default:
    - step:
        script:
-          - service mysql start
-          - ./configure --prefix=/usr --sysconfdir=/etc --localstatedir=/var --with-database=mariadb
-          - make clean all install
-          - mysql -u piler -ppiler123 piler1 < /usr/share/piler/db-mysql.sql
-          - cd unit_tests
-          - ./run.sh
-          - cd ../webui
-          - phpunit
+          - echo "Everything is awesome!"
--- a/3387
+++ b/3387
--- a/configure.in
+++ b/configure.in
@ -85,7 +85,7 @@ fi
 echo "\"Configure command: ./configure $PARAMS\"" >> $CONFIGURE_PARAMS_FILE


-SUBDIRS="src etc util init.d systemd unit_tests webui"
+SUBDIRS="src etc util init.d unit_tests"


 dnl static build
@ -115,13 +115,18 @@ AC_ARG_ENABLE(memcached,
        [  --enable-memcached      build memcached support], want_memcached=$enableval, want_memcached="no")


-dnl tcpwrappers library
+AC_ARG_ENABLE(tcpwrappers,
+	[  --enable-tcpwrappers    build tcpwrappers support], want_tcpwrappers=$enableval, want_tcpwrappers="no")

-AC_CHECK_HEADERS(tcpd.h, have_tcpwrappers=yes, echo "tcpd.h is not found")
-AC_CHECK_LIB([wrap],[hosts_access],[AC_CHECK_LIB(wrap, hosts_access, have_tcpwrappers=yes, echo "libwrap is not found"; have_tcpwrappers=no)],[],[])ac_cv_lib_wrap=ac_cv_lib_wrap_main

-if test "$have_tcpwrappers" = "no"; then
-   echo "can't find either tcpd.h or libwrap. Tcpwrappers support is disabled";
+if test "$want_tcpwrappers" = "yes"; then
+   AC_CHECK_HEADERS(tcpd.h, have_tcpwrappers=yes, have_tcpwrappers=no)
+   AC_CHECK_LIB([wrap],[hosts_access],[AC_CHECK_LIB(wrap, hosts_access, have_tcpwrappers=yes, echo "libwrap is not found"; have_tcpwrappers=no)],[],[])ac_cv_lib_wrap=ac_cv_lib_wrap_main
+
+   if test "$have_tcpwrappers" = "no"; then
+      echo "can't find either tcpd.h or libwrap";
+      exit 1;
+   fi
 fi


@ -258,10 +263,6 @@ if test z`which timeout 2>/dev/null` != "z"; then
 fi


-read -r version < VERSION
-AC_DEFINE_UNQUOTED(VERSION, "$version")
-
-
 dnl user running piler

 AC_ARG_WITH(piler-user,
@ -355,6 +356,9 @@ AC_SUBST(DATAROOTDIR)
 AC_DEFINE_UNQUOTED(DATAROOTDIR,"$dataroot_dir",[where to look for the share data files])


+AC_DEFINE_UNQUOTED(VIRUS_TEMPLATE, "$my_prefix/share/clapf/template.virus", [where the virus template is])
+AC_DEFINE_UNQUOTED(ZOMBIE_NET_REGEX, "$my_prefix/share/clapf/zombienets.regex", [where the virus template is])
+

 if test "$have_mysql" = "no" && test "$have_psql" = "no"; then
   echo
@ -383,6 +387,11 @@ if test "$os" = "SunOS"; then
   if test -x /usr/xpg4/bin/id; then id_bin="/usr/xpg4/bin/id"; fi
 fi

+if test `echo $os | grep -c CYGWIN` -eq 1; then
+   defs="$defs -DCYGWIN"
+   AC_DEFINE_UNQUOTED(ZOMBIE_NET_REGEX, "zombienets.regex", [where the virus template is])
+fi
+
 dnl whether we have antivirus support

 if test "$have_clamd" = "yes" ; then
@ -429,8 +438,19 @@ if test "$have_zip" = "yes"; then
   echo "zip library: yes"
   AC_DEFINE_UNQUOTED(HAVE_ZIP, 1, [libzip support])
   antispam_libs="$antispam_libs -lzip"
+fi
+
+if test "$have_tcpwrappers" = "yes"; then
+   echo "tcpwrappers support: yes"
+   AC_DEFINE_UNQUOTED(HAVE_LIBWRAP, 1, [tcpwrappers support])
+
+   if test "$os" = "FreeBSD"; then
+      antispam_libs="$antispam_libs -lwrap"
+   else
+      antispam_libs="$antispam_libs -lwrap -lnsl"
+   fi
 else
-   echo "zip library: no"
+   echo "tcpwrappers support: no"
 fi


@ -522,27 +542,16 @@ if test $? -eq 1; then echo "the user \"$RUNNING_USER\" does not exists, please

 echo; echo

-gcc_version="$(gcc -dumpversion)"
-extra_cflags=""
-if [[ "${gcc_version:0:1}" -gt 6 ]]; then
-   extra_cflags="-Wimplicit-fallthrough=2"
-fi
-
-CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra $extra_cflags -Wuninitialized -Wimplicit-fallthrough=2 -Wno-format-truncation -g"
+CFLAGS="$static -O2 -Wall -g"
 LIBS="$antispam_libs $sunos_libs "
-OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o screen.o $objs"
+OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o mydomains.o $objs"

-AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile])
+AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile unit_tests/Makefile contrib/imap/Makefile])
 AC_OUTPUT

-echo
-echo
-echo "IMPORTANT! If you upgrade, be sure to read https://www.mailpiler.org/upgrade-instructions/"
-echo
-echo

 echo
-echo "Did you know that piler has an enterprise edition as well?"
-echo "Check out what it can do for you at https://mailpiler.com/#features"
+echo
+echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/en/upgrade.html"
 echo
 echo
--- a/contrib/export-attachments/export-attachments.php
+++ b/contrib/export-attachments/export-attachments.php
@ -1,103 +0,0 @@
-#!/usr/bin/php
-<?php
-
-define('EXPORT_LOCK_FILE', '/var/piler/tmp/export-attachments.lock');
-
-$webuidir = "";
-
-$outdir = '';
-
-
-$opts = 'hw:d:';
-$lopts = [
-   'webui:',
-   'dir:',
-   'help'
-];
-
-
-if($options = getopt($opts, $lopts)) {
-   if(isset($options['webui'])) { $webuidir = $options['webui']; }
-   if(isset($options['dir'])) { $outdir = $options['dir']; }
-   if(isset($options['help'])) { usage(); }
-}
-
-
-if($webuidir == '' || $outdir == '') { usage(); }
-
-
-
-require_once("$webuidir/config.php");
-
-require(DIR_SYSTEM . "/startup.php");
-
-$request = new Request();
-Registry::set("request", $request);
-
-
-$start = NULL;
-
-$loader = new Loader();
-Registry::set('load', $loader);
-
-$loader->load->model('domain/domain');
-$loader->load->model('search/search');
-$loader->load->model('search/message');
-$loader->load->model('message/attachment');
-
-
-$db = new DB(DB_DRIVER, DB_HOSTNAME, DB_USERNAME, DB_PASSWORD, DB_DATABASE, DB_PREFIX);
-Registry::set('db', $db);
-
-Registry::set('auditor_user', 1);
-
-openlog("export-attachments", LOG_PID, LOG_MAIL);
-
-$fp = fopen(EXPORT_LOCK_FILE, "w");
-if(!flock($fp, LOCK_EX)) {
-   syslog(LOG_INFO, "WARN: couldn't get a lock on " . EXPORT_LOCK_FILE);
-   exit;
-}
-
-
-$domain = new ModelDomainDomain();
-$attachment = new ModelMessageAttachment();
-$message = new ModelSearchMessage();
-
-$domains = $domain->get_mapped_domains();
-
-$last_id = $attachment->get_last_attachment_id();
-$start_id = $attachment->get_checkpoint();
-
-syslog(LOG_INFO, "start: $start_id, limit: $last_id");
-
-
-for($i=$start_id; $i<$last_id; $i++) {
-   $a = $attachment->get_attachment_by_id($i);
-   $m = $message->get_message_addresses_by_piler_id($a['piler_id'], $domains);
-
-   $attachment->dump_attachment($outdir, "out", $m['sender'], $i, $a);
-
-   foreach($m['rcpt'] as $rcpt) {
-      $attachment->dump_attachment($outdir, "in", $rcpt, $i, $a);
-   }
-
-   if($i % 100 == 0) { $attachment->update_checkpoint($i); }
-}
-
-$attachment->update_checkpoint($i);
-
-// Release lock
-flock($fp, LOCK_UN);
-fclose($fp);
-
-
-
-function usage() {
-   print "\nUsage: " . __FILE__ . "\n\n";
-   print "\t--webui <path to webui directory>\n";
-   print "\t--dir <basedir to write attachments>\n";
-   print "\t--help\n\n";
-
-   exit;
-}
--- a/contrib/installer/focal.sh
+++ b/contrib/installer/focal.sh
@ -1,257 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o pipefail
-set -o nounset
-set -x
-
-MY_IP="1.2.3.4"
-PILER_HOSTNAME="${PILER_HOSTNAME:-archive.example.com}"
-MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
-MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
-SERVER_ID="${SERVER_ID:-0}"
-USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
-SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
-PHP_FPM_SOCKET="/var/run/php/php7.4-fpm.sock"
-
-MYSQL_HOSTNAME="localhost"
-MYSQL_DATABASE="piler"
-MYSQL_USERNAME="piler"
-
-SPHINX_TARGZ="sphinx-3.3.1-bin.tar.gz"
-DOWNLOAD_URL="https://download.mailpiler.com"
-PILER_DEB="piler_1.3.12-focal-eb2b22b_amd64.deb"
-PILER_USER="piler"
-CONFIG_SITE_PHP="/etc/piler/config-site.php"
-
-export DEBIAN_FRONTEND=noninteractive
-
-install_prerequisites() {
-   apt-get update
-   apt-get -y --no-install-recommends install \
-      wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \
-      php7.4-gd php7.4-curl php7.4-xml ca-certificates zip catdoc unrtf poppler-utils nginx tnef libzip5 \
-      libtre5 libwrap0 cron libmariadb-dev python3 python3-mysqldb libmariadb-dev mariadb-client-core-10.3 \
-      mariadb-server-10.3
-
-   wget -q -O "/tmp/${SPHINX_TARGZ}" "${DOWNLOAD_URL}/generic-local/${SPHINX_TARGZ}"
-   tar -C / -zxvf "/tmp/${SPHINX_TARGZ}"
-   wget -O /usr/local/bin/traefik "${DOWNLOAD_URL}/generic-local/traefik"
-   chmod +x /usr/local/bin/traefik
-   setcap cap_net_bind_service+ep /usr/local/bin/traefik
-}
-
-
-fix_mysql_settings() {
-   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
-[mysqld]
-
-innodb_buffer_pool_size=512M
-innodb_flush_log_at_trx_commit=1
-innodb_log_buffer_size=64M
-innodb_log_file_size=64M
-innodb_read_io_threads=4
-innodb_write_io_threads=4
-innodb_log_files_in_group=2
-
-innodb_file_per_table
-PILER_CNF
-}
-
-start_mysql() {
-   fix_mysql_settings
-
-   service mysql restart
-   mysqladmin -u root password "${MYSQL_ROOT_PASSWORD}"
-}
-
-
-install_piler() {
-   wget "${DOWNLOAD_URL}/piler/${PILER_DEB}"
-   dpkg -i "$PILER_DEB"
-   rm -f "$PILER_DEB"
-
-   crontab -u "$PILER_USER" /usr/share/piler/piler.cron
-
-   rm -f "$PILER_DEB" /etc/nginx/sites-enabled/default
-   ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf
-
-   touch /var/piler/.bash_history
-   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
-}
-
-
-create_my_cnf() {
-   local user=$1
-   local password=$2
-   local my_cnf=$3
-
-   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
-   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
-   chown $PILER_USER:$PILER_USER "$my_cnf"
-   chmod 600 "$my_cnf"
-}
-
-
-fix_config_site_php() {
-   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"
-
-   {
-      echo "\$config['SERVER_ID'] = $SERVER_ID;"
-      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
-      echo "\$config['SPHINX_VERSION'] = 331;"
-   } >> "$CONFIG_SITE_PHP"
-
-   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
-      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
-   fi
-
-   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
-}
-
-
-init_db() {
-   sed -e "s%MYSQL_HOSTNAME%$MYSQL_HOSTNAME%g" -e "s%MYSQL_DATABASE%$MYSQL_DATABASE%g" -e "s%MYSQL_USERNAME%$MYSQL_USERNAME%g" -e "s%MYSQL_PASSWORD%$MYSQL_PILER_PASSWORD%g" \
-      /usr/share/piler/db-mysql-root.sql.in | mysql --defaults-file=/etc/piler/.my.cnf-root -h $MYSQL_HOSTNAME
-   mysql --defaults-file=/etc/piler/.my.cnf -h $MYSQL_HOSTNAME $MYSQL_DATABASE < /usr/share/piler/db-mysql.sql
-}
-
-
-add_systemd_services() {
-   pushd /etc/systemd/system
-
-   ln -sf /usr/libexec/piler/piler.service .
-   ln -sf /usr/libexec/piler/piler-smtp.service .
-   ln -sf /usr/libexec/piler/pilersearch.service .
-
-   popd
-
-   systemctl daemon-reload
-
-   systemctl enable piler
-   systemctl enable piler-smtp
-   systemctl enable pilersearch
-   systemctl enable traefik
-}
-
-
-fix_configs() {
-   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
-      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
-      nginx -t
-      nginx -s reload
-
-      sed -i 's/server {/server {\n\tlisten 127.0.0.1:80;/' /etc/piler/piler-nginx.conf
-      systemctl stop nginx
-      sleep 5
-      systemctl start nginx
-   fi
-
-   if [[ ! -f /etc/piler/piler.conf ]]; then
-      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.example.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > /etc/piler/piler.conf
-      chmod 600 /etc/piler/piler.conf
-      chown $PILER_USER:$PILER_USER /etc/piler/piler.conf
-   fi
-
-   sed -i 's/tls_enable=0/tls_enable=1/g' /etc/piler/piler.conf
-
-   sed -i -e "s/MYSQL_HOSTNAME/${MYSQL_HOSTNAME}/g" \
-          -e "s/MYSQL_DATABASE/${MYSQL_DATABASE}/g" \
-          -e "s/MYSQL_USERNAME/${MYSQL_USERNAME}/g" \
-          -e "s/MYSQL_PASSWORD/${MYSQL_PILER_PASSWORD}/g" \
-          /etc/piler/sphinx.conf
-}
-
-setup_traefik() {
-   wget -O /etc/systemd/system/traefik.service "${DOWNLOAD_URL}/generic-local/traefik.service"
-
-   mkdir -p /usr/local/etc/traefik
-   touch /usr/local/etc/traefik/acme.json
-   chmod 600 /usr/local/etc/traefik/acme.json
-   chown www-data:www-data /usr/local/etc/traefik/acme.json
-
-   cat > /usr/local/etc/traefik/traefik.yaml << TRAEFIK
-log:
-  level: INFO
-
-entryPoints:
-  web:
-    address: "$MY_IP:80"
-    http:
-      redirections:
-        entryPoint:
-          to: websecure
-          scheme: https
-          permanent: true
-  websecure:
-    address: "$MY_IP:443"
-
-providers:
-  file:
-    filename: "/usr/local/etc/traefik/traefik.yaml"
-
-certificatesResolvers:
-  le:
-    acme:
-      storage: "/usr/local/etc/traefik/acme.json"
-      email: admin@$PILER_HOSTNAME
-      httpChallenge:
-        entryPoint: web
-
-tls:
-  options:
-    default:
-      minVersion: VersionTLS13
-
-http:
-  middlewares:
-    piler_headers:
-      headers:
-        customResponseHeaders:
-          Server: ""
-          Strict-Transport-Security: "max-age=31536000"
-          X-Content-Type-Optionsi: "nosniff"
-          Referrer-Policy: "same-origin"
-  routers:
-    master:
-      rule: "Host(\`$PILER_HOSTNAME\`)"
-      service: www
-      middlewares:
-        - "piler_headers"
-      tls:
-        certResolver: le
-  services:
-    www:
-      loadBalancer:
-        servers:
-        - url: "http://127.0.0.1:80/"
-TRAEFIK
-}
-
-
-install_prerequisites
-
-start_mysql
-
-install_piler
-
-create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
-create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf
-
-fix_config_site_php
-
-setup_traefik
-
-add_systemd_services
-
-fix_configs
-init_db
-
-su -c "indexer --all -c /etc/piler/sphinx.conf" $PILER_USER
-
-[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler
-
-systemctl start pilersearch
-systemctl start piler
-systemctl start piler-smtp
-systemctl start traefik
--- a/contrib/installer/jammy.sh
+++ b/contrib/installer/jammy.sh
@ -1,258 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o pipefail
-set -o nounset
-set -x
-
-MY_IP="1.2.3.4"
-PILER_HOSTNAME="${PILER_HOSTNAME:-archive.example.com}"
-MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
-MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
-SERVER_ID="${SERVER_ID:-0}"
-USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
-SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
-PHP_FPM_SOCKET="/var/run/php/php8.1-fpm.sock"
-
-MYSQL_HOSTNAME="localhost"
-MYSQL_DATABASE="piler"
-MYSQL_USERNAME="piler"
-
-DOWNLOAD_URL="https://download.mailpiler.com"
-PILER_DEB="piler_1.4.4-jammy-0573c6da_amd64.deb"
-PILER_USER="piler"
-PILER_CONF="/etc/piler/piler.conf"
-CONFIG_SITE_PHP="/etc/piler/config-site.php"
-SEARCHCFG="/etc/piler/manticore.conf"
-
-TRAEFIK_VERSION="v2.11.0"
-ARCH="amd64"
-
-export DEBIAN_FRONTEND=noninteractive
-
-install_prerequisites() {
-   apt-get update
-   apt-get -y --no-install-recommends install \
-      wget rsyslog openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql \
-      php8.1-fpm php8.1-zip php8.1-ldap php8.1-gd php8.1-curl php8.1-xml \
-      php8.1-mbstring ca-certificates zip catdoc unrtf poppler-utils \
-      nginx tnef libzip4 libtre5 libwrap0 cron libmariadb-dev python3 \
-      python3-mysqldb libmariadb-dev mariadb-client-core-10.6 \
-      mariadb-server-10.6
-
-   wget "https://github.com/traefik/traefik/releases/download/${TRAEFIK_VERSION}/traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz"
-   tar zxvf "traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz" traefik
-   chown root:root traefik
-   chmod 755 traefik
-   mv traefik /usr/local/bin
-   setcap cap_net_bind_service+ep /usr/local/bin/traefik
-
-   wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
-   dpkg -i manticore-repo.noarch.deb && \
-   apt-get update && \
-   apt-get install -y manticore manticore-columnar-lib manticore-extra && \
-   rm -f manticore-repo.noarch.deb
-   systemctl stop manticore
-}
-
-
-fix_mysql_settings() {
-   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
-[mysqld]
-
-innodb_buffer_pool_size=512M
-innodb_flush_log_at_trx_commit=1
-innodb_log_buffer_size=64M
-innodb_log_file_size=64M
-innodb_read_io_threads=4
-innodb_write_io_threads=4
-innodb_log_files_in_group=2
-
-innodb_file_per_table
-PILER_CNF
-}
-
-start_mysql() {
-   fix_mysql_settings
-
-   service mysql restart
-   mysqladmin -u root password "${MYSQL_ROOT_PASSWORD}"
-}
-
-
-install_piler() {
-   wget "https://bitbucket.org/jsuto/piler/downloads/${PILER_DEB}"
-   dpkg -i "$PILER_DEB"
-   rm -f "$PILER_DEB"
-
-   sed -i 's/.*indexer.*/###/g' /usr/share/piler/piler.cron
-   crontab -u "$PILER_USER" /usr/share/piler/piler.cron
-
-   rm -f "$PILER_DEB" /etc/nginx/sites-enabled/default
-   ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf
-
-   touch /var/piler/.bash_history
-   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
-}
-
-
-create_my_cnf() {
-   local user=$1
-   local password=$2
-   local my_cnf=$3
-
-   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
-   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
-   chown $PILER_USER:$PILER_USER "$my_cnf"
-   chmod 600 "$my_cnf"
-}
-
-
-fix_config_site_php() {
-   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"
-
-   {
-      echo "\$config['SERVER_ID'] = $SERVER_ID;"
-      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
-   } >> "$CONFIG_SITE_PHP"
-
-   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
-      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
-   fi
-
-   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
-   echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
-}
-
-
-init_db() {
-   sed -e "s%MYSQL_HOSTNAME%$MYSQL_HOSTNAME%g" -e "s%MYSQL_DATABASE%$MYSQL_DATABASE%g" -e "s%MYSQL_USERNAME%$MYSQL_USERNAME%g" -e "s%MYSQL_PASSWORD%$MYSQL_PILER_PASSWORD%g" \
-      /usr/share/piler/db-mysql-root.sql.in | mysql --defaults-file=/etc/piler/.my.cnf-root -h $MYSQL_HOSTNAME
-   mysql --defaults-file=/etc/piler/.my.cnf -h $MYSQL_HOSTNAME $MYSQL_DATABASE < /usr/share/piler/db-mysql.sql
-}
-
-
-add_systemd_services() {
-   pushd /etc/systemd/system
-
-   ln -sf /usr/libexec/piler/piler.service .
-   ln -sf /usr/libexec/piler/piler-smtp.service .
-   ln -sf /usr/libexec/piler/pilersearch.service .
-
-   popd
-
-   systemctl daemon-reload
-
-   systemctl enable piler
-   systemctl enable piler-smtp
-   systemctl enable pilersearch
-   systemctl enable traefik
-}
-
-
-fix_configs() {
-   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
-      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
-      nginx -t
-      nginx -s reload
-
-      sed -i 's/server {/server {\n\tlisten 127.0.0.1:80;/' /etc/piler/piler-nginx.conf
-      systemctl stop nginx
-      sleep 5
-      systemctl start nginx
-   fi
-
-   if [[ ! -f "$PILER_CONF" ]]; then
-      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.example.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > "$PILER_CONF"
-      chmod 600 "$PILER_CONF"
-      chown $PILER_USER:$PILER_USER "$PILER_CONF"
-   fi
-
-   sed -i 's/tls_enable=0/tls_enable=1/g' "$PILER_CONF"
-   sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
-
-   sed -i "s/define('RT.*/define('RT', 1);/" "$SEARCHCFG"
-}
-
-setup_traefik() {
-   wget -O /etc/systemd/system/traefik.service "${DOWNLOAD_URL}/generic-local/traefik.service"
-
-   mkdir -p /usr/local/etc/traefik
-   touch /usr/local/etc/traefik/acme.json
-   chmod 600 /usr/local/etc/traefik/acme.json
-   chown www-data:www-data /usr/local/etc/traefik/acme.json
-
-   cat > /usr/local/etc/traefik/traefik.yaml << TRAEFIK
-log:
-  level: INFO
-
-entryPoints:
-  websecure:
-    address: "$MY_IP:443"
-
-providers:
-  file:
-    filename: "/usr/local/etc/traefik/traefik.yaml"
-
-certificatesResolvers:
-  le:
-    acme:
-      storage: "/usr/local/etc/traefik/acme.json"
-      email: admin@$PILER_HOSTNAME
-      tlsChallenge: {}
-
-tls:
-  options:
-    default:
-      minVersion: VersionTLS13
-
-http:
-  middlewares:
-    piler_headers:
-      headers:
-        customResponseHeaders:
-          Server: ""
-          Strict-Transport-Security: "max-age=31536000"
-          X-Content-Type-Optionsi: "nosniff"
-          Referrer-Policy: "same-origin"
-  routers:
-    master:
-      rule: "Host(\`$PILER_HOSTNAME\`)"
-      service: www
-      middlewares:
-        - "piler_headers"
-      tls:
-        certResolver: le
-  services:
-    www:
-      loadBalancer:
-        servers:
-        - url: "http://127.0.0.1:80/"
-TRAEFIK
-}
-
-
-install_prerequisites
-
-start_mysql
-
-install_piler
-
-create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
-create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf
-
-fix_config_site_php
-
-setup_traefik
-
-add_systemd_services
-
-fix_configs
-init_db
-
-[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler
-
-systemctl start pilersearch
-systemctl start piler
-systemctl start piler-smtp
-systemctl start traefik
--- a/contrib/proxmox-lxc/README
+++ b/contrib/proxmox-lxc/README
@ -1,9 +0,0 @@
-You nedd proxmox with lxc
-
-Fill out conf file first and provide public key
-
-It automatically
- downloads the latest debian template
- provides a container
- installs piler until the post installation step
- gives a self signed certificate
--- a/contrib/proxmox-lxc/create_lxc.sh
+++ b/contrib/proxmox-lxc/create_lxc.sh
@ -1,195 +0,0 @@
-#!/bin/bash
-
-# This script wil create and fire up a standard debian buster lxc container on your proxmox pve.
-# The Script will look for the next free lxc number and take the next free and use it. So take
-# care that behind your last number is place for it. 
-
-#### SOME VARIABLES TO ADJUST ####
-
-# Storage with templates
-LXC_TMP="local"
-
-# Size and pool of rootfs / in GB
-SIZ_ROT="100"
-S_ROT_P="local-zfs"
-
-# Size and pool of Filestorage in GB will mounted to /share
-SIZ_FIL="100"
-S_FIL_P="local-zfs"
-
-#Weather or not (1 and 0) the container will createt as unpriviliged LXC
-LXC_UNP="1"
-
-# Size of the RAM assigned to the LXC
-LXC_MEM="1024"
-
-# Size of the SWAP assigned to the LXC
-LXC_SWA="1024"
-
-# The hostname (eq. zamba1 or mailpiler1)
-LXC_HOST="zamba"
-
-# The domainname (searchdomain /etc/resolf.conf & hosts)
-LXC_SDN="zmb.local"
-
-# IP-address and subnet
-LXC_IP="10.10.80.20/24"
-
-# Gateway
-LXC_GW="10.10.80.10"
-
-# DNS-server and here shoud be your AD-DC
-LXC_DNS="10.10.80.10"
-
-# Networkbridge for this machine
-LXC_BRD="vmbr80"
-
-# root password - take care to delete from this file
-LXC_PWD="MYPASSWD"
-
-LXC_KEY="ssh-rsa xxxxxxxx"
-
-############### Zamba-Server-Section ###############
-
-# Domain Entries to samba/smb.conf. Will be also uses for samba domain-provisioning when zmb-pdc will choosen.
-ZMB_REA="ZMB.LOCAL"
-ZMB_DOM="ZMB"
-
-# THE Domain-Admin and passwd for zamba-install
-ZMB_ADA="Administrator"
-ZMB_APW="MYPASSWORD"
-
-############### Mailpiler-Section ###############
-
-# The FQDN vor the Hostname. This must be exactly the same like the LXC_HOST / LXC_SDN at section above.
-PILER_DOM="piler.zmb.rocks"
-SMARTHOST="10.10.80.20"
-PILER_VER="1.3.10"
-SPHINX_VER="3.3.1"
-PHP_VER="7.4"
-
-############### Matrix-Section ###############
-
-# The FQDN vor the Hostname. This should be the same like the LXC_HOST / LXC_SDN at section above.
-MRX_DOM="matrix.zmb.rocks"
-ELE_DOM="element.zmb.rocks"
-ELE_VER="v1.7.21"
-JIT_DOM="meet.zmb.rocks"
-
-#################################
-
-# CHeck is the newest template available, else download it.
-
-DEB_LOC=$(pveam list $LXC_TMP | grep debian-10-standard | cut -d'_' -f2)
-
-DEB_REP=$(pveam available --section system | grep debian-10-standard | cut -d'_' -f2)
-
-if [[ $DEB_LOC == $DEB_REP ]];
-then
-  echo "Newest Version of Debian 10 Standard $DEP_REP exists.";
-else
-  echo "Will now download newest Debian 10 Standard $DEP_REP.";
-  pveam download $LXC_TMP debian-10-standard_$DEB_REP\_amd64.tar.gz
-fi
-
-# Get next free LXC-number
-LXC_LST=$( lxc-ls | egrep -o '.{1,5}$' )
-LXC_CHK=$((LXC_LST+1));
-
-if  [ $LXC_CHK -lt 100 ] || [ -f /etc/pve/qemu-server/$LXC_CHK.conf ]; then
-  LXC_NBR=$(pvesh get /cluster/nextid);
-else
-  LXC_NBR=$LXC_CHK;
-fi
-
-echo "Will now create LXC Container $LXC_NBR!";
-
-# Create the container
-pct create $LXC_NBR -unprivileged $LXC_UNP $LXC_TMP:vztmpl/debian-10-standard_$DEB_REP\_amd64.tar.gz -rootfs $S_ROT_P:$SIZ_ROT;
-sleep 2;
-
-pct set $LXC_NBR -memory $LXC_MEM -swap $LXC_SWA -hostname $LXC_HOST \-nameserver $LXC_DNS -searchdomain $LXC_SDN -onboot 1 -timezone Europe/Berlin -net0 name=eth0,bridge=$LXC_BRD,firewall=1,gw=$LXC_GW,ip=$LXC_IP,type=veth;
-sleep 2;
-
-PS3="Select the Server-Function: "
-
-select opt in just_lxc zmb-standalone zmb-member zmb-pdc mailpiler matrix quit; do
-  case $opt in
-    just_lxc)
-      lxc-start $LXC_NBR;
-      sleep 5;
-      # Set the root password and key
-      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
-      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
-      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
-      lxc-attach -n$LXC_NBR service ssh restart;
-      echo "Should be ready!"
-      break
-      ;;
-    zmb-standalone)
-      break
-      ;;
-    zmb-member)
-      echo "Make some additions to LXC for AD-Member-Server!"
-      pct set $LXC_NBR -mp0 $S_FIL_P:$SIZ_FIL,mp=/tank
-      sleep 2;
-      lxc-start $LXC_NBR;
-      sleep 5;
-      # Set the root password and key
-      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
-      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
-      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
-      lxc-attach -n$LXC_NBR service ssh restart;
-      cp /root/zmb_mem.orig /root/zmb_mem.sh
-      sed -i "s|#ZMB_VAR|#ZMB_VAR\nZMB_REA='$ZMB_REA'\nZMB_DOM='$ZMB_DOM'\nZMB_ADA='$ZMB_ADA'\nZMB_APW='$ZMB_APW'|" /root/zmb_mem.sh
-      pct push $LXC_NBR /root/zmb_mem.sh /root/zmb_mem.sh
-      echo "Install zamba as AD-Member-Server!"
-      lxc-attach -n$LXC_NBR bash /root/zmb_mem.sh
-      break
-      ;;
-    zmb-pdc)
-      break
-      ;;
-    mailpiler)
-      echo "Make some additions to LXC for Mailpiler!"
-      pct set $LXC_NBR -features nesting=1
-      sleep 2;
-      lxc-start $LXC_NBR;
-      sleep 5;
-      # Set the root password and key
-      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
-      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
-      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
-      lxc-attach -n$LXC_NBR service ssh restart;
-      cp /root/mailpiler.orig /root/mailpiler.sh
-      sed -i "s|#PILER_VAR|#PILER_VAR\nPILER_DOM='$PILER_DOM'\nSMARTHOST='$SMARTHOST'\nPILER_VER='$PILER_VER'\nSPHINX_VER='$SPHINX_VER'\nPHP_VER='$PHP_VER'|" /root/mailpiler.sh
-      pct push $LXC_NBR /root/mailpiler.sh /root/mailpiler.sh
-      echo "Install Mailpiler mailarchiv!"
-      lxc-attach -n$LXC_NBR bash mailpiler.sh
-      break
-      ;;
-    matrix)
-      echo "Make some additions to LXC for Matrix!"
-      lxc-start $LXC_NBR;
-      sleep 5;
-      # Set the root password and key
-      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
-      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
-      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
-      lxc-attach -n$LXC_NBR service ssh restart;
-      cp /root/matrix.orig /root/matrix.sh
-      sed -i "s|#MATRIX_VAR|#Matrix_VAR\nMRX_DOM='$MRX_DOM'\nELE_DOM='$ELE_DOM'\nELE_VER='$ELE_VER'\nJIT_DOM='$JIT_DOM'|" /root/matrix.sh
-      pct push $LXC_NBR /root/matrix.sh /root/matrix.sh
-      echo "Install Matrix Chatserver!"
-      lxc-attach -n$LXC_NBR bash matrix.sh
-      break
-      ;;
-    quit)
-      break
-      ;;
-    *)
-      echo "Invalid option!"
-      ;;
-    esac
-done
-
--- a/contrib/proxmox-lxc/mailpiler.orig
+++ b/contrib/proxmox-lxc/mailpiler.orig
@ -1,179 +0,0 @@
-#!/bin/bash
-
-#Variables will be filled in from the mainscript:
-
-#PILER_VAR
-
-
-HOSTNAME=$(hostname -f)
-
-echo "Ensure your Hostname is set to your Piler FQDN!"
-
-echo $HOSTNAME
-
-if 
-    [ "$HOSTNAME" != "$PILER_DOM" ]
-then
-        echo "Hostname doesn't match Piler_Domain! Check install.sh, /etc/hosts, /etc/hostname." && exit
-else
-        echo "Hostname matches PILER_DOMAIN, so starting installation."
-fi
-
-apt install -y gpg apt-transport-https lsb-release
-
-wget -q https://packages.sury.org/php/apt.gpg -O- | apt-key add -
-echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" | tee /etc/apt/sources.list.d/php.list
-
-apt update && apt full-upgrade -y
-
-apt install -y mc sysstat build-essential libwrap0-dev libpst-dev tnef libytnef0-dev unrtf catdoc libtre-dev tre-agrep poppler-utils libzip-dev unixodbc libpq5 software-properties-common libpoppler-dev openssl libssl-dev memcached telnet nginx mariadb-server default-libmysqlclient-dev python-mysqldb gcc libwrap0 libzip4 latex2rtf latex2html catdoc tnef libpq5 zipcmp zipmerge ziptool libsodium23
-
-apt update && apt install -y php$PHP_VER-{fpm,common,ldap,mysql,cli,opcache,phpdbg,gd,memcache,json,readline,zip}
-
-apt purge -y postfix
-
-cat > /etc/mysql/conf.d/mailpiler.conf <<EOF
-innodb_buffer_pool_size=256M
-innodb_flush_log_at_trx_commit=1
-innodb_log_buffer_size=64M
-innodb_log_file_size=16M
-query_cache_size=0
-query_cache_type=0
-query_cache_limit=2M
-EOF
-
-systemctl restart mariadb
-
-cd /tmp
-wget https://download.mailpiler.com/generic-local/sphinx-$SPHINX_VER-bin.tar.gz
-tar -xvzf sphinx-$SPHINX_VER-bin.tar.gz -C /
-
-groupadd piler
-useradd -g piler -m -s /bin/bash -d /var/piler piler
-usermod -L piler
-chmod 755 /var/piler
-
-wget https://bitbucket.org/jsuto/piler/downloads/piler-$PILER_VER.tar.gz
-tar -xvzf piler-$PILER_VER.tar.gz
-cd piler-$PILER_VER/
-./configure --localstatedir=/var --with-database=mysql --enable-tcpwrappers --enable-memcached
-make
-make install
-ldconfig
-
-cp util/postinstall.sh util/postinstall.sh.bak
-sed -i "s/   SMARTHOST=.*/   SMARTHOST="\"$SMARTHOST\""/" util/postinstall.sh
-sed -i 's/   WWWGROUP=.*/   WWWGROUP="www-data"/' util/postinstall.sh
-
-make postinstall
-
-cp /usr/local/etc/piler/piler.conf /usr/local/etc/piler/piler.conf.bak
-sed -i "s/hostid=.*/hostid=$PILER_DOM/" /usr/local/etc/piler/piler.conf
-sed -i "s/update_counters_to_memcached=.*/update_counters_to_memcached=1/" /usr/local/etc/piler/piler.conf
-
-su piler -c "indexer --all --config /usr/local/etc/piler/sphinx.conf"
-
-/etc/init.d/rc.piler start
-/etc/init.d/rc.searchd start
-
-update-rc.d rc.piler defaults
-update-rc.d rc.searchd defaults
-
-mkdir -p /etc/nginx/ssl
-openssl req -x509 -nodes -days 3650 -newkey rsa:4096 -keyout /etc/nginx/ssl/piler.key -out /etc/nginx/ssl/piler.crt -subj "/CN=$PILER_DOM" -addext "subjectAltName=DNS:$PILER_DOM"
-
-cd /etc/nginx/sites-available
-cp /tmp/piler-$PILER_VER/contrib/webserver/piler-nginx.conf /etc/nginx/sites-available/
-ln -s /etc/nginx/sites-available/piler-nginx.conf /etc/nginx/sites-enabled/piler-nginx.conf
-
-sed -i "s|PILER_HOST|$PILER_DOM|g" /etc/nginx/sites-available/piler-nginx.conf
-sed -i "s|/var/run/php/php7.4-fpm.sock|/var/run/php/php$PHP_VER-fpm.sock|g" /etc/nginx/sites-available/piler-nginx.conf
-
-sed -i "/server_name.*/a \\
-        listen 443 ssl http2;\n\n\
-        ssl_certificate /etc/nginx/ssl/piler.crt;\n\
-        ssl_certificate_key /etc/nginx/ssl/piler.key;\n\n\
-        ssl_session_timeout 1d;\n\
-        ssl_session_cache shared:SSL:15m;\n\
-        ssl_session_tickets off;\n\n\
-        # modern configuration of Mozilla SSL configurator. Tweak to your needs.\n\
-        ssl_protocols TLSv1.2 TLSv1.3;\n\
-        ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;\n\
-        ssl_prefer_server_ciphers off;\n\n\
-        add_header X-Frame-Options SAMEORIGIN;\n\
-        add_header X-Content-Type-Options nosniff;" /etc/nginx/sites-available/piler-nginx.conf
-
-sed -i "/^server {.*/i\
-server {\n\
-        listen 80;\n\
-        server_name $PILER_DOM;\n\
-        server_tokens off;\n\
-        # HTTP to HTTPS redirect.\n\
-        return 301 https://$PILER_DOM;\n\
-}" /etc/nginx/sites-available/piler-nginx.conf
-
-cp /usr/local/etc/piler/config-site.php /usr/local/etc/piler/config-site.php.bak
-sed -i "s|\$config\['SITE_URL'\] = .*|\$config\['SITE_URL'\] = 'https://$PILER_DOM/';|" /usr/local/etc/piler/config-site.php
-cat >> /usr/local/etc/piler/config-site.php <<EOF
-
-// CUSTOM
-\$config['PROVIDED_BY'] = '$PILER_DOM';
-\$config['SUPPORT_LINK'] = 'https://$PILER_DOM';
-\$config['COMPATIBILITY'] = '';
-
-// fancy features.
-\$config['ENABLE_INSTANT_SEARCH'] = 1;
-\$config['ENABLE_TABLE_RESIZE'] = 1;
-
-\$config['ENABLE_DELETE'] = 1;
-\$config['ENABLE_ON_THE_FLY_VERIFICATION'] = 1;
-
-// general settings.
-\$config['TIMEZONE'] = 'Europe/Berlin';
-
-// authentication
-// Enable authentication against an imap server
-//\$config['ENABLE_IMAP_AUTH'] = 1;
-//\$config['RESTORE_OVER_IMAP'] = 1;
-//\$config['IMAP_RESTORE_FOLDER_INBOX'] = 'INBOX';
-//\$config['IMAP_RESTORE_FOLDER_SENT'] = 'Sent';
-//\$config['IMAP_HOST'] = '$SMARTHOST';
-//\$config['IMAP_PORT'] =  993;
-//\$config['IMAP_SSL'] = true;
-
-// authentication against an ldap directory (disabled by default)
-//\$config['ENABLE_LDAP_AUTH'] = 1;
-//\$config['LDAP_HOST'] = '$SMARTHOST';
-//\$config['LDAP_PORT'] = 389;
-//\$config['LDAP_HELPER_DN'] = 'cn=administrator,cn=users,dc=mydomain,dc=local';
-//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
-//\$config['LDAP_MAIL_ATTR'] = 'mail';
-//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
-//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
-//\$config['LDAP_BASE_DN'] = 'ou=Benutzer,dc=krs,dc=local';
-
-// authentication against an Uninvention based ldap directory 
-//\$config['ENABLE_LDAP_AUTH'] = 1;
-//\$config['LDAP_HOST'] = '$SMARTHOST';
-//\$config['LDAP_PORT'] = 7389;
-//\$config['LDAP_HELPER_DN'] = 'uid=ldap-search-user,cn=users,dc=mydomain,dc=local';
-//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
-//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
-//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
-//\$config['LDAP_BASE_DN'] = 'cn=users,dc=mydomain,dc=local';
-//\$config['LDAP_MAIL_ATTR'] = 'mailPrimaryAddress';
-//\$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'person';
-//\$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'person';
-//\$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mailAlternativeAddress';
-
-// special settings.
-\$config['MEMCACHED_ENABLED'] = 1;
-\$config['SPHINX_STRICT_SCHEMA'] = 1; // required for Sphinx $SPHINX_VER, see https://bitbucket.org/jsuto/piler/issues/1085/sphinx-331.
-EOF
-
-nginx -t && systemctl restart nginx
-
-apt autoremove -y
-apt clean -y
-
-
--- a/contrib/proxmox-lxc/matrix.orig
+++ b/contrib/proxmox-lxc/matrix.orig
@ -1,150 +0,0 @@
-#!/bin/bash
-
-#MATRIX_VAR
-
-
-MRX_PKE=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
-
-ELE_DBNAME="synapse_db"
-ELE_DBUSER="synapse_user"
-ELE_DBPASS=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
-
-apt update && apt full-upgrade -y
-
-apt install -y lsb-release apt-transport-https curl gpg software-properties-common net-tools nginx mc postgresql python3-psycopg2
-
-wget wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
-echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/matrix-org.list
-apt update && apt install -y matrix-synapse-py3
-systemctl enable matrix-synapse
-
-ss -tulpen
-
-mkdir /etc/nginx/ssl
-openssl req -x509 -nodes -days 3650 -newkey rsa:4096 -keyout /etc/nginx/ssl/matrix.key -out /etc/nginx/ssl/matrix.crt -subj "/CN=$MRX_DOM" -addext "subjectAltName=DNS:$MRX_DOM"
-
-cat > /etc/nginx/sites-available/$MRX_DOM <<EOF
-# Virtual Host configuration for example.com
-#
-# You can move that to a different file under sites-available/ and symlink that
-# to sites-enabled/ to enable it.
-
-server {
-    listen 80;
-    listen [::]:80;
-    server_name $MRX_DOM;
-
-    return 301 https://$MRX_DOM;
-}
-
-server {
-    listen 443 ssl;
-    listen [::]:443 ssl;
-    server_name $MRX_DOM;
-
-    ssl on;
-    ssl_certificate /etc/nginx/ssl/matrix.crt;
-    ssl_certificate_key /etc/nginx/ssl/matrix.key;
-
-    location / {
-      proxy_pass http://127.0.0.1:8008;
-      proxy_set_header X-Forwarded-For \$remote_addr;
-    }
-}
-
-server {
-    listen 8448 ssl;
-    listen [::]:8448 ssl;
-    server_name $MRX_DOM;
-
-    ssl on;
-    ssl_certificate /etc/nginx/ssl/matrix.crt;
-    ssl_certificate_key /etc/nginx/ssl/matrix.key;
-
-    # If you don't wanna serve a site, comment this out
-    root /var/www/$MRX_DOM;
-    index index.html index.htm;
-
-    location / {
-        proxy_pass http://127.0.0.1:8008;
-        proxy_set_header X-Forwarded-For \$remote_addr;
-    }
-}
-
-EOF
-ln -s /etc/nginx/sites-available/$MRX_DOM /etc/nginx/sites-enabled/$MRX_DOM
-
-cat > /etc/nginx/sites-available/$ELE_DOM <<EOF
-# Virtual Host configuration for example.com
-#
-# You can move that to a different file under sites-available/ and symlink that
-# to sites-enabled/ to enable it.
-
-server {
-    listen 80;
-    listen [::]:80;
-    server_name $ELE_DOM;
-    return 301 https://$ELE_DOM;
-}
-
-server {
-    listen 443 ssl;
-    listen [::]:443 ssl;
-    server_name $ELE_DOM;
-
-    ssl on;
-    ssl_certificate /etc/nginx/ssl/matrix.crt;
-    ssl_certificate_key /etc/nginx/ssl/matrix.key;
-
-    # If you don't wanna serve a site, comment this out
-    root /var/www/$ELE_DOM/element;
-    index index.html index.htm;
-} 
-
-EOF
-
-ln -s /etc/nginx/sites-available/$ELE_DOM /etc/nginx/sites-enabled/$ELE_DOM
-
-systemctl restart nginx
-
-mkdir /var/www/$ELE_DOM
-cd /var/www/$ELE_DOM
-wget https://packages.riot.im/element-release-key.asc
-gpg --import element-release-key.asc
-
-wget https://github.com/vector-im/element-web/releases/download/$ELE_VER/element-$ELE_VER.tar.gz
-wget https://github.com/vector-im/element-web/releases/download/$ELE_VER/element-$ELE_VER.tar.gz.asc
-gpg --verify element-$ELE_VER.tar.gz.asc
-
-tar -xzvf element-$ELE_VER.tar.gz
-ln -s element-$ELE_VER element
-chown www-data:www-data -R element
-cp ./element/config.sample.json ./element/config.json
-sed -i "s|https://matrix-client.matrix.org|https://$MRX_DOM|" ./element/config.json
-sed -i "s|\"server_name\": \"matrix.org\"|\"server_name\": \"$MRX_DOM\"|" ./element/config.json
-
-su postgres <<EOF
-psql -c "CREATE USER $ELE_DBUSER WITH PASSWORD '$ELE_DBPASS';"
-psql -c "CREATE DATABASE $ELE_DBNAME ENCODING 'UTF8' LC_COLLATE='C' LC_CTYPE='C' template=template0 OWNER $ELE_DBUSER;"
-echo "Postgres User '$ELE_DBUSER' and database '$ELE_DBNAME' created."
-EOF
-
-cd /
-sed -i "s|#registration_shared_secret: <PRIVATE STRING>|registration_shared_secret: \"$MRX_PKE\"|" /etc/matrix-synapse/homeserver.yaml
-sed -i "s|#public_baseurl: https://example.com/|public_baseurl: https://$MRX_DOM/|" /etc/matrix-synapse/homeserver.yaml
-sed -i "s|#enable_registration: false|enable_registration: true|" /etc/matrix-synapse/homeserver.yaml
-sed -i "s|name: sqlite3|name: psycopg2|" /etc/matrix-synapse/homeserver.yaml
-sed -i "s|database: /var/lib/matrix-synapse/homeserver.db|database: $ELE_DBNAME\n    user: $ELE_DBUSER\n    password: $ELE_DBPASS\n    host: 127.0.0.1\n    cp_min: 5\n    cp_max: 10|" /etc/matrix-synapse/homeserver.yaml
-
-systemctl restart matrix-synapse
-
-register_new_matrix_user -c /etc/matrix-synapse/homeserver.yaml http://127.0.0.1:8008
-
-#curl https://download.jitsi.org/jitsi-key.gpg.key | sh -c 'gpg --dearmor > /usr/share/keyrings/jitsi-keyring.gpg'
-#echo 'deb [signed-by=/usr/share/keyrings/jitsi-keyring.gpg] https://download.jitsi.org stable/' | tee /etc/apt/sources.list.d/jitsi-stable.list > /dev/null
-
-#apt update
-#apt install -y jitsi-meet
-
-
-
--- a/contrib/proxmox-lxc/zmb_mem.orig
+++ b/contrib/proxmox-lxc/zmb_mem.orig
@ -1,100 +0,0 @@
-#!/bin/bash
-
-#ZMB_VAR
-
-
-
-apt update && apt full-upgrade -y
-echo -ne '\n' | apt install -y acl dnsutils mc samba winbind libpam-winbind libnss-winbind krb5-user krb5-config samba-dsdb-modules samba-vfs-modules 
-
-mv /etc/krb5.conf /etc/krb5.conf.bak
-cat > /etc/krb5.conf <<EOF
-[libdefaults]
-	default_realm = $ZMB_REA
-    ticket_lifetime = 600
-	dns_lookup_realm = true
-	dns_lookup_kdc = true
-	renew_lifetime = 7d
-EOF
-
-echo -e "$ZMB_APW" | kinit -V $ZMB_ADA
-klist
-
-mv /etc/samba/smb.conf /etc/samba/smb.conf.bak
-cat > /etc/samba/smb.conf <<EOF
-[global]
-	workgroup = $ZMB_DOM
-	security = ADS
-	realm = $ZMB_REA
-	server string = %h server
-
-	vfs objects = acl_xattr shadow_copy2
-    map acl inherit = Yes
-    store dos attributes = Yes
-	idmap config *:backend = tdb
-	idmap config *:range = 3000000-4000000
-	idmap config *:schema_mode = rfc2307
-
-	winbind refresh tickets = Yes
-	winbind use default domain = Yes
-	winbind separator = /
-	winbind nested groups = yes
-	winbind nss info = rfc2307
-
-	pam password change = Yes
-	passwd chat = *Enter\snew\s*\spassword:* %n\n *Retype\snew\s*\spassword:* %n\n *password\supdated\ssuccessfully* .
-	passwd program = /usr/bin/passwd %u
-
-	template homedir = /home/%U
-	template shell = /bin/bash
-	bind interfaces only = Yes
-	interfaces = lo eth0
-	log file = /var/log/samba/log.%m
-	logging = syslog
-	max log size = 1000
-	panic action = /usr/share/samba/panic-action %d
-
-	load printers = No
-	printcap name = /dev/null
-	printing = bsd
-	disable spoolss = Yes
-
-	allow trusted domains = No
-	dns proxy = No
-	shadow: snapdir = .zfs/snapshot
-	shadow: sort = desc
-	shadow: format = -%Y-%m-%d-%H%M
-	shadow: snapprefix = ^zfs-auto-snap_\(frequent\)\{0,1\}\(hourly\)\{0,1\}\(daily\)\{0,1\}\(monthly\)\{0,1\}
-	shadow: delimiter = -20
-
-[share]
-	comment = Main Share
-	path = /tank/share
-	read only = No
-	create mask = 0660
-	directory mask = 0770
-	inherit acls = Yes
-
-
-
-EOF
-
-systemctl restart smbd
-
-echo -e "$ZMB_APW" | net ads join -U $ZMB_ADA createcomputer=Computers
-sed -i "s|files systemd|files systemd winbind|g" /etc/nsswitch.conf
-sed -i "s|#WINBINDD_OPTS=|WINBINDD_OPTS=|" /etc/default/winbind
-echo -e "session optional        pam_mkhomedir.so skel=/etc/skel umask=077" >> /etc/pam.d/common-session
-
-systemctl restart winbind nmbd
-wbinfo -u
-wbinfo -g
-
-mkdir /tank/share
-chown 'administrator':'domain users' /tank/share
-
-setfacl -Rm u:administrator:rwx,g::-,o::- /tank/share
-setfacl -Rdm u:administrator:rwx,g::-,o::- /tank/share
-
-systemctl restart smbd nmbd winbind
-
--- a/contrib/smtp/smtp-source.py
+++ b/contrib/smtp/smtp-source.py
@ -168,4 +168,4 @@ while i < args.count:


 if args.debug == 0:
-    print('')
+    print
--- a/contrib/webserver/piler-apache-2.x.conf
+++ b/contrib/webserver/piler-apache-2.x.conf
@ -4,7 +4,8 @@
    DocumentRoot "/var/piler/www"

    <Directory /var/piler/www>
-       Require all granted
+       Order allow,deny
+       Allow from all

       AllowOverride all
    </Directory>
--- a/contrib/webserver/piler-nginx.conf
+++ b/contrib/webserver/piler-nginx.conf
@ -4,13 +4,10 @@ server {

        root /var/piler/www;

-        server_tokens off;
+        access_log  /var/log/nginx/access.log;
+        error_log   /var/log/nginx/error.log;

-        add_header X-XSS-Protection "1; mode=block";
-        add_header X-Content-Type-Options "nosniff";
-        add_header Referrer-Policy "same-origin";
-
-        gzip on;
+        gzip  on;
        gzip_types text/plain application/xml text/css;
        gzip_vary on;

@ -34,7 +31,8 @@ server {
               return 404;
            }

-            fastcgi_pass unix:/var/run/php/php8.1-fpm.sock;
+            #fastcgi_pass unix:/var/run/php5-fpm.sock;
+            fastcgi_pass unix:/var/run/php/php7.0-fpm.sock;
            fastcgi_index index.php;
            include fastcgi_params;
        }
@ -51,7 +49,6 @@ server {
        rewrite /message.php /index.php?route=message/view;
        rewrite /bulkrestore.php /index.php?route=message/bulkrestore;
        rewrite /bulkremove.php /index.php?route=message/bulkremove;
-        rewrite /rejectremove.php /index.php?route=message/rejectremove;
        rewrite /bulkpdf.php /index.php?route=message/bulkpdf;
        rewrite /folders.php /index.php?route=folder/list&;
        rewrite /settings.php /index.php?route=user/settings;
--- a/cppcheck.sh
+++ b/cppcheck.sh
@ -1,7 +0,0 @@
-#!/bin/bash
-
-set -o nounset
-set -o errexit
-set -o pipefail
-
-cppcheck -DHAVE_PDFTOTEXT -DHAVE_PPTHTML -DHAVE_TNEF -DHAVE_UNRTF -DHAVE_XLS2CSV -DHAVE_CATPPT -DHAVE_CATDOC -DHAVE_ZIP -D_GNU_SOURCE -DHAVE_DAEMON -DHAVE_TRE -DHAVE_CLAMD -DHAVE_LIBCLAMAV -DNEED_MYSQL --error-exitcode=1 --enable=all --suppressions-list=suppressions.txt --force src/ unit_tests/
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -1,43 +0,0 @@
-FROM ubuntu:22.04
-
-ARG PACKAGE
-
-LABEL description="piler ubuntu jammy image" \
-      maintainer="Janos SUTO, sj@acts.hu" \
-      package="${PACKAGE}"
-
-ENV DEBIAN_FRONTEND="noninteractive" \
-    DISTRO="jammy" \
-    PILER_USER="piler" \
-    MYSQL_DATABASE="piler"
-
-COPY ${PACKAGE} /
-
-RUN apt-get update && \
-    apt-get -y --no-install-recommends install \
-       wget openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql php8.1-fpm php8.1-zip php8.1-ldap \
-       php8.1-gd php8.1-curl php8.1-xml php8.1-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip4 \
-       libtre5 cron libmariadb-dev mariadb-client-core-10.6 python3 python3-mysqldb ca-certificates curl rsyslog && \
-    wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
-    dpkg -i manticore-repo.noarch.deb && \
-    rm -f manticore-repo.noarch.deb && \
-    apt-get update && \
-    apt-get install -y manticore manticore-columnar-lib && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* && \
-    sed -i '/session    required     pam_loginuid.so/c\#session    required     pam_loginuid.so' /etc/pam.d/cron && \
-    dpkg -i ${PACKAGE} && \
-    touch /etc/piler/MANTICORE && \
-    ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled && \
-    rm -f ${PACKAGE} /etc/nginx/sites-enabled/default /etc/piler/piler.key /etc/piler/piler.pem /etc/piler/config-site.php && \
-    crontab -u $PILER_USER /usr/share/piler/piler.cron
-
-VOLUME ["/etc/piler"]
-VOLUME ["/var/piler/store"]
-VOLUME ["/var/piler/manticore"]
-
-EXPOSE 25 80 443
-
-COPY start.sh /start.sh
-
-CMD ["/start.sh"]
--- a/docker/README.md
+++ b/docker/README.md
@ -1,16 +0,0 @@
-## How to run piler
-
-Edit the variables in docker-compose.yaml, then run
-
-```
-docker-compose up -d
-```
-
-## How to build the image for yourself
-
-Pick the latest deb package from Bitbucket download page (https://bitbucket.org/jsuto/piler/downloads/)
-and use it as the PACKAGE build argument, eg.
-
-```
-docker build --build-arg PACKAGE=piler_1.4.1-jammy-c860ca67_amd64.deb -t piler:1.4.1 .
-```
--- a/docker/build.sh
+++ b/docker/build.sh
@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o pipefail
-set -o nounset
-
-IMAGE_NAME="sutoj/piler:1.4.4"
-
-if [[ $# -ne 1 ]]; then echo "ERROR: missing package name" 1>&2; exit 1; fi
-
-docker build --pull --build-arg PACKAGE="$1" -t "$IMAGE_NAME" .
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@ -1,59 +0,0 @@
-version: "3"
-services:
-  mysql:
-    image: mariadb:11.1.2
-    container_name: mysql
-    restart: unless-stopped
-    cap_drop:
-      - ALL
-    cap_add:
-      - dac_override
-      - setuid
-      - setgid
-    environment:
-      - MYSQL_DATABASE=piler
-      - MYSQL_USER=piler
-      - MYSQL_PASSWORD=piler123
-      - MYSQL_RANDOM_ROOT_PASSWORD=yes
-    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
-    volumes:
-      - db_data:/var/lib/mysql
-  piler:
-    image: sutoj/piler:1.4.4
-    container_name: piler
-    init: true
-    environment:
-      - MYSQL_HOSTNAME=mysql
-      - MYSQL_DATABASE=piler
-      - MYSQL_USER=piler
-      - MYSQL_PASSWORD=piler123
-      - PILER_HOSTNAME=cust1.acts.hu
-      - RT=1
-    ports:
-      - "25:25"
-      - "80:80"
-    volumes:
-      - piler_etc:/etc/piler
-      - piler_manticore:/var/piler/manticore
-      - piler_store:/var/piler/store
-    healthcheck:
-      test: curl -s smtp://localhost/
-      interval: "60s"
-      timeout: "3s"
-      start_period: "15s"
-      retries: 3
-    deploy:
-      resources:
-        reservations:
-          memory: 512M
-        limits:
-          memory: 512M
-
-    depends_on:
-      - "mysql"
-
-volumes:
-  db_data: {}
-  piler_etc: {}
-  piler_manticore: {}
-  piler_store: {}
--- a/docker/start.sh
+++ b/docker/start.sh
@ -1,216 +0,0 @@
-#!/bin/bash
-
-set -o errexit
-set -o pipefail
-set -o nounset
-
-CONFIG_DIR="/etc/piler"
-VOLUME_DIR="/var/piler"
-PILER_CONF="${CONFIG_DIR}/piler.conf"
-PILER_KEY="${CONFIG_DIR}/piler.key"
-PILER_PEM="${CONFIG_DIR}/piler.pem"
-PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf"
-SPHINX_CONF="${CONFIG_DIR}/manticore.conf"
-CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php"
-PILER_MY_CNF="${CONFIG_DIR}/.my.cnf"
-RT="${RT:-0}"
-
-error() {
-   echo "ERROR:" "$*" 1>&2
-   exit 1
-}
-
-
-log() {
-   echo "DEBUG:" "$*"
-}
-
-
-pre_flight_check() {
-   [[ -v PILER_HOSTNAME ]] || error "Missing PILER_HOSTNAME env variable"
-   [[ -v MYSQL_HOSTNAME ]] || error "Missing MYSQL_HOSTNAME env variable"
-   [[ -v MYSQL_DATABASE ]] || error "Missing MYSQL_DATABASE env variable"
-   [[ -v MYSQL_USER ]]     || error "Missing MYSQL_USER env variable"
-   [[ -v MYSQL_PASSWORD ]] || error "Missing MYSQL_PASSWORD env variable"
-}
-
-
-give_it_to_piler() {
-   local f="$1"
-
-   [[ -f "$f" ]] || error "${f} does not exist, aborting"
-
-   chown "${PILER_USER}:${PILER_USER}" "$f"
-   chmod 600 "$f"
-}
-
-
-make_certificate() {
-   local f="$1"
-   local crt="/tmp/1.cert"
-   local SSL_CERT_DATA="/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com"
-
-   log "Making an ssl certificate"
-
-   openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 -subj "$SSL_CERT_DATA" -keyout "$f" -out "$crt" -sha1 2>/dev/null
-   cat "$crt" >> "$f"
-   rm -f "$crt"
-
-   give_it_to_piler "$f"
-}
-
-
-make_piler_key() {
-   local f="$1"
-
-   log "Generating piler.key"
-
-   dd if=/dev/urandom bs=56 count=1 of="$f" 2>/dev/null
-   [[ $(stat -c '%s' "$f") -eq 56 ]] || error "could not read 56 bytes from /dev/urandom to ${f}"
-
-   give_it_to_piler "$f"
-}
-
-
-fix_configs() {
-   [[ -f "$PILER_KEY" ]] || make_piler_key "$PILER_KEY"
-   [[ -f "$PILER_PEM" ]] || make_certificate "$PILER_PEM"
-
-   if [[ ! -f "$PILER_NGINX_CONF" ]]; then
-      log "Writing ${PILER_NGINX_CONF}"
-
-      cp "${PILER_NGINX_CONF}.dist" "$PILER_NGINX_CONF"
-      sed -i "s%PILER_HOST%${PILER_HOSTNAME}%" "$PILER_NGINX_CONF"
-   fi
-
-   if [[ ! -f "$PILER_CONF" ]]; then
-      log "Writing ${PILER_CONF}"
-
-      sed \
-         -e "s/mysqluser=.*/mysqluser=${MYSQL_USER}/g" \
-         -e "s/mysqldb=.*/mysqldb=${MYSQL_DATABASE}/g" \
-         -e "s/verystrongpassword/${MYSQL_PASSWORD}/g" \
-         -e "s/hostid=.*/hostid=${PILER_HOSTNAME}/g" \
-         -e "s/tls_enable=.*/tls_enable=1/g" \
-         -e "s/mysqlsocket=.*/mysqlsocket=/g" "${PILER_CONF}.dist" > "$PILER_CONF"
-
-      {
-         echo "mysqlhost=${MYSQL_HOSTNAME}"
-      } >> "$PILER_CONF"
-
-      give_it_to_piler "$PILER_CONF"
-   fi
-
-   if [[ ! -f "$CONFIG_SITE_PHP" ]]; then
-      log "Writing ${CONFIG_SITE_PHP}"
-
-      cp "${CONFIG_DIR}/config-site.dist.php" "$CONFIG_SITE_PHP"
-
-      sed -i "s%HOSTNAME%${PILER_HOSTNAME}%" "$CONFIG_SITE_PHP"
-
-      {
-         echo "\$config['DECRYPT_BINARY'] = '/usr/bin/pilerget';"
-         echo "\$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/bin/pileraget';"
-         echo "\$config['PILER_BINARY'] = '/usr/sbin/piler';"
-         echo "\$config['DB_HOSTNAME'] = '$MYSQL_HOSTNAME';"
-         echo "\$config['DB_DATABASE'] = '$MYSQL_DATABASE';"
-         echo "\$config['DB_USERNAME'] = '$MYSQL_USER';"
-         echo "\$config['DB_PASSWORD'] = '$MYSQL_PASSWORD';"
-         echo "\$config['ENABLE_MEMCACHED'] = 1;"
-         echo "\$memcached_server = ['memcached', 11211];"
-      } >> "$CONFIG_SITE_PHP"
-   fi
-
-   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \
-       -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \
-       -e "s%MYSQL_USERNAME%${MYSQL_USER}%" \
-       -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" \
-       -i "$SPHINX_CONF"
-
-   # Fixes for RT index
-
-   if [[ $RT -eq 1 ]]; then
-      sed -i "s/define('RT', 0)/define('RT', 1)/" "$SPHINX_CONF"
-      if ! grep "'RT'" "$CONFIG_SITE_PHP"; then
-         echo "\$config['RT'] = 1;" >> "$CONFIG_SITE_PHP"
-      fi
-
-      if ! grep "'SPHINX_MAIN_INDEX'" "$CONFIG_SITE_PHP"; then
-         echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
-      fi
-
-      sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
-   fi
-}
-
-
-wait_until_mysql_server_is_ready() {
-   while true; do if mysql "--defaults-file=${PILER_MY_CNF}" <<< "show databases"; then break; fi; log "${MYSQL_HOSTNAME} is not ready"; sleep 5; done
-
-   log "${MYSQL_HOSTNAME} is ready"
-}
-
-
-init_database() {
-   local table
-   local has_metadata_table=0
-
-   wait_until_mysql_server_is_ready
-
-   while read -r table; do
-      if [[ "$table" == metadata ]]; then has_metadata_table=1; fi
-   done < <(mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< 'show tables')
-
-   if [[ $has_metadata_table -eq 0 ]]; then
-      log "no metadata table, creating tables"
-
-      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" < /usr/share/piler/db-mysql.sql
-   else
-      log "metadata table exists"
-   fi
-
-   if [[ -v ADMIN_USER_PASSWORD_HASH ]]; then
-      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< "update user set password='${ADMIN_USER_PASSWORD_HASH}' where uid=0"
-   fi
-}
-
-
-create_my_cnf_files() {
-   printf "[client]\nhost = %s\nuser = %s\npassword = %s\n[mysqldump]\nhost = %s\nuser = %s\npassword = %s\n" \
-      "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" \
-      > "$PILER_MY_CNF"
-
-   give_it_to_piler "$PILER_MY_CNF"
-}
-
-
-start_services() {
-   service cron start
-   service php8.1-fpm start
-   service nginx start
-   rsyslogd
-}
-
-
-start_piler() {
-   if [[ $RT -eq 0 && ! -f "${VOLUME_DIR}/manticore/main1.spp" ]]; then
-      log "main1.spp does not exist, creating index files"
-      su -c "indexer --all --config ${SPHINX_CONF}" "$PILER_USER"
-   fi
-
-   # No pid file should exist for piler
-   rm -f /var/run/piler/*pid
-
-   /etc/init.d/rc.searchd start
-   /etc/init.d/rc.piler start
-}
-
-
-pre_flight_check
-fix_configs
-create_my_cnf_files
-init_database
-start_services
-start_piler
-
-sleep infinity
--- a/etc/Makefile.in
+++ b/etc/Makefile.in
@ -31,10 +31,8 @@ all:

 install:
 	$(INSTALL) -m 0640 -g $(RUNNING_GROUP) $(srcdir)/piler.conf $(DESTDIR)$(sysconfdir)/piler/piler.conf.dist
-	sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/sphinx.conf.in > sphinx.conf.dist
-	sed -e 's%@LOCALSTATEDIR@%$(localstatedir)%g' $(srcdir)/manticore.conf.in > manticore.conf.dist
-	$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/sphinx.conf.dist $(DESTDIR)$(sysconfdir)/piler/sphinx.conf.dist
-	$(INSTALL) -m 0755 -g $(RUNNING_GROUP) $(srcdir)/manticore.conf.dist $(DESTDIR)$(sysconfdir)/piler/manticore.conf.dist
+	sed -e 's%LOCALSTATEDIR%$(localstatedir)%g' $(srcdir)/sphinx.conf.in > sphinx.conf.dist
+	$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/sphinx.conf.dist $(DESTDIR)$(sysconfdir)/piler/sphinx.conf.dist
 	$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/config-site.dist.php $(DESTDIR)$(sysconfdir)/piler/config-site.dist.php
 	$(INSTALL) -m 0644 -g $(RUNNING_GROUP) $(srcdir)/cron.jobs.in $(DESTDIR)$(datarootdir)/piler/piler.cron
 	sed -i -e 's%LOCALSTATEDIR%$(localstatedir)%g' $(DESTDIR)$(datarootdir)/piler/piler.cron
--- a/etc/config-site.dist.php
+++ b/etc/config-site.dist.php
@ -1,13 +1,11 @@
 <?php

-define('SITE_NAME_CONST', 'SITE_NAME');
+$config['SITE_NAME'] = 'HOSTNAME';
+$config['SITE_URL'] = 'http://' . $config['SITE_NAME'] . '/';

-$config[SITE_NAME_CONST] = 'HOSTNAME';
-$config['SITE_URL'] = 'http://' . $config[SITE_NAME_CONST] . '/';
-
-$config['SMTP_DOMAIN'] = $config[SITE_NAME_CONST];
-$config['SMTP_FROMADDR'] = 'no-reply@' . $config[SITE_NAME_CONST];
-$config['ADMIN_EMAIL'] = 'admin@' . $config[SITE_NAME_CONST];
+$config['SMTP_DOMAIN'] = $config['SITE_NAME'];
+$config['SMTP_FROMADDR'] = 'no-reply@' . $config['SITE_NAME'];
+$config['ADMIN_EMAIL'] = 'admin@' . $config['SITE_NAME'];

 $config['DB_PASSWORD'] = 'MYSQL_PASSWORD';

--- a/etc/cron.jobs.in
+++ b/etc/cron.jobs.in
@ -1,14 +1,19 @@
 ### PILERSTART
 5,35 * * * * LIBEXECDIR/piler/indexer.delta.sh
 30   2 * * * LIBEXECDIR/piler/indexer.main.sh
-40 3 * * * LIBEXECDIR/piler/purge.sh
-*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet tag1 --rotate
-*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/manticore.conf --quiet note1 --rotate
+15,45 * * * * LIBEXECDIR/piler/indexer.attachment.sh
+*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet tag1 --rotate
+*/15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet note1 --rotate
 */5 * * * *  /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \;
-*/5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error

-### optional
-###30 6 * * * /usr/bin/php LIBEXECDIR/piler/generate_stats.php --webui LOCALSTATEDIR/piler/www >/dev/null
-###*/5 * * * * LIBEXECDIR/piler/import.sh
+### optional: the same report you can see on the health page
+###30 7 * * * /usr/bin/php LIBEXECDIR/piler/daily-report.php --webui LOCALSTATEDIR/piler/www
+
+### optional: populate accouting data
+###30 6 * * * /usr/bin/php LIBEXECDIR/piler/generate_stats.php --webui LOCALSTATEDIR/piler/www
+
+### optional: purge aged emails
+###2 0 * * * BINDIR/pilerpurge
+

 ### PILEREND
--- a/etc/example.conf
+++ b/etc/example.conf
@ -22,11 +22,10 @@ username=piler
 ; The default is 7 years + 2 days (=7*365+2=2557 days)
 default_retention_days=2557

-; The initialization vector for encryption.
-; By now it has become obsolete. Don't use it for
-; new installations. However, if you used it before
-; then you must keep it as it is.
-;iv=
+; this is a 16 character long vector
+; after the installation you must not change it ever
+; otherwise you can't access your emails
+;iv=****************

 ; whether to encrypt messages (1) or not (0).
 ; Make sure to set this value to your needs right after installing piler,
@ -46,12 +45,12 @@ number_of_worker_processes=2
 max_connections=64

 ; number of processed emails per each piler process
-max_requests_per_child=10000
+max_requests_per_child=1000

 ; SMTP HELO identification string
 ; this should be the FQDN part of the email address
-; where you copy emails, eg. archive@piler.example.com -> piler.example.com
-hostid=piler.example.com
+; where you copy emails, eg. archive@piler.yourdomain.com -> piler.yourdomain.com
+hostid=piler.yourdomain.com

 ; whether to process rcpt to addresses and add them to rcpt table (1) or not (0)
 process_rcpt_to_addresses=0
@ -67,9 +66,6 @@ listen_port=25

 clamd_socket=/tmp/clamd

-; check for client timeout interval. Default: 20 sec
-check_for_client_timeout_interval=20
-
 ; smtp timeout. Default: 60 sec
 smtp_timeout=60

@ -107,13 +103,6 @@ pemfile=
 cipher_list=ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS


-; set the minimum TLS protocol version for piler-smtp daemon
-;
-; Valid values: TLSv1, TLSv1.1, TLSv1.2, TLSv1.3
-; TLSv1 and TLSv1.1 are not recommended for security reasons
-tls_min_version=TLSv1.2
-
-
 ; piler's own header to indicate previously archived messages
 piler_header_field=X-piler-id:

@ -197,7 +186,7 @@ update_counters_to_memcached=0
 ; hint: if you are using a mysql replicated environment and you do _not_ want
 ; piler to write to the replicated database (because you do sync it some other
 ; way to the master database or you are not interested in keeping the counters
-; persistantly at all), then specify a big number here, that fits to the
+; persistantly at all), then specify a big number here, that fits to the 
 ; "long int" size, eg. 2147483647
 memcached_to_db_interval=900

@ -209,7 +198,7 @@ memcached_to_db_interval=900
 // this can be either utf8 or utf8mb4. Make sure to match the value
 // to the charset of the piler database! Also, make sure to set this
 // value in sphinx.conf
-mysqlcharset=utf8mb4
+mysqlcharset=utfmb4
 ;mysqlhost=127.0.0.1
 ;mysqlport=3306
 mysqlsocket=/var/run/mysqld/mysqld.sock
@ -219,14 +208,6 @@ mysqldb=piler
 mysql_connect_timeout=2


-;
-; manticore stuff
-;
-sphxhost=127.0.0.1
-sphxport=9306
-; whether to enable (1) or not (0) the rt index. If so, then piler
-; writes the index data directly to manticore's sql API
-rtindex=0

 ; if you want to change the 'sent' time as you archive the message
 ; set this in seconds. This can be a postive or negative value.
@ -237,38 +218,3 @@ tweak_sent_time_offset=0
 ; whether to enable (1) or not (0) the extra mmap dedup test feature
 ; if you change it, be sure to stop, then start piler
 mmap_dedup_test=0
-
-; security header that must be present in the first mail header of
-; the message. If the security_header value is not empty, then the
-; parser checks for this header line. Unless it's found it will discard
-; the given email. Note that this feature is supposed to be a security
-; mechanism against unwanted email in the archive if limiting smtp
-; clients via an IP-address list is not feasible.
-security_header=
-
-; By default the archive accepts any envelope recipient addresses.
-; If your archive's port 25 is wide open to the Internet (which it
-; shouldn't be, then spammers may find it, and fill it with spam.
-;
-; By setting this variable you may restrict the envelope address
-; to a single email address, eg. some-random-address-12345@archive.example.com
-; Then the archive will reject any other envelope recipients
-archive_address=
-
-; whether to enable (1) or not (0) an smtp access list similar to
-; postfix's postscreen. Valid actions in the acl file are "permit"
-; and "reject" (without quotes). See smtp.acl.example for more.
-;
-; Important! There's an implicit default deny at the end of the
-; rules. In other words if you decide to use the acl file, then
-; everyone is not explicitly permitted is denied.
-smtp_access_list=0
-
-; max message size in bytes
-; piler-smtp will reject any message that's bigger than this number
-max_message_size=50000000
-
-; max memory in bytes piler-smtp uses for buffering messages
-; when this limit is exceeded, no new emails will be accepted
-; until the used memory decreases below this level
-max_smtp_memory=500000000
--- a/etc/manticore.conf.in
+++ b/etc/manticore.conf.in
@ -1,268 +0,0 @@
-#!/usr/bin/php
-<?php
-
-define('LOCALSTATEDIR', '/var');
-define('NGRAM_CONFIG', "        #ngram_len               = 1\n        #ngram_chars             = U+3000..U+2FA1F\n");
-
-# See http://sphinxsearch.com/wiki/doku.php?id=charset_tables for more on the charset_table settings
-# The following settings contains English and some Latin extras
-define('CHARSET_TABLE', "0..9, english, _, \
-                                  U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
-                                  U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
-                                  U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
-                                  U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171\n");
-
-define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
-
-define('RT', 0);
-
-?>
-
-#
-# minimal manticore configuration suited to piler
-#
-
-<?php if(RT == 0) { ?>
-
-source base
-{
-   type = mysql
-   sql_host = MYSQL_HOSTNAME
-   sql_db = MYSQL_DATABASE
-   sql_user = MYSQL_USERNAME
-   sql_pass = MYSQL_PASSWORD
-
-   sql_attr_uint = size
-   sql_attr_uint = sent
-   sql_attr_uint = attachments
-}
-
-source delta : base
-{
-   sql_query_pre = SET NAMES utf8mb4
-   sql_query_pre  = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index
-   sql_query_post_index  = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
-   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
-
-   sql_query_killlist = SELECT `id` FROM `metadata` WHERE `deleted`=1
-}
-
-source main1 : base
-{
-   sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
-}
-
-source main2 : base
-{
-   sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
-}
-
-source main3 : base
-{
-   sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
-}
-
-source main4 : base
-{
-   sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
-}
-
-source dailydelta : base
-{
-   sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
-}
-
-index main1
-{
-        source                  = main1
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main1
-        min_prefix_len          = 5
-        min_word_len            = 1
-        stored_fields           =
-        charset_table           = <?php print CHARSET_TABLE; ?>
-<?php print NGRAM_CONFIG; ?>
-}
-
-index main2
-{
-        source                  = main2
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main2
-        min_prefix_len          = 5
-        min_word_len            = 1
-        stored_fields           =
-        charset_table           = <?php print CHARSET_TABLE; ?>
-<?php print NGRAM_CONFIG; ?>
-}
-
-index main3
-{
-        source                  = main3
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main3
-        min_prefix_len          = 5
-        min_word_len            = 1
-        stored_fields           =
-        charset_table           = <?php print CHARSET_TABLE; ?>
-<?php print NGRAM_CONFIG; ?>
-}
-
-index main4
-{
-        source                  = main4
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/main4
-        min_prefix_len          = 5
-        min_word_len            = 1
-        stored_fields           =
-        charset_table           = <?php print CHARSET_TABLE; ?>
-<?php print NGRAM_CONFIG; ?>
-}
-
-index dailydelta1
-{
-        source                  = dailydelta
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/dailydelta1
-        min_prefix_len          = 5
-        min_word_len            = 1
-        stored_fields           =
-        charset_table           = <?php print CHARSET_TABLE; ?>
-        killlist_target         = main1:kl, main2:kl, main3:kl, main4:kl, dailydelta1:kl
-<?php print NGRAM_CONFIG; ?>
-}
-
-index delta1
-{
-        source                  = delta
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/delta1
-        min_prefix_len          = 5
-        min_word_len            = 1
-        stored_fields           =
-        charset_table           = <?php print CHARSET_TABLE; ?>
-<?php print NGRAM_CONFIG; ?>
-}
-
-source tag : base
-{
-   sql_query_pre = SET NAMES utf8mb4
-   sql_query  = SELECT `_id`, `id` AS iid, `uid`, `tag` FROM `tag`
-
-   sql_attr_uint = iid
-   sql_attr_uint = uid
-}
-
-source note : base
-{
-   sql_query_pre = SET NAMES utf8mb4
-   sql_query  = SELECT `_id`, `id` AS iid, `uid`, `note` FROM `note`
-
-   sql_attr_uint = iid
-   sql_attr_uint = uid
-}
-
-index tag1
-{
-        source                  = tag
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/tag1
-        min_prefix_len          = 5
-        min_word_len            = 1
-        charset_table           = <?php print CHARSET_TABLE; ?>
-        stored_fields           =
-<?php print NGRAM_CONFIG; ?>
-}
-
-
-index note1
-{
-        source                  = note
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/manticore/note1
-        min_prefix_len          = 5
-        min_word_len            = 1
-        charset_table           = <?php print CHARSET_TABLE; ?>
-        stored_fields           =
-<?php print NGRAM_CONFIG; ?>
-}
-
-<?php } else { ?>
-
-index piler1
-{
-    type = rt
-    path = /var/piler/manticore/piler1
-    rt_mem_limit = 512M
-    stored_fields =
-    min_word_len = 1
-    min_prefix_len = 5
-    charset_table  = <?php print CHARSET_TABLE; ?>
-    # See https://manual.manticoresearch.com/Creating_an_index/Data_types#Row-wise-and-columnar-attribute-storages
-    # if you want to enable columnar storage
-    # columnar_attrs = *
-    rt_field = sender
-    rt_field = rcpt
-    rt_field = senderdomain
-    rt_field = rcptdomain
-    rt_field = subject
-    rt_field = body
-    rt_field = attachment_types
-    rt_attr_bigint = arrived
-    rt_attr_bigint = sent
-    rt_attr_uint = size
-    rt_attr_uint = direction
-    rt_attr_uint = folder
-    rt_attr_uint = attachments
-}
-
-index tag1
-{
-    type = rt
-    path = /var/piler/manticore/tag1
-    rt_mem_limit = 16M
-    stored_fields = tag
-    min_word_len = 2
-    min_prefix_len = 5
-    charset_table  = <?php print CHARSET_TABLE; ?>
-    rt_field = tag
-    rt_attr_bigint = mid
-    rt_attr_uint = uid
-}
-
-index note1
-{
-    type = rt
-    path = /var/piler/manticore/note1
-    rt_mem_limit = 16M
-    stored_fields = note
-    min_word_len = 2
-    min_prefix_len = 5
-    charset_table  = <?php print CHARSET_TABLE; ?>
-    rt_field = note
-    rt_attr_bigint = mid
-    rt_attr_uint = uid
-}
-
-<?php } ?>
-
-
-searchd
-{
-        listen                  = 127.0.0.1:9312
-        listen                  = 127.0.0.1:9306:mysql
-        listen                  = 127.0.0.1:9307:mysql_readonly
-        log                     = /var/piler/manticore/manticore.log
-        binlog_max_log_size     = 256M
-        binlog_path             = /var/piler/manticore
-        binlog_flush            = 2
-        query_log               = /var/piler/manticore/query.log
-        network_timeout         = 5
-        pid_file                = /var/run/piler/searchd.pid
-        seamless_rotate         = 1
-        preopen_tables          = 1
-        unlink_old              = 1
-        thread_stack            = 512k
-<?php if(RT) { ?>
-        rt_flush_period         = 300
-<?php } ?>
-
-}
--- a/etc/smtp.acl.example
+++ b/etc/smtp.acl.example
@ -1,9 +0,0 @@
-# Allow office365 servers. See the below URI for more
-# https://docs.microsoft.com/en-us/microsoft-365/enterprise/urls-and-ip-address-ranges?view=o365-worldwide
-#
-# Anything is not listed below will be rejected
-#
-40.92.0.0/15 permit
-40.107.0.0/16 permit
-52.100.0.0/14 permit
-104.47.0.0/17 permit
--- a/etc/sphinx.conf.in
+++ b/etc/sphinx.conf.in
@ -1,34 +1,3 @@
-#!/usr/bin/php
-<?php
-
-define('SPHINX_VERSION', 331); // If you have sphinx-3.3.1, then set SPHINX_VERSION to 331
-define('LOCALSTATEDIR', '@LOCALSTATEDIR@');
-define('NGRAM_CONFIG', "        #ngram_len               = 1\n        #ngram_chars             = U+3000..U+2FA1F\n");
-
-# See http://sphinxsearch.com/wiki/doku.php?id=charset_tables for more on the charset_table settings
-# The following settings contains English and some Latin extras
-define('SPHINX_CHARSET_TABLE', "0..9, english, _, \
-                                  U+C1->U+E1, U+C4->U+E4, U+C5->U+E5, U+C6->U+E6, U+C9->U+E9, U+CD->U+ED, U+D3->U+F3, U+D6->U+F6, U+D8->U+F8, \
-                                  U+DA->U+FA, U+DC->U+FC, U+0150->U+0151, U+0152->U+0153, U+0170->U+0171, U+01E2->U+E6, U+01E3->U+E6, U+01FC->U+E6, \
-                                  U+01FD->U+E6, U+1D01->U+E6, U+1D02->U+E6, U+1D2D->U+E6, U+1D46->U+E6, \
-                                  U+DF, U+E1, U+E4, U+E5, U+E6, U+E9, U+ED, U+00F3, U+F6, U+F8, U+FA, U+FC, U+0151, U+0153, U+0171\n");
-
-// Sphinx 3.3.1 introduced some strict rules for fulltext search column names
-// In order to comply with it you must enable SPHINX_STRICT_SCHEMA variable
-// Be sure to check out http://www.mailpiler.org/wiki/current:sphinx3 for more
-// NB: The SPHINX_STRICT_SCHEMA in sphinx.conf MUST BE THE SAME as in config.php (or in config-site.php)
-//
-define('SPHINX_STRICT_SCHEMA', 1);
-define('RT', 0);
-
-if(SPHINX_STRICT_SCHEMA) {
-   define('SELECT_FIELDS', 'id, `from` as sender, `to` as rcpt, `fromdomain` as senderdomain, `todomain` as rcptdomain, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
-} else {
-   define('SELECT_FIELDS', 'id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types`');
-}
-
-?>
-
 #
 # minimal sphinx configuration suited to piler
 #
@ -46,52 +15,46 @@ source base
   sql_attr_uint = attachments
 }

-<?php if(!RT) { ?>
-
 source delta : base
 {
   sql_query_pre = SET NAMES utf8mb4
   sql_query_pre  = REPLACE INTO sph_counter SELECT 1, IFNULL(MAX(id), 0) FROM sph_index
   sql_query_post_index  = DELETE FROM sph_index WHERE id<=(SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
-   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)
+   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index \
+        WHERE id <= (SELECT max_doc_id FROM sph_counter WHERE counter_id=1)

-<?php if(SPHINX_VERSION < 300) { ?>
   sql_query_killlist = SELECT `id` FROM `metadata` WHERE `deleted`=1
-<?php } else { ?>
-   sql_query_kbatch = SELECT `id` FROM `metadata` WHERE `deleted`=1
-<?php } ?>
 }

 source main1 : base
 {
   sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
 }

 source main2 : base
 {
   sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
 }

 source main3 : base
 {
   sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
 }

 source main4 : base
 {
   sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
 }

 source dailydelta : base
 {
   sql_query_pre = SET NAMES utf8mb4
-   sql_query = SELECT <?php print SELECT_FIELDS; ?> FROM sph_index WHERE id=-1
+   sql_query = SELECT id, `from`, `to`, `fromdomain`, `todomain`, `subject`, `arrived`, `sent`, `body`, `size`, `direction`, `folder`, `attachments`, `attachment_types` FROM sph_index WHERE id=-1
 }
-<?php } ?>

 source tag : base
 {
@ -113,171 +76,148 @@ source note : base

 }

-<?php if(!RT) { ?>
+source att : base
+{
+
+   sql_query_pre = SET NAMES utf8mb4
+   sql_query  =  select a.id as aid, m.id as mid, a.name AS aname, a.size, REPLACE(REPLACE(m.`from`, '@', 'X'), '.', 'X') as `from`, REPLACE(REPLACE((select group_concat(rcpt.`to` ORDER BY `to` ASC SEPARATOR ' ') from rcpt where rcpt.id=mid group by rcpt.id), '@', 'X'), '.', 'X') as `to` from attachment a, metadata m where m.piler_id=a.piler_id
+
+   sql_attr_uint = size
+   sql_attr_uint = mid
+}
+
+
 index main1
 {
-        source                  = main1
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main1
-<?php if(SPHINX_VERSION < 300) { ?>
-        docinfo                 = extern
-        dict                    = keywords
-<?php } ?>
+	source			= main1
+	path			= LOCALSTATEDIR/piler/sphinx/main1
+	docinfo			= extern
+	dict			= keywords
        min_prefix_len          = 5
        min_word_len            = 1
-        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-<?php print NGRAM_CONFIG; ?>
+        #ngram_len               = 1
+        #ngram_chars             = U+3000..U+2FA1F
 }

 index main2
 {
        source                  = main2
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main2
-<?php if(SPHINX_VERSION < 300) { ?>
+        path                    = LOCALSTATEDIR/piler/sphinx/main2
        docinfo                 = extern
-        dict                    = keywords
-<?php } ?>
+	dict			= keywords
        min_prefix_len          = 5
        min_word_len            = 1
-        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-<?php print NGRAM_CONFIG; ?>
+        #ngram_len               = 1
+        #ngram_chars             = U+3000..U+2FA1F
 }

 index main3
 {
        source                  = main3
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main3
-<?php if(SPHINX_VERSION < 300) { ?>
+        path                    = LOCALSTATEDIR/piler/sphinx/main3
        docinfo                 = extern
-        dict                    = keywords
-<?php } ?>
+	dict			= keywords
        min_prefix_len          = 5
        min_word_len            = 1
-        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-<?php print NGRAM_CONFIG; ?>
+        #ngram_len               = 1
+        #ngram_chars             = U+3000..U+2FA1F
 }

 index main4
 {
        source                  = main4
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/main4
-<?php if(SPHINX_VERSION < 300) { ?>
+        path                    = LOCALSTATEDIR/piler/sphinx/main4
        docinfo                 = extern
-        dict                    = keywords
-<?php } ?>
+	dict			= keywords
        min_prefix_len          = 5
        min_word_len            = 1
-        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-<?php print NGRAM_CONFIG; ?>
+        #ngram_len               = 1
+        #ngram_chars             = U+3000..U+2FA1F
 }

 index dailydelta1
 {
        source                  = dailydelta
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/dailydelta1
-<?php if(SPHINX_VERSION < 300) { ?>
+        path                    = LOCALSTATEDIR/piler/sphinx/dailydelta1
        docinfo                 = extern
-        dict                    = keywords
-<?php } ?>
+	dict			= keywords
        min_prefix_len          = 5
        min_word_len            = 1
-        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-<?php print NGRAM_CONFIG; ?>
+        #ngram_len               = 1
+        #ngram_chars             = U+3000..U+2FA1F
 }

 index delta1
 {
        source                  = delta
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/delta1
-<?php if(SPHINX_VERSION < 300) { ?>
+        path                    = LOCALSTATEDIR/piler/sphinx/delta1
        docinfo                 = extern
-        dict                    = keywords
-<?php } ?>
+	dict			= keywords
        min_prefix_len          = 5
        min_word_len            = 1
-        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-<?php print NGRAM_CONFIG; ?>
-<?php if(SPHINX_VERSION >= 310) { ?>
-        kbatch = main1, main2, main3, main4, dailydelta1
-<?php } ?>
+        #ngram_len               = 1
+        #ngram_chars             = U+3000..U+2FA1F
 }

-<?php } ?>

 index tag1
 {
        source                  = tag
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/tag1
-<?php if(SPHINX_VERSION < 300) { ?>
+        path                    = LOCALSTATEDIR/piler/sphinx/tag1
        docinfo                 = extern
-        dict                    = keywords
-<?php } ?>
+	dict			= keywords
        min_prefix_len          = 5
        min_word_len            = 1
-        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-<?php print NGRAM_CONFIG; ?>
+        #ngram_len               = 1
+        #ngram_chars             = U+3000..U+2FA1F
 }


 index note1
 {
        source                  = note
-        path                    = <?php print LOCALSTATEDIR; ?>/piler/sphinx/note1
-<?php if(SPHINX_VERSION < 300) { ?>
+        path                    = LOCALSTATEDIR/piler/sphinx/note1
        docinfo                 = extern
-        dict                    = keywords
-<?php } ?>
+	dict			= keywords
        min_prefix_len          = 5
        min_word_len            = 1
-        charset_table           = <?php print SPHINX_CHARSET_TABLE; ?>
-<?php print NGRAM_CONFIG; ?>
+        #ngram_len               = 1
+        #ngram_chars             = U+3000..U+2FA1F
 }

-<?php if(RT) { ?>
-index piler1
+
+index att1
 {
-        type = rt
-        path = /var/piler/sphinx/piler1
-        rt_mem_limit = 128M
-        rt_field = sender
-        rt_field = rcpt
-        rt_field = senderdomain
-        rt_field = rcptdomain
-        rt_field = subject
-        rt_field = body
-        rt_field = attachment_types
-        rt_attr_bigint = arrived
-        rt_attr_bigint = sent
-        rt_attr_uint = size
-        rt_attr_uint = direction
-        rt_attr_uint = folder
-        rt_attr_uint = attachments
+        source                  = att
+        path                    = /var/piler/sphinx/att1
+        docinfo                 = extern
+        dict                    = keywords
+        min_prefix_len          = 6
+        min_word_len            = 1
+        ngram_len               = 1
+        ngram_chars             = U+1100..U+2FA1F
 }
-<?php } ?>
+

 indexer
 {
-        mem_limit               = 256M
+	mem_limit		= 256M
 }


 searchd
 {
-        listen                  = 127.0.0.1:9312
-        listen                  = 127.0.0.1:9306:mysql41
-        log                     = /dev/null
-        binlog_path             = /var/piler/sphinx
-        binlog_max_log_size     = 16M
-        binlog_flush            = 2
-<?php if(RT) { ?>
-        rt_flush_period         = 300
-<?php } ?>
-        ##query_log             =
-        read_timeout            = 5
-        max_children            = 30
-        pid_file                = /var/run/piler/searchd.pid
-        seamless_rotate         = 1
-        preopen_indexes         = 1
-        unlink_old              = 1
-        thread_stack            = 512k
-        workers                 = thread_pool
+	listen			= 127.0.0.1:9312
+	listen			= 127.0.0.1:9306:mysql41
+	log			= /dev/null
+	binlog_path             = 
+	##query_log		=
+	read_timeout		= 5
+	max_children		= 30
+	pid_file		= /var/run/piler/searchd.pid
+	seamless_rotate		= 1
+	preopen_indexes		= 1
+	unlink_old		= 1
+	thread_stack		= 512k
+	workers			= threads # for RT to work
 }
--- a/init.d/rc.piler.in
+++ b/init.d/rc.piler.in
@ -15,30 +15,27 @@

 NAME=piler
 OPTIONS=""
-PID_FILE="$(SBINDIR/pilerconf $OPTIONS -q pidfile | cut -f2 -d=)"
-PID_NUMBER="$(test -f "$PID_FILE" && cat "$PID_FILE")"
-PILER_SMTP_PID="$(pgrep piler-smtp)"
+PID_FILE=`SBINDIR/pilerconf $OPTIONS -q pidfile | cut -f2 -d=`
+PID_NUMBER=`test -f ${PID_FILE} && cat ${PID_FILE}`
+PILER_SMTP_PID=$(ps uaxw | grep -w piler-smtp | grep -v grep | awk '{print $2}')

 start() {
-   echo "starting piler-smtp . . . "
-   SBINDIR/piler-smtp -d
+	echo "starting piler-smtp . . . "
+	SBINDIR/piler-smtp -d

-   echo "starting $NAME . . ."
-   SBINDIR/piler -d $OPTIONS
+	echo "starting $NAME . . ."
+	SBINDIR/piler -d $OPTIONS
 }

 stop() {
-   if [ "$PID_NUMBER" != "" ]; then echo "stopping piler"; kill "$PID_NUMBER"; fi
+	echo "stopping $NAME"
+	kill ${PID_NUMBER}

-   if [ "$PILER_SMTP_PID" != "" ]; then echo "stopping piler-smtp"; kill "$PILER_SMTP_PID"; fi
+	if [ $PILER_SMTP_PID != '' ]; then echo "stopping piler-smtp"; kill $PILER_SMTP_PID; fi
 }

 check_status(){
-        if [ -f "/proc/${PID_NUMBER}/status" ]; then
-           echo "piler is running, pid: ${PID_NUMBER}";
-        else
-           echo "piler is NOT running";
-        fi
+        test -f /proc/${PID_NUMBER}/status

        if [ "${PILER_SMTP_PID}" != '' ]; then
           echo "piler-smtp is running, pid: ${PILER_SMTP_PID}";
@ -48,34 +45,37 @@ check_status(){
 }

 case "$1" in
-        start)
-                start;
-        ;;
+	start)
+		start;
+   	;;

-        stop)
-                stop;
-        ;;
+	stop)
+		stop;
+   	;;

        status)
-                if check_status; then
-                   exit 0
-                else
-                   exit 1
+                if check_status;
+                 then
+                    echo "${NAME} is running."
+                    exit 0
+                 else
+                    echo "${NAME} is not running."
+                    exit 1
                fi
        ;;

-        restart)
-                stop;
-                sleep 1;
-                start;
-        ;;
+	restart)
+		stop;
+		sleep 1;
+		start;
+   	;;

-        reload)
-                kill -HUP "$PID_NUMBER"
-                echo "reloaded"
-        ;;
+	reload)
+		kill -HUP $PID_NUMBER
+		echo "reloaded"
+	;;

-        *)
-                echo "Usage: $0 start|stop|restart|reload|status"
-        ;;
+	*)
+		echo "Usage: $0 start|stop|restart|reload|status"
 esac
+
--- a/init.d/rc.searchd.in
+++ b/init.d/rc.searchd.in
@ -16,13 +16,7 @@ export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin

 NAME=searchd
 PID_FILE=/var/run/piler/searchd.pid
-PID_NUMBER=$(test -f ${PID_FILE} && cat ${PID_FILE})
-CONFIG_FILE=sphinx.conf
-
-if [ -f SYSCONFDIR/piler/MANTICORE ]; then
-   CONFIG_FILE=manticore.conf
-fi
-
+PID_NUMBER=`test -f ${PID_FILE} && cat ${PID_FILE}`

 start() {
 	echo "starting searchd . . ."
@ -32,8 +26,8 @@ start() {
 		chown piler:piler /var/run/piler
 	fi

-        if [ $(id -u) -eq 0 ]; then
-		su piler -c "searchd --config SYSCONFDIR/piler/${CONFIG_FILE}"
+        if [ `id -u` -eq 0 ]; then
+		su piler -c "searchd --config SYSCONFDIR/piler/sphinx.conf"
 	else
 		searchd
 	fi
@ -41,7 +35,7 @@ start() {

 stop() {
 	echo "stopping searchd"
-	kill "$PID_NUMBER"
+	kill ${PID_NUMBER}
 }

 check_status(){
--- a/piler-config.h.in
+++ b/piler-config.h.in
@ -2,8 +2,6 @@
 * piler-config.h.in, SJ
 */

-#define VERSION "x.x.x"
-
 #define CONFDIR "/usr/local/etc"
 #define DATADIR "/usr/local/var"
 #define DATAROOTDIR "/usr/local/share"
@ -26,6 +24,9 @@
 #undef HAVE_TNEF
 #undef HAVE_ZIP

+#undef HAVE_LIBWRAP
+
 #undef HAVE_TWEAK_SENT_TIME

 #undef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
+
--- a/src/Makefile.in
+++ b/src/Makefile.in
@ -33,7 +33,7 @@ MAKE = `which make`

 INSTALL = @INSTALL@

-all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport reindex test stats piler-smtp
+all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport reindex test piler-smtp
 install: install-piler


@ -43,12 +43,12 @@ piler: piler.c libpiler.a
 libpiler.a: $(OBJS) $(SQL_OBJS)
 	ar cr libpiler.a $(OBJS) $(SQL_OBJS)
 	ranlib libpiler.a
-	$(CC) -shared -Wl,-soname,libpiler.so.$(LIBPILER_VERSION) -o libpiler.so.$(LIBPILER_VERSION) $(OBJS) $(SQL_OBJS)
+	$(CC) -shared -o libpiler.so.$(LIBPILER_VERSION) $(OBJS) $(SQL_OBJS)
 	ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so
 	ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so.$(PILER_VERSION)

 piler-smtp: piler-smtp.c libpiler.a
-	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< cfg.o misc.o tai.o smtp.o session.o dirs.o sig.o bdat.o screen.o $(LIBS) $(LIBDIR)
+	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< cfg.o misc.o tai.o smtp.o session.o dirs.o sig.o bdat.o $(LIBS) $(LIBDIR)

 pilerget: pilerget.c libpiler.a
 	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
@ -71,11 +71,8 @@ reindex: reindex.c libpiler.a
 test: test.c libpiler.a
 	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o pilertest $< -lpiler $(LIBS) $(LIBDIR) @LDFLAGS@

-stats: stats.c libpiler.a
-	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o pilerstats $< -lpiler $(LIBS) $(LIBDIR) @LDFLAGS@
-
 %.o: $(srcdir)/%.c
-	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -c $< -o $@
+	$(CC) $(CFLAGS) -fPIC $(INCDIR) $(DEFS) -c $< -o $@


 install-piler:
@ -93,10 +90,9 @@ install-piler:
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerexport $(DESTDIR)$(bindir)
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) reindex $(DESTDIR)$(bindir)
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilertest $(DESTDIR)$(bindir)
-	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerstats $(DESTDIR)$(bindir)

 clean:
-	rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilertest pilerstats reindex piler-smtp
+	rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilertest reindex piler-smtp

 distclean: clean
 	rm -f Makefile
--- a/src/archive.c
+++ b/src/archive.c
@ -16,9 +16,6 @@
 #include <syslog.h>
 #include <openssl/blowfish.h>
 #include <openssl/evp.h>
-#if OPENSSL_VERSION_MAJOR >= 3
-   #include <openssl/provider.h>
-#endif
 #include <zlib.h>
 #include <assert.h>
 #include <piler.h>
@ -57,14 +54,11 @@ void zerr(int ret){

 int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
   int ret, pos=0;
+   unsigned have;
   z_stream strm;
   char *new_ptr;
   unsigned char out[REALLYBIGBUFSIZE];

-   /* expecting deflate with 32k window size (0x78) */
-   if(len > 0 && in[0] != 0x78)
-      return Z_DATA_ERROR;
-
   /* allocate inflate state */

   strm.zalloc = Z_NULL;
@ -102,7 +96,7 @@ int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
                return ret;
      }

-      unsigned have = REALLYBIGBUFSIZE - strm.avail_out;
+      have = REALLYBIGBUFSIZE - strm.avail_out;

      /*
       * write the uncompressed result either to stdout
@ -137,16 +131,11 @@ int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
 }


-int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct config *cfg){
+int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct __config *cfg){
   int rc=0, n, olen, tlen, len, fd=-1;
   unsigned char *s=NULL, *addr=NULL, inbuf[REALLYBIGBUFSIZE];
   struct stat st;
-#if OPENSSL_VERSION_NUMBER < 0x10100000L
   EVP_CIPHER_CTX ctx;
-#else
-   EVP_CIPHER_CTX *ctx=NULL;
-#endif
-   int blocklen;


   if(filename == NULL) return 1;
@ -165,47 +154,12 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
      return 1;
   }

-   // The new encryption scheme uses piler id starting with 5000....

   if(cfg->encrypt_messages == 1){
-   #if OPENSSL_VERSION_NUMBER < 0x10100000L
      EVP_CIPHER_CTX_init(&ctx);
-      if(strstr(filename, "/5000")){
-         rc = EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
-      } else {
-         rc = EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
-      }
+      EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);

-      if(!rc){
-         syslog(LOG_PRIORITY, "ERROR: EVP_DecryptInit_ex()");
-         goto CLEANUP;
-      }
-
-      blocklen = EVP_CIPHER_CTX_block_size(&ctx);
-   #else
-      ctx = EVP_CIPHER_CTX_new();
-      if(!ctx) goto CLEANUP;
-
-      EVP_CIPHER_CTX_init(ctx);
-      if(strstr(filename, "/5000")){
-         rc = EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
-      } else {
-      #if OPENSSL_VERSION_MAJOR >= 3
-         OSSL_PROVIDER_load(NULL, "legacy");
-         OSSL_PROVIDER_load(NULL, "default");
-      #endif
-         rc = EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
-      }
-
-      if(!rc){
-         syslog(LOG_PRIORITY, "ERROR: EVP_DecryptInit_ex()");
-         goto CLEANUP;
-      }
-
-      blocklen = EVP_CIPHER_CTX_block_size(ctx);
-   #endif
-
-      len = st.st_size+blocklen;
+      len = st.st_size+EVP_MAX_BLOCK_LENGTH;

      s = malloc(len);

@ -218,11 +172,7 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de

      while((n = read(fd, inbuf, sizeof(inbuf)))){

-      #if OPENSSL_VERSION_NUMBER < 0x10100000L
         if(!EVP_DecryptUpdate(&ctx, s+tlen, &olen, inbuf, n)){
-      #else
-         if(!EVP_DecryptUpdate(ctx, s+tlen, &olen, inbuf, n)){
-      #endif
            syslog(LOG_PRIORITY, "%s: EVP_DecryptUpdate()", filename);
            goto CLEANUP;
         }
@ -231,24 +181,14 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
      }


-   #if OPENSSL_VERSION_NUMBER < 0x10100000L
      if(EVP_DecryptFinal(&ctx, s + tlen, &olen) != 1){
-   #else
-      if(EVP_DecryptFinal(ctx, s + tlen, &olen) != 1){
-   #endif
         syslog(LOG_PRIORITY, "%s: EVP_DecryptFinal()", filename);
         goto CLEANUP;
      }


      tlen += olen;
-
-      // old fileformat with static IV
      rc = inf(s, tlen, mode, buffer, dest);
-      // new fileformat, starting with blocklen bytes of garbage
-      if(rc != Z_OK && tlen >= blocklen){
-         rc = inf(s+blocklen, tlen-blocklen, mode, buffer, dest);
-      }
   }
   else {
      addr = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
@ -261,22 +201,17 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de


 CLEANUP:
-   if(fd != -1) close(fd); //-V547
+   if(fd != -1) close(fd);
   if(s) free(s);
-   if(cfg->encrypt_messages == 1)
-   #if OPENSSL_VERSION_NUMBER < 0x10100000L
-      EVP_CIPHER_CTX_cleanup(&ctx);
-   #else
-      if(ctx) EVP_CIPHER_CTX_free(ctx);
-   #endif
+   if(cfg->encrypt_messages == 1) EVP_CIPHER_CTX_cleanup(&ctx);

   return 0;
 }


-int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct config *cfg){
-   int attachments;
-   char *buffer=NULL, *saved_buffer, *p, filename[SMALLBUFSIZE];
+int retrieve_email_from_archive(struct session_data *sdata, struct __data *data, FILE *dest, struct __config *cfg){
+   int i, attachments;
+   char *buffer=NULL, *saved_buffer, *p, filename[SMALLBUFSIZE], pointer[SMALLBUFSIZE];
   struct ptr_array ptr_arr[MAX_ATTACHMENTS];
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
   struct stat st;
@ -287,14 +222,14 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct c
      return 1;
   }

-   attachments = query_attachments(sdata, &ptr_arr[0]);
+   attachments = query_attachments(sdata, data, &ptr_arr[0]);

   if(attachments == -1){
      printf("problem querying the attachment of %s\n", sdata->ttmpfile);
      return 1;
   }

-   snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, *(sdata->ttmpfile+24), *(sdata->ttmpfile+25), *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
+   snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
   if(stat(filename, &st)){
      snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+RND_STR_LEN-6), *(sdata->ttmpfile+RND_STR_LEN-5), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
@ -310,8 +245,7 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct c
      if(buffer){
         saved_buffer = buffer;

-         for(int i=1; i<=attachments; i++){
-            char pointer[SMALLBUFSIZE];
+         for(i=1; i<=attachments; i++){
            snprintf(pointer, sizeof(pointer)-1, "ATTACHMENT_POINTER_%s.a%d_XXX_PILER", sdata->ttmpfile, i);

            p = strstr(buffer, pointer);
@ -321,7 +255,7 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct c
               buffer = p + strlen(pointer);

               if(strlen(ptr_arr[i].piler_id) == RND_STR_LEN){
-                  snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, ptr_arr[i].piler_id[24], ptr_arr[i].piler_id[25], ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);
+                  snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);

               #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
                  if(stat(filename, &st)){
@ -346,3 +280,5 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct c

   return 0;
 }
+
+
--- a/src/attachment.c
+++ b/src/attachment.c
@ -16,34 +16,35 @@
 #include <piler.h>


-int store_attachments(struct session_data *sdata, struct parser_state *state, struct config *cfg){
+int store_attachments(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg){
+   uint64 id=0;
   int i, rc=1, found, affected_rows;
-   struct sql sql, sql2;

-   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE) == ERR) return rc;
-   if(prepare_sql_statement(sdata, &sql2, SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE) == ERR) return rc;
+
+   if(prepare_sql_statement(sdata, &(data->stmt_insert_into_attachment_table), SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE) == ERR) return rc;
+   if(prepare_sql_statement(sdata, &(data->stmt_get_attachment_id_by_signature), SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE) == ERR) return rc;


   for(i=1; i<=state->n_attachments; i++){
      found = 0;
-      uint64 id = 0;
+      id = 0;

      if(state->attachments[i].size > 0){

-         p_bind_init(&sql2);
+         p_bind_init(data);

-         sql2.sql[sql2.pos] = state->attachments[i].digest; sql2.type[sql2.pos] = TYPE_STRING; sql2.pos++;
-         sql2.sql[sql2.pos] = (char *)&(state->attachments[i].size); sql2.type[sql2.pos] = TYPE_LONG; sql2.pos++;
+         data->sql[data->pos] = state->attachments[i].digest; data->type[data->pos] = TYPE_STRING; data->pos++;
+         data->sql[data->pos] = (char *)&(state->attachments[i].size); data->type[data->pos] = TYPE_LONG; data->pos++;

-         if(p_exec_stmt(sdata, &sql2) == OK){
+         if(p_exec_query(sdata, data->stmt_get_attachment_id_by_signature, data) == OK){

-            p_bind_init(&sql2);
+            p_bind_init(data);

-            sql2.sql[sql2.pos] = (char *)&id; sql2.type[sql2.pos] = TYPE_LONGLONG; sql2.len[sql2.pos] = sizeof(uint64); sql2.pos++;
+            data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;

-            p_store_results(&sql2);
-            if(p_fetch_results(&sql2) == OK) found = 1;
-            p_free_results(&sql2);
+            p_store_results(data->stmt_get_attachment_id_by_signature, data);
+            if(p_fetch_results(data->stmt_get_attachment_id_by_signature) == OK) found = 1;
+            p_free_results(data->stmt_get_attachment_id_by_signature);
         }

         if(found == 0){
@ -54,20 +55,20 @@ int store_attachments(struct session_data *sdata, struct parser_state *state, st
         }


-         p_bind_init(&sql);
+         p_bind_init(data);

-         sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-         sql.sql[sql.pos] = (char *)&i; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-         sql.sql[sql.pos] = state->attachments[i].digest; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-         sql.sql[sql.pos] = state->attachments[i].filename; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-         sql.sql[sql.pos] = state->attachments[i].type; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-         sql.sql[sql.pos] = (char *)&(state->attachments[i].size); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-         sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
+         data->sql[data->pos] = sdata->ttmpfile; data->type[data->pos] = TYPE_STRING; data->pos++;
+         data->sql[data->pos] = (char *)&i; data->type[data->pos] = TYPE_LONG; data->pos++;
+         data->sql[data->pos] = state->attachments[i].digest; data->type[data->pos] = TYPE_STRING; data->pos++;
+         data->sql[data->pos] = state->attachments[i].filename; data->type[data->pos] = TYPE_STRING; data->pos++;
+         data->sql[data->pos] = state->attachments[i].type; data->type[data->pos] = TYPE_STRING; data->pos++;
+         data->sql[data->pos] = (char *)&(state->attachments[i].size); data->type[data->pos] = TYPE_LONG; data->pos++;
+         data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->pos++;

-         if(p_exec_stmt(sdata, &sql) == ERR) goto CLOSE;
+         if(p_exec_query(sdata, data->stmt_insert_into_attachment_table, data) == ERR) goto CLOSE;


-         affected_rows = p_get_affected_rows(&sql);
+         affected_rows = p_get_affected_rows(data->stmt_insert_into_attachment_table);
         if(affected_rows != 1){
            syslog(LOG_PRIORITY, "%s attachment sql error: affected rows: %d", sdata->ttmpfile, affected_rows);
            goto CLOSE;
@ -83,70 +84,69 @@ int store_attachments(struct session_data *sdata, struct parser_state *state, st
   rc = 0;

 CLOSE:
-   close_prepared_statement(&sql);
-   close_prepared_statement(&sql2);
+   close_prepared_statement(data->stmt_insert_into_attachment_table);
+   close_prepared_statement(data->stmt_get_attachment_id_by_signature);

   return rc;
 }


-int query_attachment_pointers(struct session_data *sdata, uint64 ptr, char *piler_id, int *id){
+int query_attachment_pointers(struct session_data *sdata, struct __data *data, uint64 ptr, char *piler_id, int *id){
   int rc=0;
-   struct sql sql;

-   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER) == ERR) return rc;
+   if(prepare_sql_statement(sdata, &(data->stmt_get_attachment_pointer), SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER) == ERR) return rc;

-   p_bind_init(&sql);

-   sql.sql[sql.pos] = (char *)&ptr; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
+   p_bind_init(data);

-   if(p_exec_stmt(sdata, &sql) == OK){
+   data->sql[data->pos] = (char *)&ptr; data->type[data->pos] = TYPE_LONGLONG; data->pos++;

-      p_bind_init(&sql);
+   if(p_exec_query(sdata, data->stmt_get_attachment_pointer, data) == OK){

-      sql.sql[sql.pos] = piler_id; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++;
-      sql.sql[sql.pos] = (char *)id; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++;
+      p_bind_init(data);

-      p_store_results(&sql);
+      data->sql[data->pos] = piler_id; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = RND_STR_LEN; data->pos++;
+      data->sql[data->pos] = (char *)id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;

-      if(p_fetch_results(&sql) == OK) rc = 1;
-      p_free_results(&sql);
+      p_store_results(data->stmt_get_attachment_pointer, data);
+
+      if(p_fetch_results(data->stmt_get_attachment_pointer) == OK) rc = 1;
+      p_free_results(data->stmt_get_attachment_pointer);
   }

-   close_prepared_statement(&sql);
+   close_prepared_statement(data->stmt_get_attachment_pointer);

   return rc;
 }


-int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr){
+int query_attachments(struct session_data *sdata, struct __data *data, struct ptr_array *ptr_arr){
   int i, rc, id, attachments=0;
   uint64 ptr;
-   struct sql sql;

   for(i=0; i<MAX_ATTACHMENTS; i++) memset((char*)&ptr_arr[i], 0, sizeof(struct ptr_array));

-   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_QUERY_ATTACHMENT) == ERR) return attachments;
+   if(prepare_sql_statement(sdata, &(data->stmt_query_attachment), SQL_PREPARED_STMT_QUERY_ATTACHMENT) == ERR) return attachments;

-   p_bind_init(&sql);
+   p_bind_init(data);

-   sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   data->sql[data->pos] = sdata->ttmpfile; data->type[data->pos] = TYPE_STRING; data->pos++;

-   if(p_exec_stmt(sdata, &sql) == ERR) goto CLOSE;
+   if(p_exec_query(sdata, data->stmt_query_attachment, data) == ERR) goto CLOSE;

-   p_bind_init(&sql);
+   p_bind_init(data);

-   sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++;
-   sql.sql[sql.pos] = (char *)&ptr; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+   data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
+   data->sql[data->pos] = (char *)&ptr; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;

-   p_store_results(&sql);
+   p_store_results(data->stmt_query_attachment, data);

-   while(p_fetch_results(&sql) == OK){
+   while(p_fetch_results(data->stmt_query_attachment) == OK){

      if(id > 0 && id < MAX_ATTACHMENTS){
         if(ptr > 0){
            ptr_arr[id].ptr = ptr;
-            rc = query_attachment_pointers(sdata, ptr, &(ptr_arr[id].piler_id[0]), &(ptr_arr[id].attachment_id));
+            rc = query_attachment_pointers(sdata, data, ptr, &(ptr_arr[id].piler_id[0]), &(ptr_arr[id].attachment_id));
            if(!rc){
               attachments = -1;
               goto CLOSE;
@ -161,10 +161,12 @@ int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr){
      }
   }

-   p_free_results(&sql);
+   p_free_results(data->stmt_query_attachment);

 CLOSE:
-   close_prepared_statement(&sql);
+   close_prepared_statement(data->stmt_query_attachment);

   return attachments;
 }
+
+
--- a/src/av.h
+++ b/src/av.h
@ -20,6 +20,6 @@
 #define CLAMD_RESP_INFECTED "FOUND"
 #define CLAMD_RESP_ERROR "ERROR"

-int clamd_scan(char *tmpfile, struct config *cfg);
+int clamd_scan(char *tmpfile, struct __config *cfg);

 #endif /* _AV_H */
--- a/src/avir.c
+++ b/src/avir.c
@ -10,7 +10,7 @@
 #include <piler.h>


-int do_av_check(char *filename, struct config *cfg){
+int do_av_check(char *filename, struct __config *cfg){
   int rav = AVIR_OK;

   if(clamd_scan(filename, cfg) == AV_VIRUS) rav = AVIR_VIRUS;
--- a/src/base64.c
+++ b/src/base64.c
@ -0,0 +1,79 @@
+/*
+ * base64.c, SJ
+ */
+
+#include <stdio.h>
+#include <string.h>
+
+
+char base64_value(char c){
+   static const char *base64_table = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+
+   if((int)c > 63) return '=';
+
+   return base64_table[(int)c];
+}
+
+
+void base64_encode_block(unsigned char *in, int inlen, char *out){
+   char a, b, c, d, fragment;
+
+   sprintf(out, "====");
+
+   if(inlen <= 0) return;
+
+   fragment = *in & 0x3;
+
+   a = *in >> 2;
+
+   out[0] = base64_value(a);
+
+   b = fragment << 4;
+
+   if(inlen > 1)
+      b += *(in+1) >> 4;
+
+   out[1] = base64_value(b);
+
+   if(inlen == 1) return;
+
+
+   c = *(in+1) & 0xf;
+   c = c << 2;
+
+   if(inlen > 2){
+      fragment = *(in+2) & 0xfc;
+      c += fragment >> 6;
+
+      d = *(in+2) & 0x3f;
+      out[3] = base64_value(d);
+   }
+
+   out[2] = base64_value(c);
+}
+
+
+void base64_encode(unsigned char *in, int inlen, char *out, int outlen){
+   int i=0, j, pos=0;
+   unsigned char buf[3];
+
+   memset(buf, 0, 3);
+   memset(out, 0, outlen);
+
+   for(j=0; j<inlen; j++){
+
+      if(i == 3){
+         base64_encode_block(buf, 3, &out[pos]); pos += 4;
+
+         memset(buf, 0, 3);
+
+         i = 0;
+      }
+
+      buf[i] = *(in+j);
+      i++;       
+   }
+
+   base64_encode_block(buf, i, &out[pos]);
+}
+
--- a/src/bdat.c
+++ b/src/bdat.c
@ -21,62 +21,70 @@


 void reset_bdat_counters(struct smtp_session *session){
+   session->bdat_rounds = 0;
+   session->bdat_last_round = 0;
   session->bdat_bytes_to_read = 0;
   session->bad = 0;
 }


-void get_bdat_size_to_read(struct smtp_session *session){
+void get_bdat_size_to_read(struct smtp_session *session, char *buf){
   char *p;

+   session->bdat_rounds++;
   session->bdat_bytes_to_read = 0;

   session->protocol_state = SMTP_STATE_BDAT;

-   p = strcasestr(session->buf, " LAST");
+   // determine if this is the last BDAT command
+
+   p = strcasestr(buf, " LAST");
   if(p){
+      session->bdat_last_round = 1;
+      syslog(LOG_INFO, "%s: BDAT LAST", session->ttmpfile);
      *p = '\0';
   }

   // determine the size to be read

-   p = strchr(session->buf, ' ');
+   p = strchr(buf, ' ');
   if(p){
      session->bdat_bytes_to_read = atoi(p);
-      if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "fd=%d: BDAT len=%d", session->net.socket, session->bdat_bytes_to_read);
+      if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: BDAT len=%d", session->ttmpfile, session->bdat_bytes_to_read);
   }

   if(!p || session->bdat_bytes_to_read <= 0){
      session->bdat_bytes_to_read = 0;
-      syslog(LOG_INFO, "%s: ERROR: malformed BDAT command", session->ttmpfile);
+      syslog(LOG_INFO, "%s: malformed BDAT command", session->ttmpfile);
   }
 }


-void process_bdat(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg){
+void process_bdat(struct smtp_session *session, char *readbuf, int readlen){
+   int i;
+   char buf[SMALLBUFSIZE];
+
   if(readlen <= 0) return;

-   if(session->fd == -1){
+   //printf("readbuf in process_bdat (%d): *%s*\n", readlen, readbuf);
+
+   if(session->bdat_rounds == 1){
      session->fd = open(session->ttmpfile, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR|S_IRGRP);
      if(session->fd == -1){
         syslog(LOG_PRIORITY, "%s: %s", ERR_OPEN_TMP_FILE, session->ttmpfile);
      }
-
-      if(cfg->process_rcpt_to_addresses == 1) write_envelope_addresses(session, cfg);
   }

   session->bdat_bytes_to_read -= readlen;

   if(session->fd != -1){
-      if(write(session->fd, readbuf, readlen) != -1){
-         session->tot_len += readlen;
+      write(session->fd, readbuf, readlen);
+      session->tot_len += readlen;

-         if(session->cfg->verbosity >= _LOG_EXTREME) syslog(LOG_INFO, "%s: wrote %d bytes, %d bytes to go", session->ttmpfile, readlen, session->bdat_bytes_to_read);
-      }
-      else syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
+      if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: wrote %d bytes, %d bytes to go", session->ttmpfile, readlen, session->bdat_bytes_to_read);
   }

-
+   
   if(session->bdat_bytes_to_read < 0){
      // malformed data from client: we got more data then had been told in BDAT argument
      syslog(LOG_PRIORITY, "ERROR: invalid BDAT data. Expected %d, got %d bytes", session->bdat_bytes_to_read + readlen, readlen);
@ -84,34 +92,45 @@ void process_bdat(struct smtp_session *session, char *readbuf, int readlen, stru

      close(session->fd);
      unlink(session->ttmpfile);
-
-      session->fd = -1;
   }

+   if(session->bdat_bytes_to_read == 0){

-   // If there's nothing more to read, then send response to smtp client
+      if(session->bdat_last_round == 1){
+         if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: read all bdat data in %d rounds", session->ttmpfile, session->bdat_rounds);

-   if(session->bdat_bytes_to_read <= 0){
+         // send back the smtp answers
+         for(i=0; i<session->bdat_rounds; i++){
+            if(session->fd == -1){
+               send_smtp_response(session, SMTP_RESP_421_ERR_WRITE_FAILED);
+            }
+            else {
+               if(i == 0){
+                  fsync(session->fd);
+                  close(session->fd);

-      if(session->fd == -1){
-         send_smtp_response(session, SMTP_RESP_421_ERR_WRITE_FAILED);
-      }
-      else {
-         fsync(session->fd);
-         close(session->fd);
+                  move_email(session);
+
+                  snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
+                  send_smtp_response(session, buf);
+                  syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d", session->ttmpfile, session->mailfrom, session->tot_len);
+               }
+               else send_smtp_response(session, SMTP_RESP_250_BDAT);
+            }
+         }
+
+         // technically we are not in the PERIOD state, but it's good enough
+         // to quit the BDAT processing state
+         session->protocol_state = SMTP_STATE_PERIOD;

         session->fd = -1;
-
-         move_email(session);
-
-         char buf[SMALLBUFSIZE];
-         snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
-         send_smtp_response(session, buf);
-         syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d, client=%s, fd=%d", session->ttmpfile, session->mailfrom, session->tot_len, session->remote_host, session->net.socket);
      }

-      // technically we are not in the PERIOD state, but it's good enough
-      // to quit the BDAT processing state
-      session->protocol_state = SMTP_STATE_PERIOD;
+      else {
+         // this is not the last BDAT round, let's go back
+         // after the rcpt state, and wait for the next
+         // BDAT command
+         session->protocol_state = SMTP_STATE_RCPT_TO;
+      }
   }
 }
--- a/src/cfg.c
+++ b/src/cfg.c
@ -17,16 +17,30 @@ int string_parser(char *src, char *target, int limit){
   return 0;
 };

+int multi_line_string_parser(char *src, char *target, unsigned int limit){
+   if(strlen(src) > 0 && strlen(target) + strlen(src) + 3 < limit){
+      strncat(target, src, limit-strlen(target));
+      strncat(target, "\r\n", limit-strlen(target));
+
+      return 0;
+   }
+
+   return 1;
+};
+
 int int_parser(char *src, int *target){
   *target = strtol(src, (char **) NULL, 10);

   return 0;
 };

-int uint64_parser(char *src, uint64 *target){
-   *target = strtoull(src, (char**)NULL, 10);
+
+int float_parser(char *src, float *target){
+   *target = strtof(src, (char **) NULL);
+
   return 0;
-}
+};
+

 struct _parse_rule {
   char *name;
@ -44,71 +58,61 @@ struct _parse_rule {

 struct _parse_rule config_parse_rules[] =
 {
-   { "archive_address", "string", (void*) string_parser, offsetof(struct config, archive_address), "", MAXVAL-1},
-   { "archive_emails_not_having_message_id", "integer", (void*) int_parser, offsetof(struct config, archive_emails_not_having_message_id), "0", sizeof(int)},
-   { "archive_only_mydomains", "integer", (void*) int_parser, offsetof(struct config, archive_only_mydomains), "0", sizeof(int)},
-   { "backlog", "integer", (void*) int_parser, offsetof(struct config, backlog), "20", sizeof(int)},
-   { "check_for_client_timeout_interval", "integer", (void*) int_parser, offsetof(struct config, check_for_client_timeout_interval), "20", sizeof(int)},
-   { "cipher_list", "string", (void*) string_parser, offsetof(struct config, cipher_list), "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS", MAXVAL-1},
-   { "clamd_addr", "string", (void*) string_parser, offsetof(struct config, clamd_addr), "", MAXVAL-1},
-   { "clamd_port", "integer", (void*) int_parser, offsetof(struct config, clamd_port), "0", sizeof(int)},
-   { "clamd_socket", "string", (void*) string_parser, offsetof(struct config, clamd_socket), CLAMD_SOCKET, MAXVAL-1},
-   { "debug", "integer", (void*) int_parser, offsetof(struct config, debug), "0", sizeof(int)},
-   { "default_retention_days", "integer", (void*) int_parser, offsetof(struct config, default_retention_days), "2557", sizeof(int)},
-   { "enable_chunking", "integer", (void*) int_parser, offsetof(struct config, enable_chunking), "0", sizeof(int)},
-   { "enable_cjk", "integer", (void*) int_parser, offsetof(struct config, enable_cjk), "0", sizeof(int)},
-   { "enable_folders", "integer", (void*) int_parser, offsetof(struct config, enable_folders), "0", sizeof(int)},
-   { "encrypt_messages", "integer", (void*) int_parser, offsetof(struct config, encrypt_messages), "1", sizeof(int)},
-   { "extra_to_field", "string", (void*) string_parser, offsetof(struct config, extra_to_field), "", MAXVAL-1},
-   { "extract_attachments", "integer", (void*) int_parser, offsetof(struct config, extract_attachments), "1", sizeof(int)},
-   { "helper_timeout", "integer", (void*) int_parser, offsetof(struct config, helper_timeout), "20", sizeof(int)},
-   { "hostid", "string", (void*) string_parser, offsetof(struct config, hostid), HOSTID, MAXVAL-1},
-   { "iv", "string", (void*) string_parser, offsetof(struct config, iv), "", MAXVAL-1},
-   { "listen_addr", "string", (void*) string_parser, offsetof(struct config, listen_addr), "0.0.0.0", MAXVAL-1},
-   { "listen_port", "integer", (void*) int_parser, offsetof(struct config, listen_port), "25", sizeof(int)},
-   { "locale", "string", (void*) string_parser, offsetof(struct config, locale), "", MAXVAL-1},
-   { "max_connections", "integer", (void*) int_parser, offsetof(struct config, max_connections), "64", sizeof(int)},
-   { "max_message_size", "integer", (void*) int_parser, offsetof(struct config, max_message_size), "50000000", sizeof(int)},
-   { "max_requests_per_child", "integer", (void*) int_parser, offsetof(struct config, max_requests_per_child), "10000", sizeof(int)},
-   { "max_smtp_memory", "uint64", (void*) uint64_parser, offsetof(struct config, max_smtp_memory), "500000000", sizeof(uint64)},
-   { "memcached_servers", "string", (void*) string_parser, offsetof(struct config, memcached_servers), "127.0.0.1", MAXVAL-1},
-   { "memcached_to_db_interval", "integer", (void*) int_parser, offsetof(struct config, memcached_to_db_interval), "900", sizeof(int)},
-   { "memcached_ttl", "integer", (void*) int_parser, offsetof(struct config, memcached_ttl), "86400", sizeof(int)},
-   { "min_message_size", "integer", (void*) int_parser, offsetof(struct config, min_message_size), "100", sizeof(int)},
-   { "min_word_len", "integer", (void*) int_parser, offsetof(struct config, min_word_len), "1", sizeof(int)},
-   { "mmap_dedup_test", "integer", (void*) int_parser, offsetof(struct config, mmap_dedup_test), "0", sizeof(int)},
-   { "mysqlcharset", "string", (void*) string_parser, offsetof(struct config, mysqlcharset), "utf8mb4", MAXVAL-1},
-   { "mysqlhost", "string", (void*) string_parser, offsetof(struct config, mysqlhost), "", MAXVAL-1},
-   { "mysqlport", "integer", (void*) int_parser, offsetof(struct config, mysqlport), "", sizeof(int)},
-   { "mysqlsocket", "string", (void*) string_parser, offsetof(struct config, mysqlsocket), "/tmp/mysql.sock", MAXVAL-1},
-   { "mysqluser", "string", (void*) string_parser, offsetof(struct config, mysqluser), "piler", MAXVAL-1},
-   { "mysqlpwd", "string", (void*) string_parser, offsetof(struct config, mysqlpwd), "", MAXVAL-1},
-   { "mysqldb", "string", (void*) string_parser, offsetof(struct config, mysqldb), "piler", MAXVAL-1},
-   { "mysql_connect_timeout", "integer", (void*) int_parser, offsetof(struct config, mysql_connect_timeout), "2", sizeof(int)},
-   { "number_of_worker_processes", "integer", (void*) int_parser, offsetof(struct config, number_of_worker_processes), "2", sizeof(int)},
-   { "pemfile", "string", (void*) string_parser, offsetof(struct config, pemfile), "", MAXVAL-1},
-   { "pidfile", "string", (void*) string_parser, offsetof(struct config, pidfile), PIDFILE, MAXVAL-1},
-   { "piler_header_field", "string", (void*) string_parser, offsetof(struct config, piler_header_field), "X-piler-id:", MAXVAL-1},
-   { "process_rcpt_to_addresses", "integer", (void*) int_parser, offsetof(struct config, process_rcpt_to_addresses), "0", sizeof(int)},
-   { "queuedir", "string", (void*) string_parser, offsetof(struct config, queuedir), QUEUE_DIR, MAXVAL-1},
-   { "rtindex", "integer", (void*) int_parser, offsetof(struct config, rtindex), "0", sizeof(int)},
-   { "security_header", "string", (void*) string_parser, offsetof(struct config, security_header), "", MAXVAL-1},
-   { "server_id", "integer", (void*) int_parser, offsetof(struct config, server_id), "0", sizeof(int)},
-   { "sphxdb", "string", (void*) string_parser, offsetof(struct config, sphxdb), "piler1", MAXVAL-1},
-   { "sphxhost", "string", (void*) string_parser, offsetof(struct config, sphxhost), "127.0.0.1", MAXVAL-1},
-   { "sphxport", "integer", (void*) int_parser, offsetof(struct config, sphxport), "9306", sizeof(int)},
-   { "smtp_access_list", "integer", (void*) int_parser, offsetof(struct config, smtp_access_list), "0", sizeof(int)},
-   { "smtp_timeout", "integer", (void*) int_parser, offsetof(struct config, smtp_timeout), "60", sizeof(int)},
-   { "spam_header_line", "string", (void*) string_parser, offsetof(struct config, spam_header_line), "", MAXVAL-1},
-   { "syslog_recipients", "integer", (void*) int_parser, offsetof(struct config, syslog_recipients), "0", sizeof(int)},
-   { "tls_enable", "integer", (void*) int_parser, offsetof(struct config, tls_enable), "0", sizeof(int)},
-   { "tls_min_version", "string", (void*) string_parser, offsetof(struct config, tls_min_version), "TLSv1.2", MAXVAL-1},
-   { "tweak_sent_time_offset", "integer", (void*) int_parser, offsetof(struct config, tweak_sent_time_offset), "0", sizeof(int)},
-   { "update_counters_to_memcached", "integer", (void*) int_parser, offsetof(struct config, update_counters_to_memcached), "0", sizeof(int)},
-   { "username", "string", (void*) string_parser, offsetof(struct config, username), "piler", MAXVAL-1},
-   { "use_antivirus", "integer", (void*) int_parser, offsetof(struct config, use_antivirus), "1", sizeof(int)},
-   { "verbosity", "integer", (void*) int_parser, offsetof(struct config, verbosity), "1", sizeof(int)},
-   { "workdir", "string", (void*) string_parser, offsetof(struct config, workdir), WORK_DIR, MAXVAL-1},
+
+   { "archive_emails_not_having_message_id", "integer", (void*) int_parser, offsetof(struct __config, archive_emails_not_having_message_id), "0", sizeof(int)},
+   { "archive_only_mydomains", "integer", (void*) int_parser, offsetof(struct __config, archive_only_mydomains), "0", sizeof(int)},
+   { "backlog", "integer", (void*) int_parser, offsetof(struct __config, backlog), "20", sizeof(int)},
+   { "cipher_list", "string", (void*) string_parser, offsetof(struct __config, cipher_list), "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS", MAXVAL-1},
+   { "clamd_addr", "string", (void*) string_parser, offsetof(struct __config, clamd_addr), "", MAXVAL-1},
+   { "clamd_port", "integer", (void*) int_parser, offsetof(struct __config, clamd_port), "0", sizeof(int)},
+   { "clamd_socket", "string", (void*) string_parser, offsetof(struct __config, clamd_socket), CLAMD_SOCKET, MAXVAL-1},
+   { "debug", "integer", (void*) int_parser, offsetof(struct __config, debug), "0", sizeof(int)},
+   { "default_retention_days", "integer", (void*) int_parser, offsetof(struct __config, default_retention_days), "2557", sizeof(int)},
+   { "enable_chunking", "integer", (void*) int_parser, offsetof(struct __config, enable_chunking), "0", sizeof(int)},
+   { "enable_cjk", "integer", (void*) int_parser, offsetof(struct __config, enable_cjk), "0", sizeof(int)},
+   { "enable_folders", "integer", (void*) int_parser, offsetof(struct __config, enable_folders), "0", sizeof(int)},
+   { "encrypt_messages", "integer", (void*) int_parser, offsetof(struct __config, encrypt_messages), "1", sizeof(int)},
+   { "extra_to_field", "string", (void*) string_parser, offsetof(struct __config, extra_to_field), "", MAXVAL-1},
+   { "extract_attachments", "integer", (void*) int_parser, offsetof(struct __config, extract_attachments), "1", sizeof(int)},
+   { "helper_timeout", "integer", (void*) int_parser, offsetof(struct __config, helper_timeout), "20", sizeof(int)},
+   { "hostid", "string", (void*) string_parser, offsetof(struct __config, hostid), HOSTID, MAXVAL-1},
+   { "iv", "string", (void*) string_parser, offsetof(struct __config, iv), "", MAXVAL-1},
+   { "listen_addr", "string", (void*) string_parser, offsetof(struct __config, listen_addr), "0.0.0.0", MAXVAL-1},
+   { "listen_port", "integer", (void*) int_parser, offsetof(struct __config, listen_port), "25", sizeof(int)},
+   { "locale", "string", (void*) string_parser, offsetof(struct __config, locale), "", MAXVAL-1},
+   { "max_connections", "integer", (void*) int_parser, offsetof(struct __config, max_connections), "64", sizeof(int)},
+   { "max_requests_per_child", "integer", (void*) int_parser, offsetof(struct __config, max_requests_per_child), "1000", sizeof(int)},
+   { "memcached_servers", "string", (void*) string_parser, offsetof(struct __config, memcached_servers), "127.0.0.1", MAXVAL-1},
+   { "memcached_to_db_interval", "integer", (void*) int_parser, offsetof(struct __config, memcached_to_db_interval), "900", sizeof(int)},
+   { "memcached_ttl", "integer", (void*) int_parser, offsetof(struct __config, memcached_ttl), "86400", sizeof(int)},
+   { "min_message_size", "integer", (void*) int_parser, offsetof(struct __config, min_message_size), "100", sizeof(int)},
+   { "min_word_len", "integer", (void*) int_parser, offsetof(struct __config, min_word_len), "1", sizeof(int)},
+   { "mmap_dedup_test", "integer", (void*) int_parser, offsetof(struct __config, mmap_dedup_test), "0", sizeof(int)},
+   { "mysqlcharset", "string", (void*) string_parser, offsetof(struct __config, mysqlcharset), "utf8mb4", MAXVAL-1},
+   { "mysqlhost", "string", (void*) string_parser, offsetof(struct __config, mysqlhost), "", MAXVAL-1},
+   { "mysqlport", "integer", (void*) int_parser, offsetof(struct __config, mysqlport), "", sizeof(int)},
+   { "mysqlsocket", "string", (void*) string_parser, offsetof(struct __config, mysqlsocket), "/tmp/mysql.sock", MAXVAL-1},
+   { "mysqluser", "string", (void*) string_parser, offsetof(struct __config, mysqluser), "piler", MAXVAL-1},
+   { "mysqlpwd", "string", (void*) string_parser, offsetof(struct __config, mysqlpwd), "", MAXVAL-1},
+   { "mysqldb", "string", (void*) string_parser, offsetof(struct __config, mysqldb), "piler", MAXVAL-1},
+   { "mysql_connect_timeout", "integer", (void*) int_parser, offsetof(struct __config, mysql_connect_timeout), "2", sizeof(int)},
+   { "number_of_worker_processes", "integer", (void*) int_parser, offsetof(struct __config, number_of_worker_processes), "2", sizeof(int)},
+   { "pemfile", "string", (void*) string_parser, offsetof(struct __config, pemfile), "", MAXVAL-1},
+   { "pidfile", "string", (void*) string_parser, offsetof(struct __config, pidfile), PIDFILE, MAXVAL-1},
+   { "piler_header_field", "string", (void*) string_parser, offsetof(struct __config, piler_header_field), "X-piler-id:", MAXVAL-1},
+   { "process_rcpt_to_addresses", "integer", (void*) int_parser, offsetof(struct __config, process_rcpt_to_addresses), "0", sizeof(int)},
+   { "queuedir", "string", (void*) string_parser, offsetof(struct __config, queuedir), QUEUE_DIR, MAXVAL-1},
+   { "server_id", "integer", (void*) int_parser, offsetof(struct __config, server_id), "0", sizeof(int)},
+   { "smtp_timeout", "integer", (void*) int_parser, offsetof(struct __config, smtp_timeout), "60", sizeof(int)},
+   { "spam_header_line", "string", (void*) string_parser, offsetof(struct __config, spam_header_line), "", MAXVAL-1},
+   { "syslog_recipients", "integer", (void*) int_parser, offsetof(struct __config, syslog_recipients), "0", sizeof(int)},
+   { "tls_enable", "integer", (void*) int_parser, offsetof(struct __config, tls_enable), "0", sizeof(int)},
+   { "tweak_sent_time_offset", "integer", (void*) int_parser, offsetof(struct __config, tweak_sent_time_offset), "0", sizeof(int)},
+   { "update_counters_to_memcached", "integer", (void*) int_parser, offsetof(struct __config, update_counters_to_memcached), "0", sizeof(int)},
+   { "username", "string", (void*) string_parser, offsetof(struct __config, username), "piler", MAXVAL-1},
+   { "use_antivirus", "integer", (void*) int_parser, offsetof(struct __config, use_antivirus), "1", sizeof(int)},
+   { "verbosity", "integer", (void*) int_parser, offsetof(struct __config, verbosity), "1", sizeof(int)},
+   { "workdir", "string", (void*) string_parser, offsetof(struct __config, workdir), WORK_DIR, MAXVAL-1},

   {NULL, NULL, NULL, 0, 0, 0}
 };
@ -118,7 +122,7 @@ struct _parse_rule config_parse_rules[] =
 * parse configfile
 */

-int parse_config_file(char *configfile, struct config *target_cfg, struct _parse_rule *rules){
+int parse_config_file(char *configfile, struct __config *target_cfg, struct _parse_rule *rules){
   char line[MAXVAL], *chpos;
   FILE *f;

@ -143,7 +147,7 @@ int parse_config_file(char *configfile, struct config *target_cfg, struct _parse
                  printf("failed to parse %s: \"%s\"\n", line, chpos+1);
               }
               break;
-            }
+            }				

            i++;
         }
@ -158,27 +162,7 @@ int parse_config_file(char *configfile, struct config *target_cfg, struct _parse
 }


-int get_tls_protocol_number(char *protocol){
-   struct tls_protocol tls_protocols[] = {
-      { "TLSv1", TLS1_VERSION },
-      { "TLSv1.1", TLS1_1_VERSION },
-      { "TLSv1.2", TLS1_2_VERSION },
-   #ifdef TLS1_3_VERSION
-      { "TLSv1.3", TLS1_3_VERSION },
-   #endif
-   };
-
-   for(unsigned int i=0; i<sizeof(tls_protocols)/sizeof(struct tls_protocol); i++){
-      if(!strcmp(protocol, tls_protocols[i].proto)) {
-         return tls_protocols[i].version;
-      }
-   }
-
-   return 0;
-}
-
-
-int load_default_config(struct config *cfg, struct _parse_rule *rules){
+int load_default_config(struct __config *cfg, struct _parse_rule *rules){
   int i=0;

   while(rules[i].name){
@ -194,12 +178,12 @@ int load_default_config(struct config *cfg, struct _parse_rule *rules){
 * read configuration file variables
 */

-struct config read_config(char *configfile){
-   struct config cfg;
+struct __config read_config(char *configfile){
+   struct __config cfg;

   /* reset config structure and fill it with defaults */

-   memset((char *)&cfg, 0, sizeof(struct config));
+   memset((char *)&cfg, 0, sizeof(struct __config));

   load_default_config(&cfg, config_parse_rules);

@ -210,9 +194,6 @@ struct config read_config(char *configfile){

   cfg.hostid_len = strlen(cfg.hostid);

-   // Get the TLS protocol constant from string, ie. TLSv1.3 -> 772
-   cfg.tls_min_version_number = get_tls_protocol_number(cfg.tls_min_version);
-
   return cfg;
 }

@ -221,10 +202,9 @@ struct config read_config(char *configfile){
 * print a single configuration item as key=value
 */

-void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
+void print_config_item(struct __config *cfg, struct _parse_rule *rules, int i){
   int j;
   float f;
-   uint64 u;
   char *p, buf[MAXVAL];

   p = (char*)cfg + rules[i].offset;
@ -233,10 +213,6 @@ void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
      memcpy((char*)&j, p, sizeof(int));
      printf("%s=%d\n", rules[i].name, j);
   }
-   else if(strcmp(rules[i].type, "uint64") == 0){
-      memcpy((char*)&u, p, sizeof(uint64));
-      printf("%s=%llu\n", rules[i].name, u);
-   }
   else if(strcmp(rules[i].type, "float") == 0){
      memcpy((char*)&f, p, sizeof(float));
      printf("%s=%.4f\n", rules[i].name, f);
@ -253,7 +229,7 @@ void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
      trimBuffer(p);
      printf("%s=%s\n", rules[i].name, p);
   }
-
+ 
 }


@ -261,7 +237,7 @@ void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
 * print all known configuration items
 */

-void print_config_all(struct config *cfg, char *key){
+void print_config_all(struct __config *cfg, char *key){
   int i=0;
   struct _parse_rule *rules;

@ -285,7 +261,7 @@ void print_config_all(struct config *cfg, char *key){
 * print all configuration items found in configfile
 */

-void print_config(char *configfile, struct config *cfg){
+void print_config(char *configfile, struct __config *cfg){
   FILE *f;
   char line[MAXVAL], *chpos, previtem[MAXVAL];
   struct _parse_rule *rules;
@ -320,10 +296,12 @@ void print_config(char *configfile, struct config *cfg){

            i++;
         }
-
+ 
         if(!rules[i].name) printf("unknown key: \"%s\" \n", line);
      }
   }

   fclose(f);
 }
+
+
--- a/src/cfg.h
+++ b/src/cfg.h
@ -7,7 +7,7 @@

 #include "config.h"

-struct config {
+struct __config {
   int server_id;
   char username[MAXVAL];

@ -29,8 +29,6 @@ struct config {
   int tls_enable;
   char pemfile[MAXVAL];
   char cipher_list[MAXVAL];
-   char tls_min_version[MAXVAL];
-   int tls_min_version_number;

   int use_antivirus;

@ -51,7 +49,6 @@ struct config {
   int verbosity;
   char locale[MAXVAL];

-   int check_for_client_timeout_interval;
   int smtp_timeout;
   int helper_timeout;
   int extract_attachments;
@ -66,9 +63,6 @@ struct config {

   int default_retention_days;

-   char security_header[MAXVAL];
-   char archive_address[MAXVAL];
-
   // mysql stuff

   char mysqlcharset[MAXVAL];
@ -80,12 +74,6 @@ struct config {
   char mysqldb[MAXVAL];
   int mysql_connect_timeout;

-   // manticore stuff
-   char sphxhost[MAXVAL];
-   int sphxport;
-   char sphxdb[MAXVAL];
-   int rtindex;
-
   int update_counters_to_memcached;
   int memcached_to_db_interval;

@ -108,11 +96,6 @@ struct config {
   int enable_folders;

   int debug;
-
-   int smtp_access_list;
-
-   int max_message_size;
-   uint64 max_smtp_memory;
 };


--- a/src/clamd.c
+++ b/src/clamd.c
@ -18,9 +18,9 @@
 #include <piler.h>


-int clamd_scan(char *tmpfile, struct config *cfg){
+int clamd_scan(char *tmpfile, struct __config *cfg){
   int s, n;
-   char buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE];
+   char *p, *q, buf[MAXBUFSIZE], scan_cmd[SMALLBUFSIZE];
   struct sockaddr_un server;

   chmod(tmpfile, 0644);
@ -41,7 +41,7 @@ int clamd_scan(char *tmpfile, struct config *cfg){

   /* issue the SCAN command with full path to the temporary directory */

-
+   
   memset(scan_cmd, 0, SMALLBUFSIZE);
   snprintf(scan_cmd, SMALLBUFSIZE-1, "SCAN %s/%s\r\n", cfg->workdir, tmpfile);

@ -58,9 +58,9 @@ int clamd_scan(char *tmpfile, struct config *cfg){
   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: CLAMD DEBUG: %d %s", tmpfile, n, buf);

   if(strcasestr(buf, CLAMD_RESP_INFECTED)){
-      char *p = strchr(buf, ' ');
+      p = strchr(buf, ' ');
      if(p){
-         char *q = strrchr(p, ' ');
+         q = strrchr(p, ' ');
         if(q){
            *q = '\0';
            p++;
--- a/src/config.h
+++ b/src/config.h
@ -9,17 +9,15 @@
 #include "piler-config.h"
 #include "params.h"

-typedef unsigned long long uint64;
+#define VERSION "1.3.0-epoll"

-#define BUILD 1001
+#define BUILD 977

 #define HOSTID "mailarchiver"

 #define CONFIG_FILE CONFDIR "/piler/piler.conf"
-#define SMTP_ACL_FILE CONFDIR "/piler/smtp.acl"
 #define WORK_DIR DATADIR "/piler/tmp"
 #define QUEUE_DIR DATADIR "/piler/store"
-#define ERROR_DIR DATADIR "/piler/error"

 #define CLAMD_SOCKET "/tmp/clamd"

@ -32,7 +30,6 @@ typedef unsigned long long uint64;
 #define SMALLBUFSIZE 512
 #define BIGBUFSIZE 131072
 #define REALLYBIGBUFSIZE 524288
-#define SMTPBUFSIZE 2048000
 #define TINYBUFSIZE 128
 #define MAXVAL 256
 #define RANDOM_POOL "/dev/urandom"
@ -42,20 +39,6 @@ typedef unsigned long long uint64;
 #define IPLEN 16+1
 #define KEYLEN 56
 #define MIN_EMAIL_ADDRESS_LEN 9
-// Sphinx 3.x has an issue with tokens longer than 41 characters.
-//
-// When a regular user executes a query, then his default email address filter
-// causes the query to fail with the below error message, even when the query
-// itself is correct:
-//
-// SELECT id FROM main1,dailydelta1,delta1 WHERE MATCH(' ( (@sender thisisanextremelylongemailaddressyesareallylongoneyeahitolyouXaddressXcom ) | (@rcpt thisisanextremelylongemailaddressyesareallylongoneyeahitolyouXaddressXcom) ) ') ORDER BY `sent` DESC LIMIT 0,20 OPTION max_matches=1000'
-//
-// ERROR 1064 (42000): index dailydelta1,delta1,main1: syntax error, unexpected $end near ' '
-//
-// Note that we use 42, because the parser adds a trailing space to the tokens
-// See https://www.mailpiler.org/wiki/current:sphinx3 and
-// https://bitbucket.org/jsuto/piler/issues/1082/no-sphinx-results-with-long-email for more
-#define MAX_EMAIL_ADDRESS_SPHINX_LEN 42

 #define CRLF "\n"

@ -75,13 +58,10 @@ typedef unsigned long long uint64;
 #define MEMCACHED_MSGS_STORED_SIZE MEMCACHED_CLAPF_PREFIX "stored_size"


-#define PILEREXPORT_BEGIN_MARK "x-exported-by-pilerexport: start\n"
-
 #define LOG_PRIORITY LOG_INFO

 #define _LOG_INFO 3
 #define _LOG_DEBUG 5
-#define _LOG_EXTREME 100

 #define MAX_RCPT_TO 128

@ -125,10 +105,8 @@ typedef unsigned long long uint64;
 #define SQL_PREPARED_STMT_GET_FOLDER_ID              "SELECT `id` FROM " SQL_FOLDER_TABLE " WHERE `name`=? AND `parent_id`=?"
 #define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE   "INSERT INTO `" SQL_FOLDER_TABLE "` (`name`, `parent_id`) VALUES(?,?)"
 #define SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE  "UPDATE " SQL_METADATA_TABLE " SET reference=? WHERE message_id=? AND reference=''"
-#define SQL_PREPARED_STMT_GET_METADATA_REFERENCE     "SELECT COUNT(*) AS count FROM " SQL_METADATA_TABLE " WHERE reference=?"
 #define SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS        "SELECT id, type, username, password, server FROM " SQL_IMPORT_TABLE " WHERE started=0 ORDER BY id LIMIT 0,1"
 #define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE      "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `id`) VALUES(?,?)"
-#define SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE        "UPDATE " SQL_IMPORT_TABLE " SET started=?, status=?, imported=? WHERE id=?"

 /* Error codes */

@ -158,3 +136,4 @@ typedef unsigned long long uint64;
 #define S_STATUS_ERROR "error"

 #endif /* _CONFIG_H */
+
--- a/src/counters.c
+++ b/src/counters.c
@ -11,75 +11,71 @@
 #include <piler.h>


-struct counters load_counters(struct session_data *sdata){
+struct counters load_counters(struct session_data *sdata, struct __data *data){
   char buf[SMALLBUFSIZE];
   struct counters counters;
-   struct sql sql;

   bzero(&counters, sizeof(counters));

   snprintf(buf, SMALLBUFSIZE-1, "SELECT `rcvd`, `virus`, `duplicate`, `ignore`, `size`, `stored_size` FROM `%s`", SQL_COUNTER_TABLE);


-   if(prepare_sql_statement(sdata, &sql, buf) == ERR) return counters;
+   if(prepare_sql_statement(sdata, &(data->stmt_generic), buf) == ERR) return counters;


-   p_bind_init(&sql);
+   p_bind_init(data);

-   if(p_exec_stmt(sdata, &sql) == OK){
+   if(p_exec_query(sdata, data->stmt_generic, data) == OK){

-      p_bind_init(&sql);
+      p_bind_init(data);

-      sql.sql[sql.pos] = (char *)&counters.c_rcvd; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-      sql.sql[sql.pos] = (char *)&counters.c_virus; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-      sql.sql[sql.pos] = (char *)&counters.c_duplicate; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-      sql.sql[sql.pos] = (char *)&counters.c_ignore; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-      sql.sql[sql.pos] = (char *)&counters.c_size; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-      sql.sql[sql.pos] = (char *)&counters.c_stored_size; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+      data->sql[data->pos] = (char *)&counters.c_rcvd; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+      data->sql[data->pos] = (char *)&counters.c_virus; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+      data->sql[data->pos] = (char *)&counters.c_duplicate; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+      data->sql[data->pos] = (char *)&counters.c_ignore; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+      data->sql[data->pos] = (char *)&counters.c_size; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+      data->sql[data->pos] = (char *)&counters.c_stored_size; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;

-      p_store_results(&sql);
-      p_fetch_results(&sql);
-      p_free_results(&sql);
+      p_store_results(data->stmt_generic, data);
+      p_fetch_results(data->stmt_generic);
+      p_free_results(data->stmt_generic);
   }

-   close_prepared_statement(&sql);
+   close_prepared_statement(data->stmt_generic);

   return counters;
 }


-void update_counters(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+void update_counters(struct session_data *sdata, struct __data *data, struct counters *counters, struct __config *cfg){
   char buf[MAXBUFSIZE];
 #ifdef HAVE_MEMCACHED
-   unsigned long long mc;
+   unsigned long long mc, rcvd;
   struct counters c;
+   char key[MAX_MEMCACHED_KEY_LEN];
   unsigned int flags=0;
-#endif

-   if(counters->c_virus + counters->c_duplicate + counters->c_ignore + counters->c_size + counters->c_stored_size == 0) return;
+   if(counters->c_virus + counters->c_duplicate + counters->c_ignore + counters->c_size + counters->c_stored_size <= 0) return;

-#ifdef HAVE_MEMCACHED
   if(cfg->update_counters_to_memcached == 1){

      /* increment counters to memcached */

-      if(memcached_increment(&(data->memc), MEMCACHED_MSGS_RCVD, counters->c_rcvd, &mc) == MEMCACHED_SUCCESS){
-         unsigned long long rcvd = mc;
+      if(memcached_increment(&(data->memc), MEMCACHED_MSGS_RCVD, strlen(MEMCACHED_MSGS_RCVD), counters->c_rcvd, &mc) == MEMCACHED_SUCCESS){
+         rcvd = mc;

-         if(counters->c_virus > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_VIRUS, counters->c_virus, &mc);
-         if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_DUPLICATE, counters->c_duplicate, &mc);
-         if(counters->c_ignore > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_IGNORE, counters->c_ignore, &mc);
-         if(counters->c_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_SIZE, counters->c_size, &mc);
-         if(counters->c_stored_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_STORED_SIZE, counters->c_stored_size, &mc);
+         if(counters->c_virus > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), counters->c_virus, &mc);
+         if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), counters->c_duplicate, &mc);
+         if(counters->c_ignore > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), counters->c_ignore, &mc);
+         if(counters->c_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_SIZE, strlen(MEMCACHED_MSGS_SIZE), counters->c_size, &mc);
+         if(counters->c_stored_size > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_STORED_SIZE, strlen(MEMCACHED_MSGS_STORED_SIZE), counters->c_stored_size, &mc);


-         bzero(&c, sizeof(c));
+         bzero(&c, sizeof(c)); 

         snprintf(buf, MAXBUFSIZE-1, "%s %s %s %s %s %s %s", MEMCACHED_MSGS_RCVD, MEMCACHED_MSGS_VIRUS, MEMCACHED_MSGS_DUPLICATE, MEMCACHED_MSGS_IGNORE, MEMCACHED_MSGS_SIZE, MEMCACHED_MSGS_STORED_SIZE, MEMCACHED_COUNTERS_LAST_UPDATE);

         if(memcached_mget(&(data->memc), buf) == MEMCACHED_SUCCESS){
-            char key[MAX_MEMCACHED_KEY_LEN];
-
            while((memcached_fetch_result(&(data->memc), &key[0], &buf[0], &flags))){
               if(!strcmp(key, MEMCACHED_MSGS_RCVD)) c.c_rcvd = strtoull(buf, NULL, 10);
               else if(!strcmp(key, MEMCACHED_MSGS_VIRUS)) c.c_virus = strtoull(buf, NULL, 10);
@ -91,11 +87,11 @@ void update_counters(struct session_data *sdata, struct data *data, struct count
            }


-            if(sdata->now - mc > (unsigned long long)cfg->memcached_to_db_interval && c.c_rcvd > 0 && c.c_rcvd >= rcvd){
-               snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), "set", MEMCACHED_COUNTERS_LAST_UPDATE, buf, strlen(buf), 0, 0);
+            if(sdata->now - mc > cfg->memcached_to_db_interval && c.c_rcvd > 0 && c.c_rcvd >= rcvd){
+               snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_set(&(data->memc), MEMCACHED_COUNTERS_LAST_UPDATE, strlen(MEMCACHED_COUNTERS_LAST_UPDATE), buf, strlen(buf), 0, 0);

               snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET `rcvd`=%llu, `virus`=%llu, `duplicate`=%llu, `ignore`=%llu, `size`=%llu, `stored_size`=%llu", SQL_COUNTER_TABLE, c.c_rcvd, c.c_virus, c.c_duplicate, c.c_ignore, c.c_size, c.c_stored_size);
-
+ 
               //if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: update counters: %s", sdata->ttmpfile, buf);

               p_query(sdata, buf);
@ -105,21 +101,23 @@ void update_counters(struct session_data *sdata, struct data *data, struct count
      }
      else {

-         c = load_counters(sdata);
+         c = load_counters(sdata, data);

-         snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), "add", MEMCACHED_COUNTERS_LAST_UPDATE, buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_add(&(data->memc), MEMCACHED_COUNTERS_LAST_UPDATE, strlen(MEMCACHED_COUNTERS_LAST_UPDATE), buf, strlen(buf), 0, 0);

-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_virus + counters->c_virus); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_VIRUS, buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_rcvd + counters->c_rcvd); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_RCVD, buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_duplicate + counters->c_duplicate); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_DUPLICATE, buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_ignore + counters->c_ignore); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_IGNORE, buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_size + counters->c_size); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_SIZE, buf, strlen(buf), 0, 0);
-         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_stored_size + counters->c_stored_size); memcached_add(&(data->memc), "add", MEMCACHED_MSGS_STORED_SIZE, buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_virus + counters->c_virus); memcached_add(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_rcvd + counters->c_rcvd); memcached_add(&(data->memc), MEMCACHED_MSGS_RCVD, strlen(MEMCACHED_MSGS_RCVD), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_duplicate + counters->c_duplicate); memcached_add(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_ignore + counters->c_ignore); memcached_add(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_size + counters->c_size); memcached_add(&(data->memc), MEMCACHED_MSGS_SIZE, strlen(MEMCACHED_MSGS_SIZE), buf, strlen(buf), 0, 0);
+         snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_stored_size + counters->c_stored_size); memcached_add(&(data->memc), MEMCACHED_MSGS_STORED_SIZE, strlen(MEMCACHED_MSGS_STORED_SIZE), buf, strlen(buf), 0, 0);
      }

   }
   else {
 #endif
+      if(counters->c_virus + counters->c_duplicate + counters->c_ignore + counters->c_size + counters->c_stored_size <= 0) return;
+
      snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET `rcvd`=`rcvd`+%llu, `virus`=`virus`+%llu, `duplicate`=`duplicate`+%llu, `ignore`=`ignore`+%llu, `size`=`size`+%llu, `stored_size`=`stored_size`+%llu", SQL_COUNTER_TABLE, counters->c_rcvd, counters->c_virus, counters->c_duplicate, counters->c_ignore, counters->c_size, counters->c_stored_size);
      p_query(sdata, buf);

@ -128,3 +126,5 @@ void update_counters(struct session_data *sdata, struct data *data, struct count
 #endif

 }
+
+
--- a/src/decoder.c
+++ b/src/decoder.c
@ -78,8 +78,6 @@ inline void utf8_encode_char(unsigned char c, unsigned char *buf, int buflen, in
       * U+0000..U+007F      00..7F
       * U+0080..U+07FF      C2..DF      80..BF
       * U+0800..U+0FFF      E0          A0..BF      80..BF
-       *
-       * FIXME: See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf#G7404 for valid sequences
       */

      if(c <= 0x7F){
@ -98,6 +96,20 @@ inline void utf8_encode_char(unsigned char c, unsigned char *buf, int buflen, in
 }


+void sanitiseBase64(char *s){
+   char *p1;
+
+   if(s == NULL) return;
+
+   for(; *s; s++){
+      if(b64[(unsigned int)(*s & 0xFF)] == 255){
+         for(p1 = s; p1[0] != '\0'; p1++)
+            p1[0] = p1[1];
+      }
+   }
+}
+
+
 inline static void pack_4_into_3(char *s, char *s2){
   int j, n[4], k1, k2;

@ -143,7 +155,7 @@ int decodeBase64(char *p){


 int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
-   int i, len=0;
+   int i, len=0, decodedlen;
   char s[5], s2[3];

   if(plen < 4 || plen > blen)
@ -152,7 +164,7 @@ int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
   for(i=0; i<plen; i+=4){
      memcpy(s, p+i, 4);
      s[4] = '\0';
-      int decodedlen = 3;
+      decodedlen = 3;

      /* safety check against abnormally long lines */

@ -177,11 +189,12 @@ int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen){
 void decodeQP(char *p){
   unsigned int i;
   int k=0, a, b;
+   char c;

   if(p == NULL) return;

   for(i=0; i<strlen((char*)p); i++){
-      char c = p[i];
+      c = p[i];

      if(p[i] == '=' && isxdigit(p[i+1]) && isxdigit(p[i+2])){
         a = p[i+1];
@ -191,13 +204,9 @@ void decodeQP(char *p){

         i += 2;
      }
-      else if(p[i] == '_'){
-         c = ' ';
-      }

      p[k] = c;
      k++;
-
   }

   p[k] = '\0';
@ -314,18 +323,7 @@ int utf8_encode(char *inbuf, int inbuflen, char *outbuf, int outbuflen, char *en

   memset(outbuf, 0, outbuflen);

-   // Iconv sometimes produces an invalid utf8 sequence for gb2312.
-   // The fix is to use cp936, instead of gb2312 encoding.
-   //
-   // If there will be more similar exceptions, then we have to use
-   // a more efficient lookup method
-
-   if(strcasecmp(encoding, "gb2312") == 0)
-      cd = iconv_open("utf-8", "cp936");
-   else if(strcasecmp(encoding, "ks_c_5601-1987") == 0)
-      cd = iconv_open("utf-8", "EUC-KR");
-   else
-      cd = iconv_open("utf-8", encoding);
+   cd = iconv_open("utf-8", encoding);

   if(cd != (iconv_t)-1){
      inbytesleft = inbuflen;
@ -341,3 +339,4 @@ int utf8_encode(char *inbuf, int inbuflen, char *outbuf, int outbuflen, char *en

   return ret;
 }
+
--- a/src/decoder.h
+++ b/src/decoder.h
@ -5,6 +5,9 @@
 #ifndef _DECODER_H
 #define _DECODER_H

+void base64_encode(unsigned char *in, int inlen, char *out, int outlen);
+
+void sanitiseBase64(char *s);
 int decodeBase64(char *p);
 int decode_base64_to_buffer(char *p, int plen, unsigned char *b, int blen);
 void decodeQP(char *p);
--- a/src/defs.h
+++ b/src/defs.h
@ -9,12 +9,17 @@
  #include <mysql.h>
  #include <mysqld_error.h>
 #endif
+#ifdef NEED_PSQL
+  #include <libpq-fe.h>
+#endif
 #ifdef HAVE_TRE
   #include <tre/tre.h>
   #include <tre/regex.h>
 #endif
+#ifdef HAVE_LIBWRAP
+   #include <tcpd.h>
+#endif

-#include <openssl/md5.h>
 #include <openssl/sha.h>
 #include <openssl/ssl.h>
 #include <netinet/in.h>
@ -25,24 +30,21 @@
 #define MSG_BODY 0
 #define MSG_RECEIVED 1
 #define MSG_FROM 2
-#define MSG_SENDER 3
-#define MSG_TO 4
-#define MSG_CC 5
-#define MSG_SUBJECT 6
-#define MSG_CONTENT_TYPE 7
-#define MSG_CONTENT_TRANSFER_ENCODING 8
-#define MSG_CONTENT_DISPOSITION 9
-#define MSG_MESSAGE_ID 10
-#define MSG_REFERENCES 11
-#define MSG_RECIPIENT 12
-#define MSG_ENVELOPE_TO 13
+#define MSG_TO 3
+#define MSG_CC 4
+#define MSG_SUBJECT 5
+#define MSG_CONTENT_TYPE 6
+#define MSG_CONTENT_TRANSFER_ENCODING 7
+#define MSG_CONTENT_DISPOSITION 8
+#define MSG_MESSAGE_ID 9
+#define MSG_REFERENCES 10
+#define MSG_RECIPIENT 11

 #define MAXHASH 277

 #define BASE64_RATIO 1.33333333

-#define DIGEST_LENGTH EVP_MAX_MD_SIZE
-#define DIGEST_HEX_LENGTH 2*DIGEST_LENGTH+1
+#define DIGEST_LENGTH SHA256_DIGEST_LENGTH

 #define UNDEF 0
 #define READY 1
@ -76,7 +78,7 @@ struct attachment {
   char type[TINYBUFSIZE];
   char shorttype[TINYBUFSIZE];
   char aname[TINYBUFSIZE];
-   char filename[SMALLBUFSIZE];
+   char filename[TINYBUFSIZE];
   char internalname[TINYBUFSIZE];
   char digest[2*DIGEST_LENGTH+1];
   char dumped;
@ -97,25 +99,6 @@ struct node {
 };


-struct smtp_acl {
-   char network_str[BUFLEN];
-   in_addr_t low, high;
-   int prefix;
-   int rejected;
-   struct smtp_acl *r;
-};
-
-
-struct net {
-   int socket;
-   int use_ssl;
-   int starttls;
-   int timeout;
-   SSL_CTX *ctx;
-   SSL *ssl;
-};
-
-
 struct rule {
 #ifdef HAVE_TRE
   regex_t from;
@ -173,10 +156,8 @@ struct parser_state {
   int qp;
   int htmltag;
   int style;
-   int meta_content_type;
   int skip_html;
   int has_to_dump;
-   int has_to_dump_whole_body;
   int fd;
   int b64fd;
   int mfd;
@ -185,20 +166,19 @@ struct parser_state {
   int content_type_is_set;
   int pushed_pointer;
   int saved_size;
-   unsigned int writebufpos;
-   unsigned int abufpos;
-   unsigned int received_header;
+   int writebufpos;
+   int abufpos;
   char attachedfile[RND_STR_LEN+SMALLBUFSIZE];
   char message_id[SMALLBUFSIZE];
   char message_id_hash[2*DIGEST_LENGTH+1];
   char miscbuf[MAX_TOKEN_LEN];
   char qpbuf[MAX_TOKEN_LEN];
-   char receivedbuf[SMALLBUFSIZE];
   unsigned long n_token;
   unsigned long n_subject_token;
   unsigned long n_body_token;
   unsigned long n_chain_token;

+   char filename[TINYBUFSIZE];
   char type[TINYBUFSIZE];
   char charset[TINYBUFSIZE];

@ -215,15 +195,12 @@ struct parser_state {

   char reference[SMALLBUFSIZE];

-   char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_sender[SMALLBUFSIZE], b_sender_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE];
+   char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE];
   char b_journal_to[MAXBUFSIZE];

-   unsigned int bodylen;
-   unsigned int tolen;
-   unsigned int todomainlen;
-   unsigned int found_security_header;
-
-   long unsigned int journaltolen;
+   int bodylen;
+   int tolen;
+   int journaltolen;

   int retention;
 };
@ -253,7 +230,11 @@ struct session_data {
   int journal_envelope_length, journal_bottom_length;
   unsigned int sql_errno;
 #ifdef NEED_MYSQL
-   MYSQL mysql, sphx;
+   MYSQL mysql;
+#endif
+#ifdef NEED_PSQL
+   PGconn *psql;
+   char conninfo[SMALLBUFSIZE];
 #endif
 };

@ -313,25 +294,8 @@ struct import {
   int keep_eml;
   int timeout;
   int cap_uidplus;
-   int fd;
   long total_size;
-   int dryrun;
-   int tot_msgs;
-   int port;
-   int seq;
-   int table_id;
-   int delay;
-   char *server;
-   char *username;
-   char *password;
-   char *database;
-   char *skiplist;
-   char *folder_imap;
-   char *folder_name;
-   char *mboxdir;
-   char *folder;
-   char filename[SMALLBUFSIZE];
-   time_t started, updated, finished, after, before;
+   time_t started, updated, finished;
 };


@ -344,7 +308,7 @@ struct licence {
 };


-struct data {
+struct __data {
   int folder, quiet;
   char recursive_folder_names;
   char starttls[TINYBUFSIZE];
@ -354,6 +318,32 @@ struct data {
   struct licence licence;
   char *dedup;
   int child_serial;
+
+#ifdef NEED_MYSQL
+   MYSQL_STMT *stmt_generic;
+   MYSQL_STMT *stmt_get_meta_id_by_message_id;
+   MYSQL_STMT *stmt_insert_into_rcpt_table;
+   MYSQL_STMT *stmt_insert_into_sphinx_table;
+   MYSQL_STMT *stmt_insert_into_meta_table;
+   MYSQL_STMT *stmt_insert_into_attachment_table;
+   MYSQL_STMT *stmt_insert_into_folder_message_table;
+   MYSQL_STMT *stmt_get_attachment_id_by_signature;
+   MYSQL_STMT *stmt_get_attachment_pointer;
+   MYSQL_STMT *stmt_query_attachment;
+   MYSQL_STMT *stmt_get_folder_id;
+   MYSQL_STMT *stmt_insert_into_folder_table;
+   MYSQL_STMT *stmt_update_metadata_reference;
+   MYSQL_STMT *stmt_update_metadata;
+   MYSQL_STMT *stmt_select_from_meta_table;
+   MYSQL_STMT *stmt_select_non_referenced_attachments;
+#endif
+
+   char *sql[MAX_SQL_VARS];
+   int type[MAX_SQL_VARS];
+   int len[MAX_SQL_VARS];
+   unsigned long length[MAX_SQL_VARS];
+   my_bool is_null[MAX_SQL_VARS];
+   my_bool error[MAX_SQL_VARS];
   int pos;

 #ifdef HAVE_TRE
@ -365,27 +355,8 @@ struct data {
 #ifdef HAVE_MEMCACHED
   struct memcached_server memc;
 #endif
-
-   struct net *net;
-};
-
-
-#if !defined(MARIADB_BASE_VERSION) && !defined(MARIADB_VERSION_ID) && \
-  MYSQL_VERSION_ID >= 80001 && MYSQL_VERSION_ID != 80002
-typedef bool my_bool;
-#endif
-
-struct sql {
-#ifdef NEED_MYSQL
-   MYSQL_STMT *stmt;
-#endif
-   char *sql[MAX_SQL_VARS];
-   int type[MAX_SQL_VARS];
-   int len[MAX_SQL_VARS];
-   unsigned long length[MAX_SQL_VARS];
-   my_bool is_null[MAX_SQL_VARS];
-   my_bool error[MAX_SQL_VARS];
-   int pos;
+   SSL_CTX *ctx;
+   SSL *ssl;
 };


@ -402,30 +373,24 @@ struct counters {
 struct smtp_session {
   char ttmpfile[SMALLBUFSIZE];
   char mailfrom[SMALLBUFSIZE];
-   char rcptto[MAX_RCPT_TO][SMALLBUFSIZE];
-   char remote_host[INET6_ADDRSTRLEN+1];
-   char nullbyte;
+   char buf[SMALLBUFSIZE];
+   char remote_host[INET6_ADDRSTRLEN];
   time_t lasttime;
   int protocol_state;
   int slot;
   int fd;
   int bad;
-   int tot_len;
-   int bdat_bytes_to_read;
-   int num_of_rcpt_to;
-   struct config *cfg;
-   struct net net;
-   int max_message_size;
-   char *buf;
   int buflen;
-   int bufsize;
-   int too_big;
-   int mail_size;
-};
-
-struct tls_protocol {
-   char *proto;
-   int version;
+   int tot_len;
+   int bdat_rounds;
+   int bdat_last_round;
+   int bdat_bytes_to_read;
+   int socket;
+   struct __config *cfg;
+   SSL_CTX *ctx;
+   SSL *ssl;
+   int use_ssl;
+   int starttls;
 };

 #endif /* _DEFS_H */
--- a/src/digest.c
+++ b/src/digest.c
@ -33,39 +33,26 @@ int search_header_end(char *p, int n){
 }


-int make_digests(struct session_data *sdata, struct config *cfg){
-   int n, fd, offset=3, hdr_len=0;
+int make_digests(struct session_data *sdata, struct __config *cfg){
+   int i=0, n, fd, offset=3, hdr_len=0, len=0;
   char *body=NULL;
-   unsigned char buf[BIGBUFSIZE];
-
-   EVP_MD_CTX *ctx, *ctx2;
-   const EVP_MD *md, *md2;
-   unsigned int i=0, md_len, md_len2;
-   unsigned char md_value[EVP_MAX_MD_SIZE], md_value2[EVP_MAX_MD_SIZE];
+   unsigned char buf[BIGBUFSIZE], md[DIGEST_LENGTH], md2[DIGEST_LENGTH];
+   SHA256_CTX context, context2;

   memset(sdata->bodydigest, 0, 2*DIGEST_LENGTH+1);
   memset(sdata->digest, 0, 2*DIGEST_LENGTH+1);
+   SHA256_Init(&context);
+   SHA256_Init(&context2);

-   md = EVP_get_digestbyname("sha256");
-   md2 = EVP_get_digestbyname("sha256");
-   if(md == NULL || md2 == NULL){
-      syslog(LOG_PRIORITY, "ERROR: unknown message digest: sha256 in %s:%d", __func__, __LINE__);
-      return 1;
-   }
-
-   ctx = EVP_MD_CTX_new();
-   EVP_DigestInit_ex(ctx, md, NULL);
-
-   ctx2 = EVP_MD_CTX_new();
-   EVP_DigestInit_ex(ctx2, md2, NULL);

   fd = open(sdata->filename, O_RDONLY);
   if(fd == -1) return -1;

-   memset(buf, 0, sizeof(buf));

   while((n = read(fd, buf, sizeof(buf))) > 0){
-      EVP_DigestUpdate(ctx2, buf, n);
+      len += n;
+
+      SHA256_Update(&context2, buf, n);

      body = (char *)&buf[0];

@ -82,7 +69,7 @@ int make_digests(struct session_data *sdata, struct config *cfg){
      }


-      EVP_DigestUpdate(ctx, body, n);
+      SHA256_Update(&context, body, n);

      i++;
   }
@ -91,93 +78,59 @@ int make_digests(struct session_data *sdata, struct config *cfg){

   sdata->hdr_len = hdr_len;

-   EVP_DigestFinal_ex(ctx, md_value, &md_len);
-   EVP_MD_CTX_free(ctx);
-   EVP_DigestFinal_ex(ctx2, md_value2, &md_len2);
-   EVP_MD_CTX_free(ctx2);
+   SHA256_Final(md, &context);
+   SHA256_Final(md2, &context2);

-   for(i=0;i<md_len;i++){
-      snprintf(sdata->bodydigest + i*2, 3, "%02x", md_value[i]);
-   }
-
-   for(i=0;i<md_len2;i++){
-      snprintf(sdata->digest + i*2, 3, "%02x", md_value2[i]);
+   for(i=0;i<DIGEST_LENGTH;i++){
+      snprintf(sdata->bodydigest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
+      snprintf(sdata->digest + i*2, 2*DIGEST_LENGTH, "%02x", md2[i]);
   }

   return 0;
 }


-void raw_digest_file(char *digestname, char *filename, unsigned char *md_value){
-   int fd, n;
-   unsigned char buf[MAXBUFSIZE];
-   EVP_MD_CTX *ctx;
-   const EVP_MD *md;
-   unsigned int md_len;
+void digest_file(char *filename, char *digest){
+   int fd, i, n;
+   unsigned char buf[MAXBUFSIZE], md[DIGEST_LENGTH];
+   SHA256_CTX context;

-   md = EVP_get_digestbyname(digestname);
-   if(md == NULL){
-      syslog(LOG_PRIORITY, "ERROR: unknown message digest: '%s' in %s:%d", digestname, __func__, __LINE__);
-      return;
-   }
+   memset(digest, 0, 2*DIGEST_LENGTH+1);

   fd = open(filename, O_RDONLY);
   if(fd == -1) return;

-   ctx = EVP_MD_CTX_new();
-   EVP_DigestInit_ex(ctx, md, NULL);
+   SHA256_Init(&context);

   while((n = read(fd, buf, sizeof(buf))) > 0){
-      EVP_DigestUpdate(ctx, buf, n);
+      SHA256_Update(&context, buf, n);
   }

   close(fd);

-   EVP_DigestFinal_ex(ctx, md_value, &md_len);
-   EVP_MD_CTX_free(ctx);
+   SHA256_Final(md, &context);
+
+   for(i=0;i<DIGEST_LENGTH;i++)
+      snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
+
 }


-void digest_file(char *filename, char *digest){
+void digest_string(char *s, char *digest){
+   int i;
   unsigned char md[DIGEST_LENGTH];
-
-   raw_digest_file("sha256", filename, &md[0]);
+   SHA256_CTX context;

   memset(digest, 0, 2*DIGEST_LENGTH+1);

-   for(int i=0;i<SHA256_DIGEST_LENGTH;i++){
+   SHA256_Init(&context);
+
+   SHA256_Update(&context, s, strlen(s));
+
+   SHA256_Final(md, &context);
+
+   for(i=0;i<DIGEST_LENGTH;i++)
      snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
-   }
+
 }

-
-void digest_string(char *digestname, char *s, char *digest){
-   EVP_MD_CTX *ctx;
-   const EVP_MD *md;
-   unsigned int i, md_len;
-   unsigned char md_value[DIGEST_LENGTH];
-
-   memset(digest, 0, 2*DIGEST_LENGTH+2);
-
-   md = EVP_get_digestbyname(digestname);
-   if(md == NULL){
-      syslog(LOG_PRIORITY, "ERROR: unknown message digest: '%s' in %s:%d", digestname, __func__, __LINE__);
-      return;
-   }
-
-   ctx = EVP_MD_CTX_new();
-   EVP_DigestInit_ex(ctx, md, NULL);
-   EVP_DigestUpdate(ctx, s, strlen(s));
-   EVP_DigestFinal_ex(ctx, md_value, &md_len);
-   EVP_MD_CTX_free(ctx);
-
-   for(i=0;i<md_len;i++){
-      snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md_value[i]);
-   }
-}
-
-
-void create_md5_from_email_address(char *puf, char *md5buf){
-   digest_string("md5", puf, md5buf);
-   md5buf[2*MD5_DIGEST_LENGTH] = ' ';
-}
--- a/src/dirs.c
+++ b/src/dirs.c
@ -13,25 +13,10 @@
 #include <piler.h>


-void createdir(char *path, uid_t uid, gid_t gid, mode_t mode){
-   struct stat st;
-
-   if(strlen(path) > 2){
-      if(path[strlen(path)-1] == '/') path[strlen(path)-1] = '\0';
-
-      if(stat(path, &st)){
-         if(mkdir(path, mode) == 0){
-            if(chown(path, uid, gid))
-               syslog(LOG_PRIORITY, "ERROR: createdir(): chown() failed on %s", path);
-            syslog(LOG_PRIORITY, "created directory: *%s*", path);
-         }
-         else syslog(LOG_PRIORITY, "ERROR: could not create directory: *%s*", path);
-      }
-   }
-}
+void createdir(char *path, uid_t uid, gid_t gid, mode_t mode);


-void check_and_create_directories(struct config *cfg, uid_t uid, gid_t gid){
+void check_and_create_directories(struct __config *cfg, uid_t uid, gid_t gid){
   char *p, s[SMALLBUFSIZE];
   int i;

@ -71,3 +56,23 @@ void check_and_create_directories(struct config *cfg, uid_t uid, gid_t gid){
   }

 }
+
+
+void createdir(char *path, uid_t uid, gid_t gid, mode_t mode){
+   struct stat st;
+
+   if(strlen(path) > 2){
+      if(path[strlen(path)-1] == '/') path[strlen(path)-1] = '\0';
+
+      if(stat(path, &st)){
+         if(mkdir(path, mode) == 0){
+            chown(path, uid, gid);
+            syslog(LOG_PRIORITY, "created directory: *%s*", path);
+         }
+         else syslog(LOG_PRIORITY, "could not create directory: *%s*", path);
+      }
+
+   }
+}
+
+
--- a/src/errmsg.h
+++ b/src/errmsg.h
@ -20,7 +20,6 @@
 #define ERR_TIMED_OUT "ERR: timed out"
 #define ERR_FORK_FAILED "ERR: cannot fork()"
 #define ERR_OPEN_DEDUP_FILE "ERR: cannot open dedup file"
-#define ERR_PID_FILE_EXISTS "ERR: pidfile exists. If piler daemon is not running, and the pidfile exists (unclean shutdown?) then remove it, and start piler again"

 #define ERR_MYSQL_CONNECT "Cannot connect to mysql server"
 #define ERR_PSQL_CONNECT "Cannot connect to PSql server"
--- a/src/extract.c
+++ b/src/extract.c
@ -19,38 +19,40 @@
 #define die(e) do { syslog(LOG_INFO, "error: helper: %s", e); exit(EXIT_FAILURE); } while (0);


-int remove_xml(char *src, char *dest, int destlen, int *html){
+void remove_xml(char *buf, int *html){
   int i=0;
+   char *p;

-   memset(dest, 0, destlen);
+   p = buf;

-   for(; *src; src++){
-      if(*src == '<'){
-         *html = 1;
+   for(; *p; p++){
+      if(*p == '<'){ *html = 1; }
+
+      if(*html == 0){
+         *(buf+i) = *p;
+         i++;
      }
-      else if(*src == '>'){
+
+      if(*p == '>'){
         *html = 0;
-         // make sure there's a whitespace between tag contents
-         if(i < destlen && i > 0 && !isspace(dest[i-1]))
-            dest[i++] = ' ';
-      }
-      else{
-         if(*html == 0){
-            if(i < destlen) *(dest+i) = *src;
-            i++;
+
+         if(i > 2 && *(buf+i-1) != ' '){
+            *(buf+i) = ' '; i++;
         }
+
      }
+
   }

-   return i;
+   *(buf+i) = '\0';
 }


 #ifdef HAVE_ZIP
 int extract_opendocument(struct session_data *sdata, struct parser_state *state, char *filename, char *prefix){
-   int errorp, i=0, html=0;
-   unsigned int len2;
-   char buf[4*MAXBUFSIZE], puf[4*MAXBUFSIZE];
+   int errorp, i=0, len=0, html=0;
+   int len2;
+   char buf[MAXBUFSIZE];
   struct zip *z;
   struct zip_stat sb;
   struct zip_file *zf;
@ -68,13 +70,13 @@ int extract_opendocument(struct session_data *sdata, struct parser_state *state,

         zf = zip_fopen_index(z, i, 0);
         if(zf){
-            int len;
            while((len = zip_fread(zf, buf, sizeof(buf)-2)) > 0){

-               len2 = remove_xml(buf, puf, sizeof(puf), &html);
+               remove_xml(buf, &html);
+               len2 = strlen(buf);

               if(len2 > 0 && state->bodylen < BIGBUFSIZE-len2-1){
-                  memcpy(&(state->b_body[state->bodylen]), puf, len2);
+                  memcpy(&(state->b_body[state->bodylen]), buf, len2);
                  state->bodylen += len2;
               }

@ -97,8 +99,8 @@ int extract_opendocument(struct session_data *sdata, struct parser_state *state,
 }


-int unzip_file(struct session_data *sdata, struct parser_state *state, char *filename, int *rec, struct config *cfg){
-   int errorp, i=0, fd;
+int unzip_file(struct session_data *sdata, struct parser_state *state, char *filename, int *rec, struct __config *cfg){
+   int errorp, i=0, len=0, fd;
   char *p, extracted_filename[SMALLBUFSIZE], buf[MAXBUFSIZE];
   struct zip *z;
   struct zip_stat sb;
@ -129,9 +131,8 @@ int unzip_file(struct session_data *sdata, struct parser_state *state, char *fil
            if(fd != -1){
               zf = zip_fopen_index(z, i, 0);
               if(zf){
-                  int len;
                  while((len = zip_fread(zf, buf, sizeof(buf))) > 0){
-                     if(write(fd, buf, len) == -1) syslog(LOG_PRIORITY, "ERROR: error writing to fd in %s", __func__);
+                     write(fd, buf, len);
                  }
                  zip_fclose(zf);
               }
@ -168,13 +169,13 @@ int unzip_file(struct session_data *sdata, struct parser_state *state, char *fil

 #ifdef HAVE_TNEF

-int extract_tnef(struct session_data *sdata, struct parser_state *state, char *filename, struct config *cfg){
+int extract_tnef(struct session_data *sdata, struct parser_state *state, char *filename, struct __config *cfg){
   int rc=0, n, rec=1;
   char tmpdir[BUFLEN], buf[SMALLBUFSIZE];
   struct dirent **namelist;

   memset(tmpdir, 0, sizeof(tmpdir));
-   make_random_string((unsigned char *)&tmpdir[0], sizeof(tmpdir)-3);
+   make_random_string(&tmpdir[0], sizeof(tmpdir)-3);

   memcpy(&tmpdir[sizeof(tmpdir)-3], ".d", 2);

@ -182,7 +183,7 @@ int extract_tnef(struct session_data *sdata, struct parser_state *state, char *f

   snprintf(buf, sizeof(buf)-1, "%s --unix-paths -C %s %s", HAVE_TNEF, tmpdir, filename);

-   if(system(buf) == -1) syslog(LOG_INFO, "error: running %s", buf);
+   system(buf);

   n = scandir(tmpdir, &namelist, NULL, alphasort);
   if(n < 0) syslog(LOG_INFO, "error: reading %s", tmpdir);
@ -215,9 +216,10 @@ void kill_helper(){
 }


-void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct config *cfg){
-   int link[2];
+void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct __config *cfg){
+   int link[2], n;
   pid_t pid;
+   char outbuf[MAXBUFSIZE];

   if(strcmp(type, "other") == 0 || strcmp(type, "text") == 0) return;

@ -312,19 +314,18 @@ void extract_attachment_content(struct session_data *sdata, struct parser_state
   }
   else {
      close(link[1]);
-      ssize_t n;
-      char outbuf[MAXBUFSIZE];
-
      while((n = read(link[0], outbuf, sizeof(outbuf))) > 0){
         if(state->bodylen < BIGBUFSIZE-n-1){
            memcpy(&(state->b_body[state->bodylen]), outbuf, n);
            state->bodylen += n;
         }
+         //printf("Output: %.*s\n", n, outbuf);
      }
-      close(link[0]);
      wait(NULL);
      return;
   }


 }
+
+
--- a/src/hash.c
+++ b/src/hash.c
@ -33,10 +33,12 @@ void clearhash(struct node *xhash[]){
         p = q;

         q = q->r;
-         if(p->str){
-            free(p->str);
+         if(p){
+            if(p->str){
+               free(p->str);
+            }
+            free(p);
         }
-         free(p);
      }
      xhash[i] = NULL;
   }
@ -152,12 +154,13 @@ int is_substr_in_hash(struct node *xhash[], char *s){


 unsigned int DJBHash(char* str, unsigned int len){
-   unsigned int hashval = 5381;
+   unsigned int hash = 5381;
   unsigned int i    = 0;

   for(i=0; i < len; str++, i++){
-      hashval = ((hashval << 5) + hashval) + (*str);
+      hash = ((hash << 5) + hash) + (*str);
   }

-   return hashval;
+   return hash;
 }
+
--- a/src/htmlentities.h
+++ b/src/htmlentities.h
@ -285,3 +285,4 @@ struct mi htmlentities[] = {


 #endif /* _HTMLENTITIES_H */
+
--- a/src/imap.c
+++ b/src/imap.c
@ -23,6 +23,9 @@
 #include <piler.h>


+void update_import_job_stat(struct session_data *sdata, struct __data *data);
+
+
 int get_message_length_from_imap_answer(char *s){
   char *p, *q;
   int len=0;
@ -51,18 +54,18 @@ int get_message_length_from_imap_answer(char *s){
 }


-int read_response(char *buf, int buflen, struct data *data){
+int read_response(int sd, char *buf, int buflen, int *seq, struct __data *data, int use_ssl){
   int i=0, n, len=0, rc=0;
   char puf[MAXBUFSIZE], tagok[SMALLBUFSIZE], tagno[SMALLBUFSIZE], tagbad[SMALLBUFSIZE];

-   snprintf(tagok, sizeof(tagok)-1, "A%d OK", data->import->seq);
-   snprintf(tagno, sizeof(tagno)-1, "A%d NO", data->import->seq);
-   snprintf(tagbad, sizeof(tagbad)-1, "A%d BAD", data->import->seq);
+   snprintf(tagok, sizeof(tagok)-1, "A%d OK", *seq);
+   snprintf(tagno, sizeof(tagno)-1, "A%d NO", *seq);
+   snprintf(tagbad, sizeof(tagbad)-1, "A%d BAD", *seq);

   memset(buf, 0, buflen);

   while(!strstr(buf, tagok)){
-      n = recvtimeoutssl(data->net, puf, sizeof(puf));
+      n = recvtimeoutssl(sd, puf, sizeof(puf), data->import->timeout, use_ssl, data->ssl);

      if(n + len < buflen) strncat(buf, puf, n);
      else goto END;
@ -84,74 +87,28 @@ int read_response(char *buf, int buflen, struct data *data){

 END:

-   (data->import->seq)++;
+   (*seq)++;

   return rc;
 }


-int connect_to_imap_server(struct data *data){
-   char buf[MAXBUFSIZE];
+int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg){
+   int rc=ERR, i, n, messages=0, len, readlen, fd, nreads, readpos, finished, msglen, msg_written_len, tagoklen, tagbadlen, result;
+   char *p, tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], tagbad[SMALLBUFSIZE], buf[MAXBUFSIZE], puf[MAXBUFSIZE], filename[SMALLBUFSIZE];

-   data->import->cap_uidplus = 0;
-
-   if(data->net->use_ssl == 1){
-      init_ssl_to_server(data);
-   }
-
-
-   recvtimeoutssl(data->net, buf, sizeof(buf));
-
-
-   /* imap cmd: LOGIN */
-
-   if(strcmp(data->import->username, "ZIMBRA") == 0){
-      snprintf(buf, sizeof(buf)-1, "A%d AUTHENTICATE PLAIN %s\r\n", data->import->seq, data->import->password);
-   }
-   else {
-      snprintf(buf, sizeof(buf)-1, "A%d LOGIN %s \"%s\"\r\n", data->import->seq, data->import->username, data->import->password);
-   }
-
-   write1(data->net, buf, strlen(buf));
-   if(read_response(buf, sizeof(buf), data) == 0){
-      printf("login failed, server reponse: %s\n", buf);
-      return ERR;
-   }
-
-   if(strstr(buf, "UIDPLUS")){
-      data->import->cap_uidplus = 1;
-   }
-   else {
-
-      /* run the CAPABILITY command if the reply doesn't contain the UIDPLUS capability */
-
-      snprintf(buf, sizeof(buf)-1, "A%d CAPABILITY\r\n", data->import->seq);
-
-      write1(data->net, buf, strlen(buf));
-      read_response(buf, sizeof(buf), data);
-
-      if(strstr(buf, "UIDPLUS")) data->import->cap_uidplus = 1;
-   }
-
-
-   return OK;
-}
-
-
-int imap_select_cmd_on_folder(char *folder, struct data *data){
-   int messages=0;
-   char *p, buf[MAXBUFSIZE];
+   /* imap cmd: SELECT */

   if(strchr(folder, '"'))
-      snprintf(buf, sizeof(buf)-1, "A%d SELECT %s\r\n", data->import->seq, folder);
+      snprintf(buf, sizeof(buf)-1, "A%d SELECT %s\r\n", *seq, folder);
   else
-      snprintf(buf, sizeof(buf)-1, "A%d SELECT \"%s\"\r\n", data->import->seq, folder);
+      snprintf(buf, sizeof(buf)-1, "A%d SELECT \"%s\"\r\n", *seq, folder);

-   write1(data->net, buf, strlen(buf));
-   if(read_response(buf, sizeof(buf), data) == 0){
+   write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+   if(read_response(sd, buf, sizeof(buf), seq, data, use_ssl) == 0){
      trimBuffer(buf);
-      printf("ERROR: select cmd error: %s\n", buf);
-      return messages;
+      printf("select cmd error: %s\n", buf);
+      return rc;
   }

   p = strstr(buf, " EXISTS");
@ -164,241 +121,304 @@ int imap_select_cmd_on_folder(char *folder, struct data *data){
      }
   }

-   if(data->quiet == 0){printf("found %d messages\n", messages); }
-
-   data->import->total_messages += messages;
-
-   return messages;
-}
-
-
-int imap_download_email(struct data *data, int i){
-   int fd, len, result, tagoklen, tagbadlen;
-   int n, readlen=0, nreads=0, readpos=0, finished=0, msglen=0, msg_written_len=0;
-   char *p, buf[MAXBUFSIZE], puf[MAXBUFSIZE], tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], tagbad[SMALLBUFSIZE];
-
-   data->import->processed_messages++;
-
-   snprintf(data->import->filename, SMALLBUFSIZE-1, "%d-imap-%d.txt", getpid(), data->import->processed_messages);
-
-   unlink(data->import->filename);
-
-   fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
-   if(fd == -1){
-      printf("cannot open: %s\n", data->import->filename);
-      return ERR;
-   }
-
-   snprintf(tag, sizeof(tag)-1, "A%d", data->import->seq);
-   snprintf(tagok, sizeof(tagok)-1, "A%d OK", (data->import->seq)++);
-   snprintf(tagbad, sizeof(tagbad)-1, "%s BAD", tag);
-
-   tagoklen = strlen(tagok);
-   tagbadlen = strlen(tagbad);
-
-   snprintf(buf, sizeof(buf)-1, "%s FETCH %d (BODY.PEEK[])\r\n", tag, i);
-   write1(data->net, buf, strlen(buf));
-
-   while((n = recvtimeoutssl(data->net, &buf[readpos], sizeof(buf)-readpos)) > 0){
-
-      readlen += n;
-
-      if(strchr(buf, '\n')){
-         readpos = 0;
-         p = &buf[0];
-         do {
-            nreads++;
-            memset(puf, 0, sizeof(puf));
-            p = split(p, '\n', puf, sizeof(puf)-1, &result);
-            len = strlen(puf);
-
-            if(result == 1){
-               // process a complete line
-
-               if(nreads == 1){
-
-                  if(strcasestr(puf, " FETCH ")){
-                     msglen = get_message_length_from_imap_answer(puf);
-
-                     if(msglen == 0){
-                        finished = 1;
-                        break;
-                     }
-                     continue;
-                  }
-
-                  if(strcasestr(puf, " BYE")){
-                     printf("imap server sent BYE response: '%s'\n", puf);
-                     close(fd);
-                     unlink(data->import->filename);
-                     return ERR;
-                  }
-
-               }
-
-               if(len > 0 && msg_written_len < msglen){
-                  if(write(fd, puf, len) == -1) printf("ERROR: writing to fd\n");
-                  if(write(fd, "\n", 1) == -1) printf("ERROR: writing to fd\n");
-                  msg_written_len += len + 1;
-               }
-
-               if(strncmp(puf, tagok, tagoklen) == 0){
-                  finished = 1;
-                  break;
-               }
-
-               if(strncmp(puf, tagbad, tagbadlen) == 0){
-                  printf("ERROR happened reading the message!\n");
-                  finished = 1;
-                  break;
-               }
-
-            }
-            else {
-               // prepend the last incomplete line back to 'buf'
-
-               snprintf(buf, sizeof(buf)-2, "%s", puf);
-               readpos = len;
-               break;
-            }
-
-         } while(p);
-
-
-      }
-      else {
-         readpos += n;
-      }
-
-      if(finished == 1) break;
-   }
-
-   close(fd);
-
-   if(msglen > 10) return OK;
-
-   return ERR;
-}
-
-
-void imap_delete_message(struct data *data, int i){
-   char buf[SMALLBUFSIZE];
-
-   snprintf(buf, sizeof(buf)-1, "A%d STORE %d +FLAGS.SILENT (\\Deleted)\r\n", data->import->seq, i);
-   write1(data->net, buf, strlen(buf));
-   read_response(buf, sizeof(buf), data);
-}
-
-
-void imap_move_message_to_folder(struct data *data, int i){
-   int tagoklen;
-   char buf[SMALLBUFSIZE], tagok[SMALLBUFSIZE];
-
-   snprintf(tagok, sizeof(tagok)-1, "A%d OK", data->import->seq);
-   tagoklen = strlen(tagok);
-
-   snprintf(buf, sizeof(buf)-1, "A%d COPY %d %s\r\n", data->import->seq, i, data->import->move_folder);
-   write1(data->net, buf, strlen(buf));
-   read_response(buf, sizeof(buf), data);
-
-   if(strncmp(buf, tagok, tagoklen) == 0){
-      snprintf(buf, sizeof(buf)-1, "A%d STORE %d +FLAGS.SILENT (\\Deleted)\r\n", data->import->seq, i);
-      write1(data->net, buf, strlen(buf));
-      read_response(buf, sizeof(buf), data);
-   }
-}
-
-
-void imap_expunge_message(struct data *data){
-   char buf[SMALLBUFSIZE];
-
-   snprintf(buf, sizeof(buf)-1, "A%d EXPUNGE\r\n", data->import->seq);
-   write1(data->net, buf, strlen(buf));
-   read_response(buf, sizeof(buf), data);
-}
-
-
-int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
-   int i, messages=0;
-
-   messages = imap_select_cmd_on_folder(folder, data);
+   printf("found %d messages\n", messages);

   if(messages <= 0) return OK;

+
   if(data->recursive_folder_names == 1){
-      data->folder = get_folder_id(sdata, folder, 0);
-      if(data->folder == ERR_FOLDER) data->folder = add_new_folder(sdata, folder, 0);
+      data->folder = get_folder_id(sdata, data, folder, 0);
+      if(data->folder == ERR_FOLDER) data->folder = add_new_folder(sdata, data, folder, 0);
   }

+
+   data->import->total_messages += messages;
+
   for(i=data->import->start_position; i<=messages; i++){
-      if(imap_download_email(data, i) == OK){
-         if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/messages); fflush(stdout); }
-
-         if(data->import->dryrun == 0){
-            int rc = import_message(sdata, data, counters, cfg);
-
-            if(data->import->remove_after_import == 1 && rc == OK){
-               imap_delete_message(data, i);
-            }
-
-            if(data->import->move_folder && data->import->cap_uidplus == 1){
-               imap_move_message_to_folder(data, i);
-            }
-         }
-
-         if(data->import->download_only == 0) unlink(data->import->filename);
-      }

      /* whether to quit after processing a batch of messages */

      if(data->import->batch_processing_limit > 0 && data->import->processed_messages >= data->import->batch_processing_limit){
         break;
      }
+
+      data->import->processed_messages++;
+      if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/messages); fflush(stdout); }
+
+      snprintf(tag, sizeof(tag)-1, "A%d", *seq);
+      snprintf(tagok, sizeof(tagok)-1, "A%d OK", (*seq)++);
+      snprintf(tagbad, sizeof(tagbad)-1, "%s BAD", tag);
+
+      tagoklen = strlen(tagok);
+      tagbadlen = strlen(tagbad);
+
+      snprintf(buf, sizeof(buf)-1, "%s FETCH %d (BODY.PEEK[])\r\n", tag, i);
+
+      snprintf(filename, sizeof(filename)-1, "%d-imap-%d.txt", getpid(), data->import->processed_messages);
+      unlink(filename);
+
+      fd = open(filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
+      if(fd == -1){
+         printf("cannot open: %s\n", filename);
+         return rc;
+      }
+
+
+      write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+
+      readlen = 0;
+      nreads = 0;
+      readpos = 0;
+      finished = 0;
+      msglen = 0;
+      msg_written_len = 0;
+
+      while((n = recvtimeoutssl(sd, &buf[readpos], sizeof(buf)-readpos, data->import->timeout, use_ssl, data->ssl)) > 0){
+
+         readlen += n;
+
+         if(strchr(buf, '\n')){
+            readpos = 0;
+            p = &buf[0];
+            do {
+               nreads++;
+               memset(puf, 0, sizeof(puf));
+               p = split(p, '\n', puf, sizeof(puf)-1, &result);
+               len = strlen(puf);
+
+               if(result == 1){
+                  // process a complete line
+
+                  if(nreads == 1){
+
+                     if(strcasestr(puf, " FETCH ")){
+                        msglen = get_message_length_from_imap_answer(puf);
+
+                        if(msglen == 0){
+                           finished = 1;
+                           break;
+                        }
+                        continue;
+                     }
+
+                     if(strcasestr(puf, " BYE")){
+                        printf("imap server sent BYE response: '%s'\n", puf);
+                        close(fd);
+                        unlink(filename);
+                        return ERR;
+                     }
+
+                  }
+ 
+                  if(len > 0 && msg_written_len < msglen){
+                     write(fd, puf, len);
+                     write(fd, "\n", 1);
+                     msg_written_len += len + 1;
+                  }
+
+                  if(strncmp(puf, tagok, tagoklen) == 0){
+                     finished = 1;
+                     break;
+                  }
+
+                  if(strncmp(puf, tagbad, tagbadlen) == 0){
+                     printf("ERROR happened reading the message!\n");
+                     finished = 1;
+                     break;
+                  }
+
+               }
+               else {
+                  // prepend the last incomplete line back to 'buf'
+
+                  snprintf(buf, sizeof(buf)-2, "%s", puf);
+                  readpos = len;
+                  break;
+               }
+
+            } while(p);
+
+
+
+         }
+         else {
+            readpos += n;
+         }
+
+         if(finished == 1) break;
+      }
+
+
+      close(fd);
+
+      if(dryrun == 0 && msglen > 10){
+         rc = import_message(filename, sdata, data, cfg);
+
+         if(data->import->processed_messages % 100 == 0){
+            time(&(data->import->updated));
+            update_import_job_stat(sdata, data);
+         }
+      }
+      else rc = OK;
+
+
+      if(rc == ERR) printf("error importing '%s'\n", filename);
+      else {
+
+         if(data->import->remove_after_import == 1 && dryrun == 0){
+            snprintf(buf, sizeof(buf)-1, "A%d STORE %d +FLAGS.SILENT (\\Deleted)\r\n", *seq, i);
+            write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+            read_response(sd, buf, sizeof(buf), seq, data, use_ssl);
+         }
+
+
+         if(data->import->move_folder && data->import->cap_uidplus == 1 && dryrun == 0){
+
+            snprintf(tagok, sizeof(tagok)-1, "A%d OK", *seq);
+            tagoklen = strlen(tagok);
+
+            snprintf(buf, sizeof(buf)-1, "A%d COPY %d %s\r\n", *seq, i, data->import->move_folder);
+            write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+            read_response(sd, buf, sizeof(buf), seq, data, use_ssl);
+
+            if(strncmp(buf, tagok, tagoklen) == 0){
+               snprintf(buf, sizeof(buf)-1, "A%d STORE %d +FLAGS.SILENT (\\Deleted)\r\n", *seq, i);
+               write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+               read_response(sd, buf, sizeof(buf), seq, data, use_ssl);
+
+            }
+         }
+
+
+
+         if(data->import->download_only == 0) unlink(filename);
+      }
+
+
   }

-   if((data->import->remove_after_import == 1 || data->import->move_folder) && data->import->dryrun == 0){
-      imap_expunge_message(data);
+   if((data->import->remove_after_import == 1 || data->import->move_folder) && dryrun == 0){
+      snprintf(buf, sizeof(buf)-1, "A%d EXPUNGE\r\n", *seq);
+      write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+      read_response(sd, buf, sizeof(buf), seq, data, use_ssl);
   }


-   if(data->quiet == 0){printf("\n"); }
+   printf("\n");

   return OK;
 }


-void send_imap_close(struct data *data){
-   char puf[SMALLBUFSIZE];
-   snprintf(puf, sizeof(puf)-1, "A%d CLOSE\r\n", data->import->seq);
+int connect_to_imap_server(int sd, int *seq, char *username, char *password, struct __data *data, int use_ssl){
+   int n;
+   char buf[MAXBUFSIZE];
+   X509* server_cert;
+   char *str;

-   write1(data->net, puf, strlen(puf));
+   data->import->cap_uidplus = 0;
+
+   if(use_ssl == 1){
+
+      SSL_library_init();
+      SSL_load_error_strings();
+
+      data->ctx = SSL_CTX_new(TLSv1_client_method());
+      CHK_NULL(data->ctx, "internal SSL error");
+
+      data->ssl = SSL_new(data->ctx);
+      CHK_NULL(data->ssl, "internal ssl error");
+
+      SSL_set_fd(data->ssl, sd);
+      n = SSL_connect(data->ssl);
+      CHK_SSL(n, "internal ssl error");
+
+      printf("Cipher: %s\n", SSL_get_cipher(data->ssl));
+
+      server_cert = SSL_get_peer_certificate(data->ssl);
+      CHK_NULL(server_cert, "server cert error");
+
+      //if(verbose){
+         str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0);
+         CHK_NULL(str, "error in server cert");
+         printf("server cert:\n\t subject: %s\n", str);
+         OPENSSL_free(str);
+
+         str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0);
+         CHK_NULL(str, "error in server cert");
+         printf("\t issuer: %s\n\n", str);
+         OPENSSL_free(str);
+      //}
+
+      X509_free(server_cert);
+   }
+
+
+   recvtimeoutssl(sd, buf, sizeof(buf), data->import->timeout, use_ssl, data->ssl);
+
+
+   /* imap cmd: LOGIN */
+
+   snprintf(buf, sizeof(buf)-1, "A%d LOGIN %s \"%s\"\r\n", *seq, username, password);
+
+   write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+   if(read_response(sd, buf, sizeof(buf), seq, data, use_ssl) == 0){
+      printf("login failed, server reponse: %s\n", buf);
+      return ERR;
+   }
+
+   if(strstr(buf, "UIDPLUS")){
+      data->import->cap_uidplus = 1;
+   }
+   else {
+
+      /* run the CAPABILITY command if the reply doesn't contain the UIDPLUS capability */
+
+      snprintf(buf, sizeof(buf)-1, "A%d CAPABILITY\r\n", *seq);
+
+      write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+      read_response(sd, buf, sizeof(buf), seq, data, use_ssl);
+
+      if(strstr(buf, "UIDPLUS")) data->import->cap_uidplus = 1;
+   }
+
+
+   return OK;
 }


-int list_folders(struct data *data){
+void send_imap_close(int sd, int *seq, struct __data *data, int use_ssl){
+   char puf[SMALLBUFSIZE];  
+   snprintf(puf, sizeof(puf)-1, "A%d CLOSE\r\n", *seq);
+
+   write1(sd, puf, strlen(puf), use_ssl, data->ssl);
+}
+
+
+int list_folders(int sd, int *seq, int use_ssl, char *folder_name, struct __data *data){
   char *p, *q, *r, *buf, *ruf, tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], puf[MAXBUFSIZE];
   char attrs[SMALLBUFSIZE], folder[SMALLBUFSIZE];
   int len=MAXBUFSIZE+3, pos=0, n, rc=ERR, fldrlen=0, result;

-   if(data->quiet == 0){printf("List of IMAP folders:\n"); }
+   printf("List of IMAP folders:\n");

   buf = malloc(len);
   if(!buf) return rc;

   memset(buf, 0, len);

-   snprintf(tag, sizeof(tag)-1, "A%d", data->import->seq); snprintf(tagok, sizeof(tagok)-1, "A%d OK", (data->import->seq)++);
-   if(data->import->folder_imap == NULL)
+   snprintf(tag, sizeof(tag)-1, "A%d", *seq); snprintf(tagok, sizeof(tagok)-1, "A%d OK", (*seq)++);
+   if(folder_name == NULL)
      snprintf(puf, sizeof(puf)-1, "%s LIST \"\" \"*\"\r\n", tag);
   else
-      snprintf(puf, sizeof(puf)-1, "%s LIST \"%s\" \"*\"\r\n", tag, data->import->folder_imap);
+      snprintf(puf, sizeof(puf)-1, "%s LIST \"%s\" \"*\"\r\n", tag, folder_name);

-   write1(data->net, puf, strlen(puf));
+   write1(sd, puf, strlen(puf), use_ssl, data->ssl);

   p = NULL;

   while(1){
-      n = recvtimeoutssl(data->net, puf, sizeof(puf));
+      n = recvtimeoutssl(sd, puf, sizeof(puf), data->import->timeout, use_ssl, data->ssl);
      if(n < 0) return ERR;

      if(pos + n >= len){
@ -421,7 +441,7 @@ int list_folders(struct data *data){
   }

   // trim the "A3 OK LIST completed" trailer off
-   if(p) *p = '\0'; //-V547
+   if(p) *p = '\0';

   memset(attrs, 0, sizeof(attrs));

@ -452,29 +472,23 @@ int list_folders(struct data *data){
               q++;
               fldrlen = strtol(q, NULL, 10);
            } else {
-
+               
               if(fldrlen) {
-                  int ruflen = strlen(q) * 2;
-                  ruf = malloc(ruflen + 1);
-                  if(ruf){
-                     snprintf(ruf, ruflen, "%s", q);
-                     r = ruf;
-                     while(*r != '\0') {
-                        if(*r == '\\') {
-                           memmove(r + 1, r, strlen(r));
-                           r++;
-                        }
+                  ruf = malloc(strlen(q) * 2 + 1);
+                  memset(ruf, 0, strlen(q) * 2 + 1);
+                  memcpy(ruf, q, strlen(q));
+                  r = ruf;
+                  while(*r != '\0') {
+                     if(*r == '\\') {
+                        memmove(r + 1, r, strlen(r));
                        r++;
                     }
-
-                     snprintf(folder, sizeof(folder)-1, "%s", ruf);
-
-                     free(ruf);
-                  }
-                  else {
-                     printf("error: ruf = malloc()\n");
+                     r++;
                  }

+                  snprintf(folder, sizeof(folder)-1, "%s", ruf);
+
+                  free(ruf);
                  fldrlen = 0;
               } else {
                  snprintf(folder, sizeof(folder)-1, "%s", q);
@ -483,9 +497,9 @@ int list_folders(struct data *data){
               if(!strstr(attrs, "\\Noselect")){
                  addnode(data->imapfolders, folder);
               }
-               else if(data->quiet == 0){printf("skipping "); }
+               else printf("skipping ");

-               if(data->quiet == 0){printf("=> '%s [%s]'\n", folder, attrs); }
+               printf("=> '%s [%s]'\n", folder, attrs);

               memset(attrs, 0, sizeof(attrs));
            }
@ -504,3 +518,5 @@ ENDE_FOLDERS:

   return rc;
 }
+
+
--- a/src/import.c
+++ b/src/import.c
@ -18,29 +18,27 @@
 #include <piler.h>


-int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+int import_message(char *filename, struct session_data *sdata, struct __data *data, struct __config *cfg){
   int rc=ERR;
-   char *rule;
+   char *p, *rule, newpath[SMALLBUFSIZE];
   struct stat st;
   struct parser_state state;
+   struct counters counters;

-   if(data->import->delay > 0){
-      struct timespec req;
-
-      req.tv_sec = 0;
-      req.tv_nsec = 1000000 * data->import->delay;
-
-      nanosleep(&req, NULL);
-   }

   init_session_data(sdata, cfg);

-   if(cfg->verbosity > 1) printf("processing: %s\n", data->import->filename);
+   if(data->import->extra_recipient){
+      snprintf(sdata->rcptto[0], SMALLBUFSIZE-1, "%s", data->import->extra_recipient);
+      sdata->num_of_rcpt_to = 1;
+   }

-   if(strcmp(data->import->filename, "-") == 0){
+   if(cfg->verbosity > 1) printf("processing: %s\n", filename);
+
+   if(strcmp(filename, "-") == 0){

      if(read_from_stdin(sdata) == ERR){
-         printf("ERROR: error reading from stdin\n");
+         printf("error reading from stdin\n");
         return rc;
      }

@ -49,17 +47,17 @@ int import_message(struct session_data *sdata, struct data *data, struct counter
   }
   else {

-      if(stat(data->import->filename, &st) != 0){
-         printf("ERROR: cannot stat() %s\n", data->import->filename);
+      if(stat(filename, &st) != 0){
+         printf("cannot stat() %s\n", filename);
         return rc;
      }

      if(S_ISREG(st.st_mode) == 0){
-         printf("ERROR: %s is not a file\n", data->import->filename);
+         printf("%s is not a file\n", filename);
         return rc;
      }

-      snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", data->import->filename);
+      snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);

      sdata->tot_len = st.st_size;
   }
@ -72,166 +70,132 @@ int import_message(struct session_data *sdata, struct data *data, struct counter

   data->import->total_size += sdata->tot_len;

-
+   
   sdata->delivered = 0;

   sdata->import = 1;

   state = parse_message(sdata, 1, data, cfg);
   post_parse(sdata, &state, cfg);
-   rule = check_against_ruleset(data->archiving_rules, &state, sdata->tot_len, sdata->spam_message);
+
+   rule = check_againt_ruleset(data->archiving_rules, &state, sdata->tot_len, sdata->spam_message);

   if(rule){
-      if(data->quiet == 0) printf("discarding %s by archiving policy: %s\n", data->import->filename, rule);
+      if(data->quiet == 0) printf("discarding %s by archiving policy: %s\n", filename, rule);
      rc = OK;
   }
   else {
      make_digests(sdata, cfg);

      if(sdata->hdr_len < 10){
-         printf("%s: invalid message, hdr_len: %d\n", data->import->filename, sdata->hdr_len);
-         rc = ERR;
+         printf("%s: invalid message, hdr_len: %d\n", filename, sdata->hdr_len);
+         return ERR;
      }
-      else if(data->import->after > 0 && sdata->sent < data->import->after){
-         if(cfg->verbosity > 1) printf("discarding older email: %s (%ld/%ld)\n", sdata->filename, sdata->sent, data->import->after);
-         rc = ERR_DISCARDED;
-      }
-      else if(data->import->before > 0 && sdata->sent > data->import->before){
-         if(cfg->verbosity > 1) printf("discarding newer email: %s (%ld/%ld)\n", sdata->filename, sdata->sent, data->import->before);
-         rc = ERR_DISCARDED;
-      }
-      else {
-         // When importing emails, we should add the retention value (later) to the original sent value
-         sdata->retained = sdata->sent;

-         // backup original value of data->folder
-         int folder = data->folder;
-         rc = process_message(sdata, &state, data, cfg);
-         data->folder = folder;
-         unlink(state.message_id_hash);
-      }
+      rc = process_message(sdata, &state, data, cfg);
+      unlink(state.message_id_hash);
   }

   unlink(sdata->tmpframe);

-   remove_stripped_attachments(&state);
-
-   if(strcmp(data->import->filename, "-") == 0) unlink(sdata->ttmpfile);
+   if(strcmp(filename, "-") == 0) unlink(sdata->ttmpfile);


   switch(rc) {
      case OK:
-                        counters->c_rcvd++;
-                        counters->c_size += sdata->tot_len;
-                        counters->c_stored_size += sdata->stored_len;
+                        bzero(&counters, sizeof(counters));
+                        counters.c_rcvd = 1;
+                        counters.c_size += sdata->tot_len;
+                        counters.c_stored_size = sdata->stored_len;
+                        update_counters(sdata, data, &counters, cfg);

                        break;

      case ERR_EXISTS:
                        rc = OK;

-                        counters->c_duplicate++;
-
-                        if(data->quiet == 0) printf("duplicate: %s (duplicate id: %llu)\n", data->import->filename, sdata->duplicate_id);
-                        break;
-
-      case ERR_DISCARDED:
-                        rc = OK;
-
-                        counters->c_ignore++;
+                        bzero(&counters, sizeof(counters));
+                        counters.c_duplicate = 1;
+                        update_counters(sdata, data, &counters, cfg);

+                        if(data->quiet == 0) printf("duplicate: %s (duplicate id: %llu)\n", filename, sdata->duplicate_id);
                        break;

      default:
-                        printf("failed to import: %s (id: %s)\n", data->import->filename, sdata->ttmpfile);
+                        printf("failed to import: %s (id: %s)\n", filename, sdata->ttmpfile);
                        break;
-   }
+   } 

   if(rc != OK && data->import->failed_folder){
-      char *p = strrchr(data->import->filename, '/');
+      p = strrchr(filename, '/');
      if(p)
         p++;
      else
-         p = data->import->filename;
+         p = filename;

-      char newpath[SMALLBUFSIZE];
      snprintf(newpath, sizeof(newpath)-2, "%s/%s", data->import->failed_folder, p);

-      if(rename(data->import->filename, newpath))
-         printf("cannot move %s to %s\n", data->import->filename, newpath);
+      if(rename(filename, newpath))
+         printf("cannot move %s to %s\n", filename, newpath);
   }

   return rc;
 }


-int update_import_table(struct session_data *sdata, struct data *data) {
-   int ret=ERR, status=2, started=1;
-   struct sql sql;
-
-   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE) == ERR) return ret;
-
-   p_bind_init(&sql);
-
-   sql.sql[sql.pos] = (char *)&(started); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&(status); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&(data->import->tot_msgs); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&(data->import->table_id); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-
-   if(p_exec_stmt(sdata, &sql) == OK) ret = OK;
-
-   close_prepared_statement(&sql);
-
-   return ret;
-}
-
-
-int get_folder_id(struct session_data *sdata, char *foldername, int parent_id){
+int get_folder_id(struct session_data *sdata, struct __data *data, char *foldername, int parent_id){
   int id=ERR_FOLDER;
-   struct sql sql;

-   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_FOLDER_ID) == ERR) return id;
+   if(prepare_sql_statement(sdata, &(data->stmt_get_folder_id), SQL_PREPARED_STMT_GET_FOLDER_ID) == ERR) return id;

-   p_bind_init(&sql);
+   p_bind_init(data);
+   data->sql[data->pos] = foldername; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = (char *)&parent_id; data->type[data->pos] = TYPE_LONG; data->pos++;

-   sql.sql[sql.pos] = foldername; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = (char *)&parent_id; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
+   if(p_exec_query(sdata, data->stmt_get_folder_id, data) == OK){

-   if(p_exec_stmt(sdata, &sql) == OK){
+      p_bind_init(data);
+      data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(unsigned long); data->pos++;

-      p_bind_init(&sql);
-
-      sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(unsigned long); sql.pos++;
-
-      p_store_results(&sql);
-      p_fetch_results(&sql);
-      p_free_results(&sql);
+      p_store_results(data->stmt_get_folder_id, data);
+      p_fetch_results(data->stmt_get_folder_id);
+      p_free_results(data->stmt_get_folder_id);
   }

-   close_prepared_statement(&sql);
+   close_prepared_statement(data->stmt_get_folder_id);

   return id;
 }


-int add_new_folder(struct session_data *sdata, char *foldername, int parent_id){
+int add_new_folder(struct session_data *sdata, struct __data *data, char *foldername, int parent_id){
   int id=ERR_FOLDER;
-   struct sql sql;

   if(foldername == NULL) return id;

-   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE) == ERR) return id;
+   if(prepare_sql_statement(sdata, &(data->stmt_insert_into_folder_table), SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE) == ERR) return id;

-   p_bind_init(&sql);
+   p_bind_init(data);
+   data->sql[data->pos] = foldername; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = (char *)&parent_id; data->type[data->pos] = TYPE_LONG; data->pos++;

-   sql.sql[sql.pos] = foldername; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = (char *)&parent_id; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-
-   if(p_exec_stmt(sdata, &sql) == OK){
-      id = p_get_insert_id(&sql);
+   if(p_exec_query(sdata, data->stmt_insert_into_folder_table, data) == OK){
+      id = p_get_insert_id(data->stmt_insert_into_folder_table);
   }

-   close_prepared_statement(&sql);
+   close_prepared_statement(data->stmt_insert_into_folder_table);

   return id;
 }
+
+
+void update_import_job_stat(struct session_data *sdata, struct __data *data){
+   char buf[SMALLBUFSIZE];
+
+   snprintf(buf, sizeof(buf)-1, "update import set status=%d, started=%ld, updated=%ld, finished=%ld, total=%d, imported=%d where id=%d", data->import->status, data->import->started, data->import->updated, data->import->finished, data->import->total_messages, data->import->processed_messages, data->import->import_job_id);
+
+   p_query(sdata, buf);
+}
+
+
+
--- a/src/import.h
+++ b/src/import.h
@ -6,23 +6,24 @@
   #define _IMPORT_H


-int import_message(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
-int update_import_table(struct session_data *sdata, struct data *data);
+int import_message(char *filename, struct session_data *sdata, struct __data *data, struct __config *cfg);
+void update_import_job_stat(struct session_data *sdata, struct __data *data);

-int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg);
-int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
-int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
-void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
-int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+int read_gui_import_data(struct session_data *sdata, struct __data *data, char *folder_imap, char *skiplist, int dryrun, struct __config *cfg);

-int connect_to_pop3_server(struct data *data);
-void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+int import_from_maildir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg);
+int import_from_mailbox(char *mailbox, struct session_data *sdata, struct __data *data, struct __config *cfg);
+int import_mbox_from_dir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg);
+int import_from_pop3_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, int dryrun, struct __config *cfg);
+int import_from_imap_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, char *folder_imap, char *skiplist, int dryrun, struct __config *cfg);

-int connect_to_imap_server(struct data *data);
-int list_folders(struct data *data);
-int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
-void send_imap_close(struct data *data);
+int connect_to_pop3_server(int sd, char *username, char *password, struct __data *data, int use_ssl);
+int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg);

-void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+int connect_to_imap_server(int sd, int *seq, char *username, char *password, struct __data *data, int use_ssl);
+int list_folders(int sd, int *seq, int use_ssl, char *folder_name, struct __data *data);
+int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg);
+void send_imap_close(int sd, int *seq, struct __data *data, int use_ssl);

 #endif /* _IMPORT_H */
+
--- a/src/import_gui.c
+++ b/src/import_gui.c
@ -0,0 +1,80 @@
+/*
+ * import_gui.c, SJ
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <getopt.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+int read_gui_import_data(struct session_data *sdata, struct __data *data, char *folder_imap, char *skiplist, int dryrun, struct __config *cfg){
+   int rc=ERR;
+   char s_type[SMALLBUFSIZE], s_username[SMALLBUFSIZE], s_password[SMALLBUFSIZE], s_server[SMALLBUFSIZE];
+
+   memset(s_type, 0, sizeof(s_type));
+   memset(s_username, 0, sizeof(s_username));
+   memset(s_password, 0, sizeof(s_password));
+   memset(s_server, 0, sizeof(s_server));
+
+   if(prepare_sql_statement(sdata, &(data->stmt_generic), SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS) == ERR) return ERR;
+
+   p_bind_init(data);
+
+   if(p_exec_query(sdata, data->stmt_generic, data) == OK){
+
+      p_bind_init(data);
+
+      data->sql[data->pos] = (char *)&(data->import->import_job_id); data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
+      data->sql[data->pos] = &s_type[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_type)-2; data->pos++;
+      data->sql[data->pos] = &s_username[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_username)-2; data->pos++;
+      data->sql[data->pos] = &s_password[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_password)-2; data->pos++;
+      data->sql[data->pos] = &s_server[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_server)-2; data->pos++;
+
+      p_store_results(data->stmt_generic, data);
+
+      if(p_fetch_results(data->stmt_generic) == OK) rc = OK;
+
+      p_free_results(data->stmt_generic);
+   }
+
+   close_prepared_statement(data->stmt_generic);
+
+   data->import->processed_messages = 0;
+   data->import->total_messages = 0;
+
+   time(&(data->import->started));
+   data->import->status = 1;
+   update_import_job_stat(sdata, data);  
+
+   if(strcmp(s_type, "pop3") == 0){
+      rc = import_from_pop3_server(s_server, s_username, s_password, 110, sdata, data, dryrun, cfg);
+   }
+
+   if(strcmp(s_type, "imap") == 0){
+      rc = import_from_imap_server(s_server, s_username, s_password, 143, sdata, data, folder_imap, skiplist, dryrun, cfg);
+   }
+
+   update_import_job_stat(sdata, data);
+
+   // don't set error in case of a problem, because it
+   // will scare users looking at the gui progressbar
+
+   return rc;
+}
+
+
--- a/src/import_imap.c
+++ b/src/import_imap.c
@ -22,64 +22,61 @@
 #include <piler.h>


-int import_from_imap_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
-   int i, rc=ERR, ret=OK, skipmatch;
+int import_from_imap_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, char *folder_imap, char *skiplist, int dryrun, struct __config *cfg){
+   int i, rc=ERR, ret=OK, sd, seq=1, skipmatch, use_ssl=0;
   char port_string[8], puf[SMALLBUFSIZE];
   struct addrinfo hints, *res;
   struct node *q;

-   data->net->use_ssl = 0;
-   data->import->seq = 1;

   inithash(data->imapfolders);

-   snprintf(port_string, sizeof(port_string)-1, "%d", data->import->port);
+   snprintf(port_string, sizeof(port_string)-1, "%d", port);

   memset(&hints, 0, sizeof(hints));
   hints.ai_family = AF_UNSPEC;
   hints.ai_socktype = SOCK_STREAM;

-   if((rc = getaddrinfo(data->import->server, port_string, &hints, &res)) != 0){
-      printf("getaddrinfo for '%s': %s\n", data->import->server, gai_strerror(rc));
+   if((rc = getaddrinfo(server, port_string, &hints, &res)) != 0){
+      printf("getaddrinfo for '%s': %s\n", server, gai_strerror(rc));
      return ERR;
   }

-   if(data->import->port == 993) data->net->use_ssl = 1;
+   if(port == 993) use_ssl = 1;


-   if((data->net->socket = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
+   if((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
      printf("cannot create socket\n");
      ret = ERR;
      goto ENDE_IMAP;
   }

-   if(connect(data->net->socket, res->ai_addr, res->ai_addrlen) == -1){
+   if(connect(sd, res->ai_addr, res->ai_addrlen) == -1){
      printf("connect()\n");
      ret = ERR;
      goto ENDE_IMAP;
   }

-   if(connect_to_imap_server(data) == ERR){
-      close(data->net->socket);
+   if(connect_to_imap_server(sd, &seq, username, password, data, use_ssl) == ERR){
+      close(sd);
      ret = ERR;
      goto ENDE_IMAP;
   }


-   if(list_folders(data) == ERR) goto ENDE_IMAP;
-
+   if(list_folders(sd, &seq, use_ssl, folder_imap, data) == ERR) goto ENDE_IMAP;

   for(i=0;i<MAXHASH;i++){
      q = data->imapfolders[i];
      while(q != NULL){

-         if(q->str && strlen(q->str) > 1){
+         if(q && q->str && strlen(q->str) > 1){

            skipmatch = 0;

-            if(data->import->skiplist && strlen(data->import->skiplist) > 0){
+            if(skiplist && strlen(skiplist) > 0){
               snprintf(puf, sizeof(puf)-1, "%s,", (char *)q->str);
-               if(strstr(data->import->skiplist, puf)) skipmatch = 1;
+               if(strstr(skiplist, puf)) skipmatch = 1;
            }

            if(skipmatch == 1){
@ -88,7 +85,7 @@ int import_from_imap_server(struct session_data *sdata, struct data *data, struc
            else {
               if(data->quiet == 0) printf("processing folder: %s... ", (char *)q->str);

-               if(process_imap_folder(q->str, sdata, data, counters, cfg) == ERR) ret = ERR;
+               if(process_imap_folder(sd, &seq, q->str, sdata, data, use_ssl, dryrun, cfg) == ERR) ret = ERR;
            }

         }
@ -98,9 +95,9 @@ int import_from_imap_server(struct session_data *sdata, struct data *data, struc
      }
   }

-   send_imap_close(data);
+   send_imap_close(sd, &seq, data, use_ssl);

-   close_connection(data->net);
+   close_connection(sd, data, use_ssl);

 ENDE_IMAP:
   freeaddrinfo(res);
@ -111,3 +108,5 @@ ENDE_IMAP:

   return ret;
 }
+
+
--- a/src/import_mailbox.c
+++ b/src/import_mailbox.c
@ -22,10 +22,10 @@
 #include <piler.h>


-int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+int import_from_mailbox(char *mailbox, struct session_data *sdata, struct __data *data, struct __config *cfg){
   FILE *F, *f=NULL;
   int rc=ERR, tot_msgs=0, ret=OK;
-   char buf[MAXBUFSIZE];
+   char buf[MAXBUFSIZE], fname[SMALLBUFSIZE];
   time_t t;


@ -44,18 +44,18 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *
         if(f){
            fclose(f);
            f = NULL;
-            rc = import_message(sdata, data, counters, cfg);
+            rc = import_message(fname, sdata, data, cfg);
            if(rc == ERR){
-               printf("error importing: '%s'\n", data->import->filename);
+               printf("error importing: '%s'\n", fname);
               ret = ERR;
            }
-            else unlink(data->import->filename);
+            else unlink(fname);

-            if(data->quiet == 0){ printf("processed: %7d\r", data->import->tot_msgs); fflush(stdout); }
+            if(data->quiet == 0) printf("processed: %7d\r", tot_msgs); fflush(stdout);
         }

-         snprintf(data->import->filename, sizeof(data->import->filename)-1, "%ld-%d", t, data->import->tot_msgs);
-         f = fopen(data->import->filename, "w+");
+         snprintf(fname, sizeof(fname)-1, "%ld-%d", t, tot_msgs);
+         f = fopen(fname, "w+");
         continue;
      }

@ -64,14 +64,14 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *

   if(f){
      fclose(f);
-      rc = import_message(sdata, data, counters, cfg);
+      rc = import_message(fname, sdata, data, cfg);
      if(rc == ERR){
-         printf("ERROR: error importing: '%s'\n", data->import->filename);
+         printf("error importing: '%s'\n", fname);
         ret = ERR;
      }
-      else unlink(data->import->filename);
+      else unlink(fname);

-      if(data->quiet == 0){ printf("processed: %7d\r", data->import->tot_msgs); fflush(stdout); }
+      if(data->quiet == 0) printf("processed: %7d\r", tot_msgs); fflush(stdout);
   }

   fclose(F);
@ -80,7 +80,7 @@ int import_from_mailbox(char *mailbox, struct session_data *sdata, struct data *
 }


-int import_mbox_from_dir(char *directory, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
+int import_mbox_from_dir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg){
   DIR *dir;
   struct dirent *de;
   int rc=ERR, ret=OK, i=0;
@ -103,7 +103,7 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct dat
      if(stat(fname, &st) == 0){
         if(S_ISDIR(st.st_mode)){
            folder = data->folder;
-            rc = import_mbox_from_dir(fname, sdata, data, counters, cfg);
+            rc = import_mbox_from_dir(fname, sdata, data, tot_msgs, cfg);
            data->folder = folder;
            if(rc == ERR) ret = ERR;
         }
@ -111,9 +111,9 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct dat

            if(S_ISREG(st.st_mode)){
               if(i == 0 && data->recursive_folder_names == 1){
-                  folder = get_folder_id(sdata, fname, data->folder);
+                  folder = get_folder_id(sdata, data, fname, data->folder);
                  if(folder == ERR_FOLDER){
-                     folder = add_new_folder(sdata, fname, data->folder);
+                     folder = add_new_folder(sdata, data, fname, data->folder);

                     if(folder == ERR_FOLDER){
                        printf("error: cannot get/add folder '%s' to parent id: %d\n", fname, data->folder);
@ -126,8 +126,8 @@ int import_mbox_from_dir(char *directory, struct session_data *sdata, struct dat

               }

-               rc = import_from_mailbox(fname, sdata, data, counters, cfg);
-               if(rc == OK) (data->import->tot_msgs)++;
+               rc = import_from_mailbox(fname, sdata, data, cfg);
+               if(rc == OK) (*tot_msgs)++;
               else ret = ERR;

               i++;
--- a/src/import_maildir.c
+++ b/src/import_maildir.c
@ -22,12 +22,12 @@
 #include <piler.h>


-int import_from_maildir(struct session_data *sdata, struct data *data, char *directory, struct counters *counters, struct config *cfg){
+int import_from_maildir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg){
   DIR *dir;
   struct dirent *de;
   int rc=ERR, ret=OK, i=0;
   int folder;
-   char *p, subdir[SMALLBUFSIZE];
+   char *p, fname[SMALLBUFSIZE];
   struct stat st;

   dir = opendir(directory);
@ -40,13 +40,12 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir
   while((de = readdir(dir))){
      if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;

-      snprintf(data->import->filename, SMALLBUFSIZE-1, "%s/%s", directory, de->d_name);
+      snprintf(fname, sizeof(fname)-1, "%s/%s", directory, de->d_name);

-      if(stat(data->import->filename, &st) == 0){
+      if(stat(fname, &st) == 0){
         if(S_ISDIR(st.st_mode)){
            folder = data->folder;
-            snprintf(subdir, sizeof(subdir)-1, "%s", data->import->filename);
-            rc = import_from_maildir(sdata, data, subdir, counters, cfg);
+            rc = import_from_maildir(fname, sdata, data, tot_msgs, cfg);
            data->folder = folder;
            if(rc == ERR) ret = ERR;
         }
@ -57,13 +56,13 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir
                  p = strrchr(directory, '/');
                  if(p) p++;
                  else {
-                     printf("ERROR: invalid directory name: '%s'\n", directory);
+                     printf("invalid directory name: '%s'\n", directory);
                     return ERR;
                  }

-                  folder = get_folder_id(sdata, p, data->folder);
+                  folder = get_folder_id(sdata, data, p, data->folder);
                  if(folder == ERR_FOLDER){
-                     folder = add_new_folder(sdata, p, data->folder);
+                     folder = add_new_folder(sdata, data, p, data->folder);

                     if(folder == ERR_FOLDER){
                        printf("error: cannot get/add folder '%s' to parent id: %d\n", p, data->folder);
@ -76,36 +75,34 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir

               }

-               rc = import_message(sdata, data, counters, cfg);
+               rc = import_message(fname, sdata, data, cfg);

-               if(rc == OK) (data->import->tot_msgs)++;
+               if(rc == OK) (*tot_msgs)++;
               else if(rc == ERR){
-                  printf("ERROR: error importing: '%s'\n", data->import->filename);
+                  printf("error importing: '%s'\n", fname);
                  ret = ERR;
               }
-
-               if(data->import->remove_after_import == 1 && rc != ERR) unlink(data->import->filename);
+ 
+               if(data->import->remove_after_import == 1 && rc != ERR) unlink(fname);

               i++;

-               if(data->quiet == 0){ printf("processed: %7d\r", data->import->tot_msgs); fflush(stdout); }
+               if(data->quiet == 0) printf("processed: %7d\r", *tot_msgs); fflush(stdout);
            }
            else {
-               printf("%s is not a file\n", data->import->filename);
+               printf("%s is not a file\n", fname);
            }

         }
      }
      else {
-         printf("cannot stat() %s\n", data->import->filename);
+         printf("cannot stat() %s\n", fname);
      }

   }
   closedir(dir);

-   if(data->import->table_id > 0){
-      update_import_table(sdata, data);
-   }
-
   return ret;
 }
+
+
--- a/src/import_pilerexport.c
+++ b/src/import_pilerexport.c
@ -1,114 +0,0 @@
-/*
- * import_pop3.c
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/socket.h>
-#include <netdb.h>
-#include <arpa/inet.h>
-#include <netinet/in.h>
-#include <fcntl.h>
-#include <dirent.h>
-#include <unistd.h>
-#include <time.h>
-#include <locale.h>
-#include <getopt.h>
-#include <syslog.h>
-#include <piler.h>
-
-
-void import_the_file(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
-   close(data->import->fd);
-   data->import->fd = -1;
-
-   if(import_message(sdata, data, counters, cfg) != ERR){
-      unlink(data->import->filename);
-   }
-}
-
-
-void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
-
-   if(!strcmp(buf, PILEREXPORT_BEGIN_MARK)){
-      if((*count) > 0){
-         import_the_file(sdata, data, counters, cfg);
-      }
-
-      (*count)++;
-
-      snprintf(data->import->filename, SMALLBUFSIZE-1, "import-%llu", *count);
-
-      data->import->fd = open(data->import->filename, O_CREAT|O_WRONLY|O_TRUNC, S_IRUSR|S_IWUSR);
-      if(data->import->fd == -1){
-         // Do some error handling
-         printf("error: cannot open %s\n", data->import->filename);
-      }
-
-   } else {
-      if(write(data->import->fd, buf, buflen) != buflen){
-         printf("error: didnt write %d bytes\n", buflen);
-      }
-   }
-}
-
-
-void import_from_pilerexport(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
-   int n, rc, nullbyte, savedlen=0, puflen;
-   uint64 count=0;
-   char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE];
-
-   memset(savedbuf, 0, sizeof(savedbuf));
-
-   data->import->fd = -1;
-
-   do {
-      memset(buf, 0, sizeof(buf));
-      n = fread(buf, 1, sizeof(buf)-1, stdin);
-
-      int remaininglen = n;
-
-      if(savedlen > 0){
-         memset(copybuf, 0, sizeof(copybuf));
-
-         memcpy(copybuf, savedbuf, savedlen);
-         memcpy(&copybuf[savedlen], buf, n);
-
-         remaininglen += savedlen;
-
-         savedlen = 0;
-         memset(savedbuf, 0, sizeof(savedbuf));
-
-         p = &copybuf[0];
-      }
-      else {
-         p = &buf[0];
-      }
-
-      do {
-         puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf), &rc, &nullbyte);
-         p += puflen;
-         remaininglen -= puflen;
-
-         if(puflen > 0){
-            if(rc == OK){
-               process_buffer(puf, puflen, &count, sdata, data, counters, cfg);
-            }
-            else {
-               snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
-               savedlen = puflen;
-            }
-         }
-
-      } while(puflen > 0);
-
-   } while(n > 0);
-
-   if(data->import->fd != -1){
-      import_the_file(sdata, data, counters, cfg);
-   }
-}
--- a/src/import_pop3.c
+++ b/src/import_pop3.c
@ -22,46 +22,50 @@
 #include <piler.h>


-void import_from_pop3_server(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
-   int rc;
+int import_from_pop3_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, int dryrun, struct __config *cfg){
+   int rc, ret=OK, sd, use_ssl=0;
   char port_string[8];
   struct addrinfo hints, *res;

-   data->net->use_ssl = 0;
-
-   snprintf(port_string, sizeof(port_string)-1, "%d", data->import->port);
+   snprintf(port_string, sizeof(port_string)-1, "%d", port);

   memset(&hints, 0, sizeof(hints));
   hints.ai_family = AF_UNSPEC;
   hints.ai_socktype = SOCK_STREAM;

-   if((rc = getaddrinfo(data->import->server, port_string, &hints, &res)) != 0){
-      printf("getaddrinfo for '%s': %s\n", data->import->server, gai_strerror(rc));
-      return;
+   if((rc = getaddrinfo(server, port_string, &hints, &res)) != 0){
+      printf("getaddrinfo for '%s': %s\n", server, gai_strerror(rc));
+      return ERR;
   }

-   if(data->import->port == 995) data->net->use_ssl = 1;
+   if(port == 995) use_ssl = 1;

-   if((data->net->socket = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
+   if((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
      printf("cannot create socket\n");
+      ret = ERR;
      goto ENDE_POP3;
   }

-   if(connect(data->net->socket, res->ai_addr, res->ai_addrlen) == -1){
+   if(connect(sd, res->ai_addr, res->ai_addrlen) == -1){
      printf("connect()\n");
+      ret = ERR;
      goto ENDE_POP3;
   }


-   if(connect_to_pop3_server(data) == ERR){
-      close(data->net->socket);
+   if(connect_to_pop3_server(sd, username, password, data, use_ssl) == ERR){
+      close(sd);
+      ret = ERR;
      goto ENDE_POP3;
   }

-   process_pop3_emails(sdata, data, counters, cfg);
+   if(process_pop3_emails(sd, sdata, data, use_ssl, dryrun, cfg) == ERR) ret = ERR;

-   close_connection(data->net);
+   close_connection(sd, data, use_ssl);

 ENDE_POP3:
   freeaddrinfo(res);
+
+   return ret;
 }
+
--- a/src/list.c
+++ b/src/list.c
@ -0,0 +1,97 @@
+/*
+ * list.c, SJ
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include "list.h"
+#include "config.h"
+
+
+int append_list(struct list **list, char *p){
+   struct list *q, *t, *u=NULL;
+
+   q = *list;
+
+   while(q){
+      if(strcmp(q->s, p) == 0)
+         return 0;
+
+      u = q;
+      q = q->r;
+   }
+
+   t = create_list_item(p);
+   if(t){
+      if(*list == NULL)
+         *list = t;
+      else if(u)
+         u->r = t;
+
+      return 1;
+   }
+
+   return -1;
+}
+
+
+struct list *create_list_item(char *s){
+   struct list *h=NULL;
+
+   if((h = malloc(sizeof(struct list))) == NULL)
+      return NULL;
+
+   snprintf(h->s, SMALLBUFSIZE-1, "%s", s);
+   h->r = NULL;
+
+   return h;
+}
+
+
+int is_string_on_list(struct list *list, char *s){
+   struct list *p;
+
+   p = list;
+
+   while(p != NULL){
+      if(strcmp(p->s, s) == 0) return 1;
+      p = p->r;
+   }
+
+   return 0;
+}
+
+
+int is_item_on_string(struct list *list, char *s){
+   struct list *p;
+
+   p = list;
+
+   while(p != NULL){
+      if(strstr(s, p->s)) return 1;
+      p = p->r;
+   }
+
+   return 0;
+}
+
+
+void free_list(struct list *list){
+   struct list *p, *q;
+
+   p = list;
+
+   while(p != NULL){
+      q = p->r;
+
+      if(p)
+         free(p);
+
+      p = q;
+   }
+}
+
+
+
--- a/src/list.h
+++ b/src/list.h
@ -0,0 +1,17 @@
+/*
+ * list.h, SJ
+ */
+
+#ifndef _LIST_H
+ #define _LIST_H
+
+#include "defs.h"
+
+int append_list(struct list **list, char *p);
+struct list *create_list_item(char *s);
+int is_string_on_list(struct list *list, char *s);
+int is_item_on_string(struct list *list, char *s);
+void free_list(struct list *list);
+
+#endif /* _LIST_H */
+
--- a/src/memc.c
+++ b/src/memc.c
@ -59,7 +59,7 @@ void memcached_init(struct memcached_server *ptr, char *server_ip, int server_po


 int set_socket_options(struct memcached_server *ptr){
-   int error, flag=1, flags;
+   int error, flag=1, flags, rval;
   struct timeval waittime;
   struct linger linger;

@ -128,7 +128,8 @@ int set_socket_options(struct memcached_server *ptr){


   if((flags & O_NONBLOCK) == 0){
-      if(fcntl(ptr->fd, F_SETFL, flags | O_NONBLOCK) == -1) return MEMCACHED_FAILURE;
+      rval = fcntl(ptr->fd, F_SETFL, flags | O_NONBLOCK);
+      if(rval == -1) return MEMCACHED_FAILURE;
   }

   return MEMCACHED_SUCCESS;
@ -196,7 +197,7 @@ int memcached_connect(struct memcached_server *ptr){
        {
          (void)close(ptr->fd);
          ptr->fd= -1;
-
+          
        }

      return MEMCACHED_FAILURE;
@ -216,13 +217,12 @@ int memcached_shutdown(struct memcached_server *ptr){
 }


-int memcached_add(struct memcached_server *ptr, char *cmd, char *key, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry){
+int memcached_add(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry){
   int len=0;

   if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;

-   // cmd could be either 'add' or 'set'
-   snprintf(ptr->buf, MAXBUFSIZE-1, "%s %s %u %lu %u \r\n", cmd, key, flags, expiry, valuelen);
+   snprintf(ptr->buf, MAXBUFSIZE-1, "add %s %d %ld %d \r\n", key, flags, expiry, valuelen);
   len = strlen(ptr->buf);

   strncat(ptr->buf, value, MAXBUFSIZE-strlen(ptr->buf)-1);
@ -240,7 +240,30 @@ int memcached_add(struct memcached_server *ptr, char *cmd, char *key, char *valu
 }


-int memcached_increment(struct memcached_server *ptr, char *key, unsigned long long value, unsigned long long *result){
+int memcached_set(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry){
+   int len=0;
+
+   if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
+
+   snprintf(ptr->buf, MAXBUFSIZE-1, "set %s %d %ld %d \r\n", key, flags, expiry, valuelen);
+   len = strlen(ptr->buf);
+
+   strncat(ptr->buf, value, MAXBUFSIZE-strlen(ptr->buf)-1);
+   strncat(ptr->buf, "\r\n", MAXBUFSIZE-strlen(ptr->buf)-1);
+
+   len += valuelen + 2;
+
+   send(ptr->fd, ptr->buf, len, 0);
+
+   ptr->last_read_bytes = __recvtimeout(ptr->fd, ptr->buf, MAXBUFSIZE, ptr->rcv_timeout);
+
+   if(strcmp("STORED\r\n", ptr->buf)) return MEMCACHED_FAILURE;
+
+   return MEMCACHED_SUCCESS;
+}
+
+
+int memcached_increment(struct memcached_server *ptr, char *key, unsigned int keylen, unsigned long long value, unsigned long long *result){
   char *p;

   if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
@ -262,6 +285,52 @@ int memcached_increment(struct memcached_server *ptr, char *key, unsigned long l
 }


+char *memcached_get(struct memcached_server *ptr, char *key, unsigned int *len, unsigned int *flags){
+   char *p;
+
+   if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return NULL;
+
+   snprintf(ptr->buf, MAXBUFSIZE, "get %s \r\n", key);
+   send(ptr->fd, ptr->buf, strlen(ptr->buf), 0);
+
+   ptr->last_read_bytes = __recvtimeout(ptr->fd, ptr->buf, MAXBUFSIZE, ptr->rcv_timeout);
+
+   if(ptr->last_read_bytes <= 0){
+      memcached_shutdown(ptr);
+      return NULL;
+   }
+
+
+   if(ptr->last_read_bytes < 10) return NULL;
+
+   if(strncmp("VALUE ", ptr->buf, 6)) return NULL;
+
+   p = strchr(ptr->buf, '\r');
+   if(!p) return NULL;
+   *p = '\0';
+   ptr->result = p + 2;
+
+   p = strrchr(ptr->buf + 6, ' ');
+   if(!p) return NULL;
+   *len = atoi(p+1);
+   *p = '\0';
+
+
+   p = strrchr(ptr->buf + 6, ' ');
+   if(!p) return NULL;
+   *flags = atoi(p+1);
+   *p = '\0';
+
+ 
+   p = strchr(ptr->result, '\r');
+   if(!p) return NULL;
+
+   *p = '\0';
+
+   return ptr->result;
+}
+
+
 int memcached_mget(struct memcached_server *ptr, char *key){

   if(memcached_connect(ptr) != MEMCACHED_SUCCESS) return MEMCACHED_FAILURE;
@ -325,3 +394,5 @@ char *memcached_fetch_result(struct memcached_server *ptr, char *key, char *valu

   return p;
 }
+
+
--- a/src/memc.h
+++ b/src/memc.h
@ -9,8 +9,9 @@ void memcached_init(struct memcached_server *ptr, char *server_ip, int server_po
 int set_socket_options(struct memcached_server *ptr);
 int memcached_connect(struct memcached_server *ptr);
 int memcached_shutdown(struct memcached_server *ptr);
-int memcached_add(struct memcached_server *ptr, char *cmd, char *key, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry);
-int memcached_increment(struct memcached_server *ptr, char *key, unsigned long long value, unsigned long long *result);
+int memcached_add(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry);
+int memcached_set(struct memcached_server *ptr, char *key, unsigned int keylen, char *value, unsigned int valuelen, unsigned int flags, unsigned long expiry);
+int memcached_increment(struct memcached_server *ptr, char *key, unsigned int keylen, unsigned long long value, unsigned long long *result);
 char *memcached_get(struct memcached_server *ptr, char *key, unsigned int *len, unsigned int *flags);
 int memcached_mget(struct memcached_server *ptr, char *key);
 char *memcached_fetch_result(struct memcached_server *ptr, char *key, char *value, unsigned int *flags);
--- a/src/message.c
+++ b/src/message.c
@ -10,7 +10,6 @@
 #include <sys/mman.h>
 #include <sys/socket.h>
 #include <sys/stat.h>
-#include <errno.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <syslog.h>
@ -18,10 +17,9 @@
 #include <zlib.h>


-int store_index_data(struct session_data *sdata, struct parser_state *state, struct data *data, uint64 id, struct config *cfg){
+int store_index_data(struct session_data *sdata, struct parser_state *state, struct __data *data, uint64 id, struct __config *cfg){
   int rc=ERR;
-   char *subj, *sender=state->b_from, *sender_domain=state->b_from_domain;
-   struct sql sql;
+   char *subj;

   if(data->folder == 0){
      data->folder = get_folder_id_by_rule(data, state, sdata->tot_len, sdata->spam_message, cfg);
@ -30,142 +28,74 @@ int store_index_data(struct session_data *sdata, struct parser_state *state, str
   subj = state->b_subject;
   if(*subj == ' ') subj++;

+
+   if(prepare_sql_statement(sdata, &(data->stmt_insert_into_sphinx_table), SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc;
+
+
   fix_email_address_for_sphinx(state->b_from);
-   fix_email_address_for_sphinx(state->b_sender);
   fix_email_address_for_sphinx(state->b_to);
   fix_email_address_for_sphinx(state->b_from_domain);
-   fix_email_address_for_sphinx(state->b_sender_domain);
   fix_email_address_for_sphinx(state->b_to_domain);

-   if(state->b_sender_domain[0]){
-      sender = state->b_sender;
-      sender_domain = state->b_sender_domain;
-   }

-   if(cfg->rtindex){
-      // Manticore doesn't support prepared statements using sphinxQL
-      // so we have to go through a painful query assembly escaping
-      // the untrusted input
-      //
-      char a[4*MAXBUFSIZE+4*SMALLBUFSIZE];
-      char *query=NULL;
+   p_bind_init(data);

-      snprintf(a, sizeof(a)-1, "REPLACE INTO %s (id, arrived, sent, size, direction, folder, attachments, attachment_types, sender, rcpt, senderdomain, rcptdomain, subject, body) VALUES (%llu,%ld,%ld,%d,%d,%d,%d,'%s','", cfg->sphxdb, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, data->folder, state->n_attachments, sdata->attachments);
+   data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
+   data->sql[data->pos] = state->b_from; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = state->b_to; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = state->b_from_domain; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = state->b_to_domain; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = subj; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = state->b_body; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = (char *)&sdata->now; data->type[data->pos] = TYPE_LONG; data->pos++;
+   data->sql[data->pos] = (char *)&sdata->sent; data->type[data->pos] = TYPE_LONG; data->pos++;
+   data->sql[data->pos] = (char *)&sdata->tot_len; data->type[data->pos] = TYPE_LONG; data->pos++;
+   data->sql[data->pos] = (char *)&sdata->direction; data->type[data->pos] = TYPE_LONG; data->pos++;
+   data->sql[data->pos] = (char *)&data->folder; data->type[data->pos] = TYPE_LONG; data->pos++;
+   data->sql[data->pos] = (char *)&state->n_attachments; data->type[data->pos] = TYPE_LONG; data->pos++;
+   data->sql[data->pos] = sdata->attachments; data->type[data->pos] = TYPE_STRING; data->pos++;

-      int ret = append_string_to_buffer(&query, a);
+   if(p_exec_query(sdata, data->stmt_insert_into_sphinx_table, data) == OK) rc = OK;

-      unsigned long len = strlen(sender);
-      char *s = malloc(2*len+1);
-      mysql_real_escape_string(&(sdata->sphx), s, sender, len);
-      ret += append_string_to_buffer(&query, s);
-      free(s);
-      ret += append_string_to_buffer(&query, "','");
+   close_prepared_statement(data->stmt_insert_into_sphinx_table);

-      len = strlen(state->b_to);
-      s = malloc(2*len+1);
-      mysql_real_escape_string(&(sdata->sphx), s, state->b_to, len);
-      ret += append_string_to_buffer(&query, s);
-      free(s);
-      ret += append_string_to_buffer(&query, "','");
-
-      len = strlen(sender_domain);
-      s = malloc(2*len+1);
-      mysql_real_escape_string(&(sdata->sphx), s, sender_domain, len);
-      ret += append_string_to_buffer(&query, s);
-      free(s);
-      ret += append_string_to_buffer(&query, "','");
-
-      len = strlen(state->b_to_domain);
-      s = malloc(2*len+1);
-      mysql_real_escape_string(&(sdata->sphx), s, state->b_to_domain, len);
-      ret += append_string_to_buffer(&query, s);
-      free(s);
-      ret += append_string_to_buffer(&query, "','");
-
-      len = strlen(subj);
-      s = malloc(2*len+1);
-      mysql_real_escape_string(&(sdata->sphx), s, subj, len);
-      ret += append_string_to_buffer(&query, s);
-      free(s);
-      ret += append_string_to_buffer(&query, "','");
-
-      len = strlen(state->b_body);
-      s = malloc(2*len+1);
-      mysql_real_escape_string(&(sdata->sphx), s, state->b_body, len);
-      ret += append_string_to_buffer(&query, s);
-      free(s);
-      ret += append_string_to_buffer(&query, "')");
-
-      if(ret == 0 && mysql_real_query(&(sdata->sphx), query, strlen(query)) == OK) rc = OK;
-      else syslog(LOG_PRIORITY, "ERROR: %s failed to store index data for id=%llu, errno=%d, append ret=%d", sdata->ttmpfile, id, mysql_errno(&(sdata->sphx)), ret);
-
-      free(query);
-   }
-   else {
-      if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc;
-
-      p_bind_init(&sql);
-
-      sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
-      sql.sql[sql.pos] = sender; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-      sql.sql[sql.pos] = state->b_to; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-      sql.sql[sql.pos] = sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-      sql.sql[sql.pos] = state->b_to_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-      sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-      sql.sql[sql.pos] = state->b_body; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-      sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-      sql.sql[sql.pos] = (char *)&sdata->sent; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-      sql.sql[sql.pos] = (char *)&sdata->tot_len; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-      sql.sql[sql.pos] = (char *)&sdata->direction; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-      sql.sql[sql.pos] = (char *)&data->folder; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-      sql.sql[sql.pos] = (char *)&state->n_attachments; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-      sql.sql[sql.pos] = sdata->attachments; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-
-      if(p_exec_stmt(sdata, &sql) == OK) rc = OK;
-      else syslog(LOG_PRIORITY, "ERROR: %s failed to store index data for id=%llu, sql_errno=%d", sdata->ttmpfile, id, sdata->sql_errno);
-
-      close_prepared_statement(&sql);
-   }

   return rc;
 }


-uint64 get_metaid_by_messageid(struct session_data *sdata, char *message_id, char *piler_id){
+uint64 get_metaid_by_messageid(struct session_data *sdata, struct __data *data, char *message_id, char *piler_id){
   uint64 id=0;
-   struct sql sql;

-   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_META_ID_BY_MESSAGE_ID) == ERR) return id;
+   if(prepare_sql_statement(sdata, &(data->stmt_get_meta_id_by_message_id), SQL_PREPARED_STMT_GET_META_ID_BY_MESSAGE_ID) == ERR) return id;

-   p_bind_init(&sql);
-   sql.sql[sql.pos] = message_id; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   p_bind_init(data);
+   data->sql[data->pos] = message_id; data->type[data->pos] = TYPE_STRING; data->pos++;

-   if(p_exec_stmt(sdata, &sql) == OK){
+   if(p_exec_query(sdata, data->stmt_get_meta_id_by_message_id, data) == OK){

-      p_bind_init(&sql);
+      p_bind_init(data);
+      data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+      data->sql[data->pos] = piler_id; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = RND_STR_LEN; data->pos++;

-      sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-      sql.sql[sql.pos] = piler_id; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++;
+      p_store_results(data->stmt_get_meta_id_by_message_id, data);

-      p_store_results(&sql);
+      p_fetch_results(data->stmt_get_meta_id_by_message_id);

-      p_fetch_results(&sql);
-
-      p_free_results(&sql);
+      p_free_results(data->stmt_get_meta_id_by_message_id);
   }

-   close_prepared_statement(&sql);
+   mysql_stmt_close(data->stmt_get_meta_id_by_message_id);

   return id;
 }


-int store_recipients(struct session_data *sdata, char *to, uint64 id, struct config *cfg){
-   int rc=OK, n=0;
+int store_recipients(struct session_data *sdata, struct __data *data, char *to, uint64 id, struct __config *cfg){
+   int ret=OK, n=0;
   char *p, *q, puf[SMALLBUFSIZE];
-   struct sql sql;

-   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_RCPT_TABLE) == ERR) return ERR;
+   if(prepare_sql_statement(sdata, &(data->stmt_insert_into_rcpt_table), SQL_PREPARED_STMT_INSERT_INTO_RCPT_TABLE) == ERR) return ret;

   p = to;
   do {
@ -176,67 +106,79 @@ int store_recipients(struct session_data *sdata, char *to, uint64 id, struct con
      if(q && strlen(q) > 3 && does_it_seem_like_an_email_address(puf) == 1){
         q++;

-         p_bind_init(&sql);
+         p_bind_init(data);

-         sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
-         sql.sql[sql.pos] = &puf[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-         sql.sql[sql.pos] = q; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+         data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
+         data->sql[data->pos] = &puf[0]; data->type[data->pos] = TYPE_STRING; data->pos++;
+         data->sql[data->pos] = q; data->type[data->pos] = TYPE_STRING; data->pos++;

-         if(p_exec_stmt(sdata, &sql) == ERR){
-            if(sdata->sql_errno != ER_DUP_ENTRY){
-               syslog(LOG_PRIORITY, "ERROR: %s: failed to add '%s' for id=%llu to rcpt table, sql_errno=%d", sdata->ttmpfile, puf, id, sdata->sql_errno);
-               rc = ERR;
-            }
+
+         if(p_exec_query(sdata, data->stmt_insert_into_rcpt_table, data) == ERR){
+            if(sdata->sql_errno != ER_DUP_ENTRY) ret = ERR;
         }
         else n++;
      }

   } while(p);

-   close_prepared_statement(&sql);
+   mysql_stmt_close(data->stmt_insert_into_rcpt_table);

-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored %d recipients, rc=%d", sdata->ttmpfile, n, rc);
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: added %d recipients", sdata->ttmpfile, n);

-   return rc;
+   return ret;
 }


-int store_folder_id(struct session_data *sdata, struct data *data, uint64 id, struct config *cfg){
-   int rc=ERR;
-   struct sql sql;
+void remove_recipients(struct session_data *sdata, uint64 id){
+   char s[SMALLBUFSIZE];
+
+   snprintf(s, sizeof(s)-1, "DELETE FROM " SQL_RECIPIENT_TABLE " WHERE id=%llu", id);
+
+   p_query(sdata, s);
+}
+
+
+int store_folder_id(struct session_data *sdata, struct __data *data, uint64 id){
+   int rc = ERR;

   if(data->folder == ERR_FOLDER) return rc;

-   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE) == ERR) return rc;
+   if(prepare_sql_statement(sdata, &(data->stmt_insert_into_folder_message_table), SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE) == ERR) return rc;

-   p_bind_init(&sql);
+   p_bind_init(data);

-   sql.sql[sql.pos] = (char *)&data->folder; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
+   data->sql[data->pos] = (char *)&data->folder; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
+   data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->pos++;

-   if(p_exec_stmt(sdata, &sql) == OK) rc = OK;
-   close_prepared_statement(&sql);
-
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored folderdata, rc=%d", sdata->ttmpfile, rc);
+   if(p_exec_query(sdata, data->stmt_insert_into_folder_message_table, data) == OK) rc = OK;
+   close_prepared_statement(data->stmt_insert_into_folder_message_table);

   return rc;
 }


-int update_metadata_reference(struct session_data *sdata, struct parser_state *state, char *ref, struct config *cfg){
+void remove_folder_id(struct session_data *sdata, uint64 id){
+   char s[SMALLBUFSIZE];
+
+   snprintf(s, sizeof(s)-1, "DELETE FROM " SQL_FOLDER_MESSAGE_TABLE " WHERE id=%llu", id);
+
+   p_query(sdata, s);
+}
+
+
+int update_metadata_reference(struct session_data *sdata, struct parser_state *state, struct __data *data, char *ref, struct __config *cfg){
   int ret = ERR;
-   struct sql sql;

-   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE) == ERR) return ret;
+   if(prepare_sql_statement(sdata, &(data->stmt_update_metadata_reference), SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE) == ERR) return ret;

-   p_bind_init(&sql);
+   p_bind_init(data);

-   sql.sql[sql.pos] = ref; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = state->reference; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   data->sql[data->pos] = ref; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = state->reference; data->type[data->pos] = TYPE_STRING; data->pos++;

-   if(p_exec_stmt(sdata, &sql) == OK) ret = OK;
+   if(p_exec_query(sdata, data->stmt_update_metadata_reference, data) == OK) ret = OK;

-   close_prepared_statement(&sql);
+   mysql_stmt_close(data->stmt_update_metadata_reference);

   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: updated meta reference for '%s', rc=%d", sdata->ttmpfile, state->reference, ret);

@ -244,146 +186,90 @@ int update_metadata_reference(struct session_data *sdata, struct parser_state *s
 }


-int store_meta_data(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
-   int rc=ERR;
-   char *subj, *sender, *sender_domain, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1];
+int store_meta_data(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg){
+   int rc, ret=ERR, result;
+   char *subj, *p, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1];
   uint64 id=0;
-   struct sql sql;
+

   subj = state->b_subject;
   if(*subj == ' ') subj++;

-   if(state->b_sender_domain[0]){
-      sender = state->b_sender;
-      sender_domain = state->b_sender_domain;
-      get_first_email_address_from_string(state->b_sender, s2, sizeof(s2));
-   } else {
-      sender = state->b_from;
-      sender_domain = state->b_from_domain;
-      get_first_email_address_from_string(state->b_from, s2, sizeof(s2));
-   }
+   snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, state->b_from, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);

-   snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, sender, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
-
-   digest_string("sha256", s, &vcode[0]);
+   digest_string(s, &vcode[0]);

   memset(ref, 0, sizeof(ref));
   if(strlen(state->reference) > 10){
-      digest_string("sha256", state->reference, &ref[0]);
-      update_metadata_reference(sdata, state, &ref[0], cfg);
-   }
-   else if(state->reference[0] == 0){
-      // during import, the order of messages is often random
-      // check if this is a message which is already referenced
-      uint64 count=0;
-
-      digest_string("sha256", state->message_id, &ref[0]);
-      if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_METADATA_REFERENCE) != ERR){
-         p_bind_init(&sql);
-
-         sql.sql[sql.pos] = &ref[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-
-         if(p_exec_stmt(sdata, &sql) == OK){
-            p_bind_init(&sql);
-
-            sql.sql[sql.pos] = (char *)&count; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-            p_store_results(&sql);
-            p_fetch_results(&sql);
-            p_free_results(&sql);
-         }
-      }
-
-      close_prepared_statement(&sql);
-
-      // no reference yet
-      if(count <= 0)
-         ref[0] = 0;
+      digest_string(state->reference, &ref[0]);
+      update_metadata_reference(sdata, state, data, &ref[0], cfg);
   }


-   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR;
+   if(prepare_sql_statement(sdata, &(data->stmt_insert_into_meta_table), SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR;
+
+   memset(s2, 0, sizeof(s2));
+
+   p = state->b_from;
+   do {
+      memset(s2, 0, sizeof(s2));
+      p = split(p, ' ', s2, sizeof(s2)-1, &result);
+
+      if(s2[0] == '\0') continue;
+
+      if(does_it_seem_like_an_email_address(s2) == 1){ break; }
+   } while(p);
+

   if(strlen(state->b_to) < 5){
      snprintf(state->b_to, SMALLBUFSIZE-1, "undisclosed-recipients@no.domain");
   }


-   p_bind_init(&sql);
+   p_bind_init(data);

-   sql.sql[sql.pos] = &s2[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = (char *)&sdata->spam_message; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&sdata->sent; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&sdata->retained; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&sdata->tot_len; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&sdata->hdr_len; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&sdata->direction; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = (char *)&state->n_attachments; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
-   sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = state->message_id; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = &ref[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = sdata->digest; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = sdata->bodydigest; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = &vcode[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-
-   if(p_exec_stmt(sdata, &sql) == OK){
-      id = p_get_insert_id(&sql);
-
-      if(store_recipients(sdata, state->b_to, id, cfg) == OK){
-
-         if(store_index_data(sdata, state, data, id, cfg) == OK) rc = OK;
-
-         if(cfg->enable_folders == 1){
-            rc = store_folder_id(sdata, data, id, cfg);
-         }
-      }
-
-      // There were some sql errors, so we should rollback everything
-      if(rc == ERR){
-         rollback(sdata, state, id, cfg);
-      }
+   data->sql[data->pos] = &s2[0]; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = state->b_from_domain; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = subj; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = (char *)&sdata->spam_message; data->type[data->pos] = TYPE_LONG; data->pos++;
+   data->sql[data->pos] = (char *)&sdata->now; data->type[data->pos] = TYPE_LONG; data->pos++;
+   data->sql[data->pos] = (char *)&sdata->sent; data->type[data->pos] = TYPE_LONG; data->pos++;
+   data->sql[data->pos] = (char *)&sdata->retained; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
+   data->sql[data->pos] = (char *)&sdata->tot_len; data->type[data->pos] = TYPE_LONG; data->pos++;
+   data->sql[data->pos] = (char *)&sdata->hdr_len; data->type[data->pos] = TYPE_LONG; data->pos++;
+   data->sql[data->pos] = (char *)&sdata->direction; data->type[data->pos] = TYPE_LONG; data->pos++;
+   data->sql[data->pos] = (char *)&state->n_attachments; data->type[data->pos] = TYPE_LONG; data->pos++;
+   data->sql[data->pos] = sdata->ttmpfile; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = state->message_id; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = &ref[0]; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = sdata->digest; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = sdata->bodydigest; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = &vcode[0]; data->type[data->pos] = TYPE_STRING; data->pos++;

+   if(p_exec_query(sdata, data->stmt_insert_into_meta_table, data) == ERR){
+      ret = ERR_EXISTS;
   }
   else {
-      syslog(LOG_PRIORITY, "ERROR: %s storing metadata, sql_errno=%d", sdata->ttmpfile, sdata->sql_errno);
+      id = p_get_insert_id(data->stmt_insert_into_meta_table);
+      rc = store_recipients(sdata, data, state->b_to, id, cfg);
+
+      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored recipients, rc=%d", sdata->ttmpfile, rc);
+
+      rc = store_index_data(sdata, state, data, id, cfg);
+
+      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored indexdata, rc=%d", sdata->ttmpfile, rc);
+
+      if(cfg->enable_folders == 1){
+         rc = store_folder_id(sdata, data, id);
+         if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored folderdata, rc=%d", sdata->ttmpfile, rc);
+      }
+
+      ret = OK;
   }

-   close_prepared_statement(&sql);
+   close_prepared_statement(data->stmt_insert_into_meta_table);

-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored metadata, rc=%d",  sdata->ttmpfile, rc);
-
-   return rc;
-}
-
-
-void rollback(struct session_data *sdata, struct parser_state *state, uint64 id, struct config *cfg){
-   char buf[SMALLBUFSIZE];
-
-   snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE id=%llu", SQL_SPHINX_TABLE, id);
-   p_query(sdata, buf);
-   syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
-
-   snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE id=%llu", SQL_RECIPIENT_TABLE, id);
-   p_query(sdata, buf);
-   syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
-
-   snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE id=%llu", SQL_METADATA_TABLE, id);
-   p_query(sdata, buf);
-   syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
-
-   snprintf(buf, sizeof(buf)-2, "DELETE FROM %s WHERE piler_id='%s'", SQL_ATTACHMENT_TABLE, sdata->ttmpfile);
-   p_query(sdata, buf);
-   syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
-
-   if(cfg->enable_folders == 1){
-      snprintf(buf, sizeof(buf)-1, "DELETE FROM " SQL_FOLDER_MESSAGE_TABLE " WHERE id=%llu", id);
-      p_query(sdata, buf);
-      syslog(LOG_PRIORITY, "ERROR: %s: rollback sql stmt=%s", sdata->ttmpfile, buf);
-   }
-
-   remove_stored_message_files(sdata, state, cfg);
+   return ret;
 }


@ -394,20 +280,20 @@ void remove_stripped_attachments(struct parser_state *state){
 }


-int is_duplicated_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
-   int fd;
+int process_message(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg){
+   int rc, fd;
   char piler_id[SMALLBUFSIZE];

   /* discard if existing message_id */

-   sdata->duplicate_id = get_metaid_by_messageid(sdata, state->message_id, piler_id);
+   sdata->duplicate_id = get_metaid_by_messageid(sdata, data, state->message_id, piler_id);

   if(sdata->duplicate_id > 0){
      remove_stripped_attachments(state);

-      if(sdata->restored_copy == 0 && strlen(state->b_journal_to) > 0){
+      if(strlen(state->b_journal_to) > 0){
         if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to add journal rcpt (%s) to id=%llu for message-id: '%s'", sdata->ttmpfile, state->b_journal_to, sdata->duplicate_id, state->message_id);
-         store_recipients(sdata, state->b_journal_to, sdata->duplicate_id, cfg);
+         store_recipients(sdata, data, state->b_journal_to, sdata->duplicate_id, cfg);
      }

      return ERR_EXISTS;
@ -416,7 +302,7 @@ int is_duplicated_message(struct session_data *sdata, struct parser_state *state
   fd = open(state->message_id_hash, O_CREAT|O_EXCL, S_IRUSR|S_IWUSR);
   if(fd == -1){
      remove_stripped_attachments(state);
-      syslog(LOG_PRIORITY, "%s: touch %s FAILED (%s), error: %s", sdata->ttmpfile, state->message_id_hash, state->message_id, strerror(errno));
+      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: touch %s FAILED (%s)", sdata->ttmpfile, state->message_id_hash, state->message_id);
      return ERR_EXISTS;
   }
   close(fd);
@ -424,13 +310,12 @@ int is_duplicated_message(struct session_data *sdata, struct parser_state *state
   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: touch %s OK (%s)", sdata->ttmpfile, state->message_id_hash, state->message_id);


+
   if(cfg->mmap_dedup_test == 1 && data->dedup != MAP_FAILED && data->child_serial >= 0 && data->child_serial < MAXCHILDREN){

      if(strstr(data->dedup, state->message_id_hash)){
-         if(cfg->verbosity >= _LOG_DEBUG){
-            syslog(LOG_INFO, "%s: dedup string: %s", sdata->ttmpfile, data->dedup);
-            syslog(LOG_INFO, "%s: message-id-hash=%s, serial=%d", sdata->ttmpfile, state->message_id_hash, data->child_serial);
-         }
+         if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: dedup string: %s", sdata->ttmpfile, data->dedup);
+         if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: message-id-hash=%s, serial=%d", sdata->ttmpfile, state->message_id_hash, data->child_serial);

         remove_stripped_attachments(state);
         return ERR_EXISTS;
@ -439,26 +324,37 @@ int is_duplicated_message(struct session_data *sdata, struct parser_state *state
      memcpy(data->dedup + data->child_serial*DIGEST_LENGTH*2, state->message_id_hash, DIGEST_LENGTH*2);
   }

-   return OK;
-}


-int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){

-   if(is_duplicated_message(sdata, state, data, cfg) == ERR_EXISTS)
-      return ERR_EXISTS;
+   /* store base64 encoded file attachments */

-   sdata->retained += query_retain_period(data, state, sdata->tot_len, sdata->spam_message, cfg);
+   if(state->n_attachments > 0){
+      rc = store_attachments(sdata, state, data, cfg);
+
+      remove_stripped_attachments(state);
+
+      if(rc) return ERR;
+   }


-   if(state->n_attachments > 0 && store_attachments(sdata, state, cfg) == ERR) return ERR;
-
-
-   if(store_file(sdata, sdata->tmpframe, 0, cfg) == 0){
-      syslog(LOG_PRIORITY, "ERROR: %s: failed to store message: %s", sdata->ttmpfile, sdata->tmpframe);
+   rc = store_file(sdata, sdata->tmpframe, 0, cfg);
+   if(rc == 0){
+      syslog(LOG_PRIORITY, "%s: error storing message: %s", sdata->ttmpfile, sdata->tmpframe);
      return ERR;
   }


-   return store_meta_data(sdata, state, data, cfg);
+   sdata->retained += query_retain_period(data, state, sdata->tot_len, sdata->spam_message, cfg);
+
+   rc = store_meta_data(sdata, state, data, cfg);
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored metadata, rc=%d",  sdata->ttmpfile, rc);
+   if(rc == ERR_EXISTS){
+
+      remove_stored_message_files(sdata, state, cfg);
+
+      return ERR_EXISTS;
+   }
+
+   return OK;
 }
--- a/src/misc.c
+++ b/src/misc.c
@ -63,10 +63,6 @@ void get_extractor_list(){
      printf("%s ", HAVE_TNEF);
   #endif

-   #ifdef HAVE_ZIP
-      printf("libzip ");
-   #endif
-
   printf("\n\n");
 }

@ -77,7 +73,7 @@ void __fatal(char *s){
 }

 /*
- * calculate the difference between two timevals in [usec]
+ * calculate the difference betwwen two timevals in [usec]
 */

 long tvdiff(struct timeval a, struct timeval b){
@ -88,6 +84,81 @@ long tvdiff(struct timeval a, struct timeval b){
 }


+/*
+ * search something in a buffer
+ */
+
+int searchStringInBuffer(char *s, int len1, char *what, int len2){
+   int i, k, r;
+
+   for(i=0; i<len1; i++){
+      r = 0;
+
+      for(k=0; k<len2; k++){
+         if(*(s+i+k) == *(what+k))
+            r++;
+      }
+
+      if(r == len2)
+         return i;
+   }
+
+   return 0;
+}
+
+
+int search_char_backward(char *buf, int buflen, char c){
+   int n, m;
+
+   m = buflen - 1 - 5;
+   if(m < 0) m = 0;
+
+   for(n=m; n<buflen; n++){
+      if(*(buf + n) == c){
+         return n;
+      }
+   }
+   return -1;
+}
+
+
+/*
+ * count a character in buffer
+ */
+
+int countCharacterInBuffer(char *p, char c){
+   int i=0;
+
+   for(; *p; p++){
+      if(*p == c)
+         i++;
+   }
+
+   return i;
+}
+
+
+void replaceCharacterInBuffer(char *p, char from, char to){
+   int i, k=0;
+
+   for(i=0; i<strlen(p); i++){
+      if(p[i] == from){
+         if(to > 0){
+            p[k] = to;
+            k++;
+         }
+      }
+      else {
+         p[k] = p[i];
+         k++;
+      }
+
+   }
+
+   p[k] = '\0';
+}
+
+
 /*
 * split a string by a character as delimiter
 */
@ -143,8 +214,10 @@ char *split_str(char *row, char *what, char *s, int size){
      r += strlen(what);
   }

-   strncpy(s, row, len);
-   s[len] = '\0';
+   if(s != NULL){
+      strncpy(s, row, len);
+      s[len] = '\0';
+   }

   return r;
 }
@ -158,13 +231,13 @@ int trimBuffer(char *s){
   int n=0;
   char *p;

-   p = strchr(s, '\n');
+   p = strrchr(s, '\n');
   if(p){
      *p = '\0';
      n++;
   }

-   p = strchr(s, '\r');
+   p = strrchr(s, '\r');
   if(p){
      *p = '\0';
      n++;
@ -175,7 +248,8 @@ int trimBuffer(char *s){


 int extract_verp_address(char *email){
-   char *p1;
+   char *p, *p1, *p2;
+   char puf[SMALLBUFSIZE];

   // a VERP address is like archive+user=domain.com@myarchive.local

@ -185,14 +259,13 @@ int extract_verp_address(char *email){

   p1 = strchr(email, '+');
   if(p1){
-      char *p2 = strchr(p1, '@');
+      p2 = strchr(p1, '@');
      if(p2 && p2 > p1 + 2){
         if(strchr(p1+1, '=')){
-            char puf[SMALLBUFSIZE];
            memset(puf, 0, sizeof(puf));

            memcpy(&puf[0], p1+1, p2-p1-1);
-            char *p = strchr(puf, '=');
+            p = strchr(puf, '=');
            if(p) *p = '@';
            strcpy(email, puf);
         }
@ -229,15 +302,19 @@ int extractEmail(char *rawmail, char *email){
 * using the rand() function if not possible
 */

-void make_random_string(unsigned char *buf, int buflen){
-   const char alphanum[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
-   int i, fd, urandom=0, len = sizeof(alphanum)-1;
+void make_random_string(char *buf, int buflen){
+   int i, len, fd;
+   int urandom=0;
+   static char alphanum[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+   unsigned char s[QUEUE_ID_LEN];
+
+   len = strlen(alphanum);

   fd = open(RANDOM_POOL, O_RDONLY);
   if(fd != -1){
-      if(readFromEntropyPool(fd, buf, buflen) == buflen){
-         for(i=0; i<buflen; i++){
-            *(buf+i) = alphanum[*(buf+i) % len];
+      if(readFromEntropyPool(fd, s, sizeof(s)) == sizeof(s)){
+         for(i=0; i<QUEUE_ID_LEN; i++){
+            *(buf+i) = alphanum[s[i] % len];
         }

         urandom = 1;
@ -262,13 +339,11 @@ void create_id(char *id, unsigned char server_id){

   get_random_bytes(buf, RND_STR_LEN/2, server_id);

-   // New encryption scheme using AES-256
-   buf[0] = 0x50;
-
   for(i=0; i < RND_STR_LEN/2; i++){
      sprintf(id, "%02x", buf[i]);
      id += 2;
   }
+
 }


@ -286,7 +361,7 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
   taia_now(&now);
   taia_pack(nowpack, &now);

-   memcpy(buf, nowpack, 12); //-V512
+   memcpy(buf, nowpack, 12);

   fd = open(RANDOM_POOL, O_RDONLY);
   if(fd == -1) return ret;
@ -296,7 +371,7 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
   if(readFromEntropyPool(fd, buf+12+1, len-12-1) != len-12-1){
      syslog(LOG_PRIORITY, "%s: %s", ERR_CANNOT_READ_FROM_POOL, RANDOM_POOL);
   }
-
+   
   close(fd);
   return ret;
 }
@ -308,10 +383,10 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){

 int readFromEntropyPool(int fd, void *_s, ssize_t n){
   char *s = _s;
-   ssize_t pos = 0;
+   ssize_t res, pos = 0;

   while(n > pos){
-      ssize_t res = read(fd, s + pos, n - pos);
+      res = read(fd, s + pos, n - pos);
      switch(res){
         case  -1: continue;
         case   0: return res;
@ -347,13 +422,13 @@ int recvtimeout(int s, char *buf, int len, int timeout){
 }


-int write1(struct net *net, void *buf, int buflen){
+int write1(int sd, void *buf, int buflen, int use_ssl, SSL *ssl){
   int n;

-   if(net->use_ssl == 1)
-      n = SSL_write(net->ssl, buf, buflen);
+   if(use_ssl == 1)
+      n = SSL_write(ssl, buf, buflen);
   else
-      n = send(net->socket, buf, buflen, 0);
+      n = send(sd, buf, buflen, 0);

   return n;
 }
@ -369,28 +444,28 @@ int ssl_want_retry(SSL *ssl, int ret, int timeout){
   i = SSL_get_error(ssl, ret);
   if(i == SSL_ERROR_NONE)
      return 1;
-
+ 
   tv.tv_sec = timeout/1000;
   tv.tv_usec = 0;
   FD_ZERO(&fds);
-
+ 
   switch(i){
      case SSL_ERROR_WANT_READ: // pause until the socket is readable
          sock = SSL_get_rfd(ssl);
          FD_SET(sock, &fds);
          i = select(sock+1, &fds, 0, 0, &tv);
          break;
-
+ 
      case SSL_ERROR_WANT_WRITE: // pause until the socket is writeable
         sock = SSL_get_wfd(ssl);
         FD_SET(sock, &fds);
         i = select(sock+1, 0, &fds, 0, &tv);
         break;
-
+ 
      case SSL_ERROR_ZERO_RETURN: // the sock closed, just return quietly
         i = 0;
         break;
-
+ 
      default: // ERROR - unexpected error code
         i = -1;
         break;
@ -414,26 +489,26 @@ int ssl_read_timeout(SSL *ssl, void *buf, int len, int timeout){
 }


-int recvtimeoutssl(struct net *net, char *buf, int len){
+int recvtimeoutssl(int s, char *buf, int len, int timeout, int use_ssl, SSL *ssl){

    memset(buf, 0, len);

-    if(net->use_ssl == 1){
-       return ssl_read_timeout(net->ssl, buf, len-1, net->timeout);
+    if(use_ssl == 1){
+       return ssl_read_timeout(ssl, buf, len-1, timeout);
    }
    else {
-       return recvtimeout(net->socket, buf, len-1, net->timeout);
+       return recvtimeout(s, buf, len-1, timeout);
    }
 }


-void close_connection(struct net *net){
-   close(net->socket);
+void close_connection(int sd, struct __data *data, int use_ssl){
+   close(sd);

-   if(net->use_ssl == 1){
-      SSL_shutdown(net->ssl);
-      SSL_free(net->ssl);
-      SSL_CTX_free(net->ctx);
+   if(use_ssl == 1){
+      SSL_shutdown(data->ssl);
+      SSL_free(data->ssl);
+      SSL_CTX_free(data->ctx);
      ERR_free_strings();
   }
 }
@ -469,7 +544,7 @@ int drop_privileges(struct passwd *pwd){
 }


-void init_session_data(struct session_data *sdata, struct config *cfg){
+void init_session_data(struct session_data *sdata, struct __config *cfg){
   int i;


@ -564,6 +639,12 @@ void strtolower(char *s){
 }


+void *get_in_addr(struct sockaddr *sa){
+   if(sa->sa_family == AF_INET) return &(((struct sockaddr_in*)sa)->sin_addr);
+   return &(((struct sockaddr_in6*)sa)->sin6_addr);
+}
+
+
 int make_socket_non_blocking(int fd){
   int flags, s;

@ -652,89 +733,12 @@ void move_email(struct smtp_session *session){

   snprintf(buf, sizeof(buf)-1, "%d/%s", session->ttmpfile[0] % session->cfg->number_of_worker_processes, session->ttmpfile);

-   if(session->nullbyte){
-      snprintf(buf, sizeof(buf)-1, "%s/%s", ERROR_DIR, session->ttmpfile);
-      syslog(LOG_PRIORITY, "ERROR: %s contains an invalid NUL-byte, moving it to %s", session->ttmpfile, ERROR_DIR);
-   }
-
   if(rename(session->ttmpfile, buf)){
-      syslog(LOG_PRIORITY, "ERROR: couldn't rename %s to %s (reason: %s)", session->ttmpfile, buf, strerror(errno));
+      syslog(LOG_PRIORITY, "ERROR: couldn't rename %s to %s", session->ttmpfile, buf);
   }
 }


-int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte){
-   int i=0;
-
-   *rc = ERR;
-   *nullbyte = 0;
-
-   memset(buf, 0, buflen);
-
-   for(int j=0; j<slen; j++){
-      if(i<buflen-2){
-         if(*(s+j) == 0){
-            *nullbyte = 1;
-         }
-
-         buf[i] = *(s+j);
-         i++;
-
-         if(*(s+j) == c){
-            *rc = OK;
-            break;
-         }
-      }
-      else {
-         break;
-      }
-   }
-
-   return i;
-}
-
-
-int init_ssl_to_server(struct data *data){
-   int n;
-   X509* server_cert;
-   char *str;
-
-   SSL_library_init();
-   SSL_load_error_strings();
-
-#if OPENSSL_VERSION_NUMBER < 0x10100000L
-   data->net->ctx = SSL_CTX_new(TLSv1_2_client_method());
-#else
-   data->net->ctx = SSL_CTX_new(TLS_client_method());
-#endif
-   CHK_NULL(data->net->ctx, "internal SSL error");
-
-   data->net->ssl = SSL_new(data->net->ctx);
-   CHK_NULL(data->net->ssl, "internal ssl error");
-
-   SSL_set_fd(data->net->ssl, data->net->socket);
-   n = SSL_connect(data->net->ssl);
-   CHK_SSL(n, "internal ssl error");
-
-   //printf("Cipher: %s\n", SSL_get_cipher(data->net->ssl));
-
-   server_cert = SSL_get_peer_certificate(data->net->ssl);
-   CHK_NULL(server_cert, "server cert error");
-
-   str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0);
-   CHK_NULL(str, "error in server cert");
-   OPENSSL_free(str);
-
-   str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0);
-   CHK_NULL(str, "error in server cert");
-   OPENSSL_free(str);
-
-   X509_free(server_cert);
-
-   return OK;
-}
-
-
 #ifndef _GNU_SOURCE
 char *strcasestr(const char *s, const char *find){
   char c, sc;
@ -755,55 +759,3 @@ char *strcasestr(const char *s, const char *find){
   return((char*)s);
 }
 #endif
-
-
-int append_string_to_buffer(char **buffer, char *str){
-   int arglen;
-   char *s=NULL;
-
-   arglen = strlen(str);
-
-   if(!*buffer){
-      *buffer = malloc(arglen+1);
-      memset(*buffer, 0, arglen+1);
-      memcpy(*buffer, str, arglen);
-   }
-   else {
-      int len = strlen(*buffer);
-      s = realloc(*buffer, len + arglen+1);
-      if(!s) return 1;
-
-      *buffer = s;
-
-      memset(*buffer+len, 0, arglen+1);
-      memcpy(*buffer+len, str, arglen);
-   }
-
-   return 0;
-}
-
-
-int get_size_from_smtp_mail_from(char *s){
-   int size=0;
-   char *p;
-
-   p = strcasestr(s, "SIZE=");
-   if(p){
-      p += strlen("SIZE=");
-      char *q = p;
-      for(; *q; q++){
-         if(isspace(*q)) break;
-      }
-
-      // We extract max. 9 characters, which is just under 1GB
-      // and not overflowing an int variable
-      if(q - p <= 9){
-         char c = *q;
-         *q = '\0';
-         size = atoi(p);
-         *q = c;
-      }
-   }
-
-   return size;
-}
--- a/src/misc.h
+++ b/src/misc.h
@ -19,42 +19,40 @@ int get_build();
 void get_extractor_list();
 void __fatal(char *s);
 long tvdiff(struct timeval a, struct timeval b);
+int searchStringInBuffer(char *s, int len1, char *what, int len2);
+int search_char_backward(char *buf, int buflen, char c);
+int countCharacterInBuffer(char *p, char c);
+void replaceCharacterInBuffer(char *p, char from, char to);
 char *split(char *str, int ch, char *buf, int buflen, int *result);
 char *split_str(char *row, char *what, char *s, int size);
 int trimBuffer(char *s);
 int extractEmail(char *rawmail, char *email);
-int extract_verp_address(char *email);
-void make_random_string(unsigned char *buf, int buflen);
+void make_random_string(char *buf, int buflen);
 void create_id(char *id, unsigned char server_id);
 int get_random_bytes(unsigned char *buf, int len, unsigned char server_id);
 int readFromEntropyPool(int fd, void *_s, ssize_t n);
 int recvtimeout(int s, char *buf, int len, int timeout);
-int write1(struct net *net, void *buf, int buflen);
-int recvtimeoutssl(struct net *net, char *buf, int len);
-void close_connection(struct net *net);
+int write1(int sd, void *buf, int buflen, int use_ssl, SSL *ssl);
+int recvtimeoutssl(int s, char *buf, int len, int timeout, int use_ssl, SSL *ssl);
+void close_connection(int sd, struct __data *data, int use_ssl);

 void write_pid_file(char *pidfile);
 int drop_privileges(struct passwd *pwd);

-void init_session_data(struct session_data *sdata, struct config *cfg);
+void init_session_data(struct session_data *sdata, struct __config *cfg);
 int read_from_stdin(struct session_data *sdata);
 void strtolower(char *s);

+void *get_in_addr(struct sockaddr *sa);
 int make_socket_non_blocking(int fd);
 int create_and_bind(char *listen_addr, int listen_port);

 int can_i_write_directory(char *dir);

 void move_email(struct smtp_session *session);
-int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte);
-
-int init_ssl_to_server(struct data *data);

 #ifndef _GNU_SOURCE
   char *strcasestr(const char *s, const char *find);
 #endif

-int append_string_to_buffer(char **buffer, char *str);
-int get_size_from_smtp_mail_from(char *s);
-
 #endif /* _MISC_H */
--- a/src/mydomains.c
+++ b/src/mydomains.c
@ -10,41 +10,43 @@
 #include <piler.h>


-void load_mydomains(struct session_data *sdata, struct data *data, struct config *cfg){
+void load_mydomains(struct session_data *sdata, struct __data *data, struct __config *cfg){
+   int rc;
   char s[SMALLBUFSIZE];
-   struct sql sql;

   memset(s, 0, sizeof(s));


-   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_DOMAINS) == ERR) return;
+   if(prepare_sql_statement(sdata, &(data->stmt_generic), SQL_PREPARED_STMT_GET_DOMAINS) == ERR) return;


-   p_bind_init(&sql);
+   p_bind_init(data);

-   if(p_exec_stmt(sdata, &sql) == OK){
+   if(p_exec_query(sdata, data->stmt_generic, data) == OK){

-      p_bind_init(&sql);
+      p_bind_init(data);

-      sql.sql[sql.pos] = &s[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(s)-2; sql.pos++;
+      data->sql[data->pos] = &s[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s)-2; data->pos++;

-      p_store_results(&sql);
+      p_store_results(data->stmt_generic, data);

-      while(p_fetch_results(&sql) == OK){
-         if(addnode(data->mydomains, s) == 0) syslog(LOG_PRIORITY, "failed to append mydomain: '%s'", s);
+      while(p_fetch_results(data->stmt_generic) == OK){
+         rc = addnode(data->mydomains, s);
+
+         if(rc == 0) syslog(LOG_PRIORITY, "failed to append mydomain: '%s'", s);
         else if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "added mydomain: '%s'", s);

         memset(s, 0, sizeof(s));
      }

-      p_free_results(&sql);
+      p_free_results(data->stmt_generic);
   }

-   close_prepared_statement(&sql);
+   close_prepared_statement(data->stmt_generic);
 }


-int is_email_address_on_my_domains(char *email, struct data *data){
+int is_email_address_on_my_domains(char *email, struct __data *data){
   int rc=0;
   char *q, *s;

@ -63,3 +65,5 @@ int is_email_address_on_my_domains(char *email, struct data *data){

   return rc;
 }
+
+
--- a/src/mysql.c
+++ b/src/mysql.c
@ -9,7 +9,7 @@
 #include <piler.h>


-int open_database(struct session_data *sdata, struct config *cfg){
+int open_database(struct session_data *sdata, struct __config *cfg){
   int rc=1;
   char buf[BUFLEN];

@ -19,7 +19,7 @@ int open_database(struct session_data *sdata, struct config *cfg){
   mysql_options(&(sdata->mysql), MYSQL_OPT_RECONNECT, (const char*)&rc);

   if(mysql_real_connect(&(sdata->mysql), cfg->mysqlhost, cfg->mysqluser, cfg->mysqlpwd, cfg->mysqldb, cfg->mysqlport, cfg->mysqlsocket, 0) == 0){
-      syslog(LOG_PRIORITY, "cant connect to mysql server");
+      printf("cant connect to mysql server\n");
      return ERR;
   }

@ -33,49 +33,20 @@ int open_database(struct session_data *sdata, struct config *cfg){
 }


-int open_sphx(struct session_data *sdata, struct config *cfg){
-   int rc=1;
-   char buf[BUFLEN];
-
-   mysql_init(&(sdata->sphx));
-
-   mysql_options(&(sdata->sphx), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg->mysql_connect_timeout);
-   mysql_options(&(sdata->sphx), MYSQL_OPT_RECONNECT, (const char*)&rc);
-
-   if(mysql_real_connect(&(sdata->sphx), cfg->sphxhost, "", "", cfg->sphxdb, cfg->sphxport, "", 0) == 0){
-      syslog(LOG_PRIORITY, "cant connect to %s:%d", cfg->sphxhost, cfg->sphxport);
-      return ERR;
-   }
-
-   snprintf(buf, sizeof(buf)-2, "SET NAMES %s", cfg->mysqlcharset);
-   mysql_real_query(&(sdata->sphx), buf, strlen(buf));
-
-   snprintf(buf, sizeof(buf)-2, "SET CHARACTER SET %s", cfg->mysqlcharset);
-   mysql_real_query(&(sdata->sphx), buf, strlen(buf));
-
-   return OK;
-}
-
-
 void close_database(struct session_data *sdata){
   mysql_close(&(sdata->mysql));
 }


-void close_sphx(struct session_data *sdata){
-   mysql_close(&(sdata->sphx));
-}
-
-
-void p_bind_init(struct sql *sql){
+void p_bind_init(struct __data *data){
   int i;

-   sql->pos = 0;
+   data->pos = 0;

   for(i=0; i<MAX_SQL_VARS; i++){
-      sql->sql[i] = NULL;
-      sql->type[i] = TYPE_UNDEF;
-      sql->len[i] = 0;
+      data->sql[i] = NULL;
+      data->type[i] = TYPE_UNDEF;
+      data->len[i] = 0;
   }
 }

@ -85,7 +56,7 @@ void p_query(struct session_data *sdata, char *s){
 }


-int p_exec_stmt(struct session_data *sdata, struct sql *sql){
+int p_exec_query(struct session_data *sdata, MYSQL_STMT *stmt, struct __data *data){
   MYSQL_BIND bind[MAX_SQL_VARS];
   unsigned long length[MAX_SQL_VARS];
   int i, ret=ERR;
@ -94,10 +65,10 @@ int p_exec_stmt(struct session_data *sdata, struct sql *sql){
   memset(bind, 0, sizeof(bind));

   for(i=0; i<MAX_SQL_VARS; i++){
-      if(sql->type[i] > TYPE_UNDEF){
+      if(data->type[i] > TYPE_UNDEF){


-         switch(sql->type[i]) {
+         switch(data->type[i]) {
             case TYPE_SHORT:
                                  bind[i].buffer_type = MYSQL_TYPE_SHORT;
                                  bind[i].length = 0;
@ -117,7 +88,7 @@ int p_exec_stmt(struct session_data *sdata, struct sql *sql){

             case TYPE_STRING:
                                  bind[i].buffer_type = MYSQL_TYPE_STRING;
-                                  length[i] = strlen(sql->sql[i]);
+                                  length[i] = strlen(data->sql[i]);
                                  bind[i].length = &length[i];
                                  break;

@ -130,22 +101,23 @@ int p_exec_stmt(struct session_data *sdata, struct sql *sql){
         };


-         bind[i].buffer = sql->sql[i];
+         bind[i].buffer = data->sql[i];
         bind[i].is_null = 0;

+         
      }
      else { break; }
   }

-   if(mysql_stmt_bind_param(sql->stmt, bind)){
-      sdata->sql_errno = mysql_stmt_errno(sql->stmt);
-      syslog(LOG_PRIORITY, "ERROR: %s: mysql_stmt_bind_param() '%s' (errno: %d)", sdata->ttmpfile, mysql_stmt_error(sql->stmt), sdata->sql_errno);
+   if(mysql_stmt_bind_param(stmt, bind)){
+      sdata->sql_errno = mysql_stmt_errno(stmt);
+      syslog(LOG_PRIORITY, "%s: mysql_stmt_bind_param() error: %s (errno: %d)", sdata->ttmpfile, mysql_stmt_error(stmt), sdata->sql_errno);
      return ret;
   }

-   if(mysql_stmt_execute(sql->stmt)){
-      sdata->sql_errno = mysql_stmt_errno(sql->stmt);
-      syslog(LOG_PRIORITY, "ERROR: %s: mysql_stmt_execute() '%s' (errno: %d)", sdata->ttmpfile, mysql_error(&(sdata->mysql)), sdata->sql_errno);
+   if(mysql_stmt_execute(stmt)){
+      sdata->sql_errno = mysql_stmt_errno(stmt);
+      syslog(LOG_PRIORITY, "%s: mysql_stmt_execute error: *%s* (errno: %d)", sdata->ttmpfile, mysql_error(&(sdata->mysql)), sdata->sql_errno);
      return ret;
   }

@ -155,16 +127,16 @@ int p_exec_stmt(struct session_data *sdata, struct sql *sql){
 }


-int p_store_results(struct sql *sql){
+int p_store_results(MYSQL_STMT *stmt, struct __data *data){
   MYSQL_BIND bind[MAX_SQL_VARS];
   int i, ret=ERR;

   memset(bind, 0, sizeof(bind));

   for(i=0; i<MAX_SQL_VARS; i++){
-      if(sql->type[i] > TYPE_UNDEF){
+      if(data->type[i] > TYPE_UNDEF){

-         switch(sql->type[i]) {
+         switch(data->type[i]) {
             case TYPE_SHORT:     bind[i].buffer_type = MYSQL_TYPE_SHORT;
                                  break;

@ -180,7 +152,7 @@ int p_store_results(struct sql *sql){

             case TYPE_STRING:
                                  bind[i].buffer_type = MYSQL_TYPE_STRING;
-                                  bind[i].buffer_length = sql->len[i];
+                                  bind[i].buffer_length = data->len[i];
                                  break;

             default:
@ -190,21 +162,21 @@ int p_store_results(struct sql *sql){
         };


-         bind[i].buffer = (char *)sql->sql[i];
-         bind[i].is_null = &(sql->is_null[i]);
-         bind[i].length = &(sql->length[i]);
-         bind[i].error = &(sql->error[i]);
+         bind[i].buffer = (char *)data->sql[i];
+         bind[i].is_null = &(data->is_null[i]);
+         bind[i].length = &(data->length[i]);
+         bind[i].error = &(data->error[i]);

      }
      else { break; }
   }

-   if(mysql_stmt_bind_result(sql->stmt, bind)){
+   if(mysql_stmt_bind_result(stmt, bind)){
      return ret;
   }


-   if(mysql_stmt_store_result(sql->stmt)){
+   if(mysql_stmt_store_result(stmt)){
      return ret;
   }

@ -214,48 +186,43 @@ int p_store_results(struct sql *sql){
 }


-int p_fetch_results(struct sql *sql){
+int p_fetch_results(MYSQL_STMT *stmt){

-   if(mysql_stmt_fetch(sql->stmt) == 0) return OK;
+   if(mysql_stmt_fetch(stmt) == 0) return OK;

   return ERR;
 }


-void p_free_results(struct sql *sql){
-   mysql_stmt_free_result(sql->stmt);
+void p_free_results(MYSQL_STMT *stmt){
+   mysql_stmt_free_result(stmt);
 }


-uint64 p_get_insert_id(struct sql *sql){
-   return mysql_stmt_insert_id(sql->stmt);
+uint64 p_get_insert_id(MYSQL_STMT *stmt){
+   return mysql_stmt_insert_id(stmt);
 }


-int p_get_affected_rows(struct sql *sql){
-   return mysql_stmt_affected_rows(sql->stmt);
+int p_get_affected_rows(MYSQL_STMT *stmt){
+   return mysql_stmt_affected_rows(stmt);
 }


-int prepare_sql_statement(struct session_data *sdata, struct sql *sql, char *s){
+int prepare_sql_statement(struct session_data *sdata, MYSQL_STMT **stmt, char *s){

-   sql->stmt = mysql_stmt_init(&(sdata->mysql));
-   if(!(sql->stmt)){
-      syslog(LOG_PRIORITY, "%s: error: mysql_stmt_init()", sdata->ttmpfile);
+   *stmt = mysql_stmt_init(&(sdata->mysql));
+   if(!*stmt){
+      syslog(LOG_PRIORITY, "%s: mysql_stmt_init() error", sdata->ttmpfile);
      return ERR;
   }

-   if(mysql_stmt_prepare(sql->stmt, s, strlen(s))){
-      syslog(LOG_PRIORITY, "%s: error: mysql_stmt_prepare() %s => sql: %s", sdata->ttmpfile, mysql_stmt_error(sql->stmt), s);
+   if(mysql_stmt_prepare(*stmt, s, strlen(s))){
+      syslog(LOG_PRIORITY, "%s: mysql_stmt_prepare() error: %s => sql: %s", sdata->ttmpfile, mysql_stmt_error(*stmt), s);
      return ERR;
   }

-   return OK;
-}
-
-
-void close_prepared_statement(struct sql *sql){
-   if(sql->stmt) mysql_stmt_close(sql->stmt);
+   return OK; 
 }


@ -267,3 +234,10 @@ void insert_offset(struct session_data *sdata, int server_id){

   mysql_real_query(&(sdata->mysql), s, strlen(s));
 }
+
+
+void close_prepared_statement(MYSQL_STMT *stmt){
+   if(stmt) mysql_stmt_close(stmt);
+}
+
+
--- a/src/parser.c
+++ b/src/parser.c
@ -16,9 +16,11 @@
 #include <piler.h>


-struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct data *data, struct config *cfg){
+struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct __data *data, struct __config *cfg){
   FILE *f;
-   char buf[MAXBUFSIZE];
+   int i;
+   unsigned int len;
+   char *p, buf[MAXBUFSIZE], puf[SMALLBUFSIZE];
   char writebuffer[MAXBUFSIZE], abuffer[MAXBUFSIZE];
   struct parser_state state;

@ -30,6 +32,44 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec
      return state;
   }

+
+   if(sdata->num_of_rcpt_to > 0 && cfg->process_rcpt_to_addresses == 1){
+      for(i=0; i<sdata->num_of_rcpt_to; i++){
+
+         snprintf(puf, sizeof(puf)-1, "%s ", sdata->rcptto[i]);
+
+         if(does_it_seem_like_an_email_address(puf) == 1){
+            p = strstr(puf, cfg->hostid);
+            if(!p){
+
+               strtolower(puf);
+               len = strlen(puf);
+
+               if(state.tolen < MAXBUFSIZE-len-1){
+
+                  if(findnode(state.rcpt, puf) == NULL){
+                     addnode(state.journal_recipient, puf);
+                     addnode(state.rcpt, puf);
+
+                     memcpy(&(state.b_journal_to[state.journaltolen]), puf, len);
+                     state.journaltolen += len;
+
+                     memcpy(&(state.b_to[state.tolen]), puf, len);
+                     state.tolen += len;
+
+                     if(state.tolen < MAXBUFSIZE-len-1){
+                        split_email_address(puf);
+                        memcpy(&(state.b_to[state.tolen]), puf, len);
+                        state.tolen += len;
+                     }
+                  }
+               }
+            }
+         }
+
+      }
+   }
+
   if(take_into_pieces == 1){
      state.mfd = open(sdata->tmpframe, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR);
      if(state.mfd == -1){
@ -44,69 +84,31 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec
   }

   if(take_into_pieces == 1 && state.writebufpos > 0){
-      if(write(state.mfd, writebuffer, state.writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: %s: write(), %s, %d, %s", sdata->ttmpfile, __func__, __LINE__, __FILE__);
-      memset(writebuffer, 0, sizeof(writebuffer)); //-V597
+      write(state.mfd, writebuffer, state.writebufpos);
+      memset(writebuffer, 0, sizeof(writebuffer));
      state.writebufpos = 0;
   }

   if(take_into_pieces == 1){
      close(state.mfd); state.mfd = 0;
-
-      if(state.has_to_dump_whole_body == 1){
-         if(state.abufpos > 0){
-            flush_attachment_buffer(&state, &abuffer[0], sizeof(abuffer));
-         }
-         if(state.fd != -1) close(state.fd);
-         if(state.b64fd != -1) close(state.b64fd);
-      }
-
   }

   fclose(f);

-   if(data->import && data->import->extra_recipient){
-      char tmpbuf[SMALLBUFSIZE];
-      snprintf(tmpbuf, sizeof(tmpbuf)-1, "%s", data->import->extra_recipient);
-      add_recipient(tmpbuf, strlen(tmpbuf), sdata, &state, data, cfg);
-   }
-
-   // If both Sender: and From: headers exist, and they are different, then append
-   // the From: address to recipients list to give him access to this email as well
-
-   if(state.b_sender_domain[0]){
-      char frombuf[SMALLBUFSIZE];
-      char senderbuf[SMALLBUFSIZE];
-      get_first_email_address_from_string(state.b_from, frombuf, sizeof(frombuf));
-      get_first_email_address_from_string(state.b_sender, senderbuf, sizeof(senderbuf));
-      if(strcmp(frombuf, senderbuf)){
-         frombuf[strlen(frombuf)] = ' ';
-         add_recipient(frombuf, strlen(frombuf), sdata, &state, data, cfg);
-      }
-   }
-
   return state;
 }


-void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg){
-   int i;
+void post_parse(struct session_data *sdata, struct parser_state *state, struct __config *cfg){
+   int i, rec=0;
+   unsigned int len;
+   char *p;

   clearhash(state->boundaries);
   clearhash(state->rcpt);
   clearhash(state->rcpt_domain);
   clearhash(state->journal_recipient);

-   // Fix From: and Sender: lines if they are too long
-   if(strlen(state->b_from) > 255) state->b_from[255] = '\0';
-   if(strlen(state->b_from_domain) > 255) state->b_from_domain[255] = '\0';
-
-   if(strlen(state->b_sender) > 255) state->b_sender[255] = '\0';
-   if(strlen(state->b_sender_domain) > 255) state->b_sender_domain[255] = '\0';
-
-   // Truncate the message_id if it's >255 characters
-   if(strlen(state->message_id) > 255) state->message_id[255] = '\0';
-
-   fixupEncodedHeaderLine(state->b_subject, MAXBUFSIZE);
   trimBuffer(state->b_subject);

   if(sdata->internal_sender == 0) sdata->direction = DIRECTION_INCOMING;
@ -116,50 +118,20 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c
      if(sdata->internal_recipient == 1 && sdata->external_recipient == 1) sdata->direction = DIRECTION_INTERNAL_AND_OUTGOING;
   }

-   char *q = strrchr(state->receivedbuf, ';');
-   // I've seen some odd 1st Received: lines where the date
-   // wasn't the last item after a semicolon, eg.
-   //
-   // Received: from some.server.local [172.16.102.18]
-   //   by some.server.com with POP3 (Message Archiver)
-   //   for <nobody@127.0.0.1> (single-drop); Fri, 22 Nov 2019 11:24:49 -0500 (EST)
-   //   smtp.auth=3Daaaaaaaa@yourdomain.com|Received:=20by=20yourdomain
-   //   .com=20with=20ESMTPA=20id=20md50124862414.msg=3B=20Fri,=2022=20
-   //   ...
-   //
-   // Such line messes with the parse_date_header() function
-   // so the workaround is to discard such Received: line
-   if(q && strlen(q+1) < 45){
-      time_t received_timestamp = parse_date_header(q+1);
-      if(received_timestamp > 10000000){
-         // If the calculated date based on Date: header line differs more than 1 week
-         // then we'll override it with the data parsed from the first Received: line
-         if(labs(received_timestamp - sdata->sent) > 604800) sdata->sent = received_timestamp;
-      }
-   }

   for(i=1; i<=state->n_attachments; i++){
-      char puf[SMALLBUFSIZE];
-      snprintf(puf, sizeof(puf)-1, "%s ", state->attachments[i].filename);
-
-      unsigned int len = strlen(puf);
-      if(state->bodylen < BIGBUFSIZE-len-1){
-         memcpy(&(state->b_body[state->bodylen]), puf, len);
-         state->bodylen += len;
-      }
-
      digest_file(state->attachments[i].internalname, &(state->attachments[i].digest[0]));

-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s, dumped: %d", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest, state->attachments[i].dumped);
+      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest);

-      char *p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type);
+      p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type);
      len = strlen(p);
      if(strlen(sdata->attachments) < SMALLBUFSIZE-len-1 && !strstr(sdata->attachments, p)) memcpy(&(sdata->attachments[strlen(sdata->attachments)]), p, len);

      if(state->attachments[i].dumped == 1){
-         int rec = 0;
+         rec = 0;
         if(cfg->extract_attachments == 1 && state->bodylen < BIGBUFSIZE-1024) extract_attachment_content(sdata, state, state->attachments[i].aname, get_attachment_extractor_by_filename(state->attachments[i].filename), &rec, cfg);
-
+      
         unlink(state->attachments[i].aname);
      }

@ -173,7 +145,7 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c
   }


-   digest_string("sha256", state->message_id, &(state->message_id_hash[0]));
+   digest_string(state->message_id, &(state->message_id_hash[0]));

   if(sdata->sent == 0) sdata->sent = sdata->now;
 }
@ -190,7 +162,7 @@ void storno_attachment(struct parser_state *state){
   memset(state->attachments[state->n_attachments].type, 0, TINYBUFSIZE);
   memset(state->attachments[state->n_attachments].shorttype, 0, TINYBUFSIZE);
   memset(state->attachments[state->n_attachments].aname, 0, TINYBUFSIZE);
-   memset(state->attachments[state->n_attachments].filename, 0, SMALLBUFSIZE);
+   memset(state->attachments[state->n_attachments].filename, 0, TINYBUFSIZE);
   memset(state->attachments[state->n_attachments].internalname, 0, TINYBUFSIZE);
   memset(state->attachments[state->n_attachments].digest, 0, 2*DIGEST_LENGTH+1);

@ -199,29 +171,11 @@ void storno_attachment(struct parser_state *state){
 }


-void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned int abuffersize){
-   if(write(state->fd, abuffer, state->abufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
-
-   if(state->b64fd != -1){
-      abuffer[state->abufpos] = '\0';
-      if(state->base64 == 1){
-         unsigned char b64buffer[MAXBUFSIZE];
-         int n64 = base64_decode_attachment_buffer(abuffer, &b64buffer[0], sizeof(b64buffer));
-         if(write(state->b64fd, b64buffer, n64) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
-      }
-      else if(write(state->b64fd, abuffer, state->abufpos) == -1){
-         syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
-      }
-   }
-
-   state->abufpos = 0;
-   memset(abuffer, 0, abuffersize);
-}
-
-
-int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg){
-   char *p;
-   int boundary_line=0;
+int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, int writebuffersize, char *abuffer, int abuffersize, struct __data *data, struct __config *cfg){
+   char *p, *q, puf[SMALLBUFSIZE];
+   unsigned char b64buffer[MAXBUFSIZE];
+   char tmpbuf[MAXBUFSIZE];
+   int n64, writelen, boundary_line=0, result;
   unsigned int len;

   if(cfg->debug == 1) printf("line: %s", buf);
@ -239,24 +193,11 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
         sdata->restored_copy = 1;
      }

-      if(cfg->security_header[0] && state->found_security_header == 0 && strstr(buf, cfg->security_header)){
-         state->found_security_header = 1;
-      }
-
      if(*(cfg->piler_header_field) != 0 && strncmp(buf, cfg->piler_header_field, strlen(cfg->piler_header_field)) == 0){
         sdata->restored_copy = 1;
      }

-      if(sdata->ms_journal == 0 && ( strncmp(buf, "X-MS-Journal-Report:", strlen("X-MS-Journal-Report:")) == 0 || strncmp(buf, "X-WM-Journal-Report: journal", strlen("X-WM-Journal-Report: journal")) == 0) ){
-
-         memset(state->b_to, 0, MAXBUFSIZE);
-         state->tolen = 0;
-         memset(state->b_to_domain, 0, SMALLBUFSIZE);
-         state->todomainlen = 0;
-
-         clearhash(state->rcpt);
-         clearhash(state->rcpt_domain);
-
+      if(sdata->ms_journal == 0 && strncmp(buf, "X-MS-Journal-Report:", strlen("X-MS-Journal-Report:")) == 0){
         //if(sdata->import == 0){
            sdata->ms_journal = 1;
            memset(state->message_id, 0, SMALLBUFSIZE);
@ -271,55 +212,74 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata

      if(state->is_header == 1) state->is_header = 0;
      state->is_1st_header = 0;
+
+      if(state->anamepos > 0){
+         extractNameFromHeaderLine(state->attachment_name_buf, "name", state->filename);
+      }
+
   }


   if(take_into_pieces == 1){
-      if(state->message_state == MSG_BODY && state->fd != -1 && (state->has_to_dump_whole_body == 1 || is_substr_in_hash(state->boundaries, buf) == 0) ){
+      if(state->message_state == MSG_BODY && state->fd != -1 && is_substr_in_hash(state->boundaries, buf) == 0){
+         //n = write(state->fd, buf, len); // WRITE
         if(len + state->abufpos > abuffersize-1){
-            flush_attachment_buffer(state, abuffer, abuffersize);
+            write(state->fd, abuffer, state->abufpos); 
+
+            if(state->b64fd != -1){
+               abuffer[state->abufpos] = '\0';
+               if(state->base64 == 1){
+                  n64 = base64_decode_attachment_buffer(abuffer, &b64buffer[0], sizeof(b64buffer));
+                  write(state->b64fd, b64buffer, n64);
+               }
+               else {
+                  write(state->b64fd, abuffer, state->abufpos);
+               }
+            }
+
+            state->abufpos = 0; memset(abuffer, 0, abuffersize);
         }
         memcpy(abuffer+state->abufpos, buf, len); state->abufpos += len;

         state->attachments[state->n_attachments].size += len;
-
-         // When processing the body and writing to an attachment file, then we finish here
-         return 0;
      }
      else {
         state->saved_size += len;
+         //n = write(state->mfd, buf, len); // WRITE
         if(len + state->writebufpos > writebuffersize-1){
-            if(write(state->mfd, writebuffer, state->writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
-            state->writebufpos = 0;
-            memset(writebuffer, 0, writebuffersize);
+            write(state->mfd, writebuffer, state->writebufpos); state->writebufpos = 0; memset(writebuffer, 0, writebuffersize);
         }
         memcpy(writebuffer+state->writebufpos, buf, len); state->writebufpos += len;
      }
   }


-   if(state->message_state == MSG_BODY && state->has_to_dump == 1 && state->pushed_pointer == 0){
+   if(state->message_state == MSG_BODY && state->has_to_dump == 1 &&  state->pushed_pointer == 0){
+      //printf("####name: %s, type: %s, base64: %d\n", state->filename, state->type, state->base64);
+
      state->pushed_pointer = 1;


      // this is a real attachment to dump, it doesn't have to be base64 encoded!
-      if(state->attachment_name_buf[0] != 0 && strcasestr(state->attachment_name_buf, "name") && strlen(state->type) > 3 && state->n_attachments < MAX_ATTACHMENTS-1){
+      if(strlen(state->filename) > 4 && strlen(state->type) > 3 && state->n_attachments < MAX_ATTACHMENTS-1){
         state->n_attachments++;

-         extractNameFromHeaderLine(state->attachment_name_buf, "name", state->attachments[state->n_attachments].filename, SMALLBUFSIZE);
+         snprintf(state->attachments[state->n_attachments].filename, TINYBUFSIZE-1, "%s", state->filename);
         snprintf(state->attachments[state->n_attachments].type, TINYBUFSIZE-1, "%s", state->type);
         snprintf(state->attachments[state->n_attachments].internalname, TINYBUFSIZE-1, "%s.a%d", sdata->ttmpfile, state->n_attachments);
         snprintf(state->attachments[state->n_attachments].aname, TINYBUFSIZE-1, "%s.a%d.bin", sdata->ttmpfile, state->n_attachments);

+         //printf("DUMP FILE: %s\n", state->attachments[state->n_attachments].internalname);
+
         if(take_into_pieces == 1){
            state->fd = open(state->attachments[state->n_attachments].internalname, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR);

-            fixupEncodedHeaderLine(state->attachments[state->n_attachments].filename, SMALLBUFSIZE);
+            fixupEncodedHeaderLine(state->attachments[state->n_attachments].filename, TINYBUFSIZE);

            p = get_attachment_extractor_by_filename(state->attachments[state->n_attachments].filename);

            snprintf(state->attachments[state->n_attachments].shorttype, TINYBUFSIZE-1, "%s", p);
-
+ 
            if(strcmp("other", p)){
               state->b64fd = open(state->attachments[state->n_attachments].aname, O_CREAT|O_RDWR, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
               state->attachments[state->n_attachments].dumped = 1;
@ -332,16 +292,13 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
               syslog(LOG_PRIORITY, "%s: error opening %s", sdata->ttmpfile, state->attachments[state->n_attachments].internalname);
            }
            else {
-               char puf[SMALLBUFSIZE];
               snprintf(puf, sizeof(puf)-1, "ATTACHMENT_POINTER_%s.a%d_XXX_PILER", sdata->ttmpfile, state->n_attachments);
-               int writelen = strlen(puf);
+               //n = write(state->mfd, puf, strlen(puf)); // WRITE
+               writelen = strlen(puf);
               if(writelen + state->writebufpos > writebuffersize-1){
-                  if(write(state->mfd, writebuffer, state->writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: write(), %s, %d, %s", __func__, __LINE__, __FILE__);
-                  state->writebufpos = 0;
-                  memset(writebuffer, 0, writebuffersize);
+                  write(state->mfd, writebuffer, state->writebufpos); state->writebufpos = 0; memset(writebuffer, 0, writebuffersize);
               }
-               memcpy(writebuffer+state->writebufpos, puf, writelen);
-               state->writebufpos += writelen;
+               memcpy(writebuffer+state->writebufpos, puf, writelen); state->writebufpos += writelen;
            }
         }

@ -390,62 +347,37 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
         sdata->spam_message = 1;
      }

-      if(strncasecmp(buf, "X-Piler-Envelope-To:", strlen("X-Piler-Envelope-To:")) == 0){
-         state->message_state = MSG_ENVELOPE_TO;
-         buf += strlen("X-Piler-Envelope-To:");
-      }
-      else if(strncasecmp(buf, "From:", strlen("From:")) == 0){
-         state->message_state = MSG_FROM;
-         buf += strlen("From:");
-      }
-      else if(strncasecmp(buf, "Sender:", strlen("Sender:")) == 0){
-         state->message_state = MSG_SENDER;
-         buf += strlen("Sender:");
-      }
+      if(strncasecmp(buf, "From:", strlen("From:")) == 0) state->message_state = MSG_FROM;
+
      else if(strncasecmp(buf, "Content-Type:", strlen("Content-Type:")) == 0){
         state->message_state = MSG_CONTENT_TYPE;
-      }
-      else if(strncasecmp(buf, "Content-Transfer-Encoding:", strlen("Content-Transfer-Encoding:")) == 0){
-         state->message_state = MSG_CONTENT_TRANSFER_ENCODING;
-         if(state->is_1st_header == 1 && strcasestr(buf, "base64")){
-            state->has_to_dump = 1;
-            state->has_to_dump_whole_body = 1;
-         }
-      }

-      /*
-       * We only enter MSG_CONTENT_DISPOSITION state if we couldn't find
-       * the filename in MSG_CONTENT_TYPE state. We also assume that
-       * Content-Type: comes first, then Content-Disposition:
-       */
-      else if(strncasecmp(buf, "Content-Disposition:", strlen("Content-Disposition:")) == 0 && strcasestr(state->attachment_name_buf, "name") == NULL){
+         if(state->anamepos > 0){
+            extractNameFromHeaderLine(state->attachment_name_buf, "name", state->filename);
+            memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
+            state->anamepos = 0;
+         }
+
+      }
+      else if(strncasecmp(buf, "Content-Transfer-Encoding:", strlen("Content-Transfer-Encoding:")) == 0) state->message_state = MSG_CONTENT_TRANSFER_ENCODING;
+      else if(strncasecmp(buf, "Content-Disposition:", strlen("Content-Disposition:")) == 0){
         state->message_state = MSG_CONTENT_DISPOSITION;
+
+         if(state->anamepos > 0){
+            extractNameFromHeaderLine(state->attachment_name_buf, "name", state->filename);
+            memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
+            state->anamepos = 0;
+         }
+
      }
-      else if(strncasecmp(buf, "To:", 3) == 0){
-         state->message_state = MSG_TO;
-         buf += strlen("To:");
-      }
-      else if(strncasecmp(buf, "Cc:", 3) == 0){
-         state->message_state = MSG_CC;
-         buf += strlen("Cc:");
-      }
-      else if(strncasecmp(buf, "Bcc:", 4) == 0){
-         state->message_state = MSG_CC;
-         buf += strlen("Bcc:");
-      }
-      else if(strncasecmp(buf, "Message-Id:", 11) == 0){
-         state->message_state = MSG_MESSAGE_ID;
-         buf += strlen("Message-Id:");
-      }
+      else if(strncasecmp(buf, "To:", 3) == 0) state->message_state = MSG_TO;
+      else if(strncasecmp(buf, "Cc:", 3) == 0) state->message_state = MSG_CC;
+      else if(strncasecmp(buf, "Bcc:", 4) == 0) state->message_state = MSG_CC;
+      else if(strncasecmp(buf, "Message-Id:", 11) == 0) state->message_state = MSG_MESSAGE_ID;
      else if(strncasecmp(buf, "References:", 11) == 0) state->message_state = MSG_REFERENCES;
-      else if(strncasecmp(buf, "Subject:", strlen("Subject:")) == 0){
-         state->message_state = MSG_SUBJECT;
-         buf += strlen("Subject:");
-      }
-      else if(strncasecmp(buf, "Recipient:", strlen("Recipient:")) == 0){
-         state->message_state = MSG_RECIPIENT;
-         buf += strlen("Recipient:");
-      }
+      else if(strncasecmp(buf, "Subject:", strlen("Subject:")) == 0) state->message_state = MSG_SUBJECT;
+      else if(strncasecmp(buf, "Recipient:", strlen("Recipient:")) == 0) state->message_state = MSG_RECIPIENT;
+
      if(sdata->ms_journal == 1 && (state->message_state == MSG_TO || state->message_state == MSG_RECIPIENT) ){
         p = strstr(buf, "Expanded:");
         if(p) *p = '\0';
@ -460,37 +392,23 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata

         if(strstr(buf, "=?") && strstr(buf, "?=")) fixupEncodedHeaderLine(buf, MAXBUFSIZE);

-         sdata->sent = parse_date_header(buf+5);
+         sdata->sent = parse_date_header(buf);

         /* allow +2 days drift in the parsed Date: value */

         if(sdata->sent - sdata->now > 2*86400) sdata->sent = sdata->now;
      }

-      else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf+14);
-      else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0){
-         state->message_state = MSG_RECEIVED;
-         state->received_header++;
-      }
-      else if(cfg->extra_to_field[0] != '\0' && strncasecmp(buf, cfg->extra_to_field, strlen(cfg->extra_to_field)) == 0){
-         state->message_state = MSG_RECIPIENT;
-         buf += strlen(cfg->extra_to_field);
-      }
+      else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf);
+      else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0) state->message_state = MSG_RECEIVED;
+      else if(cfg->extra_to_field[0] != '\0' && strncasecmp(buf, cfg->extra_to_field, strlen(cfg->extra_to_field)) == 0) state->message_state = MSG_TO;

      if(state->message_state == MSG_MESSAGE_ID && state->message_id[0] == 0){
-         while(isspace(*buf)){
-            buf++;
-         }
+         p = strchr(buf+11, ' ');
+         if(p) p = buf + 12;
+         else p = buf + 11;

-         snprintf(state->message_id, SMALLBUFSIZE-1, "%s", buf);
-      }
-
-      if(state->message_state == MSG_CONTENT_TYPE || state->message_state == MSG_CONTENT_DISPOSITION){
-         fill_attachment_name_buf(state, buf);
-      }
-
-      if(state->received_header == 1 && state->message_state == MSG_RECEIVED && strlen(state->receivedbuf) + len < sizeof(state->receivedbuf)){
-         memcpy(&(state->receivedbuf[strlen(state->receivedbuf)]), buf, len);
+         snprintf(state->message_id, SMALLBUFSIZE-1, "%s", p);
      }

      /* we are interested in only From:, To:, Subject:, Received:, Content-*: header lines */
@ -505,7 +423,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
   }


-   /*
+   /* 
    * A normal journal looks like this:
    *
    *   Sender: sender@domain
@ -545,8 +463,31 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
   if(state->is_1st_header == 1){

      if(state->message_state == MSG_SUBJECT && strlen(state->b_subject) + strlen(buf) < MAXBUFSIZE-1){
-         // buffer the subject lines, and decode it later
-         strncat(state->b_subject, buf, MAXBUFSIZE-strlen(state->b_subject)-1);
+
+         if(state->b_subject[0] == '\0'){
+            p = &buf[0];
+            if(strncmp(buf, "Subject:", strlen("Subject:")) == 0) p += strlen("Subject:");
+            if(*p == ' ') p++;
+
+            fixupEncodedHeaderLine(p, MAXBUFSIZE);
+            strncat(state->b_subject, p, MAXBUFSIZE-strlen(state->b_subject)-1);
+         }
+         else {
+
+            /*
+             * if the next subject line is encoded, then strip the whitespace characters at the beginning of the line
+             */
+
+            p = buf;
+
+            if(strcasestr(buf, "?Q?") || strcasestr(buf, "?B?")){
+               while(isspace(*p)) p++;
+            }
+
+            fixupEncodedHeaderLine(p, MAXBUFSIZE);
+
+            strncat(state->b_subject, p, MAXBUFSIZE-strlen(state->b_subject)-1);
+         }
      }
      else { fixupEncodedHeaderLine(buf, MAXBUFSIZE); }
   }
@ -590,8 +531,6 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
         state->message_rfc822 = 1;
         state->is_header = 1;

-         state->has_to_dump = 0;
-
         if(sdata->ms_journal == 1){
            state->is_1st_header = 1;

@ -601,8 +540,6 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
            memset(state->b_body, 0, BIGBUFSIZE);
            memset(state->b_from, 0, SMALLBUFSIZE);
            memset(state->b_from_domain, 0, SMALLBUFSIZE);
-            memset(state->b_sender, 0, SMALLBUFSIZE);
-            memset(state->b_sender_domain, 0, SMALLBUFSIZE);
            memset(state->message_id, 0, SMALLBUFSIZE);

            sdata->ms_journal = 0;
@ -610,11 +547,27 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
      }


-      if(strcasestr(buf, "charset")) extractNameFromHeaderLine(buf, "charset", state->charset, TINYBUFSIZE);
+      if(strcasestr(buf, "charset")) extractNameFromHeaderLine(buf, "charset", state->charset);
      if(strcasestr(state->charset, "UTF-8")) state->utf8 = 1;
   }


+   if((state->message_state == MSG_CONTENT_TYPE || state->message_state == MSG_CONTENT_DISPOSITION) && strlen(state->filename) < 5){
+
+      p = &buf[0];
+      for(; *p; p++){
+         if(*p != ' ' && *p != '\t') break;
+      }
+
+      len = strlen(p);
+
+      if(len + state->anamepos < SMALLBUFSIZE-2){
+         memcpy(&(state->attachment_name_buf[state->anamepos]), p, len);
+         state->anamepos += len;
+      }
+   }
+
+
   if(state->message_state == MSG_CONTENT_TRANSFER_ENCODING){
      if(strcasestr(buf, "base64")) state->base64 = 1;
      if(strcasestr(buf, "quoted-printable")) state->qp = 1;
@ -635,7 +588,20 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
      if(state->has_to_dump == 1){
         if(take_into_pieces == 1 && state->fd != -1){
            if(state->abufpos > 0){
-               flush_attachment_buffer(state, abuffer, abuffersize);
+               write(state->fd, abuffer, state->abufpos);
+
+               if(state->b64fd != -1){
+                  abuffer[state->abufpos] = '\0';
+                  if(state->base64 == 1){
+                     n64 = base64_decode_attachment_buffer(abuffer, &b64buffer[0], sizeof(b64buffer));
+                     write(state->b64fd, b64buffer, n64);
+                  }
+                  else {
+                     write(state->b64fd, abuffer, state->abufpos);
+                  }
+               }
+
+               state->abufpos = 0; memset(abuffer, 0, abuffersize); 
            }
            close(state->fd);
            close(state->b64fd);
@ -656,15 +622,16 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata

      state->pushed_pointer = 0;

+      memset(state->filename, 0, TINYBUFSIZE);
      memset(state->type, 0, TINYBUFSIZE);
-      memset(state->charset, 0, TINYBUFSIZE);
+      snprintf(state->charset, TINYBUFSIZE-1, "unknown");

      memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
      state->anamepos = 0;

      state->message_state = MSG_UNDEF;

-      return 0;
+      return 0;      
   }

   if(boundary_line == 1){ return 0; }
@ -674,7 +641,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata


   /* skip irrelevant headers */
-   if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_SENDER && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT && state->message_state != MSG_ENVELOPE_TO) return 0;
+   if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT) return 0;


   /* don't process body if it's not a text or html part */
@ -687,6 +654,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
   }


+   /* remove all HTML tags */
+   if(state->texthtml == 1 && state->message_state == MSG_BODY) markHTML(buf, state);

   if(state->message_state == MSG_BODY && state->qp == 1){
      fixupSoftBreakInQuotedPritableLine(buf, state); // 2011.12.07
@ -696,31 +665,128 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
   /* I believe that we can live without this function call */
   //decodeURL(buf);

-   /* remove all HTML tags */
-   if(state->texthtml == 1 && state->message_state == MSG_BODY) markHTML(buf, state);
-
-
-   if(state->texthtml == 1){
-      size_t buflen = strlen(buf);
-      decodeHTML(buf, state->utf8);
-      /* decodeHTML converted some entities to iso-8859-1 */
-      if(state->utf8 != 1 && strlen(buf) != buflen){
-        /* no charset or us-ascii: switch to iso-8859-1 */
-        if (state->charset[0] == 0 || strcasecmp(state->charset, "us-ascii") == 0){
-          syslog(LOG_PRIORITY, "%s: assuming iso-8859-1 encoding for HTML (was '%s')", sdata->ttmpfile, state->charset);
-          snprintf(state->charset, TINYBUFSIZE-1, "ISO8859-1");
-        }
-      }
-   }
+   if(state->texthtml == 1) decodeHTML(buf, state->utf8);

   /* encode the body if it's not utf-8 encoded */
   if(state->message_state == MSG_BODY && state->utf8 != 1){
-      char tmpbuf[MAXBUFSIZE];
-      int result = utf8_encode(buf, strlen(buf), &tmpbuf[0], sizeof(tmpbuf), state->charset);
+      result = utf8_encode(buf, strlen(buf), &tmpbuf[0], sizeof(tmpbuf), state->charset);
      if(result == OK) snprintf(buf, MAXBUFSIZE-1, "%s", tmpbuf);
   }

-   tokenize(buf, state, sdata, data, cfg);
+   translateLine((unsigned char*)buf, state);
+
+   reassembleToken(buf);
+
+
+   if(state->is_header == 1) p = strchr(buf, ' ');
+   else p = buf;
+
+   //printf("a: %d/%d/%d/%d/j=%d %s\n", state->is_1st_header, state->is_header, state->message_rfc822, state->message_state, sdata->ms_journal, buf);
+
+   do {
+      memset(puf, 0, sizeof(puf));
+      p = split(p, ' ', puf, sizeof(puf)-1, &result);
+
+      if(puf[0] == '\0') continue;
+
+      degenerateToken((unsigned char*)puf);
+
+      if(puf[0] == '\0') continue;
+
+      strncat(puf, " ", sizeof(puf)-strlen(puf)-1);
+
+      if(strncasecmp(puf, "http://", 7) == 0 || strncasecmp(puf, "https://", 8) == 0) fixURL(puf, sizeof(puf)-1);
+
+      len = strlen(puf);
+
+      // skip body tokens if not an URL && (empty token || too long)
+      if(state->is_header == 0 && strncmp(puf, "__URL__", 7) && (puf[0] == ' ' || (len > MAX_WORD_LEN && cfg->enable_cjk == 0)) ){
+         continue;
+      }
+
+      if(state->message_state == MSG_FROM && state->is_1st_header == 1 && strlen(state->b_from) < SMALLBUFSIZE-len-1){
+         strtolower(puf);
+
+         q = strchr(puf, '@');
+         if(q) fix_plus_sign_in_email_address(puf, &q, &len);
+
+         memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
+
+         if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(puf) == 1 && state->b_from_domain[0] == '\0'){
+            if(q && strlen(q) > 5){
+               memcpy(&(state->b_from_domain), q+1, strlen(q+1)-1);
+               if(strstr(sdata->mailfrom, "<>")){
+                  snprintf(sdata->fromemail, SMALLBUFSIZE-1, "%s", puf);
+                  sdata->fromemail[len-1] = '\0';
+               }
+            }
+
+            if(is_email_address_on_my_domains(puf, data) == 1) sdata->internal_sender = 1;
+
+            if(strlen(state->b_from) < SMALLBUFSIZE-len-1){
+               split_email_address(puf);
+               memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
+            }
+         }
+      }
+      else if((state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && state->is_1st_header == 1 && state->tolen < MAXBUFSIZE-len-1){
+         strtolower(puf);
+
+         /* fix aaa+bbb@ccc.fu address to aaa@ccc.fu, 2017.02.04, SJ */
+         q = strchr(puf, '@');
+         if(q) fix_plus_sign_in_email_address(puf, &q, &len);
+
+         if(state->message_state == MSG_RECIPIENT && findnode(state->journal_recipient, puf) == NULL){
+            addnode(state->journal_recipient, puf);
+            memcpy(&(state->b_journal_to[state->journaltolen]), puf, len);
+            if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: journal rcpt: '%s'", sdata->ttmpfile, puf);
+         }
+
+
+         if(findnode(state->rcpt, puf) == NULL){
+
+            /* skip any address matching ...@cfg->hostid, 2013.10.29, SJ */
+            if(q && strncmp(q+1, cfg->hostid, cfg->hostid_len) == 0){
+               continue;
+            }
+
+            addnode(state->rcpt, puf);
+            memcpy(&(state->b_to[state->tolen]), puf, len);
+            state->tolen += len;
+
+            if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(puf) == 1){
+
+               if(is_email_address_on_my_domains(puf, data) == 1) sdata->internal_recipient = 1;
+               else sdata->external_recipient = 1;
+
+               if(q){
+                  if(findnode(state->rcpt_domain, q+1) == NULL){
+                     addnode(state->rcpt_domain, q+1);
+                     memcpy(&(state->b_to_domain[strlen(state->b_to_domain)]), q+1, strlen(q+1));
+                  }
+               }
+
+               if(state->tolen < MAXBUFSIZE-len-1){
+                  split_email_address(puf);
+                  memcpy(&(state->b_to[state->tolen]), puf, len);
+                  state->tolen += len;
+               }
+
+            }
+         }
+
+      }
+      else if(state->message_state == MSG_BODY && len >= cfg->min_word_len && state->bodylen < BIGBUFSIZE-len-1){
+         // 99% of email addresses are longer than 8 characters
+         if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(puf)){
+            fix_email_address_for_sphinx(puf);
+         }
+
+         memcpy(&(state->b_body[state->bodylen]), puf, len);
+         state->bodylen += len;
+      }
+
+   } while(p);

   return 0;
 }
--- a/src/parser.h
+++ b/src/parser.h
@ -9,9 +9,9 @@
 #include "config.h"
 #include "defs.h"

-struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct data *data, struct config *cfg);
-void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg);
-int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg);
+struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct __data *data, struct __config *cfg);
+void post_parse(struct session_data *sdata, struct parser_state *state, struct __config *cfg);
+int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, int writebuffersize, char *abuffer, int abuffersize, struct __data *data, struct __config *cfg);

 void init_state(struct parser_state *state);
 time_t parse_date_header(char *s);
@ -20,24 +20,19 @@ void fixupEncodedHeaderLine(char *buf, int buflen);
 void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state);
 void fixupBase64EncodedLine(char *buf, struct parser_state *state);
 void markHTML(char *buf, struct parser_state *state);
-void setStateHTML(char *htmlbuf, int pos, struct parser_state *state);
+int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state);
 void translateLine(unsigned char *p, struct parser_state *state);
 void fix_email_address_for_sphinx(char *s);
 void split_email_address(char *s);
 int does_it_seem_like_an_email_address(char *email);
-void add_recipient(char *email, unsigned int len, struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
 void reassembleToken(char *p);
 void degenerateToken(unsigned char *p);
 void fixURL(char *buf, int buflen);
-void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultbuflen);
+void extractNameFromHeaderLine(char *s, char *name, char *resultbuf);
 char *determine_attachment_type(char *filename, char *type);
 char *get_attachment_extractor_by_filename(char *filename);
 void parse_reference(struct parser_state *state, char *s);
 int base64_decode_attachment_buffer(char *p, unsigned char *b, int blen);
 void fix_plus_sign_in_email_address(char *puf, char **at_sign, unsigned int *len);
-void tokenize(char *buf, struct parser_state *state, struct session_data *sdata, struct data *data, struct config *cfg);
-void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned int abuffersize);
-void fill_attachment_name_buf(struct parser_state *state, char *buf);
-int get_first_email_address_from_string(char *str, char *buf, int buflen);

 #endif /* _PARSER_H */
--- a/src/parser_utils.c
+++ b/src/parser_utils.c
@ -40,7 +40,6 @@ void init_state(struct parser_state *state){

   state->htmltag = 0;
   state->style = 0;
-   state->meta_content_type = 0;

   state->skip_html = 0;

@ -50,16 +49,14 @@ void init_state(struct parser_state *state){
   memset(state->message_id_hash, 0, 2*DIGEST_LENGTH+1);
   memset(state->miscbuf, 0, MAX_TOKEN_LEN);
   memset(state->qpbuf, 0, MAX_TOKEN_LEN);
-   memset(state->receivedbuf, 0, sizeof(state->receivedbuf));

+   memset(state->filename, 0, TINYBUFSIZE);
   memset(state->type, 0, TINYBUFSIZE);
-   memset(state->charset, 0, TINYBUFSIZE);

   memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
   state->anamepos = 0;

   state->has_to_dump = 0;
-   state->has_to_dump_whole_body = 0;
   state->fd = -1;
   state->b64fd = -1;
   state->mfd = -1;
@ -70,7 +67,6 @@ void init_state(struct parser_state *state){

   state->writebufpos = 0;
   state->abufpos = 0;
-   state->received_header = 0;

   inithash(state->boundaries);
   inithash(state->rcpt);
@ -85,7 +81,7 @@ void init_state(struct parser_state *state){
      memset(state->attachments[i].type, 0, TINYBUFSIZE);
      memset(state->attachments[i].shorttype, 0, TINYBUFSIZE);
      memset(state->attachments[i].aname, 0, TINYBUFSIZE);
-      memset(state->attachments[i].filename, 0, SMALLBUFSIZE);
+      memset(state->attachments[i].filename, 0, TINYBUFSIZE);
      memset(state->attachments[i].internalname, 0, TINYBUFSIZE);
      memset(state->attachments[i].digest, 0, 2*DIGEST_LENGTH+1);
   }
@ -94,8 +90,6 @@ void init_state(struct parser_state *state){

   memset(state->b_from, 0, SMALLBUFSIZE);
   memset(state->b_from_domain, 0, SMALLBUFSIZE);
-   memset(state->b_sender, 0, SMALLBUFSIZE);
-   memset(state->b_sender_domain, 0, SMALLBUFSIZE);
   memset(state->b_to, 0, MAXBUFSIZE);
   memset(state->b_to_domain, 0, SMALLBUFSIZE);
   memset(state->b_subject, 0, MAXBUFSIZE);
@ -103,13 +97,10 @@ void init_state(struct parser_state *state){
   memset(state->b_journal_to, 0, MAXBUFSIZE);

   state->tolen = 0;
-   state->todomainlen = 0;
   state->bodylen = 0;
   state->journaltolen = 0;

   state->retention = 0;
-
-   state->found_security_header = 0;
 }


@ -122,12 +113,13 @@ long get_local_timezone_offset(){


 time_t parse_date_header(char *datestr){
-   int n=0;
+   int n=0, len;
   long offset=0;
   time_t ts=0;
   char *p, *q, *r, *tz, s[SMALLBUFSIZE], tzh[4], tzm[3];
   struct tm tm;

+   datestr += 5;
   p = datestr;

   for(; *datestr; datestr++){
@ -157,7 +149,7 @@ time_t parse_date_header(char *datestr){
   do {
      p = split_str(p, " ", s, sizeof(s)-1);

-      int len = strlen(s);
+      len = strlen(s);

      if(len > 0){
         n++;
@ -204,7 +196,6 @@ time_t parse_date_header(char *datestr){
         else if(strncasecmp(s, "Sat", 3) == 0) tm.tm_wday = 6;
         else if(strncasecmp(s, "Sun", 3) == 0) tm.tm_wday = 0;

-
         if(len <= 2 && tm.tm_mday == 0){ tm.tm_mday = atoi(s); continue; }

         if(len <= 2 && tm.tm_mon == -1){ tm.tm_mon = atoi(s) - 1; continue; }
@ -254,7 +245,7 @@ time_t parse_date_header(char *datestr){
            } else {
               tm.tm_min = atoi(r);
            }
-            continue;
+            continue; 
         }
      }

@ -266,24 +257,12 @@ time_t parse_date_header(char *datestr){

   ts += get_local_timezone_offset() - offset;

-   if(ts < 700000000){
-      // If the Date: field contains some garbage, eg.
-      // "Date: [mail_datw]" or similar, and the date
-      // is before Sat Mar  7 20:26:40 UTC 1992, then
-      // return the current timestamp
-
-      time_t now;
-      time(&now);
-
-      return now;
-   }
-
   return ts;
 }


 int extract_boundary(char *p, struct parser_state *state){
-   char *q;
+   char *q, *q2;

   p += strlen("boundary");

@ -310,7 +289,7 @@ int extract_boundary(char *p, struct parser_state *state){
            break;
      }

-      char *q2 = strchr(p, ';');
+      q2 = strchr(p, ';');
      if(q2) *q2 = '\0';

      q = strrchr(p, '"');
@ -334,13 +313,13 @@ int extract_boundary(char *p, struct parser_state *state){


 void fixupEncodedHeaderLine(char *buf, int buflen){
-   char *q, *r, *s, *e, *end;
+   char *sb, *sq, *p, *q, *r, *s, *e, *start, *end;
   /*
    * I thought SMALLBUFSIZE would be enough for v, encoding and tmpbuf(2*),
    * but then I saw a 6-7000 byte long subject line, so I've switched to MAXBUFSIZE
    */
-   char v[MAXBUFSIZE], u[MAXBUFSIZE], puf[MAXBUFSIZE], encoding[MAXBUFSIZE], tmpbuf[2*MAXBUFSIZE];
-   int need_encoding, ret, prev_encoded=0, n_tokens=0;
+   char v[MAXBUFSIZE], puf[MAXBUFSIZE], encoding[MAXBUFSIZE], tmpbuf[2*MAXBUFSIZE];
+   int need_encoding, ret;

   if(buflen < 5) return;

@ -351,111 +330,72 @@ void fixupEncodedHeaderLine(char *buf, int buflen){
   do {
      q = split_str(q, " ", v, sizeof(v)-1);

-      char *p = v;
+      p = v;
+
+      memset(encoding, 0, sizeof(encoding));

      do {
-         memset(u, 0, sizeof(u));
+         start = strstr(p, "=?");
+         if(start){
+            *start = '\0';
+            if(strlen(p) > 0){
+               strncat(puf, p, sizeof(puf)-strlen(puf)-1);
+            }

-         /*
-          * We can't use split_str(p, "=?", ...) it will fail with the following pattern
-          *    =?UTF-8?B?SG9neWFuIMOtcmp1bmsgcGFuYXN6bGV2ZWxldD8=?=
-          *
-          * Also the below pattern requires special care:
-          *    =?gb2312?B?<something>?==?gb2312?Q?<something else>?=
-          *
-          * And we have to check the following cases as well:
-          *    Happy New Year! =?utf-8?q?=F0=9F=8E=86?=
-          */
+            start++;

-         int b64=0, qp=0;
-         memset(encoding, 0, sizeof(encoding));
-
-         r = strstr(p, "=?");
-         if(r){
-            p = r + 2;
-
-            e = strchr(p, '?');
+            e = strchr(start+2, '?');
            if(e){
               *e = '\0';
-               snprintf(encoding, sizeof(encoding)-1, "%s", p);
+               snprintf(encoding, sizeof(encoding)-1, "%s", start+1);
               *e = '?';
+            }

-               s = strcasestr(e, "?B?");
-               if(s){
-                  b64 = 1;
-                  p = s + 3;
-               }
-               else {
-                  s = strcasestr(e, "?Q?");
-                  if(s){
-                     qp = 1;
-                     p = s + 3;
+            s = NULL;
+            sb = strcasestr(start, "?B?"); if(sb) s = sb;
+            sq = strcasestr(start, "?Q?"); if(sq) s = sq;
+
+            if(s){
+               end = strstr(s+3, "?=");
+               if(end){
+                  *end = '\0';
+
+                  if(sb){ decodeBase64(s+3); }
+                  if(sq){ decodeQP(s+3); r = s + 3; for(; *r; r++){ if(*r == '_') *r = ' '; } }
+
+                  /* encode everything if it's not utf-8 encoded */
+
+                  need_encoding = 0;
+                  ret = ERR;
+
+                  if(strlen(encoding) > 2 && strcasecmp(encoding, "utf-8")){
+                     need_encoding = 1;
+                     ret = utf8_encode(s+3, strlen(s+3), &tmpbuf[0], sizeof(tmpbuf), encoding);
                  }
+
+                  if(need_encoding == 1 && ret == OK)
+                     strncat(puf, tmpbuf, sizeof(puf)-strlen(puf)-1);
+                  else 
+                     strncat(puf, s+3, sizeof(puf)-strlen(puf)-1);
+
+                  p = end + 2;
               }
            }
-
-            end = strstr(p, "?=");
-            if(end){
-               *end = '\0';
-            }
-
-
-            snprintf(u, sizeof(u)-1, "%s", p);
-
-            if(end) {
-               p = end + 2;
-            }
-         }
-         else {
-            snprintf(u, sizeof(u)-1, "%s", p);
-            p = NULL;
-         }
-
-         if(u[0] == 0) continue;
-
-         n_tokens++;
-
-         if(b64 == 1) decodeBase64(u);
-         else if(qp == 1) decodeQP(u);
-
-
-         /*
-          * https://www.ietf.org/rfc/rfc2047.txt says that
-          *
-          * "When displaying a particular header field that contains multiple
-          *  'encoded-word's, any 'linear-white-space' that separates a pair of
-          *  adjacent 'encoded-word's is ignored." (6.2)
-          */
-         if(prev_encoded == 1 && (b64 == 1 || qp == 1)) {}
-         else if(n_tokens > 1){
-            strncat(puf, " ", sizeof(puf)-strlen(puf)-1);
-         }
-
-         if(b64 == 1 || qp == 1){
-            prev_encoded = 1;
-            need_encoding = 0;
-            ret = ERR;
-
-            if(encoding[0] && strcasecmp(encoding, "utf-8")){
-               need_encoding = 1;
-               ret = utf8_encode(u, strlen(u), &tmpbuf[0], sizeof(tmpbuf), encoding);
-            }
-
-            if(need_encoding == 1 && ret == OK){
-               #pragma GCC diagnostic push
-               #pragma GCC diagnostic ignored "-Wstringop-truncation"
-               strncat(puf, tmpbuf, sizeof(puf)-strlen(puf)-1);
-            }
            else {
-               strncat(puf, u, sizeof(puf)-strlen(puf)-1);
+               strncat(puf, start, sizeof(puf)-strlen(puf)-1);
+
+               break;
            }
         }
         else {
-            strncat(puf, u, sizeof(puf)-strlen(puf)-1);
+            strncat(puf, p, sizeof(puf)-strlen(puf)-1);
+            break;
         }

      } while(p);

+      if(q) strncat(puf, " ", sizeof(puf)-strlen(puf)-1);
+
   } while(q);

   snprintf(buf, buflen-1, "%s", puf);
@ -464,9 +404,9 @@ void fixupEncodedHeaderLine(char *buf, int buflen){

 void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state){
   int i=0;
+   char *p, puf[MAXBUFSIZE];

   if(strlen(state->qpbuf) > 0){
-      char puf[MAXBUFSIZE];
      memset(puf, 0, sizeof(puf));
      snprintf(puf, sizeof(puf)-1, "%s%s", state->qpbuf, buf);
      snprintf(buf, MAXBUFSIZE-1, "%s", puf);
@ -479,7 +419,7 @@ void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state){
   }

   if(i == 1){
-      char *p = strrchr(buf, ' ');
+      p = strrchr(buf, ' ');
      if(p){
         memset(state->qpbuf, 0, MAX_TOKEN_LEN);
         if(strlen(p) < MAX_TOKEN_LEN-1){
@ -493,8 +433,9 @@ void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state){


 void fixupBase64EncodedLine(char *buf, struct parser_state *state){
+   char *p, puf[MAXBUFSIZE];
+
   if(strlen(state->miscbuf) > 0){
-      char puf[MAXBUFSIZE];
      memset(puf, 0, sizeof(puf));
      strncpy(puf, state->miscbuf, sizeof(puf)-strlen(puf)-1);
      strncat(puf, buf, sizeof(puf)-strlen(puf)-1);
@ -506,7 +447,7 @@ void fixupBase64EncodedLine(char *buf, struct parser_state *state){
   }

   if(buf[strlen(buf)-1] != '\n'){
-      char *p = strrchr(buf, ' ');
+      p = strrchr(buf, ' ');
      if(p){
         memcpy(&(state->miscbuf[0]), p+1, MAX_TOKEN_LEN-1);
         *p = '\0';
@ -538,13 +479,13 @@ void markHTML(char *buf, struct parser_state *state){
      }

      if(state->htmltag == 1){
-
+    
         if(j == 0 && *s == '!'){
            state->skip_html = 1;
            //printf("skiphtml=1\n");
         }

-         if(state->skip_html == 0){
+         if(state->skip_html == 0){ 
            if(*s != '>' && *s != '<' && *s != '"'){
               //printf("j=%d/%c", j, *s);
               html[j] = tolower(*s);
@ -553,7 +494,7 @@ void markHTML(char *buf, struct parser_state *state){

            if(isspace(*s)){
               if(j > 0){
-                  setStateHTML(html, pos, state);
+                  k += appendHTMLTag(puf, html, pos, state);
                  memset(html, 0, SMALLBUFSIZE); j=0;
               }
               pos++;
@ -578,51 +519,55 @@ void markHTML(char *buf, struct parser_state *state){

         if(j > 0){
            strncat(html, " ", SMALLBUFSIZE-1);
-            setStateHTML(html, pos, state);
+            k += appendHTMLTag(puf, html, pos, state);
            memset(html, 0, SMALLBUFSIZE); j=0;
         }
-         state->meta_content_type = 0;
      }

   }

   //printf("append last in line:*%s*, html=+%s+, j=%d\n", puf, html, j);
-   if(j > 0){ setStateHTML(html, pos, state); }
+   if(j > 0){ appendHTMLTag(puf, html, pos, state); }

   strcpy(buf, puf);
 }


-void setStateHTML(char *htmlbuf, int pos, struct parser_state *state){
+int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state){
+   char *p, html[SMALLBUFSIZE];
+   int len;
+
   if(pos == 0 && strncmp(htmlbuf, "style ", 6) == 0) state->style = 1;
   if(pos == 0 && strncmp(htmlbuf, "/style ", 7) == 0) state->style = 0;

-   if(pos == 0 && state->charset[0] == 0 && strncmp(htmlbuf, "meta ", 5) == 0) state->meta_content_type = 0x1;
-   if(state->meta_content_type){
-     if((state->meta_content_type & 0x2) == 0 && strstr(htmlbuf, "http-equiv=content-type "))
-       state->meta_content_type |= 0x2;
+   return 0;

-     if((state->meta_content_type & 0x4) == 0 && strstr(htmlbuf, "content=text/html;"))
-       state->meta_content_type |= 0x4;
+   //printf("appendHTML: pos:%d, +%s+\n", pos, htmlbuf);

-     if(state->meta_content_type == 0x7){
-       char *p, *q;
+   if(state->style == 1) return 0;

-       p = strstr(htmlbuf, "charset=");
-       if(p){
-         p += 8;
-         for(q = p; isalnum(*q) || index("-_", *q); q++)
-           ;
+   if(strlen(htmlbuf) == 0) return 0;

-         if(q > p && q-p+1 < (int) sizeof(state->charset)){
-           syslog(LOG_PRIORITY, "Changing HTML charset from '%s' to '%*s' due to meta tag", state->charset, (int)(q-p), p);
-           strncpy(state->charset, p, q-p);
-           state->charset[q-p+1] = '\0';
-           state->meta_content_type = 0;
-         }
-       }
-     }
+   snprintf(html, SMALLBUFSIZE-1, "HTML*%s", htmlbuf);
+   len = strlen(html);
+
+   if(len > 8 && strchr(html, '=')){
+      p = strstr(html, "cid:");
+      if(p){
+         *(p+3) = '\0';
+         strncat(html, " ", SMALLBUFSIZE-1);
+      }
+
+      strncat(buf, html, MAXBUFSIZE-1);
+      return len;
   }
+
+   if(strstr(html, "http") ){
+      strncat(buf, html+5, MAXBUFSIZE-1);
+      return len-5;
+   }
+
+   return 0;
 }


@ -635,9 +580,9 @@ void translateLine(unsigned char *p, struct parser_state *state){

   for(; *p; p++){

-      if( (state->message_state == MSG_RECEIVED || state->message_state == MSG_FROM || state->message_state == MSG_SENDER || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && *p == '@'){ continue; }
+      if( (state->message_state == MSG_RECEIVED || state->message_state == MSG_FROM || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && *p == '@'){ continue; }

-      if(state->message_state == MSG_FROM || state->message_state == MSG_SENDER || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT){
+      if(state->message_state == MSG_FROM || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT){

         /* To fix some unusual addresses, eg.
          *    "'user@domain'"    -> user@domain
@ -654,7 +599,6 @@ void translateLine(unsigned char *p, struct parser_state *state){
         prev = *p;
      }

-
      if(state->message_state == MSG_SUBJECT && (*p == '%' || *p == '_' || *p == '&') ){ continue; }

      if(state->message_state == MSG_CONTENT_TYPE && *p == '_' ){ continue; }
@ -709,53 +653,13 @@ int does_it_seem_like_an_email_address(char *email){
 }


-void add_recipient(char *email, unsigned int len, struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
-   if(findnode(state->rcpt, email) == NULL){
-
-      char *q = strchr(email, '@');
-
-      /* skip any address matching ...@cfg->hostid, 2013.10.29, SJ */
-      if(q && strncmp(q+1, cfg->hostid, cfg->hostid_len) == 0){
-         return;
-      }
-
-      addnode(state->rcpt, email);
-      memcpy(&(state->b_to[state->tolen]), email, len);
-      state->tolen += len;
-
-      if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(email) == 1){
-
-         if(is_email_address_on_my_domains(email, data) == 1) sdata->internal_recipient = 1;
-         else sdata->external_recipient = 1;
-
-         if(q){
-            if(findnode(state->rcpt_domain, q+1) == NULL){
-               addnode(state->rcpt_domain, q+1);
-               unsigned int domainlen = strlen(q+1);
-
-               if(state->todomainlen < SMALLBUFSIZE-domainlen-1){
-                  memcpy(&(state->b_to_domain[state->todomainlen]), q+1, domainlen);
-                  state->todomainlen += domainlen;
-               }
-            }
-         }
-
-         if(state->tolen < MAXBUFSIZE-len-1){
-            split_email_address(email);
-            memcpy(&(state->b_to[state->tolen]), email, len);
-            state->tolen += len;
-         }
-      }
-   }
-}
-
-
 /*
 * reassemble 'V i a g r a' to 'Viagra'
 */

 void reassembleToken(char *p){
-   unsigned int i, k=0;
+   unsigned int i;
+   int k=0;

   for(i=0; i<strlen(p); i++){

@ -780,7 +684,7 @@ void degenerateToken(unsigned char *p){
   int i=1, d=0, dp=0;
   unsigned char *s;

-   /* quit if the string does not end with a punctuation character */
+   /* quit if this the string does not end with a punctuation character */

   if(!ispunct(*(p+strlen((char *)p)-1)))
      return;
@ -835,13 +739,12 @@ void fixURL(char *buf, int buflen){
 }


-void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultbuflen){
-   char buf[SMALLBUFSIZE], *p, *q;
+void extractNameFromHeaderLine(char *s, char *name, char *resultbuf){
+   int extended=0;
+   char buf[SMALLBUFSIZE], puf[SMALLBUFSIZE], *p, *q, *encoding;

   snprintf(buf, sizeof(buf)-1, "%s", s);

-   memset(resultbuf, 0, resultbuflen);
-
   p = strstr(buf, name);
   if(p){

@ -865,15 +768,8 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb
       *
       *      foo: bar; title*=UTF-8''%c2%a3%20and%20%e2%82%ac%20rates
       *
-       *   Odd one having two filename definitions, and having a semicolon (;) in the filename:
-       *
-       *      filename*=utf-8''P;LAN%20Holden%204.docx;filename="P;LAN Holden 4.docx"*
-       *
-       *
       */

-      int extended=0;
-
      p += strlen(name);
      if(*p == '*'){
         extended = 1;
@ -882,26 +778,19 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb
      p = strchr(p, '=');
      if(p){
         p++;
-
-         // skip any whitespace after name=, ie. name = "
-         while(*p==' ' || *p=='\t') p++;
-
-         // if there's a double quote after the equal symbol (=), ie. name*="utf-8....
-         if(*p == '"'){
-            p++;
-            q = strchr(p, '"');
-
-            if(q) *q = '\0';
+         q = strrchr(p, ';');
+         if(q) *q = '\0';
+         q = strrchr(p, '"');
+         if(q){
+            *q = '\0';
+            p = strchr(p, '"');
+            if(p){
+               p++;
+            }
         }
-         else {
-            // no " after =, so split on ;
-            q = strchr(p, ';');
-            if(q) *q = '\0';
-         }
-

         if(extended == 1){
-            char *encoding = p;
+            encoding = p;
            q = strchr(p, '\'');
            if(q){
               *q = '\0';
@ -913,16 +802,15 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb
            decodeURL(p);

            if(strlen(encoding) > 2 && strcasecmp(encoding, "utf-8"))
-               utf8_encode(p, strlen(p), resultbuf, resultbuflen-2, encoding);
+               utf8_encode(p, strlen(p), resultbuf, TINYBUFSIZE-1, encoding);
            else
-               snprintf(resultbuf, resultbuflen-2, "%s", p);
+               snprintf(resultbuf, TINYBUFSIZE-1, "%s", p);
         }
         else {
-            char puf[SMALLBUFSIZE];
            snprintf(puf, sizeof(puf)-1, "%s", p);
            fixupEncodedHeaderLine(puf, sizeof(puf));

-            snprintf(resultbuf, resultbuflen-2, "%s", puf);
+            snprintf(resultbuf, TINYBUFSIZE-1, "%s", puf);
         }

      }
@ -932,6 +820,8 @@ void extractNameFromHeaderLine(char *s, char *name, char *resultbuf, int resultb


 char *determine_attachment_type(char *filename, char *type){
+   char *p;
+
   if(strncasecmp(type, "text/", strlen("text/")) == 0) return "text,";
   if(strncasecmp(type, "image/", strlen("image/")) == 0) return "image,";
   if(strncasecmp(type, "audio/", strlen("audio/")) == 0) return "audio,";
@ -962,7 +852,7 @@ char *determine_attachment_type(char *filename, char *type){

   if(strncasecmp(type, "application/", 12) == 0){

-      char *p = strrchr(filename, '.');
+      p = strrchr(filename, '.');
      if(p){
         p++;

@ -972,7 +862,7 @@ char *determine_attachment_type(char *filename, char *type){
         if(strncasecmp(p, "rar", 3) == 0) return "compressed,";

         // tar.gz has the same type
-         if(strncasecmp(p, "gz", 2) == 0) return "compressed,";
+         if(strncasecmp(p, "x-gzip", 3) == 0) return "compressed,";

         if(strncasecmp(p, "rtf", 3) == 0) return "word,";
         if(strncasecmp(p, "doc", 3) == 0) return "word,";
@ -987,7 +877,7 @@ char *determine_attachment_type(char *filename, char *type){
         if(strncasecmp(p, "jpg", 3) == 0) return "image,";
         if(strncasecmp(p, "jpeg", 4) == 0) return "image,";
         if(strncasecmp(p, "tiff", 4) == 0) return "image,";
-      }
+      } 
   }

   return "other,";
@ -1024,13 +914,14 @@ char *get_attachment_extractor_by_filename(char *filename){


 void parse_reference(struct parser_state *state, char *s){
+   int len;
   char puf[SMALLBUFSIZE];

   if(strlen(state->reference) > 10) return;

   do {
      s = split_str(s, " ", puf, sizeof(puf)-1);
-      int len = strlen(puf);
+      len = strlen(puf);

      if(len > 10 && len < SMALLBUFSIZE-1){
         memcpy(&(state->reference[strlen(state->reference)]), puf, len);
@ -1056,52 +947,15 @@ int base64_decode_attachment_buffer(char *p, unsigned char *b, int blen){


 void fix_plus_sign_in_email_address(char *puf, char **at_sign, unsigned int *len){
+   int n;
   char *r;

   r = strchr(puf, '+');
   if(r){
-      int n = strlen(*at_sign);
+      n = strlen(*at_sign);
      memmove(r, *at_sign, n);
      *(r+n) = '\0';
      *len = strlen(puf);
      *at_sign = r;
   }
 }
-
-
-void fill_attachment_name_buf(struct parser_state *state, char *buf){
-   char *p = &buf[0];
-
-   for(; *p; p++){
-      if(*p != ' ' && *p != '\t') break;
-   }
-
-   int len = strlen(p);
-
-   if(len + state->anamepos < SMALLBUFSIZE-3){
-      memcpy(&(state->attachment_name_buf[state->anamepos]), p, len);
-      state->anamepos += len;
-
-      // add a trailing separator semicolon to make sure there's separation
-      // with the next item
-      state->attachment_name_buf[state->anamepos] = ';';
-      state->anamepos++;
-   }
-}
-
-
-int get_first_email_address_from_string(char *str, char *buf, int buflen){
-   int result;
-
-   char *p = str;
-   do {
-      memset(buf, 0, buflen);
-      p = split(p, ' ', buf, buflen-1, &result);
-
-      if(*buf == '\0') continue;
-
-      if(does_it_seem_like_an_email_address(buf) == 1){ return 1; }
-   } while(p);
-
-   return 0;
-}
--- a/src/piler-smtp.c
+++ b/src/piler-smtp.c
@ -31,37 +31,23 @@ extern char *optarg;
 extern int optind;

 struct epoll_event event, *events=NULL;
+int timeout = 20; // checking for timeout this often [sec]
 int num_connections = 0;
 int listenerfd = -1;
-int loglevel = 1;

 char *configfile = CONFIG_FILE;
-struct config cfg;
+struct __config cfg;
 struct passwd *pwd;
 struct smtp_session *session, **sessions=NULL;
-struct smtp_acl *smtp_acl[MAXHASH];
-
-time_t prev_timeout_check = 0;
-
-void usage(){
-   printf("\nusage: piler\n\n");
-   printf("    -c <config file>                  Config file to use if not the default\n");
-   printf("    -d                                Fork to the background\n");
-   printf("    -v                                Return the version and build number\n");
-   printf("    -V                                Return the version and some build parameters\n");
-   printf("    -L <log level>                    Set the log level: 1-5\n");
-
-   exit(0);
-}


-void p_clean_exit(int sig){
-   if(sig > 0) syslog(LOG_PRIORITY, "got signal: %d, %s", sig, strsignal(sig));
+void p_clean_exit(){
+   int i;

   if(listenerfd != -1) close(listenerfd);

   if(sessions){
-      for(int i=0; i<cfg.max_connections; i++){
+      for(i=0; i<cfg.max_connections; i++){
         if(sessions[i]) free_smtp_session(sessions[i]);
      }

@ -70,10 +56,10 @@ void p_clean_exit(int sig){

   if(events) free(events);

-   clear_smtp_acl(smtp_acl);
-
   syslog(LOG_PRIORITY, "%s has been terminated", PROGNAME);

+   //unlink(cfg.pidfile);
+
   ERR_free_strings();

   exit(1);
@ -82,29 +68,26 @@ void p_clean_exit(int sig){

 void fatal(char *s){
   syslog(LOG_PRIORITY, "%s", s);
-   p_clean_exit(0);
+   p_clean_exit();
 }


 void check_for_client_timeout(){
   time_t now;
-
-   time(&now);
-
-   if(cfg.verbosity >= LOG_DEBUG) syslog(LOG_PRIORITY, "%s @%ld", __func__, now);
-
-   if(now - prev_timeout_check < cfg.smtp_timeout) return;
+   int i;

   if(num_connections > 0){
-      for(int i=0; i<cfg.max_connections; i++){
+      time(&now);
+
+      for(i=0; i<cfg.max_connections; i++){
         if(sessions[i] && now - sessions[i]->lasttime >= cfg.smtp_timeout){
-            syslog(LOG_PRIORITY, "client %s timeout, lasttime: %ld", sessions[i]->remote_host, sessions[i]->lasttime);
-            tear_down_session(sessions, sessions[i]->slot, &num_connections, "timeout");
+            syslog(LOG_PRIORITY, "client %s timeout", sessions[i]->remote_host);
+            tear_down_session(sessions, sessions[i]->slot, &num_connections);
         }
      }
   }

-   time(&prev_timeout_check);
+   alarm(timeout);
 }


@ -128,32 +111,27 @@ void initialise_configuration(){
   setlocale(LC_MESSAGES, cfg.locale);
   setlocale(LC_CTYPE, cfg.locale);

-   load_smtp_acl(smtp_acl);
-
   syslog(LOG_PRIORITY, "reloaded config: %s", configfile);
 }


 int main(int argc, char **argv){
-   int client_sockfd;
-   int i, daemonise=0;
+   int listenerfd, client_sockfd;
+   int i, n, daemonise=0;
   int client_len = sizeof(struct sockaddr_storage);
   ssize_t readlen;
   struct sockaddr_storage client_address;
-   char readbuf[REALLYBIGBUFSIZE];
+   char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
+   char readbuf[BIGBUFSIZE];
   int efd;

-   while((i = getopt(argc, argv, "c:L:dvVh")) > 0){
+   while((i = getopt(argc, argv, "c:dvVh")) > 0){
      switch(i){

        case 'c' :
                   configfile = optarg;
                   break;

-        case 'L':
-                   loglevel = atoi(optarg);
-                   break;
-
        case 'd' :
                   daemonise = 1;
                   break;
@ -164,8 +142,8 @@ int main(int argc, char **argv){
                   return 0;

        case 'h' :
-        default  :
-                   usage();
+        default  : 
+                   __fatal("usage: ...");
      }
   }

@ -201,14 +179,11 @@ int main(int argc, char **argv){

   set_signal_handler(SIGINT, p_clean_exit);
   set_signal_handler(SIGTERM, p_clean_exit);
-   set_signal_handler(SIGKILL, p_clean_exit);
-   set_signal_handler(SIGSEGV, p_clean_exit);
-
-   set_signal_handler(SIGPIPE, SIG_IGN);
-   set_signal_handler(SIGALRM, SIG_IGN);
-
+   set_signal_handler(SIGALRM, check_for_client_timeout);
   set_signal_handler(SIGHUP, initialise_configuration);

+   alarm(timeout);
+
   // calloc() initialitizes the allocated memory

   sessions = calloc(cfg.max_connections, sizeof(struct smtp_session));
@ -228,20 +203,12 @@ int main(int argc, char **argv){
 #endif

   for(;;){
-      int n = epoll_wait(efd, events, cfg.max_connections, 1000);
+      n = epoll_wait(efd, events, cfg.max_connections, -1);
      for(i=0; i<n; i++){

-         // Office365 sometimes behaves oddly: when it receives the 250 OK
-         // message after sending the email, it doesn't send the QUIT command
-         // rather it aborts the connection
-
         if((events[i].events & EPOLLERR) || (events[i].events & EPOLLHUP) || (!(events[i].events & EPOLLIN))){
-            if(cfg.verbosity >= _LOG_EXTREME) syslog(LOG_PRIORITY, "ERROR: the remote end hung up without sending QUIT");
-            session = get_session_by_socket(sessions, cfg.max_connections, events[i].data.fd);
-            if(session)
-               tear_down_session(sessions, session->slot, &num_connections, "hungup");
-            else
-               close(events[i].data.fd);
+            syslog(LOG_PRIORITY, "ERROR: epoll error");
+            close(events[i].data.fd);
            continue;
         }

@ -258,19 +225,13 @@ int main(int argc, char **argv){
                     break;
                  }
                  else {
-                     syslog(LOG_PRIORITY, "ERROR: accept(): '%s'", strerror(errno));
+                     syslog(LOG_PRIORITY, "ERROR: accept()");
                     break;
                  }
               }

-               char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
-               memset(hbuf, 0, sizeof(hbuf));
-               memset(sbuf, 0, sizeof(sbuf));
-
               if(getnameinfo((struct sockaddr *)&client_address, client_len, hbuf, sizeof(hbuf), sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV) == 0){
-                  // Strictly speaking it's not correct to log num_connections+1 connections
-                  // but it still gives a good clue how many connections we have at the moment
-                  syslog(LOG_PRIORITY, "connected from %s:%s on fd=%d (active connections: %d)", hbuf, sbuf, client_sockfd, num_connections + 1);
+                  syslog(LOG_PRIORITY, "connected from %s:%s on descriptor %d", hbuf, sbuf, client_sockfd);
               }

               if(make_socket_non_blocking(client_sockfd) == -1){
@ -285,7 +246,7 @@ int main(int argc, char **argv){
                  break;
               }

-               start_new_session(sessions, client_sockfd, &num_connections, smtp_acl, hbuf, &cfg);
+               start_new_session(sessions, client_sockfd, &num_connections, &cfg);
            }

            continue;
@ -307,12 +268,14 @@ int main(int argc, char **argv){
            time(&(session->lasttime));

            while(1){
-               if(session->net.use_ssl == 1)
-                  readlen = SSL_read(session->net.ssl, (char*)&readbuf[0], sizeof(readbuf)-1);
+               memset(readbuf, 0, sizeof(readbuf));
+
+               if(session->use_ssl == 1)
+                  readlen = SSL_read(session->ssl, (char*)&readbuf[0], sizeof(readbuf)-1);
               else
                  readlen = read(events[i].data.fd, (char*)&readbuf[0], sizeof(readbuf)-1);

-               if(cfg.verbosity >= _LOG_EXTREME && readlen > 0) syslog(LOG_PRIORITY, "got %ld bytes to read", readlen);
+               if(cfg.verbosity >= _LOG_DEBUG && readlen > 0) syslog(LOG_PRIORITY, "got %ld bytes to read", readlen);

               if(readlen == -1){
                  /* If errno == EAGAIN, that means we have read all data. So go back to the main loop. */
@ -327,8 +290,7 @@ int main(int argc, char **argv){
                  break;
               }

-               readbuf[readlen] = '\0';
-               handle_data(session, &readbuf[0], readlen, &cfg);
+               handle_data(session, &readbuf[0], readlen);

               if(session->protocol_state == SMTP_STATE_BDAT && session->bad == 1){
                  done = 1;
@ -339,14 +301,12 @@ int main(int argc, char **argv){
            /* Don't wait until the remote client closes the connection after he sent the QUIT command */

            if(done || session->protocol_state == SMTP_STATE_FINISHED){
-               tear_down_session(sessions, session->slot, &num_connections, "done");
+               tear_down_session(sessions, session->slot, &num_connections);
            }
         }


      }
-
-      check_for_client_timeout();
   }

   return 0;
--- a/src/piler.c
+++ b/src/piler.c
@ -34,27 +34,13 @@ extern int optind;
 int quit = 0;
 int received_sighup = 0;
 char *configfile = CONFIG_FILE;
-struct config cfg;
-struct data data;
+struct __config cfg;
+struct __data data;
 struct passwd *pwd;

 struct child children[MAXCHILDREN];


-void p_clean_exit();
-
-
-void usage(){
-   printf("\nusage: piler\n\n");
-   printf("    -c <config file>                  Config file to use if not the default\n");
-   printf("    -d                                Fork to the background\n");
-   printf("    -v                                Return the version and build number\n");
-   printf("    -V                                Return the version and some build parameters\n");
-
-   exit(0);
-}
-
-
 void takesig(int sig){
   int i, status;
   pid_t pid;
@ -101,48 +87,12 @@ void child_sighup_handler(int sig){
 }


-int perform_checks(char *filename, struct session_data *sdata, struct data *data, struct parser_state *parser_state, struct config *cfg){
-
-   if(cfg->security_header[0] && parser_state->found_security_header == 0){
-      syslog(LOG_PRIORITY, "%s: discarding: missing security header", filename);
-      return ERR_DISCARDED;
-   }
-
-   char *arule = check_against_ruleset(data->archiving_rules, parser_state, sdata->tot_len, sdata->spam_message);
-
-   if(arule){
-      syslog(LOG_PRIORITY, "%s: discarding: archiving policy: *%s*", filename, arule);
-      return ERR_DISCARDED;
-   }
-
-
-   if(cfg->archive_only_mydomains == 1 && sdata->internal_sender == 0 && sdata->internal_recipient == 0){
-      syslog(LOG_PRIORITY, "%s: discarding: not on mydomains", filename);
-      return ERR_DISCARDED;
-   }
-
-   make_digests(sdata, cfg);
-
-   // A normal header is much bigger than 10 bytes. We get here for header-only
-   // messages without a Message-ID: line. I believe that no such message is valid, and
-   // it's a reasonable to discard it, and not allowing it to fill up the error directory.
-
-   if(sdata->hdr_len < 10){
-      syslog(LOG_PRIORITY, "%s: discarding: a header-only message without a Message-ID line", filename);
-      return ERR_DISCARDED;
-   }
-
-   int rc = process_message(sdata, parser_state, data, cfg);
-   unlink(parser_state->message_id_hash);
-
-   return rc;
-}
-
-
-int process_email(char *filename, struct session_data *sdata, struct data *data, int size, struct config *cfg){
+int process_email(char *filename, struct session_data *sdata, struct __data *data, int size, struct __config *cfg){
+   int rc;
   char tmpbuf[SMALLBUFSIZE];
   char *status=S_STATUS_UNDEF;
-   char *p;
+   char *arule;
+   char *rcpt;
   struct timezone tz;
   struct timeval tv1, tv2;
   struct parser_state parser_state;
@ -166,11 +116,10 @@ int process_email(char *filename, struct session_data *sdata, struct data *data,
   snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);

   parser_state = parse_message(sdata, 1, data, cfg);
-
   post_parse(sdata, &parser_state, cfg);

   if(cfg->syslog_recipients == 1){
-      char *rcpt = parser_state.b_to;
+      rcpt = parser_state.b_to;
      do {
         rcpt = split_str(rcpt, " ", tmpbuf, sizeof(tmpbuf)-1);

@ -180,10 +129,28 @@ int process_email(char *filename, struct session_data *sdata, struct data *data,
      } while(rcpt);
   }

-   int rc = perform_checks(filename, sdata, data, &parser_state, cfg);
+
+   arule = check_againt_ruleset(data->archiving_rules, &parser_state, sdata->tot_len, sdata->spam_message);
+
+   if(arule){
+      syslog(LOG_PRIORITY, "%s: discarding: archiving policy: *%s*", filename, arule);
+      rc = ERR_DISCARDED;
+      remove_stripped_attachments(&parser_state);
+   }
+   else {
+      make_digests(sdata, cfg);
+
+      if(sdata->hdr_len < 10){
+         syslog(LOG_PRIORITY, "%s: invalid message, hdr_len: %d", filename, sdata->hdr_len);
+         rc = ERR;
+      }
+
+      rc = process_message(sdata, &parser_state, data, cfg);
+      unlink(parser_state.message_id_hash);
+   }
+
   unlink(sdata->tmpframe);

-   remove_stripped_attachments(&parser_state);

   if(rc == OK){
      status = S_STATUS_STORED;
@ -202,19 +169,6 @@ int process_email(char *filename, struct session_data *sdata, struct data *data,
   }
   else {
      status = S_STATUS_ERROR;
-
-      // move the file from piler/tmp/[0-xxx] dir to piler/error directory
-      p = strchr(filename, '/');
-      if(p)
-         p++;
-      else
-         p = filename;
-
-      snprintf(tmpbuf, sizeof(tmpbuf)-1, "%s/%s", ERROR_DIR, p);
-      if(rename(filename, tmpbuf) == 0)
-         syslog(LOG_PRIORITY, "%s: moved to %s", filename, tmpbuf);
-      else
-         syslog(LOG_PRIORITY, "%s: failed to moved to %s", filename, tmpbuf);
   }

   if(rc != ERR) unlink(filename);
@ -233,7 +187,7 @@ int process_email(char *filename, struct session_data *sdata, struct data *data,
 }


-int process_dir(char *directory, struct session_data *sdata, struct data *data, struct config *cfg){
+int process_dir(char *directory, struct session_data *sdata, struct __data *data, struct __config *cfg){
   DIR *dir;
   struct dirent *de;
   int tot_msgs=0;
@ -287,19 +241,10 @@ void child_main(struct child *ptr){

      sig_block(SIGHUP);

-      int sphxopen = 0;
-      if(cfg.rtindex && open_sphx(&sdata, &cfg) == OK){
-         sphxopen = 1;
-      }
-
-      if((cfg.rtindex == 0 || sphxopen == 1) && open_database(&sdata, &cfg) == OK){
+      if(open_database(&sdata, &cfg) == OK){
         ptr->messages += process_dir(dir, &sdata, &data, &cfg);
         close_database(&sdata);

-         if(cfg.rtindex){
-            close_sphx(&sdata);
-         }
-
         sleep(1);
      }
      else {
@ -309,13 +254,13 @@ void child_main(struct child *ptr){

      sig_unblock(SIGHUP);

-      // Let the child quit after processing max_requests_per_child messages
+      // TODO: do we want to quit after processing a certain number of messages?
+
+      //if(cfg.max_requests_per_child > 0 && ptr->messages >= cfg.max_requests_per_child){
+      //   if(cfg.verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "child (pid: %d, serial: %d) served enough: %d", getpid(), ptr->messages, ptr->serial);
+      //   break;
+      //}

-      if(cfg.max_requests_per_child > 0 && ptr->messages >= cfg.max_requests_per_child){
-         if(cfg.verbosity >= _LOG_DEBUG)
-            syslog(LOG_PRIORITY, "child (pid: %d, serial: %d) served enough: %d", getpid(), ptr->messages, ptr->serial);
-         break;
-      }
   }

 #ifdef HAVE_MEMCACHED
@ -470,9 +415,9 @@ void initialise_configuration(){
      return;
   }

-   load_rules(&sdata, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
-   load_rules(&sdata, data.retention_rules, SQL_RETENTION_RULE_TABLE);
-   load_rules(&sdata, data.folder_rules, SQL_FOLDER_RULE_TABLE);
+   load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
+   load_rules(&sdata, &data, data.retention_rules, SQL_RETENTION_RULE_TABLE);
+   load_rules(&sdata, &data, data.folder_rules, SQL_FOLDER_RULE_TABLE);

   load_mydomains(&sdata, &data, &cfg);

@ -490,8 +435,7 @@ void initialise_configuration(){


 int main(int argc, char **argv){
-   int i, daemonise=0;
-   struct stat st;
+   int i, daemonise=0, dedupfd;


   while((i = getopt(argc, argv, "c:dvVh")) > 0){
@ -516,7 +460,7 @@ int main(int argc, char **argv){

        case 'h' :
        default  : 
-                   usage();
+                   __fatal("usage: ...");
      }
   }

@ -529,7 +473,6 @@ int main(int argc, char **argv){
   initrules(data.retention_rules);
   initrules(data.folder_rules);
   data.dedup = MAP_FAILED;
-   data.import = NULL;

   initialise_configuration();

@ -541,10 +484,8 @@ int main(int argc, char **argv){

   if(drop_privileges(pwd)) fatal(ERR_SETUID);

-   if(stat(cfg.pidfile, &st) == 0) fatal(ERR_PID_FILE_EXISTS);
-
   if(cfg.mmap_dedup_test == 1){
-      int dedupfd = open(MESSAGE_ID_DEDUP_FILE, O_RDWR);
+      dedupfd = open(MESSAGE_ID_DEDUP_FILE, O_RDWR);
      if(dedupfd == -1) fatal(ERR_OPEN_DEDUP_FILE);

      data.dedup = mmap(NULL, MAXCHILDREN*DIGEST_LENGTH*2, PROT_READ|PROT_WRITE, MAP_SHARED, dedupfd, 0);
--- a/src/piler.h
+++ b/src/piler.h
@ -17,7 +17,6 @@
 #include <sig.h>
 #include <av.h>
 #include <rules.h>
-#include <screen.h>
 #include <sql.h>
 #include <import.h>
 #include <smtp.h>
@ -28,54 +27,50 @@
   #include "memc.h"
 #endif

-int read_key(struct config *cfg);
+int read_key(struct __config *cfg);
 void insert_offset(struct session_data *sdata, int server_id);

 void tear_down_client(int n);

-int do_av_check(char *filename, struct config *cfg);
+int do_av_check(char *filename, struct __config *cfg);

-int make_digests(struct session_data *sdata, struct config *cfg);
+int make_digests(struct session_data *sdata, struct __config *cfg);
 void digest_file(char *filename, char *digest);
-void digest_string(char *digestname, char *s, char *digest);
-void create_md5_from_email_address(char *puf, char *md5buf);
+void digest_string(char *s, char *digest);

 void remove_stripped_attachments(struct parser_state *state);
-int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
-void rollback(struct session_data *sdata, struct parser_state *state, uint64 id, struct config *cfg);
-int reimport_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
-int store_file(struct session_data *sdata, char *filename, int len, struct config *cfg);
-int remove_stored_message_files(struct session_data *sdata, struct parser_state *state, struct config *cfg);
-int store_attachments(struct session_data *sdata, struct parser_state *state, struct config *cfg);
-int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr);
+int process_message(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg);
+int reimport_message(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg);
+int store_file(struct session_data *sdata, char *filename, int len, struct __config *cfg);
+int remove_stored_message_files(struct session_data *sdata, struct parser_state *state, struct __config *cfg);
+int store_attachments(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg);
+int query_attachments(struct session_data *sdata, struct __data *data, struct ptr_array *ptr_arr);

-struct config read_config(char *configfile);
+struct __config read_config(char *configfile);

-void createdir(char *path, uid_t uid, gid_t gid, mode_t mode);
-void check_and_create_directories(struct config *cfg, uid_t uid, gid_t gid);
+void check_and_create_directories(struct __config *cfg, uid_t uid, gid_t gid);

-void update_counters(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg);
+void update_counters(struct session_data *sdata, struct __data *data, struct counters *counters, struct __config *cfg);

-int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct config *cfg);
-int file_from_archive_to_network(char *filename, int sd, int tls_enable, struct data *data, struct config *cfg);
+int retrieve_email_from_archive(struct session_data *sdata, struct __data *data, FILE *dest, struct __config *cfg);
+int file_from_archive_to_network(char *filename, int sd, int tls_enable, struct __data *data, struct __config *cfg);

-int get_folder_id(struct session_data *sdata, char *foldername, int parent_id);
-int add_new_folder(struct session_data *sdata, char *foldername, int parent_id);
+int get_folder_id(struct session_data *sdata, struct __data *data, char *foldername, int parent_id);
+int add_new_folder(struct session_data *sdata, struct __data *data, char *foldername, int parent_id);

-int store_index_data(struct session_data *sdata, struct parser_state *state, struct data *data, uint64 id, struct config *cfg);
+int store_index_data(struct session_data *sdata, struct parser_state *state, struct __data *data, uint64 id, struct __config *cfg);

-void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct config *cfg);
+void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct __config *cfg);

-int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct config *cfg);
+int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *dest, struct __config *cfg);

-void load_mydomains(struct session_data *sdata, struct data *data, struct config *cfg);
-int is_email_address_on_my_domains(char *email, struct data *data);
+void load_mydomains(struct session_data *sdata, struct __data *data, struct __config *cfg);
+int is_email_address_on_my_domains(char *email, struct __data *data);

-int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg);
-void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections, char *reason);
+int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct __config *cfg);
+void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections);
 struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int max_connections, int socket);
-void write_envelope_addresses(struct smtp_session *session, struct config *cfg);
-void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg);
+void handle_data(struct smtp_session *session, char *readbuf, int readlen);
 void free_smtp_session(struct smtp_session *session);

 void child_sighup_handler(int sig);
@ -83,6 +78,7 @@ void child_main(struct child *ptr);
 pid_t child_make(struct child *ptr);
 int search_slot_by_pid(pid_t pid);
 void kill_children(int sig);
+void p_clean_exit();
 void fatal(char *s);
 void initialise_configuration();

--- a/src/pileraget.c
+++ b/src/pileraget.c
@ -22,7 +22,7 @@ int main(int argc, char **argv){
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
   struct stat st;
 #endif
-   struct config cfg;
+   struct __config cfg;


   if(argc < 3){
@ -46,7 +46,7 @@ int main(int argc, char **argv){
      return 1;
   }

-   snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, argv[1][24], argv[1][25], argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
+   snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
   if(stat(filename, &st)){
      snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][RND_STR_LEN-6], argv[1][RND_STR_LEN-5], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
@ -57,3 +57,5 @@ int main(int argc, char **argv){

   return 0;
 }
+
+
--- a/src/pilerconf.c
+++ b/src/pilerconf.c
@ -12,13 +12,13 @@ extern char *optarg;
 extern int optind;


-void print_config_all(struct config *cfg, char *key);
-void print_config(char *configfile, struct config *cfg);
+void print_config_all(struct __config *cfg, char *key);
+void print_config(char *configfile, struct __config *cfg);

 int main(int argc, char **argv){
   int i, print_from_file=0;
   char *configfile=CONFIG_FILE, *query=NULL;
-   struct config cfg;
+   struct __config cfg;

   while((i = getopt(argc, argv, "c:q:nh?")) > 0){
       switch(i){
@ -41,7 +41,7 @@ int main(int argc, char **argv){
                    break;


-         default  :
+         default  : 
                    break;
       }
   }
--- a/src/pilerexport.c
+++ b/src/pilerexport.c
@ -8,13 +8,11 @@
 #include <sys/time.h>
 #include <sys/types.h>
 #include <sys/stat.h>
-#include <sys/mman.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <time.h>
 #include <locale.h>
 #include <syslog.h>
-#include <zip.h>
 #include <getopt.h>
 #include <piler.h>

@ -24,22 +22,15 @@ extern int optind;

 int dryrun = 0;
 int exportall = 0;
-int verification_status = 0;
-int export_to_stdout = 0;
+int rc = 0;
 char *query=NULL;
 int verbosity = 0;
 int max_matches = 1000;
 char *index_list = "main1,dailydelta1,delta1";
-struct passwd *pwd;
 regex_t regexp;
-char *zipfile = NULL;
-struct zip *zip = NULL;
-uint64 *zip_ids = NULL;
-int zip_counter = 0;
-int zip_batch = 2000;
-int max_files_in_export_dir = 0;

-int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg);
+
+int export_emails_matching_to_query(struct session_data *sdata, struct __data *data, char *s, struct __config *cfg);


 void usage(){
@ -57,13 +48,7 @@ void usage(){
   printf("    -w <where condition>              Where condition to pass to sphinx, eg. \"match('@subject: piler')\"\n");
   printf("    -m <max. matches>                 Max. matches to apply to sphinx query (default: %d)\n", max_matches);
   printf("    -i <index list>                   Sphinx indices to use  (default: %s)\n", index_list);
-#if LIBZIP_VERSION_MAJOR >= 1
-   printf("    -z <zip file>                     Write exported EML files to a zip file\n");
-   printf("    -Z <batch size>                   Zip batch size. Valid range: 10-10000, default: 2000\n");
-#endif
   printf("    -A                                Export all emails from archive\n");
-   printf("    -D <max files>                    Max. number of files to put in a single directory, default: 2000\n");
-   printf("    -o                                Export emails to stdout\n");
   printf("    -d                                Dry run\n");

   regfree(&regexp);
@ -102,12 +87,15 @@ unsigned long convert_time(char *yyyymmdd, int h, int m, int s){

   tm.tm_mday = atoi(yyyymmdd);

+
+   tm.tm_isdst = -1;
+
   return mktime(&tm);
 }


 int append_email_to_buffer(char **buffer, char *email){
-   int arglen;
+   int len, arglen;
   char *s=NULL, emailaddress[SMALLBUFSIZE];

   snprintf(emailaddress, sizeof(emailaddress)-1, "'%s'", email);
@ -119,7 +107,7 @@ int append_email_to_buffer(char **buffer, char *email){
      memcpy(*buffer, emailaddress, arglen);
   }
   else {
-      int len = strlen(*buffer);
+      len = strlen(*buffer);
      s = realloc(*buffer, len + arglen+2);
      if(!s){
         printf("malloc problem!\n");
@ -137,29 +125,50 @@ int append_email_to_buffer(char **buffer, char *email){
 }


-uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *where_condition, uint64 last_id, int *num, struct config *cfg){
+int append_string_to_buffer(char **buffer, char *str){
+   int len, arglen;
+   char *s=NULL;
+
+   arglen = strlen(str);
+
+   if(!*buffer){
+      *buffer = malloc(arglen+1);
+      memset(*buffer, 0, arglen+1);
+      memcpy(*buffer, str, arglen);
+   }
+   else {
+      len = strlen(*buffer);
+      s = realloc(*buffer, len + arglen+1);
+      if(!s) return 1;
+
+      *buffer = s;
+
+      memset(*buffer+len, 0, arglen+1);
+      memcpy(*buffer+len, str, arglen);
+   }
+
+   return 0;
+}
+
+
+uint64 run_query(struct session_data *sdata, struct session_data *sdata2, struct __data *data, char *where_condition, uint64 last_id, int *num, struct __config *cfg){
+   MYSQL_RES *res;
   MYSQL_ROW row;
-   uint64 id=0;
-   char s[MAXBUFSIZE];
   int rc=0;
+   uint64 id=0;
+   char s[SMALLBUFSIZE];

   *num = 0;

   if(!where_condition) return id;

-   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE id IN (", SQL_METADATA_TABLE);
+   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE id IN (", SQL_METADATA_TABLE);
   rc += append_string_to_buffer(&query, s);

   snprintf(s, sizeof(s)-1, "SELECT id FROM %s WHERE %s AND id > %llu ORDER BY id ASC LIMIT 0,%d", index_list, where_condition, last_id, max_matches);

-   if(dryrun){
-      printf("sphinx query: %s\n", s);
-   }
-
-   syslog(LOG_PRIORITY, "sphinx query: %s", s);
-
   if(mysql_real_query(&(sdata2->mysql), s, strlen(s)) == 0){
-      MYSQL_RES *res = mysql_store_result(&(sdata2->mysql));
+      res = mysql_store_result(&(sdata2->mysql));
      if(res != NULL){
         while((row = mysql_fetch_row(res))){
            id = strtoull(row[0], NULL, 10);
@ -174,8 +183,7 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *
      }
   }

-   if(!rc) export_emails_matching_to_query(sdata, query, cfg);
-   else printf("error: append_string_to_buffer() in run_query()\n");
+   if(!rc) export_emails_matching_to_query(sdata, data, query, cfg);

   free(query);
   query = NULL;
@ -186,11 +194,12 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *


 uint64 get_total_found(struct session_data *sdata){
+   MYSQL_RES *res;
   MYSQL_ROW row;
   uint64 total_found=0;

   if(mysql_real_query(&(sdata->mysql), "SHOW META LIKE 'total_found'", 28) == 0){
-      MYSQL_RES *res = mysql_store_result(&(sdata->mysql));
+      res = mysql_store_result(&(sdata->mysql));
      if(res != NULL){
         while((row = mysql_fetch_row(res))){
            total_found = strtoull(row[1], NULL, 10);
@ -203,17 +212,17 @@ uint64 get_total_found(struct session_data *sdata){
 }


-void export_emails_matching_id_list(struct session_data *sdata, struct session_data *sdata2, char *where_condition, struct config *cfg){
+void export_emails_matching_id_list(struct session_data *sdata, struct session_data *sdata2, struct __data *data, char *where_condition, struct __config *cfg){
   int n;
   uint64 count=0, last_id=0, total_found=0;

-   last_id = run_query(sdata, sdata2, where_condition, last_id, &n, cfg);
+   last_id = run_query(sdata, sdata2, data, where_condition, last_id, &n, cfg);
   count += n;

   total_found = get_total_found(sdata2);

   while(count < total_found){
-      last_id = run_query(sdata, sdata2, where_condition, last_id, &n, cfg);
+      last_id = run_query(sdata, sdata2, data, where_condition, last_id, &n, cfg);
      count += n;
   }

@ -221,86 +230,102 @@ void export_emails_matching_id_list(struct session_data *sdata, struct session_d


 int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain, int minsize, int maxsize, unsigned long startdate, unsigned long stopdate){
+   int where_condition=1;
   char s[SMALLBUFSIZE];
-   int rc=0;

   if(exportall == 1){
-      rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM ");
+      rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM ");
      rc += append_string_to_buffer(&query, SQL_METADATA_TABLE);
      rc += append_string_to_buffer(&query, " WHERE deleted=0 ");
      return rc;
   }

-   snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE deleted=0 ", SQL_MESSAGES_VIEW);
+   snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE deleted=0 ", SQL_MESSAGES_VIEW);

   rc = append_string_to_buffer(&query, s);

   if(from){
-      rc = append_string_to_buffer(&query, " AND ");
+      if(where_condition) rc = append_string_to_buffer(&query, " AND ");

      rc += append_string_to_buffer(&query, "`from` IN (");
      rc += append_string_to_buffer(&query, from);
      rc += append_string_to_buffer(&query, ")");

      free(from);
+
+      where_condition++;
   }

   if(to){
-      rc = append_string_to_buffer(&query, " AND ");
+      if(where_condition) rc = append_string_to_buffer(&query, " AND ");

      rc += append_string_to_buffer(&query, "`to` IN (");
      rc += append_string_to_buffer(&query, to);
      rc += append_string_to_buffer(&query, ")");

      free(to);
+
+      where_condition++;
   }

   if(fromdomain){
-      rc = append_string_to_buffer(&query, " AND ");
+      if(where_condition) rc = append_string_to_buffer(&query, " AND ");

      rc += append_string_to_buffer(&query, "`fromdomain` IN (");
      rc += append_string_to_buffer(&query, fromdomain);
      rc += append_string_to_buffer(&query, ")");

      free(fromdomain);
+
+      where_condition++;
   }


   if(todomain){
-      rc = append_string_to_buffer(&query, " AND ");
+      if(where_condition) rc = append_string_to_buffer(&query, " AND ");

      rc += append_string_to_buffer(&query, "`todomain` IN (");
      rc += append_string_to_buffer(&query, todomain);
      rc += append_string_to_buffer(&query, ")");

      free(todomain);
+
+      where_condition++;
   }

   if(minsize > 0){
-      rc = append_string_to_buffer(&query, " AND ");
+      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
      snprintf(s, sizeof(s)-1, " `size` >= %d", minsize);
      rc += append_string_to_buffer(&query, s);
+
+      where_condition++;
   }


   if(maxsize > 0){
-      rc = append_string_to_buffer(&query, " AND ");
+      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
      snprintf(s, sizeof(s)-1, " `size` <= %d", maxsize);
      rc += append_string_to_buffer(&query, s);
+
+      where_condition++;
   }


   if(startdate > 0){
-      rc = append_string_to_buffer(&query, " AND ");
-      snprintf(s, sizeof(s)-1, " `sent` >= %lu", startdate);
+      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      snprintf(s, sizeof(s)-1, " `sent` >= %ld", startdate);
      rc += append_string_to_buffer(&query, s);
+
+      where_condition++;
   }


   if(stopdate > 0){
-      rc = append_string_to_buffer(&query, " AND ");
-      snprintf(s, sizeof(s)-1, " `sent` <= %lu", stopdate);
+      if(where_condition) rc = append_string_to_buffer(&query, " AND ");
+      snprintf(s, sizeof(s)-1, " `sent` <= %ld", stopdate);
      rc += append_string_to_buffer(&query, s);
+
+      where_condition++;
   }


@ -309,87 +334,44 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain
   return rc;
 }

-#if LIBZIP_VERSION_MAJOR >= 1
-void zip_flush(){
-   zip_close(zip);

-   zip = NULL;
-   zip_counter = 0;
-
-   if(!zip_ids) return;
-
-   for(int i=0; i<zip_batch; i++){
-      if(*(zip_ids+i)){
-         char filename[SMALLBUFSIZE];
-         snprintf(filename, sizeof(filename)-1, "%llu.eml", *(zip_ids+i));
-         unlink(filename);
-      }
-   }
-
-   free(zip_ids);
-   zip_ids = NULL;
-}
-#endif
-
-int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg){
+int export_emails_matching_to_query(struct session_data *sdata, struct __data *data, char *s, struct __config *cfg){
   FILE *f;
-   uint64 id, n=0, dir_counter=0;
+   uint64 id, n=0;
   char digest[SMALLBUFSIZE], bodydigest[SMALLBUFSIZE];
   char filename[SMALLBUFSIZE];
-   char export_subdir[SMALLBUFSIZE];
-   struct sql sql;
-   int errorp, rc=0, attachments;
-   unsigned long total_attachments=0;
-
-   if(prepare_sql_statement(sdata, &sql, s) == ERR) return ERR;
+   int rc=0;


-   p_bind_init(&sql);
+   if(prepare_sql_statement(sdata, &(data->stmt_generic), s) == ERR) return ERR;

-   if(p_exec_stmt(sdata, &sql) == ERR) goto ENDE;
+
+   p_bind_init(data);
+
+   if(p_exec_query(sdata, data->stmt_generic, data) == ERR) goto ENDE;



-   p_bind_init(&sql);
+   p_bind_init(data);

-   sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-   sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++;
-   sql.sql[sql.pos] = &digest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(digest)-2; sql.pos++;
-   sql.sql[sql.pos] = &bodydigest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(bodydigest)-2; sql.pos++;
-   sql.sql[sql.pos] = (char *)&attachments; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++;
+   data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+   data->sql[data->pos] = sdata->ttmpfile; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = RND_STR_LEN; data->pos++;
+   data->sql[data->pos] = &digest[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(digest)-2; data->pos++;
+   data->sql[data->pos] = &bodydigest[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(bodydigest)-2; data->pos++;

-   p_store_results(&sql);
+   p_store_results(data->stmt_generic, data);

-   while(p_fetch_results(&sql) == OK){
+   while(p_fetch_results(data->stmt_generic) == OK){

      if(id > 0){

         if(dryrun == 0){

-            if(export_to_stdout){
-               printf("%s", PILEREXPORT_BEGIN_MARK);
-               rc = retrieve_email_from_archive(sdata, stdout, cfg);
-               continue;
-            }
-
-            if(max_files_in_export_dir > 0 && n % max_files_in_export_dir == 0){
-               dir_counter++;
-               snprintf(export_subdir, sizeof(export_subdir)-1, "export-%llu", dir_counter);
-               if(n > 0 && chdir("..")){
-                  p_clean_exit("error chdir(\"..\")", 1);
-               }
-
-               createdir(export_subdir, pwd->pw_uid, pwd->pw_gid, 0700);
-               if(chdir(export_subdir)){
-                  p_clean_exit("error chdir to export-* dir", 1);
-               }
-            }
-
            snprintf(filename, sizeof(filename)-1, "%llu.eml", id);

            f = fopen(filename, "w");
            if(f){
-               rc = retrieve_email_from_archive(sdata, f, cfg);
+               rc = retrieve_email_from_archive(sdata, data, f, cfg);
               fclose(f);

               n++;
@ -401,48 +383,13 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct
               if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0){
                  printf("exported: %10llu\r", n); fflush(stdout);
               }
-               else {
+               else
                  printf("verification FAILED. %s\n", filename);
-                  verification_status = 1;
-               }

-               if(zipfile){
-               #if LIBZIP_VERSION_MAJOR >= 1
-                  // Open zip file if handler is NULL
-                  if(!zip){
-                     zip = zip_open(zipfile, ZIP_CREATE, &errorp);
-                     if(!zip){
-                        printf("error: error creating zip file=%s, error code=%d\n", zipfile, errorp);
-                        return ERR;
-                     }
-                  }
-
-                  if(!zip_ids) zip_ids = (uint64*) calloc(sizeof(uint64), zip_batch);
-
-                  if(!zip_ids){
-                     printf("calloc error for zip_ids\n");
-                     return ERR;
-                  }
-
-                  zip_source_t *zs = zip_source_file(zip, filename, 0, 0);
-                  if(zs && zip_file_add(zip, filename, zs, ZIP_FL_ENC_UTF_8) >= 0){
-                     *(zip_ids+zip_counter) = id;
-                     zip_counter++;
-                  } else {
-                     printf("error adding file %s: %s\n", filename, zip_strerror(zip));
-                     return ERR;
-                  }
-
-                  if(zip_counter == zip_batch){
-                     zip_flush();
-                  }
-               #endif
-               }
            }
            else printf("cannot open: %s\n", filename);
         }
         else {
-            total_attachments += attachments;
            printf("id:%llu\n", id);
         }

@ -450,14 +397,11 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct

   }

-   p_free_results(&sql);
+   p_free_results(data->stmt_generic);

 ENDE:
-   close_prepared_statement(&sql);
+   close_prepared_statement(data->stmt_generic);

-   if(dryrun){
-      printf("attachments: %lu\n", total_attachments);
-   }

   printf("\n");

@ -466,13 +410,14 @@ ENDE:


 int main(int argc, char **argv){
-   int minsize=0, maxsize=0;
+   int c, minsize=0, maxsize=0;
   size_t nmatch=0;
   unsigned long startdate=0, stopdate=0;
   char *configfile=CONFIG_FILE;
   char *to=NULL, *from=NULL, *todomain=NULL, *fromdomain=NULL, *where_condition=NULL;
   struct session_data sdata, sdata2;
-   struct config cfg;
+   struct __data data;
+   struct __config cfg;


   if(regcomp(&regexp, "^([\\+a-z0-9_\\.@\\-]+)$", REG_ICASE | REG_EXTENDED)){
@ -491,7 +436,6 @@ int main(int argc, char **argv){
            {"all",          no_argument,        0,  'A' },
            {"dry-run",      no_argument,        0,  'd' },
            {"dryrun",       no_argument,        0,  'd' },
-            {"stdout",       no_argument,        0,  'o' },
            {"help",         no_argument,        0,  'h' },
            {"version",      no_argument,        0,  'v' },
            {"from",         required_argument,  0,  'f' },
@ -500,10 +444,7 @@ int main(int argc, char **argv){
            {"to-domain",    required_argument,  0,  'R' },
            {"start-date",   required_argument,  0,  'a' },
            {"stop-date",    required_argument,  0,  'b' },
-            {"zip",          required_argument,  0,  'z' },
-            {"zip-batch",    required_argument,  0,  'Z' },
            {"where-condition", required_argument,  0,  'w' },
-            {"max-files",    required_argument,  0,  'D' },
            {"max-matches",  required_argument,  0,  'm' },
            {"index-list",   required_argument,  0,  'i' },
            {0,0,0,0}
@ -511,9 +452,9 @@ int main(int argc, char **argv){

      int option_index = 0;

-      int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:D:oAdhv?", long_options, &option_index);
+      c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?", long_options, &option_index);
 #else
-      int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:D:oAdhv?");
+      c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:Adhv?");
 #endif

      if(c == -1) break;
@ -544,10 +485,7 @@ int main(int argc, char **argv){
                       break;
                    }

-                    if(append_email_to_buffer(&from, optarg)){
-                       printf("error: append_email_to_buffer() for from\n");
-                       return 1;
-                    }
+                    rc = append_email_to_buffer(&from, optarg);

                    break;

@ -558,10 +496,7 @@ int main(int argc, char **argv){
                       break;
                    }

-                    if(append_email_to_buffer(&to, optarg)){
-                       printf("error: append_email_to_buffer() for to\n");
-                       return 1;
-                    }
+                    rc = append_email_to_buffer(&to, optarg);

                    break;

@ -572,12 +507,9 @@ int main(int argc, char **argv){
                       break;
                    }

-                    if(append_email_to_buffer(&fromdomain, optarg)){
-                       printf("error: append_email_to_buffer() for fromdomain\n");
-                       return 1;
-                    }
+                    rc = append_email_to_buffer(&fromdomain, optarg);

-                    break;
+                    break;           

         case 'R' :

@ -586,13 +518,10 @@ int main(int argc, char **argv){
                       break;
                    }

-                    if(append_email_to_buffer(&todomain, optarg)){
-                       printf("error: append_email_to_buffer() for todomain\n");
-                       return 1;
-                    }
-
-                    break;
+                    rc = append_email_to_buffer(&todomain, optarg);

+                    break;                     
+                    
         case 'a' :
                    startdate = convert_time(optarg, 0, 0, 0);
                    break;
@ -614,23 +543,6 @@ int main(int argc, char **argv){
                    index_list = optarg;
                    break;

-         case 'z':  zipfile = optarg;
-                    break;
-
-         case 'Z':  zip_batch = atoi(optarg);
-                    if(zip_batch < 10 || zip_batch > 10000)
-                       zip_batch = 2000;
-                    break;
-
-         case 'D':  max_files_in_export_dir = atoi(optarg);
-                    if(max_files_in_export_dir < 10 || max_files_in_export_dir > 100000)
-                       max_files_in_export_dir = 2000;
-                    break;
-
-         case 'o':
-                    export_to_stdout = 1;
-                    break;
-
         case 'd' :
                    dryrun = 1;
                    break;
@ -658,11 +570,6 @@ int main(int argc, char **argv){

   if(read_key(&cfg)) p_clean_exit(ERR_READING_KEY, 1);

-   if(strlen(cfg.username) > 1){
-      pwd = getpwnam(cfg.username);
-      if(!pwd) __fatal(ERR_NON_EXISTENT_USER);
-   }
-

   init_session_data(&sdata, &cfg);

@ -684,23 +591,19 @@ int main(int argc, char **argv){
         p_clean_exit("cannot connect to 127.0.0.1:9306", 1);
      }

-      export_emails_matching_id_list(&sdata, &sdata2, where_condition, &cfg);
+      export_emails_matching_id_list(&sdata, &sdata2, &data, where_condition, &cfg);

      close_database(&sdata2);
   }
   else {
      if(build_query_from_args(from, to, fromdomain, todomain, minsize, maxsize, startdate, stopdate) > 0) p_clean_exit("malloc problem building query", 1);
-      export_emails_matching_to_query(&sdata, query, &cfg);
+      export_emails_matching_to_query(&sdata, &data, query, &cfg);
      free(query);
   }

   close_database(&sdata);

-   if(zipfile){
-   #if LIBZIP_VERSION_MAJOR >= 1
-      zip_flush();
-   #endif
-   }
-
-   return verification_status;
+   return 0;
 }
+
+
--- a/src/pilerget.c
+++ b/src/pilerget.c
@ -19,7 +19,8 @@
 int main(int argc, char **argv){
   int readkey=1;
   struct session_data sdata;
-   struct config cfg;
+   struct __data data;
+   struct __config cfg;


   if(argc < 2){
@ -47,10 +48,12 @@ int main(int argc, char **argv){

   snprintf(sdata.ttmpfile, SMALLBUFSIZE-1, "%s", argv[1]);
   snprintf(sdata.filename, SMALLBUFSIZE-1, "%s", sdata.ttmpfile);
-   retrieve_email_from_archive(&sdata, stdout, &cfg);
+   retrieve_email_from_archive(&sdata, &data, stdout, &cfg);


   close_database(&sdata);

   return 0;
 }
+
+
--- a/src/pilerimport.c
+++ b/src/pilerimport.c
@ -30,6 +30,9 @@
 extern char *optarg;
 extern int optind;

+int dryrun=0;
+int import_from_gui=0;
+

 void usage(){
   printf("\nusage: pilerimport\n\n");
@ -52,32 +55,23 @@ void usage(){
   printf("    -s <start position>               Start importing POP3 emails from this position\n");
   printf("    -j <failed folder>                Move failed to import emails to this folder\n");
   printf("    -a <recipient>                    Add recipient to the To:/Cc: list\n");
-   printf("    -T <id>                           Update import table at id=<id>\n");
-   printf("    -Z <ms>                           Delay Z milliseconds in between emails being imported\n");
   printf("    -D                                Dry-run, do not import anything\n");
-   printf("    -y                                Read pilerexport data from stdin\n");
   printf("    -o                                Only download emails for POP3/IMAP import\n");
   printf("    -r                                Remove imported emails\n");
   printf("    -q                                Quiet mode\n");
-   printf("    -A <timestamp>                    Import emails sent after this timestamp\n");
-   printf("    -B <timestamp>                    Import emails sent before this timestamp\n");

   exit(0);
 }


 int main(int argc, char **argv){
-   int i, n_mbox=0, read_from_pilerexport=0;
-   char *configfile=CONFIG_FILE, *mbox[MBOX_ARGS], *directory=NULL;
-   char puf[SMALLBUFSIZE], *imapserver=NULL, *pop3server=NULL;
+   int i, c, rc=0, n_mbox=0, tot_msgs=0, port=143;
+   char *configfile=CONFIG_FILE, *emlfile=NULL, *mboxdir=NULL, *mbox[MBOX_ARGS], *directory=NULL;
+   char *imapserver=NULL, *pop3server=NULL, *username=NULL, *password=NULL, *skiplist=SKIPLIST, *folder=NULL, *folder_imap=NULL;
   struct session_data sdata;
-   struct config cfg;
-   struct data data;
+   struct __config cfg;
+   struct __data data;
   struct import import;
-   struct net net;
-   struct counters counters;
-
-   bzero(&counters, sizeof(counters));

   for(i=0; i<MBOX_ARGS; i++) mbox[i] = NULL;

@ -93,30 +87,10 @@ int main(int argc, char **argv){
   import.extra_recipient = import.move_folder = import.failed_folder = NULL;
   import.start_position = 1;
   import.download_only = 0;
-   import.dryrun = 0;
-   import.port = 143;
-   import.server = NULL;
-   import.username = NULL;
-   import.password = NULL;
-   import.database = NULL;
-   import.skiplist = SKIPLIST;
-   import.folder_imap = NULL;
-   memset(import.filename, 0, SMALLBUFSIZE);
-   import.mboxdir = NULL;
-   import.tot_msgs = 0;
-   import.table_id = 0;
-   import.folder = NULL;
-   import.delay = 0;
-   import.after = 0;
-   import.before = 0;
+   import.timeout = 30;

   data.import = &import;

-   net.socket = -1;
-   net.timeout = 30;
-
-   data.net = &net;
-
   inithash(data.mydomains);
   initrules(data.archiving_rules);
   initrules(data.retention_rules);
@ -144,17 +118,13 @@ int main(int argc, char **argv){
            {"batch-limit",  required_argument,  0,  'b' },
            {"timeout",      required_argument,  0,  't' },
            {"start-position",required_argument,  0,  's' },
-            {"table-id",     required_argument,  0,  'T' },
-            {"delay",        required_argument,  0,  'Z' },
            {"quiet",        no_argument,        0,  'q' },
-            {"recursive",    no_argument,        0,  'R' },
+            {"recursive",    required_argument,  0,  'R' },
            {"remove-after-import",no_argument,  0,  'r' },
            {"failed-folder",  required_argument,  0,  'j' },
            {"move-folder",  required_argument,  0,  'g' },
-            {"after",        required_argument,  0,  'A' },
-            {"before",       required_argument,  0,  'B' },
            {"only-download",no_argument,        0,  'o' },
-            {"read-from-export",no_argument,     0,  'y' },
+            {"gui-import",   no_argument,        0,  'G' },
            {"dry-run",      no_argument,        0,  'D' },
            {"help",         no_argument,        0,  'h' },
            {0,0,0,0}
@ -162,9 +132,9 @@ int main(int argc, char **argv){

      int option_index = 0;

-      int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:A:B:yDRroqh?", long_options, &option_index);
+      c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:GDRroqh?", long_options, &option_index);
 #else
-      int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:A:B:yDRroqh?");
+      c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:GDRroqh?");
 #endif

      if(c == -1) break;
@ -176,7 +146,7 @@ int main(int argc, char **argv){
                    break;

         case 'e' :
-                    snprintf(data.import->filename, SMALLBUFSIZE-1, "%s", optarg);
+                    emlfile = optarg;
                    break;

         case 'd' :
@ -193,42 +163,40 @@ int main(int argc, char **argv){
                    break;

         case 'M' :
-                    data.import->mboxdir = optarg;
+                    mboxdir = optarg;
                    break;

         case 'i' :
                    imapserver = optarg;
-                    data.import->server = optarg;
                    break;

         case 'K' :
                    pop3server = optarg;
-                    data.import->server = optarg;
-                    if(data.import->port == 143) data.import->port = 110;
+                    if(port == 143) port = 110;
                    break;

         case 'u' :
-                    data.import->username = optarg;
+                    username = optarg;
                    break;

         case 'p' :
-                    data.import->password = optarg;
+                    password = optarg;
                    break;

         case 'P' :
-                    data.import->port = atoi(optarg);
+                    port = atoi(optarg);
                    break;

         case 'x' :
-                    data.import->skiplist = optarg;
+                    skiplist = optarg;
                    break;

         case 'F' :
-                    data.import->folder = optarg;
+                    folder = optarg;
                    break;

         case 'f' :
-                    data.import->folder_imap = optarg;
+                    folder_imap = optarg;
                    break;

         case 'R' :
@ -249,7 +217,7 @@ int main(int argc, char **argv){

         case 'o' :
                    data.import->download_only = 1;
-                    data.import->dryrun = 1;
+                    dryrun = 1;
                    break;

         case 'b' :
@ -270,59 +238,35 @@ int main(int argc, char **argv){
                    break;

         case 'a' :
-                    snprintf(puf, sizeof(puf)-1, "%s ", optarg);
-                    data.import->extra_recipient = puf;
+                    data.import->extra_recipient = optarg;
                    break;

-         case 'T' :
-                    if(atoi(optarg) < 1){
-                       printf("invalid import table id: %s\n", optarg);
-                       return -1;
-                    }
-
-                    data.import->table_id = atoi(optarg);
-                    break;
-
-         case 'Z' :
-                    if(atoi(optarg) < 1){
-                       printf("invalid delay value: %s\n", optarg);
-                       return -1;
-                    }
-
-                    data.import->delay = atoi(optarg);
-                    break;
-
-         case 'y' :
-                    read_from_pilerexport = 1;
+         case 'G' :
+                    import_from_gui = 1;
                    break;

         case 'D' :
-                    data.import->dryrun = 1;
+                    dryrun = 1;
                    break;

         case 'q' :
                    data.quiet = 1;
                    break;

-         case 'A' : data.import->after = atol(optarg);
-                    break;
-
-         case 'B' : data.import->before = atol(optarg);
-                    break;
-
         case 'h' :
         case '?' :
                    usage();
                    break;


-         default  :
+         default  : 
                    break;
       }
   }


-   if(!mbox[0] && !data.import->mboxdir && !data.import->filename[0] && !directory && !imapserver && !pop3server && !read_from_pilerexport) usage();
+
+   if(!mbox[0] && !mboxdir && !emlfile && !directory && !imapserver && !pop3server && import_from_gui == 0) usage();

   if(data.import->failed_folder && !can_i_write_directory(data.import->failed_folder)){
      printf("cannot write failed directory '%s'\n", data.import->failed_folder);
@ -333,9 +277,7 @@ int main(int argc, char **argv){

   cfg = read_config(configfile);

-   memset(cfg.security_header, 0, MAXVAL);
-
-   if((data.recursive_folder_names == 1 || data.import->folder) && cfg.enable_folders == 0){
+   if((data.recursive_folder_names == 1 || folder) && cfg.enable_folders == 0){
      printf("please set enable_folders=1 in piler.conf to use the folder options\n");
      return ERR;
   }
@ -343,9 +285,6 @@ int main(int argc, char **argv){
   /* make sure we don't discard messages without a valid Message-Id when importing manually */
   cfg.archive_emails_not_having_message_id = 1;

-   /* The mmap_dedup_test feature is expected to work with the piler daemon only */
-   cfg.mmap_dedup_test = 0;
-
   if(read_key(&cfg)){
      printf("%s\n", ERR_READING_KEY);
      return ERR;
@ -356,7 +295,6 @@ int main(int argc, char **argv){

   if(open_database(&sdata, &cfg) == ERR) return 0;

-   if(cfg.rtindex && open_sphx(&sdata, &cfg) == ERR) return 0;

   setlocale(LC_CTYPE, cfg.locale);

@ -366,39 +304,39 @@ int main(int argc, char **argv){
   memcached_init(&(data.memc), cfg.memcached_servers, 11211);
 #endif

-   if(data.import->folder){
-      data.folder = get_folder_id(&sdata, data.import->folder, 0);
+   if(folder){
+      data.folder = get_folder_id(&sdata, &data, folder, 0);

      if(data.folder == ERR_FOLDER){
-         data.folder = add_new_folder(&sdata, data.import->folder, 0);
+         data.folder = add_new_folder(&sdata, &data, folder, 0);
      }

      if(data.folder == ERR_FOLDER){
-         printf("ERROR: cannot get/add folder '%s'\n", data.import->folder);
+         printf("error: cannot get/add folder '%s'\n", folder);
         close_database(&sdata);
         return 0;
      }

   }

-   load_rules(&sdata, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
-   load_rules(&sdata, data.retention_rules, SQL_RETENTION_RULE_TABLE);
-   load_rules(&sdata, data.folder_rules, SQL_FOLDER_RULE_TABLE);
+   load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
+   load_rules(&sdata, &data, data.retention_rules, SQL_RETENTION_RULE_TABLE);
+   load_rules(&sdata, &data, data.folder_rules, SQL_FOLDER_RULE_TABLE);

   load_mydomains(&sdata, &data, &cfg);

-   if(data.import->filename[0] != '\0') import_message(&sdata, &data, &counters, &cfg);
+   if(emlfile) rc = import_message(emlfile, &sdata, &data, &cfg);

   if(mbox[0]){
      for(i=0; i<n_mbox; i++){
-         import_from_mailbox(mbox[i], &sdata, &data, &counters, &cfg);
+         rc = import_from_mailbox(mbox[i], &sdata, &data, &cfg);
      }
   }
-   if(data.import->mboxdir) import_mbox_from_dir(data.import->mboxdir, &sdata, &data, &counters, &cfg);
-   if(directory) import_from_maildir(&sdata, &data, directory, &counters, &cfg);
-   if(imapserver) import_from_imap_server(&sdata, &data, &counters, &cfg);
-   if(pop3server) import_from_pop3_server(&sdata, &data, &counters, &cfg);
-   if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &counters, &cfg);
+   if(mboxdir) rc = import_mbox_from_dir(mboxdir, &sdata, &data, &tot_msgs, &cfg);
+   if(directory) rc = import_from_maildir(directory, &sdata, &data, &tot_msgs, &cfg);
+   if(imapserver && username && password) rc = import_from_imap_server(imapserver, username, password, port, &sdata, &data, folder_imap, skiplist, dryrun, &cfg);
+   if(pop3server && username && password) rc = import_from_pop3_server(pop3server, username, password, port, &sdata, &data, dryrun, &cfg);
+   if(import_from_gui == 1) rc = read_gui_import_data(&sdata, &data, folder_imap, skiplist, dryrun, &cfg);

   clearrules(data.archiving_rules);
   clearrules(data.retention_rules);
@ -406,15 +344,11 @@ int main(int argc, char **argv){

   clearhash(data.mydomains);

-   update_counters(&sdata, &data, &counters, &cfg);
-
-   syslog(LOG_PRIORITY, "server=%s, user=%s, directory=%s, imported=%lld, duplicated=%lld, discarded=%lld", data.import->server, data.import->username, directory, counters.c_rcvd, counters.c_duplicate, counters.c_ignore);
-
   close_database(&sdata);

-   if(cfg.rtindex) close_sphx(&sdata);
-
   if(data.quiet == 0) printf("\n");

-   return 0;
+   return rc;
 }
+
+
--- a/src/pilerpurge.c
+++ b/src/pilerpurge.c
@ -0,0 +1,439 @@
+/*
+ * pilerpurge.c, SJ
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+extern char *optarg;
+extern int optind;
+
+int dryrun = 0;
+unsigned long purged_size=0;
+
+#define NUMBER_OF_ATTACHMENTS_TO_REMOVE_IN_ONE_ROUND 100
+
+#define SQL_STMT_SELECT_PURGE_FROM_OPTION_TABLE "SELECT `value` FROM `" SQL_OPTION_TABLE "` WHERE `key`='enable_purge'"
+#define SQL_STMT_DELETE_FROM_META_TABLE "UPDATE `" SQL_METADATA_TABLE "` SET `deleted`=1 WHERE `id` IN ("
+#define SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID "UPDATE `" SQL_METADATA_TABLE "` SET `deleted`=1 WHERE `piler_id` IN ('"
+#define SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS "SELECT `piler_id`, `attachment_id`, `i` FROM `" SQL_ATTACHMENTS_VIEW "` WHERE `refcount`=0 AND `piler_id` IN ('"
+#define SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE "DELETE FROM `" SQL_ATTACHMENT_TABLE "` WHERE `id` IN ("
+
+
+int is_purge_allowed(struct session_data *sdata, struct __data *data){
+   int rc=0;
+
+   if(prepare_sql_statement(sdata, &(data->stmt_generic), SQL_STMT_SELECT_PURGE_FROM_OPTION_TABLE) == ERR) return rc;
+
+
+   p_bind_init(data);
+
+   if(p_exec_query(sdata, data->stmt_generic, data) == OK){
+
+      p_bind_init(data);
+
+      data->sql[data->pos] = (char *)&rc; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
+
+      p_store_results(data->stmt_generic, data);
+      p_fetch_results(data->stmt_generic);
+      p_free_results(data->stmt_generic);
+   }
+
+   close_prepared_statement(data->stmt_generic);
+
+   return rc;
+}
+
+
+int remove_message_frame_files(char *s, char *update_meta_sql, struct session_data *sdata, struct __config *cfg){
+   char *p, puf[SMALLBUFSIZE], filename[SMALLBUFSIZE];
+   int n=0, result;
+#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
+   struct stat st;
+#endif
+
+   p = s;
+   do {
+      p = split(p, ' ', puf, sizeof(puf)-1, &result);
+
+      if(strlen(puf) == RND_STR_LEN){
+         snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, puf[8], puf[9], puf[10], puf[RND_STR_LEN-4], puf[RND_STR_LEN-3], puf[RND_STR_LEN-2], puf[RND_STR_LEN-1], puf);
+
+      #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
+         if(stat(filename, &st)){
+            snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, puf[RND_STR_LEN-6], puf[RND_STR_LEN-5], puf[RND_STR_LEN-4], puf[RND_STR_LEN-3], puf[RND_STR_LEN-2], puf[RND_STR_LEN-1], puf);
+         }
+      #endif
+
+         if(dryrun == 1){
+            n++;
+            printf("removing messagefile: %s\n", filename);
+         }
+         else {
+            if(unlink(filename) == 0) n++;
+         }
+
+      }
+   } while(p);
+
+
+   update_meta_sql[strlen(update_meta_sql)-1] = ')';
+
+   if(dryrun == 1){
+      printf("update metadata query: *%s*\n\n", update_meta_sql);
+   } else {
+      p_query(sdata, update_meta_sql);
+   }
+
+
+   return n;
+}
+
+
+int remove_attachments(char *in, struct session_data *sdata, struct __data *data, struct __config *cfg){
+   char filename[SMALLBUFSIZE];
+   char *a, buf[NUMBER_OF_ATTACHMENTS_TO_REMOVE_IN_ONE_ROUND*(RND_STR_LEN+1)+10], update_meta_sql[strlen(SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID)+NUMBER_OF_ATTACHMENTS_TO_REMOVE_IN_ONE_ROUND*(RND_STR_LEN+3)+10], delete_attachment_stmt[MAXBUFSIZE];
+   char piler_id[SMALLBUFSIZE], i[BUFLEN];
+   int n=0, m=0, len, attachment_id=0, piler_id_len;
+   unsigned int blen=0, ulen=0, dlen=0;
+#ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
+   struct stat st;
+#endif
+
+   if(strlen(in) < 10) return 0;
+
+   len = strlen(SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS) + strlen(in) + 2;
+
+   a = malloc(len);
+   if(!a) return 0;
+
+   memset(a, 0, len);
+
+   in[strlen(in)-2] = ')';
+   in[strlen(in)-1] = '\0';
+   snprintf(a, len-1, "%s%s", SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS, in);
+
+   if(prepare_sql_statement(sdata, &(data->stmt_select_non_referenced_attachments), a) == ERR){ free(a); return n; }
+
+   if(dryrun == 1) printf("attachment select sql: *%s*\n\n", a);
+
+   memset(buf, 0, sizeof(buf));
+   memset(update_meta_sql, 0, sizeof(update_meta_sql));
+   memset(delete_attachment_stmt, 0, sizeof(delete_attachment_stmt));
+
+   snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
+   ulen = strlen(SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
+
+   snprintf(delete_attachment_stmt, sizeof(delete_attachment_stmt)-1, "%s", SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
+   dlen = strlen(SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
+
+   p_bind_init(data);
+   if(p_exec_query(sdata, data->stmt_select_non_referenced_attachments, data) == ERR) goto ENDE;
+
+
+   p_bind_init(data);
+
+   data->sql[data->pos] = &piler_id[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(piler_id)-2; data->pos++;
+   data->sql[data->pos] = (char *)&attachment_id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
+   data->sql[data->pos] = &i[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(i)-2; data->pos++;
+
+   p_store_results(data->stmt_select_non_referenced_attachments, data);
+
+   while(p_fetch_results(data->stmt_select_non_referenced_attachments) == OK){
+
+      piler_id_len = strlen(piler_id);
+
+      if(piler_id_len != RND_STR_LEN || attachment_id <= 0){
+         printf("invalid piler_id: '%s.a%d'\n", piler_id, attachment_id);
+         continue;
+      }
+
+      snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, piler_id[8], piler_id[9], piler_id[10], piler_id[RND_STR_LEN-4], piler_id[RND_STR_LEN-3], piler_id[RND_STR_LEN-2], piler_id[RND_STR_LEN-1], piler_id, attachment_id);
+   #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
+      if(stat(filename, &st)){
+         snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, piler_id[RND_STR_LEN-6], piler_id[RND_STR_LEN-5], piler_id[RND_STR_LEN-4], piler_id[RND_STR_LEN-3], piler_id[RND_STR_LEN-2], piler_id[RND_STR_LEN-1], piler_id, attachment_id);
+      }
+   #endif
+
+      if(dryrun == 1){
+         printf("removing attachment: *%s*\n", filename);
+      } else {
+         unlink(filename);
+      }
+
+
+      if(strlen(i) > 0){
+         if(dlen > sizeof(delete_attachment_stmt) - 200){
+
+            delete_attachment_stmt[dlen-1] = ')';
+            if(dryrun == 1){
+               printf("delete sql: *%s*\n", delete_attachment_stmt);
+            } else {
+               p_query(sdata, delete_attachment_stmt);
+            }
+
+            memset(delete_attachment_stmt, 0, sizeof(delete_attachment_stmt));
+            snprintf(delete_attachment_stmt, sizeof(delete_attachment_stmt)-1, "%s", SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
+            dlen = strlen(SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
+         }
+
+         memcpy(&delete_attachment_stmt[dlen], i, strlen(i)); dlen += strlen(i);
+         memcpy(&delete_attachment_stmt[dlen], ",", 1); dlen++;
+      }
+
+
+      if(attachment_id == 1){
+
+         m++;
+
+         if(m >= NUMBER_OF_ATTACHMENTS_TO_REMOVE_IN_ONE_ROUND){
+            if(ulen > strlen(SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID)+10){
+               update_meta_sql[ulen-2] = ')';
+               update_meta_sql[ulen-1] = '\0';
+            }
+
+            n += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
+
+            memset(buf, 0, sizeof(buf));
+            memset(update_meta_sql, 0, sizeof(update_meta_sql));
+
+            snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
+
+            ulen = strlen(SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
+            blen = 0;
+
+            m = 0;
+         }
+
+         memcpy(&buf[blen], piler_id, piler_id_len); blen += piler_id_len;
+         memcpy(&buf[blen], " ", 1); blen++;
+
+         memcpy(&update_meta_sql[ulen], piler_id, piler_id_len); ulen += piler_id_len;
+         memcpy(&update_meta_sql[ulen], "','", 3); ulen += 3;
+      }
+
+   }
+
+   p_free_results(data->stmt_select_non_referenced_attachments);
+
+   if(ulen > strlen(SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID)+10){
+
+      update_meta_sql[ulen-2] = ')';
+      update_meta_sql[ulen-1] = '\0';
+
+      n += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
+   }
+
+   if(dlen > strlen(SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE)){
+      delete_attachment_stmt[dlen-1] = ')';
+      if(dryrun == 1){
+         printf("delete sql: *%s*\n", delete_attachment_stmt);
+      } else {
+         p_query(sdata, delete_attachment_stmt);
+      }
+
+   }
+
+ENDE:
+   free(a);
+
+   close_prepared_statement(data->stmt_select_non_referenced_attachments);
+
+   return n;
+}
+
+
+int purge_messages_round1(struct session_data *sdata, struct __data *data, char *attachment_condition, struct __config *cfg){
+   int purged=0, size;
+   unsigned int blen=0, ulen=0;
+   char id[BUFLEN], s[SMALLBUFSIZE], buf[MAXBUFSIZE], update_meta_sql[MAXBUFSIZE];
+
+   memset(buf, 0, sizeof(buf));
+   memset(update_meta_sql, 0, sizeof(update_meta_sql));
+
+   snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE);
+   ulen = strlen(SQL_STMT_DELETE_FROM_META_TABLE);
+
+   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `size` FROM `%s` WHERE `deleted`=0 AND `retained` < %ld AND %s AND id NOT IN (SELECT id FROM `%s` WHERE `to` IN (SELECT email FROM `%s`)) AND id NOT IN (SELECT id FROM `%s` WHERE `from` IN (SELECT email FROM `%s`))", SQL_METADATA_TABLE, sdata->now, attachment_condition, SQL_RECIPIENT_TABLE, SQL_LEGAL_HOLD_TABLE, SQL_METADATA_TABLE, SQL_LEGAL_HOLD_TABLE);
+
+   if(dryrun == 1) printf("purge sql: *%s*\n", s);
+
+   if(prepare_sql_statement(sdata, &(data->stmt_select_from_meta_table), s) == ERR) return purged;
+
+   p_bind_init(data);
+
+   if(p_exec_query(sdata, data->stmt_select_from_meta_table, data) == OK){
+
+      p_bind_init(data);
+
+      data->sql[data->pos] = &id[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(id)-2; data->pos++;
+      data->sql[data->pos] = &s[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s)-2; data->pos++;
+      data->sql[data->pos] = (char *)&size; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
+
+      p_store_results(data->stmt_select_from_meta_table, data);
+
+      while(p_fetch_results(data->stmt_select_from_meta_table) == OK){
+
+         memcpy(&update_meta_sql[ulen], id, strlen(id)); ulen += strlen(id);
+         memcpy(&update_meta_sql[ulen], ",", 1); ulen++;
+
+         purged_size += size;
+
+         if(blen >= sizeof(buf)-RND_STR_LEN-2-1){
+
+            purged += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
+
+            memset(buf, 0, sizeof(buf));
+            memset(update_meta_sql, 0, sizeof(update_meta_sql));
+
+            snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE);
+
+            blen = 0;
+            ulen = strlen(SQL_STMT_DELETE_FROM_META_TABLE);
+         }
+
+         memcpy(&buf[blen], s, strlen(s)); blen += strlen(s);
+         memcpy(&buf[blen], " ", 1); blen++;
+
+      }
+
+      p_free_results(data->stmt_select_from_meta_table);
+
+      if(strlen(buf) > 5 && strlen(update_meta_sql) > strlen(SQL_STMT_DELETE_FROM_META_TABLE)+5){
+         purged += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
+      }
+
+   }
+
+   close_prepared_statement(data->stmt_select_from_meta_table);
+
+   return purged;
+}
+
+
+int purge_messages_with_attachments(struct session_data *sdata, struct __data *data, struct __config *cfg){
+   int purged=0, size;
+   unsigned int idlist_len=0;
+   char s[SMALLBUFSIZE], idlist[MAXBUFSIZE];
+
+   memset(idlist, 0, sizeof(idlist));
+
+   snprintf(s, sizeof(s)-1, "SELECT `piler_id`, `size` FROM `%s` WHERE `deleted`=0 AND `retained` < %ld AND attachments > 0", SQL_METADATA_TABLE, sdata->now);
+
+   if(dryrun == 1) printf("purge sql: *%s*\n", s);
+
+   if(prepare_sql_statement(sdata, &(data->stmt_select_from_meta_table), s) == ERR) return purged;
+
+   p_bind_init(data);
+   if(p_exec_query(sdata, data->stmt_select_from_meta_table, data) == OK){
+
+      p_bind_init(data);
+
+      data->sql[data->pos] = &s[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s)-2; data->pos++;
+      data->sql[data->pos] = (char *)&size; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
+
+      p_store_results(data->stmt_select_from_meta_table, data);
+
+      while(p_fetch_results(data->stmt_select_from_meta_table) == OK){
+         memcpy(&idlist[idlist_len], s, strlen(s)); idlist_len += strlen(s);
+         memcpy(&idlist[idlist_len], "','", 3); idlist_len += 3;
+
+         purged_size += size;
+
+         if(idlist_len >= sizeof(idlist)-2*RND_STR_LEN){
+            purged += remove_attachments(idlist, sdata, data, cfg);
+
+            memset(idlist, 0, sizeof(idlist));
+            idlist_len = 0;
+         }
+      }
+
+      p_free_results(data->stmt_select_from_meta_table);
+
+      if(idlist_len > 5){
+         purged += remove_attachments(idlist, sdata, data, cfg);
+      }
+
+   }
+
+   close_prepared_statement(data->stmt_select_from_meta_table);
+
+   return purged;
+}
+
+
+int main(int argc, char **argv){
+   int i, purged=0;
+   char *configfile=CONFIG_FILE, buf[SMALLBUFSIZE];
+   struct session_data sdata;
+   struct __data data;
+   struct __config cfg;
+
+
+   while((i = getopt(argc, argv, "c:dh?")) > 0){
+       switch(i){
+
+         case 'c' :
+                    configfile = optarg;
+                    break;
+
+         case 'd' :
+                    dryrun = 1;
+                    break;
+
+         case 'h' :
+         case '?' :
+         default  : 
+                    break;
+       }
+   }
+
+
+
+   (void) openlog("pilerpurge", LOG_PID, LOG_MAIL);
+
+   cfg = read_config(configfile);
+
+
+   if(open_database(&sdata, &cfg) == ERR) return 0;
+
+
+   setlocale(LC_CTYPE, cfg.locale);
+
+   init_session_data(&sdata, &cfg);
+
+   i = is_purge_allowed(&sdata, &data);
+   if(i == 1){
+      purged += purge_messages_round1(&sdata, &data, "attachments=0", &cfg);
+      purged += purge_messages_with_attachments(&sdata, &data, &cfg);
+      purged += purge_messages_round1(&sdata, &data, "attachments > 0", &cfg);
+
+      syslog(LOG_INFO, "purged %d messages, %ld bytes", purged, purged_size);
+   }
+   else printf("purge is not allowed by configuration, enable_purge=%d\n", i);
+
+   
+   if(purged_size > 100){
+      snprintf(buf, sizeof(buf)-1, "UPDATE `%s` SET size = size - %ld", SQL_COUNTER_TABLE, purged_size);
+
+      if(dryrun == 0) p_query(&sdata, buf);
+   }
+
+
+   close_database(&sdata);
+
+   return 0;
+}
+
+
--- a/src/pop3.c
+++ b/src/pop3.c
@ -23,26 +23,71 @@
 #include <piler.h>


-int connect_to_pop3_server(struct data *data){
-   char buf[MAXBUFSIZE];
+void update_import_job_stat(struct session_data *sdata, struct __data *data);

-   if(data->net->use_ssl == 1){
-      init_ssl_to_server(data);
+
+int is_last_complete_pop3_packet(char *s, int len){
+
+   if(*(s+len-5) == '\r' && *(s+len-4) == '\n' && *(s+len-3) == '.' && *(s+len-2) == '\r' && *(s+len-1) == '\n'){
+      return 1;
   }

-   recvtimeoutssl(data->net, buf, sizeof(buf));
+   return 0;
+}


-   snprintf(buf, sizeof(buf)-1, "USER %s\r\n", data->import->username);
-
-   write1(data->net, buf, strlen(buf));
-   recvtimeoutssl(data->net, buf, sizeof(buf));
+int connect_to_pop3_server(int sd, char *username, char *password, struct __data *data, int use_ssl){
+   int n;
+   char buf[MAXBUFSIZE];
+   X509* server_cert;
+   char *str;


-   snprintf(buf, sizeof(buf)-1, "PASS %s\r\n", data->import->password);
+   if(use_ssl == 1){

-   write1(data->net, buf, strlen(buf));
-   recvtimeoutssl(data->net, buf, sizeof(buf));
+      SSL_library_init();
+      SSL_load_error_strings();
+
+      data->ctx = SSL_CTX_new(TLSv1_client_method());
+      CHK_NULL(data->ctx, "internal SSL error");
+
+      data->ssl = SSL_new(data->ctx);
+      CHK_NULL(data->ssl, "internal ssl error");
+
+      SSL_set_fd(data->ssl, sd);
+      n = SSL_connect(data->ssl);
+      CHK_SSL(n, "internal ssl error");
+
+      printf("Cipher: %s\n", SSL_get_cipher(data->ssl));
+
+      server_cert = SSL_get_peer_certificate(data->ssl);
+      CHK_NULL(server_cert, "server cert error");
+
+      str = X509_NAME_oneline(X509_get_subject_name(server_cert), 0, 0);
+      CHK_NULL(str, "error in server cert");
+      OPENSSL_free(str);
+
+      str = X509_NAME_oneline(X509_get_issuer_name(server_cert), 0, 0);
+      CHK_NULL(str, "error in server cert");
+      OPENSSL_free(str);
+
+      X509_free(server_cert);
+   }
+
+
+   recvtimeoutssl(sd, buf, sizeof(buf), data->import->timeout, use_ssl, data->ssl);
+
+
+   snprintf(buf, sizeof(buf)-1, "USER %s\r\n", username);
+
+   write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+   recvtimeoutssl(sd, buf, sizeof(buf), data->import->timeout, use_ssl, data->ssl);
+
+
+   snprintf(buf, sizeof(buf)-1, "PASS %s\r\n", password);
+
+   write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+   recvtimeoutssl(sd, buf, sizeof(buf), data->import->timeout, use_ssl, data->ssl);

   if(strncmp(buf, "+OK", 3) == 0) return OK;

@ -52,154 +97,123 @@ int connect_to_pop3_server(struct data *data){
 }


-void get_number_of_total_messages(struct data *data){
-   char buf[MAXBUFSIZE];
+int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg){
+   int i=0, rc=ERR, n, pos, readlen, fd, lastpos, nreads;
+   char *p, buf[MAXBUFSIZE], filename[SMALLBUFSIZE];
+   char aggrbuf[3*MAXBUFSIZE];

+   data->import->processed_messages = 0;
   data->import->total_messages = 0;

   snprintf(buf, sizeof(buf)-1, "STAT\r\n");
-   write1(data->net, buf, strlen(buf));
+   write1(sd, buf, strlen(buf), use_ssl, data->ssl);

-   recvtimeoutssl(data->net, buf, sizeof(buf));
+   recvtimeoutssl(sd, buf, sizeof(buf), data->import->timeout, use_ssl, data->ssl);

   if(strncmp(buf, "+OK ", 4) == 0){
-      char *p = strchr(&buf[4], ' ');
+      p = strchr(&buf[4], ' ');
      if(p){
         *p = '\0';
         data->import->total_messages = atoi(&buf[4]);
      }
   }
-   else {
-      printf("ERROR: '%s'", buf);
-   }
-}
+   else return ERR;


-int pop3_download_email(struct data *data, int i){
-   char *p, buf[MAXBUFSIZE], savedbuf[MAXBUFSIZE], copybuf[2*MAXBUFSIZE];
-
-   data->import->processed_messages++;
-
-   snprintf(data->import->filename, SMALLBUFSIZE-1, "pop3-tmp-%d-%d.txt", getpid(), i);
-   unlink(data->import->filename);
-
-   int fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
-   if(fd == -1){
-      printf("cannot open: %s\n", data->import->filename);
-      return ERR;
-   }
-
-   memset(savedbuf, 0, sizeof(savedbuf));
-
-   snprintf(buf, sizeof(buf)-1, "RETR %d\r\n", i);
-   write1(data->net, buf, strlen(buf));
-
-   int nlines = 0;
-   int endofmessage = 0;
-   int savedlen = 0;
-   int n = 0;
-
-   while((n = recvtimeoutssl(data->net, buf, sizeof(buf))) > 0){
-      int remaininglen = n;
-
-      if(savedlen){
-         memset(copybuf, 0, sizeof(copybuf));
-         memcpy(copybuf, savedbuf, savedlen);
-         memcpy(&copybuf[savedlen], buf, n);
-
-         remaininglen += savedlen;
-
-         savedlen = 0;
-         memset(savedbuf, 0, sizeof(savedbuf));
-
-         p = &copybuf[0];
-      } else {
-         p = &buf[0];
-      }
-
-      int puflen=0;
-      int rc=OK;
-      int nullbyte=0;
-
-      do {
-         char puf[MAXBUFSIZE];
-
-         puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf)-1, &rc, &nullbyte);
-         remaininglen -= puflen;
-         nlines++;
-
-         if(nlines == 1){
-            if(strncmp(puf, "+OK", 3)){
-               printf("error: %s", puf);
-               return ERR;
-            }
-         } else {
-            if(puf[puflen-1] == '\n'){
-               if(puflen == 3 && puf[0] == '.' && puf[1] == '\r' && puf[2] == '\n'){
-                  endofmessage = 1;
-                  break;
-               }
-
-               int dotstuff = 0;
-               if(puf[0] == '.' && puf[1] != '\r' && puf[1] != '\n') dotstuff = 1;
-
-               if(write(fd, &puf[dotstuff], puflen-dotstuff) == -1) printf("ERROR: writing to fd\n");
-
-            } else if(puflen > 0) {
-               savedlen = puflen;
-               snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
-            }
-         }
-
-         p += puflen;
-
-      } while(puflen > 0);
-
-      if(endofmessage){
-         break;
-      }
-   }
-
-   close(fd);
-
-   return OK;
-}
-
-
-void pop3_delete_message(struct data *data, int i){
-   char buf[SMALLBUFSIZE];
-
-   snprintf(buf, sizeof(buf)-1, "DELE %d\r\n", i);
-   write1(data->net, buf, strlen(buf));
-   recvtimeoutssl(data->net, buf, sizeof(buf));
-}
-
-
-void process_pop3_emails(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg){
-   char buf[MAXBUFSIZE];
-
-   data->import->processed_messages = 0;
-
-   get_number_of_total_messages(data);
-
   if(data->quiet == 0) printf("found %d messages\n", data->import->total_messages);

-   if(data->import->total_messages <= 0) return;
+   if(data->import->total_messages <= 0) return OK;

-   for(int i=data->import->start_position; i<=data->import->total_messages; i++){
-      if(pop3_download_email(data, i) == OK){
-         if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/data->import->total_messages); fflush(stdout); }
+   for(i=data->import->start_position; i<=data->import->total_messages; i++){
+      data->import->processed_messages++;
+      if(data->quiet == 0){ printf("processed: %7d [%3d%%]\r", data->import->processed_messages, 100*i/data->import->total_messages); fflush(stdout); }

-         if(data->import->dryrun == 0){
-            int rc = import_message(sdata, data, counters, cfg);

-            if(data->import->remove_after_import == 1 && rc == OK){
-               pop3_delete_message(data, i);
-            }
-         }
+      snprintf(buf, sizeof(buf)-1, "RETR %d\r\n", i);
+
+      snprintf(filename, sizeof(filename)-1, "pop3-tmp-%d-%d.txt", getpid(), i);
+      unlink(filename);
+
+      fd = open(filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
+      if(fd == -1){
+         printf("cannot open: %s\n", filename);
+         return rc;
      }

-      if(data->import->download_only == 0) unlink(data->import->filename);
+      write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+
+      readlen = 0;
+      pos = 0;
+      nreads = 0;
+
+      memset(aggrbuf, 0, sizeof(aggrbuf));
+      lastpos = 0;
+
+
+      while((n = recvtimeoutssl(sd, buf, sizeof(buf), data->import->timeout, use_ssl, data->ssl)) > 0){
+         nreads++;
+         readlen += n;
+
+         if(nreads == 1){
+
+            if(strncmp(buf, "+OK", 3) == 0){
+               p = strchr(&buf[3], '\n');
+               if(p){
+                  *p = '\0';
+                  pos = strlen(buf)+1;
+                  *p = '\n';
+               }
+            }
+            else { printf("error: %s", buf); return ERR; }
+
+         }
+
+         if(lastpos + 1 + n < sizeof(aggrbuf)){
+
+            if(nreads == 1){
+               memcpy(aggrbuf+lastpos, buf+pos, n-pos);
+               lastpos += n-pos;
+            }
+            else {
+               memcpy(aggrbuf+lastpos, buf, n);
+               lastpos += n;
+            }
+         }
+         else {
+            write(fd, aggrbuf, sizeof(buf));
+
+            memmove(aggrbuf, aggrbuf+sizeof(buf), lastpos-sizeof(buf));
+            lastpos -= sizeof(buf);
+
+            memcpy(aggrbuf+lastpos, buf, n);
+            lastpos += n;
+         }
+
+         if(is_last_complete_pop3_packet(aggrbuf, lastpos) == 1){
+            write(fd, aggrbuf, lastpos-3);
+            break;
+         }
+
+      } 
+
+      close(fd);
+
+      if(dryrun == 0) rc = import_message(filename, sdata, data, cfg);
+      else rc = OK;
+
+      if(dryrun == 0 && rc == OK && data->import->remove_after_import == 1){
+         snprintf(buf, sizeof(buf)-1, "DELE %d\r\n", i);
+         write1(sd, buf, strlen(buf), use_ssl, data->ssl);
+         recvtimeoutssl(sd, buf, sizeof(buf), data->import->timeout, use_ssl, data->ssl);
+      }
+
+      if(i % 100 == 0){
+         time(&(data->import->updated));
+         update_import_job_stat(sdata, data);
+      }
+
+      if(data->import->download_only == 0) unlink(filename);
+

      /* whether to quit after processing a batch of messages */

@ -210,7 +224,16 @@ void process_pop3_emails(struct session_data *sdata, struct data *data, struct c


   snprintf(buf, sizeof(buf)-1, "QUIT\r\n");
-   write1(data->net, buf, strlen(buf));
+   write1(sd, buf, strlen(buf), use_ssl, data->ssl);

   if(data->quiet == 0) printf("\n");
+
+   time(&(data->import->finished));
+   data->import->status = 2;
+   update_import_job_stat(sdata, data);
+
+
+   return OK;
 }
+
+
--- a/src/reindex.c
+++ b/src/reindex.c
@ -42,42 +42,45 @@ void p_clean_exit(char *msg, int rc){
 }


-uint64 get_max_meta_id(struct session_data *sdata){
+uint64 get_max_meta_id(struct session_data *sdata, struct __data *data){
   char s[SMALLBUFSIZE];
   uint64 id=0;
-   struct sql sql;

   snprintf(s, sizeof(s)-1, "SELECT MAX(`id`) FROM %s", SQL_METADATA_TABLE);


-   if(prepare_sql_statement(sdata, &sql, s) == ERR) return id;
+   if(prepare_sql_statement(sdata, &(data->stmt_generic), s) == ERR) return id;


-   p_bind_init(&sql);
+   p_bind_init(data);

-   if(p_exec_stmt(sdata, &sql) == OK){
+   if(p_exec_query(sdata, data->stmt_generic, data) == OK){

-      p_bind_init(&sql);
+      p_bind_init(data);

-      sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+      data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;

-      p_store_results(&sql);
-      p_fetch_results(&sql);
-      p_free_results(&sql);
+      p_store_results(data->stmt_generic, data);
+      p_fetch_results(data->stmt_generic);
+      p_free_results(data->stmt_generic);
   }

-   close_prepared_statement(&sql);
+   close_prepared_statement(data->stmt_generic);


   return id;
 }


-uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *data, uint64 from_id, uint64 to_id, struct config *cfg){
+uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct __data *data, uint64 from_id, uint64 to_id, struct __config *cfg){
+   FILE *f;
+   char filename[SMALLBUFSIZE];
   char s[SMALLBUFSIZE];
-   uint64 stored_id=0, reindexed=0;
+   int rc=0;
+   uint64 stored_id=0, reindexed=0, delta;
   struct parser_state state;
-   struct sql sql;
+
+   delta = to_id - from_id;

   if(cfg->enable_folders == 1)
      snprintf(s, sizeof(s)-1, "SELECT m.`id`, `piler_id`, `arrived`, `sent`, f.folder_id FROM %s m, %s f WHERE m.id=f.id AND (m.id BETWEEN %llu AND %llu) AND `deleted`=0", SQL_METADATA_TABLE, SQL_FOLDER_MESSAGE_TABLE, from_id, to_id);
@ -85,78 +88,69 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da
      snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `arrived`, `sent` FROM %s WHERE (id BETWEEN %llu AND %llu) AND `deleted`=0", SQL_METADATA_TABLE, from_id, to_id);


-   if(prepare_sql_statement(sdata, &sql, s) == ERR) return reindexed;
+   if(prepare_sql_statement(sdata, &(data->stmt_generic), s) == ERR) return reindexed;

-   p_bind_init(&sql);
+   p_bind_init(data);

-   if(p_exec_stmt(sdata, &sql) == OK){
+   if(p_exec_query(sdata, data->stmt_generic, data) == OK){

-      p_bind_init(&sql);
+      p_bind_init(data);

-      sql.sql[sql.pos] = (char *)&stored_id; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
-      sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN+2; sql.pos++;
-      sql.sql[sql.pos] = (char *)&(sdata->now); sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(unsigned long); sql.pos++;
-      sql.sql[sql.pos] = (char *)&(sdata->sent); sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(unsigned long); sql.pos++;
+      data->sql[data->pos] = (char *)&stored_id; data->type[data->pos] = TYPE_LONGLONG; data->len[data->pos] = sizeof(uint64); data->pos++;
+      data->sql[data->pos] = sdata->ttmpfile; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = RND_STR_LEN+2; data->pos++;
+      data->sql[data->pos] = (char *)&(sdata->now); data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(unsigned long); data->pos++;
+      data->sql[data->pos] = (char *)&(sdata->sent); data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(unsigned long); data->pos++;
      if(cfg->enable_folders == 1){
-         sql.sql[sql.pos] = (char *)&(data->folder); sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(unsigned long); sql.pos++;
+         data->sql[data->pos] = (char *)&(data->folder); data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(unsigned long); data->pos++;
      }

-      p_store_results(&sql);
+      p_store_results(data->stmt_generic, data);

-      while(p_fetch_results(&sql) == OK){
+      while(p_fetch_results(data->stmt_generic) == OK){

-         char filename[SMALLBUFSIZE];
-         snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id);
+         if(stored_id > 0){

-         FILE *f = fopen(filename, "w");
-         if(f){
-            int rc = retrieve_email_from_archive(sdata, f, cfg);
-            fclose(f);
+            snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id);
+
+            f = fopen(filename, "w");
+            if(f){
+               rc = retrieve_email_from_archive(sdata, data, f, cfg);
+               fclose(f);
+
+               if(rc){
+                  printf("cannot retrieve: %s\n", filename);
+                  unlink(filename);
+                  continue;
+               }
+
+               snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
+
+               state = parse_message(sdata, 0, data, cfg);
+               post_parse(sdata, &state, cfg);
+
+               rc = store_index_data(sdata, &state, data, stored_id, cfg);
+
+               if(rc == OK) reindexed++;
+               else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename);

-            if(rc){
-               printf("cannot retrieve: %s\n", filename);
               unlink(filename);
-               continue;
-            }
-
-            snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
-
-            struct stat st;
-            sdata->tot_len = stat(filename, &st) == 0 ? st.st_size : 0;
-
-            sdata->internal_sender = sdata->internal_recipient = sdata->external_recipient = sdata->direction = 0;
-            memset(sdata->attachments, 0, SMALLBUFSIZE);
-
-            state = parse_message(sdata, 1, data, cfg);
-            post_parse(sdata, &state, cfg);
-
-            rc = store_index_data(sdata, &state, data, stored_id, cfg);
-
-            unlink(sdata->tmpframe);
-            remove_stripped_attachments(&state);
-
-            if(rc == OK) reindexed++;
-            else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename);
-
-            unlink(filename);
-
-            if(progressbar){
-               uint64 delta = to_id - from_id + 1;
-
-               printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta));
-               fflush(stdout);
+
+               if(progressbar){
+                  printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta));
+                  fflush(stdout);
+               }
+
            }
+            else printf("cannot open: %s\n", filename);

         }
-         else printf("cannot open: %s\n", filename);

      }

-
-      p_free_results(&sql);
+      p_free_results(data->stmt_generic);
   }

-   close_prepared_statement(&sql);
+   close_prepared_statement(data->stmt_generic);


   if(progressbar) printf("\n");
@ -166,16 +160,16 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da


 int main(int argc, char **argv){
-   int all=0;
+   int c, all=0;
   uint64 from_id=0, to_id=0, n=0;
   char *configfile=CONFIG_FILE, *folder=NULL;
   struct session_data sdata;
-   struct data data;
-   struct config cfg;
+   struct __data data;
+   struct __config cfg;


   while(1){
-      int c = getopt(argc, argv, "c:f:t:F:pahv?");
+      c = getopt(argc, argv, "c:f:t:F:pahv?");

      if(c == -1) break;

@ -215,7 +209,7 @@ int main(int argc, char **argv){
   }


-   if(all == 0 && (from_id == 0 || to_id == 0) ) usage();
+   if(all == 0 && (from_id <= 0 || to_id <= 0) ) usage();

   if(!can_i_write_directory(NULL)) __fatal("cannot write current directory!");

@ -245,15 +239,10 @@ int main(int argc, char **argv){
      p_clean_exit("cannot connect to mysql server", 1);
   }

-   if(cfg.rtindex && open_sphx(&sdata, &cfg) == ERR){
-      p_clean_exit("cannot connect to 127.0.0.1:9306", 1);
-   }
-
-
-   load_rules(&sdata, data.folder_rules, SQL_FOLDER_RULE_TABLE);
+   load_rules(&sdata, &data, data.folder_rules, SQL_FOLDER_RULE_TABLE);

   if(folder){
-      data.folder = get_folder_id(&sdata, folder, 0);
+      data.folder = get_folder_id(&sdata, &data, folder, 0);
      if(data.folder == 0){
         printf("error: could not get folder id for '%s'\n", folder);
         return 0;
@ -265,7 +254,7 @@ int main(int argc, char **argv){

   if(all == 1){
      from_id = 1;
-      to_id = get_max_meta_id(&sdata);
+      to_id = get_max_meta_id(&sdata, &data);
   }

   n = retrieve_email_by_metadata_id(&sdata, &data, from_id, to_id, &cfg);
@ -277,7 +266,8 @@ int main(int argc, char **argv){
   clearhash(data.mydomains);

   close_database(&sdata);
-   if(cfg.rtindex) close_sphx(&sdata);

   return 0;
 }
+
+
--- a/src/rules.c
+++ b/src/rules.c
@ -10,75 +10,75 @@
 #include "rules.h"


-void reset_rule_condition(struct rule_cond *rule_cond){
-   memset(rule_cond->domain, 0, SMALLBUFSIZE);
-   memset(rule_cond->from, 0, SMALLBUFSIZE);
-   memset(rule_cond->to, 0, SMALLBUFSIZE);
-   memset(rule_cond->subject, 0, SMALLBUFSIZE);
-   memset(rule_cond->body, 0, SMALLBUFSIZE);
-   memset(rule_cond->_size, 0, SMALLBUFSIZE);
-   memset(rule_cond->attachment_name, 0, SMALLBUFSIZE);
-   memset(rule_cond->attachment_type, 0, SMALLBUFSIZE);
-   memset(rule_cond->_attachment_size, 0, SMALLBUFSIZE);
-
-   rule_cond->size = 0;
-   rule_cond->attachment_size = 0;
-   rule_cond->spam = 0;
-   rule_cond->days = 0;
-   rule_cond->folder_id = 0;
-}
-
-
-void load_rules(struct session_data *sdata, struct node *xhash[], char *table){
+void load_rules(struct session_data *sdata, struct __data *data, struct node *xhash[], char *table){
   char s[SMALLBUFSIZE];
   struct rule_cond rule_cond;
-   struct sql sql;

-   reset_rule_condition(&rule_cond);
+   memset(rule_cond.domain, 0, SMALLBUFSIZE);
+   memset(rule_cond.from, 0, SMALLBUFSIZE);
+   memset(rule_cond.to, 0, SMALLBUFSIZE);
+   memset(rule_cond.subject, 0, SMALLBUFSIZE);
+   memset(rule_cond.body, 0, SMALLBUFSIZE);
+   memset(rule_cond._size, 0, SMALLBUFSIZE);
+   memset(rule_cond.attachment_name, 0, SMALLBUFSIZE);
+   memset(rule_cond.attachment_type, 0, SMALLBUFSIZE);
+   memset(rule_cond._attachment_size, 0, SMALLBUFSIZE);
+
+   rule_cond.size = rule_cond.attachment_size = rule_cond.spam = rule_cond.days = rule_cond.folder_id = 0;

   snprintf(s, sizeof(s)-1, "SELECT `domain`, `from`, `to`, `subject`, `body`, `_size`, `size`, `attachment_name`, `attachment_type`, `_attachment_size`, `attachment_size`, `spam`, `days`, `folder_id` FROM `%s`", table);

-   if(prepare_sql_statement(sdata, &sql, s) == ERR) return;
+   if(prepare_sql_statement(sdata, &(data->stmt_generic), s) == ERR) return;


-   p_bind_init(&sql);
+   p_bind_init(data);

-   if(p_exec_stmt(sdata, &sql) == OK){
+   if(p_exec_query(sdata, data->stmt_generic, data) == OK){

-      p_bind_init(&sql);
+      p_bind_init(data);

-      sql.sql[sql.pos] = &rule_cond.domain[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond.domain)-2; sql.pos++;
-      sql.sql[sql.pos] = &rule_cond.from[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond.from)-2; sql.pos++;
-      sql.sql[sql.pos] = &rule_cond.to[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond.to)-2; sql.pos++;
-      sql.sql[sql.pos] = &rule_cond.subject[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond.subject)-2; sql.pos++;
-      sql.sql[sql.pos] = &rule_cond.body[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond.body)-2; sql.pos++;
-      sql.sql[sql.pos] = &rule_cond._size[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond._size)-2; sql.pos++;
-      sql.sql[sql.pos] = (char *)&rule_cond.size; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(rule_cond.size); sql.pos++;
-      sql.sql[sql.pos] = &rule_cond.attachment_name[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond.attachment_name)-2; sql.pos++;
-      sql.sql[sql.pos] = &rule_cond.attachment_type[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond.attachment_type)-2; sql.pos++;
-      sql.sql[sql.pos] = &rule_cond._attachment_size[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(rule_cond._attachment_size)-2; sql.pos++;
-      sql.sql[sql.pos] = (char *)&rule_cond.attachment_size; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(rule_cond.attachment_size); sql.pos++;
-      sql.sql[sql.pos] = (char *)&rule_cond.spam; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(rule_cond.spam); sql.pos++;
-      sql.sql[sql.pos] = (char *)&rule_cond.days; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(rule_cond.days); sql.pos++;
-      sql.sql[sql.pos] = (char *)&rule_cond.folder_id; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(rule_cond.folder_id); sql.pos++;
+      data->sql[data->pos] = &rule_cond.domain[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
+      data->sql[data->pos] = &rule_cond.from[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
+      data->sql[data->pos] = &rule_cond.to[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
+      data->sql[data->pos] = &rule_cond.subject[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
+      data->sql[data->pos] = &rule_cond.body[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
+      data->sql[data->pos] = &rule_cond._size[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
+      data->sql[data->pos] = (char *)&rule_cond.size; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
+      data->sql[data->pos] = &rule_cond.attachment_name[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
+      data->sql[data->pos] = &rule_cond.attachment_type[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
+      data->sql[data->pos] = &rule_cond._attachment_size[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = SMALLBUFSIZE-2; data->pos++;
+      data->sql[data->pos] = (char *)&rule_cond.attachment_size; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
+      data->sql[data->pos] = (char *)&rule_cond.spam; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
+      data->sql[data->pos] = (char *)&rule_cond.days; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
+      data->sql[data->pos] = (char *)&rule_cond.folder_id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;

+      p_store_results(data->stmt_generic, data);

-      p_store_results(&sql);
+      while(p_fetch_results(data->stmt_generic) == OK){
+         append_rule(xhash, &rule_cond, data);

-      while(p_fetch_results(&sql) == OK){
-         append_rule(xhash, &rule_cond);
-         reset_rule_condition(&rule_cond);
+         memset(rule_cond.domain, 0, SMALLBUFSIZE);
+         memset(rule_cond.from, 0, SMALLBUFSIZE);
+         memset(rule_cond.to, 0, SMALLBUFSIZE);
+         memset(rule_cond.subject, 0, SMALLBUFSIZE);
+         memset(rule_cond.body, 0, SMALLBUFSIZE);
+         memset(rule_cond._size, 0, SMALLBUFSIZE);
+         memset(rule_cond.attachment_name, 0, SMALLBUFSIZE);
+         memset(rule_cond.attachment_type, 0, SMALLBUFSIZE);
+         memset(rule_cond._attachment_size, 0, SMALLBUFSIZE);
+
+         rule_cond.size = rule_cond.attachment_size = rule_cond.spam = rule_cond.days = rule_cond.folder_id = 0;
      }

-      p_free_results(&sql);
+      p_free_results(data->stmt_generic);

   }

-   close_prepared_statement(&sql);
+   close_prepared_statement(data->stmt_generic);
 }


-int append_rule(struct node *xhash[], struct rule_cond *rule_cond){
+int append_rule(struct node *xhash[], struct rule_cond *rule_cond, struct __data *data){
   struct node *q, *Q=NULL, *node;
   struct rule *rule;
   int rc=0;
@ -88,7 +88,7 @@ int append_rule(struct node *xhash[], struct rule_cond *rule_cond){
   memset(node, 0, sizeof(struct node));
   node->r = NULL;

-   rule = create_rule_item(rule_cond);
+   rule = create_rule_item(rule_cond, data);

   if(rule == NULL){
      free(node);
@ -116,8 +116,9 @@ int append_rule(struct node *xhash[], struct rule_cond *rule_cond){
 }


-struct rule *create_rule_item(struct rule_cond *rule_cond){
+struct rule *create_rule_item(struct rule_cond *rule_cond, struct __data *data){
   struct rule *h=NULL;
+   char empty = '\0';
   int len;

   if(rule_cond == NULL) return NULL;
@ -129,7 +130,7 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){
   h->compiled = 1;

   h->domain = NULL;
-   h->domainlen = strlen(rule_cond->domain);
+   h->domainlen = data->length[0];

   if(h->domainlen > 2){
      h->domain = malloc(h->domainlen+2);
@ -143,16 +144,16 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){
   h->emptyfrom = h->emptyto = h->emptysubject = h->emptyaname = h->emptyatype = 0;


-   if(rule_cond->from[0] == 0){ h->emptyfrom = 1; }
+   if(rule_cond->from == NULL || strlen(rule_cond->from) < 1){ rule_cond->from[0] = empty; h->emptyfrom = 1; }
   if(regcomp(&(h->from), rule_cond->from, REG_ICASE | REG_EXTENDED)) h->compiled = 0;

-   if(rule_cond->to[0] == 0){ h->emptyto = 1; }
+   if(rule_cond->to == NULL || strlen(rule_cond->to) < 1){ rule_cond->to[0] = empty; h->emptyto = 1; }
   if(regcomp(&(h->to), rule_cond->to, REG_ICASE | REG_EXTENDED)) h->compiled = 0;

-   if(rule_cond->subject[0] == 0){ h->emptysubject = 1; }
+   if(rule_cond->subject == NULL || strlen(rule_cond->subject) < 1){ rule_cond->subject[0] = empty; h->emptysubject = 1; }
   if(regcomp(&(h->subject), rule_cond->subject, REG_ICASE | REG_EXTENDED)) h->compiled = 0;

-   if(rule_cond->body[0] == 0){ h->emptybody = 1; }
+   if(rule_cond->body == NULL || strlen(rule_cond->body) < 1){ rule_cond->body[0] = empty; h->emptybody = 1; }
   if(regcomp(&(h->body), rule_cond->body, REG_ICASE | REG_EXTENDED)) h->compiled = 0;

   h->spam = rule_cond->spam;
@ -160,16 +161,20 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){
   h->folder_id = rule_cond->folder_id;

   h->size = rule_cond->size;
+
+   if(rule_cond->_size == NULL) rule_cond->_size[0] = empty;
   snprintf(h->_size, 3, "%s", rule_cond->_size);

-   if(rule_cond->attachment_name[0] == 0){ h->emptyaname = 1; }
+   if(rule_cond->attachment_name == NULL || strlen(rule_cond->attachment_name) < 1){ rule_cond->attachment_name[0] = empty; h->emptyaname = 1; }
   if(regcomp(&(h->attachment_name), rule_cond->attachment_name, REG_ICASE | REG_EXTENDED)) h->compiled = 0;

-   if(rule_cond->attachment_type[0] == 0){ h->emptyatype = 1; }
+   if(rule_cond->attachment_type == NULL || strlen(rule_cond->attachment_type) < 1){ rule_cond->attachment_type[0] = empty; h->emptyatype = 1; }
   if(regcomp(&(h->attachment_type), rule_cond->attachment_type, REG_ICASE | REG_EXTENDED)) h->compiled = 0;


   h->attachment_size = rule_cond->attachment_size;
+
+   if(rule_cond->_attachment_size == NULL) rule_cond->_attachment_size[0] = empty;
   snprintf(h->_attachment_size, 3, "%s", rule_cond->_attachment_size);

   len = strlen(rule_cond->domain)+8 + strlen(rule_cond->from)+6 + strlen(rule_cond->to)+4 + strlen(rule_cond->subject)+9 + strlen(rule_cond->body)+6 + strlen(rule_cond->_size)+6 + strlen(rule_cond->attachment_name)+10 + strlen(rule_cond->attachment_type)+10 + strlen(rule_cond->_attachment_size)+10 + 9 + 15 + 15;
@ -177,10 +182,7 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){



-   if(h->rulestr){
-      snprintf(h->rulestr, len-1, "domain=%s,from=%s,to=%s,subject=%s,body=%s,size%s%d,att.name=%s,att.type=%s,att.size%s%d,spam=%d", rule_cond->domain, rule_cond->from, rule_cond->to, rule_cond->subject, rule_cond->body, rule_cond->_size, rule_cond->size, rule_cond->attachment_name, rule_cond->attachment_type, rule_cond->_attachment_size, rule_cond->attachment_size, rule_cond->spam);
-      syslog(LOG_INFO, "adding rule: %s", h->rulestr);
-   }
+   if(h->rulestr) snprintf(h->rulestr, len-1, "domain=%s,from=%s,to=%s,subject=%s,body=%s,size%s%d,att.name=%s,att.type=%s,att.size%s%d,spam=%d", rule_cond->domain, rule_cond->from, rule_cond->to, rule_cond->subject, rule_cond->body, rule_cond->_size, rule_cond->size, rule_cond->attachment_name, rule_cond->attachment_type, rule_cond->_attachment_size, rule_cond->attachment_size, rule_cond->spam);
   else h->compiled = 0;

   h->r = NULL;
@ -189,44 +191,11 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){
 }


-int count_match(struct rule *p, struct parser_state *state, int size, int spam){
-   int ismatch=0;
-
-   ismatch += check_spam_rule(spam, p->spam);
-   ismatch += check_size_rule(size, p->size, p->_size);
-   ismatch += check_attachment_rule(state, p);
-
-   if(p->compiled == 1){
-      size_t nmatch=0;
-
-      if(p->emptyfrom == 1){
-         ismatch += RULE_UNDEF;
-      }
-      else if(regexec(&(p->from), state->b_from, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-      if(p->emptyto == 1){
-         ismatch += RULE_UNDEF;
-      }
-      else if(regexec(&(p->to), state->b_to, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-      if(p->emptysubject == 1){
-         ismatch += RULE_UNDEF;
-      }
-      else if(regexec(&(p->subject), state->b_subject, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-
-      if(p->emptybody == 1){
-         ismatch += RULE_UNDEF;
-      }
-      else if(regexec(&(p->body), state->b_body, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
-   }
-
-   return ismatch;
-}
-
-
-char *check_against_ruleset(struct node *xhash[], struct parser_state *state, int size, int spam){
+char *check_againt_ruleset(struct node *xhash[], struct parser_state *state, int size, int spam){
+   size_t nmatch=0;
   struct rule *p;
   struct node *q;
+   int ismatch;

   q = xhash[0];

@ -235,8 +204,40 @@ char *check_against_ruleset(struct node *xhash[], struct parser_state *state, in
      if(q->str){
         p = q->str;

-         if(count_match(p, state, size, spam) > 0){
-            return p->rulestr;
+         if(p){
+            ismatch = 0;
+
+            ismatch += check_spam_rule(spam, p->spam);
+            ismatch += check_size_rule(size, p->size, p->_size);
+            ismatch += check_attachment_rule(state, p);
+
+            if(p->compiled == 1){
+               if(p->emptyfrom == 1){
+                  ismatch += RULE_UNDEF;
+               }
+               else if(regexec(&(p->from), state->b_from, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+               if(p->emptyto == 1){
+                  ismatch += RULE_UNDEF;
+               }
+               else if(regexec(&(p->to), state->b_to, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+               if(p->emptysubject == 1){
+                  ismatch += RULE_UNDEF;
+               }
+               else if(regexec(&(p->subject), state->b_subject, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+               if(p->emptybody == 1){
+                  ismatch += RULE_UNDEF;
+               }
+               else if(regexec(&(p->body), state->b_body, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+            }
+
+            if(ismatch > 0){
+               return p->rulestr;
+            }
+
         }
      }

@ -247,9 +248,11 @@ char *check_against_ruleset(struct node *xhash[], struct parser_state *state, in
 }


-time_t query_retain_period(struct data *data, struct parser_state *state, int size, int spam, struct config *cfg){
+time_t query_retain_period(struct __data *data, struct parser_state *state, int size, int spam, struct __config *cfg){
+   size_t nmatch=0;
   struct rule *p;
   struct node *q;
+   int ismatch;

   q = data->retention_rules[0];

@ -258,15 +261,47 @@ time_t query_retain_period(struct data *data, struct parser_state *state, int si
      if(q->str){
         p = q->str;

+         ismatch = 0;
+
         if(p->domainlen > 2){
            if(strcasestr(state->b_to_domain, p->domain) || strcasestr(state->b_from_domain, p->domain)){
               state->retention = p->days;
-               return (time_t)(state->retention) * (time_t)86400;
+               return (time_t)p->days * (time_t)86400;
            }
         }
-         else if(count_match(p, state, size, spam) > 0){
-            state->retention = p->days;
-            return (time_t)(state->retention) * (time_t)86400;
+         else {
+
+            ismatch += check_spam_rule(spam, p->spam);
+            ismatch += check_size_rule(size, p->size, p->_size);
+            ismatch += check_attachment_rule(state, p);
+
+            if(p->compiled == 1){
+               if(p->emptyfrom == 1){
+                  ismatch += RULE_UNDEF;
+               }
+               else if(regexec(&(p->from), state->b_from, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+               if(p->emptyto == 1){
+                  ismatch += RULE_UNDEF;
+               }
+               else if(regexec(&(p->to), state->b_to, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+               if(p->emptysubject == 1){
+                  ismatch += RULE_UNDEF;
+               }
+               else if(regexec(&(p->subject), state->b_subject, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+               if(p->emptybody == 1){
+                  ismatch += RULE_UNDEF;
+               }
+               else if(regexec(&(p->body), state->b_body, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+            }
+
+            if(ismatch > 0){
+               state->retention = p->days;
+               return (time_t)p->days * (time_t)86400;
+            }
         }

      }
@ -277,13 +312,15 @@ time_t query_retain_period(struct data *data, struct parser_state *state, int si

   state->retention = cfg->default_retention_days;

-   return (time_t)(state->retention) * (time_t)86400;
+   return (time_t)cfg->default_retention_days * (time_t)86400;
 }


-int get_folder_id_by_rule(struct data *data, struct parser_state *state, int size, int spam, struct config *cfg){
+int get_folder_id_by_rule(struct __data *data, struct parser_state *state, int size, int spam, struct __config *cfg){
+   size_t nmatch=0;
   struct rule *p;
   struct node *q;
+   int ismatch;

   if(cfg->enable_folders == 0) return 0;

@ -294,13 +331,45 @@ int get_folder_id_by_rule(struct data *data, struct parser_state *state, int siz
      if(q->str){
         p = q->str;

+         ismatch = 0;
+
         if(p->domainlen > 2){
            if(strcasestr(state->b_to_domain, p->domain) || strcasestr(state->b_from_domain, p->domain)){
               return p->folder_id;
            }
         }
-         else if(count_match(p, state, size, spam) > 0){
-            return p->folder_id;
+         else {
+
+            ismatch += check_spam_rule(spam, p->spam);
+            ismatch += check_size_rule(size, p->size, p->_size);
+            ismatch += check_attachment_rule(state, p);
+
+            if(p->compiled == 1){
+               if(p->emptyfrom == 1){
+                  ismatch += RULE_UNDEF;
+               }
+               else if(regexec(&(p->from), state->b_from, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+               if(p->emptyto == 1){
+                  ismatch += RULE_UNDEF;
+               }
+               else if(regexec(&(p->to), state->b_to, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+               if(p->emptysubject == 1){
+                  ismatch += RULE_UNDEF;
+               }
+               else if(regexec(&(p->subject), state->b_subject, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+               if(p->emptybody == 1){
+                  ismatch += RULE_UNDEF;
+               }
+               else if(regexec(&(p->body), state->b_body, nmatch, NULL, 0) == 0) ismatch += RULE_MATCH; else ismatch += RULE_NO_MATCH;
+
+            }
+
+            if(ismatch > 0){
+               return p->folder_id;
+            }
         }

      }
@ -334,16 +403,14 @@ int check_spam_rule(int is_spam, int spam){
 int check_attachment_rule(struct parser_state *state, struct rule *rule){
   int i;
   size_t nmatch=0;
+   int ismatch = 0;

-   // If no attachment rule, then return RULE_UNDEF
-   if(rule->emptyaname == 1 && rule->emptyatype == 1 && rule->attachment_size == 0) return RULE_UNDEF;
-
-   // If we have attachments, but no attachment rules, then return RULE_NO_MATCH
-   if(state->n_attachments == 0 && (rule->emptyaname == 0 || rule->emptyatype == 0 || rule->attachment_size)) return RULE_NO_MATCH;
+   if(state->n_attachments == 0) return RULE_UNDEF;

+   if(rule->emptyaname == 1 && rule->emptyatype == 1) return RULE_UNDEF;

   for(i=1; i<=state->n_attachments; i++){
-      int ismatch = 0;
+      ismatch = 0;

      if(rule->emptyaname == 0){
         if(regexec(&(rule->attachment_name), state->attachments[i].filename, nmatch, NULL, 0) == 0)
@ -374,33 +441,36 @@ void initrules(struct node *xhash[]){


 void clearrules(struct node *xhash[]){
-   struct node *q;
+   struct node *p, *q;
   struct rule *rule;

   q = xhash[0];

   while(q != NULL){
-      struct node *p = q;
+      p = q;
      q = q->r;

-      if(p->str){
-         rule = (struct rule*)p->str;
+      if(p){
+         if(p->str){
+            rule = (struct rule*)p->str;

-         regfree(&(rule->from));
-         regfree(&(rule->to));
-         regfree(&(rule->subject));
-         regfree(&(rule->body));
-         regfree(&(rule->attachment_name));
-         regfree(&(rule->attachment_type));
+            regfree(&(rule->from));
+            regfree(&(rule->to));
+            regfree(&(rule->subject));
+            regfree(&(rule->body));
+            regfree(&(rule->attachment_name));
+            regfree(&(rule->attachment_type));

-         free(rule->rulestr);
+            free(rule->rulestr);

-         if(rule->domain) free(rule->domain);
+            if(rule->domain) free(rule->domain);

-         free(rule);
+            free(rule);
+         }
+         free(p);
      }
-      free(p);
   }

   xhash[0] = NULL;
 }
+
--- a/src/rules.h
+++ b/src/rules.h
@ -7,12 +7,12 @@

 #include "defs.h"

-void load_rules(struct session_data *sdata, struct node *xhash[], char *table);
-int append_rule(struct node *xhash[], struct rule_cond *rule_cond);
-struct rule *create_rule_item(struct rule_cond *rule_cond);
-char *check_against_ruleset(struct node *xhash[], struct parser_state *state, int size, int spam);
-time_t query_retain_period(struct data *data, struct parser_state *state, int size, int spam, struct config *cfg);
-int get_folder_id_by_rule(struct data *data, struct parser_state *state, int size, int spam, struct config *cfg);
+void load_rules(struct session_data *sdata, struct __data *data, struct node *xhash[], char *table);
+int append_rule(struct node *xhash[], struct rule_cond *rule_cond, struct __data *data);
+struct rule *create_rule_item(struct rule_cond *rule_cond, struct __data *data);
+char *check_againt_ruleset(struct node *xhash[], struct parser_state *state, int size, int spam);
+time_t query_retain_period(struct __data *data, struct parser_state *state, int size, int spam, struct __config *cfg);
+int get_folder_id_by_rule(struct __data *data, struct parser_state *state, int size, int spam, struct __config *cfg);
 int check_size_rule(int message_size, int size, char *_size);
 int check_spam_rule(int is_spam, int spam);
 int check_attachment_rule(struct parser_state *state, struct rule *rule);
@ -21,3 +21,4 @@ void initrules(struct node *xhash[]);
 void clearrules(struct node *xhash[]);

 #endif /* _RULES_H */
+
--- a/src/screen.c
+++ b/src/screen.c
@ -1,254 +0,0 @@
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <sys/socket.h>
-#include <netinet/in.h>
-#include <arpa/inet.h>
-#include <sys/types.h>
-#include <netdb.h>
-#include <piler.h>
-
-
-void init_smtp_acl(struct smtp_acl *smtp_acl[]){
-   smtp_acl[0] = NULL;
-}
-
-
-void clear_smtp_acl(struct smtp_acl *smtp_acl[]){
-   struct smtp_acl *q;
-
-   q = smtp_acl[0];
-
-   while(q){
-      struct smtp_acl *p = q;
-      q = q->r;
-
-      free(p);
-   }
-
-   smtp_acl[0] = NULL;
-}
-
-
-int add_smtp_acl(struct smtp_acl *smtp_acl[], char *network_str, struct smtp_acl *acl){
-   struct smtp_acl *q, *p=NULL, *node;
-
-   if((node = malloc(sizeof(struct smtp_acl))) == NULL) return 0;
-
-   memset(node, 0, sizeof(struct smtp_acl));
-
-   node->low = acl->low;
-   node->high = acl->high;
-   node->prefix = acl->prefix;
-   node->rejected = acl->rejected;
-   snprintf(node->network_str, sizeof(node->network_str)-1, "%s", network_str);
-
-   node->r = NULL;
-
-   q = smtp_acl[0];
-
-   while(q){
-      p = q;
-      q = q->r;
-   }
-
-   if(!p){
-      smtp_acl[0] = node;
-   } else {
-      p->r = node;
-   }
-
-   return 1;
-}
-
-
-int is_valid_line(char *line){
-   // Currently we support ipv4 stuff only, ie. valid characters are: 0-9./
-   // and a line should look like "1.2.3.4/24 permit" or similar (without quotes)
-
-   if(!strchr(line, '.') || !strchr(line, '/') || (!strchr(line, ' ') && !strchr(line, '\t')) ){
-      return 0;
-   }
-
-   if(!strstr(line, "permit") && !strstr(line, "reject")){
-      return 0;
-   }
-
-   // ascii values:
-   //   46: .
-   //   47: /
-   //   48-57: 0-9
-   //   97-122: a-z
-   //
-   for(; *line; line++){
-      if(isalnum(*line) == 0 && isblank(*line) == 0 && *line != 46 && *line != 47) return 0;
-   }
-
-   return 1;
-}
-
-
-int a_to_hl(char *ipstr, in_addr_t *addr){
-  struct in_addr in;
-
-  if(inet_aton(ipstr, &in) == 1){
-     *addr = ntohl(in.s_addr);
-     return 1;
-  }
-
-  syslog(LOG_PRIORITY, "invalid ipv4 address string: *%s*", ipstr);
-
-  return 0;
-}
-
-
-in_addr_t netmask(int prefix){
-   if(prefix == 0)
-      return( ~((in_addr_t) -1) );
-   else
-      return ~((1 << (32 - prefix)) - 1);
-}
-
-
-in_addr_t network(in_addr_t addr, int prefix){
-   return addr & netmask(prefix);
-}
-
-
-in_addr_t broadcast(in_addr_t addr, int prefix){
-   return addr | ~netmask(prefix);
-}
-
-
-int str_to_net_range(char *network_addr_prefix, struct smtp_acl *smtp_acl){
-   in_addr_t net = 0;
-   int prefix = 0;
-
-   smtp_acl->low = 0;
-   smtp_acl->high = 0;
-
-   // By default we permit unless you specify "reject" (without quotes)
-   // To be on the safer side we permit even if you misspell the word "reject"
-
-   smtp_acl->rejected = 0;
-
-   if(strcasestr(network_addr_prefix, "reject")){
-      smtp_acl->rejected = 1;
-   }
-
-   char *p = strchr(network_addr_prefix, '/');
-   if(!p) return 0;
-
-   if(strlen(network_addr_prefix) > sizeof(smtp_acl->network_str)){
-      syslog(LOG_PRIORITY, "line *%s* is longer than %ld bytes, discarded", network_addr_prefix, sizeof(smtp_acl->network_str));
-      return 0;
-   }
-
-   char buf[SMALLBUFSIZE];
-   snprintf(buf, sizeof(buf)-1, "%s", network_addr_prefix);
-
-   *p = '\0';
-   p++;
-
-   // Even though the remaining part of the acl line continues with some number
-   // then whitespace characters and the permit/reject action atoi() can still
-   // figure out the numeric part properly, that's why I'm lazy here.
-   prefix = atoi(p);
-
-   // The prefix string (p) must start with a digit and the prefix integer must be in 0..32 range
-   if(*p < 48 || *p > 57 || prefix < 0 || prefix > 32){
-      syslog(LOG_PRIORITY, "error: invalid prefix: %s", p);
-      return 0;
-   }
-
-   if(a_to_hl(network_addr_prefix, &net)){
-      smtp_acl->low = network(net, prefix);
-      smtp_acl->high = broadcast(net, prefix);
-      smtp_acl->prefix = prefix;
-
-      syslog(LOG_PRIORITY, "info: parsed acl *%s*", buf);
-
-      return 1;
-   }
-
-   return 0;
-}
-
-
-void load_smtp_acl(struct smtp_acl *smtp_acl[]){
-   int count=0;
-
-   clear_smtp_acl(smtp_acl);
-   init_smtp_acl(smtp_acl);
-
-   FILE *f = fopen(SMTP_ACL_FILE, "r");
-   if(!f){
-      syslog(LOG_PRIORITY, "info: cannot open %s, piler-smtp accepts smtp connections from everywhere", SMTP_ACL_FILE);
-      return;
-   }
-
-   char line[SMALLBUFSIZE];
-   struct smtp_acl acl;
-
-   while(fgets(line, sizeof(line)-1, f)){
-      // Skip comments
-      if(line[0] == ';' || line[0] == '#') continue;
-
-      trimBuffer(line);
-
-      // Skip empty line
-      if(line[0] == 0) continue;
-
-      char line2[SMALLBUFSIZE];
-      int rc = 0;
-      snprintf(line2, sizeof(line2)-1, "%s", line);
-
-      if(is_valid_line(line) == 1 && str_to_net_range(line, &acl) == 1){
-         add_smtp_acl(smtp_acl, line, &acl);
-         count++;
-         rc = 1;
-      }
-
-      if(!rc) syslog(LOG_PRIORITY, "error: failed to parse line: *%s*", line2);
-   }
-
-   fclose(f);
-
-   // If we have entries on the smtp acl list, then add 127.0.0.1/8
-   // to let the GUI health page connect to the piler-smtp daemon
-   if(count){
-      snprintf(line, sizeof(line)-1, "127.0.0.1/8 permit");
-
-      if(str_to_net_range(line, &acl) == 1){
-         add_smtp_acl(smtp_acl, line, &acl);
-      }
-   }
-}
-
-
-int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr){
-   struct smtp_acl *q=smtp_acl[0];
-   in_addr_t addr = 0;
-
-   // Empty acl, let it pass
-   if(!q) return 0;
-
-   if(a_to_hl(ipaddr, &addr) == 0){
-      syslog(LOG_PRIORITY, "error: invalid smtp client address: *%s*", ipaddr);
-      return 1;
-   }
-
-   while(q){
-      if(addr >= q->low && addr <= q->high){
-         if(q->rejected) syslog(LOG_PRIORITY, "denied connection from %s, acl: %s/%d reject", ipaddr, q->network_str, q->prefix);
-         return q->rejected;
-      }
-
-      q = q->r;
-   }
-
-   syslog(LOG_PRIORITY, "denied connection from %s by implicit default deny", ipaddr);
-
-   return 1;
-}
--- a/src/screen.h
+++ b/src/screen.h
@ -1,16 +0,0 @@
-/*
- * screen.h, SJ
- */
-
-#ifndef _SCREEN_H
- #define _SCREEN_H
-
-#include "defs.h"
-
-void init_smtp_acl(struct smtp_acl *smtp_acl[]);
-void clear_smtp_acl(struct smtp_acl *smtp_acl[]);
-int add_smtp_acl(struct smtp_acl *smtp_acl[], char *network_str, struct smtp_acl *acl);
-void load_smtp_acl(struct smtp_acl *smtp_acl[]);
-int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr);
-
-#endif /* _SCREEN_H */
--- a/src/session.c
+++ b/src/session.c
@ -2,45 +2,34 @@
 #include <stdlib.h>
 #include <string.h>
 #include <netdb.h>
-#include <signal.h>
 #include <piler.h>


 int get_session_slot(struct smtp_session **sessions, int max_connections);
-void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg);
+void init_smtp_session(struct smtp_session *session, int slot, int sd, struct __config *cfg);

-#define GOT_CRLF_DOT_CRLF(p) *p == '\r' && *(p+1) == '\n' && *(p+2) == '.' && *(p+3) == '\r' && *(p+4) == '\n' ? 1 : 0

-uint64 get_sessions_total_memory(struct smtp_session **sessions, int max_connections){
-   uint64 total = 0;
+#ifdef HAVE_LIBWRAP
+int is_blocked_by_tcp_wrappers(int sd){
+   struct request_info req;

-   for(int i=0; i<max_connections; i++){
-      if(sessions[i]) total += sessions[i]->bufsize;
+   request_init(&req, RQ_DAEMON, "piler", RQ_FILE, sd, 0);
+
+   fromhost(&req);
+
+   if(!hosts_access(&req)){
+      send(sd, SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY, strlen(SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY), 0);
+      syslog(LOG_PRIORITY, "denied connection from %s by tcp_wrappers", eval_client(&req));
+      return 1;
   }

-   return total;
+   return 0;
 }
+#endif


-/*
- * If the sending party sets the email size when it sends the "mail from"
- * part in the smtp transaction, eg. MAIL FROM:<jajaja@akakak.lo> size=509603
- * then piler-smtp could know the email size in advance and could do
- * a better estimate on the allowed number of smtp sessions.
- */
-
-uint64 get_sessions_total_expected_mail_size(struct smtp_session **sessions, int max_connections){
-   uint64 total = 0;
-
-   for(int i=0; i<max_connections; i++){
-      if(sessions[i]) total += sessions[i]->mail_size;
-   }
-
-   return total;
-}
-
-
-int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg){
+int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct __config *cfg){
+   char smtp_banner[SMALLBUFSIZE];
   int slot;

   /*
@ -48,42 +37,25 @@ int start_new_session(struct smtp_session **sessions, int socket, int *num_conne
    */

   if(*num_connections >= cfg->max_connections){
-      syslog(LOG_PRIORITY, "ERROR: too many connections (%d), cannot accept socket %d", *num_connections, socket);
+      syslog(LOG_PRIORITY, "too many connections (%d), cannot accept socket %d", *num_connections, socket);
      send(socket, SMTP_RESP_421_ERR_ALL_PORTS_ARE_BUSY, strlen(SMTP_RESP_421_ERR_ALL_PORTS_ARE_BUSY), 0);
      close(socket);
      return -1;
   }

-   // Check remote client against the allowed network ranges
-   if(cfg->smtp_access_list && is_blocked_by_pilerscreen(smtp_acl, client_addr)){
-      send(socket, SMTP_RESP_550_ERR, strlen(SMTP_RESP_550_ERR), 0);
+#ifdef HAVE_LIBWRAP
+   if(is_blocked_by_tcp_wrappers(socket) == 1){
      close(socket);
      return -1;
   }
-
-   /*
-    * We are under the max_smtp_memory threshold
-    */
-
-   uint64 expected_total_mail_size = get_sessions_total_expected_mail_size(sessions, cfg->max_connections);
-   uint64 total_memory = get_sessions_total_memory(sessions, cfg->max_connections);
-
-   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "DEBUG: total smtp memory allocated: %llu, expected total size: %llu", total_memory, expected_total_mail_size);
-
-   if(total_memory > cfg->max_smtp_memory || expected_total_mail_size > cfg->max_smtp_memory){
-      syslog(LOG_PRIORITY, "ERROR: too much memory consumption: %llu", total_memory);
-      send(socket, SMTP_RESP_451_ERR_TOO_MANY_REQUESTS, strlen(SMTP_RESP_451_ERR_TOO_MANY_REQUESTS), 0);
-      return -1;
-   }
+#endif

   slot = get_session_slot(sessions, cfg->max_connections);

   if(slot >= 0 && sessions[slot] == NULL){
      sessions[slot] = malloc(sizeof(struct smtp_session));
      if(sessions[slot]){
-         init_smtp_session(sessions[slot], slot, socket, client_addr, cfg);
-
-         char smtp_banner[SMALLBUFSIZE];
+         init_smtp_session(sessions[slot], slot, socket, cfg);
         snprintf(smtp_banner, sizeof(smtp_banner)-1, SMTP_RESP_220_BANNER, cfg->hostid);
         send(socket, smtp_banner, strlen(smtp_banner), 0);

@ -121,217 +93,131 @@ struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int m
   int i;

   for(i=0; i<max_connections; i++){
-      if(sessions[i] && sessions[i]->net.socket == socket) return sessions[i];
+      if(sessions[i] && sessions[i]->socket == socket) return sessions[i];
   }

   return NULL;
 }


-void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg){
+void init_smtp_session(struct smtp_session *session, int slot, int sd, struct __config *cfg){
+   struct sockaddr_in addr;
+   socklen_t addr_size = sizeof(struct sockaddr_in);
+   char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
+
   session->slot = slot;

+   session->socket = sd;
+   session->buflen = 0;
   session->protocol_state = SMTP_STATE_INIT;

   session->cfg = cfg;

-   session->net.socket = sd;
-   session->net.use_ssl = 0;  // use SSL/TLS
-   session->net.starttls = 0; // SSL/TLS communication is active (1) or not (0)
-   session->net.ctx = NULL;
-   session->net.ssl = NULL;
+   session->use_ssl = 0;  // use SSL/TLS
+   session->starttls = 0; // SSL/TLS communication is active (1) or not (0)
+   session->ctx = NULL;
+   session->ssl = NULL;

-   session->nullbyte = 0;
+   memset(session->buf, 0, SMALLBUFSIZE);
+   memset(session->remote_host, 0, INET6_ADDRSTRLEN);

-   snprintf(session->remote_host, sizeof(session->remote_host)-1, "%s", client_addr);
+   reset_bdat_counters(session);

-   session->buf = NULL;
-   session->buflen = 0;
-   session->bufsize = 0;
+   time(&(session->lasttime));

-   reset_smtp_session(session);
+   if(getpeername(sd, (struct sockaddr *)&addr, &addr_size) == 0 &&
+      getnameinfo((struct sockaddr *)&addr, addr_size, hbuf, sizeof(hbuf), sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV) == 0){
+         snprintf(session->remote_host, INET6_ADDRSTRLEN-1, "%s", hbuf);
+   }
 }


 void free_smtp_session(struct smtp_session *session){
+
   if(session){

-      if(session->buf != NULL){
-         free(session->buf);
+      if(session->use_ssl == 1){
+         SSL_shutdown(session->ssl);
+         SSL_free(session->ssl);
      }

-      if(session->net.use_ssl == 1){
-         SSL_shutdown(session->net.ssl);
-         SSL_free(session->net.ssl);
-      }
-
-      if(session->net.ctx){
-         SSL_CTX_free(session->net.ctx);
-      }
+      if(session->ctx) SSL_CTX_free(session->ctx);

      free(session);
   }
 }


-void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections, char *reason){
-   if(sessions[slot] == NULL){
-      syslog(LOG_PRIORITY, "session already torn down, slot=%d, reason=%s (%d active connections)", slot, reason, *num_connections);
-      return;
-   }
+void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections){
+   syslog(LOG_PRIORITY, "disconnected from %s", sessions[slot]->remote_host);

-   if(*num_connections > 0) (*num_connections)--;
-
-   syslog(LOG_PRIORITY, "disconnected from %s on fd=%d, slot=%d, reason=%s (%d active connections)",
-          sessions[slot]->remote_host, sessions[slot]->net.socket, slot, reason, *num_connections);
-
-   close(sessions[slot]->net.socket);
-
-   if(sessions[slot]->fd != -1){
-      syslog(LOG_PRIORITY, "Removing %s", sessions[slot]->ttmpfile);
-      close(sessions[slot]->fd);
-      unlink(sessions[slot]->ttmpfile);
-      sessions[slot]->fd = -1;
-   }
+   close(sessions[slot]->socket);

   free_smtp_session(sessions[slot]);
   sessions[slot] = NULL;
+
+   (*num_connections)--;
 }


-inline int get_last_newline_position(char *buf, int buflen){
-   int i;
+void handle_data(struct smtp_session *session, char *readbuf, int readlen){
+   char *p, puf[MAXBUFSIZE];
+   int result;

-   for(i=buflen; i>0; i--){
-      if(*(buf+i) == '\n'){
-         i++;
-         break;
-      }
-   }
-
-   return i;
-}
-
-
-void flush_buffer(struct smtp_session *session){
-   // In the DATA phase skip the 1st character if it's a dot (.)
-   // and there are more characters before the trailing CR-LF
-   //
-   // See https://www.ietf.org/rfc/rfc5321.html#section-4.5.2 for more
-   for(int i=0; i<session->buflen; i++){
-      if(*(session->buf+i) == '\n' && *(session->buf+i+1) == '.' && *(session->buf+i+2) == '.'){
-         int dst = i + 2;
-         int src = dst + 1;
-         int l = session->buflen - src;
-         memmove(session->buf + dst, session->buf + src, l);
-         session->buflen -= 1;
-      }
-   }
-
-   // Exclude the trailing \r\n.\r\n sequence
-
-   session->buflen -= 5;
-
-   if(write(session->fd, session->buf, session->buflen) != session->buflen){
-      session->bad = 1;
-      syslog(LOG_PRIORITY, "ERROR (line: %d) %s: failed to write %d bytes", __LINE__, __func__, session->buflen);
-   }
-
-   session->tot_len = session->buflen;
-}
-
-
-void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg){
-   // Update lasttime if we have something to process
-   time(&(session->lasttime));
+   // process BDAT stuff

   if(session->protocol_state == SMTP_STATE_BDAT){
-      process_bdat(session, readbuf, readlen, cfg);
-      return;
-   }
-
-   // realloc memory if the new chunk doesn't fit in
-
-   if(session->buflen + readlen + 10 > session->bufsize){
-      // Handle if the current memory allocation for this email is above the max_message_size threshold
-
-      if(session->buflen > cfg->max_message_size){
-         if(session->too_big == 0) syslog(LOG_PRIORITY, "ERROR: too big email: %d vs %d", session->buflen, cfg->max_message_size);
-         session->bad = 1;
-         session->too_big = 1;
-      }
-
-      if(session->bad == 0){
-         char *q = realloc(session->buf, session->bufsize + SMTPBUFSIZE);
-         if(q){
-            session->buf = q;
-            memset(session->buf+session->bufsize, 0, SMTPBUFSIZE);
-            session->bufsize += SMTPBUFSIZE;
-         } else {
-            syslog(LOG_PRIORITY, "ERROR: realloc %s %s %d", session->ttmpfile, __func__, __LINE__);
-            session->bad = 1;
-         }
-      }
-   }
-
-   // process smtp command
-   if(session->protocol_state != SMTP_STATE_DATA){
-
-      // We got ~2 MB of garbage and no valid smtp command
-      // Terminate the connection
-      if(session->buflen + readlen > SMTPBUFSIZE - 10){
-         session->bad = 1;
-      }
-
-      // We are at the beginning of the smtp transaction
      if(session->bad == 1){
-         write1(&(session->net), SMTP_RESP_451_ERR, strlen(SMTP_RESP_451_ERR));
-         syslog(LOG_PRIORITY, "ERROR: sent 451 temp error back to client %s", session->ttmpfile);
+         // something bad happened in the BDAT processing
         return;
      }

-      //printf("got %d *%s*\n", readlen, readbuf);
-
-      memcpy(session->buf + session->buflen, readbuf, readlen);
-      session->buflen += readlen;
-
-      int pos = get_last_newline_position(session->buf, session->buflen);
-
-      if(pos < readlen) return; // no complete command
-
-      process_smtp_command(session, cfg);
-
-      memset(session->buf, 0, session->bufsize);
-      session->buflen = 0;
-
-      return;
+      process_bdat(session, readbuf, readlen);
   }

-   if(session->bad == 0){
-      memcpy(session->buf + session->buflen, readbuf, readlen);
-      session->buflen += readlen;
+   // process DATA

-      char *p = session->buf + session->buflen - 5;
-      if(session->buflen >= 5 && GOT_CRLF_DOT_CRLF(p)){
-         flush_buffer(session);
-         process_command_period(session);
+   else if(session->protocol_state == SMTP_STATE_DATA){
+      process_data(session, readbuf, readlen);
+   }
+
+   // process other SMTP commands
+
+   else {
+
+      if(session->buflen > 0){
+         snprintf(puf, sizeof(puf)-1, "%s%s", session->buf, readbuf);
+         snprintf(readbuf, BIGBUFSIZE-1, "%s", puf);
+
+         session->buflen = 0;
+         memset(session->buf, 0, SMALLBUFSIZE);
      }
-   } else if(strstr(readbuf, "\r\n.\r\n")){
-      process_command_period(session);
+
+      p = readbuf;
+
+      do {
+         memset(puf, 0, sizeof(puf));
+         p = split(p, '\n', puf, sizeof(puf)-1, &result);
+
+         if(puf[0] == '\0') continue;
+
+         if(result == 1){
+            process_smtp_command(session, puf);
+
+            // if chunking is enabled and we have data after BDAT <len>
+            // then process the rest
+
+            if(session->cfg->enable_chunking == 1 && p && session->protocol_state == SMTP_STATE_BDAT){
+               process_bdat(session, p, strlen(p));
+               break;
+            }
+         }
+         else {
+            snprintf(session->buf, SMALLBUFSIZE-1, "%s", puf);
+            session->buflen = strlen(puf);
+         }
+      } while(p);
+
   }

 }
-
-
-void write_envelope_addresses(struct smtp_session *session, struct config *cfg){
-   if(session->fd == -1) return;
-
-   for(int i=0; i<session->num_of_rcpt_to; i++){
-      char *p = strchr(session->rcptto[i], '@');
-      if(p && strncmp(p+1, cfg->hostid, cfg->hostid_len)){
-         char s[SMALLBUFSIZE];
-         snprintf(s, sizeof(s)-1, "X-Piler-Envelope-To: %s\n", session->rcptto[i]);
-         if(write(session->fd, s, strlen(s)) == -1) syslog(LOG_PRIORITY, "ERROR: %s: cannot write envelope to address", session->ttmpfile);
-      }
-   }
-}
--- a/src/sig.c
+++ b/src/sig.c
@ -56,3 +56,4 @@ signal_func *set_signal_handler(int signo, signal_func * func){

   return oact.sa_handler;
 }
+
--- a/src/smtp.c
+++ b/src/smtp.c
@ -15,59 +15,53 @@
 #include "smtp.h"


-void process_smtp_command(struct smtp_session *session, struct config *cfg){
+void process_smtp_command(struct smtp_session *session, char *buf){
   char response[SMALLBUFSIZE];

-   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "got on fd=%d: *%s*", session->net.socket, session->buf);
+   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "processing command: *%s*", buf);

-   if(strncasecmp(session->buf, SMTP_CMD_HELO, strlen(SMTP_CMD_HELO)) == 0){
+   if(strncasecmp(buf, SMTP_CMD_HELO, strlen(SMTP_CMD_HELO)) == 0){
      process_command_helo(session, response, sizeof(response));
      return;
   }

-   if(strncasecmp(session->buf, SMTP_CMD_EHLO, strlen(SMTP_CMD_EHLO)) == 0 ||
-         strncasecmp(session->buf, LMTP_CMD_LHLO, strlen(LMTP_CMD_LHLO)) == 0){
-      process_command_ehlo_lhlo(session, response, sizeof(response), cfg);
+   if(strncasecmp(buf, SMTP_CMD_EHLO, strlen(SMTP_CMD_EHLO)) == 0 ||
+         strncasecmp(buf, LMTP_CMD_LHLO, strlen(LMTP_CMD_LHLO)) == 0){
+      process_command_ehlo_lhlo(session, response, sizeof(response));
      return;
   }

-   if(strncasecmp(session->buf, SMTP_CMD_HELP, strlen(SMTP_CMD_HELP)) == 0){
-      send_smtp_response(session, SMTP_RESP_221_PILER_SMTP_OK);
+   if(strncasecmp(buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){
+      process_command_mail_from(session, buf);
      return;
   }

-   if(strncasecmp(session->buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){
-      process_command_mail_from(session);
+   if(strncasecmp(buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){
+      process_command_rcpt_to(session, buf);
      return;
   }

-   if(strncasecmp(session->buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){
-      process_command_rcpt_to(session, cfg);
+   if(strncasecmp(buf, SMTP_CMD_DATA, strlen(SMTP_CMD_DATA)) == 0){
+      process_command_data(session);
      return;
   }

-   if(strncasecmp(session->buf, SMTP_CMD_DATA, strlen(SMTP_CMD_DATA)) == 0){
-      process_command_data(session, cfg);
+   if(session->cfg->enable_chunking == 1 && strncasecmp(buf, SMTP_CMD_BDAT, strlen(SMTP_CMD_BDAT)) == 0){
+      get_bdat_size_to_read(session, buf);
      return;
   }

-   /* Support only BDAT xxxx LAST command */
-   if(session->cfg->enable_chunking == 1 && strncasecmp(session->buf, SMTP_CMD_BDAT, strlen(SMTP_CMD_BDAT)) == 0 && strcasestr(session->buf, "LAST")){
-      get_bdat_size_to_read(session);
-      return;
-   }
-
-   if(strncasecmp(session->buf, SMTP_CMD_QUIT, strlen(SMTP_CMD_QUIT)) == 0){
+   if(strncasecmp(buf, SMTP_CMD_QUIT, strlen(SMTP_CMD_QUIT)) == 0){
      process_command_quit(session, response, sizeof(response));
      return;
   }

-   if(strncasecmp(session->buf, SMTP_CMD_RESET, strlen(SMTP_CMD_RESET)) == 0){
+   if(strncasecmp(buf, SMTP_CMD_RESET, strlen(SMTP_CMD_RESET)) == 0){
      process_command_reset(session);
      return;
   }

-   if(session->cfg->tls_enable == 1 && strncasecmp(session->buf, SMTP_CMD_STARTTLS, strlen(SMTP_CMD_STARTTLS)) == 0 && session->net.use_ssl == 0){
+   if(session->cfg->tls_enable == 1 && strncasecmp(buf, SMTP_CMD_STARTTLS, strlen(SMTP_CMD_STARTTLS)) == 0 && session->use_ssl == 0){
      process_command_starttls(session);
      return;
   }
@ -76,12 +70,54 @@ void process_smtp_command(struct smtp_session *session, struct config *cfg){
 }


+void process_data(struct smtp_session *session, char *readbuf, int readlen){
+   char puf[SMALLBUFSIZE+BIGBUFSIZE];
+   int n, pos, len;
+
+   memset(puf, 0, sizeof(puf));
+
+   if(session->buflen > 0){
+      memcpy(&puf[0], session->buf, session->buflen);
+      memcpy(&puf[session->buflen], readbuf, readlen);
+      len = session->buflen + readlen;
+   }
+   else {
+      memcpy(&puf[0], readbuf, readlen);
+      len = readlen;
+   }
+
+   pos = searchStringInBuffer(&puf[0], len, SMTP_CMD_PERIOD, 5);
+
+   if(pos > 0){
+      write(session->fd, puf, pos);
+      session->tot_len += pos;
+      process_command_period(session);
+   }
+   else {
+      n = search_char_backward(&puf[0], len, '\r');
+
+      if(n == -1 || len - n > 4){
+         write(session->fd, puf, len);
+         session->tot_len += len;
+      }
+      else {
+         write(session->fd, puf, n);
+         session->tot_len += n;
+
+         snprintf(session->buf, SMALLBUFSIZE-1, "%s", &puf[n]);
+         session->buflen = len - n;
+      }
+   }
+}
+
+
 void wait_for_ssl_accept(struct smtp_session *session){
   int rc;
+   char ssl_error[SMALLBUFSIZE];

   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "waiting for ssl handshake");

-   rc = SSL_accept(session->net.ssl);
+   rc = SSL_accept(session->ssl);

   // Since we use non-blocking IO, SSL_accept() is likely to return with -1
   // "In this case a call to SSL_get_error() with the return value of SSL_accept()
@ -89,35 +125,33 @@ void wait_for_ssl_accept(struct smtp_session *session){
   //
   // In this case we may proceed.

-   if(rc == 1 || SSL_get_error(session->net.ssl, rc) == SSL_ERROR_WANT_READ){
-      session->net.use_ssl = 1;
+   if(rc == 1 || SSL_get_error(session->ssl, rc) == SSL_ERROR_WANT_READ){
+      session->use_ssl = 1;
   }

-   if(session->cfg->verbosity >= _LOG_DEBUG || session->net.use_ssl == 0){
-      char ssl_error[SMALLBUFSIZE];
-
+   if(session->cfg->verbosity >= _LOG_DEBUG || session->use_ssl == 0){
      ERR_error_string_n(ERR_get_error(), ssl_error, SMALLBUFSIZE);
      syslog(LOG_PRIORITY, "SSL_accept() result, rc=%d, errorcode: %d, error text: %s",
-             rc, SSL_get_error(session->net.ssl, rc), ssl_error);
+             rc, SSL_get_error(session->ssl, rc), ssl_error);
   }
 }


 void send_smtp_response(struct smtp_session *session, char *buf){
-   write1(&(session->net), buf, strlen(buf));
-   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "sent on fd=%d: %s", session->net.socket, buf);
+   write1(session->socket, buf, strlen(buf), session->use_ssl, session->ssl);
+   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "sent: %s", buf);
 }


 void process_command_helo(struct smtp_session *session, char *buf, int buflen){
   if(session->protocol_state == SMTP_STATE_INIT) session->protocol_state = SMTP_STATE_HELO;

-   snprintf(buf, buflen-1, "250 %s\r\n", session->cfg->hostid);
+   snprintf(buf, buflen-1, "220 %s ESMTP\r\n", session->cfg->hostid);
   send_smtp_response(session, buf);
 }


-void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen, struct config *cfg){
+void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen){
   char extensions[SMALLBUFSIZE];

   memset(extensions, 0, sizeof(extensions));
@ -125,49 +159,34 @@ void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int bufl
   if(session->protocol_state == SMTP_STATE_INIT) session->protocol_state = SMTP_STATE_HELO;

   // if tls is not started, but it's enabled in the config
-   if(session->net.use_ssl == 0 && session->cfg->tls_enable == 1) snprintf(extensions, sizeof(extensions)-1, "%s", SMTP_EXTENSION_STARTTLS);
+   if(session->use_ssl == 0 && session->cfg->tls_enable == 1) snprintf(extensions, sizeof(extensions)-1, "%s", SMTP_EXTENSION_STARTTLS);
   if(session->cfg->enable_chunking == 1) strncat(extensions, SMTP_EXTENSION_CHUNKING, sizeof(extensions)-strlen(extensions)-2);

-   //#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n%s250-SIZE %d\r\n250 8BITMIME\r\n"
-
-   snprintf(buf, buflen-1, SMTP_RESP_250_EXTENSIONS, session->cfg->hostid, cfg->max_message_size, extensions);
+   snprintf(buf, buflen-1, SMTP_RESP_250_EXTENSIONS, session->cfg->hostid, extensions);

   send_smtp_response(session, buf);
 }


 int init_ssl(struct smtp_session *session){
-#if OPENSSL_VERSION_NUMBER < 0x10100000L
-   session->net.ctx = SSL_CTX_new(TLSv1_server_method());
-#else
-   session->net.ctx = SSL_CTX_new(TLS_server_method());
-#endif
+   session->ctx = SSL_CTX_new(TLSv1_server_method());

-   if(session->net.ctx == NULL){
+   if(session->ctx == NULL){
      syslog(LOG_PRIORITY, "SSL ctx is null");
      return 0;
   }

-#if OPENSSL_VERSION_NUMBER < 0x10100000L
-   SSL_CTX_set_options(session->net.ctx, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3|SSL_OP_NO_TLSv1|SSL_OP_NO_TLSv1_1);
-#else
-   if(SSL_CTX_set_min_proto_version(session->net.ctx, session->cfg->tls_min_version_number) == 0){
-      syslog(LOG_PRIORITY, "failed SSL_CTX_set_min_proto_version() to %s/%d", session->cfg->tls_min_version, session->cfg->tls_min_version_number);
-      return 0;
-   }
-#endif
-
-   if(SSL_CTX_set_cipher_list(session->net.ctx, session->cfg->cipher_list) == 0){
+   if(SSL_CTX_set_cipher_list(session->ctx, session->cfg->cipher_list) == 0){
      syslog(LOG_PRIORITY, "failed to set cipher list: '%s'", session->cfg->cipher_list);
      return 0;
   }

-   if(SSL_CTX_use_PrivateKey_file(session->net.ctx, session->cfg->pemfile, SSL_FILETYPE_PEM) != 1){
+   if(SSL_CTX_use_PrivateKey_file(session->ctx, session->cfg->pemfile, SSL_FILETYPE_PEM) != 1){
      syslog(LOG_PRIORITY, "cannot load private key from %s", session->cfg->pemfile);
      return 0;
   }

-   if(SSL_CTX_use_certificate_file(session->net.ctx, session->cfg->pemfile, SSL_FILETYPE_PEM) != 1){
+   if(SSL_CTX_use_certificate_file(session->ctx, session->cfg->pemfile, SSL_FILETYPE_PEM) != 1){
      syslog(LOG_PRIORITY, "cannot load certificate from %s", session->cfg->pemfile);
      return 0;
   }
@ -181,69 +200,56 @@ void process_command_starttls(struct smtp_session *session){

   if(init_ssl(session) == 1){

-      session->net.ssl = SSL_new(session->net.ctx);
-      if(session->net.ssl){
+      session->ssl = SSL_new(session->ctx);
+      if(session->ssl){

-         if(SSL_set_fd(session->net.ssl, session->net.socket) == 1){
-            session->net.starttls = 1;
+         SSL_set_options(session->ssl, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3);
+
+         if(SSL_set_fd(session->ssl, session->socket) == 1){
+            session->starttls = 1;
            send_smtp_response(session, SMTP_RESP_220_READY_TO_START_TLS);
            session->protocol_state = SMTP_STATE_INIT;

-            if(session->net.starttls == 1 && session->net.use_ssl == 0)
+            if(session->starttls == 1 && session->use_ssl == 0)
               wait_for_ssl_accept(session);

            return;
-         } syslog(LOG_PRIORITY, "ERROR: %s: SSL_set_fd() failed", session->ttmpfile);
-      } syslog(LOG_PRIORITY, "ERROR: %s: SSL_new() failed", session->ttmpfile);
-   } syslog(LOG_PRIORITY, "ERROR: init_ssl()");
+         } syslog(LOG_PRIORITY, "%s: SSL_set_fd() failed", session->ttmpfile);
+      } syslog(LOG_PRIORITY, "%s: SSL_new() failed", session->ttmpfile);
+   } syslog(LOG_PRIORITY, "SSL ctx is null!");

   send_smtp_response(session, SMTP_RESP_454_ERR_TLS_TEMP_ERROR);
 }


-void process_command_mail_from(struct smtp_session *session){
+void process_command_mail_from(struct smtp_session *session, char *buf){
   memset(session->mailfrom, 0, SMALLBUFSIZE);

   if(session->protocol_state != SMTP_STATE_HELO && session->protocol_state != SMTP_STATE_PERIOD && session->protocol_state != SMTP_STATE_BDAT){
-      send(session->net.socket, SMTP_RESP_503_ERR, strlen(SMTP_RESP_503_ERR), 0);
+      send(session->socket, SMTP_RESP_503_ERR, strlen(SMTP_RESP_503_ERR), 0);
   }
   else {
+      memset(&(session->ttmpfile[0]), 0, SMALLBUFSIZE);
+      make_random_string(&(session->ttmpfile[0]), QUEUE_ID_LEN);
      session->protocol_state = SMTP_STATE_MAIL_FROM;

-      extractEmail(session->buf, session->mailfrom);
+      extractEmail(buf, session->mailfrom);

-      int mailsize = get_size_from_smtp_mail_from(session->buf);
-
-      reset_smtp_session(session);
-
-      session->mail_size = mailsize;
+      reset_bdat_counters(session);
+      session->tot_len = 0;

      send_smtp_response(session, SMTP_RESP_250_OK);
   }
 }


-void process_command_rcpt_to(struct smtp_session *session, struct config *cfg){
+void process_command_rcpt_to(struct smtp_session *session, char *buf){

   if(session->protocol_state == SMTP_STATE_MAIL_FROM || session->protocol_state == SMTP_STATE_RCPT_TO){

      // For now, we are not interested in the envelope recipients

      session->protocol_state = SMTP_STATE_RCPT_TO;
-
-      if(session->num_of_rcpt_to < MAX_RCPT_TO){
-         extractEmail(session->buf, session->rcptto[session->num_of_rcpt_to]);
-
-         // Check if we should accept archive_address only
-         if(cfg->archive_address[0] && !strstr(cfg->archive_address, session->rcptto[session->num_of_rcpt_to])){
-            syslog(LOG_PRIORITY, "ERROR: Invalid recipient: *%s*", session->rcptto[session->num_of_rcpt_to]);
-            send_smtp_response(session, SMTP_RESP_550_ERR_INVALID_RECIPIENT);
-            return;
-         }
-
-         session->num_of_rcpt_to++;
-      }
-
      send_smtp_response(session, SMTP_RESP_250_OK);
   }
   else {
@ -252,7 +258,7 @@ void process_command_rcpt_to(struct smtp_session *session, struct config *cfg){
 }


-void process_command_data(struct smtp_session *session, struct config *cfg){
+void process_command_data(struct smtp_session *session){
   session->tot_len = 0;

   if(session->protocol_state != SMTP_STATE_RCPT_TO){
@ -267,8 +273,6 @@ void process_command_data(struct smtp_session *session, struct config *cfg){
      else {
         session->protocol_state = SMTP_STATE_DATA;
         send_smtp_response(session, SMTP_RESP_354_DATA_OK);
-
-         if(cfg->process_rcpt_to_addresses == 1) write_envelope_addresses(session, cfg);
      }
   }

@ -276,39 +280,24 @@ void process_command_data(struct smtp_session *session, struct config *cfg){

 void process_command_period(struct smtp_session *session){
   char buf[SMALLBUFSIZE];
-   struct timezone tz;
-   struct timeval tv1, tv2;

   session->protocol_state = SMTP_STATE_PERIOD;

   // TODO: add some error handling

-   gettimeofday(&tv1, &tz);
   fsync(session->fd);
   close(session->fd);
-   gettimeofday(&tv2, &tz);

   session->fd = -1;

-   syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d, client=%s, fd=%d, fsync=%ld", session->ttmpfile, session->mailfrom, session->tot_len, session->remote_host, session->net.socket, tvdiff(tv2, tv1));
+   syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d", session->ttmpfile, session->mailfrom, session->tot_len);

-   if(session->bad == 1 || session->too_big == 1){
-      snprintf(buf, sizeof(buf)-1, SMTP_RESP_451_ERR);
-      unlink(session->ttmpfile);
-      syslog(LOG_PRIORITY, "ERROR: problem in processing, removing %s", session->ttmpfile);
-      if(session->too_big)
-         snprintf(buf, sizeof(buf)-1, "%s", SMTP_RESP_552_ERR_TOO_BIG_EMAIL);
-      else
-         snprintf(buf, sizeof(buf)-1, "%s", SMTP_RESP_451_ERR);
-   } else {
-      move_email(session);
-      snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
-   }
+   move_email(session);

-   if(session->buf){
-      memset(session->buf, 0, session->bufsize);
-      session->buflen = 0;
-   }
+   snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
+
+   session->buflen = 0;
+   memset(session->buf, 0, SMALLBUFSIZE);

   send_smtp_response(session, buf);
 }
@ -324,32 +313,14 @@ void process_command_quit(struct smtp_session *session, char *buf, int buflen){


 void process_command_reset(struct smtp_session *session){
-   reset_smtp_session(session);
   send_smtp_response(session, SMTP_RESP_250_OK);
-   session->protocol_state = SMTP_STATE_HELO;
-}

-
-void reset_smtp_session(struct smtp_session *session){
   session->tot_len = 0;
-   session->mail_size = 0;
-   session->too_big = 0;
-   session->bad = 0;
-
   session->fd = -1;
-
-   session->num_of_rcpt_to = 0;
-   for(int i=0; i<MAX_RCPT_TO; i++) memset(session->rcptto[i], 0, SMALLBUFSIZE);
+   session->protocol_state = SMTP_STATE_HELO;

   reset_bdat_counters(session);

-   time(&(session->lasttime));
-
   memset(&(session->ttmpfile[0]), 0, SMALLBUFSIZE);
-   make_random_string((unsigned char*)&(session->ttmpfile[0]), QUEUE_ID_LEN);
-
-   if(session->buf){
-      memset(session->buf, 0, session->bufsize);
-   }
-   session->buflen = 0;
+   make_random_string(&(session->ttmpfile[0]), QUEUE_ID_LEN);
 }
--- a/src/smtp.h
+++ b/src/smtp.h
@ -7,24 +7,22 @@

 #include <piler.h>

-void process_smtp_command(struct smtp_session *session, struct config *cfg);
-void process_data(struct smtp_session *session, char *buf, int buflen);
+void process_smtp_command(struct smtp_session *session, char *buf);
+void process_data(struct smtp_session *session, char *readbuf, int readlen);

 void send_smtp_response(struct smtp_session *session, char *buf);
 void process_command_helo(struct smtp_session *session, char *buf, int buflen);
-void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen, struct config *cfg);
+void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen);
 void process_command_quit(struct smtp_session *session, char *buf, int buflen);
 void process_command_reset(struct smtp_session *session);
-void process_command_mail_from(struct smtp_session *session);
-void process_command_rcpt_to(struct smtp_session *session, struct config *cfg);
-void process_command_data(struct smtp_session *session, struct config *cfg);
+void process_command_mail_from(struct smtp_session *session, char *buf);
+void process_command_rcpt_to(struct smtp_session *session, char *buf);
+void process_command_data(struct smtp_session *session);
 void process_command_period(struct smtp_session *session);
 void process_command_starttls(struct smtp_session *session);

 void reset_bdat_counters(struct smtp_session *session);
-void get_bdat_size_to_read(struct smtp_session *session);
-void process_bdat(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg);
-
-void reset_smtp_session(struct smtp_session *session);
+void get_bdat_size_to_read(struct smtp_session *session, char *buf);
+void process_bdat(struct smtp_session *session, char *readbuf, int readlen);

 #endif
--- a/src/smtpcodes.h
+++ b/src/smtpcodes.h
@ -18,7 +18,6 @@

 #define SMTP_CMD_HELO "HELO"
 #define SMTP_CMD_EHLO "EHLO"
-#define SMTP_CMD_HELP "HELP"
 #define SMTP_CMD_MAIL_FROM "MAIL FROM:"
 #define SMTP_CMD_RCPT_TO "RCPT TO:"
 #define SMTP_CMD_DATA "DATA"
@ -33,12 +32,11 @@

 // SMTP responses

-#define SMTP_RESP_221_PILER_SMTP_OK "221 piler-smtp is OK\r\n"
 #define SMTP_RESP_220_BANNER "220 %s ESMTP\r\n"
 #define SMTP_RESP_220_READY_TO_START_TLS "220 Ready to start TLS\r\n"
 #define SMTP_RESP_221_GOODBYE "221 %s Goodbye\r\n"
 #define SMTP_RESP_250_OK "250 Ok\r\n"
-#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n250-SIZE %d\r\n%s250 8BITMIME\r\n"
+#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n250-PIPELINING\r\n%s250-SIZE\r\n250 8BITMIME\r\n"

 #define SMTP_EXTENSION_STARTTLS "250-STARTTLS\r\n"
 #define SMTP_EXTENSION_CHUNKING "250-CHUNKING\r\n"
@ -51,16 +49,18 @@
 #define SMTP_RESP_421_ERR_TMP "421 %s service not available\r\n"
 #define SMTP_RESP_421_ERR_WRITE_FAILED "421 writing queue file failed\r\n"
 #define SMTP_RESP_421_ERR_ALL_PORTS_ARE_BUSY "421 All server ports are busy\r\n"
-#define SMTP_RESP_451_ERR_TOO_MANY_REQUESTS "451 Too many requests, try again later\r\n"
+#define SMTP_RESP_450_ERR_CMD_NOT_IMPLEMENTED "450 command not implemented\r\n"
 #define SMTP_RESP_451_ERR "451 Error in processing, try again later\r\n"
 #define SMTP_RESP_454_ERR_TLS_TEMP_ERROR "454 TLS not available currently\r\n"

 #define SMTP_RESP_502_ERR "502 Command not implemented\r\n"
 #define SMTP_RESP_503_ERR "503 Bad command sequence\r\n"
-#define SMTP_RESP_550_ERR_INVALID_RECIPIENT "550 Invalid recipient\r\n"
+#define SMTP_RESP_530_ERR_MUST_ISSUE_STARTTLS_FIRST "530 MUST issue STARTTLS command first\r\n"
+#define SMTP_RESP_550_ERR "550 Access denied.\r\n"
+#define SMTP_RESP_550_INVALID_RECIPIENT "550 Unknown recipient\r\n"
+#define SMTP_RESP_550_ERR_TOO_LONG_RCPT_TO "550 too long recipient\r\n"
 #define SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY "550 You are banned by local policy\r\n"
-#define SMTP_RESP_550_ERR "550 Service currently unavailable\r\n"
-#define SMTP_RESP_552_ERR_TOO_BIG_EMAIL "552 Too big email\r\n"
+#define SMTP_RESP_552_ERR "552 Too many recipients\r\n"


 // LMTP commands
@ -70,3 +70,4 @@
 // LMTP responses

 #define LMTP_RESP_220_BANNER "220 %s LMTP\r\n"
+
--- a/src/sql.h
+++ b/src/sql.h
@ -6,20 +6,19 @@
 #define _SQL_H


-int open_database(struct session_data *sdata, struct config *cfg);
-int open_sphx(struct session_data *sdata, struct config *cfg);
+int open_database(struct session_data *sdata, struct __config *cfg);
 void close_database(struct session_data *sdata);
-void close_sphx(struct session_data *sdata);
-int prepare_sql_statement(struct session_data *sdata, struct sql *sql, char *s);
+int prepare_sql_statement(struct session_data *sdata, MYSQL_STMT **stmt, char *s);
 void p_query(struct session_data *sdata, char *s);
-int p_exec_stmt(struct session_data *sdata, struct sql *sql);
-int p_store_results(struct sql *sql);
-int p_fetch_results(struct sql *sql);
-void p_free_results(struct sql *sql);
-void p_bind_init(struct sql *sql);
-uint64 p_get_insert_id(struct sql *sql);
-int p_get_affected_rows(struct sql *sql);
-void close_prepared_statement(struct sql *sql);
+int p_exec_query(struct session_data *sdata, MYSQL_STMT *stmt, struct __data *data);
+int p_store_results(MYSQL_STMT *stmt, struct __data *data);
+int p_fetch_results(MYSQL_STMT *stmt);
+void p_free_results(MYSQL_STMT *stmt);
+void p_bind_init(struct __data *data);
+uint64 p_get_insert_id(MYSQL_STMT *stmt);
+int p_get_affected_rows(MYSQL_STMT *stmt);
+void close_prepared_statement(MYSQL_STMT *stmt);


 #endif /* _PILER_H */
+
--- a/Show More
+++ b/Show More
 @ -1 +1 @@
 .4.5
 .3.0
				`@ -285,3 +285,4 @@ struct mi htmlentities[] = {`


				`#endif /* _HTMLENTITIES_H */`