Merge remote-tracking branch 'upstream/master'

LICENSE

@@ -1,6 +1,6 @@
 piler, an enterprise level email archiving application
 
-Copyright (C) 2012-2016, Janos SUTO <sj@acts.hu>
+Copyright (C) 2012-2021, Janos SUTO <sj@acts.hu>
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

Makefile.in

@@ -64,7 +64,8 @@ $(RECURSIVE_TARGETS):
 
 
 config-php:
-	sed "s%SYSCONFDIR%$(sysconfdir)%" config.php.in > webui/config.php
+	sed -e "s%SYSCONFDIR%$(sysconfdir)%" -e "s%SBINDIR%$(sbindir)%g" config.php.in > webui/config.php
+	sed -e "s%BINDIR%$(bindir)%g" -i webui/config.php
 
 
 installdirs: mkinstalldirs

RELEASE_NOTES

@@ -1,3 +1,34 @@
+- Introduced the archive_address feature, see etc/example.conf for the details
+- Introduced the raw: search label
+
+1.3.12:
+-------
+
+- Introduced new piler.conf variable: tls_min_version
+
+  It sets the minimum TLS protocol version the piler-smtp daemon supports.
+
+  Possible values:
+  - TLSv1   (not recommended)
+  - TLSv1.1 (not recommended)
+  - TLSv1.2 (default)
+  - TLSv1.3
+
+
+1.3.11:
+-------
+
+- [BUGFIX] Refactored the smtp timeout check
+- Obsoleted the LDAP port parameter. Specify the ldap host in the form
+  of protocol://hostname:port, eg. ldaps://ldap.yourdomain.com:636
+
+1.3.10:
+-------
+
+- Added security header feature
+- Introduced the smtp acl list, and obsoleted the tcp_wrappers check
+- Switched from Blowfish encryption to AES-256
+
 1.3.9:
 ------
 

SECURITY.md

@@ -11,9 +11,10 @@ Security update policy
 
 If a security vulnerability has found, the details, possible mitigations,
 workarounds, etc. will be shared on the piler mailing list (piler-user@list.acts.hu)
-and on the wiki: http://www.mailpiler.org/
+and on the wiki: https://www.mailpiler.org/
 
 Security configurations
 
  - Use https for the GUI
  - Reset the default passwords for admin and auditor
+ - Use the smtp acl feature to restrict SMTP access to the archive, see https://mailpiler.com/smtp-acl-list/

VERSION

@@ -1 +1 @@
-1.3.9
+1.3.12

config.php.in

@@ -82,7 +82,6 @@ $config['WKHTMLTOPDF_COMMAND'] = "wkhtmltopdf";
 
 $config['ENABLE_LDAP_AUTH'] = 0;
 $config['LDAP_HOST'] = 'ldap.yourdomain.com';
-$config['LDAP_PORT'] = 389;
 $config['LDAP_HELPER_DN'] = 'cn=....';
 $config['LDAP_HELPER_PASSWORD'] = 'xxxxxxx';
 $config['LDAP_MAIL_ATTR'] = 'mail';
@@ -217,8 +216,8 @@ $config['DIR_STAT'] = '/var/piler/stat';
 $config['DIR_IMAP'] = '/var/piler/imap';
 $config['DIR_TMP'] = '/var/piler/tmp';
 
-$config['DECRYPT_BINARY'] = '/usr/local/bin/pilerget';
-$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/local/bin/pileraget';
+$config['DECRYPT_BINARY'] = 'BINDIR/pilerget';
+$config['DECRYPT_ATTACHMENT_BINARY'] = 'BINDIR/pileraget';
 $config['DECRYPT_BUFFER_LENGTH'] = 65536;
 
 $config['OPENSSL_BINARY'] = '/usr/bin/openssl';
@@ -249,7 +248,7 @@ $config['MAX_EMAIL_LEN'] = 41;
 $config['RELOAD_COMMAND'] = 'sudo -n /etc/init.d/rc.piler reload';
 $config['PILERIMPORT_IMAP_COMMAND'] = '/usr/local/bin/pilerimport -d /var/piler/imap -q -r';
 $config['CPU_USAGE_COMMAND'] = "LC_ALL=C mpstat | tail -1 | rev | awk '{ print $1 }' | rev";
-$config['PILER_BINARY'] = "/usr/local/sbin/piler";
+$config['PILER_BINARY'] = "SBINDIR/piler";
 
 $config['LDAP_IMPORT_CONFIG_FILE'] = '/usr/local/etc/ldap-import.cfg';
 
@@ -473,6 +472,8 @@ define('HEALTH_WORKER_URL', SITE_URL . 'index.php?route=health/worker');
 
 define('LDAP_TYPE_GENERIC', 'generic_ldap');
 
+define('ATTACHMENT_DUMP_CHECKPOINT', 'attachment_dump_checkpoint');
+
 define('ACTION_ALL', 0);
 define('ACTION_UNKNOWN', 1);
 define('ACTION_LOGIN', 2);
@@ -494,6 +495,7 @@ define('ACTION_VIEW_JOURNAL', 17);
 define('ACTION_NOT_SPAM', 18);
 define('ACTION_MARK_AS_PRIVATE', 19);
 define('ACTION_MARK_MESSAGE_FOR_REMOVAL', 20);
+define('ACTION_REJECT_REMOVAL', 21);
 
 $actions = array(
                   'unknown' => 1,
@@ -508,7 +510,9 @@ $actions = array(
                   'save_search' => 11,
                   'download_attachment' => 15,
                   'journal' => 17,
-                  'private' => 19
+                  'private' => 19,
+                  'marked_for_removal', 20,
+                  'reject_removal', 21
                 );
 
 

configure

@@ -4714,23 +4714,6 @@ else
    echo "zip library: no"
 fi
 
-if test "$have_tcpwrappers" = "yes"; then
-   echo "tcpwrappers support: yes"
-
-cat >>confdefs.h <<_ACEOF
-#define HAVE_LIBWRAP 1
-_ACEOF
-
-
-   if test "$os" = "FreeBSD"; then
-      antispam_libs="$antispam_libs -lwrap"
-   else
-      antispam_libs="$antispam_libs -lwrap -lnsl"
-   fi
-else
-   echo "tcpwrappers support: no"
-fi
-
 
 echo
 
@@ -4852,9 +4835,15 @@ if test $? -eq 1; then echo "the user \"$RUNNING_USER\" does not exists, please
 
 echo; echo
 
-CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wimplicit-fallthrough=2 -Wuninitialized -Wno-format-truncation -g"
+gcc_version="$(gcc -dumpversion)"
+extra_cflags=""
+if [ "${gcc_version:0:1}" -gt 6 ]; then
+   extra_cflags="-Wimplicit-fallthrough=2"
+fi
+
+CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra $extra_cflags -Wuninitialized -Wno-format-truncation -g"
 LIBS="$antispam_libs $sunos_libs "
-OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs"
+OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o screen.o $objs"
 
 ac_config_files="$ac_config_files Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile"
 
@@ -6184,3 +6173,9 @@ echo
 echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/wiki/current:upgrade"
 echo
 echo
+
+echo
+echo "Did you know that piler has an enterprise edition as well?"
+echo "Check out what it can do for you at https://mailpiler.com/piler-enterprise-email-archiver-features/"
+echo
+echo

configure.in

@@ -433,19 +433,6 @@ else
    echo "zip library: no"
 fi
 
-if test "$have_tcpwrappers" = "yes"; then
-   echo "tcpwrappers support: yes"
-   AC_DEFINE_UNQUOTED(HAVE_LIBWRAP, 1, [tcpwrappers support])
-
-   if test "$os" = "FreeBSD"; then
-      antispam_libs="$antispam_libs -lwrap"
-   else
-      antispam_libs="$antispam_libs -lwrap -lnsl"
-   fi
-else
-   echo "tcpwrappers support: no"
-fi
-
 
 echo
 
@@ -535,9 +522,15 @@ if test $? -eq 1; then echo "the user \"$RUNNING_USER\" does not exists, please
 
 echo; echo
 
-CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wimplicit-fallthrough=2 -Wuninitialized -Wno-format-truncation -g"
+gcc_version="$(gcc -dumpversion)"
+extra_cflags=""
+if [[ "${gcc_version:0:1}" -gt 6 ]]; then
+   extra_cflags="-Wimplicit-fallthrough=2"
+fi
+
+CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra $extra_cflags -Wuninitialized -Wno-format-truncation -g"
 LIBS="$antispam_libs $sunos_libs "
-OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs"
+OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o screen.o $objs"
 
 AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile])
 AC_OUTPUT
@@ -547,3 +540,9 @@ echo
 echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/wiki/current:upgrade"
 echo
 echo
+
+echo
+echo "Did you know that piler has an enterprise edition as well?"
+echo "Check out what it can do for you at https://mailpiler.com/piler-enterprise-email-archiver-features/"
+echo
+echo

contrib/export-attachments/export-attachments.php

@@ -0,0 +1,103 @@
+#!/usr/bin/php
+<?php
+
+define('EXPORT_LOCK_FILE', '/var/piler/tmp/export-attachments.lock');
+
+$webuidir = "";
+
+$outdir = '';
+
+
+$opts = 'hw:d:';
+$lopts = [
+   'webui:',
+   'dir:',
+   'help'
+];
+
+
+if($options = getopt($opts, $lopts)) {
+   if(isset($options['webui'])) { $webuidir = $options['webui']; }
+   if(isset($options['dir'])) { $outdir = $options['dir']; }
+   if(isset($options['help'])) { usage(); }
+}
+
+
+if($webuidir == '' || $outdir == '') { usage(); }
+
+
+
+require_once("$webuidir/config.php");
+
+require(DIR_SYSTEM . "/startup.php");
+
+$request = new Request();
+Registry::set("request", $request);
+
+
+$start = NULL;
+
+$loader = new Loader();
+Registry::set('load', $loader);
+
+$loader->load->model('domain/domain');
+$loader->load->model('search/search');
+$loader->load->model('search/message');
+$loader->load->model('message/attachment');
+
+
+$db = new DB(DB_DRIVER, DB_HOSTNAME, DB_USERNAME, DB_PASSWORD, DB_DATABASE, DB_PREFIX);
+Registry::set('db', $db);
+
+Registry::set('auditor_user', 1);
+
+openlog("export-attachments", LOG_PID, LOG_MAIL);
+
+$fp = fopen(EXPORT_LOCK_FILE, "w");
+if(!flock($fp, LOCK_EX)) {
+   syslog(LOG_INFO, "WARN: couldn't get a lock on " . EXPORT_LOCK_FILE);
+   exit;
+}
+
+
+$domain = new ModelDomainDomain();
+$attachment = new ModelMessageAttachment();
+$message = new ModelSearchMessage();
+
+$domains = $domain->get_mapped_domains();
+
+$last_id = $attachment->get_last_attachment_id();
+$start_id = $attachment->get_checkpoint();
+
+syslog(LOG_INFO, "start: $start_id, limit: $last_id");
+
+
+for($i=$start_id; $i<$last_id; $i++) {
+   $a = $attachment->get_attachment_by_id($i);
+   $m = $message->get_message_addresses_by_piler_id($a['piler_id'], $domains);
+
+   $attachment->dump_attachment($outdir, "out", $m['sender'], $i, $a);
+
+   foreach($m['rcpt'] as $rcpt) {
+      $attachment->dump_attachment($outdir, "in", $rcpt, $i, $a);
+   }
+
+   if($i % 100 == 0) { $attachment->update_checkpoint($i); }
+}
+
+$attachment->update_checkpoint($i);
+
+// Release lock
+flock($fp, LOCK_UN);
+fclose($fp);
+
+
+
+function usage() {
+   print "\nUsage: " . __FILE__ . "\n\n";
+   print "\t--webui <path to webui directory>\n";
+   print "\t--dir <basedir to write attachments>\n";
+   print "\t--help\n\n";
+
+   exit;
+}

contrib/proxmox-lxc/README

@@ -0,0 +1,9 @@
+You nedd proxmox with lxc
+
+Fill out conf file first and provide public key
+
+It automatically
+- downloads the latest debian template
+- provides a container
+- installs piler until the post installation step
+- gives a self signed certificate

contrib/proxmox-lxc/create_lxc.sh

@@ -0,0 +1,195 @@
+#!/bin/bash
+
+# This script wil create and fire up a standard debian buster lxc container on your proxmox pve.
+# The Script will look for the next free lxc number and take the next free and use it. So take
+# care that behind your last number is place for it. 
+
+#### SOME VARIABLES TO ADJUST ####
+
+# Storage with templates
+LXC_TMP="local"
+
+# Size and pool of rootfs / in GB
+SIZ_ROT="100"
+S_ROT_P="local-zfs"
+
+# Size and pool of Filestorage in GB will mounted to /share
+SIZ_FIL="100"
+S_FIL_P="local-zfs"
+
+#Weather or not (1 and 0) the container will createt as unpriviliged LXC
+LXC_UNP="1"
+
+# Size of the RAM assigned to the LXC
+LXC_MEM="1024"
+
+# Size of the SWAP assigned to the LXC
+LXC_SWA="1024"
+
+# The hostname (eq. zamba1 or mailpiler1)
+LXC_HOST="zamba"
+
+# The domainname (searchdomain /etc/resolf.conf & hosts)
+LXC_SDN="zmb.local"
+
+# IP-address and subnet
+LXC_IP="10.10.80.20/24"
+
+# Gateway
+LXC_GW="10.10.80.10"
+
+# DNS-server and here shoud be your AD-DC
+LXC_DNS="10.10.80.10"
+
+# Networkbridge for this machine
+LXC_BRD="vmbr80"
+
+# root password - take care to delete from this file
+LXC_PWD="MYPASSWD"
+
+LXC_KEY="ssh-rsa xxxxxxxx"
+
+############### Zamba-Server-Section ###############
+
+# Domain Entries to samba/smb.conf. Will be also uses for samba domain-provisioning when zmb-pdc will choosen.
+ZMB_REA="ZMB.LOCAL"
+ZMB_DOM="ZMB"
+
+# THE Domain-Admin and passwd for zamba-install
+ZMB_ADA="Administrator"
+ZMB_APW="MYPASSWORD"
+
+############### Mailpiler-Section ###############
+
+# The FQDN vor the Hostname. This must be exactly the same like the LXC_HOST / LXC_SDN at section above.
+PILER_DOM="piler.zmb.rocks"
+SMARTHOST="10.10.80.20"
+PILER_VER="1.3.10"
+SPHINX_VER="3.3.1"
+PHP_VER="7.4"
+
+############### Matrix-Section ###############
+
+# The FQDN vor the Hostname. This should be the same like the LXC_HOST / LXC_SDN at section above.
+MRX_DOM="matrix.zmb.rocks"
+ELE_DOM="element.zmb.rocks"
+ELE_VER="v1.7.21"
+JIT_DOM="meet.zmb.rocks"
+
+#################################
+
+# CHeck is the newest template available, else download it.
+
+DEB_LOC=$(pveam list $LXC_TMP | grep debian-10-standard | cut -d'_' -f2)
+
+DEB_REP=$(pveam available --section system | grep debian-10-standard | cut -d'_' -f2)
+
+if [[ $DEB_LOC == $DEB_REP ]];
+then
+  echo "Newest Version of Debian 10 Standard $DEP_REP exists.";
+else
+  echo "Will now download newest Debian 10 Standard $DEP_REP.";
+  pveam download $LXC_TMP debian-10-standard_$DEB_REP\_amd64.tar.gz
+fi
+
+# Get next free LXC-number
+LXC_LST=$( lxc-ls | egrep -o '.{1,5}$' )
+LXC_CHK=$((LXC_LST+1));
+
+if  [ $LXC_CHK -lt 100 ] || [ -f /etc/pve/qemu-server/$LXC_CHK.conf ]; then
+  LXC_NBR=$(pvesh get /cluster/nextid);
+else
+  LXC_NBR=$LXC_CHK;
+fi
+
+echo "Will now create LXC Container $LXC_NBR!";
+
+# Create the container
+pct create $LXC_NBR -unprivileged $LXC_UNP $LXC_TMP:vztmpl/debian-10-standard_$DEB_REP\_amd64.tar.gz -rootfs $S_ROT_P:$SIZ_ROT;
+sleep 2;
+
+pct set $LXC_NBR -memory $LXC_MEM -swap $LXC_SWA -hostname $LXC_HOST \-nameserver $LXC_DNS -searchdomain $LXC_SDN -onboot 1 -timezone Europe/Berlin -net0 name=eth0,bridge=$LXC_BRD,firewall=1,gw=$LXC_GW,ip=$LXC_IP,type=veth;
+sleep 2;
+
+PS3="Select the Server-Function: "
+
+select opt in just_lxc zmb-standalone zmb-member zmb-pdc mailpiler matrix quit; do
+  case $opt in
+    just_lxc)
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      echo "Should be ready!"
+      break
+      ;;
+    zmb-standalone)
+      break
+      ;;
+    zmb-member)
+      echo "Make some additions to LXC for AD-Member-Server!"
+      pct set $LXC_NBR -mp0 $S_FIL_P:$SIZ_FIL,mp=/tank
+      sleep 2;
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      cp /root/zmb_mem.orig /root/zmb_mem.sh
+      sed -i "s|#ZMB_VAR|#ZMB_VAR\nZMB_REA='$ZMB_REA'\nZMB_DOM='$ZMB_DOM'\nZMB_ADA='$ZMB_ADA'\nZMB_APW='$ZMB_APW'|" /root/zmb_mem.sh
+      pct push $LXC_NBR /root/zmb_mem.sh /root/zmb_mem.sh
+      echo "Install zamba as AD-Member-Server!"
+      lxc-attach -n$LXC_NBR bash /root/zmb_mem.sh
+      break
+      ;;
+    zmb-pdc)
+      break
+      ;;
+    mailpiler)
+      echo "Make some additions to LXC for Mailpiler!"
+      pct set $LXC_NBR -features nesting=1
+      sleep 2;
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      cp /root/mailpiler.orig /root/mailpiler.sh
+      sed -i "s|#PILER_VAR|#PILER_VAR\nPILER_DOM='$PILER_DOM'\nSMARTHOST='$SMARTHOST'\nPILER_VER='$PILER_VER'\nSPHINX_VER='$SPHINX_VER'\nPHP_VER='$PHP_VER'|" /root/mailpiler.sh
+      pct push $LXC_NBR /root/mailpiler.sh /root/mailpiler.sh
+      echo "Install Mailpiler mailarchiv!"
+      lxc-attach -n$LXC_NBR bash mailpiler.sh
+      break
+      ;;
+    matrix)
+      echo "Make some additions to LXC for Matrix!"
+      lxc-start $LXC_NBR;
+      sleep 5;
+      # Set the root password and key
+      echo -e "$LXC_PWD\n$LXC_PWD" | lxc-attach -n$LXC_NBR passwd;
+      lxc-attach -n$LXC_NBR mkdir /root/.ssh;
+      echo -e "$LXC_KEY" | lxc-attach -n$LXC_NBR tee /root/.ssh/authorized_keys;
+      lxc-attach -n$LXC_NBR service ssh restart;
+      cp /root/matrix.orig /root/matrix.sh
+      sed -i "s|#MATRIX_VAR|#Matrix_VAR\nMRX_DOM='$MRX_DOM'\nELE_DOM='$ELE_DOM'\nELE_VER='$ELE_VER'\nJIT_DOM='$JIT_DOM'|" /root/matrix.sh
+      pct push $LXC_NBR /root/matrix.sh /root/matrix.sh
+      echo "Install Matrix Chatserver!"
+      lxc-attach -n$LXC_NBR bash matrix.sh
+      break
+      ;;
+    quit)
+      break
+      ;;
+    *)
+      echo "Invalid option!"
+      ;;
+    esac
+done
+

contrib/proxmox-lxc/mailpiler.orig

@@ -0,0 +1,179 @@
+#!/bin/bash
+
+#Variables will be filled in from the mainscript:
+
+#PILER_VAR
+
+
+HOSTNAME=$(hostname -f)
+
+echo "Ensure your Hostname is set to your Piler FQDN!"
+
+echo $HOSTNAME
+
+if 
+    [ "$HOSTNAME" != "$PILER_DOM" ]
+then
+        echo "Hostname doesn't match Piler_Domain! Check install.sh, /etc/hosts, /etc/hostname." && exit
+else
+        echo "Hostname matches PILER_DOMAIN, so starting installation."
+fi
+
+apt install -y gpg apt-transport-https lsb-release
+
+wget -q https://packages.sury.org/php/apt.gpg -O- | apt-key add -
+echo "deb https://packages.sury.org/php/ $(lsb_release -sc) main" | tee /etc/apt/sources.list.d/php.list
+
+apt update && apt full-upgrade -y
+
+apt install -y mc sysstat build-essential libwrap0-dev libpst-dev tnef libytnef0-dev unrtf catdoc libtre-dev tre-agrep poppler-utils libzip-dev unixodbc libpq5 software-properties-common libpoppler-dev openssl libssl-dev memcached telnet nginx mariadb-server default-libmysqlclient-dev python-mysqldb gcc libwrap0 libzip4 latex2rtf latex2html catdoc tnef libpq5 zipcmp zipmerge ziptool libsodium23
+
+apt update && apt install -y php$PHP_VER-{fpm,common,ldap,mysql,cli,opcache,phpdbg,gd,memcache,json,readline,zip}
+
+apt purge -y postfix
+
+cat > /etc/mysql/conf.d/mailpiler.conf <<EOF
+innodb_buffer_pool_size=256M
+innodb_flush_log_at_trx_commit=1
+innodb_log_buffer_size=64M
+innodb_log_file_size=16M
+query_cache_size=0
+query_cache_type=0
+query_cache_limit=2M
+EOF
+
+systemctl restart mariadb
+
+cd /tmp
+wget https://download.mailpiler.com/generic-local/sphinx-$SPHINX_VER-bin.tar.gz
+tar -xvzf sphinx-$SPHINX_VER-bin.tar.gz -C /
+
+groupadd piler
+useradd -g piler -m -s /bin/bash -d /var/piler piler
+usermod -L piler
+chmod 755 /var/piler
+
+wget https://bitbucket.org/jsuto/piler/downloads/piler-$PILER_VER.tar.gz
+tar -xvzf piler-$PILER_VER.tar.gz
+cd piler-$PILER_VER/
+./configure --localstatedir=/var --with-database=mysql --enable-tcpwrappers --enable-memcached
+make
+make install
+ldconfig
+
+cp util/postinstall.sh util/postinstall.sh.bak
+sed -i "s/   SMARTHOST=.*/   SMARTHOST="\"$SMARTHOST\""/" util/postinstall.sh
+sed -i 's/   WWWGROUP=.*/   WWWGROUP="www-data"/' util/postinstall.sh
+
+make postinstall
+
+cp /usr/local/etc/piler/piler.conf /usr/local/etc/piler/piler.conf.bak
+sed -i "s/hostid=.*/hostid=$PILER_DOM/" /usr/local/etc/piler/piler.conf
+sed -i "s/update_counters_to_memcached=.*/update_counters_to_memcached=1/" /usr/local/etc/piler/piler.conf
+
+su piler -c "indexer --all --config /usr/local/etc/piler/sphinx.conf"
+
+/etc/init.d/rc.piler start
+/etc/init.d/rc.searchd start
+
+update-rc.d rc.piler defaults
+update-rc.d rc.searchd defaults
+
+mkdir -p /etc/nginx/ssl
+openssl req -x509 -nodes -days 3650 -newkey rsa:4096 -keyout /etc/nginx/ssl/piler.key -out /etc/nginx/ssl/piler.crt -subj "/CN=$PILER_DOM" -addext "subjectAltName=DNS:$PILER_DOM"
+
+cd /etc/nginx/sites-available
+cp /tmp/piler-$PILER_VER/contrib/webserver/piler-nginx.conf /etc/nginx/sites-available/
+ln -s /etc/nginx/sites-available/piler-nginx.conf /etc/nginx/sites-enabled/piler-nginx.conf
+
+sed -i "s|PILER_HOST|$PILER_DOM|g" /etc/nginx/sites-available/piler-nginx.conf
+sed -i "s|/var/run/php/php7.4-fpm.sock|/var/run/php/php$PHP_VER-fpm.sock|g" /etc/nginx/sites-available/piler-nginx.conf
+
+sed -i "/server_name.*/a \\
+        listen 443 ssl http2;\n\n\
+        ssl_certificate /etc/nginx/ssl/piler.crt;\n\
+        ssl_certificate_key /etc/nginx/ssl/piler.key;\n\n\
+        ssl_session_timeout 1d;\n\
+        ssl_session_cache shared:SSL:15m;\n\
+        ssl_session_tickets off;\n\n\
+        # modern configuration of Mozilla SSL configurator. Tweak to your needs.\n\
+        ssl_protocols TLSv1.2 TLSv1.3;\n\
+        ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384;\n\
+        ssl_prefer_server_ciphers off;\n\n\
+        add_header X-Frame-Options SAMEORIGIN;\n\
+        add_header X-Content-Type-Options nosniff;" /etc/nginx/sites-available/piler-nginx.conf
+
+sed -i "/^server {.*/i\
+server {\n\
+        listen 80;\n\
+        server_name $PILER_DOM;\n\
+        server_tokens off;\n\
+        # HTTP to HTTPS redirect.\n\
+        return 301 https://$PILER_DOM;\n\
+}" /etc/nginx/sites-available/piler-nginx.conf
+
+cp /usr/local/etc/piler/config-site.php /usr/local/etc/piler/config-site.php.bak
+sed -i "s|\$config\['SITE_URL'\] = .*|\$config\['SITE_URL'\] = 'https://$PILER_DOM/';|" /usr/local/etc/piler/config-site.php
+cat >> /usr/local/etc/piler/config-site.php <<EOF
+
+// CUSTOM
+\$config['PROVIDED_BY'] = '$PILER_DOM';
+\$config['SUPPORT_LINK'] = 'https://$PILER_DOM';
+\$config['COMPATIBILITY'] = '';
+
+// fancy features.
+\$config['ENABLE_INSTANT_SEARCH'] = 1;
+\$config['ENABLE_TABLE_RESIZE'] = 1;
+
+\$config['ENABLE_DELETE'] = 1;
+\$config['ENABLE_ON_THE_FLY_VERIFICATION'] = 1;
+
+// general settings.
+\$config['TIMEZONE'] = 'Europe/Berlin';
+
+// authentication
+// Enable authentication against an imap server
+//\$config['ENABLE_IMAP_AUTH'] = 1;
+//\$config['RESTORE_OVER_IMAP'] = 1;
+//\$config['IMAP_RESTORE_FOLDER_INBOX'] = 'INBOX';
+//\$config['IMAP_RESTORE_FOLDER_SENT'] = 'Sent';
+//\$config['IMAP_HOST'] = '$SMARTHOST';
+//\$config['IMAP_PORT'] =  993;
+//\$config['IMAP_SSL'] = true;
+
+// authentication against an ldap directory (disabled by default)
+//\$config['ENABLE_LDAP_AUTH'] = 1;
+//\$config['LDAP_HOST'] = '$SMARTHOST';
+//\$config['LDAP_PORT'] = 389;
+//\$config['LDAP_HELPER_DN'] = 'cn=administrator,cn=users,dc=mydomain,dc=local';
+//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
+//\$config['LDAP_MAIL_ATTR'] = 'mail';
+//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
+//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
+//\$config['LDAP_BASE_DN'] = 'ou=Benutzer,dc=krs,dc=local';
+
+// authentication against an Uninvention based ldap directory 
+//\$config['ENABLE_LDAP_AUTH'] = 1;
+//\$config['LDAP_HOST'] = '$SMARTHOST';
+//\$config['LDAP_PORT'] = 7389;
+//\$config['LDAP_HELPER_DN'] = 'uid=ldap-search-user,cn=users,dc=mydomain,dc=local';
+//\$config['LDAP_HELPER_PASSWORD'] = 'myxxxxpasswd';
+//\$config['LDAP_AUDITOR_MEMBER_DN'] = '';
+//\$config['LDAP_ADMIN_MEMBER_DN'] = '';
+//\$config['LDAP_BASE_DN'] = 'cn=users,dc=mydomain,dc=local';
+//\$config['LDAP_MAIL_ATTR'] = 'mailPrimaryAddress';
+//\$config['LDAP_ACCOUNT_OBJECTCLASS'] = 'person';
+//\$config['LDAP_DISTRIBUTIONLIST_OBJECTCLASS'] = 'person';
+//\$config['LDAP_DISTRIBUTIONLIST_ATTR'] = 'mailAlternativeAddress';
+
+// special settings.
+\$config['MEMCACHED_ENABLED'] = 1;
+\$config['SPHINX_STRICT_SCHEMA'] = 1; // required for Sphinx $SPHINX_VER, see https://bitbucket.org/jsuto/piler/issues/1085/sphinx-331.
+EOF
+
+nginx -t && systemctl restart nginx
+
+apt autoremove -y
+apt clean -y
+
+

contrib/proxmox-lxc/matrix.orig

@@ -0,0 +1,150 @@
+#!/bin/bash
+
+#MATRIX_VAR
+
+
+MRX_PKE=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+
+ELE_DBNAME="synapse_db"
+ELE_DBUSER="synapse_user"
+ELE_DBPASS=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1)
+
+apt update && apt full-upgrade -y
+
+apt install -y lsb-release apt-transport-https curl gpg software-properties-common net-tools nginx mc postgresql python3-psycopg2
+
+wget wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
+echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/matrix-org.list
+apt update && apt install -y matrix-synapse-py3
+systemctl enable matrix-synapse
+
+ss -tulpen
+
+mkdir /etc/nginx/ssl
+openssl req -x509 -nodes -days 3650 -newkey rsa:4096 -keyout /etc/nginx/ssl/matrix.key -out /etc/nginx/ssl/matrix.crt -subj "/CN=$MRX_DOM" -addext "subjectAltName=DNS:$MRX_DOM"
+
+cat > /etc/nginx/sites-available/$MRX_DOM <<EOF
+# Virtual Host configuration for example.com
+#
+# You can move that to a different file under sites-available/ and symlink that
+# to sites-enabled/ to enable it.
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name $MRX_DOM;
+
+    return 301 https://$MRX_DOM;
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    server_name $MRX_DOM;
+
+    ssl on;
+    ssl_certificate /etc/nginx/ssl/matrix.crt;
+    ssl_certificate_key /etc/nginx/ssl/matrix.key;
+
+    location / {
+      proxy_pass http://127.0.0.1:8008;
+      proxy_set_header X-Forwarded-For \$remote_addr;
+    }
+}
+
+server {
+    listen 8448 ssl;
+    listen [::]:8448 ssl;
+    server_name $MRX_DOM;
+
+    ssl on;
+    ssl_certificate /etc/nginx/ssl/matrix.crt;
+    ssl_certificate_key /etc/nginx/ssl/matrix.key;
+
+    # If you don't wanna serve a site, comment this out
+    root /var/www/$MRX_DOM;
+    index index.html index.htm;
+
+    location / {
+        proxy_pass http://127.0.0.1:8008;
+        proxy_set_header X-Forwarded-For \$remote_addr;
+    }
+}
+
+EOF
+ln -s /etc/nginx/sites-available/$MRX_DOM /etc/nginx/sites-enabled/$MRX_DOM
+
+cat > /etc/nginx/sites-available/$ELE_DOM <<EOF
+# Virtual Host configuration for example.com
+#
+# You can move that to a different file under sites-available/ and symlink that
+# to sites-enabled/ to enable it.
+
+server {
+    listen 80;
+    listen [::]:80;
+    server_name $ELE_DOM;
+    return 301 https://$ELE_DOM;
+}
+
+server {
+    listen 443 ssl;
+    listen [::]:443 ssl;
+    server_name $ELE_DOM;
+
+    ssl on;
+    ssl_certificate /etc/nginx/ssl/matrix.crt;
+    ssl_certificate_key /etc/nginx/ssl/matrix.key;
+
+    # If you don't wanna serve a site, comment this out
+    root /var/www/$ELE_DOM/element;
+    index index.html index.htm;
+} 
+
+EOF
+
+ln -s /etc/nginx/sites-available/$ELE_DOM /etc/nginx/sites-enabled/$ELE_DOM
+
+systemctl restart nginx
+
+mkdir /var/www/$ELE_DOM
+cd /var/www/$ELE_DOM
+wget https://packages.riot.im/element-release-key.asc
+gpg --import element-release-key.asc
+
+wget https://github.com/vector-im/element-web/releases/download/$ELE_VER/element-$ELE_VER.tar.gz
+wget https://github.com/vector-im/element-web/releases/download/$ELE_VER/element-$ELE_VER.tar.gz.asc
+gpg --verify element-$ELE_VER.tar.gz.asc
+
+tar -xzvf element-$ELE_VER.tar.gz
+ln -s element-$ELE_VER element
+chown www-data:www-data -R element
+cp ./element/config.sample.json ./element/config.json
+sed -i "s|https://matrix-client.matrix.org|https://$MRX_DOM|" ./element/config.json
+sed -i "s|\"server_name\": \"matrix.org\"|\"server_name\": \"$MRX_DOM\"|" ./element/config.json
+
+su postgres <<EOF
+psql -c "CREATE USER $ELE_DBUSER WITH PASSWORD '$ELE_DBPASS';"
+psql -c "CREATE DATABASE $ELE_DBNAME ENCODING 'UTF8' LC_COLLATE='C' LC_CTYPE='C' template=template0 OWNER $ELE_DBUSER;"
+echo "Postgres User '$ELE_DBUSER' and database '$ELE_DBNAME' created."
+EOF
+
+cd /
+sed -i "s|#registration_shared_secret: <PRIVATE STRING>|registration_shared_secret: \"$MRX_PKE\"|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|#public_baseurl: https://example.com/|public_baseurl: https://$MRX_DOM/|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|#enable_registration: false|enable_registration: true|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|name: sqlite3|name: psycopg2|" /etc/matrix-synapse/homeserver.yaml
+sed -i "s|database: /var/lib/matrix-synapse/homeserver.db|database: $ELE_DBNAME\n    user: $ELE_DBUSER\n    password: $ELE_DBPASS\n    host: 127.0.0.1\n    cp_min: 5\n    cp_max: 10|" /etc/matrix-synapse/homeserver.yaml
+
+systemctl restart matrix-synapse
+
+register_new_matrix_user -c /etc/matrix-synapse/homeserver.yaml http://127.0.0.1:8008
+
+#curl https://download.jitsi.org/jitsi-key.gpg.key | sh -c 'gpg --dearmor > /usr/share/keyrings/jitsi-keyring.gpg'
+#echo 'deb [signed-by=/usr/share/keyrings/jitsi-keyring.gpg] https://download.jitsi.org stable/' | tee /etc/apt/sources.list.d/jitsi-stable.list > /dev/null
+
+#apt update
+#apt install -y jitsi-meet
+
+
+

contrib/proxmox-lxc/zmb_mem.orig

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+#ZMB_VAR
+
+
+
+apt update && apt full-upgrade -y
+echo -ne '\n' | apt install -y acl dnsutils mc samba winbind libpam-winbind libnss-winbind krb5-user krb5-config samba-dsdb-modules samba-vfs-modules 
+
+mv /etc/krb5.conf /etc/krb5.conf.bak
+cat > /etc/krb5.conf <<EOF
+[libdefaults]
+	default_realm = $ZMB_REA
+    ticket_lifetime = 600
+	dns_lookup_realm = true
+	dns_lookup_kdc = true
+	renew_lifetime = 7d
+EOF
+
+echo -e "$ZMB_APW" | kinit -V $ZMB_ADA
+klist
+
+mv /etc/samba/smb.conf /etc/samba/smb.conf.bak
+cat > /etc/samba/smb.conf <<EOF
+[global]
+	workgroup = $ZMB_DOM
+	security = ADS
+	realm = $ZMB_REA
+	server string = %h server
+
+	vfs objects = acl_xattr shadow_copy2
+    map acl inherit = Yes
+    store dos attributes = Yes
+	idmap config *:backend = tdb
+	idmap config *:range = 3000000-4000000
+	idmap config *:schema_mode = rfc2307
+
+	winbind refresh tickets = Yes
+	winbind use default domain = Yes
+	winbind separator = /
+	winbind nested groups = yes
+	winbind nss info = rfc2307
+
+	pam password change = Yes
+	passwd chat = *Enter\snew\s*\spassword:* %n\n *Retype\snew\s*\spassword:* %n\n *password\supdated\ssuccessfully* .
+	passwd program = /usr/bin/passwd %u
+
+	template homedir = /home/%U
+	template shell = /bin/bash
+	bind interfaces only = Yes
+	interfaces = lo eth0
+	log file = /var/log/samba/log.%m
+	logging = syslog
+	max log size = 1000
+	panic action = /usr/share/samba/panic-action %d
+
+	load printers = No
+	printcap name = /dev/null
+	printing = bsd
+	disable spoolss = Yes
+
+	allow trusted domains = No
+	dns proxy = No
+	shadow: snapdir = .zfs/snapshot
+	shadow: sort = desc
+	shadow: format = -%Y-%m-%d-%H%M
+	shadow: snapprefix = ^zfs-auto-snap_\(frequent\)\{0,1\}\(hourly\)\{0,1\}\(daily\)\{0,1\}\(monthly\)\{0,1\}
+	shadow: delimiter = -20
+
+[share]
+	comment = Main Share
+	path = /tank/share
+	read only = No
+	create mask = 0660
+	directory mask = 0770
+	inherit acls = Yes
+
+
+
+EOF
+
+systemctl restart smbd
+
+echo -e "$ZMB_APW" | net ads join -U $ZMB_ADA createcomputer=Computers
+sed -i "s|files systemd|files systemd winbind|g" /etc/nsswitch.conf
+sed -i "s|#WINBINDD_OPTS=|WINBINDD_OPTS=|" /etc/default/winbind
+echo -e "session optional        pam_mkhomedir.so skel=/etc/skel umask=077" >> /etc/pam.d/common-session
+
+systemctl restart winbind nmbd
+wbinfo -u
+wbinfo -g
+
+mkdir /tank/share
+chown 'administrator':'domain users' /tank/share
+
+setfacl -Rm u:administrator:rwx,g::-,o::- /tank/share
+setfacl -Rdm u:administrator:rwx,g::-,o::- /tank/share
+
+systemctl restart smbd nmbd winbind
+

contrib/webserver/piler-nginx.conf

@@ -4,7 +4,13 @@ server {
 
         root /var/piler/www;
 
-        gzip  on;
+        server_tokens off;
+
+        add_header X-XSS-Protection "1; mode=block";
+        add_header X-Content-Type-Options "nosniff";
+        add_header Referrer-Policy "same-origin";
+
+        gzip on;
         gzip_types text/plain application/xml text/css;
         gzip_vary on;
 
@@ -28,7 +34,7 @@ server {
                return 404;
             }
 
-            fastcgi_pass unix:/var/run/php/php7.2-fpm.sock;
+            fastcgi_pass unix:/var/run/php/php7.4-fpm.sock;
             fastcgi_index index.php;
             include fastcgi_params;
         }

docker/Dockerfile

@@ -2,48 +2,40 @@ FROM ubuntu:20.04
 
 ARG PACKAGE
 
-LABEL description="piler container" \
+LABEL description="piler ubuntu focal image" \
       maintainer="Janos SUTO, sj@acts.hu" \
       package="${PACKAGE}"
 
 ENV DEBIAN_FRONTEND="noninteractive" \
-    DISTRO="bionic" \
+    DISTRO="focal" \
     DOWNLOAD_URL="https://download.mailpiler.com" \
     PILER_USER="piler" \
-    MYSQL_HOSTNAME="localhost" \
     MYSQL_DATABASE="piler" \
-    MYSQL_PILER_PASSWORD="piler123" \
-    MYSQL_ROOT_PASSWORD="abcde123" \
-    SPHINX_BIN_TARGZ="sphinx-3.1.1-bin.tar.gz"
+    SPHINX_BIN_TARGZ="sphinx-3.3.1-bin.tar.gz"
 
-ADD "https://bitbucket.org/jsuto/piler/downloads/${PACKAGE}" "/${PACKAGE}"
-COPY start.sh /start.sh
+COPY ${PACKAGE} /
 
 RUN apt-get update && \
     apt-get -y --no-install-recommends install \
-       wget rsyslog openssl sysstat php7.2-cli php7.2-cgi php7.2-mysql php7.2-fpm php7.2-zip php7.2-ldap \
-       php7.2-gd php7.2-curl php7.2-xml catdoc unrtf poppler-utils nginx tnef sudo libodbc1 libpq5 libzip4 \
-       libtre5 libwrap0 cron libmariadb3 libmysqlclient-dev python python-mysqldb mariadb-server && \
+       wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \
+       php7.4-gd php7.4-curl php7.4-xml php7.4-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip5 \
+       libtre5 cron libmariadb-dev mariadb-client-core-10.3 python3 python3-mysqldb ca-certificates curl && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* && \
-    service mysql start && mysqladmin -u root password ${MYSQL_ROOT_PASSWORD} && \
     wget --no-check-certificate -q -O ${SPHINX_BIN_TARGZ} ${DOWNLOAD_URL}/generic-local/${SPHINX_BIN_TARGZ} && \
     tar zxvf ${SPHINX_BIN_TARGZ} && \
-    rm -f ${SPHINX_BIN_TARGZ} && \
     sed -i 's/mail.[iwe].*//' /etc/rsyslog.conf && \
     sed -i '/session    required     pam_loginuid.so/c\#session    required     pam_loginuid.so' /etc/pam.d/cron && \
-    mkdir /etc/piler && \
-    printf "[mysql]\nuser = piler\npassword = %s\n" ${MYSQL_PILER_PASSWORD} > /etc/piler/.my.cnf && \
-    printf "[mysql]\nuser = root\npassword = %s\n" ${MYSQL_ROOT_PASSWORD} > /root/.my.cnf && \
-    echo "alias mysql='mysql --defaults-file=/etc/piler/.my.cnf'" > /root/.bashrc && \
-    echo "alias t='tail -f /var/log/syslog'" >> /root/.bashrc && \
-    dpkg -i $PACKAGE && \
-    crontab -u $PILER_USER /usr/share/piler/piler.cron && \
-    touch /var/log/mail.log && \
-    rm -f $PACKAGE /etc/nginx/sites-enabled/default && \
-    sed -i 's/#ngram/ngram/g' /etc/piler/sphinx.conf.dist && \
-    sed -i 's/220/311/g' /etc/piler/sphinx.conf.dist
+    dpkg -i ${PACKAGE} && \
+    ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled && \
+    rm -f ${PACKAGE} ${SPHINX_BIN_TARGZ} /etc/nginx/sites-enabled/default /etc/piler/piler.key /etc/piler/piler.pem /etc/piler/config-site.php && \
+    crontab -u $PILER_USER /usr/share/piler/piler.cron
+
+VOLUME ["/etc/piler"]
+VOLUME ["/var/piler"]
 
 EXPOSE 25 80 443
-VOLUME ["/var/piler"]
+
+COPY start.sh /start.sh
+
 CMD ["/start.sh"]

docker/README

@@ -1,12 +0,0 @@
-How to build
-
-  Pick the latest deb package from Bitbucket download page (https://bitbucket.org/jsuto/piler/downloads/)
-  and use it as the PACKAGE build argument, eg.
-
-  docker build --build-arg PACKAGE=piler_1.3.6~bionic-65cc7eb_amd64.deb -t sutoj/piler .
-
-How to run the image
-
-  Set the PILER_HOST env variable to match your hostname, eg.
-
-  docker run -d --name piler1 -p 25:25 -p 80:80 -e PILER_HOST=archive.yourdomain.com sutoj/piler

docker/README.md

@@ -0,0 +1,16 @@
+## How to run piler
+
+Edit the variables in docker-compose.yaml, then run
+
+```
+docker-compose up -d
+```
+
+## How to build the image for yourself
+
+Pick the latest deb package from Bitbucket download page (https://bitbucket.org/jsuto/piler/downloads/)
+and use it as the PACKAGE build argument, eg.
+
+```
+docker build --build-arg PACKAGE=piler_1.3.11-focal-5c2ceb1_amd64.deb -t piler:1.3.11 .
+```

docker/build.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+IMAGE_NAME="sutoj/piler:1.3.11"
+
+if [[ $# -ne 1 ]]; then echo "ERROR: missing package name" 1>&2; exit 1; fi
+
+docker build --pull --build-arg PACKAGE="$1" -t "$IMAGE_NAME" .

docker/docker-compose.yaml

@@ -0,0 +1,72 @@
+version: "3"
+services:
+
+  mysql:
+    image: mariadb:10.5
+    restart: unless-stopped
+    cap_drop:
+      - ALL
+    cap_add:
+      - dac_override
+      - setuid
+      - setgid
+    environment:
+      - MYSQL_DATABASE=piler
+      - MYSQL_USER=piler
+      - MYSQL_PASSWORD=piler123
+      - MYSQL_RANDOM_ROOT_PASSWORD=yes
+    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
+    healthcheck:
+      test: mysql --user=piler --password=piler123 piler --execute "show tables"
+      interval: "60s"
+      timeout: "5s"
+      start_period: "15s"
+      retries: 3
+    volumes:
+      - db_data:/var/lib/mysql
+
+  memcached:
+    image: memcached:latest
+    restart: unless-stopped
+    cap_drop:
+      - ALL
+    command: -m 64
+
+  piler:
+    image: sutoj/piler:1.3.10
+    init: true
+    environment:
+      - MYSQL_DATABASE=piler
+      - MYSQL_USER=piler
+      - MYSQL_PASSWORD=piler123
+      - MYSQL_HOSTNAME=mysql
+      - PILER_HOSTNAME=archive.yourdomain.com
+      - MEMCACHED_HOST=memcached
+    ports:
+      - "25:25"
+      - "80:80"
+      - "443:443"
+    volumes:
+      - piler_etc:/etc/piler
+      - piler_var:/var/piler
+    healthcheck:
+      test: curl -s smtp://localhost/
+      interval: "60s"
+      timeout: "3s"
+      start_period: "15s"
+      retries: 3
+    deploy:
+      resources:
+        reservations:
+          memory: 512M
+        limits:
+          memory: 512M
+
+    depends_on:
+      - "memcached"
+      - "mysql"
+
+volumes:
+  db_data: {}
+  piler_etc: {}
+  piler_var: {}

docker/start.sh

@@ -4,91 +4,198 @@ set -o errexit
 set -o pipefail
 set -o nounset
 
-DATAROOTDIR="/usr/share"
-SYSCONFDIR="/etc"
-SPHINXCFG="/etc/piler/sphinx.conf"
-PILER_HOST=${PILER_HOST:-archive.yourdomain.com}
-PILER_CONF="/etc/piler/piler.conf"
-CONFIG_SITE_PHP="/etc/piler/config-site.php"
-CONFIG_PHP="/var/piler/www/config.php"
+CONFIG_DIR="/etc/piler"
+VOLUME_DIR="/var/piler"
+PILER_CONF="${CONFIG_DIR}/piler.conf"
+PILER_KEY="${CONFIG_DIR}/piler.key"
+PILER_PEM="${CONFIG_DIR}/piler.pem"
+PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf"
+SPHINX_CONF="${CONFIG_DIR}/sphinx.conf"
+CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php"
+PILER_MY_CNF="${CONFIG_DIR}/.my.cnf"
 
-create_mysql_db() {
-   echo "Creating mysql database"
 
-   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%g" \
-       -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%g" \
-       -e "s%MYSQL_USERNAME%${PILER_USER}%g" \
-       -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" \
-       "${DATAROOTDIR}/piler/db-mysql-root.sql.in" | \
-       mysql -h "$MYSQL_HOSTNAME" -u root --password="$MYSQL_ROOT_PASSWORD"
-
-   mysql -h "$MYSQL_HOSTNAME" -u "$PILER_USER" --password="$MYSQL_PILER_PASSWORD" "$MYSQL_DATABASE" < "${DATAROOTDIR}/piler/db-mysql.sql"
-
-   echo "Done."
+error() {
+   echo "ERROR:" "$*" 1>&2
+   exit 1
 }
 
 
-pre_seed_sphinx() {
-   echo "Writing sphinx configuration"
-   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \
-       -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \
-       -e "s%MYSQL_USERNAME%${PILER_USER}%" \
-       -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%" \
-       -e "s%220%311%" \
-       -e "s%type = mysql%type = mysql\n   sql_sock = /var/run/mysqld/mysqld.sock%" \
-       "${SYSCONFDIR}/piler/sphinx.conf.dist" > "$SPHINXCFG"
+log() {
+   echo "DEBUG:" "$*"
+}
 
-   echo "Done."
 
-   echo "Initializing sphinx indices"
-   su "$PILER_USER" -c "indexer --all --config ${SYSCONFDIR}/piler/sphinx.conf"
-   echo "Done."
+pre_flight_check() {
+   [[ -v PILER_HOSTNAME ]] || error "Missing PILER_HOSTNAME env variable"
+   [[ -v MYSQL_HOSTNAME ]] || error "Missing MYSQL_HOSTNAME env variable"
+   [[ -v MYSQL_DATABASE ]] || error "Missing MYSQL_DATABASE env variable"
+   [[ -v MYSQL_USER ]]     || error "Missing MYSQL_USER env variable"
+   [[ -v MYSQL_PASSWORD ]] || error "Missing MYSQL_PASSWORD env variable"
+}
+
+
+give_it_to_piler() {
+   local f="$1"
+
+   [[ -f "$f" ]] || error "${f} does not exist, aborting"
+
+   chown "${PILER_USER}:${PILER_USER}" "$f"
+   chmod 600 "$f"
+}
+
+
+make_certificate() {
+   local f="$1"
+   local crt="/tmp/1.cert"
+   local SSL_CERT_DATA="/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com"
+
+   log "Making an ssl certificate"
+
+   openssl req -new -newkey rsa:4096 -days 3650 -nodes -x509 -subj "$SSL_CERT_DATA" -keyout "$f" -out "$crt" -sha1 2>/dev/null
+   cat "$crt" >> "$f"
+   rm -f "$crt"
+
+   give_it_to_piler "$f"
+}
+
+
+make_piler_key() {
+   local f="$1"
+
+   log "Generating piler.key"
+
+   dd if=/dev/urandom bs=56 count=1 of="$f" 2>/dev/null
+   [[ $(stat -c '%s' "$f") -eq 56 ]] || error "could not read 56 bytes from /dev/urandom to ${f}"
+
+   give_it_to_piler "$f"
 }
 
 
 fix_configs() {
-   local piler_nginx_conf="/etc/piler/piler-nginx.conf"
+   [[ -f "$PILER_KEY" ]] || make_piler_key "$PILER_KEY"
+   [[ -f "$PILER_PEM" ]] || make_certificate "$PILER_PEM"
+
+   if [[ ! -f "$PILER_NGINX_CONF" ]]; then
+      log "Writing ${PILER_NGINX_CONF}"
+
+      cp "${PILER_NGINX_CONF}.dist" "$PILER_NGINX_CONF"
+      sed -i "s%PILER_HOST%${PILER_HOSTNAME}%" "$PILER_NGINX_CONF"
+   fi
 
    if [[ ! -f "$PILER_CONF" ]]; then
-      cp /etc/piler/piler.conf.dist "$PILER_CONF"
-      chmod 640 "$PILER_CONF"
-      chown root:piler "$PILER_CONF"
-      sed -i "s%hostid=.*%hostid=${PILER_HOST%%:*}%" "$PILER_CONF"
-      sed -i "s%tls_enable=.*%tls_enable=1%" "$PILER_CONF"
-      sed -i "s%mysqlpwd=.*%mysqlpwd=${MYSQL_PILER_PASSWORD}%" "$PILER_CONF"
+      log "Writing ${PILER_CONF}"
+
+      sed \
+         -e "s/mysqluser=.*/mysqluser=${MYSQL_USER}/g" \
+         -e "s/mysqldb=.*/mysqldb=${MYSQL_DATABASE}/g" \
+         -e "s/verystrongpassword/${MYSQL_PASSWORD}/g" \
+         -e "s/hostid=.*/hostid=${PILER_HOSTNAME}/g" \
+         -e "s/tls_enable=.*/tls_enable=1/g" \
+         -e "s/mysqlsocket=.*/mysqlsocket=/g" "${PILER_CONF}.dist" > "$PILER_CONF"
+
+      {
+         echo "mysqlhost=${MYSQL_HOSTNAME}"
+      } >> "$PILER_CONF"
+
+      give_it_to_piler "$PILER_CONF"
    fi
 
-   if [[ ! -f "$piler_nginx_conf" ]]; then
-      cp /etc/piler/piler-nginx.conf.dist "$piler_nginx_conf"
-      sed -i "s%PILER_HOST%${PILER_HOST}%" "$piler_nginx_conf"
+   if [[ ! -f "$CONFIG_SITE_PHP" ]]; then
+      log "Writing ${CONFIG_SITE_PHP}"
+
+      cp "${CONFIG_DIR}/config-site.dist.php" "$CONFIG_SITE_PHP"
+
+      sed -i "s%HOSTNAME%${PILER_HOSTNAME}%" "$CONFIG_SITE_PHP"
+
+      {
+         echo "\$config['DECRYPT_BINARY'] = '/usr/bin/pilerget';"
+         echo "\$config['DECRYPT_ATTACHMENT_BINARY'] = '/usr/bin/pileraget';"
+         echo "\$config['PILER_BINARY'] = '/usr/sbin/piler';"
+         echo "\$config['DB_HOSTNAME'] = '$MYSQL_HOSTNAME';"
+         echo "\$config['DB_DATABASE'] = '$MYSQL_DATABASE';"
+         echo "\$config['DB_USERNAME'] = '$MYSQL_USER';"
+         echo "\$config['DB_PASSWORD'] = '$MYSQL_PASSWORD';"
+         echo "\$config['ENABLE_MEMCACHED'] = 1;"
+         echo "\$memcached_server = ['memcached', 11211];"
+      } >> "$CONFIG_SITE_PHP"
    fi
 
-   ln -sf "$piler_nginx_conf" /etc/nginx/sites-enabled/piler
-
-   sed -i "s%HOSTNAME%${PILER_HOST}%" "$CONFIG_SITE_PHP"
-   sed -i "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%" "$CONFIG_SITE_PHP"
-
-   sed -i "s%^\$config\['DECRYPT_BINARY'\].*%\$config\['DECRYPT_BINARY'\] = '/usr/bin/pilerget';%" "$CONFIG_PHP"
-   sed -i "s%^\$config\['DECRYPT_ATTACHMENT_BINARY'\].*%\$config\['DECRYPT_ATTACHMENT_BINARY'\] = '/usr/bin/pileraget';%" "$CONFIG_PHP"
-   sed -i "s%^\$config\['PILER_BINARY'\].*%\$config\['PILER_BINARY'\] = '/usr/sbin/piler';%" "$CONFIG_PHP"
+   sed -e "s%MYSQL_HOSTNAME%${MYSQL_HOSTNAME}%" \
+       -e "s%MYSQL_DATABASE%${MYSQL_DATABASE}%" \
+       -e "s%MYSQL_USERNAME%${MYSQL_USER}%" \
+       -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" \
+       -i "$SPHINX_CONF"
 }
 
 
-service rsyslog start
-service mysql start
+wait_until_mysql_server_is_ready() {
+   while true; do if mysql "--defaults-file=${PILER_MY_CNF}" <<< "show databases"; then break; fi; log "${MYSQL_HOSTNAME} is not ready"; sleep 5; done
 
-create_mysql_db
-pre_seed_sphinx
+   log "${MYSQL_HOSTNAME} is ready"
+}
+
+
+init_database() {
+   local table
+   local has_metadata_table=0
+
+   wait_until_mysql_server_is_ready
+
+   while read -r table; do
+      if [[ "$table" == metadata ]]; then has_metadata_table=1; fi
+   done < <(mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< 'show tables')
+
+   if [[ $has_metadata_table -eq 0 ]]; then
+      log "no metadata table, creating tables"
+
+      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" < /usr/share/piler/db-mysql.sql
+   else
+      log "metadata table exists"
+   fi
+
+   if [[ -v ADMIN_USER_PASSWORD_HASH ]]; then
+      mysql "--defaults-file=${PILER_MY_CNF}" "$MYSQL_DATABASE" <<< "update user set password='${ADMIN_USER_PASSWORD_HASH}' where uid=0"
+   fi
+}
+
+
+create_my_cnf_files() {
+   printf "[client]\nhost = %s\nuser = %s\npassword = %s\n[mysqldump]\nhost = %s\nuser = %s\npassword = %s\n" \
+      "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" "$MYSQL_HOSTNAME" "$MYSQL_USER" "$MYSQL_PASSWORD" \
+      > "$PILER_MY_CNF"
+
+   give_it_to_piler "$PILER_MY_CNF"
+}
+
+
+start_services() {
+   service rsyslog start
+   service cron start
+   service php7.4-fpm start
+   service nginx start
+}
+
+
+start_piler() {
+   if [[ ! -f "${VOLUME_DIR}/sphinx/main1.spp" ]]; then
+      log "main1.spp does not exist, creating index files"
+      su -c "indexer --all --config ${SPHINX_CONF}" piler
+   fi
+
+   # No pid file should exist for piler
+   rm -f /var/run/piler/*pid
+
+   /etc/init.d/rc.searchd start
+   /etc/init.d/rc.piler start
+}
+
+
+pre_flight_check
 fix_configs
+create_my_cnf_files
+init_database
+start_services
+start_piler
 
-service cron start
-service php7.2-fpm start
-service nginx start
-/etc/init.d/rc.searchd start
-
-# fix for overlay, https://github.com/phusion/baseimage-docker/issues/198
-touch /var/spool/cron/crontabs/piler
-
-/etc/init.d/rc.piler start
-
-while true; do sleep 120; done
+while true; do sleep 3600; done

etc/cron.jobs.in

@@ -6,7 +6,6 @@
 */15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet note1 --rotate
 */5 * * * *  /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \;
 */5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error
-3 * * * * LIBEXECDIR/piler/watch_sphinx_main_index.sh
 2 0 * * * LIBEXECDIR/piler/pilerpurge.py
 
 ### optional: populate accouting data

etc/example.conf

@@ -22,10 +22,11 @@ username=piler
 ; The default is 7 years + 2 days (=7*365+2=2557 days)
 default_retention_days=2557
 
-; this is a 16 character long vector
-; after the installation you must not change it ever
-; otherwise you can't access your emails
-;iv=****************
+; The initialization vector for encryption.
+; By now it has become obsolete. Don't use it for
+; new installations. However, if you used it before
+; then you must keep it as it is.
+;iv=
 
 ; whether to encrypt messages (1) or not (0).
 ; Make sure to set this value to your needs right after installing piler,
@@ -106,6 +107,13 @@ pemfile=
 cipher_list=ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS
 
 
+; set the minimum TLS protocol version for piler-smtp daemon
+;
+; Valid values: TLSv1, TLSv1.1, TLSv1.2, TLSv1.3
+; TLSv1 and TLSv1.1 are not recommended for security reasons
+tls_min_version=TLSv1.2
+
+
 ; piler's own header to indicate previously archived messages
 piler_header_field=X-piler-id:
 
@@ -221,3 +229,29 @@ tweak_sent_time_offset=0
 ; whether to enable (1) or not (0) the extra mmap dedup test feature
 ; if you change it, be sure to stop, then start piler
 mmap_dedup_test=0
+
+; security header that must be present in the first mail header of
+; the message. If the security_header value is not empty, then the
+; parser checks for this header line. Unless it's found it will discard
+; the given email. Note that this feature is supposed to be a security
+; mechanism against unwanted email in the archive if limiting smtp
+; clients via an IP-address list is not feasible.
+security_header=
+
+; By default the archive accepts any envelope recipient addresses.
+; If your archive's port 25 is wide open to the Internet (which it
+; shouldn't be, then spammers may find it, and fill it with spam.
+;
+; By setting this variable you may restrict the envelope address
+; to a single email address, eg. some-random-address-12345@archive.yourdomain.com
+; Then the archive will reject any other envelope recipients
+archive_address=
+
+; whether to enable (1) or not (0) an smtp access list similar to
+; postfix's postscreen. Valid actions in the acl file are "permit"
+; and "reject" (without quotes). See smtp.acl.example for more.
+;
+; Important! There's an implicit default deny at the end of the
+; rules. In other words if you decide to use the acl file, then
+; everyone is not explicitly permitted is denied.
+smtp_access_list=0

etc/smtp.acl.example

@@ -0,0 +1,9 @@
+# Allow office365 servers. See the below URI for more
+# https://docs.microsoft.com/en-us/microsoft-365/enterprise/urls-and-ip-address-ranges?view=o365-worldwide
+#
+# Anything is not listed below will be rejected
+#
+40.92.0.0/15 permit
+40.107.0.0/16 permit
+52.100.0.0/14 permit
+104.47.0.0/17 permit

piler-config.h.in

@@ -26,8 +26,6 @@
 #undef HAVE_TNEF
 #undef HAVE_ZIP
 
-#undef HAVE_LIBWRAP
-
 #undef HAVE_TWEAK_SENT_TIME
 
 #undef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT

src/Makefile.in

@@ -48,7 +48,7 @@ libpiler.a: $(OBJS) $(SQL_OBJS)
 	ln -sf libpiler.so.$(LIBPILER_VERSION) libpiler.so.$(PILER_VERSION)
 
 piler-smtp: piler-smtp.c libpiler.a
-	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< cfg.o misc.o tai.o smtp.o session.o dirs.o sig.o bdat.o $(LIBS) $(LIBDIR)
+	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< cfg.o misc.o tai.o smtp.o session.o dirs.o sig.o bdat.o screen.o $(LIBS) $(LIBDIR)
 
 pilerget: pilerget.c libpiler.a
 	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)

src/archive.c

@@ -58,6 +58,10 @@ int inf(unsigned char *in, int len, int mode, char **buffer, FILE *dest){
    char *new_ptr;
    unsigned char out[REALLYBIGBUFSIZE];
 
+   /* expecting deflate with 32k window size (0x78) */
+   if(len > 0 && in[0] != 0x78)
+      return Z_DATA_ERROR;
+
    /* allocate inflate state */
 
    strm.zalloc = Z_NULL;
@@ -139,6 +143,7 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
 #else
    EVP_CIPHER_CTX *ctx=NULL;
 #endif
+   int blocklen;
 
 
    if(filename == NULL) return 1;
@@ -157,20 +162,33 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
       return 1;
    }
 
+   // The new encryption scheme uses piler id starting with 5000....
 
    if(cfg->encrypt_messages == 1){
    #if OPENSSL_VERSION_NUMBER < 0x10100000L
       EVP_CIPHER_CTX_init(&ctx);
-      EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      if(strstr(filename, "/5000")){
+         EVP_DecryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+      } else {
+         EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      }
+
+      blocklen = EVP_CIPHER_CTX_block_size(&ctx);
    #else
       ctx = EVP_CIPHER_CTX_new();
       if(!ctx) goto CLEANUP;
 
       EVP_CIPHER_CTX_init(ctx);
-      EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      if(strstr(filename, "/5000")){
+         EVP_DecryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+      } else {
+         EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      }
+
+      blocklen = EVP_CIPHER_CTX_block_size(ctx);
    #endif
 
-      len = st.st_size+EVP_MAX_BLOCK_LENGTH;
+      len = st.st_size+blocklen;
 
       s = malloc(len);
 
@@ -207,7 +225,13 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
 
 
       tlen += olen;
+
+      // old fileformat with static IV
       rc = inf(s, tlen, mode, buffer, dest);
+      // new fileformat, starting with blocklen bytes of garbage
+      if(rc != Z_OK && tlen >= blocklen){
+         rc = inf(s+blocklen, tlen-blocklen, mode, buffer, dest);
+      }
    }
    else {
       addr = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
@@ -220,7 +244,7 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
 
 
 CLEANUP:
-   if(fd != -1) close(fd);
+   if(fd != -1) close(fd); //-V547
    if(s) free(s);
    if(cfg->encrypt_messages == 1)
    #if OPENSSL_VERSION_NUMBER < 0x10100000L

src/cfg.c

@@ -39,7 +39,7 @@ struct _parse_rule {
 
 struct _parse_rule config_parse_rules[] =
 {
-
+   { "archive_address", "string", (void*) string_parser, offsetof(struct config, archive_address), "", MAXVAL-1},
    { "archive_emails_not_having_message_id", "integer", (void*) int_parser, offsetof(struct config, archive_emails_not_having_message_id), "0", sizeof(int)},
    { "archive_only_mydomains", "integer", (void*) int_parser, offsetof(struct config, archive_only_mydomains), "0", sizeof(int)},
    { "backlog", "integer", (void*) int_parser, offsetof(struct config, backlog), "20", sizeof(int)},
@@ -84,11 +84,14 @@ struct _parse_rule config_parse_rules[] =
    { "piler_header_field", "string", (void*) string_parser, offsetof(struct config, piler_header_field), "X-piler-id:", MAXVAL-1},
    { "process_rcpt_to_addresses", "integer", (void*) int_parser, offsetof(struct config, process_rcpt_to_addresses), "0", sizeof(int)},
    { "queuedir", "string", (void*) string_parser, offsetof(struct config, queuedir), QUEUE_DIR, MAXVAL-1},
+   { "security_header", "string", (void*) string_parser, offsetof(struct config, security_header), "", MAXVAL-1},
    { "server_id", "integer", (void*) int_parser, offsetof(struct config, server_id), "0", sizeof(int)},
+   { "smtp_access_list", "integer", (void*) int_parser, offsetof(struct config, smtp_access_list), "0", sizeof(int)},
    { "smtp_timeout", "integer", (void*) int_parser, offsetof(struct config, smtp_timeout), "60", sizeof(int)},
    { "spam_header_line", "string", (void*) string_parser, offsetof(struct config, spam_header_line), "", MAXVAL-1},
    { "syslog_recipients", "integer", (void*) int_parser, offsetof(struct config, syslog_recipients), "0", sizeof(int)},
    { "tls_enable", "integer", (void*) int_parser, offsetof(struct config, tls_enable), "0", sizeof(int)},
+   { "tls_min_version", "string", (void*) string_parser, offsetof(struct config, tls_min_version), "TLSv1.2", MAXVAL-1},
    { "tweak_sent_time_offset", "integer", (void*) int_parser, offsetof(struct config, tweak_sent_time_offset), "0", sizeof(int)},
    { "update_counters_to_memcached", "integer", (void*) int_parser, offsetof(struct config, update_counters_to_memcached), "0", sizeof(int)},
    { "username", "string", (void*) string_parser, offsetof(struct config, username), "piler", MAXVAL-1},
@@ -144,6 +147,24 @@ int parse_config_file(char *configfile, struct config *target_cfg, struct _parse
 }
 
 
+int get_tls_protocol_number(char *protocol){
+   struct tls_protocol tls_protocols[] = {
+      { "TLSv1", TLS1_VERSION },
+      { "TLSv1.1", TLS1_1_VERSION },
+      { "TLSv1.2", TLS1_2_VERSION },
+      { "TLSv1.3", TLS1_3_VERSION },
+   };
+
+   for(unsigned int i=0; i<sizeof(tls_protocols)/sizeof(struct tls_protocol); i++){
+      if(!strcmp(protocol, tls_protocols[i].proto)) {
+         return tls_protocols[i].version;
+      }
+   }
+
+   return 0;
+}
+
+
 int load_default_config(struct config *cfg, struct _parse_rule *rules){
    int i=0;
 
@@ -176,6 +197,9 @@ struct config read_config(char *configfile){
 
    cfg.hostid_len = strlen(cfg.hostid);
 
+   // Get the TLS protocol constant from string, ie. TLSv1.3 -> 772
+   cfg.tls_min_version_number = get_tls_protocol_number(cfg.tls_min_version);
+
    return cfg;
 }
 

src/cfg.h

@@ -29,6 +29,8 @@ struct config {
    int tls_enable;
    char pemfile[MAXVAL];
    char cipher_list[MAXVAL];
+   char tls_min_version[MAXVAL];
+   int tls_min_version_number;
 
    int use_antivirus;
 
@@ -64,6 +66,9 @@ struct config {
 
    int default_retention_days;
 
+   char security_header[MAXVAL];
+   char archive_address[MAXVAL];
+
    // mysql stuff
 
    char mysqlcharset[MAXVAL];
@@ -97,6 +102,8 @@ struct config {
    int enable_folders;
 
    int debug;
+
+   int smtp_access_list;
 };
 
 

src/config.h

@@ -9,11 +9,12 @@
 #include "piler-config.h"
 #include "params.h"
 
-#define BUILD 998
+#define BUILD 1001
 
 #define HOSTID "mailarchiver"
 
 #define CONFIG_FILE CONFDIR "/piler/piler.conf"
+#define SMTP_ACL_FILE CONFDIR "/piler/smtp.acl"
 #define WORK_DIR DATADIR "/piler/tmp"
 #define QUEUE_DIR DATADIR "/piler/store"
 #define ERROR_DIR DATADIR "/piler/error"
@@ -57,6 +58,8 @@
 #define MEMCACHED_MSGS_STORED_SIZE MEMCACHED_CLAPF_PREFIX "stored_size"
 
 
+#define PILEREXPORT_BEGIN_MARK "x-exported-by-pilerexport: start\n"
+
 #define LOG_PRIORITY LOG_INFO
 
 #define _LOG_INFO 3

src/defs.h

@@ -13,9 +13,6 @@
    #include <tre/tre.h>
    #include <tre/regex.h>
 #endif
-#ifdef HAVE_LIBWRAP
-   #include <tcpd.h>
-#endif
 
 #include <openssl/sha.h>
 #include <openssl/ssl.h>
@@ -27,16 +24,17 @@
 #define MSG_BODY 0
 #define MSG_RECEIVED 1
 #define MSG_FROM 2
-#define MSG_TO 3
-#define MSG_CC 4
-#define MSG_SUBJECT 5
-#define MSG_CONTENT_TYPE 6
-#define MSG_CONTENT_TRANSFER_ENCODING 7
-#define MSG_CONTENT_DISPOSITION 8
-#define MSG_MESSAGE_ID 9
-#define MSG_REFERENCES 10
-#define MSG_RECIPIENT 11
-#define MSG_ENVELOPE_TO 12
+#define MSG_SENDER 3
+#define MSG_TO 4
+#define MSG_CC 5
+#define MSG_SUBJECT 6
+#define MSG_CONTENT_TYPE 7
+#define MSG_CONTENT_TRANSFER_ENCODING 8
+#define MSG_CONTENT_DISPOSITION 9
+#define MSG_MESSAGE_ID 10
+#define MSG_REFERENCES 11
+#define MSG_RECIPIENT 12
+#define MSG_ENVELOPE_TO 13
 
 #define MAXHASH 277
 
@@ -97,6 +95,15 @@ struct node {
 };
 
 
+struct smtp_acl {
+   char network_str[BUFLEN];
+   in_addr_t low, high;
+   int prefix;
+   int rejected;
+   struct smtp_acl *r;
+};
+
+
 struct net {
    int socket;
    int use_ssl;
@@ -203,12 +210,14 @@ struct parser_state {
 
    char reference[SMALLBUFSIZE];
 
-   char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE];
+   char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_sender[SMALLBUFSIZE], b_sender_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE];
    char b_journal_to[MAXBUFSIZE];
 
    unsigned int bodylen;
    unsigned int tolen;
    unsigned int todomainlen;
+   unsigned int found_security_header;
+
    int journaltolen;
 
    int retention;
@@ -299,6 +308,7 @@ struct import {
    int keep_eml;
    int timeout;
    int cap_uidplus;
+   int fd;
    long total_size;
    int dryrun;
    int tot_msgs;
@@ -388,7 +398,7 @@ struct smtp_session {
    char mailfrom[SMALLBUFSIZE];
    char rcptto[MAX_RCPT_TO][SMALLBUFSIZE];
    char buf[MAXBUFSIZE];
-   char remote_host[INET6_ADDRSTRLEN];
+   char remote_host[INET6_ADDRSTRLEN+1];
    time_t lasttime;
    int protocol_state;
    int slot;
@@ -403,4 +413,9 @@ struct smtp_session {
    struct net net;
 };
 
+struct tls_protocol {
+   char *proto;
+   int version;
+};
+
 #endif /* _DEFS_H */

src/digest.c

@@ -48,6 +48,7 @@ int make_digests(struct session_data *sdata, struct config *cfg){
    fd = open(sdata->filename, O_RDONLY);
    if(fd == -1) return -1;
 
+   memset(buf, 0, sizeof(buf));
 
    while((n = read(fd, buf, sizeof(buf))) > 0){
       SHA256_Update(&context2, buf, n);
@@ -80,8 +81,8 @@ int make_digests(struct session_data *sdata, struct config *cfg){
    SHA256_Final(md2, &context2);
 
    for(i=0;i<DIGEST_LENGTH;i++){
-      snprintf(sdata->bodydigest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);
-      snprintf(sdata->digest + i*2, 2*DIGEST_LENGTH, "%02x", md2[i]);
+      snprintf(sdata->bodydigest + i*2, 3, "%02x", md[i]);
+      snprintf(sdata->digest + i*2, 3, "%02x", md2[i]);
    }
 
    return 0;

src/extract.c

@@ -25,12 +25,20 @@ int remove_xml(char *src, char *dest, int destlen, int *html){
    memset(dest, 0, destlen);
 
    for(; *src; src++){
-      if(*src == '<'){ *html = 1; continue; }
-      if(*src == '>'){ *html = 0; continue; }
-
-      if(*html == 0){
-         if(i < destlen) *(dest+i) = *src;
-         i++;
+      if(*src == '<'){
+         *html = 1;
+      }
+      else if(*src == '>'){
+         *html = 0;
+         // make sure there's a whitespace between tag contents
+         if(i < destlen && i > 0 && !isspace(dest[i-1]))
+            dest[i++] = ' ';
+      }
+      else{
+         if(*html == 0){
+            if(i < destlen) *(dest+i) = *src;
+            i++;
+         }
       }
    }
 

src/imap.c

@@ -416,7 +416,7 @@ int list_folders(struct data *data){
    }
 
    // trim the "A3 OK LIST completed" trailer off
-   if(p) *p = '\0';
+   if(p) *p = '\0'; //-V547
 
    memset(attrs, 0, sizeof(attrs));
 
@@ -449,21 +449,27 @@ int list_folders(struct data *data){
             } else {
 
                if(fldrlen) {
-                  ruf = malloc(strlen(q) * 2 + 1);
-                  memset(ruf, 0, strlen(q) * 2 + 1);
-                  memcpy(ruf, q, strlen(q));
-                  r = ruf;
-                  while(*r != '\0') {
-                     if(*r == '\\') {
-                        memmove(r + 1, r, strlen(r));
+                  int ruflen = strlen(q) * 2;
+                  ruf = malloc(ruflen + 1);
+                  if(ruf){
+                     snprintf(ruf, ruflen, "%s", q);
+                     r = ruf;
+                     while(*r != '\0') {
+                        if(*r == '\\') {
+                           memmove(r + 1, r, strlen(r));
+                           r++;
+                        }
                         r++;
                      }
-                     r++;
+
+                     snprintf(folder, sizeof(folder)-1, "%s", ruf);
+
+                     free(ruf);
+                  }
+                  else {
+                     printf("error: ruf = malloc()\n");
                   }
 
-                  snprintf(folder, sizeof(folder)-1, "%s", ruf);
-
-                  free(ruf);
                   fldrlen = 0;
                } else {
                   snprintf(folder, sizeof(folder)-1, "%s", q);

src/import.h

@@ -23,4 +23,6 @@ int list_folders(struct data *data);
 int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct config *cfg);
 void send_imap_close(struct data *data);
 
+void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg);
+
 #endif /* _IMPORT_H */

src/import_pilerexport.c

@@ -0,0 +1,109 @@
+/*
+ * import_pop3.c
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <getopt.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+void import_the_file(struct session_data *sdata, struct data *data, struct config *cfg){
+   close(data->import->fd);
+   data->import->fd = -1;
+
+   if(import_message(sdata, data, cfg) != ERR){
+      unlink(data->import->filename);
+   }
+}
+
+
+void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct config *cfg){
+
+   if(!strcmp(buf, PILEREXPORT_BEGIN_MARK)){
+      if((*count) > 0){
+         import_the_file(sdata, data, cfg);
+      }
+
+      (*count)++;
+
+      snprintf(data->import->filename, SMALLBUFSIZE-1, "import-%llu", *count);
+
+      data->import->fd = open(data->import->filename, O_CREAT|O_WRONLY|O_TRUNC, S_IRUSR|S_IWUSR);
+      if(data->import->fd == -1){
+         // Do some error handling
+         printf("error: cannot open %s\n", data->import->filename);
+      }
+
+   } else {
+      if(write(data->import->fd, buf, buflen) != buflen){
+         printf("error: didnt write %d bytes\n", buflen);
+      }
+   }
+}
+
+
+void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg){
+   int n, rc, savedlen=0, puflen;
+   uint64 count=0;
+   char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE];
+
+   memset(savedbuf, 0, sizeof(savedbuf));
+
+   data->import->fd = -1;
+
+   do {
+      memset(buf, 0, sizeof(buf));
+      n = fread(buf, 1, sizeof(buf)-1, stdin);
+
+      if(savedlen > 0){
+         memset(copybuf, 0, sizeof(copybuf));
+
+         memcpy(copybuf, savedbuf, savedlen);
+         memcpy(&copybuf[savedlen], buf, n);
+
+         savedlen = 0;
+         memset(savedbuf, 0, sizeof(savedbuf));
+
+         p = &copybuf[0];
+      }
+      else {
+         p = &buf[0];
+      }
+
+      do {
+         puflen = read_one_line(p, '\n', puf, sizeof(puf), &rc);
+         p += puflen;
+
+         if(puflen > 0){
+            if(rc == OK){
+               process_buffer(puf, puflen, &count, sdata, data, cfg);
+            }
+            else {
+               snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
+               savedlen = puflen;
+            }
+         }
+
+      } while(puflen > 0);
+
+   } while(n > 0);
+
+   if(data->import->fd != -1){
+      import_the_file(sdata, data, cfg);
+   }
+}

src/message.c

@@ -20,7 +20,7 @@
 
 int store_index_data(struct session_data *sdata, struct parser_state *state, struct data *data, uint64 id, struct config *cfg){
    int rc=ERR;
-   char *subj;
+   char *subj, *sender=state->b_from, *sender_domain=state->b_from_domain;
    struct sql sql;
 
    if(data->folder == 0){
@@ -35,17 +35,23 @@ int store_index_data(struct session_data *sdata, struct parser_state *state, str
 
 
    fix_email_address_for_sphinx(state->b_from);
+   fix_email_address_for_sphinx(state->b_sender);
    fix_email_address_for_sphinx(state->b_to);
    fix_email_address_for_sphinx(state->b_from_domain);
+   fix_email_address_for_sphinx(state->b_sender_domain);
    fix_email_address_for_sphinx(state->b_to_domain);
 
+   if(state->b_sender_domain[0]){
+      sender = state->b_sender;
+      sender_domain = state->b_sender_domain;
+   }
 
    p_bind_init(&sql);
 
    sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
-   sql.sql[sql.pos] = state->b_from; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = sender; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
    sql.sql[sql.pos] = state->b_to; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = state->b_from_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
    sql.sql[sql.pos] = state->b_to_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
    sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
    sql.sql[sql.pos] = state->b_body; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
@@ -180,15 +186,25 @@ int update_metadata_reference(struct session_data *sdata, struct parser_state *s
 
 
 int store_meta_data(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
-   int rc=ERR, result;
-   char *subj, *p, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1];
+   int rc=ERR;
+   char *subj, *sender, *sender_domain, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1];
    uint64 id=0;
    struct sql sql;
 
    subj = state->b_subject;
    if(*subj == ' ') subj++;
 
-   snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, state->b_from, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
+   if(state->b_sender_domain[0]){
+      sender = state->b_sender;
+      sender_domain = state->b_sender_domain;
+      get_first_email_address_from_string(state->b_sender, s2, sizeof(s2));
+   } else {
+      sender = state->b_from;
+      sender_domain = state->b_from_domain;
+      get_first_email_address_from_string(state->b_from, s2, sizeof(s2));
+   }
+
+   snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, sender, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
 
    digest_string(s, &vcode[0]);
 
@@ -201,19 +217,6 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru
 
    if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR;
 
-   memset(s2, 0, sizeof(s2));
-
-   p = state->b_from;
-   do {
-      memset(s2, 0, sizeof(s2));
-      p = split(p, ' ', s2, sizeof(s2)-1, &result);
-
-      if(s2[0] == '\0') continue;
-
-      if(does_it_seem_like_an_email_address(s2) == 1){ break; }
-   } while(p);
-
-
    if(strlen(state->b_to) < 5){
       snprintf(state->b_to, SMALLBUFSIZE-1, "undisclosed-recipients@no.domain");
    }
@@ -222,7 +225,7 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru
    p_bind_init(&sql);
 
    sql.sql[sql.pos] = &s2[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
-   sql.sql[sql.pos] = state->b_from_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+   sql.sql[sql.pos] = sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
    sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
    sql.sql[sql.pos] = (char *)&sdata->spam_message; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
    sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++;

src/misc.c

@@ -230,11 +230,8 @@ int extractEmail(char *rawmail, char *email){
  */
 
 void make_random_string(unsigned char *buf, int buflen){
-   int i, len, fd;
-   int urandom=0;
-   static char alphanum[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
-
-   len = strlen(alphanum);
+   const char alphanum[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+   int i, fd, urandom=0, len = sizeof(alphanum)-1;
 
    fd = open(RANDOM_POOL, O_RDONLY);
    if(fd != -1){
@@ -265,11 +262,13 @@ void create_id(char *id, unsigned char server_id){
 
    get_random_bytes(buf, RND_STR_LEN/2, server_id);
 
+   // New encryption scheme using AES-256
+   buf[0] = 0x50;
+
    for(i=0; i < RND_STR_LEN/2; i++){
       sprintf(id, "%02x", buf[i]);
       id += 2;
    }
-
 }
 
 
@@ -287,7 +286,7 @@ int get_random_bytes(unsigned char *buf, int len, unsigned char server_id){
    taia_now(&now);
    taia_pack(nowpack, &now);
 
-   memcpy(buf, nowpack, 12);
+   memcpy(buf, nowpack, 12); //-V512
 
    fd = open(RANDOM_POOL, O_RDONLY);
    if(fd == -1) return ret;

src/parser.c

@@ -45,7 +45,7 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec
 
    if(take_into_pieces == 1 && state.writebufpos > 0){
       if(write(state.mfd, writebuffer, state.writebufpos) == -1) syslog(LOG_PRIORITY, "ERROR: %s: write(), %s, %d, %s", sdata->ttmpfile, __func__, __LINE__, __FILE__);
-      memset(writebuffer, 0, sizeof(writebuffer));
+      memset(writebuffer, 0, sizeof(writebuffer)); //-V597
       state.writebufpos = 0;
    }
 
@@ -68,6 +68,16 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec
       add_recipient(data->import->extra_recipient, strlen(data->import->extra_recipient), sdata, &state, data, cfg);
    }
 
+   // If both Sender: and From: headers exist, and they are different, then append
+   // the From: address to recipients list to give him access to this email as well
+
+   if(state.b_sender_domain[0] && strcmp(state.b_from, state.b_sender)){
+      char tmpbuf[SMALLBUFSIZE];
+      get_first_email_address_from_string(state.b_from, tmpbuf, sizeof(tmpbuf));
+      tmpbuf[strlen(tmpbuf)] = ' ';
+      add_recipient(tmpbuf, strlen(tmpbuf), sdata, &state, data, cfg);
+   }
+
    return state;
 }
 
@@ -80,10 +90,13 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c
    clearhash(state->rcpt_domain);
    clearhash(state->journal_recipient);
 
-   // Fix From: line if it's too long
+   // Fix From: and Sender: lines if they are too long
    if(strlen(state->b_from) > 255) state->b_from[255] = '\0';
    if(strlen(state->b_from_domain) > 255) state->b_from_domain[255] = '\0';
 
+   if(strlen(state->b_sender) > 255) state->b_sender[255] = '\0';
+   if(strlen(state->b_sender_domain) > 255) state->b_sender_domain[255] = '\0';
+
    // Truncate the message_id if it's >255 characters
    if(strlen(state->message_id) > 255) state->message_id[255] = '\0';
 
@@ -199,6 +212,10 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
          sdata->restored_copy = 1;
       }
 
+      if(cfg->security_header[0] && state->found_security_header == 0 && strstr(buf, cfg->security_header)){
+         state->found_security_header = 1;
+      }
+
       if(*(cfg->piler_header_field) != 0 && strncmp(buf, cfg->piler_header_field, strlen(cfg->piler_header_field)) == 0){
          sdata->restored_copy = 1;
       }
@@ -354,6 +371,10 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
          state->message_state = MSG_FROM;
          buf += strlen("From:");
       }
+      else if(strncasecmp(buf, "Sender:", strlen("Sender:")) == 0){
+         state->message_state = MSG_SENDER;
+         buf += strlen("Sender:");
+      }
       else if(strncasecmp(buf, "Content-Type:", strlen("Content-Type:")) == 0){
          state->message_state = MSG_CONTENT_TYPE;
       }
@@ -541,6 +562,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
             memset(state->b_body, 0, BIGBUFSIZE);
             memset(state->b_from, 0, SMALLBUFSIZE);
             memset(state->b_from_domain, 0, SMALLBUFSIZE);
+            memset(state->b_sender, 0, SMALLBUFSIZE);
+            memset(state->b_sender_domain, 0, SMALLBUFSIZE);
             memset(state->message_id, 0, SMALLBUFSIZE);
 
             sdata->ms_journal = 0;
@@ -612,7 +635,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
 
 
    /* skip irrelevant headers */
-   if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT && state->message_state != MSG_ENVELOPE_TO) return 0;
+   if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_SENDER && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT && state->message_state != MSG_ENVELOPE_TO) return 0;
 
 
    /* don't process body if it's not a text or html part */

src/parser.h

@@ -20,7 +20,7 @@ void fixupEncodedHeaderLine(char *buf, int buflen);
 void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state);
 void fixupBase64EncodedLine(char *buf, struct parser_state *state);
 void markHTML(char *buf, struct parser_state *state);
-int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state);
+void setStateHTMLStyle(char *htmlbuf, int pos, struct parser_state *state);
 void translateLine(unsigned char *p, struct parser_state *state);
 void fix_email_address_for_sphinx(char *s);
 void split_email_address(char *s);
@@ -38,5 +38,6 @@ void fix_plus_sign_in_email_address(char *puf, char **at_sign, unsigned int *len
 void tokenize(char *buf, struct parser_state *state, struct session_data *sdata, struct data *data, struct config *cfg);
 void flush_attachment_buffer(struct parser_state *state, char *abuffer, unsigned int abuffersize);
 void fill_attachment_name_buf(struct parser_state *state, char *buf);
+int get_first_email_address_from_string(char *str, char *buf, int buflen);
 
 #endif /* _PARSER_H */

src/parser_utils.c

@@ -90,6 +90,8 @@ void init_state(struct parser_state *state){
 
    memset(state->b_from, 0, SMALLBUFSIZE);
    memset(state->b_from_domain, 0, SMALLBUFSIZE);
+   memset(state->b_sender, 0, SMALLBUFSIZE);
+   memset(state->b_sender_domain, 0, SMALLBUFSIZE);
    memset(state->b_to, 0, MAXBUFSIZE);
    memset(state->b_to_domain, 0, SMALLBUFSIZE);
    memset(state->b_subject, 0, MAXBUFSIZE);
@@ -102,6 +104,8 @@ void init_state(struct parser_state *state){
    state->journaltolen = 0;
 
    state->retention = 0;
+
+   state->found_security_header = 0;
 }
 
 
@@ -259,6 +263,18 @@ time_t parse_date_header(char *datestr){
 
    ts += get_local_timezone_offset() - offset;
 
+   if(ts < 700000000){
+      // If the Date: field contains some garbage, eg.
+      // "Date: [mail_datw]" or similar, and the date
+      // is before Sat Mar  7 20:26:40 UTC 1992, then
+      // return the current timestamp
+
+      time_t now;
+      time(&now);
+
+      return now;
+   }
+
    return ts;
 }
 
@@ -534,7 +550,7 @@ void markHTML(char *buf, struct parser_state *state){
 
             if(isspace(*s)){
                if(j > 0){
-                  k += appendHTMLTag(puf, html, pos, state);
+                  setStateHTMLStyle(html, pos, state);
                   memset(html, 0, SMALLBUFSIZE); j=0;
                }
                pos++;
@@ -559,7 +575,7 @@ void markHTML(char *buf, struct parser_state *state){
 
          if(j > 0){
             strncat(html, " ", SMALLBUFSIZE-1);
-            k += appendHTMLTag(puf, html, pos, state);
+            setStateHTMLStyle(html, pos, state);
             memset(html, 0, SMALLBUFSIZE); j=0;
          }
       }
@@ -567,47 +583,15 @@ void markHTML(char *buf, struct parser_state *state){
    }
 
    //printf("append last in line:*%s*, html=+%s+, j=%d\n", puf, html, j);
-   if(j > 0){ appendHTMLTag(puf, html, pos, state); }
+   if(j > 0){ setStateHTMLStyle(html, pos, state); }
 
    strcpy(buf, puf);
 }
 
 
-int appendHTMLTag(char *buf, char *htmlbuf, int pos, struct parser_state *state){
-   char html[SMALLBUFSIZE];
-   int len;
-
+void setStateHTMLStyle(char *htmlbuf, int pos, struct parser_state *state){
    if(pos == 0 && strncmp(htmlbuf, "style ", 6) == 0) state->style = 1;
    if(pos == 0 && strncmp(htmlbuf, "/style ", 7) == 0) state->style = 0;
-
-   return 0;
-
-   //printf("appendHTML: pos:%d, +%s+\n", pos, htmlbuf);
-
-   if(state->style == 1) return 0;
-
-   if(strlen(htmlbuf) == 0) return 0;
-
-   snprintf(html, SMALLBUFSIZE-1, "HTML*%s", htmlbuf);
-   len = strlen(html);
-
-   if(len > 8 && strchr(html, '=')){
-      char *p = strstr(html, "cid:");
-      if(p){
-         *(p+3) = '\0';
-         strncat(html, " ", SMALLBUFSIZE-1);
-      }
-
-      strncat(buf, html, MAXBUFSIZE-1);
-      return len;
-   }
-
-   if(strstr(html, "http") ){
-      strncat(buf, html+5, MAXBUFSIZE-1);
-      return len-5;
-   }
-
-   return 0;
 }
 
 
@@ -620,9 +604,9 @@ void translateLine(unsigned char *p, struct parser_state *state){
 
    for(; *p; p++){
 
-      if( (state->message_state == MSG_RECEIVED || state->message_state == MSG_FROM || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && *p == '@'){ continue; }
+      if( (state->message_state == MSG_RECEIVED || state->message_state == MSG_FROM || state->message_state == MSG_SENDER || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && *p == '@'){ continue; }
 
-      if(state->message_state == MSG_FROM || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT){
+      if(state->message_state == MSG_FROM || state->message_state == MSG_SENDER || state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT){
 
          /* To fix some unusual addresses, eg.
           *    "'user@domain'"    -> user@domain
@@ -957,7 +941,7 @@ char *determine_attachment_type(char *filename, char *type){
          if(strncasecmp(p, "rar", 3) == 0) return "compressed,";
 
          // tar.gz has the same type
-         if(strncasecmp(p, "x-gzip", 3) == 0) return "compressed,";
+         if(strncasecmp(p, "gz", 2) == 0) return "compressed,";
 
          if(strncasecmp(p, "rtf", 3) == 0) return "word,";
          if(strncasecmp(p, "doc", 3) == 0) return "word,";
@@ -1073,3 +1057,20 @@ void fill_attachment_name_buf(struct parser_state *state, char *buf){
       state->anamepos++;
    }
 }
+
+
+int get_first_email_address_from_string(char *str, char *buf, int buflen){
+   int result;
+
+   char *p = str;
+   do {
+      memset(buf, 0, buflen);
+      p = split(p, ' ', buf, buflen-1, &result);
+
+      if(*buf == '\0') continue;
+
+      if(does_it_seem_like_an_email_address(buf) == 1){ return 1; }
+   } while(p);
+
+   return 0;
+}

src/piler-smtp.c

@@ -38,7 +38,9 @@ char *configfile = CONFIG_FILE;
 struct config cfg;
 struct passwd *pwd;
 struct smtp_session *session, **sessions=NULL;
+struct smtp_acl *smtp_acl[MAXHASH];
 
+time_t prev_timeout_check = 0;
 
 void usage(){
    printf("\nusage: piler\n\n");
@@ -66,6 +68,8 @@ void p_clean_exit(int sig){
 
    if(events) free(events);
 
+   clear_smtp_acl(smtp_acl);
+
    syslog(LOG_PRIORITY, "%s has been terminated", PROGNAME);
 
    ERR_free_strings();
@@ -87,16 +91,18 @@ void check_for_client_timeout(){
 
    if(cfg.verbosity >= LOG_DEBUG) syslog(LOG_PRIORITY, "%s @%ld", __func__, now);
 
+   if(now - prev_timeout_check < cfg.smtp_timeout) return;
+
    if(num_connections > 0){
       for(int i=0; i<cfg.max_connections; i++){
          if(sessions[i] && now - sessions[i]->lasttime >= cfg.smtp_timeout){
             syslog(LOG_PRIORITY, "client %s timeout, lasttime: %ld", sessions[i]->remote_host, sessions[i]->lasttime);
-            tear_down_session(sessions, sessions[i]->slot, &num_connections);
+            tear_down_session(sessions, sessions[i]->slot, &num_connections, "timeout");
          }
       }
    }
 
-   alarm(cfg.check_for_client_timeout_interval);
+   time(&prev_timeout_check);
 }
 
 
@@ -120,6 +126,8 @@ void initialise_configuration(){
    setlocale(LC_MESSAGES, cfg.locale);
    setlocale(LC_CTYPE, cfg.locale);
 
+   load_smtp_acl(smtp_acl);
+
    syslog(LOG_PRIORITY, "reloaded config: %s", configfile);
 }
 
@@ -130,7 +138,6 @@ int main(int argc, char **argv){
    int client_len = sizeof(struct sockaddr_storage);
    ssize_t readlen;
    struct sockaddr_storage client_address;
-   char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
    char readbuf[BIGBUFSIZE];
    int efd;
 
@@ -192,8 +199,8 @@ int main(int argc, char **argv){
    set_signal_handler(SIGSEGV, p_clean_exit);
 
    set_signal_handler(SIGPIPE, SIG_IGN);
+   set_signal_handler(SIGALRM, SIG_IGN);
 
-   set_signal_handler(SIGALRM, check_for_client_timeout);
    set_signal_handler(SIGHUP, initialise_configuration);
 
    // calloc() initialitizes the allocated memory
@@ -214,10 +221,8 @@ int main(int argc, char **argv){
    if(daemonise == 1 && daemon(1, 0) == -1) fatal(ERR_DAEMON);
 #endif
 
-   alarm(cfg.check_for_client_timeout_interval);
-
    for(;;){
-      int n = epoll_wait(efd, events, cfg.max_connections, -1);
+      int n = epoll_wait(efd, events, cfg.max_connections, 1000);
       for(i=0; i<n; i++){
 
          // Office365 sometimes behaves oddly: when it receives the 250 OK
@@ -228,7 +233,7 @@ int main(int argc, char **argv){
             if(cfg.verbosity >= _LOG_EXTREME) syslog(LOG_PRIORITY, "ERROR: the remote end hung up without sending QUIT");
             session = get_session_by_socket(sessions, cfg.max_connections, events[i].data.fd);
             if(session)
-               tear_down_session(sessions, session->slot, &num_connections);
+               tear_down_session(sessions, session->slot, &num_connections, "hungup");
             else
                close(events[i].data.fd);
             continue;
@@ -252,6 +257,10 @@ int main(int argc, char **argv){
                   }
                }
 
+               char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
+               memset(hbuf, 0, sizeof(hbuf));
+               memset(sbuf, 0, sizeof(sbuf));
+
                if(getnameinfo((struct sockaddr *)&client_address, client_len, hbuf, sizeof(hbuf), sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV) == 0){
                   // Strictly speaking it's not correct to log num_connections+1 connections
                   // but it still gives a good clue how many connections we have at the moment
@@ -270,7 +279,7 @@ int main(int argc, char **argv){
                   break;
                }
 
-               start_new_session(sessions, client_sockfd, &num_connections, &cfg);
+               start_new_session(sessions, client_sockfd, &num_connections, smtp_acl, hbuf, &cfg);
             }
 
             continue;
@@ -292,8 +301,6 @@ int main(int argc, char **argv){
             time(&(session->lasttime));
 
             while(1){
-               memset(readbuf, 0, sizeof(readbuf));
-
                if(session->net.use_ssl == 1)
                   readlen = SSL_read(session->net.ssl, (char*)&readbuf[0], sizeof(readbuf)-1);
                else
@@ -325,12 +332,14 @@ int main(int argc, char **argv){
             /* Don't wait until the remote client closes the connection after he sent the QUIT command */
 
             if(done || session->protocol_state == SMTP_STATE_FINISHED){
-               tear_down_session(sessions, session->slot, &num_connections);
+               tear_down_session(sessions, session->slot, &num_connections, "done");
             }
          }
 
 
       }
+
+      check_for_client_timeout();
    }
 
    return 0;

src/piler.c

@@ -101,11 +101,47 @@ void child_sighup_handler(int sig){
 }
 
 
+int perform_checks(char *filename, struct session_data *sdata, struct data *data, struct parser_state *parser_state, struct config *cfg){
+
+   if(cfg->security_header[0] && parser_state->found_security_header == 0){
+      syslog(LOG_PRIORITY, "%s: discarding: missing security header", filename);
+      return ERR_DISCARDED;
+   }
+
+   char *arule = check_against_ruleset(data->archiving_rules, parser_state, sdata->tot_len, sdata->spam_message);
+
+   if(arule){
+      syslog(LOG_PRIORITY, "%s: discarding: archiving policy: *%s*", filename, arule);
+      return ERR_DISCARDED;
+   }
+
+
+   if(cfg->archive_only_mydomains == 1 && sdata->internal_sender == 0 && sdata->internal_recipient == 0){
+      syslog(LOG_PRIORITY, "%s: discarding: not on mydomains", filename);
+      return ERR_DISCARDED;
+   }
+
+   make_digests(sdata, cfg);
+
+   // A normal header is much bigger than 10 bytes. We get here for header-only
+   // messages without a Message-ID: line. I believe that no such message is valid, and
+   // it's a reasonable to discard it, and not allowing it to fill up the error directory.
+
+   if(sdata->hdr_len < 10){
+      syslog(LOG_PRIORITY, "%s: discarding: a header-only message without a Message-ID line", filename);
+      return ERR_DISCARDED;
+   }
+
+   int rc = process_message(sdata, parser_state, data, cfg);
+   unlink(parser_state->message_id_hash);
+
+   return rc;
+}
+
+
 int process_email(char *filename, struct session_data *sdata, struct data *data, int size, struct config *cfg){
-   int rc;
    char tmpbuf[SMALLBUFSIZE];
    char *status=S_STATUS_UNDEF;
-   char *arule;
    char *p;
    struct timezone tz;
    struct timeval tv1, tv2;
@@ -144,29 +180,7 @@ int process_email(char *filename, struct session_data *sdata, struct data *data,
       } while(rcpt);
    }
 
-   arule = check_against_ruleset(data->archiving_rules, &parser_state, sdata->tot_len, sdata->spam_message);
-
-   if(arule){
-      syslog(LOG_PRIORITY, "%s: discarding: archiving policy: *%s*", filename, arule);
-      rc = ERR_DISCARDED;
-   }
-   else if(cfg->archive_only_mydomains == 1 && sdata->internal_sender == 0 && sdata->internal_recipient == 0){
-      syslog(LOG_PRIORITY, "%s: discarding: not on mydomains", filename);
-      rc = ERR_DISCARDED;
-   }
-   else {
-      make_digests(sdata, cfg);
-
-      if(sdata->hdr_len < 10){
-         syslog(LOG_PRIORITY, "%s: invalid message, hdr_len: %d", filename, sdata->hdr_len);
-         rc = ERR;
-      }
-      else {
-         rc = process_message(sdata, &parser_state, data, cfg);
-         unlink(parser_state.message_id_hash);
-      }
-   }
-
+   int rc = perform_checks(filename, sdata, data, &parser_state, cfg);
    unlink(sdata->tmpframe);
 
    remove_stripped_attachments(&parser_state);

src/piler.h

@@ -17,6 +17,7 @@
 #include <sig.h>
 #include <av.h>
 #include <rules.h>
+#include <screen.h>
 #include <sql.h>
 #include <import.h>
 #include <smtp.h>
@@ -68,8 +69,8 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
 void load_mydomains(struct session_data *sdata, struct data *data, struct config *cfg);
 int is_email_address_on_my_domains(char *email, struct data *data);
 
-int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct config *cfg);
-void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections);
+int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg);
+void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections, char *reason);
 struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int max_connections, int socket);
 void write_envelope_addresses(struct smtp_session *session, struct config *cfg);
 void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg);

src/pilerexport.c

@@ -25,13 +25,17 @@ extern int optind;
 int dryrun = 0;
 int exportall = 0;
 int verification_status = 0;
-int rc = 0;
+int export_to_stdout = 0;
 char *query=NULL;
 int verbosity = 0;
 int max_matches = 1000;
 char *index_list = "main1,dailydelta1,delta1";
 regex_t regexp;
 char *zipfile = NULL;
+struct zip *zip = NULL;
+uint64 *zip_ids = NULL;
+int zip_counter = 0;
+int zip_batch = 2000;
 
 int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg);
 
@@ -51,8 +55,12 @@ void usage(){
    printf("    -w <where condition>              Where condition to pass to sphinx, eg. \"match('@subject: piler')\"\n");
    printf("    -m <max. matches>                 Max. matches to apply to sphinx query (default: %d)\n", max_matches);
    printf("    -i <index list>                   Sphinx indices to use  (default: %s)\n", index_list);
+#if LIBZIP_VERSION_MAJOR >= 1
    printf("    -z <zip file>                     Write exported EML files to a zip file\n");
+   printf("    -Z <batch size>                   Zip batch size. Valid range: 10-10000, default: 2000\n");
+#endif
    printf("    -A                                Export all emails from archive\n");
+   printf("    -o                                Export emails to stdout\n");
    printf("    -d                                Dry run\n");
 
    regfree(&regexp);
@@ -155,17 +163,24 @@ int append_string_to_buffer(char **buffer, char *str){
 uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *where_condition, uint64 last_id, int *num, struct config *cfg){
    MYSQL_ROW row;
    uint64 id=0;
-   char s[SMALLBUFSIZE];
+   char s[MAXBUFSIZE];
+   int rc=0;
 
    *num = 0;
 
    if(!where_condition) return id;
 
-   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE id IN (", SQL_METADATA_TABLE);
+   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE id IN (", SQL_METADATA_TABLE);
    rc += append_string_to_buffer(&query, s);
 
    snprintf(s, sizeof(s)-1, "SELECT id FROM %s WHERE %s AND id > %llu ORDER BY id ASC LIMIT 0,%d", index_list, where_condition, last_id, max_matches);
 
+   if(dryrun){
+      printf("sphinx query: %s\n", s);
+   }
+
+   syslog(LOG_PRIORITY, "sphinx query: %s", s);
+
    if(mysql_real_query(&(sdata2->mysql), s, strlen(s)) == 0){
       MYSQL_RES *res = mysql_store_result(&(sdata2->mysql));
       if(res != NULL){
@@ -183,6 +198,7 @@ uint64 run_query(struct session_data *sdata, struct session_data *sdata2, char *
    }
 
    if(!rc) export_emails_matching_to_query(sdata, query, cfg);
+   else printf("error: append_string_to_buffer() in run_query()\n");
 
    free(query);
    query = NULL;
@@ -229,15 +245,16 @@ void export_emails_matching_id_list(struct session_data *sdata, struct session_d
 
 int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain, int minsize, int maxsize, unsigned long startdate, unsigned long stopdate){
    char s[SMALLBUFSIZE];
+   int rc=0;
 
    if(exportall == 1){
-      rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest` FROM ");
+      rc = append_string_to_buffer(&query, "SELECT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM ");
       rc += append_string_to_buffer(&query, SQL_METADATA_TABLE);
       rc += append_string_to_buffer(&query, " WHERE deleted=0 ");
       return rc;
    }
 
-   snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest` FROM %s WHERE deleted=0 ", SQL_MESSAGES_VIEW);
+   snprintf(s, sizeof(s)-1, "SELECT DISTINCT `id`, `piler_id`, `digest`, `bodydigest`, `attachments` FROM %s WHERE deleted=0 ", SQL_MESSAGES_VIEW);
 
    rc = append_string_to_buffer(&query, s);
 
@@ -315,28 +332,27 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain
    return rc;
 }
 
+#if LIBZIP_VERSION_MAJOR >= 1
+void zip_flush(){
+   zip_close(zip);
 
-int write_to_zip_file(char *filename){
-   struct zip *z=NULL;
-   int errorp, ret=ERR;
+   zip = NULL;
+   zip_counter = 0;
 
-   z = zip_open(zipfile, ZIP_CREATE, &errorp);
-   if(!z){
-      printf("error: error creating zip file=%s, error code=%d\n", zipfile, errorp);
-      return ret;
+   if(!zip_ids) return;
+
+   for(int i=0; i<zip_batch; i++){
+      if(*(zip_ids+i)){
+         char filename[SMALLBUFSIZE];
+         snprintf(filename, sizeof(filename)-1, "%llu.eml", *(zip_ids+i));
+         unlink(filename);
+      }
    }
 
-   zip_source_t *zs = zip_source_file(z, filename, 0, 0);
-   if(zs && zip_file_add(z, filename, zs, ZIP_FL_ENC_UTF_8) >= 0){
-      ret = OK;
-   } else {
-      printf("error adding file %s: %s\n", filename, zip_strerror(z));
-   }
-
-   zip_close(z);
-
-   return ret;
+   free(zip_ids);
+   zip_ids = NULL;
 }
+#endif
 
 int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg){
    FILE *f;
@@ -344,6 +360,8 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct
    char digest[SMALLBUFSIZE], bodydigest[SMALLBUFSIZE];
    char filename[SMALLBUFSIZE];
    struct sql sql;
+   int errorp, rc=0, attachments;
+   unsigned long total_attachments=0;
 
    if(prepare_sql_statement(sdata, &sql, s) == ERR) return ERR;
 
@@ -360,6 +378,7 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct
    sql.sql[sql.pos] = sdata->ttmpfile; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = RND_STR_LEN; sql.pos++;
    sql.sql[sql.pos] = &digest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(digest)-2; sql.pos++;
    sql.sql[sql.pos] = &bodydigest[0]; sql.type[sql.pos] = TYPE_STRING; sql.len[sql.pos] = sizeof(bodydigest)-2; sql.pos++;
+   sql.sql[sql.pos] = (char *)&attachments; sql.type[sql.pos] = TYPE_LONG; sql.len[sql.pos] = sizeof(int); sql.pos++;
 
    p_store_results(&sql);
 
@@ -369,6 +388,12 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct
 
          if(dryrun == 0){
 
+            if(export_to_stdout){
+               printf("%s", PILEREXPORT_BEGIN_MARK);
+               rc = retrieve_email_from_archive(sdata, stdout, cfg);
+               continue;
+            }
+
             snprintf(filename, sizeof(filename)-1, "%llu.eml", id);
 
             f = fopen(filename, "w");
@@ -390,14 +415,43 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct
                   verification_status = 1;
                }
 
-               if(zipfile && write_to_zip_file(filename) == OK){
-                  unlink(filename);
-               }
+               if(zipfile){
+               #if LIBZIP_VERSION_MAJOR >= 1
+                  // Open zip file if handler is NULL
+                  if(!zip){
+                     zip = zip_open(zipfile, ZIP_CREATE, &errorp);
+                     if(!zip){
+                        printf("error: error creating zip file=%s, error code=%d\n", zipfile, errorp);
+                        return ERR;
+                     }
+                  }
 
+                  if(!zip_ids) zip_ids = (uint64*) calloc(sizeof(uint64), zip_batch);
+
+                  if(!zip_ids){
+                     printf("calloc error for zip_ids\n");
+                     return ERR;
+                  }
+
+                  zip_source_t *zs = zip_source_file(zip, filename, 0, 0);
+                  if(zs && zip_file_add(zip, filename, zs, ZIP_FL_ENC_UTF_8) >= 0){
+                     *(zip_ids+zip_counter) = id;
+                     zip_counter++;
+                  } else {
+                     printf("error adding file %s: %s\n", filename, zip_strerror(zip));
+                     return ERR;
+                  }
+
+                  if(zip_counter == zip_batch){
+                     zip_flush();
+                  }
+               #endif
+               }
             }
             else printf("cannot open: %s\n", filename);
          }
          else {
+            total_attachments += attachments;
             printf("id:%llu\n", id);
          }
 
@@ -410,6 +464,10 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct
 ENDE:
    close_prepared_statement(&sql);
 
+   if(dryrun){
+      printf("attachments: %lu\n", total_attachments);
+   }
+
    printf("\n");
 
    return rc;
@@ -442,6 +500,7 @@ int main(int argc, char **argv){
             {"all",          no_argument,        0,  'A' },
             {"dry-run",      no_argument,        0,  'd' },
             {"dryrun",       no_argument,        0,  'd' },
+            {"stdout",       no_argument,        0,  'o' },
             {"help",         no_argument,        0,  'h' },
             {"version",      no_argument,        0,  'v' },
             {"from",         required_argument,  0,  'f' },
@@ -451,6 +510,7 @@ int main(int argc, char **argv){
             {"start-date",   required_argument,  0,  'a' },
             {"stop-date",    required_argument,  0,  'b' },
             {"zip",          required_argument,  0,  'z' },
+            {"zip-batch",    required_argument,  0,  'Z' },
             {"where-condition", required_argument,  0,  'w' },
             {"max-matches",  required_argument,  0,  'm' },
             {"index-list",   required_argument,  0,  'i' },
@@ -459,9 +519,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Adhv?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:oAdhv?", long_options, &option_index);
 #else
-      int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Adhv?");
+      int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:oAdhv?");
 #endif
 
       if(c == -1) break;
@@ -492,7 +552,10 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&from, optarg);
+                    if(append_email_to_buffer(&from, optarg)){
+                       printf("error: append_email_to_buffer() for from\n");
+                       return 1;
+                    }
 
                     break;
 
@@ -503,7 +566,10 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&to, optarg);
+                    if(append_email_to_buffer(&to, optarg)){
+                       printf("error: append_email_to_buffer() for to\n");
+                       return 1;
+                    }
 
                     break;
 
@@ -514,7 +580,10 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&fromdomain, optarg);
+                    if(append_email_to_buffer(&fromdomain, optarg)){
+                       printf("error: append_email_to_buffer() for fromdomain\n");
+                       return 1;
+                    }
 
                     break;
 
@@ -525,7 +594,10 @@ int main(int argc, char **argv){
                        break;
                     }
 
-                    rc = append_email_to_buffer(&todomain, optarg);
+                    if(append_email_to_buffer(&todomain, optarg)){
+                       printf("error: append_email_to_buffer() for todomain\n");
+                       return 1;
+                    }
 
                     break;
 
@@ -553,6 +625,14 @@ int main(int argc, char **argv){
          case 'z':  zipfile = optarg;
                     break;
 
+         case 'Z':  zip_batch = atoi(optarg);
+                    if(zip_batch < 10 || zip_batch > 10000)
+                       zip_batch = 2000;
+                    break;
+
+         case 'o':
+                    export_to_stdout = 1;
+                    break;
 
          case 'd' :
                     dryrun = 1;
@@ -614,5 +694,11 @@ int main(int argc, char **argv){
 
    close_database(&sdata);
 
+   if(zipfile){
+   #if LIBZIP_VERSION_MAJOR >= 1
+      zip_flush();
+   #endif
+   }
+
    return verification_status;
 }

src/pilerimport.c

@@ -54,6 +54,7 @@ void usage(){
    printf("    -a <recipient>                    Add recipient to the To:/Cc: list\n");
    printf("    -T <id>                           Update import table at id=<id>\n");
    printf("    -D                                Dry-run, do not import anything\n");
+   printf("    -y                                Read pilerexport data from stdin\n");
    printf("    -o                                Only download emails for POP3/IMAP import\n");
    printf("    -r                                Remove imported emails\n");
    printf("    -q                                Quiet mode\n");
@@ -63,7 +64,7 @@ void usage(){
 
 
 int main(int argc, char **argv){
-   int i, n_mbox=0;
+   int i, n_mbox=0, read_from_pilerexport=0;
    char *configfile=CONFIG_FILE, *mbox[MBOX_ARGS], *directory=NULL;
    char puf[SMALLBUFSIZE], *imapserver=NULL, *pop3server=NULL;
    struct session_data sdata;
@@ -141,6 +142,7 @@ int main(int argc, char **argv){
             {"failed-folder",  required_argument,  0,  'j' },
             {"move-folder",  required_argument,  0,  'g' },
             {"only-download",no_argument,        0,  'o' },
+            {"read-from-export",no_argument,     0,  'y' },
             {"dry-run",      no_argument,        0,  'D' },
             {"help",         no_argument,        0,  'h' },
             {0,0,0,0}
@@ -148,9 +150,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:DRroqh?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:yDRroqh?", long_options, &option_index);
 #else
-      int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:DRroqh?");
+      int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:yDRroqh?");
 #endif
 
       if(c == -1) break;
@@ -269,6 +271,10 @@ int main(int argc, char **argv){
                     data.import->table_id = atoi(optarg);
                     break;
 
+         case 'y' :
+                    read_from_pilerexport = 1;
+                    break;
+
          case 'D' :
                     data.import->dryrun = 1;
                     break;
@@ -289,7 +295,7 @@ int main(int argc, char **argv){
    }
 
 
-   if(!mbox[0] && !data.import->mboxdir && !data.import->filename && !directory && !imapserver && !pop3server) usage();
+   if(!mbox[0] && !data.import->mboxdir && !data.import->filename[0] && !directory && !imapserver && !pop3server) usage();
 
    if(data.import->failed_folder && !can_i_write_directory(data.import->failed_folder)){
       printf("cannot write failed directory '%s'\n", data.import->failed_folder);
@@ -300,6 +306,8 @@ int main(int argc, char **argv){
 
    cfg = read_config(configfile);
 
+   memset(cfg.security_header, 0, MAXVAL);
+
    if((data.recursive_folder_names == 1 || data.import->folder) && cfg.enable_folders == 0){
       printf("please set enable_folders=1 in piler.conf to use the folder options\n");
       return ERR;
@@ -359,6 +367,7 @@ int main(int argc, char **argv){
    if(directory) import_from_maildir(&sdata, &data, directory, &cfg);
    if(imapserver) import_from_imap_server(&sdata, &data, &cfg);
    if(pop3server) import_from_pop3_server(&sdata, &data, &cfg);
+   if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &cfg);
 
    clearrules(data.archiving_rules);
    clearrules(data.retention_rules);

src/reindex.c

@@ -123,11 +123,14 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da
 
             snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
 
-            state = parse_message(sdata, 0, data, cfg);
+            state = parse_message(sdata, 1, data, cfg);
             post_parse(sdata, &state, cfg);
 
             rc = store_index_data(sdata, &state, data, stored_id, cfg);
 
+            unlink(sdata->tmpframe);
+            remove_stripped_attachments(&state);
+
             if(rc == OK) reindexed++;
             else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename);
 

src/rules.c

@@ -118,7 +118,6 @@ int append_rule(struct node *xhash[], struct rule_cond *rule_cond){
 
 struct rule *create_rule_item(struct rule_cond *rule_cond){
    struct rule *h=NULL;
-   char empty = '\0';
    int len;
 
    if(rule_cond == NULL) return NULL;
@@ -144,16 +143,16 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){
    h->emptyfrom = h->emptyto = h->emptysubject = h->emptyaname = h->emptyatype = 0;
 
 
-   if(rule_cond->from == NULL || strlen(rule_cond->from) < 1){ rule_cond->from[0] = empty; h->emptyfrom = 1; }
+   if(rule_cond->from[0] == 0){ h->emptyfrom = 1; }
    if(regcomp(&(h->from), rule_cond->from, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->to == NULL || strlen(rule_cond->to) < 1){ rule_cond->to[0] = empty; h->emptyto = 1; }
+   if(rule_cond->to[0] == 0){ h->emptyto = 1; }
    if(regcomp(&(h->to), rule_cond->to, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->subject == NULL || strlen(rule_cond->subject) < 1){ rule_cond->subject[0] = empty; h->emptysubject = 1; }
+   if(rule_cond->subject[0] == 0){ h->emptysubject = 1; }
    if(regcomp(&(h->subject), rule_cond->subject, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->body == NULL || strlen(rule_cond->body) < 1){ rule_cond->body[0] = empty; h->emptybody = 1; }
+   if(rule_cond->body[0] == 0){ h->emptybody = 1; }
    if(regcomp(&(h->body), rule_cond->body, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
    h->spam = rule_cond->spam;
@@ -161,20 +160,16 @@ struct rule *create_rule_item(struct rule_cond *rule_cond){
    h->folder_id = rule_cond->folder_id;
 
    h->size = rule_cond->size;
-
-   if(rule_cond->_size == NULL) rule_cond->_size[0] = empty;
    snprintf(h->_size, 3, "%s", rule_cond->_size);
 
-   if(rule_cond->attachment_name == NULL || strlen(rule_cond->attachment_name) < 1){ rule_cond->attachment_name[0] = empty; h->emptyaname = 1; }
+   if(rule_cond->attachment_name[0] == 0){ h->emptyaname = 1; }
    if(regcomp(&(h->attachment_name), rule_cond->attachment_name, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
-   if(rule_cond->attachment_type == NULL || strlen(rule_cond->attachment_type) < 1){ rule_cond->attachment_type[0] = empty; h->emptyatype = 1; }
+   if(rule_cond->attachment_type[0] == 0){ h->emptyatype = 1; }
    if(regcomp(&(h->attachment_type), rule_cond->attachment_type, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
 
 
    h->attachment_size = rule_cond->attachment_size;
-
-   if(rule_cond->_attachment_size == NULL) rule_cond->_attachment_size[0] = empty;
    snprintf(h->_attachment_size, 3, "%s", rule_cond->_attachment_size);
 
    len = strlen(rule_cond->domain)+8 + strlen(rule_cond->from)+6 + strlen(rule_cond->to)+4 + strlen(rule_cond->subject)+9 + strlen(rule_cond->body)+6 + strlen(rule_cond->_size)+6 + strlen(rule_cond->attachment_name)+10 + strlen(rule_cond->attachment_type)+10 + strlen(rule_cond->_attachment_size)+10 + 9 + 15 + 15;

src/screen.c

@@ -0,0 +1,254 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <sys/types.h>
+#include <netdb.h>
+#include <piler.h>
+
+
+void init_smtp_acl(struct smtp_acl *smtp_acl[]){
+   smtp_acl[0] = NULL;
+}
+
+
+void clear_smtp_acl(struct smtp_acl *smtp_acl[]){
+   struct smtp_acl *q;
+
+   q = smtp_acl[0];
+
+   while(q){
+      struct smtp_acl *p = q;
+      q = q->r;
+
+      free(p);
+   }
+
+   smtp_acl[0] = NULL;
+}
+
+
+int add_smtp_acl(struct smtp_acl *smtp_acl[], char *network_str, struct smtp_acl *acl){
+   struct smtp_acl *q, *p=NULL, *node;
+
+   if((node = malloc(sizeof(struct smtp_acl))) == NULL) return 0;
+
+   memset(node, 0, sizeof(struct smtp_acl));
+
+   node->low = acl->low;
+   node->high = acl->high;
+   node->prefix = acl->prefix;
+   node->rejected = acl->rejected;
+   snprintf(node->network_str, sizeof(node->network_str)-1, "%s", network_str);
+
+   node->r = NULL;
+
+   q = smtp_acl[0];
+
+   while(q){
+      p = q;
+      q = q->r;
+   }
+
+   if(!p){
+      smtp_acl[0] = node;
+   } else {
+      p->r = node;
+   }
+
+   return 1;
+}
+
+
+int is_valid_line(char *line){
+   // Currently we support ipv4 stuff only, ie. valid characters are: 0-9./
+   // and a line should look like "1.2.3.4/24 permit" or similar (without quotes)
+
+   if(!strchr(line, '.') || !strchr(line, '/') || (!strchr(line, ' ') && !strchr(line, '\t')) ){
+      return 0;
+   }
+
+   if(!strstr(line, "permit") && !strstr(line, "reject")){
+      return 0;
+   }
+
+   // ascii values:
+   //   46: .
+   //   47: /
+   //   48-57: 0-9
+   //   97-122: a-z
+   //
+   for(; *line; line++){
+      if(isalnum(*line) == 0 && isblank(*line) == 0 && *line != 46 && *line != 47) return 0;
+   }
+
+   return 1;
+}
+
+
+int a_to_hl(char *ipstr, in_addr_t *addr){
+  struct in_addr in;
+
+  if(inet_aton(ipstr, &in) == 1){
+     *addr = ntohl(in.s_addr);
+     return 1;
+  }
+
+  syslog(LOG_PRIORITY, "invalid ipv4 address string: *%s*", ipstr);
+
+  return 0;
+}
+
+
+in_addr_t netmask(int prefix){
+   if(prefix == 0)
+      return( ~((in_addr_t) -1) );
+   else
+      return ~((1 << (32 - prefix)) - 1);
+}
+
+
+in_addr_t network(in_addr_t addr, int prefix){
+   return addr & netmask(prefix);
+}
+
+
+in_addr_t broadcast(in_addr_t addr, int prefix){
+   return addr | ~netmask(prefix);
+}
+
+
+int str_to_net_range(char *network_addr_prefix, struct smtp_acl *smtp_acl){
+   in_addr_t net = 0;
+   int prefix = 0;
+
+   smtp_acl->low = 0;
+   smtp_acl->high = 0;
+
+   // By default we permit unless you specify "reject" (without quotes)
+   // To be on the safer side we permit even if you misspell the word "reject"
+
+   smtp_acl->rejected = 0;
+
+   if(strcasestr(network_addr_prefix, "reject")){
+      smtp_acl->rejected = 1;
+   }
+
+   char *p = strchr(network_addr_prefix, '/');
+   if(!p) return 0;
+
+   if(strlen(network_addr_prefix) > sizeof(smtp_acl->network_str)){
+      syslog(LOG_PRIORITY, "line *%s* is longer than %ld bytes, discarded", network_addr_prefix, sizeof(smtp_acl->network_str));
+      return 0;
+   }
+
+   char buf[SMALLBUFSIZE];
+   snprintf(buf, sizeof(buf)-1, "%s", network_addr_prefix);
+
+   *p = '\0';
+   p++;
+
+   // Even though the remaining part of the acl line continues with some number
+   // then whitespace characters and the permit/reject action atoi() can still
+   // figure out the numeric part properly, that's why I'm lazy here.
+   prefix = atoi(p);
+
+   // The prefix string (p) must start with a digit and the prefix integer must be in 0..32 range
+   if(*p < 48 || *p > 57 || prefix < 0 || prefix > 32){
+      syslog(LOG_PRIORITY, "error: invalid prefix: %s", p);
+      return 0;
+   }
+
+   if(a_to_hl(network_addr_prefix, &net)){
+      smtp_acl->low = network(net, prefix);
+      smtp_acl->high = broadcast(net, prefix);
+      smtp_acl->prefix = prefix;
+
+      syslog(LOG_PRIORITY, "info: parsed acl *%s*", buf);
+
+      return 1;
+   }
+
+   return 0;
+}
+
+
+void load_smtp_acl(struct smtp_acl *smtp_acl[]){
+   int count=0;
+
+   clear_smtp_acl(smtp_acl);
+   init_smtp_acl(smtp_acl);
+
+   FILE *f = fopen(SMTP_ACL_FILE, "r");
+   if(!f){
+      syslog(LOG_PRIORITY, "info: cannot open %s, piler-smtp accepts smtp connections from everywhere", SMTP_ACL_FILE);
+      return;
+   }
+
+   char line[SMALLBUFSIZE];
+   struct smtp_acl acl;
+
+   while(fgets(line, sizeof(line)-1, f)){
+      // Skip comments
+      if(line[0] == ';' || line[0] == '#') continue;
+
+      trimBuffer(line);
+
+      // Skip empty line
+      if(line[0] == 0) continue;
+
+      char line2[SMALLBUFSIZE];
+      int rc = 0;
+      snprintf(line2, sizeof(line2)-1, "%s", line);
+
+      if(is_valid_line(line) == 1 && str_to_net_range(line, &acl) == 1){
+         add_smtp_acl(smtp_acl, line, &acl);
+         count++;
+         rc = 1;
+      }
+
+      if(!rc) syslog(LOG_PRIORITY, "error: failed to parse line: *%s*", line2);
+   }
+
+   fclose(f);
+
+   // If we have entries on the smtp acl list, then add 127.0.0.1/8
+   // to let the GUI health page connect to the piler-smtp daemon
+   if(count){
+      snprintf(line, sizeof(line)-1, "127.0.0.1/8 permit");
+
+      if(str_to_net_range(line, &acl) == 1){
+         add_smtp_acl(smtp_acl, line, &acl);
+      }
+   }
+}
+
+
+int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr){
+   struct smtp_acl *q=smtp_acl[0];
+   in_addr_t addr = 0;
+
+   // Empty acl, let it pass
+   if(!q) return 0;
+
+   if(a_to_hl(ipaddr, &addr) == 0){
+      syslog(LOG_PRIORITY, "error: invalid smtp client address: *%s*", ipaddr);
+      return 1;
+   }
+
+   while(q){
+      if(addr >= q->low && addr <= q->high){
+         if(q->rejected) syslog(LOG_PRIORITY, "denied connection from %s, acl: %s/%d reject", ipaddr, q->network_str, q->prefix);
+         return q->rejected;
+      }
+
+      q = q->r;
+   }
+
+   syslog(LOG_PRIORITY, "denied connection from %s by implicit default deny", ipaddr);
+
+   return 1;
+}

src/screen.h

@@ -0,0 +1,16 @@
+/*
+ * screen.h, SJ
+ */
+
+#ifndef _SCREEN_H
+ #define _SCREEN_H
+
+#include "defs.h"
+
+void init_smtp_acl(struct smtp_acl *smtp_acl[]);
+void clear_smtp_acl(struct smtp_acl *smtp_acl[]);
+int add_smtp_acl(struct smtp_acl *smtp_acl[], char *network_str, struct smtp_acl *acl);
+void load_smtp_acl(struct smtp_acl *smtp_acl[]);
+int is_blocked_by_pilerscreen(struct smtp_acl *smtp_acl[], char *ipaddr);
+
+#endif /* _SCREEN_H */

src/session.c

@@ -7,29 +7,10 @@
 
 
 int get_session_slot(struct smtp_session **sessions, int max_connections);
-void init_smtp_session(struct smtp_session *session, int slot, int sd, struct config *cfg);
+void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg);
 
 
-#ifdef HAVE_LIBWRAP
-int is_blocked_by_tcp_wrappers(int sd){
-   struct request_info req;
-
-   request_init(&req, RQ_DAEMON, "piler", RQ_FILE, sd, 0);
-
-   fromhost(&req);
-
-   if(!hosts_access(&req)){
-      send(sd, SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY, strlen(SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY), 0);
-      syslog(LOG_PRIORITY, "denied connection from %s by tcp_wrappers", eval_client(&req));
-      return 1;
-   }
-
-   return 0;
-}
-#endif
-
-
-int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct config *cfg){
+int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg){
    int slot;
 
    /*
@@ -43,19 +24,19 @@ int start_new_session(struct smtp_session **sessions, int socket, int *num_conne
       return -1;
    }
 
-#ifdef HAVE_LIBWRAP
-   if(is_blocked_by_tcp_wrappers(socket) == 1){
+   // Check remote client against the allowed network ranges
+   if(cfg->smtp_access_list && is_blocked_by_pilerscreen(smtp_acl, client_addr)){
+      send(socket, SMTP_RESP_550_ERR, strlen(SMTP_RESP_550_ERR), 0);
       close(socket);
       return -1;
    }
-#endif
 
    slot = get_session_slot(sessions, cfg->max_connections);
 
    if(slot >= 0 && sessions[slot] == NULL){
       sessions[slot] = malloc(sizeof(struct smtp_session));
       if(sessions[slot]){
-         init_smtp_session(sessions[slot], slot, socket, cfg);
+         init_smtp_session(sessions[slot], slot, socket, client_addr, cfg);
 
          char smtp_banner[SMALLBUFSIZE];
          snprintf(smtp_banner, sizeof(smtp_banner)-1, SMTP_RESP_220_BANNER, cfg->hostid);
@@ -102,10 +83,7 @@ struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int m
 }
 
 
-void init_smtp_session(struct smtp_session *session, int slot, int sd, struct config *cfg){
-   struct sockaddr_in addr;
-   socklen_t addr_size = sizeof(struct sockaddr_in);
-   char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
+void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg){
    int i;
 
    session->slot = slot;
@@ -131,21 +109,15 @@ void init_smtp_session(struct smtp_session *session, int slot, int sd, struct co
    for(i=0; i<MAX_RCPT_TO; i++) memset(session->rcptto[i], 0, SMALLBUFSIZE);
 
    memset(session->buf, 0, MAXBUFSIZE);
-   memset(session->remote_host, 0, INET6_ADDRSTRLEN);
+   snprintf(session->remote_host, sizeof(session->remote_host)-1, "%s", client_addr);
 
    reset_bdat_counters(session);
 
    time(&(session->lasttime));
-
-   if(getpeername(session->net.socket, (struct sockaddr *)&addr, &addr_size) == 0 &&
-      getnameinfo((struct sockaddr *)&addr, addr_size, hbuf, sizeof(hbuf), sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV) == 0){
-         snprintf(session->remote_host, INET6_ADDRSTRLEN-1, "%s", hbuf);
-   }
 }
 
 
 void free_smtp_session(struct smtp_session *session){
-
    if(session){
 
       if(session->net.use_ssl == 1){
@@ -153,15 +125,25 @@ void free_smtp_session(struct smtp_session *session){
          SSL_free(session->net.ssl);
       }
 
-      if(session->net.ctx) SSL_CTX_free(session->net.ctx);
+      if(session->net.ctx){
+         SSL_CTX_free(session->net.ctx);
+      }
 
       free(session);
    }
 }
 
 
-void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections){
-   syslog(LOG_PRIORITY, "disconnected from %s on fd=%d (%d active connections)", sessions[slot]->remote_host, sessions[slot]->net.socket, (*num_connections)-1);
+void tear_down_session(struct smtp_session **sessions, int slot, int *num_connections, char *reason){
+   if(sessions[slot] == NULL){
+      syslog(LOG_PRIORITY, "session already torn down, slot=%d, reason=%s (%d active connections)", slot, reason, *num_connections);
+      return;
+   }
+
+   if(*num_connections > 0) (*num_connections)--;
+
+   syslog(LOG_PRIORITY, "disconnected from %s on fd=%d, slot=%d, reason=%s (%d active connections)",
+          sessions[slot]->remote_host, sessions[slot]->net.socket, slot, reason, *num_connections);
 
    close(sessions[slot]->net.socket);
 
@@ -174,8 +156,6 @@ void tear_down_session(struct smtp_session **sessions, int slot, int *num_connec
 
    free_smtp_session(sessions[slot]);
    sessions[slot] = NULL;
-
-   (*num_connections)--;
 }
 
 
@@ -197,6 +177,7 @@ void handle_data(struct smtp_session *session, char *readbuf, int readlen, struc
       p = &copybuf[0];
    }
    else {
+      readbuf[readlen] = 0;
       p = readbuf;
    }
 

src/smtp.c

@@ -31,13 +31,18 @@ void process_smtp_command(struct smtp_session *session, char *buf, struct config
       return;
    }
 
+   if(strncasecmp(buf, SMTP_CMD_HELP, strlen(SMTP_CMD_HELP)) == 0){
+      send_smtp_response(session, SMTP_RESP_221_PILER_SMTP_OK);
+      return;
+   }
+
    if(strncasecmp(buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){
       process_command_mail_from(session, buf);
       return;
    }
 
    if(strncasecmp(buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){
-      process_command_rcpt_to(session, buf);
+      process_command_rcpt_to(session, buf, cfg);
       return;
    }
 
@@ -166,6 +171,11 @@ int init_ssl(struct smtp_session *session){
       return 0;
    }
 
+   if(SSL_CTX_set_min_proto_version(session->net.ctx, session->cfg->tls_min_version_number) == 0){
+      syslog(LOG_PRIORITY, "failed SSL_CTX_set_min_proto_version() to %s/%d", session->cfg->tls_min_version, session->cfg->tls_min_version_number);
+      return 0;
+   }
+
    if(SSL_CTX_set_cipher_list(session->net.ctx, session->cfg->cipher_list) == 0){
       syslog(LOG_PRIORITY, "failed to set cipher list: '%s'", session->cfg->cipher_list);
       return 0;
@@ -193,8 +203,6 @@ void process_command_starttls(struct smtp_session *session){
       session->net.ssl = SSL_new(session->net.ctx);
       if(session->net.ssl){
 
-         SSL_set_options(session->net.ssl, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3);
-
          if(SSL_set_fd(session->net.ssl, session->net.socket) == 1){
             session->net.starttls = 1;
             send_smtp_response(session, SMTP_RESP_220_READY_TO_START_TLS);
@@ -204,9 +212,9 @@ void process_command_starttls(struct smtp_session *session){
                wait_for_ssl_accept(session);
 
             return;
-         } syslog(LOG_PRIORITY, "%s: SSL_set_fd() failed", session->ttmpfile);
-      } syslog(LOG_PRIORITY, "%s: SSL_new() failed", session->ttmpfile);
-   } syslog(LOG_PRIORITY, "SSL ctx is null!");
+         } syslog(LOG_PRIORITY, "ERROR: %s: SSL_set_fd() failed", session->ttmpfile);
+      } syslog(LOG_PRIORITY, "ERROR: %s: SSL_new() failed", session->ttmpfile);
+   } syslog(LOG_PRIORITY, "ERROR: init_ssl()");
 
    send_smtp_response(session, SMTP_RESP_454_ERR_TLS_TEMP_ERROR);
 }
@@ -231,7 +239,7 @@ void process_command_mail_from(struct smtp_session *session, char *buf){
 }
 
 
-void process_command_rcpt_to(struct smtp_session *session, char *buf){
+void process_command_rcpt_to(struct smtp_session *session, char *buf, struct config *cfg){
 
    if(session->protocol_state == SMTP_STATE_MAIL_FROM || session->protocol_state == SMTP_STATE_RCPT_TO){
 
@@ -241,6 +249,14 @@ void process_command_rcpt_to(struct smtp_session *session, char *buf){
 
       if(session->num_of_rcpt_to < MAX_RCPT_TO){
          extractEmail(buf, session->rcptto[session->num_of_rcpt_to]);
+
+         // Check if we should accept archive_address only
+         if(cfg->archive_address[0] && !strstr(cfg->archive_address, session->rcptto[session->num_of_rcpt_to])){
+            syslog(LOG_PRIORITY, "ERROR: Invalid recipient: *%s*", session->rcptto[session->num_of_rcpt_to]);
+            send_smtp_response(session, SMTP_RESP_550_ERR_INVALID_RECIPIENT);
+            return;
+         }
+
          session->num_of_rcpt_to++;
       }
 
@@ -298,7 +314,7 @@ void process_command_period(struct smtp_session *session){
 
    session->buflen = 0;
    session->last_data_char = 0;
-   memset(session->buf, 0, SMALLBUFSIZE);
+   memset(session->buf, 0, sizeof(session->buf));
 
    send_smtp_response(session, buf);
 }

src/smtp.h

@@ -16,7 +16,7 @@ void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int bufl
 void process_command_quit(struct smtp_session *session, char *buf, int buflen);
 void process_command_reset(struct smtp_session *session);
 void process_command_mail_from(struct smtp_session *session, char *buf);
-void process_command_rcpt_to(struct smtp_session *session, char *buf);
+void process_command_rcpt_to(struct smtp_session *session, char *buf, struct config *cfg);
 void process_command_data(struct smtp_session *session, struct config *cfg);
 void process_command_period(struct smtp_session *session);
 void process_command_starttls(struct smtp_session *session);

src/smtpcodes.h

@@ -18,6 +18,7 @@
 
 #define SMTP_CMD_HELO "HELO"
 #define SMTP_CMD_EHLO "EHLO"
+#define SMTP_CMD_HELP "HELP"
 #define SMTP_CMD_MAIL_FROM "MAIL FROM:"
 #define SMTP_CMD_RCPT_TO "RCPT TO:"
 #define SMTP_CMD_DATA "DATA"
@@ -32,6 +33,7 @@
 
 // SMTP responses
 
+#define SMTP_RESP_221_PILER_SMTP_OK "221 piler-smtp is OK\r\n"
 #define SMTP_RESP_220_BANNER "220 %s ESMTP\r\n"
 #define SMTP_RESP_220_READY_TO_START_TLS "220 Ready to start TLS\r\n"
 #define SMTP_RESP_221_GOODBYE "221 %s Goodbye\r\n"
@@ -54,7 +56,9 @@
 
 #define SMTP_RESP_502_ERR "502 Command not implemented\r\n"
 #define SMTP_RESP_503_ERR "503 Bad command sequence\r\n"
+#define SMTP_RESP_550_ERR_INVALID_RECIPIENT "550 Invalid recipient\r\n"
 #define SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY "550 You are banned by local policy\r\n"
+#define SMTP_RESP_550_ERR "550 Service currently unavailable\r\n"
 
 
 // LMTP commands

src/store.c

@@ -51,6 +51,8 @@ int store_file(struct session_data *sdata, char *filename, int len, struct confi
 #else
    EVP_CIPHER_CTX *ctx;
 #endif
+   int blocklen;
+   unsigned char rnd[EVP_MAX_BLOCK_LENGTH];
    unsigned char *outbuf=NULL;
    int outlen=0, writelen, tmplen;
 
@@ -107,26 +109,46 @@ int store_file(struct session_data *sdata, char *filename, int len, struct confi
 
    #if OPENSSL_VERSION_NUMBER < 0x10100000L
       EVP_CIPHER_CTX_init(&ctx);
-      EVP_EncryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      EVP_EncryptInit_ex(&ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+      blocklen = EVP_CIPHER_CTX_block_size(&ctx);
    #else
       ctx = EVP_CIPHER_CTX_new();
       if(!ctx) goto ENDE;
 
       EVP_CIPHER_CTX_init(ctx);
-      EVP_EncryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
+      EVP_EncryptInit_ex(ctx, EVP_aes_256_cbc(), NULL, cfg->key, cfg->iv);
+      blocklen = EVP_CIPHER_CTX_block_size(ctx);
    #endif
 
-      outbuf = malloc(dstlen + EVP_MAX_BLOCK_LENGTH);
+      // prepend a block with random data as replacement for dynamic iv
+      // see e.g. https://crypto.stackexchange.com/questions/5421/using-cbc-with-a-fixed-iv-and-a-random-first-plaintext-block
+      fd = open(RANDOM_POOL, O_RDONLY);
+      if(fd == -1) goto ENDE;
+      tmplen = readFromEntropyPool(fd, rnd, blocklen);
+      close(fd);
+      if(tmplen != blocklen) goto ENDE;
+      // make sure, random data does not start with zlib magic 0x78
+      if(rnd[0] == 0x78) rnd[0] =~ rnd[0];
+
+      outbuf = malloc(dstlen + blocklen * 2);
       if(outbuf == NULL) goto ENDE;
 
    #if OPENSSL_VERSION_NUMBER < 0x10100000L
-      if(!EVP_EncryptUpdate(&ctx, outbuf, &outlen, z, dstlen)) goto ENDE;
+      if(!EVP_EncryptUpdate(&ctx, outbuf, &outlen, rnd, blocklen)) goto ENDE;
+      if(!EVP_EncryptUpdate(&ctx, outbuf + outlen, &tmplen, z, dstlen)) goto ENDE;
+   #else
+      if(!EVP_EncryptUpdate(ctx, outbuf, &outlen, rnd, blocklen)) goto ENDE;
+      if(!EVP_EncryptUpdate(ctx, outbuf + outlen, &tmplen, z, dstlen)) goto ENDE;
+   #endif
+      outlen += tmplen;
+
+   #if OPENSSL_VERSION_NUMBER < 0x10100000L
       if(!EVP_EncryptFinal_ex(&ctx, outbuf + outlen, &tmplen)) goto ENDE;
    #else
-      if(!EVP_EncryptUpdate(ctx, outbuf, &outlen, z, dstlen)) goto ENDE;
       if(!EVP_EncryptFinal_ex(ctx, outbuf + outlen, &tmplen)) goto ENDE;
    #endif
       outlen += tmplen;
+
    #if OPENSSL_VERSION_NUMBER < 0x10100000L
       EVP_CIPHER_CTX_cleanup(&ctx);
    #else
@@ -206,7 +228,7 @@ int store_file(struct session_data *sdata, char *filename, int len, struct confi
 
 ENDE:
    if(outbuf) free(outbuf);
-   if(z) free(z);
+   if(z) free(z); //-V547
 
    return ret;
 }

src/test.c

@@ -151,6 +151,7 @@ int main(int argc, char **argv){
 
    printf("message-id: %s / %s\n", state.message_id, state.message_id_hash);
    printf("from: *%s (%s)*\n", state.b_from, state.b_from_domain);
+   printf("sender: *%s (%s)*\n", state.b_sender, state.b_sender_domain);
    printf("to: *%s (%s)*\n", state.b_to, state.b_to_domain);
    printf("reference: *%s*\n", state.reference);
    printf("subject: *%s*\n", state.b_subject);
@@ -169,7 +170,7 @@ int main(int argc, char **argv){
    printf("rules check: %s\n", rule);
 
    retention_seconds = query_retain_period(&data, &state, st.st_size, sdata.spam_message, &cfg);
-   sdata.retained = sdata.now + retention_seconds;
+   sdata.retained = sdata.sent + retention_seconds;
 
    printf("folder: %d\n", get_folder_id_by_rule(&data, &state, st.st_size, sdata.spam_message, &cfg));
 

src/tokenizer.c

@@ -75,6 +75,25 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata,
             }
          }
       }
+      else if(state->message_state == MSG_SENDER && state->is_1st_header == 1 && strlen(state->b_sender) < SMALLBUFSIZE-len-1){
+         strtolower(puf);
+
+         q = strchr(puf, '@');
+         if(q) fix_plus_sign_in_email_address(puf, &q, &len);
+
+         memcpy(&(state->b_sender[strlen(state->b_sender)]), puf, len);
+
+         if(len >= MIN_EMAIL_ADDRESS_LEN && does_it_seem_like_an_email_address(puf) == 1 && state->b_sender_domain[0] == '\0'){
+            if(q && strlen(q) > 5){
+               memcpy(&(state->b_sender_domain), q+1, strlen(q+1)-1);
+            }
+
+            if(strlen(state->b_sender) < SMALLBUFSIZE-len-1){
+               split_email_address(puf);
+               memcpy(&(state->b_sender[strlen(state->b_sender)]), puf, len);
+            }
+         }
+      }
       else if((state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT || state->message_state == MSG_ENVELOPE_TO) && state->is_1st_header == 1 && state->tolen < MAXBUFSIZE-len-1){
          strtolower(puf);
 

tests/cases/05-smtp.inc

@@ -12,21 +12,22 @@ case1() {
    "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet2" --socket --no-counter
    "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam0" --socket --no-counter
    "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam1" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam2" --socket --no-counter
    "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/journal" --socket --no-counter
    "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/deduptest" --socket --no-counter
    "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus" --socket --no-counter
 
 
-   wait_until_emails_are_processed "piler1" 3000
+   wait_until_emails_are_processed "piler1" 3007
    docker exec "piler1" su piler -c /usr/libexec/piler/indexer.delta.sh 2>/dev/null
 
-   count_status_values 3000 2892 108 0
+   count_status_values 3007 2896 111 0
 
    test_retrieved_messages_are_the_same "piler1" "piler"
 
    run_05_sphinx_tests
 
-   docker exec "piler1" su piler -c 'php /usr/libexec/piler/generate_stats.php --webui /var/piler/www --start=2015/01/01 --stop=2020/12/31'
+   docker exec "piler1" su piler -c 'php /usr/libexec/piler/generate_stats.php --webui /var/piler/www --start=2015/01/01 --stop=2021/12/31'
 
    docker exec "piler1" su piler -c 'php /usr/libexec/piler/sign.php --webui /var/piler/www --mode time'
 }

tests/setup.inc

@@ -33,13 +33,13 @@ launch_containers() {
 create_rules() {
    local container="$1"
 
-   echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"acts.hu\",\"acts.hu\"),(\"gtsce.com\",\"gtsce.com\"),(\"datanet.hu\",\"datanet.hu\"),(\"gtsdatanet.hu\",\"gtsdatanet.hu\"),(\"gts.hu\",\"gts.hu\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh
+   echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"acts.hu\",\"acts.hu\"),(\"gtsce.com\",\"gtsce.com\"),(\"datanet.hu\",\"datanet.hu\"),(\"gtsdatanet.hu\",\"gtsdatanet.hu\"),(\"gts.hu\",\"gts.hu\"),(\"aaa.fu\",\"aaa.fu\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh
 
    echo 'echo "insert into archiving_rule (subject) values (\"Android táblagép\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
    echo 'echo "insert into archiving_rule (\`from\`) values (\"@gmail.com\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
    echo 'echo "insert into archiving_rule (\`from\`,attachment_type, _attachment_size, attachment_size) values (\"finderis.co.ua\", \"image\", \">\", 100000)"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
    echo 'echo "insert into archiving_rule (\`to\`) values (\"undisclosed-recipients\")"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
-   echo 'echo "insert into import (\`type\`, username, password, server) values (\"imap\", \"sanyi@aaa.fu\", \"abcde123\", \"imap\")"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
+   echo 'echo "insert into import (\`type\`, username, password, server) values (\"imap\", \"sanyi@aaa.fu\", \"abcde123\", \"imap.aaa.fu\")"|mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
 
    echo 'echo "update user set password=\"\$6\$GKL00T\$8jqoFOe3PyAbOCLwKB7JwndwC.IinHrZRkdoQDZUc8vybZ88sA2qomlz5JceNif8fFpkGzZ03ilvQa7tqQx0v1\""| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
 

unit_tests/check_misc.c

@@ -112,7 +112,7 @@ static void test_create_id(){
 
    for(i=0; i<10; i++){
       create_id(buf, 0xf);
-      ASSERT(strncmp(buf, "40000000", strlen("40000000")) == 0, buf);
+      ASSERT(strncmp(buf, "50000000", strlen("50000000")) == 0, buf);
       ASSERT(buf[24] == '0' && buf[25] == 'f', buf);
    }
 

unit_tests/check_parser.c

@@ -10,6 +10,8 @@ struct parser_test {
    char message_id[SMALLBUFSIZE];
    char from[SMALLBUFSIZE];
    char from_domain[SMALLBUFSIZE];
+   char sender[SMALLBUFSIZE];
+   char sender_domain[SMALLBUFSIZE];
    char to[SMALLBUFSIZE];
    char to_domain[SMALLBUFSIZE];
    char reference[SMALLBUFSIZE];
@@ -26,22 +28,22 @@ static void test_parser(struct config *cfg){
    struct data data;
    struct parser_test tests[] = {
 
-      {"1.eml", "<ajahhdddhjdhddh@jatekokbirodalma.hu>", "játékok birodalma játékbolt hirlevel@jatekokbirodalma.hu hirlevel jatekokbirodalma hu ", "jatekokbirodalma.hu", "architerv m sj@acts.hu sj acts hu ", "acts.hu ", "", "BLACK FRIDAY - Hihetetlen kedvezmények csak 1 napig november 27-én", 2},
-      {"2.eml", "<20151101142653.111156815AF6D@acts.hu>", "jml lighting huixinsoft67@foxmail.com huixinsoft67 foxmail com ", "foxmail.com", "sj@acts.hu sj acts hu ", "acts.hu ", "", "New design ultra slim led  panel light", 0},
-      {"5-ibm-images.eml", "<OFC1576266.0E8F7B7D-ONC12577CB.00303030-C12577CB.003053CD@hu.ibm.com>", "ibm rendezveny rendezveny@hu.ibm.com rendezveny hu ibm com ", "hu.ibm.com", "cim1@aaaa.bbb.fu cim1 aaaa bbb fu ajajaj@piler.aaa.fu ajajaj piler aaa fu ibm rendezveny rendezveny@hu.ibm.com rendezveny hu ibm com ", "aaaa.bbb.fu piler.aaa.fu hu.ibm.com ", "", "***Emlékeztető*** - Egészségipar - eEgészségügy (Út a jövőbe, párbeszéd a gazdaságélénkítésről)  2010. november 4.", 5},
-      {"9-attached-text.eml", "<list-507327664@mail.aaa.fu>", "dr lucky amechi clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com", "usuarios-no-listados ", "", "", "Please read my attached letter", 1},
-      {"13-xlsx.eml", "<alpine.LNX.2.00.1209261517040.17054@aaa.fu>", "aaaaa@aaa.fu aaaaa aaa fu ", "aaa.fu", "sj@acts.hu sj acts hu ", "acts.hu ", "", "ez egy teszt", 1},
-      {"15-image-only-spam.eml", "<av5f1fCf5XO0oBab757826337RSFKvu@pnmarketing.com>", "kriegel paff sketches@pnmarketing.com sketches pnmarketing com ", "pnmarketing.com", "holmon knobel aaaaa@acts.hu aaaaa acts hu ", "acts.hu ", "", "Lack of concentration, backed up by a vocabulary of tremendous scope, a", 1},
-      {"16-rfc822-attachment-1.eml", "<list-423974736@mail.aaa.fu>", "martonagnes martonagnes@lajt.hu martonagnes lajt hu erős istván eistvan@marosheviz.info ", "lajt.hu", "martonagnes@lajt.hu martonagnes lajt hu ", "lajt.hu ", "", "Féláras akció! 31000Ft/2fő/3nap húsvétkor is a Park Inn****-ben!", 2 },
-      {"17-attached-text-bogus-mime.eml", "<list-507327664@mail.aaa.fu>", "dr lucky amechi clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com", "usuarios-no-listados ", "", "", "Please read my attached letter", 1},
-      {"18-spam-html-encoding.eml", "<list-435458392@mail.aaa.fu>", "a1 hitelcentrum kft Üveges szilvia a1hitelcentrum@t-online.hu a1hitelcentrum t online hu ", "t-online.hu", "postmaster@aaa.fu postmaster aaa fu ", "aaa.fu ", "", "TÁJÉKOZTATÁSVargay Péter", 0},
-      {"19-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-186.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1},
-      {"20-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-187.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1},
-      {"21-register-tricky-urls.eml", "<E1IBifn-0001un-MD@admin4.theregister.co.uk>", "the register update-49363-08f0f768@list.theregister.co.uk update 49363 08f0f768 list theregister co uk ", "list.theregister.co.uk", "hello@mail.aaa.fu hello mail aaa fu ", "mail.aaa.fu ", "", "[sp@m] Reg Headlines Friday July 20", 0},
-      {"30-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "hello@acts.hu hello acts hu ", "acts.hu ", "", "RE: hxx-ajajajaja.com_ Aaagágyi és kia ttt_webstat hiba", 0},
-      {"31-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "hello@acts.hu hello acts hu ", "acts.hu ", "", "Re: stanhu \"domain not found\"-dal eldobja a @fohu-ra küldött leveleket...", 0},
-      {"32-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "hello@acts.hu hello acts hu ", "acts.hu ", "", "<GD-XXXX/1-2015> www.ujsag.hu new virtual host reg. --> Aaaaaaaaa", 0},
-      {"33-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "hello@acts.hu hello acts hu ", "acts.hu ", "", "[JIRA] Commented: (AAAA-151) A aaa-nek kerek egy XXX-et, ZH74617282, ACC27363484944", 0},
+      {"1.eml", "<ajahhdddhjdhddh@jatekokbirodalma.hu>", "játékok birodalma játékbolt hirlevel@jatekokbirodalma.hu hirlevel jatekokbirodalma hu ", "jatekokbirodalma.hu", "", "", "architerv m sj@acts.hu sj acts hu ", "acts.hu ", "", "BLACK FRIDAY - Hihetetlen kedvezmények csak 1 napig november 27-én", 2},
+      {"2.eml", "<20151101142653.111156815AF6D@acts.hu>", "jml lighting huixinsoft67@foxmail.com huixinsoft67 foxmail com ", "foxmail.com", "jml lighting hwtwi@mcqw.com hwtwi mcqw com ", "mcqw.com", "sj@acts.hu sj acts hu huixinsoft67@foxmail.com huixinsoft67 foxmail com ", "acts.hu foxmail.com ", "", "New design ultra slim led  panel light", 0},
+      {"5-ibm-images.eml", "<OFC1576266.0E8F7B7D-ONC12577CB.00303030-C12577CB.003053CD@hu.ibm.com>", "ibm rendezveny rendezveny@hu.ibm.com rendezveny hu ibm com ", "hu.ibm.com", "marta riman rimanmarta@hu.ibm.com rimanmarta hu ibm com ", "hu.ibm.com", "cim1@aaaa.bbb.fu cim1 aaaa bbb fu ajajaj@piler.aaa.fu ajajaj piler aaa fu ibm rendezveny rendezveny@hu.ibm.com rendezveny hu ibm com ", "aaaa.bbb.fu piler.aaa.fu hu.ibm.com ", "", "***Emlékeztető*** - Egészségipar - eEgészségügy (Út a jövőbe, párbeszéd a gazdaságélénkítésről)  2010. november 4.", 5},
+      {"9-attached-text.eml", "<list-507327664@mail.aaa.fu>", "dr lucky amechi clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com", "aaa aaa aaa@aaa.fu aaa aaa fu ", "aaa.fu", "usuarios-no-listados clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com ", "", "Please read my attached letter", 1},
+      {"13-xlsx.eml", "<alpine.LNX.2.00.1209261517040.17054@aaa.fu>", "aaaaa@aaa.fu aaaaa aaa fu ", "aaa.fu", "", "", "sj@acts.hu sj acts hu ", "acts.hu ", "", "ez egy teszt", 1},
+      {"15-image-only-spam.eml", "<av5f1fCf5XO0oBab757826337RSFKvu@pnmarketing.com>", "kriegel paff sketches@pnmarketing.com sketches pnmarketing com ", "pnmarketing.com", "", "", "holmon knobel aaaaa@acts.hu aaaaa acts hu ", "acts.hu ", "", "Lack of concentration, backed up by a vocabulary of tremendous scope, a", 1},
+      {"16-rfc822-attachment-1.eml", "<list-423974736@mail.aaa.fu>", "martonagnes martonagnes@lajt.hu martonagnes lajt hu erős istván eistvan@marosheviz.info ", "lajt.hu", "postmaster postmaster@aaa.fu postmaster aaa fu ", "aaa.fu", "martonagnes@lajt.hu martonagnes lajt hu ", "lajt.hu ", "", "Féláras akció! 31000Ft/2fő/3nap húsvétkor is a Park Inn****-ben!", 2 },
+      {"17-attached-text-bogus-mime.eml", "<list-507327664@mail.aaa.fu>", "dr lucky amechi clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com", "postmaster postmaster@aaa.fu postmaster aaa fu ", "aaa.fu", "usuarios-no-listados clubzenit@zenithoteles.com clubzenit zenithoteles com ", "zenithoteles.com ", "", "Please read my attached letter", 1},
+      {"18-spam-html-encoding.eml", "<list-435458392@mail.aaa.fu>", "a1 hitelcentrum kft Üveges szilvia a1hitelcentrum@t-online.hu a1hitelcentrum t online hu ", "t-online.hu", "postmaster postmaster@aaa.fu postmaster aaa fu ", "aaa.fu", "postmaster@aaa.fu postmaster aaa fu a1hitelcentrum@t-online.hu a1hitelcentrum t online hu ", "aaa.fu t-online.hu ", "", "TÁJÉKOZTATÁSVargay Péter", 0},
+      {"19-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-186.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "", "", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1},
+      {"20-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-187.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "", "", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1},
+      {"21-register-tricky-urls.eml", "<E1IBifn-0001un-MD@admin4.theregister.co.uk>", "the register update-49363-08f0f768@list.theregister.co.uk update 49363 08f0f768 list theregister co uk ", "list.theregister.co.uk", "", "", "hello@mail.aaa.fu hello mail aaa fu ", "mail.aaa.fu ", "", "[sp@m] Reg Headlines Friday July 20", 0},
+      {"30-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", "RE: hxx-ajajajaja.com_ Aaagágyi és kia ttt_webstat hiba", 0},
+      {"31-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", "Re: stanhu \"domain not found\"-dal eldobja a @fohu-ra küldött leveleket...", 0},
+      {"32-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", "<GD-XXXX/1-2015> www.ujsag.hu new virtual host reg. --> Aaaaaaaaa", 0},
+      {"33-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", "[JIRA] Commented: (AAAA-151) A aaa-nek kerek egy XXX-et, ZH74617282, ACC27363484944", 0},
    };
 
 
@@ -66,7 +68,9 @@ static void test_parser(struct config *cfg){
 
       assert(strcmp(state.message_id, tests[i].message_id) == 0 && "test_parser()1");
       assert(strcmp(state.b_from, tests[i].from) == 0 && "test_parser()2a");
-      assert(strcmp(state.b_from_domain, tests[i].from_domain) == 0 && "test_parser()2b");
+      assert(strcmp(state.b_sender, tests[i].sender) == 0 && "test_parser()2b");
+      assert(strcmp(state.b_from_domain, tests[i].from_domain) == 0 && "test_parser()2c");
+      assert(strcmp(state.b_sender_domain, tests[i].sender_domain) == 0 && "test_parser()2d");
       assert(strcmp(state.b_to, tests[i].to) == 0 && "test_parser()3a");
       assert(strcmp(state.b_to_domain, tests[i].to_domain) == 0 && "test_parser()3b");
       assert(strcmp(state.b_subject, tests[i].subject) == 0 && "test_parser()4");

unit_tests/php/FormatTest.php

@@ -3,6 +3,7 @@
 use PHPUnit\Framework\TestCase;
 
 include_once("webui/system/model.php");
+include_once("webui/system/misc.php");
 include_once("webui/model/health/health.php");
 
 final class FormatTest extends TestCase
@@ -25,4 +26,28 @@ final class FormatTest extends TestCase
       $this->assertEquals($result, $expected_result);
    }
 
+
+   public function providerTestConvertDateStringToYmdByTemplateValues() {
+      return [
+         ['2021.05.01', 'y.m.d', ['0','0','0']],
+         ['2021.05.01', 'Y.m', ['0','0','0']],
+         ['2021.05.01', 'Y.m.d.e', ['0','0','0']],
+         ['2021.05.01', 'Y.m.d', ['2021','05','01']],
+         ['2021.05.01', 'Y.m.d.', ['0','0','0']],
+         ['2021.05.01', 'Y-m-d', ['2021','05','01']],
+         ['12/01/2008', 'm/d/Y', ['2008','12','01']],
+         ['12-01-2008', 'm-d-Y', ['2008','12','01']],
+      ];
+   }
+
+   /**
+    * @dataProvider providerTestConvertDateStringToYmdByTemplateValues
+    */
+
+   public function test_convert_date_string_to_ymd_by_template($date, $date_template, $expected_result) {
+      $result = convert_date_string_to_ymd_by_template($date, $date_template);
+      $this->assertEquals($result, $expected_result);
+   }
+
+
 }

util/Makefile.in

@@ -46,7 +46,6 @@ install:
 	$(INSTALL) -m 0755 $(srcdir)/purge.sh $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/pilerpurge.py $(DESTDIR)$(libexecdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/postinstall.sh $(DESTDIR)$(libexecdir)/piler
-	$(INSTALL) -m 0755 $(srcdir)/watch_sphinx_main_index.sh $(DESTDIR)$(libexecdir)/piler
 
 	$(INSTALL) -m 0755 $(srcdir)/db-mysql.sql $(DESTDIR)$(datarootdir)/piler
 	$(INSTALL) -m 0755 $(srcdir)/db-mysql-root.sql.in $(DESTDIR)$(datarootdir)/piler

util/imapfetch.py

@@ -44,17 +44,20 @@ def read_folder_list(conn):
         if isinstance(folder, type(b'')):
             folder = folder.decode('utf-8')
         elif isinstance(folder, type(())):
-            folder = re.sub(r'\{\d+\}$', '', folder[0]) + folder[1]
+            folder = re.sub(r'\{\d+\}$', '', folder[0].decode('utf-8')) + folder[1].decode('utf-8')
 
         # The regex should match ' "/" ' and ' "." '
         if folder:
-            f = re.split(r' \"[\/\.]\" ', folder)
+            f = re.split(r' \"[\/\.\\]+\" ', folder)
             result.append(f[1])
 
     return [x for x in result if x not in opts['skip_folders']]
 
 
 def process_folder(conn, folder):
+    # Space in the folder name must be escaped
+    folder = re.sub(r' ', '\\ ', folder)
+
     if opts['verbose']:
         print("Processing {}".format(folder))
 
@@ -75,9 +78,10 @@ def process_folder(conn, folder):
             rc, data = conn.fetch(num, '(RFC822)')
             if opts['verbose']:
                 print(rc, num)
-            opts['counter'] += 1
-            with open("{}.eml".format(opts['counter']), "wb") as f:
-                f.write(data[0][1])
+            if isinstance(data[0], tuple):
+                opts['counter'] += 1
+                with open("{}.eml".format(opts['counter']), "wb") as f:
+                    f.write(data[0][1])
 
 
 def main():

util/indexer.delta.sh.in

@@ -4,7 +4,7 @@ export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
 MAINTMPFILE=/var/run/piler/main.indexer.tmp
 DELTATMPFILE=/var/run/piler/delta.indexer.tmp
 INDEXER="indexer --config SYSCONFDIR/piler/sphinx.conf"
-PRIORITY=mail.error
+PRIORITY=mail.info
 TOUCHFILE=/var/piler/stat/indexer
 
 if [ -f $MAINTMPFILE ]; then echo "INDEXER ERROR: indexer merging to main index is already running. It started at "`cat $MAINTMPFILE` | logger -p $PRIORITY ; exit 1; fi

util/indexer.main.sh.in

@@ -1,34 +1,49 @@
 #!/bin/bash
 
-export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
-MAINTMPFILE=/var/run/piler/main.indexer.tmp
-INDEXER="indexer --config SYSCONFDIR/piler/sphinx.conf"
-PRIORITY=mail.error
-TOUCHFILE=/var/piler/stat/indexer
+set -o nounset
+set -o errexit
+set -o pipefail
 
-if [ -f $MAINTMPFILE ]; then echo "INDEXER ERROR: indexer merging to main index is already running. It started at "`cat $MAINTMPFILE` | logger -p $PRIORITY ; exit 1; fi
+MAINTMPFILE="/var/run/piler/main.indexer.tmp"
+SPHINX_CONFIG="SYSCONFDIR/piler/sphinx.conf"
+PRIORITY="mail.info"
+TOUCHFILE="/var/piler/stat/indexer"
+MAIN_INDEX="main1"
 
-date > $MAINTMPFILE
+export PATH="/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
 
-touch $TOUCHFILE
-
-function finish {
-   rm -f $MAINTMPFILE
+finish() {
+   rm -f "$MAINTMPFILE"
 }
 
+
+if [[ -f "$MAINTMPFILE" ]]; then
+   echo "INDEXER ERROR: indexer merging to main index is already running. It started at $(cat "$MAINTMPFILE")" | logger -p "$PRIORITY"
+   exit 1
+fi
+
+date > "$MAINTMPFILE"
+
+touch "$TOUCHFILE"
+
 trap finish EXIT
 
-echo "INDEXER INFO: merging to main started" | logger -p $PRIORITY
+echo "INDEXER INFO: merging to main started" | logger -p "$PRIORITY"
 
-$INDEXER --quiet --merge main1 dailydelta1 --merge-dst-range deleted 0 0 --rotate
+indexer --config "$SPHINX_CONFIG" --quiet --merge "$MAIN_INDEX" dailydelta1 --merge-dst-range deleted 0 0 --rotate
 
-echo "INDEXER INFO: merging to main finished" | logger -p $PRIORITY
+echo "INDEXER INFO: merging to main finished" | logger -p "$PRIORITY"
 
 sleep 5
 
-echo "INDEXER INFO: resetting daily delta started" | logger -p $PRIORITY
+echo "INDEXER INFO: resetting daily delta started" | logger -p "$PRIORITY"
 
-$INDEXER --quiet dailydelta1 --rotate
+indexer --config "$SPHINX_CONFIG" --quiet dailydelta1 --rotate
 
-echo "INDEXER INFO: resetting daily delta finished" | logger -p $PRIORITY
+echo "INDEXER INFO: resetting daily delta finished" | logger -p "$PRIORITY"
 
+sum=0
+while read -r a b; do
+   sum=$(( sum + b ))
+done < <( find /var/piler/sphinx/ -type f -name main\*.spd -printf "%TY%Tm%Td %s\\n" )
+printf "%d" $sum > /var/piler/stat/main_index_size

util/watch_sphinx_main_index.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-find /var/piler/sphinx/ -type f -name main\*.spd -printf "%TY%Tm%Td %s\\n" | sort -r | head -1 | cut -f2 -d ' ' > /var/piler/stat/main_index_size

webui/.htaccess

@@ -35,3 +35,24 @@ RewriteRule ^view/javascript/piler.js /js.php [QSA,L]
    </FilesMatch>
 </IfModule>
 
+<IfModule auth_gssapi_module>
+   # ktpass -princ HTTP/<webserver-fqdn>@<WINDOWS AD DOMAIN IN CAPITALS> \
+   #        -mapuser <ldap helper user>@<WINDOWS AD DOMAIN IN CAPITALS> \
+   #        -pass * \
+   #        -crypto AES256-SHA1 \
+   #        -ptype KRB5_NT_PRINCIPAL \
+   #        -out /etc/krb5/http.keytab \
+   #
+   # setspn -s HTTP/<webserver-fqdn> <ldap helper user>
+
+   <FilesMatch "sso\.php$">
+      RewriteEngine on
+      RewriteCond %{HTTP:Authorization}  !^$
+      RewriteRule .* - [E=HTTP_AUTHORIZATION:%{HTTP:Authorization},L]
+      AuthName "User with domain part (separated by @) in CAPITALS - e.g. 'user@DOMAIN'"
+      AuthType GSSAPI
+      GssapiBasicAuth On
+      GssapiCredStore keytab:/etc/krb5/http.keytab
+      Require valid-user
+   </FilesMatch>
+</IfModule>

webui/controller/audit/helper.php

@@ -64,6 +64,9 @@ class ControllerAuditHelper extends Controller {
       $this->data['actions'][ACTION_DOWNLOAD_ATTACHMENT] = $this->data['text_download_attachment2'];
       $this->data['actions'][ACTION_UNAUTHORIZED_DOWNLOAD_ATTACHMENT] = $this->data['text_unauthorized_download_attachment'];
       $this->data['actions'][ACTION_VIEW_JOURNAL] = $this->data['text_view_journal'];
+      $this->data['actions'][ACTION_MARK_MESSAGE_FOR_REMOVAL] = $this->data['text_remove_request'];
+      $this->data['actions'][ACTION_MARK_AS_PRIVATE] = $this->data['text_mark_private'];
+      $this->data['actions'][ACTION_REJECT_REMOVAL] = $this->data['text_rejected_removal'];
 
 
 

webui/controller/common/layout-audit-removal.php

@@ -17,13 +17,9 @@ class ControllerCommonLayoutAuditRemoval extends Controller {
          $this->children = array(
                       "common/menu",
                       "search/folder",
-                      "search/popup",
                       "common/footer"
          );
 
          $this->render();
-
       }
-
-
 }

webui/controller/common/layout-audit.php

@@ -17,7 +17,6 @@ class ControllerCommonLayoutAudit extends Controller {
          $this->children = array(
                       "common/menu",
                       "search/folder",
-                      "search/popup",
                       "common/footer"
          );
 

webui/controller/common/layout-search.php

@@ -19,7 +19,6 @@ class ControllerCommonLayoutSearch extends Controller {
          $this->children = array(
                       "common/menu",
                       "search/folder",
-                      "search/popup",
                       "common/footer"
          );
 

webui/controller/message/rejectremove.php

@@ -37,6 +37,8 @@ class ControllerMessageRejectRemove extends Controller {
 
       // Shouldn't we ask for a token or something as well?
 
+      AUDIT(ACTION_REJECT_REMOVAL, '', '', $id, '');
+
       $db->query("UPDATE " . TABLE_DELETED . " SET deleted=0, approver=?, date2=?, reason2=? WHERE id=?", [$this->data['username'], NOW, $this->request->post['reason2'], $id]);
       syslog(LOG_INFO, $this->data['username'] . " rejected removing message: $id");
 

webui/controller/search/helper.php

@@ -15,6 +15,7 @@ class ControllerSearchHelper extends Controller {
                     'folders'         => '',
                     'extra_folders'   => '',
                     'id'              => '',
+                    'raw'              => '',
                     'match'           => array()
                      );
 
@@ -49,7 +50,7 @@ class ControllerSearchHelper extends Controller {
 
       if($this->request->post['searchtype'] == 'expert'){
 
-         if(isset($this->request->post['search']) && preg_match("/(from|to|subject|body|direction|d|size|date1|date2|attachment|a|tag|note|id)\:/", $this->request->post['search'])) {
+         if(isset($this->request->post['search']) && preg_match("/(from|to|subject|body|direction|d|size|date1|date2|attachment|a|tag|note|id|raw)\:/", $this->request->post['search'])) {
             $this->a = $this->model_search_search->preprocess_post_expert_request($this->request->post);
          }
          else {

webui/controller/search/popup.php

@@ -1,17 +0,0 @@
-<?php  
-
-
-class ControllerSearchPopup extends Controller {
-
-   protected function index() {
-
-      $this->id = "popup";
-      $this->template = "search/popup.tpl";
-
-      $this->render();
-   }
-
-
-}
-
-?>

webui/language/cz/messages.php

@@ -211,6 +211,7 @@ $_['text_logout2'] = "odhlášení";
 $_['text_maillog_status'] = "status maillog collectoru";
 $_['text_main_title'] = "clapf web UI";
 $_['text_mapped_domain'] = "Mapovaná doména";
+$_['text_mark_private'] = "private";
 $_['text_marked_for_removal'] = "Zpráva označena k odstranění";
 $_['text_memory_usage'] = "Využití paměti";
 $_['text_message'] = "zpráva";
@@ -492,3 +493,5 @@ $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
 
 $_['text_wildcard_domains'] = "Wildcard domains";
+$_['text_remove_request'] = "remove request";
+$_['text_rejected_removal'] = "rejected removal";

webui/language/de/messages.php

@@ -197,6 +197,7 @@ $_['text_logout2'] = "Abmeldung";
 $_['text_maillog_status'] = "Sammelstatus Maillogbuch";
 $_['text_main_title'] = "Clapf Benutzeroberfläche";
 $_['text_mapped_domain'] = "Domänenalias";
+$_['text_mark_private'] = "private";
 $_['text_marked_for_removal'] = "Nachricht zum Löschen markiert";
 $_['text_memory_usage'] = "Arbeitsspeicher";
 $_['text_message'] = "Nachricht";
@@ -499,3 +500,5 @@ $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
 
 $_['text_wildcard_domains'] = "Wildcard domains";
+$_['text_remove_request'] = "remove request";
+$_['text_rejected_removal'] = "rejected removal";

webui/language/en/messages.php

@@ -216,6 +216,7 @@ $_['text_logout2'] = "logout";
 $_['text_maillog_status'] = "maillog collector status";
 $_['text_main_title'] = "clapf web UI";
 $_['text_mapped_domain'] = "Mapped domain";
+$_['text_mark_private'] = "private";
 $_['text_marked_for_removal'] = "Message marked for removal";
 $_['text_memory_usage'] = "Memory usage";
 $_['text_message'] = "message";
@@ -287,6 +288,7 @@ $_['text_ref'] = "Reference";
 $_['text_refresh_period'] = "Refresh period";
 $_['text_refresh_qr_code'] = "Refresh QR code";
 $_['text_reject'] = "Reject";
+$_['text_rejected_removal'] = "rejected removal";
 $_['text_reason_of_rejection'] = "Reason of rejection";
 $_['text_relay_details'] = "Relay details";
 $_['text_relay_status'] = "Relay status";
@@ -294,6 +296,7 @@ $_['text_remove'] = "Remove";
 $_['text_remove_domain'] = "Remove domain";
 $_['text_remove_message'] = "Remove message";
 $_['text_remove_message2'] = "remove message";
+$_['text_remove_request'] = "remove request";
 $_['text_remove_selected_uids'] = "Remove selected uids";
 $_['text_remove_policy'] = "Remove policy";
 $_['text_remove_rule'] = "Remove rule";

webui/language/es/messages.php

@@ -197,6 +197,7 @@ $_['text_logout2'] = "cerrar sesión";
 $_['text_maillog_status'] = "estado del colector de logs de mails";
 $_['text_main_title'] = "IU web clapf";
 $_['text_mapped_domain'] = "Dominio mapeado";
+$_['text_mark_private'] = "private";
 $_['text_marked_for_removal'] = "Mensaje marcado para remover";
 $_['text_memory_usage'] = "Uso de memoria";
 $_['text_message'] = "mensaje";
@@ -497,3 +498,5 @@ $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
 
 $_['text_wildcard_domains'] = "Wildcard domains";
+$_['text_remove_request'] = "remove request";
+$_['text_rejected_removal'] = "rejected removal";

webui/language/fr/messages.php

@@ -210,6 +210,7 @@ $_['text_logout2'] = "déconnexion";
 $_['text_maillog_status'] = "statut du collecteur de logs mails";
 $_['text_main_title'] = "clapf web UI";
 $_['text_mapped_domain'] = "Domaine mappé";
+$_['text_mark_private'] = "private";
 $_['text_marked_for_removal'] = "Message marqué pour suppression";
 $_['text_memory_usage'] = "Utilisation mémoire";
 $_['text_message'] = "message";
@@ -494,3 +495,5 @@ $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
 
 $_['text_wildcard_domains'] = "Wildcard domains";
+$_['text_remove_request'] = "remove request";
+$_['text_rejected_removal'] = "rejected removal";

webui/language/hu/messages.php

@@ -217,6 +217,7 @@ $_['text_logout2'] = "kijelentkezés";
 $_['text_maillog_status'] = "maillog gyűjtő státusz";
 $_['text_main_title'] = "clapf web UI";
 $_['text_mapped_domain'] = "Hozzárendelt domain";
+$_['text_mark_private'] = "privát";
 $_['text_marked_for_removal'] = "Levél törlésre jelölve";
 $_['text_memory_usage'] = "Memória használat";
 $_['text_message'] = "üzenet";
@@ -289,12 +290,14 @@ $_['text_ref'] = "Hivatkozás";
 $_['text_refresh_period'] = "Frissítési periódus";
 $_['text_refresh_qr_code'] = "QR kód frissítése";
 $_['text_reject'] = "Elutasítás";
+$_['text_rejected_removal'] = "törlés elutasítva";
 $_['text_relay_details'] = "Relay részletek";
 $_['text_relay_status'] = "Relay státusz";
 $_['text_remove'] = "Törlés";
 $_['text_remove_domain'] = "Domain törlése";
 $_['text_remove_message'] = "Levél törlése";
 $_['text_remove_message2'] = "levél törlése";
+$_['text_remove_request'] = "törlésre jelölve";
 $_['text_remove_selected_uids'] = "Kijelölt azonosítók törlése";
 $_['text_remove_policy'] = "Házirend törlése";
 $_['text_remove_rule'] = "Szabály törlése";

webui/language/pl/messages.php

@@ -212,6 +212,7 @@ $_['text_logout2'] = "wylogowany";
 $_['text_maillog_status'] = "program od zbierania wiadomości e-mail";
 $_['text_main_title'] = "clapf interfejsu użytkownika strony";
 $_['text_mapped_domain'] = "Zmapowana domena";
+$_['text_mark_private'] = "private";
 $_['text_marked_for_removal'] = "Wiadomość oznaczona do usunięcia";
 $_['text_memory_usage'] = "Użycie pamięci";
 $_['text_message'] = "wiadomość";
@@ -493,3 +494,5 @@ $_['text_user_data_officer'] = "Data officer";
 $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
 $_['text_wildcard_domains'] = "Wildcard domains";
+$_['text_remove_request'] = "remove request";
+$_['text_rejected_removal'] = "rejected removal";

webui/language/pt/messages.php

@@ -486,3 +486,6 @@ $_['text_user_data_officer'] = "Data officer";
 $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
 $_['text_wildcard_domains'] = "Wildcard domains";
+$_['text_remove_request'] = "remove request";
+
+$_['text_rejected_removal'] = "rejected removal";

webui/language/ru/messages.php

@@ -210,6 +210,7 @@ $_['text_logout2'] = "выход";
 $_['text_maillog_status'] = "статус сборщика maillog";
 $_['text_main_title'] = "Основной заголовок";
 $_['text_mapped_domain'] = "Отображаемый домен";
+$_['text_mark_private'] = "private";
 $_['text_marked_for_removal'] = "Помеченные для удаления сообщения";
 $_['text_memory_usage'] = "Использование памяти";
 $_['text_message'] = "сообщение";
@@ -494,3 +495,6 @@ $_['text_user_data_officer'] = "Data officer";
 $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
 $_['text_wildcard_domains'] = "Wildcard domains";
+$_['text_remove_request'] = "remove request";
+
+$_['text_rejected_removal'] = "rejected removal";

webui/language/tr/messages.php

@@ -209,6 +209,7 @@ $_['text_logout2'] = "çıkış";
 $_['text_maillog_status'] = "e-posta logu toplama durumu";
 $_['text_main_title'] = "clapf web arabirimi";
 $_['text_mapped_domain'] = "Eşleşmiş alan adı";
+$_['text_mark_private'] = "private";
 $_['text_marked_for_removal'] = "Mesaj kaldırılmak üzere işaretlenmiş";
 $_['text_memory_usage'] = "Hafıza kullanımı";
 $_['text_message'] = "mesaj";
@@ -494,3 +495,6 @@ $_['text_user_data_officer'] = "Data officer";
 $_['text_no_selected_message'] = "no selected message";
 $_['text_create_note'] = "Create note";
 $_['text_wildcard_domains'] = "Wildcard domains";
+$_['text_remove_request'] = "remove request";
+
+$_['text_rejected_removal'] = "rejected removal";

webui/model/group/group.php

@@ -194,7 +194,7 @@ class ModelGroupGroup extends Model {
             $query = $ldap->query(LDAP_BASE_DN, "(&(objectClass=" . LDAP_ACCOUNT_OBJECTCLASS . ")(" . LDAP_MAIL_ATTR . "=" . $username_prefix . $s . "*))", array());
 
             if(isset($query->rows)) {
-               $emails = $this->model_user_auth->get_email_array_from_ldap_attr($query->rows);
+               $emails = $this->model_user_auth->get_email_array_from_ldap_attr($query->rows, LDAP_DISTRIBUTIONLIST_OBJECTCLASS);
             }
          }
       }

webui/model/mail/mail.php

@@ -41,7 +41,7 @@ class ModelMailMail extends Model {
 
       $l = fgets($r, 4096);
 
-      if(preg_match("/^250/", $l)){ $queue_id = $l; $ok = 1; }
+      if(preg_match("/^250/", $l)){ $queue_id = trim($l); $ok = 1; }
 
       fputs($r, "QUIT\r\n");
       $l = fgets($r, 4096);

webui/model/message/attachment.php

@@ -2,11 +2,10 @@
 
 class ModelMessageAttachment extends Model {
 
-
    public function get_attachment_by_id($id = 0) {
       if($id <= 0) { return []; }
 
-      $query = $this->db->query("SELECT id, piler_id, attachment_id, name, type FROM " . TABLE_ATTACHMENT . " WHERE id=?", [$id]);
+      $query = $this->db->query("SELECT id, piler_id, attachment_id, name, type, ptr FROM " . TABLE_ATTACHMENT . " WHERE id=?", [$id]);
 
       if(isset($query->row)) {
          if($query->row['ptr'] > 0) {
@@ -77,4 +76,54 @@ class ModelMessageAttachment extends Model {
       return $images;
    }
 
+
+   public function dump_attachment($basedir='', $in_or_out="in", $email='', $id=0, $attachment=[]) {
+      if($basedir == '' || $email == '') {
+         return;
+      }
+
+      $dir = sprintf("%s/%s/%s", $basedir, $email, $in_or_out);
+
+      if(!is_dir($dir)) {
+         if(!mkdir($dir, 0700, true)) {
+            die("Failed to create folder $dir");
+         }
+      }
+
+      $fname = sprintf("%s/%d-%s", $dir, $id, $attachment['filename']);
+      $fp = fopen($fname, "w+");
+      if($fp) {
+         fwrite($fp, $attachment['attachment']);
+         fclose($fp);
+      } else {
+         syslog(LOG_INFO, "ERROR: could not write $fname");
+      }
+   }
+
+
+   public function get_last_attachment_id() {
+      $query = $this->db->query("SELECT id FROM " . TABLE_ATTACHMENT . " ORDER BY id DESC LIMIT 1");
+
+      if(isset($query->row['id'])) {
+         return $query->row['id'];
+      }
+
+      return 0;
+   }
+
+
+   public function get_checkpoint() {
+      $query = $this->db->query("SELECT value FROM `" . TABLE_OPTION . "` WHERE `key`=?", [ATTACHMENT_DUMP_CHECKPOINT]);
+      if(isset($query->row['value'])) {
+         return $query->row['value'];
+      } else {
+         $this->db->query("INSERT INTO `" . TABLE_OPTION . "` (`key`, value) VALUES(?,0)", [ATTACHMENT_DUMP_CHECKPOINT]);
+         return 1;
+      }
+   }
+
+
+   public function update_checkpoint($value=0) {
+      $this->db->query("UPDATE `" . TABLE_OPTION . "` SET value=? WHERE `key`=?", [$value, ATTACHMENT_DUMP_CHECKPOINT]);
+   }
 }

webui/model/search/message.php

@@ -224,7 +224,7 @@ class ModelSearchMessage extends Model {
 
       if($html == 0) {
          while(list($k, $v) = each($terms)) {
-            $s = preg_replace("/$v/i", "<span class=\"message_highlight\">$v</span>", $s);
+            $s = preg_replace("/$v/i", "<span class=\"mssghglght\">$v</span>", $s);
          }
 
          return $s;
@@ -246,7 +246,7 @@ class ModelSearchMessage extends Model {
 
                reset($terms);
                while(list($k, $v) = each($terms)) {
-                  $str = preg_replace("/$v/i", "<span class=\"message_highlight\">$v</span>", $str);
+                  $str = preg_replace("/$v/i", "<span class=\"mssghglght\">$v</span>", $str);
                }
 
                $s .= $str;
@@ -296,6 +296,30 @@ class ModelSearchMessage extends Model {
    }
 
 
+   public function get_message_addresses_by_piler_id($piler_id='', $domains=[]) {
+      $id = 0;
+      $sender = '';
+      $rcpt = [];
+
+      $query = $this->db->query("SELECT id, `from`, `fromdomain` FROM " . TABLE_META . " WHERE piler_id=?", [$piler_id]);
+      if(isset($query->row)) {
+         $id = $query->row['id'];
+         if(in_array($query->row['fromdomain'], $domains)) {
+            $sender = $query->row['from'];
+         }
+      }
+
+      $query = $this->db->query("SELECT `to`, `todomain` FROM " . TABLE_RCPT . " WHERE id=?", [$id]);
+      foreach($query->rows as $row) {
+         if(in_array($row['todomain'], $domains)) {
+            $rcpt[] = $row['to'];
+         }
+      }
+
+      return ['sender' => $sender, 'rcpt' => $rcpt];
+   }
+
+
    public function get_attachment_by_id($id = 0) {
       if($id <= 0) { return array(); }
 

webui/model/search/search.php

@@ -112,7 +112,6 @@ class ModelSearchSearch extends Model {
 
       $session = Registry::get('session');
 
-
       $i = 0;
       while(list($k, $v) = each($data['match'])) {
          if($v == "@attachment_types") {
@@ -163,6 +162,10 @@ class ModelSearchSearch extends Model {
 
       $match = implode(" ", $data['match']);
 
+      if(Registry::get('auditor_user') == 1 && RESTRICTED_AUDITOR == 0 && $data['raw']) {
+         $match .= $data['raw'];
+      }
+
       if($emailfilter) {
          if(strlen($match) > 2) { $match = "( $match ) & $emailfilter"; }
          else { $match = $emailfilter; }
@@ -327,6 +330,7 @@ class ModelSearchSearch extends Model {
                     'folders'         => '',
                     'extra_folders'   => '',
                     'id'              => '',
+                    'raw'             => '',
                     'match'           => array()
       );
 
@@ -351,15 +355,12 @@ class ModelSearchSearch extends Model {
          else if($v == 'subject:') { $token = 'match'; $a['match'][] = '@subject'; continue; }
          else if($v == 'body:') { $token = 'match'; $a['match'][] = '@body'; continue; }
          else if($v == 'direction:' || $v == 'd:') { $token = 'direction'; continue; }
-         else if($v == 'size:') { $token = 'size'; continue; }
-         else if($v == 'date1:') { $token = 'date1'; continue; }
-         else if($v == 'date2:') { $token = 'date2'; continue; }
          else if($v == 'attachment:' || $v == 'a:') { $token = 'match'; $a['match'][] = '@attachment_types'; continue; }
-         else if($v == 'size') { $token = 'size'; continue; }
-         else if($v == 'tag:') { $token = 'tag'; continue; }
-         else if($v == 'note:') { $token = 'note'; continue; }
-         else if($v == 'ref:') { $token = 'ref'; continue; }
-         else if($v == 'id:') { $token = 'id'; continue; }
+
+         else if(in_array($v, ['size:', 'date1:', 'date2:', 'tag:', 'note:', 'ref:', 'id:', 'raw:'])) {
+            $token = substr($v, 0, strlen($v)-1); continue;
+         }
+
          else if($token != 'date1' && $token != 'date2') {
             if(preg_match("/\d{4}\-\d{1,2}\-\d{1,2}/", $v) || preg_match("/\d{1,2}\/\d{1,2}\/\d{4}/", $v)) {
                $ndate++;
@@ -368,12 +369,7 @@ class ModelSearchSearch extends Model {
          }
 
          if($token == 'match') { $a['match'][] = $v; }
-         else if($token == 'date1') { $a['date1'] = ' ' . $v; }
-         else if($token == 'date2') { $a['date2'] = ' ' . $v; }
-         else if($token == 'tag') { $a['tag'] .= ' ' . $v; }
-         else if($token == 'note') { $a['note'] .= ' ' . $v; }
-         else if($token == 'ref') { $a['ref'] = ' ' . $v; }
-         else if($token == 'id') { $a['id'] .= ' ' . $v; }
+         else if(in_array($token, ['date1', 'date2', 'ref', 'tag', 'note', 'id', 'raw'])) { $a[$token] .= ' ' . $v; }
 
          else if($token == 'direction') {
             if($v == 'inbound') { $a['direction'] = "0"; }
@@ -473,6 +469,7 @@ class ModelSearchSearch extends Model {
       $note = array();
       $private = [];
       $deleted = [];
+      $marked_for_removal = [];
       $q = '';
       global $SUPPRESS_RECIPIENTS;
 
@@ -489,12 +486,10 @@ class ModelSearchSearch extends Model {
 
       if(isset($query->rows)) {
          foreach($query->rows as $r) {
-            if(!isset($rcpt[$r['id']]) && !in_array($r['to'], $SUPPRESS_RECIPIENTS)) {
-               $srcpt[$r['id']] = $r['to'];
-               $rcpt[$r['id']] = $r['to'];
-            }
-            else {
-               if(Registry::get('auditor_user') == 1) { $rcpt[$r['id']] .= ",\n" . $r['to']; }
+            if(!isset($rcpt[$r['id']])) { $rcpt[$r['id']] = []; }
+
+            if(Registry::get('auditor_user') == 1 || !in_array($r['to'], $SUPPRESS_RECIPIENTS)) {
+               array_push($rcpt[$r['id']], $r['to']);
             }
          }
       }
@@ -511,10 +506,12 @@ class ModelSearchSearch extends Model {
          }
 
          if(ENABLE_DELETE) {
-            $s = $this->db->query("SELECT `id` FROM `" . TABLE_DELETED . "` WHERE deleted=1 AND id IN ($q)", $ids);
-
+            $s = $this->db->query("SELECT `id`, `deleted` FROM `" . TABLE_DELETED . "` WHERE id IN ($q)", $ids);
             foreach ($s->rows as $p) {
-               $deleted[$p['id']] = 1;
+               if($p['id'] == 1) {
+                  $deleted[$p['id']] = 1;
+               }
+               $marked_for_removal[$p['id']] = 1;
             }
          }
 
@@ -538,12 +535,13 @@ class ModelSearchSearch extends Model {
          foreach($query->rows as $m) {
             // We mark it as deleted even if it's only marked for removal
             if(ENABLE_DELETE == 1 && ($m['retained'] < NOW || isset($deleted[$m['id']])) ) $m['deleted'] = 1; else $m['deleted'] = 0;
+            if(ENABLE_DELETE == 1 && isset($marked_for_removal[$m['id']])) $m['marked_for_removal'] = 1; else $m['marked_for_removal'] = 0;
 
             $m['shortfrom'] = make_short_string($m['from'], MAX_CGI_FROM_SUBJ_LEN);
             $m['from'] = escape_gt_lt_quote_symbols($m['from']);
 
-            isset($srcpt[$m['id']]) ? $m['shortto'] = $srcpt[$m['id']] : $m['shortto'] = '';
             isset($rcpt[$m['id']]) ? $m['to'] = $rcpt[$m['id']] : $m['to'] = '';
+            $m['shortto'] = make_short_string($this->get_preferred_recipient($rcpt[$m['id']]), MAX_CGI_FROM_SUBJ_LEN);
             $m['to'] = escape_gt_lt_quote_symbols($m['to']);
 
 
@@ -586,32 +584,23 @@ class ModelSearchSearch extends Model {
    }
 
 
-   public function get_message_recipients($id = '') {
-      $rcpt = array();
-      $domains = array();
+   private function get_preferred_recipient($arr = []) {
+      $result = '';
 
-      if(Registry::get('auditor_user') == 0) { return $rcpt; }
+      $session = Registry::get('session');
+      $group_emails = $session->get('group_emails');
+      $user_emails = $session->get('user_emails');
 
-      $query = $this->db->query("SELECT `domain` FROM " . TABLE_DOMAIN);
-      foreach($query->rows as $q) {
-         array_push($domains, $q['domain']);
+      if(count($arr) < 2 || (!$group_emails && !$user_emails) ) { return $arr[0]; }
+
+      foreach ($arr as $a) {
+         if($result == '' && in_array($a, $group_emails)) { $result = $a; }
+         if(in_array($a, $user_emails)) { $result = $a; }
       }
 
-      $query = $this->db->query("SELECT `to` FROM " . VIEW_MESSAGES . " WHERE id=?", array($id));
+      if($result == '') { $result = $arr[0]; }
 
-      foreach($query->rows as $q) {
-         $mydomain = 0;
-
-         foreach ($domains as $domain) {
-            if(preg_match("/\@$domain$/", $q['to'])) { $mydomain = 1; break; }
-         }
-
-         if($mydomain == 1) {
-            array_push($rcpt, $q['to']);
-         }
-      }
-
-      return $rcpt;
+      return $result;
    }
 
 

webui/model/user/auth.php

@@ -180,7 +180,6 @@ class ModelUserAuth extends Model {
 
       $ldap_type = '';
       $ldap_host = LDAP_HOST;
-      $ldap_port = LDAP_PORT;
       $ldap_base_dn = LDAP_BASE_DN;
       $ldap_helper_dn = LDAP_HELPER_DN;
       $ldap_helper_password = LDAP_HELPER_PASSWORD;
@@ -212,7 +211,7 @@ class ModelUserAuth extends Model {
 
       if($ldap_host == '' || $ldap_helper_password == '') { return 0; }
 
-      $ldap = new LDAP($ldap_host, $ldap_port, $ldap_helper_dn, $ldap_helper_password);
+      $ldap = new LDAP($ldap_host, $ldap_helper_dn, $ldap_helper_password);
 
       if($ldap->is_bind_ok()) {
 
@@ -221,7 +220,7 @@ class ModelUserAuth extends Model {
          if(isset($query->row['dn']) && $query->row['dn']) {
             $a = $query->row;
 
-            $ldap_auth = new LDAP($ldap_host, $ldap_port, $a['dn'], $password);
+            $ldap_auth = new LDAP($ldap_host, $a['dn'], $password);
 
             if(LOG_LEVEL >= NORMAL) { syslog(LOG_INFO, "ldap auth against '" . $ldap_host . "', dn: '" . $a['dn'] . "', result: " . $ldap_auth->is_bind_ok()); }
 
@@ -234,7 +233,7 @@ class ModelUserAuth extends Model {
                if($this->check_ldap_membership($ldap_auditor_member_dn, $query->rows) == 1) { $role = 2; }
                if($this->check_ldap_membership($ldap_admin_member_dn, $query->rows) == 1) { $role = 1; }
 
-               $emails = $this->get_email_array_from_ldap_attr($query->rows);
+               $emails = $this->get_email_array_from_ldap_attr($query->rows, $ldap_distributionlist_objectclass);
 
                $extra_emails = $this->model_user_user->get_email_addresses_from_groups($emails);
                $emails = array_merge($emails, $extra_emails);
@@ -292,11 +291,19 @@ class ModelUserAuth extends Model {
    }
 
 
-   public function get_email_array_from_ldap_attr($e = array()) {
+   public function get_email_array_from_ldap_attr($e = array(), $group_object_class) {
       global $mailattrs;
       $data = [];
+      $group_emails = [];
+      $user_emails = [];
 
       foreach($e as $a) {
+         $group_object = 0;
+
+         if($group_object_class && in_array($group_object_class, $a['objectclass'])) {
+            $group_object = 1;
+         }
+
          if(LOG_LEVEL >= DEBUG) { syslog(LOG_INFO, "checking ldap entry dn: " . $a['dn'] . ", cn: " . $a['cn']); }
 
          foreach ($mailattrs as $mailattr) {
@@ -316,7 +323,15 @@ class ModelUserAuth extends Model {
                         }
 
                         $email = preg_replace("/^([\w]+)\:/i", "", $a[$mailattr][$i]);
-                        if(validemail($email) && !in_array($email, $data)) { array_push($data, $email); }
+                        if(validemail($email)) {
+                           if(!in_array($email, $data)) { array_push($data, $email); }
+
+                           if($group_object) {
+                              if(!in_array($email, $group_emails)) { array_push($group_emails, $email); }
+                           } else {
+                              if(!in_array($email, $user_emails)) { array_push($user_emails, $email); }
+                           }
+                        }
                      }
                   }
                }
@@ -324,12 +339,25 @@ class ModelUserAuth extends Model {
                   if(LOG_LEVEL >= DEBUG) { syslog(LOG_INFO, "checking entry #2: " . $a[$mailattr]); }
 
                   $email = strtolower(preg_replace("/^([\w]+)\:/i", "", $a[$mailattr]));
-                  if(validemail($email) && !in_array($email, $data)) { array_push($data, $email); }
+                  if(validemail($email)) {
+                     if(!in_array($email, $data)) { array_push($data, $email); }
+
+                     if($group_object) {
+                        if(!in_array($email, $group_emails)) { array_push($group_emails, $email); }
+                     } else {
+                        if(!in_array($email, $user_emails)) { array_push($user_emails, $email); }
+                     }
+                  }
                }
             }
          }
       }
 
+      $session = Registry::get('session');
+
+      $session->set("user_emails", $user_emails);
+      $session->set("group_emails", $group_emails);
+
       return $data;
    }
 
@@ -349,7 +377,7 @@ class ModelUserAuth extends Model {
 
       $uid = $this->model_user_user->get_uid_by_email($email);
       if($uid < 1) {
-         $uid = $this->model_user_user->get_next_uid(TABLE_EMAIL);
+         $uid = $this->model_user_user->get_next_uid();
          $query = $this->db->query("INSERT INTO " . TABLE_EMAIL . " (uid, email) VALUES(?,?)", array($uid, $email));
       }
 
@@ -478,7 +506,7 @@ class ModelUserAuth extends Model {
 
       if(LOG_LEVEL >= NORMAL) { syslog(LOG_INFO, "sso login: $sso_user"); }
 
-      $ldap = new LDAP(LDAP_HOST, LDAP_PORT, LDAP_HELPER_DN, LDAP_HELPER_PASSWORD);
+      $ldap = new LDAP(LDAP_HOST, LDAP_HELPER_DN, LDAP_HELPER_PASSWORD);
 
       if($ldap->is_bind_ok()) {
 
@@ -506,7 +534,7 @@ class ModelUserAuth extends Model {
 
             $query = $ldap->query(LDAP_BASE_DN, "(|(&(objectClass=user)(" . $ldap_mail_attr . "$username))(&(objectClass=group)(member=$username))(&(objectClass=group)(member=" . stripslashes($a['dn']) . ")))", array());
 
-            $emails = $this->get_email_array_from_ldap_attr($query->rows);
+            $emails = $this->get_email_array_from_ldap_attr($query->rows, LDAP_DISTRIBUTIONLIST_OBJECTCLASS);
 
             $extra_emails = $this->model_user_user->get_email_addresses_from_groups($emails);
             $emails = array_merge($emails, $extra_emails);

webui/model/user/google.php

@@ -37,8 +37,8 @@ class ModelUserGoogle extends Model {
       $session->set("uid", $user['uid']);
       $session->set("admin_user", 0);
       $session->set("email", $user['username']);
-      $session->set("domain", $query->row['domain']);
-      $session->set("realname", $query->row['realname']);
+      $session->set("domain", $user['domain']);
+      $session->set("realname", $user['realname']);
 
       $session->set("emails", $this->model_user_user->get_users_all_email_addresses($user['uid']));
       $session->set("folders", $this->model_folder_folder->get_folder_id_array_for_user($user['uid']));

webui/model/user/user.php

@@ -306,9 +306,9 @@ class ModelUserUser extends Model {
    }
 
 
-   public function get_next_uid($table = TABLE_USER) {
+   public function get_next_uid() {
 
-      $query = $this->db->query("SELECT MAX(uid) AS last_id FROM " . $table);
+      $query = $this->db->query("SELECT MAX(uid) AS last_id FROM " . TABLE_EMAIL);
 
       if(isset($query->row['last_id']) && $query->row['last_id'] > 0) {
          return (int)$query->row['last_id'] + 1;

webui/system/database/ldap.php

@@ -6,9 +6,9 @@ class LDAP {
    private $link;
    private $bind;
 
-   public function __construct($ldaphost, $ldapport, $binddn, $bindpw) {
+   public function __construct($ldaphost, $binddn, $bindpw) {
 
-      $this->link = ldap_connect($ldaphost, $ldapport) or exit('Error: ldap_connect()');
+      $this->link = ldap_connect($ldaphost) or exit('Error: ldap_connect()');
       ldap_set_option($this->link, LDAP_OPT_PROTOCOL_VERSION, 3);
       ldap_set_option($this->link, LDAP_OPT_REFERRALS, 0);
 

webui/system/misc.php

@@ -1,5 +1,10 @@
 <?php
 
+function H($s = '') {
+   print htmlentities($s);
+}
+
+
 function LOGGER($event = '', $username = '') {
    $ipaddr = '';
 
@@ -436,6 +441,26 @@ function fetch_url($url = '') {
 }
 
 
+function convert_date_string_to_ymd_by_template($date_string, $date_template) {
+   $Y = $m = $d = 0;
+
+   $s = $template_array = preg_split("/(\.|\-|\/)/", $date_template);
+   sort($s);
+
+   $date_array = preg_split("/(\.|\-|\/)/", $date_string);
+
+   if($s != ['Y','d','m'] || count($template_array) != 3 || count($date_array) != 3) {
+      return [$Y, $m, $d];
+   }
+
+   while(list($k, $v) = each($template_array)) {
+      $$v = $date_array[$k];
+   }
+
+   return [$Y, $m, $d];
+}
+
+
 function fixup_date_condition($field = '', $date1 = 0, $date2 = 0) {
    global $session;
 
@@ -449,12 +474,7 @@ function fixup_date_condition($field = '', $date1 = 0, $date2 = 0) {
 
 
    if($date1) {
-      list($y,$m,$d) = preg_split("/(\.|\-|\/)/", $date1);
-
-      if(DATE_TEMPLATE == 'd/m/Y') { $a = $y; $y = $d; $d = $a; }
-
-      if($m == '*') { $m = 0; }
-      if($d == '*') { $d = 0; }
+      list($y,$m,$d) = convert_date_string_to_ymd_by_template($date1, DATE_TEMPLATE);
 
       $date1 = mktime(0, 0, 0, $m, $d, $y);
 
@@ -462,9 +482,7 @@ function fixup_date_condition($field = '', $date1 = 0, $date2 = 0) {
    }
 
    if($date2) {
-      list($y,$m,$d) = preg_split("/(\.|\-|\/)/", $date2);
-
-      if(DATE_TEMPLATE == 'd/m/Y') { $a = $y; $y = $d; $d = $a; }
+      list($y,$m,$d) = convert_date_string_to_ymd_by_template($date2, DATE_TEMPLATE);
 
       $date2 = mktime(23, 59, 59, $m, $d, $y);
 
@@ -480,7 +498,31 @@ function fixup_date_condition($field = '', $date1 = 0, $date2 = 0) {
 
 
 function make_short_string($what, $length) {
-   return strlen($what) > $length ? substr($what, 0, $length) . "..." : $what;
+   if($length < 1) { return ''; }
+
+   if(strlen($what) <= $length) { return $what; }
+
+   $arr = preg_split("/\s/", $what);
+   $s = '';
+
+   $i = 0;
+   foreach($arr as $a) {
+      if($i == 0) {
+         if($length > 0 && strlen($a) > $length) {
+            return substr($a, 0, $length) . '...';
+         }
+      }
+
+      if(strlen($s) + strlen($a) <= $length) {
+         $s .= $a . ' ';
+      } else {
+         break;
+      }
+
+      $i++;
+   }
+
+   return $s . '...';
 }
 
 

webui/view/javascript/piler-in.js

@@ -357,6 +357,13 @@ var Piler =
     },
 
 
+    show_advanced_search_modal:function()
+    {
+        Piler.log("[show_advanced_search_modal]");
+        $('#advancedsearch-modal').modal('show');
+    },
+
+
     show_bulk_remove_modal:function()
     {
         Piler.log("[show_bulk_remove_modal]");
@@ -693,7 +700,7 @@ var Piler =
         // a = $( a );// a == DOM element
         // a = Piler.getSource( a );// a == Javascript event
 
-        var z = $('div#searchpopup1');
+        var z = $('div#advancedsearch-modal');
 
         Piler.search = 'Complex';
 
@@ -727,9 +734,9 @@ var Piler =
             extra_folders: Piler.extra_folders
         }
 
-        Piler.load_search_results();
+        $('input#_search').val('');
 
-        $('#searchpopup1').hide();
+        Piler.load_search_results();
     },
 
 
@@ -1001,6 +1008,8 @@ var Piler =
     {
         Piler.log("[add_shortcuts]");
 
+        $("#button_search").click();
+
         $(document).keypress(function(e){
            if(e.which == 13){
               $("#button_search").click();

webui/view/theme/default/assets/css/metro-bootstrap.css

@@ -1660,7 +1660,7 @@ html,body{height:auto !important;height:100%;min-height:100%;}
 #notesbox{position:absolute;top:8px;right:8px;}
 #folderbox{position:absolute;top:8px;right:380px;}
 #sspinner{display:none;}
-.message_highlight{background:lightblue;}
+.mssghglght{background:lightblue;}
 #searchcontainer{text-align:center;min-width:320px;}
 #searchcontainer label{display:none;height:0;}
 #searchcontainer input{height:26px;width:100%;}

webui/view/theme/default/templates/common/layout-audit-removal.tpl

@@ -60,7 +60,7 @@
       </div>
       <div class="modal-footer">
         <a href="#" class="btn" data-dismiss="modal" aria-hidden="true"><?php print $text_close; ?></a>
-        <a href="#" class="btn btn-primary" id="reject_id2"><?php print $text_delete; ?></a>
+        <a href="#" class="btn btn-primary" id="reject_id2"><?php print $text_reject; ?></a>
       </div>
     </div>
 

webui/view/theme/default/templates/common/layout-search.tpl

@@ -49,6 +49,88 @@
   </div>
 </div>
 
+<div id="advancedsearch-modal" class="modal hide fade">
+  <div class="modal-header">
+    <button type="button" class="close" data-dismiss="modal" role="dialog" aria-hidden="true"><i class="icon-remove"></i></button>
+    <h3><?php print $text_advanced_search; ?></h3>
+  </div>
+  <div class="modal-body">
+
+     <input type="hidden" name="xsearchtype" id="xsearchtype" value="simple" />
+     <input type="hidden" name="xsort" id="xsort" value="date" />
+     <input type="hidden" name="xorder" id="xorder" value="0" />
+     <input type="hidden" name="xref" id="xref" value="" />
+
+    <table class="table">
+      <tr>
+        <td><?php print $text_from; ?></td>
+        <td><input type="text" name="xfrom" id="xfrom" value="" /></td>
+      </tr>
+
+      <tr>
+        <td><?php print $text_to; ?></td>
+        <td><input type="text" name="xto" id="xto" value="<?php if(isset($to)) { print $to; } ?>" /></td>
+      </tr>
+
+      <tr>
+        <td><?php print $text_subject; ?></td>
+        <td><input type="text" name="xsubject" id="xsubject" value="<?php if(isset($subject)) { print $subject; } ?>" /></td>
+      </tr>
+
+      <tr>
+        <td><?php print $text_body; ?></td>
+        <td><input type="text" name="xbody" id="xbody" value="<?php if(isset($body)) { print $body; } ?>" /></td>
+      </tr>
+
+      <tr>
+        <td><?php print $text_tags; ?></td>
+        <td><input type="text" name="xtag" id="xtag" value="<?php if(isset($tag)) { print $tag; } ?>" /></td>
+      </tr>
+
+      <tr>
+        <td><?php print $text_notes; ?></td>
+        <td><input type="text" name="xnote" id="xnote" value="<?php if(isset($note)) { print $note; } ?>" /></td>
+      </tr>
+
+      <tr>
+        <td><?php print $text_attachment; ?></td>
+        <td>
+          <img src="view/theme/default/assets/images/fileicons/doc.png" alt="Word" title="Word" />
+          <input type="checkbox" class="checkbox popup" name="xhas_attachment_doc" id="xhas_attachment_doc" <?php if(isset($has_attachment_doc) && $has_attachment_doc == 1) { ?>checked="checked"<?php } ?> onclick="Piler.clear_attachment_any();" />
+
+          <img src="view/theme/default/assets/images/fileicons/xls.png" alt="Excel" title="Excel" />
+          <input type="checkbox" class="checkbox popup" name="xhas_attachment_xls" id="xhas_attachment_xls" <?php if(isset($has_attachment_xls) && $has_attachment_xls == 1) { ?>checked="checked"<?php } ?> onclick="Piler.clear_attachment_any();" />
+
+          <img src="view/theme/default/assets/images/fileicons/pdf.png" alt="PDF" title="PDF" />
+          <input type="checkbox" class="checkbox popup" name="xhas_attachment_pdf" id="xhas_attachment_pdf" <?php if(isset($has_attachment_pdf) && $has_attachment_pdf == 1) { ?>checked="checked"<?php } ?> onclick="Piler.clear_attachment_any();" />
+
+          <img src="view/theme/default/assets/images/fileicons/image.png" alt="image" title="image" />
+          <input type="checkbox" class="checkbox popup" name="xhas_attachment_image" id="xhas_attachment_image" <?php if(isset($has_attachment_image) && $has_attachment_image == 1) { ?>checked="checked"<?php } ?> onclick="Piler.clear_attachment_any();" />
+
+          <img src="view/theme/default/assets/images/fileicons/file.png" alt="any" title="any" />
+          <input type="checkbox" class="checkbox popup" name="xhas_attachment_any" id="xhas_attachment_any" <?php if(isset($has_attachment_any) && $has_attachment_any == 1) { ?>checked="checked"<?php } ?> onclick="Piler.clear_attachment_others();" />
+        </td>
+      </tr>
+
+      <tr>
+        <td><?php print $text_date_from; ?></td>
+        <td><input type="text" name="date1" id="date1" size="11" value="<?php if(isset($date1)) { print $date1; } ?>" placeholder="<?php print DATE_FORMAT; ?>" /></td>
+      </tr>
+
+      <tr>
+        <td><?php print $text_date_to; ?></td>
+        <td><input type="text" name="date2" id="date2" size="11" value="<?php if(isset($date2)) { print $date2; } ?>" placeholder="<?php print DATE_FORMAT; ?>" /></td>
+      </tr>
+    </table>
+
+  </div>
+
+  <div class="modal-footer">
+    <a href="#" class="btn" data-dismiss="modal" aria-hidden="true"><?php print $text_close; ?></a>
+    <a href="#" class="btn btn-primary" onclick="Piler.complex();" data-dismiss="modal" aria-hidden="true">OK</a>
+  </div>
+</div>
+
 <?php if(Registry::get('auditor_user') == 1) { ?>
 <div id="deletebox-modal" class="modal hide fade">
   <div class="modal-header">
@@ -65,7 +147,7 @@
 </div>
 <?php } ?>
 
-<?php if( (OUTLOOK == 1 && SHOW_MENU_FOR_OUTLOOK == 1) || MOBILE_DEVICE == 0) { ?>
+<?php if( (OUTLOOK == 1 && SHOW_MENU_FOR_OUTLOOK == 1) || (OUTLOOK == 0 && MOBILE_DEVICE == 0) ) { ?>
     <div id="menu">
         <?php print $menu; ?>
     </div>
@@ -92,7 +174,7 @@
              <?php if(MOBILE_DEVICE == 0) { ?>
                 <div class="span6 input-append btn-group">
                     <button id="button_search" class="btn btn-large btn-danger" onclick="Piler.expert(this); return false;"><i class="icon-search icon-large"></i>&nbsp;<?php print $text_search; ?></button>
-                    <button id="button_expert" class="btn btn-large btn-inverse" onclick="$('#searchpopup1').show();"><?php print $text_advanced_search; ?> &nbsp;<span class="caret"></span></button>
+                    <button id="button_expert" class="btn btn-large btn-inverse" onclick="Piler.show_advanced_search_modal();"><?php print $text_advanced_search; ?> &nbsp;<span class="caret"></span></button>
                     <button id="button_options" class="btn btn-large btn-inverse dropdown-toggle" data-toggle="dropdown"><?php print $text_options; ?> &nbsp;<span class="caret"></span></button>
                     <ul class="dropdown-menu">
                         <li><a href="#" onclick="Piler.saved_search_terms('<?php print $text_saved; ?>');"><?php print $text_save; ?></a></li>
@@ -104,7 +186,7 @@
             </div>
          </div>
     </div>
-    <?php print $popup; ?>
+
     <div id="mainscreen">
         <div id="mailleftcontainer">
         <?php if(ENABLE_FOLDER_RESTRICTIONS == 1) { ?>

webui/view/theme/default/templates/search/helper.tpl

@@ -49,9 +49,9 @@
             <td id="c2_r<?php print $i; ?>" class="resultcell id"><?php print ($page*$page_len) + $i + 1; ?></td>
             <td id="c3_r<?php print $i; ?>" class="resultcell date"><?php print $message['date']; ?></td>
             <td id="c4_r<?php print $i; ?>" class="resultcell from"><?php if($message['from'] != $message['shortfrom']) { ?><span title="<?php print $message['from']; ?>"><?php print $message['shortfrom']; ?></span><?php } else { print $message['from']; } ?></td>
-            <td id="c5_r<?php print $i; ?>" class="resultcell to"><?php if($message['to'] != $message['shortto']) { ?><span title="<?php print $message['to']; ?>"><?php print $message['shortto']; ?>&nbsp;<i class=" muted icon-group"></i></span><?php } else { print $message['to']; } ?></td>
+            <td id="c5_r<?php print $i; ?>" class="resultcell to"><?php if(count($message['to']) > 1) { ?><span title="<?php print implode("\n", $message['to']); ?>"><?php print $message['shortto']; ?>&nbsp;<i class="muted icon-group"></i></span><?php } else { print $message['shortto']; } ?></td>
 
-            <td id="c6_r<?php print $i; ?>" class="resultcell subject"><a href="#" <?php if($message['deleted'] == 1) { ?>class="xxx"<?php } ?>><?php print $message['subject']; ?></a><?php if(ENABLE_REFERENCES == 1 && $message['reference']) { ?> <a href="#" <?php if($message['deleted'] == 1) { ?>class="xxx"<?php } ?> title="<?php print $text_conversation_available; ?>" onclick="$('#ref').val('<?php print $message['reference']; ?>'); Piler.expert(this);">[+]</span></a><?php } ?><?php if($message['private'] == 1) { ?> <span class="private">P</span><?php } ?></td>
+            <td id="c6_r<?php print $i; ?>" class="resultcell subject"><a href="#" <?php if($message['deleted'] == 1) { ?>class="xxx"<?php } ?>><?php print $message['subject']; ?></a><?php if(ENABLE_REFERENCES == 1 && $message['reference']) { ?> <a href="#" <?php if($message['deleted'] == 1) { ?>class="xxx"<?php } ?> title="<?php print $text_conversation_available; ?>" onclick="$('#ref').val('<?php print $message['reference']; ?>'); Piler.expert(this);">[+]</span></a><?php } ?><?php if($message['private'] == 1) { ?> <span class="private">P</span><?php } ?> <?php if($message['marked_for_removal'] == 1) { ?> <span class="private">R</span><?php } ?></td>
 
             <td id="c7_r<?php print $i; ?>" class="resultcell size"><?php print $message['size']; ?></td>
             <td id="c8_r<?php print $i; ?>" class="resultcell end"><?php if($message['spam'] == 1) { ?><i class="spam icon-warning-sign icon-large" title="<?php print $text_spam_flag; ?>"></i><?php } else { ?>&nbsp;<?php } ?></td>
@@ -135,7 +135,7 @@
             &nbsp;
 
          <?php if(Registry::get('auditor_user') == 1 && $session->get("sphx_query")) { ?>
-            <span style="margin-left: 30px;"><a href="#" onclick="Piler.show_message('messagebox1', '<?php print $session->get("sphx_query"); ?>', 5);">sphinx</a></span>
+            <span style="margin-left: 30px;"><a href="#" onclick="Piler.show_message('messagebox1', '<?php H($session->get("sphx_query")); ?>', 5);">sphinx</a></span>
          <?php } ?>
 
     <?php } else { print $text_none_found; } ?>