Bumped up version number to 1.4.5

Signed-off-by: Janos SUTO <sj@acts.hu>
piler-smtp refactor
2025-04-06 01:10:32 +02:00 · 2024-04-06 21:02:07 +02:00 · 2024-04-06 20:59:50 +02:00 · 2024-04-06 17:36:56 +02:00 · 2024-04-04 12:57:32 +02:00 · 2024-03-25 13:37:37 +01:00
39 changed files with 793 additions and 324 deletions
--- a/Makefile.in
+++ b/Makefile.in
@ -82,7 +82,7 @@ installdirs: mkinstalldirs
 		$(DESTDIR)$(localstatedir)/piler/stat \
 		$(DESTDIR)$(localstatedir)/piler/tmp \
 		$(DESTDIR)$(localstatedir)/piler/error \
-		$(DESTDIR)$(localstatedir)/piler/sphinx \
+		$(DESTDIR)$(localstatedir)/piler/export \
 		$(DESTDIR)$(localstatedir)/piler/manticore

 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/run/piler
@ -91,7 +91,7 @@ installdirs: mkinstalldirs
 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/stat
 	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/tmp
 	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/error
-	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/sphinx
+	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/export
 	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/manticore


--- a/40
+++ b/40
@ -1,8 +1,44 @@
+1.4.5:
+------
+
+- Introduced 2 new variables in /etc/piler/piler.conf affecting piler-smtp
+
+  ; max message size in bytes
+  ; piler-smtp will reject any message that's bigger than this number
+  max_message_size=50000000
+
+  ; max memory in bytes piler-smtp uses for buffering messages
+  ; when this limit is exceeded, no new emails will be accepted
+  ; until the used memory for all in progress emails decreases
+  ; below this level
+  max_smtp_memory=500000000
+
+  Be sure to adjust these values to your environment!
+
+- Added read-only connection support for manticore
+  If using sphinx, add the following to config-site.php:
+  $config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9306';
+
+- pilerimport supports Zimbra IMAP impersonation
+
+  Generate the following base64 encoded string:
+  (Be sure to use the actual usernames and password):
+
+  pw="$( printf '%s\0%s\0%s' 'username' 'zimbra_admin_username' 'zimbra_admin_password' | base64 )"
+
+  Then specify -u ZIMBRA -p "$pw" for pilerimport, eg.
+
+  pilerimport -i imap.server -u ZIMBRA -p "$pw" ...
+
+  Note that "ZIMBRA" is a special username, it indicates for pilerimport
+  to actually use the imap impersonation for Zimbra.
+
+
 1.4.4:
 ------

-Renamed "group" table to "usergroup"
-Be sure to run util/db-upgrade.sql on the mysql piler database
+- Renamed "group" table to "usergroup"
+  Be sure to run util/db-upgrade.sql on the mysql piler database


 1.4.3:
--- a/2
+++ b/2
@ -1 +1 @@
-1.4.4
+1.4.5
--- a/config.php.in
+++ b/config.php.in
@ -232,6 +232,11 @@ $config['TSA_START_ID'] = 1;
 $config['TSA_STAMP_REQUEST_UNIT_SIZE'] = 10000;
 $config['TSA_VERIFY_CERTIFICATE'] = true;
 $config['TSA_RELAXED_CHECK'] = false;
+$config['TSA_AUTH_USER'] = '';
+$config['TSA_AUTH_PASSWORD'] = '';
+$config['TSA_AUTH_CERT_FILE'] = '';
+$config['TSA_AUTH_KEY_FILE'] = '';
+$config['TSA_AUTH_KEY_PASSWORD'] = '';

 $config['DB_DRIVER'] = 'mysql';
 $config['DB_PREFIX'] = '';
@ -244,6 +249,7 @@ $config['DB_CHARSET'] = 'utf8mb4';
 $config['SPHINX_DRIVER'] = 'sphinx';
 $config['SPHINX_DATABASE'] = 'sphinx';
 $config['SPHINX_HOSTNAME'] = '127.0.0.1:9306';
+$config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9307';
 $config['SPHINX_MAIN_INDEX'] = 'main1,dailydelta1,delta1';
 $config['SPHINX_ATTACHMENT_INDEX'] = 'att1';
 $config['SPHINX_TAG_INDEX'] = 'tag1';
--- a/4
+++ b/4
@ -6426,13 +6426,13 @@ fi

 echo
 echo
-echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/wiki/current:upgrade"
+echo "IMPORTANT! If you upgrade, be sure to read https://www.mailpiler.org/upgrade-instructions/"
 echo
 echo

 echo
 echo "Did you know that piler has an enterprise edition as well?"
-echo "Check out what it can do for you at https://mailpiler.com/piler-enterprise-email-archiver-features/"
+echo "Check out what it can do for you at https://mailpiler.com/#features"
 echo
 echo

--- a/configure.in
+++ b/configure.in
@ -537,12 +537,12 @@ AC_OUTPUT

 echo
 echo
-echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/wiki/current:upgrade"
+echo "IMPORTANT! If you upgrade, be sure to read https://www.mailpiler.org/upgrade-instructions/"
 echo
 echo

 echo
 echo "Did you know that piler has an enterprise edition as well?"
-echo "Check out what it can do for you at https://mailpiler.com/piler-enterprise-email-archiver-features/"
+echo "Check out what it can do for you at https://mailpiler.com/#features"
 echo
 echo
--- a/contrib/installer/jammy.sh
+++ b/contrib/installer/jammy.sh
@ -25,6 +25,9 @@ PILER_CONF="/etc/piler/piler.conf"
 CONFIG_SITE_PHP="/etc/piler/config-site.php"
 SEARCHCFG="/etc/piler/manticore.conf"

+TRAEFIK_VERSION="v2.11.0"
+ARCH="amd64"
+
 export DEBIAN_FRONTEND=noninteractive

 install_prerequisites() {
@ -37,8 +40,11 @@ install_prerequisites() {
      python3-mysqldb libmariadb-dev mariadb-client-core-10.6 \
      mariadb-server-10.6

-   wget -O /usr/local/bin/traefik "${DOWNLOAD_URL}/generic-local/traefik"
-   chmod +x /usr/local/bin/traefik
+   wget "https://github.com/traefik/traefik/releases/download/${TRAEFIK_VERSION}/traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz"
+   tar zxvf "traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz" traefik
+   chown root:root traefik
+   chmod 755 traefik
+   mv traefik /usr/local/bin
   setcap cap_net_bind_service+ep /usr/local/bin/traefik

   wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -17,7 +17,7 @@ RUN apt-get update && \
    apt-get -y --no-install-recommends install \
       wget openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql php8.1-fpm php8.1-zip php8.1-ldap \
       php8.1-gd php8.1-curl php8.1-xml php8.1-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip4 \
-       libtre5 cron libmariadb-dev mariadb-client-core-10.6 python3 python3-mysqldb ca-certificates curl && \
+       libtre5 cron libmariadb-dev mariadb-client-core-10.6 python3 python3-mysqldb ca-certificates curl rsyslog && \
    wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
    dpkg -i manticore-repo.noarch.deb && \
    rm -f manticore-repo.noarch.deb && \
--- a/docker/docker-compose.yaml
+++ b/docker/docker-compose.yaml
@ -1,8 +1,7 @@
 version: "3"
 services:
-
  mysql:
-    image: mariadb:10.6
+    image: mariadb:11.1.2
    container_name: mysql
    restart: unless-stopped
    cap_drop:
@ -17,43 +16,26 @@ services:
      - MYSQL_PASSWORD=piler123
      - MYSQL_RANDOM_ROOT_PASSWORD=yes
    command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
-    healthcheck:
-      test: mysql --user=piler --password=piler123 piler --execute "show tables"
-      interval: "60s"
-      timeout: "5s"
-      start_period: "15s"
-      retries: 3
    volumes:
      - db_data:/var/lib/mysql
-
-  memcached:
-    container_name: memcached
-    image: memcached:latest
-    restart: unless-stopped
-    cap_drop:
-      - ALL
-    command: -m 64
-
  piler:
    image: sutoj/piler:1.4.4
    container_name: piler
    init: true
    environment:
+      - MYSQL_HOSTNAME=mysql
      - MYSQL_DATABASE=piler
      - MYSQL_USER=piler
      - MYSQL_PASSWORD=piler123
-      - MYSQL_HOSTNAME=mysql
-      - PILER_HOSTNAME=archive.example.com
-      - MEMCACHED_HOST=memcached
+      - PILER_HOSTNAME=cust1.acts.hu
+      - RT=1
    ports:
      - "25:25"
      - "80:80"
-      - "443:443"
    volumes:
      - piler_etc:/etc/piler
-      - piler_var_store:/var/piler/store
-      - piler_var_manticore:/var/piler/manticore
-      - /dev/log:/dev/log
+      - piler_manticore:/var/piler/manticore
+      - piler_store:/var/piler/store
    healthcheck:
      test: curl -s smtp://localhost/
      interval: "60s"
@ -68,11 +50,10 @@ services:
          memory: 512M

    depends_on:
-      - "memcached"
      - "mysql"

 volumes:
  db_data: {}
  piler_etc: {}
-  piler_var_store: {}
-  piler_var_manticore: {}
+  piler_manticore: {}
+  piler_store: {}
--- a/docker/start.sh
+++ b/docker/start.sh
@ -13,7 +13,7 @@ PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf"
 SPHINX_CONF="${CONFIG_DIR}/manticore.conf"
 CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php"
 PILER_MY_CNF="${CONFIG_DIR}/.my.cnf"
-
+RT="${RT:-0}"

 error() {
   echo "ERROR:" "$*" 1>&2
@ -126,6 +126,21 @@ fix_configs() {
       -e "s%MYSQL_USERNAME%${MYSQL_USER}%" \
       -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" \
       -i "$SPHINX_CONF"
+
+   # Fixes for RT index
+
+   if [[ $RT -eq 1 ]]; then
+      sed -i "s/define('RT', 0)/define('RT', 1)/" "$SPHINX_CONF"
+      if ! grep "'RT'" "$CONFIG_SITE_PHP"; then
+         echo "\$config['RT'] = 1;" >> "$CONFIG_SITE_PHP"
+      fi
+
+      if ! grep "'SPHINX_MAIN_INDEX'" "$CONFIG_SITE_PHP"; then
+         echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
+      fi
+
+      sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
+   fi
 }


@ -173,11 +188,12 @@ start_services() {
   service cron start
   service php8.1-fpm start
   service nginx start
+   rsyslogd
 }


 start_piler() {
-   if [[ ! -f "${VOLUME_DIR}/manticore/main1.spp" ]]; then
+   if [[ $RT -eq 0 && ! -f "${VOLUME_DIR}/manticore/main1.spp" ]]; then
      log "main1.spp does not exist, creating index files"
      su -c "indexer --all --config ${SPHINX_CONF}" "$PILER_USER"
   fi
--- a/etc/example.conf
+++ b/etc/example.conf
@ -263,3 +263,12 @@ archive_address=
 ; rules. In other words if you decide to use the acl file, then
 ; everyone is not explicitly permitted is denied.
 smtp_access_list=0
+
+; max message size in bytes
+; piler-smtp will reject any message that's bigger than this number
+max_message_size=50000000
+
+; max memory in bytes piler-smtp uses for buffering messages
+; when this limit is exceeded, no new emails will be accepted
+; until the used memory decreases below this level
+max_smtp_memory=500000000
--- a/etc/manticore.conf.in
+++ b/etc/manticore.conf.in
@ -248,7 +248,8 @@ index note1
 searchd
 {
        listen                  = 127.0.0.1:9312
-        listen                  = 127.0.0.1:9306:mysql41
+        listen                  = 127.0.0.1:9306:mysql
+        listen                  = 127.0.0.1:9307:mysql_readonly
        log                     = /var/piler/manticore/manticore.log
        binlog_max_log_size     = 256M
        binlog_path             = /var/piler/manticore
--- a/src/archive.c
+++ b/src/archive.c
@ -192,6 +192,7 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
      } else {
      #if OPENSSL_VERSION_MAJOR >= 3
         OSSL_PROVIDER_load(NULL, "legacy");
+         OSSL_PROVIDER_load(NULL, "default");
      #endif
         rc = EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
      }
--- a/src/bdat.c
+++ b/src/bdat.c
@ -26,21 +26,21 @@ void reset_bdat_counters(struct smtp_session *session){
 }


-void get_bdat_size_to_read(struct smtp_session *session, char *buf){
+void get_bdat_size_to_read(struct smtp_session *session){
   char *p;

   session->bdat_bytes_to_read = 0;

   session->protocol_state = SMTP_STATE_BDAT;

-   p = strcasestr(buf, " LAST");
+   p = strcasestr(session->buf, " LAST");
   if(p){
      *p = '\0';
   }

   // determine the size to be read

-   p = strchr(buf, ' ');
+   p = strchr(session->buf, ' ');
   if(p){
      session->bdat_bytes_to_read = atoi(p);
      if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "fd=%d: BDAT len=%d", session->net.socket, session->bdat_bytes_to_read);
--- a/src/cfg.c
+++ b/src/cfg.c
@ -23,6 +23,11 @@ int int_parser(char *src, int *target){
   return 0;
 };

+int uint64_parser(char *src, uint64 *target){
+   *target = strtoull(src, (char**)NULL, 10);
+   return 0;
+}
+
 struct _parse_rule {
   char *name;
   char *type;
@ -63,7 +68,9 @@ struct _parse_rule config_parse_rules[] =
   { "listen_port", "integer", (void*) int_parser, offsetof(struct config, listen_port), "25", sizeof(int)},
   { "locale", "string", (void*) string_parser, offsetof(struct config, locale), "", MAXVAL-1},
   { "max_connections", "integer", (void*) int_parser, offsetof(struct config, max_connections), "64", sizeof(int)},
+   { "max_message_size", "integer", (void*) int_parser, offsetof(struct config, max_message_size), "50000000", sizeof(int)},
   { "max_requests_per_child", "integer", (void*) int_parser, offsetof(struct config, max_requests_per_child), "10000", sizeof(int)},
+   { "max_smtp_memory", "uint64", (void*) uint64_parser, offsetof(struct config, max_smtp_memory), "500000000", sizeof(uint64)},
   { "memcached_servers", "string", (void*) string_parser, offsetof(struct config, memcached_servers), "127.0.0.1", MAXVAL-1},
   { "memcached_to_db_interval", "integer", (void*) int_parser, offsetof(struct config, memcached_to_db_interval), "900", sizeof(int)},
   { "memcached_ttl", "integer", (void*) int_parser, offsetof(struct config, memcached_ttl), "86400", sizeof(int)},
@ -217,6 +224,7 @@ struct config read_config(char *configfile){
 void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
   int j;
   float f;
+   uint64 u;
   char *p, buf[MAXVAL];

   p = (char*)cfg + rules[i].offset;
@ -225,6 +233,10 @@ void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
      memcpy((char*)&j, p, sizeof(int));
      printf("%s=%d\n", rules[i].name, j);
   }
+   else if(strcmp(rules[i].type, "uint64") == 0){
+      memcpy((char*)&u, p, sizeof(uint64));
+      printf("%s=%llu\n", rules[i].name, u);
+   }
   else if(strcmp(rules[i].type, "float") == 0){
      memcpy((char*)&f, p, sizeof(float));
      printf("%s=%.4f\n", rules[i].name, f);
--- a/src/cfg.h
+++ b/src/cfg.h
@ -110,6 +110,9 @@ struct config {
   int debug;

   int smtp_access_list;
+
+   int max_message_size;
+   uint64 max_smtp_memory;
 };


--- a/src/config.h
+++ b/src/config.h
@ -9,6 +9,8 @@
 #include "piler-config.h"
 #include "params.h"

+typedef unsigned long long uint64;
+
 #define BUILD 1001

 #define HOSTID "mailarchiver"
@ -30,6 +32,7 @@
 #define SMALLBUFSIZE 512
 #define BIGBUFSIZE 131072
 #define REALLYBIGBUFSIZE 524288
+#define SMTPBUFSIZE 2048000
 #define TINYBUFSIZE 128
 #define MAXVAL 256
 #define RANDOM_POOL "/dev/urandom"
--- a/src/defs.h
+++ b/src/defs.h
@ -403,7 +403,6 @@ struct smtp_session {
   char ttmpfile[SMALLBUFSIZE];
   char mailfrom[SMALLBUFSIZE];
   char rcptto[MAX_RCPT_TO][SMALLBUFSIZE];
-   char buf[MAXBUFSIZE];
   char remote_host[INET6_ADDRSTRLEN+1];
   char nullbyte;
   time_t lasttime;
@ -411,13 +410,17 @@ struct smtp_session {
   int slot;
   int fd;
   int bad;
-   int buflen;
-   int last_data_char;
   int tot_len;
   int bdat_bytes_to_read;
   int num_of_rcpt_to;
   struct config *cfg;
   struct net net;
+   int max_message_size;
+   char *buf;
+   int buflen;
+   int bufsize;
+   int too_big;
+   int mail_size;
 };

 struct tls_protocol {
--- a/src/imap.c
+++ b/src/imap.c
@ -105,7 +105,12 @@ int connect_to_imap_server(struct data *data){

   /* imap cmd: LOGIN */

+   if(strcmp(data->import->username, "ZIMBRA") == 0){
+      snprintf(buf, sizeof(buf)-1, "A%d AUTHENTICATE PLAIN %s\r\n", data->import->seq, data->import->password);
+   }
+   else {
      snprintf(buf, sizeof(buf)-1, "A%d LOGIN %s \"%s\"\r\n", data->import->seq, data->import->username, data->import->password);
+   }

   write1(data->net, buf, strlen(buf));
   if(read_response(buf, sizeof(buf), data) == 0){
--- a/src/message.c
+++ b/src/message.c
@ -50,7 +50,7 @@ int store_index_data(struct session_data *sdata, struct parser_state *state, str
      char a[4*MAXBUFSIZE+4*SMALLBUFSIZE];
      char *query=NULL;

-      snprintf(a, sizeof(a)-1, "INSERT INTO %s (id, arrived, sent, size, direction, folder, attachments, attachment_types, sender, rcpt, senderdomain, rcptdomain, subject, body) VALUES (%llu,%ld,%ld,%d,%d,%d,%d,'%s','", cfg->sphxdb, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, data->folder, state->n_attachments, sdata->attachments);
+      snprintf(a, sizeof(a)-1, "REPLACE INTO %s (id, arrived, sent, size, direction, folder, attachments, attachment_types, sender, rcpt, senderdomain, rcptdomain, subject, body) VALUES (%llu,%ld,%ld,%d,%d,%d,%d,'%s','", cfg->sphxdb, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, data->folder, state->n_attachments, sdata->attachments);

      int ret = append_string_to_buffer(&query, a);

--- a/src/misc.c
+++ b/src/misc.c
@ -781,3 +781,29 @@ int append_string_to_buffer(char **buffer, char *str){

   return 0;
 }
+
+
+int get_size_from_smtp_mail_from(char *s){
+   int size=0;
+   char *p;
+
+   p = strcasestr(s, "SIZE=");
+   if(p){
+      p += strlen("SIZE=");
+      char *q = p;
+      for(; *q; q++){
+         if(isspace(*q)) break;
+      }
+
+      // We extract max. 9 characters, which is just under 1GB
+      // and not overflowing an int variable
+      if(q - p <= 9){
+         char c = *q;
+         *q = '\0';
+         size = atoi(p);
+         *q = c;
+      }
+   }
+
+   return size;
+}
--- a/src/misc.h
+++ b/src/misc.h
@ -55,5 +55,6 @@ int init_ssl_to_server(struct data *data);
 #endif

 int append_string_to_buffer(char **buffer, char *str);
+int get_size_from_smtp_mail_from(char *s);

 #endif /* _MISC_H */
--- a/src/piler-smtp.c
+++ b/src/piler-smtp.c
@ -33,6 +33,7 @@ extern int optind;
 struct epoll_event event, *events=NULL;
 int num_connections = 0;
 int listenerfd = -1;
+int loglevel = 1;

 char *configfile = CONFIG_FILE;
 struct config cfg;
@ -48,6 +49,7 @@ void usage(){
   printf("    -d                                Fork to the background\n");
   printf("    -v                                Return the version and build number\n");
   printf("    -V                                Return the version and some build parameters\n");
+   printf("    -L <log level>                    Set the log level: 1-5\n");

   exit(0);
 }
@ -138,16 +140,20 @@ int main(int argc, char **argv){
   int client_len = sizeof(struct sockaddr_storage);
   ssize_t readlen;
   struct sockaddr_storage client_address;
-   char readbuf[BIGBUFSIZE];
+   char readbuf[REALLYBIGBUFSIZE];
   int efd;

-   while((i = getopt(argc, argv, "c:dvVh")) > 0){
+   while((i = getopt(argc, argv, "c:L:dvVh")) > 0){
      switch(i){

        case 'c' :
                   configfile = optarg;
                   break;

+        case 'L':
+                   loglevel = atoi(optarg);
+                   break;
+
        case 'd' :
                   daemonise = 1;
                   break;
@ -321,6 +327,7 @@ int main(int argc, char **argv){
                  break;
               }

+               readbuf[readlen] = '\0';
               handle_data(session, &readbuf[0], readlen, &cfg);

               if(session->protocol_state == SMTP_STATE_BDAT && session->bad == 1){
--- a/src/session.c
+++ b/src/session.c
@ -9,6 +9,36 @@
 int get_session_slot(struct smtp_session **sessions, int max_connections);
 void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg);

+#define GOT_CRLF_DOT_CRLF(p) *p == '\r' && *(p+1) == '\n' && *(p+2) == '.' && *(p+3) == '\r' && *(p+4) == '\n' ? 1 : 0
+
+uint64 get_sessions_total_memory(struct smtp_session **sessions, int max_connections){
+   uint64 total = 0;
+
+   for(int i=0; i<max_connections; i++){
+      if(sessions[i]) total += sessions[i]->bufsize;
+   }
+
+   return total;
+}
+
+
+/*
+ * If the sending party sets the email size when it sends the "mail from"
+ * part in the smtp transaction, eg. MAIL FROM:<jajaja@akakak.lo> size=509603
+ * then piler-smtp could know the email size in advance and could do
+ * a better estimate on the allowed number of smtp sessions.
+ */
+
+uint64 get_sessions_total_expected_mail_size(struct smtp_session **sessions, int max_connections){
+   uint64 total = 0;
+
+   for(int i=0; i<max_connections; i++){
+      if(sessions[i]) total += sessions[i]->mail_size;
+   }
+
+   return total;
+}
+

 int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg){
   int slot;
@ -31,6 +61,21 @@ int start_new_session(struct smtp_session **sessions, int socket, int *num_conne
      return -1;
   }

+   /*
+    * We are under the max_smtp_memory threshold
+    */
+
+   uint64 expected_total_mail_size = get_sessions_total_expected_mail_size(sessions, cfg->max_connections);
+   uint64 total_memory = get_sessions_total_memory(sessions, cfg->max_connections);
+
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "DEBUG: total smtp memory allocated: %llu, expected total size: %llu", total_memory, expected_total_mail_size);
+
+   if(total_memory > cfg->max_smtp_memory || expected_total_mail_size > cfg->max_smtp_memory){
+      syslog(LOG_PRIORITY, "ERROR: too much memory consumption: %llu", total_memory);
+      send(socket, SMTP_RESP_451_ERR_TOO_MANY_REQUESTS, strlen(SMTP_RESP_451_ERR_TOO_MANY_REQUESTS), 0);
+      return -1;
+   }
+
   slot = get_session_slot(sessions, cfg->max_connections);

   if(slot >= 0 && sessions[slot] == NULL){
@ -84,11 +129,8 @@ struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int m


 void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg){
-   int i;
-
   session->slot = slot;

-   session->buflen = 0;
   session->protocol_state = SMTP_STATE_INIT;

   session->cfg = cfg;
@ -100,27 +142,24 @@ void init_smtp_session(struct smtp_session *session, int slot, int sd, char *cli
   session->net.ssl = NULL;

   session->nullbyte = 0;
-   session->last_data_char = 0;

-   session->fd = -1;
-
-   memset(session->mailfrom, 0, SMALLBUFSIZE);
-
-   session->num_of_rcpt_to = 0;
-   for(i=0; i<MAX_RCPT_TO; i++) memset(session->rcptto[i], 0, SMALLBUFSIZE);
-
-   memset(session->buf, 0, MAXBUFSIZE);
   snprintf(session->remote_host, sizeof(session->remote_host)-1, "%s", client_addr);

-   reset_bdat_counters(session);
+   session->buf = NULL;
+   session->buflen = 0;
+   session->bufsize = 0;

-   time(&(session->lasttime));
+   reset_smtp_session(session);
 }


 void free_smtp_session(struct smtp_session *session){
   if(session){

+      if(session->buf != NULL){
+         free(session->buf);
+      }
+
      if(session->net.use_ssl == 1){
         SSL_shutdown(session->net.ssl);
         SSL_free(session->net.ssl);
@ -160,71 +199,126 @@ void tear_down_session(struct smtp_session **sessions, int slot, int *num_connec
 }


+inline int get_last_newline_position(char *buf, int buflen){
+   int i;
+
+   for(i=buflen; i>0; i--){
+      if(*(buf+i) == '\n'){
+         i++;
+         break;
+      }
+   }
+
+   return i;
+}
+
+
+void flush_buffer(struct smtp_session *session){
+   // In the DATA phase skip the 1st character if it's a dot (.)
+   // and there are more characters before the trailing CR-LF
+   //
+   // See https://www.ietf.org/rfc/rfc5321.html#section-4.5.2 for more
+   for(int i=0; i<session->buflen; i++){
+      if(*(session->buf+i) == '\n' && *(session->buf+i+1) == '.' && *(session->buf+i+2) == '.'){
+         int dst = i + 2;
+         int src = dst + 1;
+         int l = session->buflen - src;
+         memmove(session->buf + dst, session->buf + src, l);
+         session->buflen -= 1;
+      }
+   }
+
+   // Exclude the trailing \r\n.\r\n sequence
+
+   session->buflen -= 5;
+
+   if(write(session->fd, session->buf, session->buflen) != session->buflen){
+      session->bad = 1;
+      syslog(LOG_PRIORITY, "ERROR (line: %d) %s: failed to write %d bytes", __LINE__, __func__, session->buflen);
+   }
+
+   session->tot_len = session->buflen;
+}
+
+
 void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg){
-   int puflen, rc, nullbyte;
-   char *p, copybuf[BIGBUFSIZE+MAXBUFSIZE], puf[MAXBUFSIZE];
-
-   // if there's something in the saved buffer, then let's merge them
-
-   int remaininglen = readlen + session->buflen;
-
-   if(session->buflen > 0){
-      memset(copybuf, 0, sizeof(copybuf));
-
-      memcpy(copybuf, session->buf, session->buflen);
-      memcpy(&copybuf[session->buflen], readbuf, readlen);
-
-      session->buflen = 0;
-      memset(session->buf, 0, MAXBUFSIZE);
-
-      p = &copybuf[0];
-   }
-   else {
-      readbuf[readlen] = 0;
-      p = readbuf;
-   }
-
-
-   do {
-      puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf)-1, &rc, &nullbyte);
-      p += puflen;
-      remaininglen -= puflen;
-
-      if(nullbyte){
-         session->nullbyte = 1;
-      }
-
-      // complete line: rc == OK and puflen > 0
-      // incomplete line with something in the buffer: rc == ERR and puflen > 0
-
-      if(puflen > 0){
-         // Update lasttime if we have a line to process
+   // Update lasttime if we have something to process
   time(&(session->lasttime));

-         // Save incomplete line to buffer
-         if(rc == ERR){
-            memcpy(session->buf, puf, puflen);
-            session->buflen = puflen;
-         }
-
-         // We have a complete line to process
-
-         if(rc == OK){
   if(session->protocol_state == SMTP_STATE_BDAT){
-               process_bdat(session, puf, puflen, cfg);
+      process_bdat(session, readbuf, readlen, cfg);
+      return;
   }
-            else if(session->protocol_state == SMTP_STATE_DATA){
-               sig_block(SIGALRM);
-               process_data(session, puf, puflen);
-               sig_unblock(SIGALRM);
+
+   // realloc memory if the new chunk doesn't fit in
+
+   if(session->buflen + readlen + 10 > session->bufsize){
+      // Handle if the current memory allocation for this email is above the max_message_size threshold
+
+      if(session->buflen > cfg->max_message_size){
+         if(session->too_big == 0) syslog(LOG_PRIORITY, "ERROR: too big email: %d vs %d", session->buflen, cfg->max_message_size);
+         session->bad = 1;
+         session->too_big = 1;
      }
-            else {
-               process_smtp_command(session, puf, cfg);
+
+      if(session->bad == 0){
+         char *q = realloc(session->buf, session->bufsize + SMTPBUFSIZE);
+         if(q){
+            session->buf = q;
+            memset(session->buf+session->bufsize, 0, SMTPBUFSIZE);
+            session->bufsize += SMTPBUFSIZE;
+         } else {
+            syslog(LOG_PRIORITY, "ERROR: realloc %s %s %d", session->ttmpfile, __func__, __LINE__);
+            session->bad = 1;
         }
      }
   }

-   } while(puflen > 0);
+   // process smtp command
+   if(session->protocol_state != SMTP_STATE_DATA){
+
+      // We got ~2 MB of garbage and no valid smtp command
+      // Terminate the connection
+      if(session->buflen + readlen > SMTPBUFSIZE - 10){
+         session->bad = 1;
+      }
+
+      // We are at the beginning of the smtp transaction
+      if(session->bad == 1){
+         write1(&(session->net), SMTP_RESP_451_ERR, strlen(SMTP_RESP_451_ERR));
+         syslog(LOG_PRIORITY, "ERROR: sent 451 temp error back to client %s", session->ttmpfile);
+         return;
+      }
+
+      //printf("got %d *%s*\n", readlen, readbuf);
+
+      memcpy(session->buf + session->buflen, readbuf, readlen);
+      session->buflen += readlen;
+
+      int pos = get_last_newline_position(session->buf, session->buflen);
+
+      if(pos < readlen) return; // no complete command
+
+      process_smtp_command(session, cfg);
+
+      memset(session->buf, 0, session->bufsize);
+      session->buflen = 0;
+
+      return;
+   }
+
+   if(session->bad == 0){
+      memcpy(session->buf + session->buflen, readbuf, readlen);
+      session->buflen += readlen;
+
+      char *p = session->buf + session->buflen - 5;
+      if(session->buflen >= 5 && GOT_CRLF_DOT_CRLF(p)){
+         flush_buffer(session);
+         process_command_period(session);
+      }
+   } else if(strstr(readbuf, "\r\n.\r\n")){
+      process_command_period(session);
+   }

 }

--- a/src/smtp.c
+++ b/src/smtp.c
@ -15,59 +15,59 @@
 #include "smtp.h"


-void process_smtp_command(struct smtp_session *session, char *buf, struct config *cfg){
+void process_smtp_command(struct smtp_session *session, struct config *cfg){
   char response[SMALLBUFSIZE];

-   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "got on fd=%d: *%s*", session->net.socket, buf);
+   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "got on fd=%d: *%s*", session->net.socket, session->buf);

-   if(strncasecmp(buf, SMTP_CMD_HELO, strlen(SMTP_CMD_HELO)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_HELO, strlen(SMTP_CMD_HELO)) == 0){
      process_command_helo(session, response, sizeof(response));
      return;
   }

-   if(strncasecmp(buf, SMTP_CMD_EHLO, strlen(SMTP_CMD_EHLO)) == 0 ||
-         strncasecmp(buf, LMTP_CMD_LHLO, strlen(LMTP_CMD_LHLO)) == 0){
-      process_command_ehlo_lhlo(session, response, sizeof(response));
+   if(strncasecmp(session->buf, SMTP_CMD_EHLO, strlen(SMTP_CMD_EHLO)) == 0 ||
+         strncasecmp(session->buf, LMTP_CMD_LHLO, strlen(LMTP_CMD_LHLO)) == 0){
+      process_command_ehlo_lhlo(session, response, sizeof(response), cfg);
      return;
   }

-   if(strncasecmp(buf, SMTP_CMD_HELP, strlen(SMTP_CMD_HELP)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_HELP, strlen(SMTP_CMD_HELP)) == 0){
      send_smtp_response(session, SMTP_RESP_221_PILER_SMTP_OK);
      return;
   }

-   if(strncasecmp(buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){
-      process_command_mail_from(session, buf);
+   if(strncasecmp(session->buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){
+      process_command_mail_from(session);
      return;
   }

-   if(strncasecmp(buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){
-      process_command_rcpt_to(session, buf, cfg);
+   if(strncasecmp(session->buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){
+      process_command_rcpt_to(session, cfg);
      return;
   }

-   if(strncasecmp(buf, SMTP_CMD_DATA, strlen(SMTP_CMD_DATA)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_DATA, strlen(SMTP_CMD_DATA)) == 0){
      process_command_data(session, cfg);
      return;
   }

   /* Support only BDAT xxxx LAST command */
-   if(session->cfg->enable_chunking == 1 && strncasecmp(buf, SMTP_CMD_BDAT, strlen(SMTP_CMD_BDAT)) == 0 && strcasestr(buf, "LAST")){
-      get_bdat_size_to_read(session, buf);
+   if(session->cfg->enable_chunking == 1 && strncasecmp(session->buf, SMTP_CMD_BDAT, strlen(SMTP_CMD_BDAT)) == 0 && strcasestr(session->buf, "LAST")){
+      get_bdat_size_to_read(session);
      return;
   }

-   if(strncasecmp(buf, SMTP_CMD_QUIT, strlen(SMTP_CMD_QUIT)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_QUIT, strlen(SMTP_CMD_QUIT)) == 0){
      process_command_quit(session, response, sizeof(response));
      return;
   }

-   if(strncasecmp(buf, SMTP_CMD_RESET, strlen(SMTP_CMD_RESET)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_RESET, strlen(SMTP_CMD_RESET)) == 0){
      process_command_reset(session);
      return;
   }

-   if(session->cfg->tls_enable == 1 && strncasecmp(buf, SMTP_CMD_STARTTLS, strlen(SMTP_CMD_STARTTLS)) == 0 && session->net.use_ssl == 0){
+   if(session->cfg->tls_enable == 1 && strncasecmp(session->buf, SMTP_CMD_STARTTLS, strlen(SMTP_CMD_STARTTLS)) == 0 && session->net.use_ssl == 0){
      process_command_starttls(session);
      return;
   }
@ -76,41 +76,6 @@ void process_smtp_command(struct smtp_session *session, char *buf, struct config
 }


-void process_data(struct smtp_session *session, char *buf, int buflen){
-   if(session->last_data_char == '\n' && strcmp(buf, ".\r\n") == 0){
-      process_command_period(session);
-   }
-   else {
-      // write line to file
-      int written=0, n_writes=0;
-
-      // In the DATA phase skip the 1st character if it's a dot (.)
-      // and there are more characters before the trailing CR-LF
-      //
-      // See https://www.ietf.org/rfc/rfc5321.html#section-4.5.2 for more.
-
-      int dotstuff = 0;
-      if(*buf == '.' && buflen > 1 && *(buf+1) != '\r' && *(buf+1) != '\n') dotstuff = 1;
-
-      while(written < buflen) {
-         int len = write(session->fd, buf+dotstuff+written, buflen-dotstuff-written);
-
-         n_writes++;
-
-         if(len > 0){
-            if(len != buflen-dotstuff) syslog(LOG_PRIORITY, "WARN: partial write: %d/%d bytes (round: %d)", len, buflen-dotstuff, n_writes);
-            written += len + dotstuff;
-            session->tot_len += len;
-            dotstuff = 0;
-         }
-         else syslog(LOG_PRIORITY, "ERROR (line: %d) process_data(): written %d bytes", __LINE__, len);
-      }
-   }
-
-   session->last_data_char = buf[buflen-1];
-}
-
-
 void wait_for_ssl_accept(struct smtp_session *session){
   int rc;

@ -152,7 +117,7 @@ void process_command_helo(struct smtp_session *session, char *buf, int buflen){
 }


-void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen){
+void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen, struct config *cfg){
   char extensions[SMALLBUFSIZE];

   memset(extensions, 0, sizeof(extensions));
@ -163,7 +128,9 @@ void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int bufl
   if(session->net.use_ssl == 0 && session->cfg->tls_enable == 1) snprintf(extensions, sizeof(extensions)-1, "%s", SMTP_EXTENSION_STARTTLS);
   if(session->cfg->enable_chunking == 1) strncat(extensions, SMTP_EXTENSION_CHUNKING, sizeof(extensions)-strlen(extensions)-2);

-   snprintf(buf, buflen-1, SMTP_RESP_250_EXTENSIONS, session->cfg->hostid, extensions);
+   //#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n%s250-SIZE %d\r\n250 8BITMIME\r\n"
+
+   snprintf(buf, buflen-1, SMTP_RESP_250_EXTENSIONS, session->cfg->hostid, cfg->max_message_size, extensions);

   send_smtp_response(session, buf);
 }
@ -234,26 +201,29 @@ void process_command_starttls(struct smtp_session *session){
 }


-void process_command_mail_from(struct smtp_session *session, char *buf){
+void process_command_mail_from(struct smtp_session *session){
+   memset(session->mailfrom, 0, SMALLBUFSIZE);
+
   if(session->protocol_state != SMTP_STATE_HELO && session->protocol_state != SMTP_STATE_PERIOD && session->protocol_state != SMTP_STATE_BDAT){
      send(session->net.socket, SMTP_RESP_503_ERR, strlen(SMTP_RESP_503_ERR), 0);
   }
   else {
-      memset(&(session->ttmpfile[0]), 0, SMALLBUFSIZE);
-      make_random_string((unsigned char*)&(session->ttmpfile[0]), QUEUE_ID_LEN);
      session->protocol_state = SMTP_STATE_MAIL_FROM;

-      extractEmail(buf, session->mailfrom);
+      extractEmail(session->buf, session->mailfrom);

-      reset_bdat_counters(session);
-      session->tot_len = 0;
+      int mailsize = get_size_from_smtp_mail_from(session->buf);
+
+      reset_smtp_session(session);
+
+      session->mail_size = mailsize;

      send_smtp_response(session, SMTP_RESP_250_OK);
   }
 }


-void process_command_rcpt_to(struct smtp_session *session, char *buf, struct config *cfg){
+void process_command_rcpt_to(struct smtp_session *session, struct config *cfg){

   if(session->protocol_state == SMTP_STATE_MAIL_FROM || session->protocol_state == SMTP_STATE_RCPT_TO){

@ -262,7 +232,7 @@ void process_command_rcpt_to(struct smtp_session *session, char *buf, struct con
      session->protocol_state = SMTP_STATE_RCPT_TO;

      if(session->num_of_rcpt_to < MAX_RCPT_TO){
-         extractEmail(buf, session->rcptto[session->num_of_rcpt_to]);
+         extractEmail(session->buf, session->rcptto[session->num_of_rcpt_to]);

         // Check if we should accept archive_address only
         if(cfg->archive_address[0] && !strstr(cfg->archive_address, session->rcptto[session->num_of_rcpt_to])){
@ -322,13 +292,23 @@ void process_command_period(struct smtp_session *session){

   syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d, client=%s, fd=%d, fsync=%ld", session->ttmpfile, session->mailfrom, session->tot_len, session->remote_host, session->net.socket, tvdiff(tv2, tv1));

+   if(session->bad == 1 || session->too_big == 1){
+      snprintf(buf, sizeof(buf)-1, SMTP_RESP_451_ERR);
+      unlink(session->ttmpfile);
+      syslog(LOG_PRIORITY, "ERROR: problem in processing, removing %s", session->ttmpfile);
+      if(session->too_big)
+         snprintf(buf, sizeof(buf)-1, "%s", SMTP_RESP_552_ERR_TOO_BIG_EMAIL);
+      else
+         snprintf(buf, sizeof(buf)-1, "%s", SMTP_RESP_451_ERR);
+   } else {
      move_email(session);
-
      snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
+   }

+   if(session->buf){
+      memset(session->buf, 0, session->bufsize);
      session->buflen = 0;
-   session->last_data_char = 0;
-   memset(session->buf, 0, sizeof(session->buf));
+   }

   send_smtp_response(session, buf);
 }
@ -344,15 +324,32 @@ void process_command_quit(struct smtp_session *session, char *buf, int buflen){


 void process_command_reset(struct smtp_session *session){
+   reset_smtp_session(session);
   send_smtp_response(session, SMTP_RESP_250_OK);
-
-   session->tot_len = 0;
-   session->fd = -1;
   session->protocol_state = SMTP_STATE_HELO;
-   session->last_data_char = 0;
+}
+
+
+void reset_smtp_session(struct smtp_session *session){
+   session->tot_len = 0;
+   session->mail_size = 0;
+   session->too_big = 0;
+   session->bad = 0;
+
+   session->fd = -1;
+
+   session->num_of_rcpt_to = 0;
+   for(int i=0; i<MAX_RCPT_TO; i++) memset(session->rcptto[i], 0, SMALLBUFSIZE);

   reset_bdat_counters(session);

+   time(&(session->lasttime));
+
   memset(&(session->ttmpfile[0]), 0, SMALLBUFSIZE);
-   make_random_string((unsigned char *)&(session->ttmpfile[0]), QUEUE_ID_LEN);
+   make_random_string((unsigned char*)&(session->ttmpfile[0]), QUEUE_ID_LEN);
+
+   if(session->buf){
+      memset(session->buf, 0, session->bufsize);
+   }
+   session->buflen = 0;
 }
--- a/src/smtp.h
+++ b/src/smtp.h
@ -7,22 +7,24 @@

 #include <piler.h>

-void process_smtp_command(struct smtp_session *session, char *buf, struct config *cfg);
+void process_smtp_command(struct smtp_session *session, struct config *cfg);
 void process_data(struct smtp_session *session, char *buf, int buflen);

 void send_smtp_response(struct smtp_session *session, char *buf);
 void process_command_helo(struct smtp_session *session, char *buf, int buflen);
-void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen);
+void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen, struct config *cfg);
 void process_command_quit(struct smtp_session *session, char *buf, int buflen);
 void process_command_reset(struct smtp_session *session);
-void process_command_mail_from(struct smtp_session *session, char *buf);
-void process_command_rcpt_to(struct smtp_session *session, char *buf, struct config *cfg);
+void process_command_mail_from(struct smtp_session *session);
+void process_command_rcpt_to(struct smtp_session *session, struct config *cfg);
 void process_command_data(struct smtp_session *session, struct config *cfg);
 void process_command_period(struct smtp_session *session);
 void process_command_starttls(struct smtp_session *session);

 void reset_bdat_counters(struct smtp_session *session);
-void get_bdat_size_to_read(struct smtp_session *session, char *buf);
+void get_bdat_size_to_read(struct smtp_session *session);
 void process_bdat(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg);

+void reset_smtp_session(struct smtp_session *session);
+
 #endif
--- a/src/smtpcodes.h
+++ b/src/smtpcodes.h
@ -38,7 +38,7 @@
 #define SMTP_RESP_220_READY_TO_START_TLS "220 Ready to start TLS\r\n"
 #define SMTP_RESP_221_GOODBYE "221 %s Goodbye\r\n"
 #define SMTP_RESP_250_OK "250 Ok\r\n"
-#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n250-PIPELINING\r\n%s250-SIZE\r\n250 8BITMIME\r\n"
+#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n250-SIZE %d\r\n%s250 8BITMIME\r\n"

 #define SMTP_EXTENSION_STARTTLS "250-STARTTLS\r\n"
 #define SMTP_EXTENSION_CHUNKING "250-CHUNKING\r\n"
@ -51,6 +51,7 @@
 #define SMTP_RESP_421_ERR_TMP "421 %s service not available\r\n"
 #define SMTP_RESP_421_ERR_WRITE_FAILED "421 writing queue file failed\r\n"
 #define SMTP_RESP_421_ERR_ALL_PORTS_ARE_BUSY "421 All server ports are busy\r\n"
+#define SMTP_RESP_451_ERR_TOO_MANY_REQUESTS "451 Too many requests, try again later\r\n"
 #define SMTP_RESP_451_ERR "451 Error in processing, try again later\r\n"
 #define SMTP_RESP_454_ERR_TLS_TEMP_ERROR "454 TLS not available currently\r\n"

@ -59,6 +60,7 @@
 #define SMTP_RESP_550_ERR_INVALID_RECIPIENT "550 Invalid recipient\r\n"
 #define SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY "550 You are banned by local policy\r\n"
 #define SMTP_RESP_550_ERR "550 Service currently unavailable\r\n"
+#define SMTP_RESP_552_ERR_TOO_BIG_EMAIL "552 Too big email\r\n"


 // LMTP commands
--- a/src/tai.h
+++ b/src/tai.h
@ -5,12 +5,12 @@
 #ifndef _TAI_H
   #define _TAI_H

+#include "config.h"
+
 #define TAI_PACK 8
 #define TAIA_PACK 16
 #define TIMESTAMP 25

-typedef unsigned long long uint64;
-
 struct tai {
   uint64 x;
 };
--- a/tests/cases/05-smtp.inc
+++ b/tests/cases/05-smtp.inc
@ -6,23 +6,23 @@ case1() {

   setup

-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox2" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet2" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam0" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam1" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam2" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/journal" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/deduptest" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/special" --socket --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox2"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet2"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam0"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam1"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam2"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/journal"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/deduptest"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/special"  --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus"  --no-counter


-   wait_until_emails_are_processed "piler1" 3019
+   wait_until_emails_are_processed "piler1" 3020
   [[ $RT -eq 1 ]] || docker exec "piler1" su piler -c /usr/libexec/piler/indexer.delta.sh 2>/dev/null

-   count_status_values 3019 2908 111 0
+   count_status_values 3020 2909 111 0

   test_retrieved_messages_are_the_same "piler1" "piler"

--- a/tests/setup.inc
+++ b/tests/setup.inc
@ -27,13 +27,13 @@ launch_containers() {

   destroy_containers

-   log "running docker-compose up"
+   log "running docker compose up"

-   docker-compose -f "$composefile" up -d
+   docker compose -f "$composefile" up -d

   wait_until_container_is_healthy "piler1"

-   docker-compose -f "$composefile" ps
+   docker compose -f "$composefile" ps

   popd
 }
--- a/unit_tests/run.sh
+++ b/unit_tests/run.sh
@ -24,6 +24,13 @@ setup_mysql() {
   mysql -u piler -ppiler123 piler1 < ../util/db-mysql.sql
 }

+run_smtp_tests() {
+   mkdir -p /var/piler/store/00/piler /var/piler/tmp /var/piler/manticore
+   chown -R piler:piler /var/piler/
+   ../src/piler-smtp -L 5 -d
+   ./smtp -s 127.0.0.1
+}
+
 if [[ -v BUILD_NUMBER ]]; then
   setup_mysql
 fi
@ -37,3 +44,5 @@ fi
 ./check_hash
 ./check_decoder
 ./check_attachments
+
+if [[ -v BUILD_NUMBER ]]; then run_smtp_tests; fi
--- a/unit_tests/smtp.c
+++ b/unit_tests/smtp.c
@ -4,11 +4,13 @@

 #include "test.h"

-
 extern char *optarg;
 extern int optind;

-char *testmessage = "From: aaa@aaa.fu\nTo: bela@aaa.fu\nMessage-Id: ajajajaja\nSubject: this is a test\n\nAaaaaa.";
+char *testmessage = "From: aaa@aaa.fu\r\nTo: bela@aaa.fu\r\nMessage-Id: ajajajaja\r\nSubject: this is a test\r\n\r\nAaaaaa";
+char *testmessage2 = " and the last line\r\n";
+
+char *recipient = "aaa@worker0";

 int helo = 0; // 0=HELO, 1=EHLO

@ -16,12 +18,24 @@ void usage(){
   printf("\nusage: smtp\n\n");
   printf("    -s <smtp server>                  SMTP server\n");
   printf("    -p <smtp port>                    SMTP port (25)\n");
+   printf("    -r <recipient>                    Envelope recipient\n");
   printf("    -t <timeout>                      Timeout in sec (10)\n");

   exit(0);
 }


+int countreplies(char *s){
+   int replies = 0;
+
+   for(; *s; s++){
+      if(*s == '\n') replies++;
+   }
+
+   return replies;
+}
+
+
 void connect_to_smtp_server(char *server, int port, struct data *data){
   int rc;
   char port_string[8], buf[MAXBUFSIZE];
@ -53,14 +67,15 @@ void connect_to_smtp_server(char *server, int port, struct data *data){
   }

   recvtimeoutssl(data->net, buf, sizeof(buf));
-   printf("rcvd: %s", buf);

 ENDE:
   freeaddrinfo(res);
 }


-void send_smtp_command(struct net *net, char *cmd, char *buf, int buflen){
+void send_smtp_command(struct net *net, char *cmd, char *buf, int buflen, int expectedreplies){
+   int tot=0;
+
   if(net == NULL || cmd == NULL) return;

   if(net->socket == -1){
@ -68,25 +83,32 @@ void send_smtp_command(struct net *net, char *cmd, char *buf, int buflen){
      return;
   }

-   printf("sent: %s", cmd);
   write1(net, cmd, strlen(cmd));
-   recvtimeoutssl(net, buf, buflen);
-   printf("rcvd: %s", buf);
+
+   while(1){
+      tot += recvtimeoutssl(net, buf+tot, buflen);
+      if(countreplies(buf) == expectedreplies) break;
+   }
 }


 void send_helo_command(struct net *net){
+   int replies=1;
   char recvbuf[MAXBUFSIZE];

   if(helo == 0){
-      send_smtp_command(net, "HELO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1);
-      assert(strncmp(recvbuf, "250 ", 4) == 0 && "HELO");
+      send_smtp_command(net, "HELO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1, replies);
+      ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
   }
   else {
-      send_smtp_command(net, "EHLO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1);
-      assert(strncmp(recvbuf, "250-", 4) == 0 && "EHLO");
-      if(net->use_ssl == 0) assert(strstr(recvbuf, "250-STARTTLS") && "STARTTLS");
-      else assert(strstr(recvbuf, "250-STARTTLS") == NULL && "STARTTLS");
+      //replies = 6;
+      replies = 5;
+      if(net->use_ssl == 1) replies--;
+
+      send_smtp_command(net, "EHLO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1, replies);
+      ASSERT(strncmp(recvbuf, "250-", 4) == 0, recvbuf);
+      if(net->use_ssl == 0){ ASSERT(strstr(recvbuf, "250-STARTTLS"), recvbuf); }
+      else { ASSERT(strstr(recvbuf, "250-STARTTLS") == NULL, recvbuf); }
   }
 }

@ -94,209 +116,392 @@ void send_helo_command(struct net *net){
 static void test_smtp_commands_one_at_a_time(char *server, int port, struct data *data){
   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];

+   TEST_HEADER();
+
   connect_to_smtp_server(server, port, data);

   send_helo_command(data->net);

-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RCPT");
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

-   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "354 ", 4) == 0 && "DATA");
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);

   snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage);

-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "PERIOD");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);

   close(data->net->socket);
+
+   TEST_FOOTER();
 }


-static void test_smtp_commands_pipelining(char *server, int port, struct data *data){
+static void test_smtp_commands_one_at_a_time_data_in_2_parts(char *server, int port, struct data *data){
   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];

+   TEST_HEADER();
+
   connect_to_smtp_server(server, port, data);

   send_helo_command(data->net);

-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+
+   write1(data->net, testmessage, strlen(testmessage));
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage2);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
+
+   close(data->net->socket);
+
+   TEST_FOOTER();
+}
+
+
+/*static void test_smtp_commands_pipelining(char *server, int port, struct data *data){
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+
+   TEST_HEADER();
+
+   connect_to_smtp_server(server, port, data);
+
+   send_helo_command(data->net);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

   snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\nQUIT\r\n", testmessage);

-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 2);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

   close(data->net->socket);
-}
+
+   TEST_FOOTER();
+}*/


 static void test_smtp_commands_with_reset_command(char *server, int port, struct data *data){
-   char recvbuf[MAXBUFSIZE];
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+
+   TEST_HEADER();

   connect_to_smtp_server(server, port, data);

   send_helo_command(data->net);

-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

-   send_smtp_command(data->net, "RSET\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RSET");
+   send_smtp_command(data->net, "RSET\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "503 ", 4) == 0 && "RCPT");
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "503 ", 4) == 0, recvbuf);

-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);

   close(data->net->socket);
+
+   TEST_FOOTER();
 }


 static void test_smtp_commands_partial_command(char *server, int port, struct data *data){
   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];

+   TEST_HEADER();
+
   connect_to_smtp_server(server, port, data);

   send_helo_command(data->net);

   write1(data->net, "M", 1);
-   printf("sent: M\n");

-   send_smtp_command(data->net, "AIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, "AIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RCPT");
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

-   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "354 ", 4) == 0 && "DATA");
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);

   snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage);

-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "PERIOD");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);

   close(data->net->socket);
+
+   TEST_FOOTER();
 }


-static void test_smtp_commands_partial_command_pipelining(char *server, int port, struct data *data){
+/*static void test_smtp_commands_partial_command_pipelining(char *server, int port, struct data *data){
   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];

+   TEST_HEADER();
+
   connect_to_smtp_server(server, port, data);

   send_helo_command(data->net);

   write1(data->net, "M", 1);
-   printf("sent: M\n");

-   send_smtp_command(data->net, "AIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   snprintf(sendbuf, sizeof(sendbuf)-1, "AIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

   snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\nQUIT\r\n", testmessage);

-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 2);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

   close(data->net->socket);
-}
+
+   TEST_FOOTER();
+}*/


 static void test_smtp_commands_starttls(char *server, int port, struct data *data){
   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];

+   TEST_HEADER();
+
   connect_to_smtp_server(server, port, data);

   send_helo_command(data->net);

-   send_smtp_command(data->net, "STARTTLS\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "220 ", 4) == 0 && "STARTTLS");
+   send_smtp_command(data->net, "STARTTLS\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "220 ", 4) == 0, recvbuf);

   init_ssl_to_server(data);
   data->net->use_ssl = 1;

   send_helo_command(data->net);

-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RCPT");
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

-   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "354 ", 4) == 0 && "DATA");
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);

   snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage);

-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "PERIOD");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);

   close(data->net->socket);
+
+   TEST_FOOTER();
 }


 static void test_smtp_commands_period_command_in_2_parts(char *server, int port, char *part1, char *part2, struct data *data){
   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];

+   TEST_HEADER();
+
   connect_to_smtp_server(server, port, data);

   send_helo_command(data->net);

-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   // As long as pipelining support is not reintroduced
+   //
+   /*snprintf(sendbuf, sizeof(sendbuf)-1, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   if(!strstr(recvbuf, "354 ")) recvtimeoutssl(data->net, recvbuf, sizeof(recvbuf)-1);
+   ASSERT(strstr(recvbuf, "354 "), recvbuf);*/
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+

   snprintf(sendbuf, sizeof(sendbuf)-1, "%s%s", testmessage, part1);
   write1(data->net, sendbuf, strlen(sendbuf));

   snprintf(sendbuf, sizeof(sendbuf), "%s", part2);
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

   snprintf(sendbuf, sizeof(sendbuf)-1, "QUIT\r\n");
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);

-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);

   close(data->net->socket);
+
+   TEST_FOOTER();
 }


 static void test_smtp_commands_period_command_in_its_own_packet(char *server, int port, struct data *data){
   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];

+   TEST_HEADER();
+
   connect_to_smtp_server(server, port, data);

   send_helo_command(data->net);

-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   // As long as pipelining support is not reintroduced
+   //
+   /*snprintf(sendbuf, sizeof(sendbuf)-1, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   if(!strstr(recvbuf, "354 ")) recvtimeoutssl(data->net, recvbuf, sizeof(recvbuf)-1);
+   ASSERT(strstr(recvbuf, "354 "), recvbuf);*/
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+
+

   snprintf(sendbuf, sizeof(sendbuf)-1, "%s", testmessage);
   write1(data->net, sendbuf, strlen(sendbuf));

   snprintf(sendbuf, sizeof(sendbuf), "\r\n.\r\n");
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);

   snprintf(sendbuf, sizeof(sendbuf)-1, "QUIT\r\n");
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);

-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);

   close(data->net->socket);
+
+   TEST_FOOTER();
+}
+
+
+static void test_smtp_commands_with_partial_data_lines(char *server, int port, struct data *data){
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+   int yes=1;
+
+   TEST_HEADER();
+
+   connect_to_smtp_server(server, port, data);
+
+   setsockopt(data->net->socket, IPPROTO_TCP, TCP_NODELAY, &yes, (socklen_t)sizeof(int));
+
+   send_helo_command(data->net);
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "From: aaa@aaa.fu\r\nTo: bela@aaa.fu\r\nMessage-Id: ajajajaja\r\nSubject: this is a test\r\n\r\nAaaaaa");
+   write1(data->net, sendbuf, strlen(sendbuf)); sleep(2);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "jjdkdjkd dkd some garbage.");
+   write1(data->net, sendbuf, strlen(sendbuf)); sleep(2);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "\r\nSome shit again  au aua ua au aua uuu");
+   write1(data->net, sendbuf, strlen(sendbuf)); sleep(2);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, ".\r\nAnd the last line.\r\n.\r\n");
+   write1(data->net, sendbuf, strlen(sendbuf));
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\nQUIT\r\n", testmessage);
+   recvtimeoutssl(data->net, recvbuf, sizeof(recvbuf));
+
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+
+   close(data->net->socket);
+
+   TEST_FOOTER();
+}
+
+
+static void test_smtp_bdat_last_one_at_a_time(char *server, int port, struct data *data){
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+
+   TEST_HEADER();
+
+   connect_to_smtp_server(server, port, data);
+
+   send_helo_command(data->net);
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "BDAT %ld LAST\r\n", strlen(testmessage)+strlen(testmessage2));
+   write1(data->net, sendbuf, strlen(sendbuf));
+
+   write1(data->net, testmessage, strlen(testmessage));
+
+   send_smtp_command(data->net, testmessage2, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
+
+   close(data->net->socket);
+
+   TEST_FOOTER();
 }


@ -317,6 +522,7 @@ int main(int argc, char **argv){
         {
            {"server",       required_argument,  0,  's' },
            {"port",         required_argument,  0,  'p' },
+            {"rcpt",         required_argument,  0,  'r' },
            {"timeout",      required_argument,  0,  't' },
            {"lhlo",         no_argument,        0,  'l' },
            {"help",         no_argument,        0,  'h' },
@ -325,9 +531,9 @@ int main(int argc, char **argv){

      int option_index = 0;

-      int c = getopt_long(argc, argv, "c:s:p:t:lh?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:s:p:t:r:lh?", long_options, &option_index);
 #else
-      int c = getopt(argc, argv, "c:s:p:t:lh?");
+      int c = getopt(argc, argv, "c:s:p:t:r:lh?");
 #endif


@ -343,6 +549,10 @@ int main(int argc, char **argv){
                    port = atoi(optarg);
                    break;

+         case 'r' :
+                    recipient = optarg;
+                    break;
+
         case 't' :
                    net.timeout = atoi(optarg);
                    break;
@ -367,21 +577,26 @@ int main(int argc, char **argv){
   data.net = &net;

   test_smtp_commands_one_at_a_time(server, port, &data);
-   test_smtp_commands_pipelining(server, port, &data);
+   test_smtp_commands_one_at_a_time_data_in_2_parts(server, port, &data);
+   ////test_smtp_commands_pipelining(server, port, &data);
   test_smtp_commands_with_reset_command(server, port, &data);
   test_smtp_commands_partial_command(server, port, &data);
-   test_smtp_commands_partial_command_pipelining(server, port, &data);
+   ////test_smtp_commands_partial_command_pipelining(server, port, &data);
+
   test_smtp_commands_period_command_in_2_parts(server, port, "\r", "\n.\r\n", &data);
   test_smtp_commands_period_command_in_2_parts(server, port, "\r\n", ".\r\n", &data);
   test_smtp_commands_period_command_in_2_parts(server, port, "\r\n.", "\r\n", &data);
   test_smtp_commands_period_command_in_2_parts(server, port, "\r\n.\r", "\n", &data);
   test_smtp_commands_period_command_in_its_own_packet(server, port, &data);

-   helo = 1; // we must use EHLO to get the STARTTLS in the response
-   test_smtp_commands_starttls(server, port, &data);
+   test_smtp_commands_with_partial_data_lines(server, port, &data);

+   helo = 1; // we must use EHLO to get the STARTTLS in the response
+
+   test_smtp_commands_starttls(server, port, &data);
+   data.net->use_ssl = 0;
+
+   test_smtp_bdat_last_one_at_a_time(server, port, &data);

   return 0;
 }
-
-
--- a/unit_tests/test.h
+++ b/unit_tests/test.h
@ -8,6 +8,7 @@
 #include <netdb.h>
 #include <arpa/inet.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <fcntl.h>
 #include <locale.h>
 #include <getopt.h>
--- a/util/pilerpurge.py
+++ b/util/pilerpurge.py
@ -9,7 +9,7 @@ import sys
 import syslog
 import time

-SQL_PURGE_SELECT_QUERY = "SELECT piler_id, size FROM " +\
+SQL_PURGE_SELECT_QUERY = "SELECT id, piler_id, size FROM " +\
    "metadata WHERE deleted=0 AND retained < UNIX_TIMESTAMP(NOW()) " +\
    "AND id NOT IN (SELECT id FROM rcpt WHERE `to` IN " +\
    "(SELECT email FROM legal_hold)) AND id NOT IN (SELECT " +\
@ -33,7 +33,9 @@ def read_options(filename="", opts={}):
    opts['password'] = config.get('piler', 'mysqlpwd')
    opts['database'] = config.get('piler', 'mysqldb')
    opts['storedir'] = config.get('piler', 'queuedir')
-
+    opts['rtindex'] = config.getint('piler', 'rtindex', fallback=0)
+    opts['sphxhost'] = config.get('piler', 'sphxhost', fallback='127.0.0.1')
+    opts['sphxport'] = config.getint('piler', 'sphxport', fallback=9306)
    opts['server_id'] = "%02x" % config.getint('piler', 'server_id')


@ -184,6 +186,17 @@ def get_attachment_file_path(piler_id='', attachment_id=0, opts={}):
                    str(attachment_id)])


+def purge_index_data(ids=[], opts={}):
+    '''
+    Delete from index data in case of RT index
+    '''
+
+    if opts['rtindex'] == 1 and opts['dry_run'] is False:
+        cursor = opts['sphx'].cursor()
+        a = "," . join([str(x) for x in ids])
+        cursor.execute("DELETE FROM piler WHERE id IN (%s)" % (a))
+
+
 def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-c", "--config", type=str, help="piler.conf path",
@ -203,6 +216,7 @@ def main():
    opts['dry_run'] = args.dry_run
    opts['verbose'] = args.verbose
    opts['db'] = None
+    opts['sphx'] = None
    opts['messages'] = 0
    opts['files'] = 0
    opts['size'] = 0
@ -217,6 +231,8 @@ def main():
        opts['db'] = dbapi.connect(opts['dbhost'], opts['username'],
                                   opts['password'], opts['database'])

+        opts['sphx'] = dbapi.connect(host=opts['sphxhost'], port=opts['sphxport'])
+
        if is_purge_enabled(opts) is False:
            syslog.syslog("Purging emails is disabled")
            sys.exit(1)
@ -229,13 +245,15 @@ def main():
            if rows == ():
                break

-            piler_id = [x[0] for x in rows]
-            size = [x[1] for x in rows]
+            id = [x[0] for x in rows]
+            piler_id = [x[1] for x in rows]
+            size = [x[2] for x in rows]

            opts['purged_size'] = opts['purged_size'] + sum(size)

            purge_m_files(piler_id, opts)
            purge_attachments_by_piler_id(piler_id, opts)
+            purge_index_data(id, opts)

            # It's possible that there's attachment duplication, thus
            # refcount > 0, even though after deleting the duplicates
--- a/webui/controller/message/bulkremove.php
+++ b/webui/controller/message/bulkremove.php
@ -49,8 +49,8 @@ class ControllerMessageBulkremove extends Controller {
            AUDIT(ACTION_REMOVE_MESSAGE, '', '', $id, '');
            $db->query("UPDATE " . TABLE_META . " SET retained=? WHERE id=?", [NOW, $id]);
            if(RT) {
-               $sphx = Registry::get('sphx');
-               $sphx->query("DELETE FROM " . SPHINX_MAIN_INDEX . " WHERE id=" . (int)$this->data['id']);
+               $sphxrw = Registry::get('sphxrw');
+               $sphxrw->query("DELETE FROM " . SPHINX_MAIN_INDEX . " WHERE id=" . (int)$this->data['id']);
            }

            syslog(LOG_INFO, $this->data['username'] . " removed message: $id");
--- a/webui/controller/message/remove.php
+++ b/webui/controller/message/remove.php
@ -41,8 +41,8 @@ class ControllerMessageRemove extends Controller {
      $db->query("UPDATE " . TABLE_META . " SET retained=? WHERE id=?", [NOW, $id]);
      $db->query("UPDATE " . TABLE_DELETED . " SET deleted=1 WHERE id=?", [$id]);
      if(RT) {
-         $sphx = Registry::get('sphx');
-         $sphx->query("DELETE FROM " . SPHINX_MAIN_INDEX . " WHERE id=", (int)$this->data['id']);
+         $sphxrw = Registry::get('sphxrw');
+         $sphxrw->query("DELETE FROM " . SPHINX_MAIN_INDEX . " WHERE id=", (int)$this->data['id']);
      }

      syslog(LOG_INFO, $this->data['username'] . " removed message: $id");
--- a/webui/index.php
+++ b/webui/index.php
@ -41,12 +41,14 @@ Registry::set('db', $db);

 Registry::set('DB_DRIVER', DB_DRIVER);

-$sphx = new DB(SPHINX_DRIVER, SPHINX_HOSTNAME, "", "", SPHINX_DATABASE, "");
+$sphx = new DB(SPHINX_DRIVER, SPHINX_HOSTNAME_READONLY, "", "", SPHINX_DATABASE, "");
+$sphxrw = new DB(SPHINX_DRIVER, SPHINX_HOSTNAME, "", "", SPHINX_DATABASE, "");
 Registry::set('sphx', $sphx);
+Registry::set('sphxrw', $sphxrw);


 if(MEMCACHED_ENABLED) {
-   $memcache = new Memcache();
+   $memcache = new Memcached();
   foreach ($memcached_servers as $m){
      $memcache->addServer($m[0], $m[1]);
   }
@ -112,6 +114,3 @@ else {

 $controller = new Front();
 $controller->dispatch($action, new Router('common/not_found'));
-
-
-?>
--- a/webui/model/search/message.php
+++ b/webui/model/search/message.php
@ -486,13 +486,13 @@ class ModelSearchMessage extends Model {

      if($tag == '') {
         if(RT) {
-            $this->sphx->query("DELETE FROM " . SPHINX_TAG_INDEX . " WHERE uid=$uid AND mid=$id");
+            $this->sphxrw->query("DELETE FROM " . SPHINX_TAG_INDEX . " WHERE uid=$uid AND mid=$id");
         } else {
            $query = $this->db->query("DELETE FROM " . TABLE_TAG . " WHERE uid=? AND id=?", array($uid, $id));
         }
      } else {
         if(RT) {
-            $this->sphx->query("REPLACE INTO " . SPHINX_TAG_INDEX . " (mid, uid, tag) VALUES (?,?,?)",[$id,$uid,$tag]);
+            $this->sphxrw->query("REPLACE INTO " . SPHINX_TAG_INDEX . " (mid, uid, tag) VALUES (?,?,?)",[$id,$uid,$tag]);
         } else {
            $query = $this->db->query("UPDATE " . TABLE_TAG . " SET tag=? WHERE uid=? AND id=?", array($tag, $uid, $id));
            if($this->db->countAffected() == 0) {
@ -510,7 +510,7 @@ class ModelSearchMessage extends Model {

      if(RT) {
         $ids_str = implode(",", $ids);
-         $this->sphx->query("DELETE FROM " . SPHINX_TAG_INDEX . " WHERE uid=$uid AND mid IN ($ids_str)");
+         $this->sphxrw->query("DELETE FROM " . SPHINX_TAG_INDEX . " WHERE uid=$uid AND mid IN ($ids_str)");
      } else {
         $query = $this->db->query("DELETE FROM " . TABLE_TAG . " WHERE uid=? AND id IN ($q)", $arr);
      }
@ -518,7 +518,7 @@ class ModelSearchMessage extends Model {
      if($tag) {
         foreach ($ids as $id) {
            if(RT) {
-               $this->sphx->query("INSERT INTO " . SPHINX_TAG_INDEX . " (mid, uid, tag) VALUES(?,?,?)", [$id, $uid, $tag]);
+               $this->sphxrw->query("INSERT INTO " . SPHINX_TAG_INDEX . " (mid, uid, tag) VALUES(?,?,?)", [$id, $uid, $tag]);
            } else {
               $query = $this->db->query("INSERT INTO " . TABLE_TAG . " (id, uid, tag) VALUES(?,?,?)", array($id, $uid, $tag));
            }
@ -561,10 +561,10 @@ class ModelSearchMessage extends Model {
   public function add_message_rt_note($id = '', $uid = 0, $note = '') {
      if($id == '' || $uid <= 0) { return 0; }

-      $this->sphx->query("DELETE FROM " . SPHINX_NOTE_INDEX . " WHERE uid=$uid AND mid=$id");
+      $this->sphxrw->query("DELETE FROM " . SPHINX_NOTE_INDEX . " WHERE uid=$uid AND mid=$id");

      if($note) {
-         $this->sphx->query("INSERT INTO " . SPHINX_NOTE_INDEX . " (mid, uid, note) VALUES (?,?,?)",[$id,$uid,$note]);
+         $this->sphxrw->query("INSERT INTO " . SPHINX_NOTE_INDEX . " (mid, uid, note) VALUES (?,?,?)",[$id,$uid,$note]);
      }

      return 1;
--- a/webui/system/helper/TrustedTimestamps.php
+++ b/webui/system/helper/TrustedTimestamps.php
@ -71,6 +71,10 @@ class TrustedTimestamps
        if (!file_exists($requestfile_path))
            throw new Exception("The Requestfile was not found");

+        $header = array('Content-Type: application/timestamp-query');
+        if(TSA_AUTH_USER)
+            $header[] = "Authorization: Basic " . base64_encode(TSA_AUTH_USER . ':' . TSA_AUTH_PASSWORD);
+
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $tsa_url);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
@ -78,10 +82,22 @@ class TrustedTimestamps
        curl_setopt($ch, CURLOPT_POST, 1);
        curl_setopt($ch, CURLOPT_BINARYTRANSFER, 1);
        curl_setopt($ch, CURLOPT_POSTFIELDS, file_get_contents($requestfile_path));
-        curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/timestamp-query'));
+        curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
        curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)");
        curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, TSA_VERIFY_CERTIFICATE);

+        if(TSA_AUTH_CERT_FILE && TSA_AUTH_KEY_FILE)
+        {
+            if(!file_exists(TSA_AUTH_CERT_FILE))
+                throw new Exception("Client certificate file " . TSA_AUTH_CERT_FILE . " not found");
+            curl_setopt($ch, CURLOPT_SSLCERT, TSA_AUTH_CERT_FILE);
+            if(!file_exists(TSA_AUTH_KEY_FILE))
+                throw new Exception("Client key file " . TSA_AUTH_KEY_FILE . " not found");
+            curl_setopt($ch, CURLOPT_SSLKEY, TSA_AUTH_KEY_FILE);
+            if(TSA_AUTH_KEY_PASSWORD)
+                curl_setopt($ch, CURLOPT_KEYPASSWD, TSA_AUTH_KEY_PASSWORD);
+        }
+
        $binary_response_string = curl_exec($ch);

        $error = curl_error($ch);
Author	SHA1	Message	Date
Janos SUTO	65ed2f798e	Bumped up version number to 1.4.5 Signed-off-by: Janos SUTO <sj@acts.hu>	2024-04-06 21:02:07 +02:00
Janos SUTO	1377cbdae6	piler-smtp refactor Signed-off-by: Janos SUTO <sj@acts.hu>	2024-04-06 20:59:50 +02:00
Janos SUTO	3d70a25d35	Updated jammy installer Signed-off-by: Janos SUTO <sj@acts.hu>	2024-04-06 17:36:56 +02:00
Janos SUTO	26627b1a20	pilerimport supports Zimbra IMAP impersonation (Credits: SEMATPL) Signed-off-by: Janos SUTO <sj@acts.hu>	2024-04-04 12:57:32 +02:00
Janos SUTO	f1cda8f6a1	Fixes #1356 , Reindex fix Signed-off-by: Janos SUTO <sj@acts.hu>	2024-03-25 13:37:37 +01:00
Janos SUTO	78835ae566	Updated counter values Signed-off-by: Janos SUTO <sj@acts.hu>	2024-03-22 06:22:51 +01:00
Janos SUTO	292ec45bee	Create export dir Signed-off-by: Janos SUTO <sj@acts.hu>	2024-03-22 06:03:28 +01:00
Janos SUTO	ba4868a068	Fixed start.sh to apply RT settings once only Signed-off-by: Janos SUTO <sj@acts.hu>	2024-03-12 16:16:23 +01:00
Janos SUTO	7f74bad08a	Added docker-compose file Signed-off-by: Janos SUTO <sj@acts.hu>	2024-02-18 12:29:38 +01:00
Janos SUTO	643a7fba9b	Fixed typo in index.php invoking memcached Signed-off-by: Janos SUTO <sj@acts.hu>	2024-02-18 12:28:11 +01:00
Janos SUTO	e3189c56ae	Added rsyslog to docker image Signed-off-by: Janos SUTO <sj@acts.hu>	2024-02-10 05:24:00 +01:00
Janos SUTO	d4a57e386c	Fixed manticore.conf Signed-off-by: Janos SUTO <sj@acts.hu>	2024-01-02 08:49:15 +01:00
Janos SUTO	f9c1d1d1d1	Added sphx readonly host to config.php.in Signed-off-by: Janos SUTO <sj@acts.hu>	2024-01-02 08:12:52 +01:00
Janos SUTO	f9e8c3f828	Removed unused docker-compose.yaml Signed-off-by: Janos SUTO <sj@acts.hu>	2024-01-02 08:11:58 +01:00
Janos SUTO	8da731c174	Added read-only connection support for manticore Signed-off-by: Janos SUTO <sj@acts.hu>	2024-01-02 08:09:53 +01:00
Janos SUTO	3637d59942	Added support to timestamp service authentication Signed-off-by: Janos SUTO <sj@acts.hu>	2023-12-26 06:52:54 +01:00
Janos SUTO	68fed34a53	Added RT support for docker image Signed-off-by: Janos SUTO <sj@acts.hu>	2023-11-22 08:16:50 +01:00
Janos SUTO	4f948d3d5c	Fixed retrieving emails archived with legacy openssl 1.x Signed-off-by: Janos SUTO <sj@acts.hu>	2023-11-03 13:47:15 +01:00
Janos SUTO	e2f6a71827	Updated pilerpurge to remove data from manticore in case of RT index Signed-off-by: Janos SUTO <sj@acts.hu>	2023-11-01 08:28:38 +01:00
Janos SUTO	91b3c73b02	Updated docker compose usage Signed-off-by: Janos SUTO <sj@acts.hu>	2023-10-16 18:26:43 +02:00
 @ -1 +1 @@
 .4.4
 .4.5