Bumped up version number to 1.4.5

Signed-off-by: Janos SUTO <sj@acts.hu>

Makefile.in

@@ -82,7 +82,7 @@ installdirs: mkinstalldirs
 		$(DESTDIR)$(localstatedir)/piler/stat \
 		$(DESTDIR)$(localstatedir)/piler/tmp \
 		$(DESTDIR)$(localstatedir)/piler/error \
-		$(DESTDIR)$(localstatedir)/piler/sphinx \
+		$(DESTDIR)$(localstatedir)/piler/export \
 		$(DESTDIR)$(localstatedir)/piler/manticore
 
 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/run/piler
@@ -91,7 +91,7 @@ installdirs: mkinstalldirs
 	$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/stat
 	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/tmp
 	$(INSTALL) -d -m 0711 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/error
-	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/sphinx
+	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/export
 	$(INSTALL) -d -m 0700 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/manticore
 
 

RELEASE_NOTES

@@ -1,8 +1,44 @@
+1.4.5:
+------
+
+- Introduced 2 new variables in /etc/piler/piler.conf affecting piler-smtp
+
+  ; max message size in bytes
+  ; piler-smtp will reject any message that's bigger than this number
+  max_message_size=50000000
+
+  ; max memory in bytes piler-smtp uses for buffering messages
+  ; when this limit is exceeded, no new emails will be accepted
+  ; until the used memory for all in progress emails decreases
+  ; below this level
+  max_smtp_memory=500000000
+
+  Be sure to adjust these values to your environment!
+
+- Added read-only connection support for manticore
+  If using sphinx, add the following to config-site.php:
+  $config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9306';
+
+- pilerimport supports Zimbra IMAP impersonation
+
+  Generate the following base64 encoded string:
+  (Be sure to use the actual usernames and password):
+
+  pw="$( printf '%s\0%s\0%s' 'username' 'zimbra_admin_username' 'zimbra_admin_password' | base64 )"
+
+  Then specify -u ZIMBRA -p "$pw" for pilerimport, eg.
+
+  pilerimport -i imap.server -u ZIMBRA -p "$pw" ...
+
+  Note that "ZIMBRA" is a special username, it indicates for pilerimport
+  to actually use the imap impersonation for Zimbra.
+
+
 1.4.4:
 ------
 
-Renamed "group" table to "usergroup"
+- Renamed "group" table to "usergroup"
-Be sure to run util/db-upgrade.sql on the mysql piler database
+  Be sure to run util/db-upgrade.sql on the mysql piler database
 
 
 1.4.3:

VERSION

@@ -1 +1 @@
-1.4.4
+1.4.5

config.php.in

@@ -232,6 +232,11 @@ $config['TSA_START_ID'] = 1;
 $config['TSA_STAMP_REQUEST_UNIT_SIZE'] = 10000;
 $config['TSA_VERIFY_CERTIFICATE'] = true;
 $config['TSA_RELAXED_CHECK'] = false;
+$config['TSA_AUTH_USER'] = '';
+$config['TSA_AUTH_PASSWORD'] = '';
+$config['TSA_AUTH_CERT_FILE'] = '';
+$config['TSA_AUTH_KEY_FILE'] = '';
+$config['TSA_AUTH_KEY_PASSWORD'] = '';
 
 $config['DB_DRIVER'] = 'mysql';
 $config['DB_PREFIX'] = '';
@@ -244,6 +249,7 @@ $config['DB_CHARSET'] = 'utf8mb4';
 $config['SPHINX_DRIVER'] = 'sphinx';
 $config['SPHINX_DATABASE'] = 'sphinx';
 $config['SPHINX_HOSTNAME'] = '127.0.0.1:9306';
+$config['SPHINX_HOSTNAME_READONLY'] = '127.0.0.1:9307';
 $config['SPHINX_MAIN_INDEX'] = 'main1,dailydelta1,delta1';
 $config['SPHINX_ATTACHMENT_INDEX'] = 'att1';
 $config['SPHINX_TAG_INDEX'] = 'tag1';

configure

@@ -6426,13 +6426,13 @@ fi
 
 echo
 echo
-echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/wiki/current:upgrade"
+echo "IMPORTANT! If you upgrade, be sure to read https://www.mailpiler.org/upgrade-instructions/"
 echo
 echo
 
 echo
 echo "Did you know that piler has an enterprise edition as well?"
-echo "Check out what it can do for you at https://mailpiler.com/piler-enterprise-email-archiver-features/"
+echo "Check out what it can do for you at https://mailpiler.com/#features"
 echo
 echo
 

configure.in

@@ -537,12 +537,12 @@ AC_OUTPUT
 
 echo
 echo
-echo "IMPORTANT! If you upgrade, be sure to read http://www.mailpiler.org/wiki/current:upgrade"
+echo "IMPORTANT! If you upgrade, be sure to read https://www.mailpiler.org/upgrade-instructions/"
 echo
 echo
 
 echo
 echo "Did you know that piler has an enterprise edition as well?"
-echo "Check out what it can do for you at https://mailpiler.com/piler-enterprise-email-archiver-features/"
+echo "Check out what it can do for you at https://mailpiler.com/#features"
 echo
 echo

contrib/installer/jammy.sh

@@ -25,6 +25,9 @@ PILER_CONF="/etc/piler/piler.conf"
 CONFIG_SITE_PHP="/etc/piler/config-site.php"
 SEARCHCFG="/etc/piler/manticore.conf"
 
+TRAEFIK_VERSION="v2.11.0"
+ARCH="amd64"
+
 export DEBIAN_FRONTEND=noninteractive
 
 install_prerequisites() {
@@ -37,8 +40,11 @@ install_prerequisites() {
       python3-mysqldb libmariadb-dev mariadb-client-core-10.6 \
       mariadb-server-10.6
 
-   wget -O /usr/local/bin/traefik "${DOWNLOAD_URL}/generic-local/traefik"
+   wget "https://github.com/traefik/traefik/releases/download/${TRAEFIK_VERSION}/traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz"
-   chmod +x /usr/local/bin/traefik
+   tar zxvf "traefik_${TRAEFIK_VERSION}_linux_${ARCH}.tar.gz" traefik
+   chown root:root traefik
+   chmod 755 traefik
+   mv traefik /usr/local/bin
    setcap cap_net_bind_service+ep /usr/local/bin/traefik
 
    wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \

docker/Dockerfile

@@ -17,7 +17,7 @@ RUN apt-get update && \
     apt-get -y --no-install-recommends install \
        wget openssl sysstat php8.1-cli php8.1-cgi php8.1-mysql php8.1-fpm php8.1-zip php8.1-ldap \
        php8.1-gd php8.1-curl php8.1-xml php8.1-memcached catdoc unrtf poppler-utils nginx tnef sudo libzip4 \
-       libtre5 cron libmariadb-dev mariadb-client-core-10.6 python3 python3-mysqldb ca-certificates curl && \
+       libtre5 cron libmariadb-dev mariadb-client-core-10.6 python3 python3-mysqldb ca-certificates curl rsyslog && \
     wget https://repo.manticoresearch.com/manticore-repo.noarch.deb && \
     dpkg -i manticore-repo.noarch.deb && \
     rm -f manticore-repo.noarch.deb && \

docker/docker-compose.yaml

@@ -1,8 +1,7 @@
 version: "3"
 services:
-
   mysql:
-    image: mariadb:10.6
+    image: mariadb:11.1.2
     container_name: mysql
     restart: unless-stopped
     cap_drop:
@@ -17,43 +16,26 @@ services:
       - MYSQL_PASSWORD=piler123
       - MYSQL_RANDOM_ROOT_PASSWORD=yes
     command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
-    healthcheck:
-      test: mysql --user=piler --password=piler123 piler --execute "show tables"
-      interval: "60s"
-      timeout: "5s"
-      start_period: "15s"
-      retries: 3
     volumes:
       - db_data:/var/lib/mysql
-
-  memcached:
-    container_name: memcached
-    image: memcached:latest
-    restart: unless-stopped
-    cap_drop:
-      - ALL
-    command: -m 64
-
   piler:
     image: sutoj/piler:1.4.4
     container_name: piler
     init: true
     environment:
+      - MYSQL_HOSTNAME=mysql
       - MYSQL_DATABASE=piler
       - MYSQL_USER=piler
       - MYSQL_PASSWORD=piler123
-      - MYSQL_HOSTNAME=mysql
+      - PILER_HOSTNAME=cust1.acts.hu
-      - PILER_HOSTNAME=archive.example.com
+      - RT=1
-      - MEMCACHED_HOST=memcached
     ports:
       - "25:25"
       - "80:80"
-      - "443:443"
     volumes:
       - piler_etc:/etc/piler
-      - piler_var_store:/var/piler/store
+      - piler_manticore:/var/piler/manticore
-      - piler_var_manticore:/var/piler/manticore
+      - piler_store:/var/piler/store
-      - /dev/log:/dev/log
     healthcheck:
       test: curl -s smtp://localhost/
       interval: "60s"
@@ -68,11 +50,10 @@ services:
           memory: 512M
 
     depends_on:
-      - "memcached"
       - "mysql"
 
 volumes:
   db_data: {}
   piler_etc: {}
-  piler_var_store: {}
+  piler_manticore: {}
-  piler_var_manticore: {}
+  piler_store: {}

docker/start.sh

@@ -13,7 +13,7 @@ PILER_NGINX_CONF="${CONFIG_DIR}/piler-nginx.conf"
 SPHINX_CONF="${CONFIG_DIR}/manticore.conf"
 CONFIG_SITE_PHP="${CONFIG_DIR}/config-site.php"
 PILER_MY_CNF="${CONFIG_DIR}/.my.cnf"
-
+RT="${RT:-0}"
 
 error() {
    echo "ERROR:" "$*" 1>&2
@@ -126,6 +126,21 @@ fix_configs() {
        -e "s%MYSQL_USERNAME%${MYSQL_USER}%" \
        -e "s%MYSQL_PASSWORD%${MYSQL_PASSWORD}%" \
        -i "$SPHINX_CONF"
+
+   # Fixes for RT index
+
+   if [[ $RT -eq 1 ]]; then
+      sed -i "s/define('RT', 0)/define('RT', 1)/" "$SPHINX_CONF"
+      if ! grep "'RT'" "$CONFIG_SITE_PHP"; then
+         echo "\$config['RT'] = 1;" >> "$CONFIG_SITE_PHP"
+      fi
+
+      if ! grep "'SPHINX_MAIN_INDEX'" "$CONFIG_SITE_PHP"; then
+         echo "\$config['SPHINX_MAIN_INDEX'] = 'piler1';" >> "$CONFIG_SITE_PHP"
+      fi
+
+      sed -i "s%rtindex=.*%rtindex=1%" "$PILER_CONF"
+   fi
 }
 
 
@@ -173,11 +188,12 @@ start_services() {
    service cron start
    service php8.1-fpm start
    service nginx start
+   rsyslogd
 }
 
 
 start_piler() {
-   if [[ ! -f "${VOLUME_DIR}/manticore/main1.spp" ]]; then
+   if [[ $RT -eq 0 && ! -f "${VOLUME_DIR}/manticore/main1.spp" ]]; then
       log "main1.spp does not exist, creating index files"
       su -c "indexer --all --config ${SPHINX_CONF}" "$PILER_USER"
    fi

etc/example.conf

@@ -263,3 +263,12 @@ archive_address=
 ; rules. In other words if you decide to use the acl file, then
 ; everyone is not explicitly permitted is denied.
 smtp_access_list=0
+
+; max message size in bytes
+; piler-smtp will reject any message that's bigger than this number
+max_message_size=50000000
+
+; max memory in bytes piler-smtp uses for buffering messages
+; when this limit is exceeded, no new emails will be accepted
+; until the used memory decreases below this level
+max_smtp_memory=500000000

etc/manticore.conf.in

@@ -248,7 +248,8 @@ index note1
 searchd
 {
         listen                  = 127.0.0.1:9312
-        listen                  = 127.0.0.1:9306:mysql41
+        listen                  = 127.0.0.1:9306:mysql
+        listen                  = 127.0.0.1:9307:mysql_readonly
         log                     = /var/piler/manticore/manticore.log
         binlog_max_log_size     = 256M
         binlog_path             = /var/piler/manticore

src/archive.c

@@ -192,6 +192,7 @@ int retrieve_file_from_archive(char *filename, int mode, char **buffer, FILE *de
       } else {
       #if OPENSSL_VERSION_MAJOR >= 3
          OSSL_PROVIDER_load(NULL, "legacy");
+         OSSL_PROVIDER_load(NULL, "default");
       #endif
          rc = EVP_DecryptInit_ex(ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
       }

src/bdat.c

@@ -26,21 +26,21 @@ void reset_bdat_counters(struct smtp_session *session){
 }
 
 
-void get_bdat_size_to_read(struct smtp_session *session, char *buf){
+void get_bdat_size_to_read(struct smtp_session *session){
    char *p;
 
    session->bdat_bytes_to_read = 0;
 
    session->protocol_state = SMTP_STATE_BDAT;
 
-   p = strcasestr(buf, " LAST");
+   p = strcasestr(session->buf, " LAST");
    if(p){
       *p = '\0';
    }
 
    // determine the size to be read
 
-   p = strchr(buf, ' ');
+   p = strchr(session->buf, ' ');
    if(p){
       session->bdat_bytes_to_read = atoi(p);
       if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "fd=%d: BDAT len=%d", session->net.socket, session->bdat_bytes_to_read);

src/cfg.c

@@ -23,6 +23,11 @@ int int_parser(char *src, int *target){
    return 0;
 };
 
+int uint64_parser(char *src, uint64 *target){
+   *target = strtoull(src, (char**)NULL, 10);
+   return 0;
+}
+
 struct _parse_rule {
    char *name;
    char *type;
@@ -63,7 +68,9 @@ struct _parse_rule config_parse_rules[] =
    { "listen_port", "integer", (void*) int_parser, offsetof(struct config, listen_port), "25", sizeof(int)},
    { "locale", "string", (void*) string_parser, offsetof(struct config, locale), "", MAXVAL-1},
    { "max_connections", "integer", (void*) int_parser, offsetof(struct config, max_connections), "64", sizeof(int)},
+   { "max_message_size", "integer", (void*) int_parser, offsetof(struct config, max_message_size), "50000000", sizeof(int)},
    { "max_requests_per_child", "integer", (void*) int_parser, offsetof(struct config, max_requests_per_child), "10000", sizeof(int)},
+   { "max_smtp_memory", "uint64", (void*) uint64_parser, offsetof(struct config, max_smtp_memory), "500000000", sizeof(uint64)},
    { "memcached_servers", "string", (void*) string_parser, offsetof(struct config, memcached_servers), "127.0.0.1", MAXVAL-1},
    { "memcached_to_db_interval", "integer", (void*) int_parser, offsetof(struct config, memcached_to_db_interval), "900", sizeof(int)},
    { "memcached_ttl", "integer", (void*) int_parser, offsetof(struct config, memcached_ttl), "86400", sizeof(int)},
@@ -217,6 +224,7 @@ struct config read_config(char *configfile){
 void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
    int j;
    float f;
+   uint64 u;
    char *p, buf[MAXVAL];
 
    p = (char*)cfg + rules[i].offset;
@@ -225,6 +233,10 @@ void print_config_item(struct config *cfg, struct _parse_rule *rules, int i){
       memcpy((char*)&j, p, sizeof(int));
       printf("%s=%d\n", rules[i].name, j);
    }
+   else if(strcmp(rules[i].type, "uint64") == 0){
+      memcpy((char*)&u, p, sizeof(uint64));
+      printf("%s=%llu\n", rules[i].name, u);
+   }
    else if(strcmp(rules[i].type, "float") == 0){
       memcpy((char*)&f, p, sizeof(float));
       printf("%s=%.4f\n", rules[i].name, f);

src/cfg.h

@@ -110,6 +110,9 @@ struct config {
    int debug;
 
    int smtp_access_list;
+
+   int max_message_size;
+   uint64 max_smtp_memory;
 };
 
 

src/config.h

@@ -9,6 +9,8 @@
 #include "piler-config.h"
 #include "params.h"
 
+typedef unsigned long long uint64;
+
 #define BUILD 1001
 
 #define HOSTID "mailarchiver"
@@ -30,6 +32,7 @@
 #define SMALLBUFSIZE 512
 #define BIGBUFSIZE 131072
 #define REALLYBIGBUFSIZE 524288
+#define SMTPBUFSIZE 2048000
 #define TINYBUFSIZE 128
 #define MAXVAL 256
 #define RANDOM_POOL "/dev/urandom"

src/defs.h

@@ -403,7 +403,6 @@ struct smtp_session {
    char ttmpfile[SMALLBUFSIZE];
    char mailfrom[SMALLBUFSIZE];
    char rcptto[MAX_RCPT_TO][SMALLBUFSIZE];
-   char buf[MAXBUFSIZE];
    char remote_host[INET6_ADDRSTRLEN+1];
    char nullbyte;
    time_t lasttime;
@@ -411,13 +410,17 @@ struct smtp_session {
    int slot;
    int fd;
    int bad;
-   int buflen;
-   int last_data_char;
    int tot_len;
    int bdat_bytes_to_read;
    int num_of_rcpt_to;
    struct config *cfg;
    struct net net;
+   int max_message_size;
+   char *buf;
+   int buflen;
+   int bufsize;
+   int too_big;
+   int mail_size;
 };
 
 struct tls_protocol {

src/imap.c

@@ -105,7 +105,12 @@ int connect_to_imap_server(struct data *data){
 
    /* imap cmd: LOGIN */
 
-   snprintf(buf, sizeof(buf)-1, "A%d LOGIN %s \"%s\"\r\n", data->import->seq, data->import->username, data->import->password);
+   if(strcmp(data->import->username, "ZIMBRA") == 0){
+      snprintf(buf, sizeof(buf)-1, "A%d AUTHENTICATE PLAIN %s\r\n", data->import->seq, data->import->password);
+   }
+   else {
+      snprintf(buf, sizeof(buf)-1, "A%d LOGIN %s \"%s\"\r\n", data->import->seq, data->import->username, data->import->password);
+   }
 
    write1(data->net, buf, strlen(buf));
    if(read_response(buf, sizeof(buf), data) == 0){

src/message.c

@@ -50,7 +50,7 @@ int store_index_data(struct session_data *sdata, struct parser_state *state, str
       char a[4*MAXBUFSIZE+4*SMALLBUFSIZE];
       char *query=NULL;
 
-      snprintf(a, sizeof(a)-1, "INSERT INTO %s (id, arrived, sent, size, direction, folder, attachments, attachment_types, sender, rcpt, senderdomain, rcptdomain, subject, body) VALUES (%llu,%ld,%ld,%d,%d,%d,%d,'%s','", cfg->sphxdb, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, data->folder, state->n_attachments, sdata->attachments);
+      snprintf(a, sizeof(a)-1, "REPLACE INTO %s (id, arrived, sent, size, direction, folder, attachments, attachment_types, sender, rcpt, senderdomain, rcptdomain, subject, body) VALUES (%llu,%ld,%ld,%d,%d,%d,%d,'%s','", cfg->sphxdb, id, sdata->now, sdata->sent, sdata->tot_len, sdata->direction, data->folder, state->n_attachments, sdata->attachments);
 
       int ret = append_string_to_buffer(&query, a);
 

src/misc.c

@@ -781,3 +781,29 @@ int append_string_to_buffer(char **buffer, char *str){
 
    return 0;
 }
+
+
+int get_size_from_smtp_mail_from(char *s){
+   int size=0;
+   char *p;
+
+   p = strcasestr(s, "SIZE=");
+   if(p){
+      p += strlen("SIZE=");
+      char *q = p;
+      for(; *q; q++){
+         if(isspace(*q)) break;
+      }
+
+      // We extract max. 9 characters, which is just under 1GB
+      // and not overflowing an int variable
+      if(q - p <= 9){
+         char c = *q;
+         *q = '\0';
+         size = atoi(p);
+         *q = c;
+      }
+   }
+
+   return size;
+}

src/misc.h

@@ -55,5 +55,6 @@ int init_ssl_to_server(struct data *data);
 #endif
 
 int append_string_to_buffer(char **buffer, char *str);
+int get_size_from_smtp_mail_from(char *s);
 
 #endif /* _MISC_H */

src/piler-smtp.c

@@ -33,6 +33,7 @@ extern int optind;
 struct epoll_event event, *events=NULL;
 int num_connections = 0;
 int listenerfd = -1;
+int loglevel = 1;
 
 char *configfile = CONFIG_FILE;
 struct config cfg;
@@ -48,6 +49,7 @@ void usage(){
    printf("    -d                                Fork to the background\n");
    printf("    -v                                Return the version and build number\n");
    printf("    -V                                Return the version and some build parameters\n");
+   printf("    -L <log level>                    Set the log level: 1-5\n");
 
    exit(0);
 }
@@ -138,16 +140,20 @@ int main(int argc, char **argv){
    int client_len = sizeof(struct sockaddr_storage);
    ssize_t readlen;
    struct sockaddr_storage client_address;
-   char readbuf[BIGBUFSIZE];
+   char readbuf[REALLYBIGBUFSIZE];
    int efd;
 
-   while((i = getopt(argc, argv, "c:dvVh")) > 0){
+   while((i = getopt(argc, argv, "c:L:dvVh")) > 0){
       switch(i){
 
         case 'c' :
                    configfile = optarg;
                    break;
 
+        case 'L':
+                   loglevel = atoi(optarg);
+                   break;
+
         case 'd' :
                    daemonise = 1;
                    break;
@@ -321,6 +327,7 @@ int main(int argc, char **argv){
                   break;
                }
 
+               readbuf[readlen] = '\0';
                handle_data(session, &readbuf[0], readlen, &cfg);
 
                if(session->protocol_state == SMTP_STATE_BDAT && session->bad == 1){

src/session.c

@@ -9,6 +9,36 @@
 int get_session_slot(struct smtp_session **sessions, int max_connections);
 void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg);
 
+#define GOT_CRLF_DOT_CRLF(p) *p == '\r' && *(p+1) == '\n' && *(p+2) == '.' && *(p+3) == '\r' && *(p+4) == '\n' ? 1 : 0
+
+uint64 get_sessions_total_memory(struct smtp_session **sessions, int max_connections){
+   uint64 total = 0;
+
+   for(int i=0; i<max_connections; i++){
+      if(sessions[i]) total += sessions[i]->bufsize;
+   }
+
+   return total;
+}
+
+
+/*
+ * If the sending party sets the email size when it sends the "mail from"
+ * part in the smtp transaction, eg. MAIL FROM:<jajaja@akakak.lo> size=509603
+ * then piler-smtp could know the email size in advance and could do
+ * a better estimate on the allowed number of smtp sessions.
+ */
+
+uint64 get_sessions_total_expected_mail_size(struct smtp_session **sessions, int max_connections){
+   uint64 total = 0;
+
+   for(int i=0; i<max_connections; i++){
+      if(sessions[i]) total += sessions[i]->mail_size;
+   }
+
+   return total;
+}
+
 
 int start_new_session(struct smtp_session **sessions, int socket, int *num_connections, struct smtp_acl *smtp_acl[], char *client_addr, struct config *cfg){
    int slot;
@@ -31,6 +61,21 @@ int start_new_session(struct smtp_session **sessions, int socket, int *num_conne
       return -1;
    }
 
+   /*
+    * We are under the max_smtp_memory threshold
+    */
+
+   uint64 expected_total_mail_size = get_sessions_total_expected_mail_size(sessions, cfg->max_connections);
+   uint64 total_memory = get_sessions_total_memory(sessions, cfg->max_connections);
+
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "DEBUG: total smtp memory allocated: %llu, expected total size: %llu", total_memory, expected_total_mail_size);
+
+   if(total_memory > cfg->max_smtp_memory || expected_total_mail_size > cfg->max_smtp_memory){
+      syslog(LOG_PRIORITY, "ERROR: too much memory consumption: %llu", total_memory);
+      send(socket, SMTP_RESP_451_ERR_TOO_MANY_REQUESTS, strlen(SMTP_RESP_451_ERR_TOO_MANY_REQUESTS), 0);
+      return -1;
+   }
+
    slot = get_session_slot(sessions, cfg->max_connections);
 
    if(slot >= 0 && sessions[slot] == NULL){
@@ -84,11 +129,8 @@ struct smtp_session *get_session_by_socket(struct smtp_session **sessions, int m
 
 
 void init_smtp_session(struct smtp_session *session, int slot, int sd, char *client_addr, struct config *cfg){
-   int i;
-
    session->slot = slot;
 
-   session->buflen = 0;
    session->protocol_state = SMTP_STATE_INIT;
 
    session->cfg = cfg;
@@ -100,27 +142,24 @@ void init_smtp_session(struct smtp_session *session, int slot, int sd, char *cli
    session->net.ssl = NULL;
 
    session->nullbyte = 0;
-   session->last_data_char = 0;
 
-   session->fd = -1;
-
-   memset(session->mailfrom, 0, SMALLBUFSIZE);
-
-   session->num_of_rcpt_to = 0;
-   for(i=0; i<MAX_RCPT_TO; i++) memset(session->rcptto[i], 0, SMALLBUFSIZE);
-
-   memset(session->buf, 0, MAXBUFSIZE);
    snprintf(session->remote_host, sizeof(session->remote_host)-1, "%s", client_addr);
 
-   reset_bdat_counters(session);
+   session->buf = NULL;
+   session->buflen = 0;
+   session->bufsize = 0;
 
-   time(&(session->lasttime));
+   reset_smtp_session(session);
 }
 
 
 void free_smtp_session(struct smtp_session *session){
    if(session){
 
+      if(session->buf != NULL){
+         free(session->buf);
+      }
+
       if(session->net.use_ssl == 1){
          SSL_shutdown(session->net.ssl);
          SSL_free(session->net.ssl);
@@ -160,71 +199,126 @@ void tear_down_session(struct smtp_session **sessions, int slot, int *num_connec
 }
 
 
+inline int get_last_newline_position(char *buf, int buflen){
+   int i;
+
+   for(i=buflen; i>0; i--){
+      if(*(buf+i) == '\n'){
+         i++;
+         break;
+      }
+   }
+
+   return i;
+}
+
+
+void flush_buffer(struct smtp_session *session){
+   // In the DATA phase skip the 1st character if it's a dot (.)
+   // and there are more characters before the trailing CR-LF
+   //
+   // See https://www.ietf.org/rfc/rfc5321.html#section-4.5.2 for more
+   for(int i=0; i<session->buflen; i++){
+      if(*(session->buf+i) == '\n' && *(session->buf+i+1) == '.' && *(session->buf+i+2) == '.'){
+         int dst = i + 2;
+         int src = dst + 1;
+         int l = session->buflen - src;
+         memmove(session->buf + dst, session->buf + src, l);
+         session->buflen -= 1;
+      }
+   }
+
+   // Exclude the trailing \r\n.\r\n sequence
+
+   session->buflen -= 5;
+
+   if(write(session->fd, session->buf, session->buflen) != session->buflen){
+      session->bad = 1;
+      syslog(LOG_PRIORITY, "ERROR (line: %d) %s: failed to write %d bytes", __LINE__, __func__, session->buflen);
+   }
+
+   session->tot_len = session->buflen;
+}
+
+
 void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg){
-   int puflen, rc, nullbyte;
+   // Update lasttime if we have something to process
-   char *p, copybuf[BIGBUFSIZE+MAXBUFSIZE], puf[MAXBUFSIZE];
+   time(&(session->lasttime));
 
-   // if there's something in the saved buffer, then let's merge them
+   if(session->protocol_state == SMTP_STATE_BDAT){
+      process_bdat(session, readbuf, readlen, cfg);
+      return;
+   }
 
-   int remaininglen = readlen + session->buflen;
+   // realloc memory if the new chunk doesn't fit in
 
-   if(session->buflen > 0){
+   if(session->buflen + readlen + 10 > session->bufsize){
-      memset(copybuf, 0, sizeof(copybuf));
+      // Handle if the current memory allocation for this email is above the max_message_size threshold
 
-      memcpy(copybuf, session->buf, session->buflen);
+      if(session->buflen > cfg->max_message_size){
-      memcpy(&copybuf[session->buflen], readbuf, readlen);
+         if(session->too_big == 0) syslog(LOG_PRIORITY, "ERROR: too big email: %d vs %d", session->buflen, cfg->max_message_size);
+         session->bad = 1;
+         session->too_big = 1;
+      }
 
+      if(session->bad == 0){
+         char *q = realloc(session->buf, session->bufsize + SMTPBUFSIZE);
+         if(q){
+            session->buf = q;
+            memset(session->buf+session->bufsize, 0, SMTPBUFSIZE);
+            session->bufsize += SMTPBUFSIZE;
+         } else {
+            syslog(LOG_PRIORITY, "ERROR: realloc %s %s %d", session->ttmpfile, __func__, __LINE__);
+            session->bad = 1;
+         }
+      }
+   }
+
+   // process smtp command
+   if(session->protocol_state != SMTP_STATE_DATA){
+
+      // We got ~2 MB of garbage and no valid smtp command
+      // Terminate the connection
+      if(session->buflen + readlen > SMTPBUFSIZE - 10){
+         session->bad = 1;
+      }
+
+      // We are at the beginning of the smtp transaction
+      if(session->bad == 1){
+         write1(&(session->net), SMTP_RESP_451_ERR, strlen(SMTP_RESP_451_ERR));
+         syslog(LOG_PRIORITY, "ERROR: sent 451 temp error back to client %s", session->ttmpfile);
+         return;
+      }
+
+      //printf("got %d *%s*\n", readlen, readbuf);
+
+      memcpy(session->buf + session->buflen, readbuf, readlen);
+      session->buflen += readlen;
+
+      int pos = get_last_newline_position(session->buf, session->buflen);
+
+      if(pos < readlen) return; // no complete command
+
+      process_smtp_command(session, cfg);
+
+      memset(session->buf, 0, session->bufsize);
       session->buflen = 0;
-      memset(session->buf, 0, MAXBUFSIZE);
 
-      p = &copybuf[0];
+      return;
-   }
-   else {
-      readbuf[readlen] = 0;
-      p = readbuf;
    }
 
+   if(session->bad == 0){
+      memcpy(session->buf + session->buflen, readbuf, readlen);
+      session->buflen += readlen;
 
-   do {
+      char *p = session->buf + session->buflen - 5;
-      puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf)-1, &rc, &nullbyte);
+      if(session->buflen >= 5 && GOT_CRLF_DOT_CRLF(p)){
-      p += puflen;
+         flush_buffer(session);
-      remaininglen -= puflen;
+         process_command_period(session);
-
-      if(nullbyte){
-         session->nullbyte = 1;
       }
-
+   } else if(strstr(readbuf, "\r\n.\r\n")){
-      // complete line: rc == OK and puflen > 0
+      process_command_period(session);
-      // incomplete line with something in the buffer: rc == ERR and puflen > 0
+   }
-
-      if(puflen > 0){
-         // Update lasttime if we have a line to process
-         time(&(session->lasttime));
-
-         // Save incomplete line to buffer
-         if(rc == ERR){
-            memcpy(session->buf, puf, puflen);
-            session->buflen = puflen;
-         }
-
-         // We have a complete line to process
-
-         if(rc == OK){
-            if(session->protocol_state == SMTP_STATE_BDAT){
-               process_bdat(session, puf, puflen, cfg);
-            }
-            else if(session->protocol_state == SMTP_STATE_DATA){
-               sig_block(SIGALRM);
-               process_data(session, puf, puflen);
-               sig_unblock(SIGALRM);
-            }
-            else {
-               process_smtp_command(session, puf, cfg);
-            }
-         }
-      }
-
-   } while(puflen > 0);
 
 }
 

src/smtp.c

@@ -15,59 +15,59 @@
 #include "smtp.h"
 
 
-void process_smtp_command(struct smtp_session *session, char *buf, struct config *cfg){
+void process_smtp_command(struct smtp_session *session, struct config *cfg){
    char response[SMALLBUFSIZE];
 
-   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "got on fd=%d: *%s*", session->net.socket, buf);
+   if(session->cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "got on fd=%d: *%s*", session->net.socket, session->buf);
 
-   if(strncasecmp(buf, SMTP_CMD_HELO, strlen(SMTP_CMD_HELO)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_HELO, strlen(SMTP_CMD_HELO)) == 0){
       process_command_helo(session, response, sizeof(response));
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_EHLO, strlen(SMTP_CMD_EHLO)) == 0 ||
+   if(strncasecmp(session->buf, SMTP_CMD_EHLO, strlen(SMTP_CMD_EHLO)) == 0 ||
-         strncasecmp(buf, LMTP_CMD_LHLO, strlen(LMTP_CMD_LHLO)) == 0){
+         strncasecmp(session->buf, LMTP_CMD_LHLO, strlen(LMTP_CMD_LHLO)) == 0){
-      process_command_ehlo_lhlo(session, response, sizeof(response));
+      process_command_ehlo_lhlo(session, response, sizeof(response), cfg);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_HELP, strlen(SMTP_CMD_HELP)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_HELP, strlen(SMTP_CMD_HELP)) == 0){
       send_smtp_response(session, SMTP_RESP_221_PILER_SMTP_OK);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_MAIL_FROM, strlen(SMTP_CMD_MAIL_FROM)) == 0){
-      process_command_mail_from(session, buf);
+      process_command_mail_from(session);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_RCPT_TO, strlen(SMTP_CMD_RCPT_TO)) == 0){
-      process_command_rcpt_to(session, buf, cfg);
+      process_command_rcpt_to(session, cfg);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_DATA, strlen(SMTP_CMD_DATA)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_DATA, strlen(SMTP_CMD_DATA)) == 0){
       process_command_data(session, cfg);
       return;
    }
 
    /* Support only BDAT xxxx LAST command */
-   if(session->cfg->enable_chunking == 1 && strncasecmp(buf, SMTP_CMD_BDAT, strlen(SMTP_CMD_BDAT)) == 0 && strcasestr(buf, "LAST")){
+   if(session->cfg->enable_chunking == 1 && strncasecmp(session->buf, SMTP_CMD_BDAT, strlen(SMTP_CMD_BDAT)) == 0 && strcasestr(session->buf, "LAST")){
-      get_bdat_size_to_read(session, buf);
+      get_bdat_size_to_read(session);
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_QUIT, strlen(SMTP_CMD_QUIT)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_QUIT, strlen(SMTP_CMD_QUIT)) == 0){
       process_command_quit(session, response, sizeof(response));
       return;
    }
 
-   if(strncasecmp(buf, SMTP_CMD_RESET, strlen(SMTP_CMD_RESET)) == 0){
+   if(strncasecmp(session->buf, SMTP_CMD_RESET, strlen(SMTP_CMD_RESET)) == 0){
       process_command_reset(session);
       return;
    }
 
-   if(session->cfg->tls_enable == 1 && strncasecmp(buf, SMTP_CMD_STARTTLS, strlen(SMTP_CMD_STARTTLS)) == 0 && session->net.use_ssl == 0){
+   if(session->cfg->tls_enable == 1 && strncasecmp(session->buf, SMTP_CMD_STARTTLS, strlen(SMTP_CMD_STARTTLS)) == 0 && session->net.use_ssl == 0){
       process_command_starttls(session);
       return;
    }
@@ -76,41 +76,6 @@ void process_smtp_command(struct smtp_session *session, char *buf, struct config
 }
 
 
-void process_data(struct smtp_session *session, char *buf, int buflen){
-   if(session->last_data_char == '\n' && strcmp(buf, ".\r\n") == 0){
-      process_command_period(session);
-   }
-   else {
-      // write line to file
-      int written=0, n_writes=0;
-
-      // In the DATA phase skip the 1st character if it's a dot (.)
-      // and there are more characters before the trailing CR-LF
-      //
-      // See https://www.ietf.org/rfc/rfc5321.html#section-4.5.2 for more.
-
-      int dotstuff = 0;
-      if(*buf == '.' && buflen > 1 && *(buf+1) != '\r' && *(buf+1) != '\n') dotstuff = 1;
-
-      while(written < buflen) {
-         int len = write(session->fd, buf+dotstuff+written, buflen-dotstuff-written);
-
-         n_writes++;
-
-         if(len > 0){
-            if(len != buflen-dotstuff) syslog(LOG_PRIORITY, "WARN: partial write: %d/%d bytes (round: %d)", len, buflen-dotstuff, n_writes);
-            written += len + dotstuff;
-            session->tot_len += len;
-            dotstuff = 0;
-         }
-         else syslog(LOG_PRIORITY, "ERROR (line: %d) process_data(): written %d bytes", __LINE__, len);
-      }
-   }
-
-   session->last_data_char = buf[buflen-1];
-}
-
-
 void wait_for_ssl_accept(struct smtp_session *session){
    int rc;
 
@@ -152,7 +117,7 @@ void process_command_helo(struct smtp_session *session, char *buf, int buflen){
 }
 
 
-void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen){
+void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen, struct config *cfg){
    char extensions[SMALLBUFSIZE];
 
    memset(extensions, 0, sizeof(extensions));
@@ -163,7 +128,9 @@ void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int bufl
    if(session->net.use_ssl == 0 && session->cfg->tls_enable == 1) snprintf(extensions, sizeof(extensions)-1, "%s", SMTP_EXTENSION_STARTTLS);
    if(session->cfg->enable_chunking == 1) strncat(extensions, SMTP_EXTENSION_CHUNKING, sizeof(extensions)-strlen(extensions)-2);
 
-   snprintf(buf, buflen-1, SMTP_RESP_250_EXTENSIONS, session->cfg->hostid, extensions);
+   //#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n%s250-SIZE %d\r\n250 8BITMIME\r\n"
+
+   snprintf(buf, buflen-1, SMTP_RESP_250_EXTENSIONS, session->cfg->hostid, cfg->max_message_size, extensions);
 
    send_smtp_response(session, buf);
 }
@@ -234,26 +201,29 @@ void process_command_starttls(struct smtp_session *session){
 }
 
 
-void process_command_mail_from(struct smtp_session *session, char *buf){
+void process_command_mail_from(struct smtp_session *session){
+   memset(session->mailfrom, 0, SMALLBUFSIZE);
+
    if(session->protocol_state != SMTP_STATE_HELO && session->protocol_state != SMTP_STATE_PERIOD && session->protocol_state != SMTP_STATE_BDAT){
       send(session->net.socket, SMTP_RESP_503_ERR, strlen(SMTP_RESP_503_ERR), 0);
    }
    else {
-      memset(&(session->ttmpfile[0]), 0, SMALLBUFSIZE);
-      make_random_string((unsigned char*)&(session->ttmpfile[0]), QUEUE_ID_LEN);
       session->protocol_state = SMTP_STATE_MAIL_FROM;
 
-      extractEmail(buf, session->mailfrom);
+      extractEmail(session->buf, session->mailfrom);
 
-      reset_bdat_counters(session);
+      int mailsize = get_size_from_smtp_mail_from(session->buf);
-      session->tot_len = 0;
+
+      reset_smtp_session(session);
+
+      session->mail_size = mailsize;
 
       send_smtp_response(session, SMTP_RESP_250_OK);
    }
 }
 
 
-void process_command_rcpt_to(struct smtp_session *session, char *buf, struct config *cfg){
+void process_command_rcpt_to(struct smtp_session *session, struct config *cfg){
 
    if(session->protocol_state == SMTP_STATE_MAIL_FROM || session->protocol_state == SMTP_STATE_RCPT_TO){
 
@@ -262,7 +232,7 @@ void process_command_rcpt_to(struct smtp_session *session, char *buf, struct con
       session->protocol_state = SMTP_STATE_RCPT_TO;
 
       if(session->num_of_rcpt_to < MAX_RCPT_TO){
-         extractEmail(buf, session->rcptto[session->num_of_rcpt_to]);
+         extractEmail(session->buf, session->rcptto[session->num_of_rcpt_to]);
 
          // Check if we should accept archive_address only
          if(cfg->archive_address[0] && !strstr(cfg->archive_address, session->rcptto[session->num_of_rcpt_to])){
@@ -322,13 +292,23 @@ void process_command_period(struct smtp_session *session){
 
    syslog(LOG_PRIORITY, "received: %s, from=%s, size=%d, client=%s, fd=%d, fsync=%ld", session->ttmpfile, session->mailfrom, session->tot_len, session->remote_host, session->net.socket, tvdiff(tv2, tv1));
 
-   move_email(session);
+   if(session->bad == 1 || session->too_big == 1){
+      snprintf(buf, sizeof(buf)-1, SMTP_RESP_451_ERR);
+      unlink(session->ttmpfile);
+      syslog(LOG_PRIORITY, "ERROR: problem in processing, removing %s", session->ttmpfile);
+      if(session->too_big)
+         snprintf(buf, sizeof(buf)-1, "%s", SMTP_RESP_552_ERR_TOO_BIG_EMAIL);
+      else
+         snprintf(buf, sizeof(buf)-1, "%s", SMTP_RESP_451_ERR);
+   } else {
+      move_email(session);
+      snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
+   }
 
-   snprintf(buf, sizeof(buf)-1, "250 OK <%s>\r\n", session->ttmpfile);
+   if(session->buf){
-
+      memset(session->buf, 0, session->bufsize);
-   session->buflen = 0;
+      session->buflen = 0;
-   session->last_data_char = 0;
+   }
-   memset(session->buf, 0, sizeof(session->buf));
 
    send_smtp_response(session, buf);
 }
@@ -344,15 +324,32 @@ void process_command_quit(struct smtp_session *session, char *buf, int buflen){
 
 
 void process_command_reset(struct smtp_session *session){
+   reset_smtp_session(session);
    send_smtp_response(session, SMTP_RESP_250_OK);
-
-   session->tot_len = 0;
-   session->fd = -1;
    session->protocol_state = SMTP_STATE_HELO;
-   session->last_data_char = 0;
+}
+
+
+void reset_smtp_session(struct smtp_session *session){
+   session->tot_len = 0;
+   session->mail_size = 0;
+   session->too_big = 0;
+   session->bad = 0;
+
+   session->fd = -1;
+
+   session->num_of_rcpt_to = 0;
+   for(int i=0; i<MAX_RCPT_TO; i++) memset(session->rcptto[i], 0, SMALLBUFSIZE);
 
    reset_bdat_counters(session);
 
+   time(&(session->lasttime));
+
    memset(&(session->ttmpfile[0]), 0, SMALLBUFSIZE);
-   make_random_string((unsigned char *)&(session->ttmpfile[0]), QUEUE_ID_LEN);
+   make_random_string((unsigned char*)&(session->ttmpfile[0]), QUEUE_ID_LEN);
+
+   if(session->buf){
+      memset(session->buf, 0, session->bufsize);
+   }
+   session->buflen = 0;
 }

src/smtp.h

@@ -7,22 +7,24 @@
 
 #include <piler.h>
 
-void process_smtp_command(struct smtp_session *session, char *buf, struct config *cfg);
+void process_smtp_command(struct smtp_session *session, struct config *cfg);
 void process_data(struct smtp_session *session, char *buf, int buflen);
 
 void send_smtp_response(struct smtp_session *session, char *buf);
 void process_command_helo(struct smtp_session *session, char *buf, int buflen);
-void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen);
+void process_command_ehlo_lhlo(struct smtp_session *session, char *buf, int buflen, struct config *cfg);
 void process_command_quit(struct smtp_session *session, char *buf, int buflen);
 void process_command_reset(struct smtp_session *session);
-void process_command_mail_from(struct smtp_session *session, char *buf);
+void process_command_mail_from(struct smtp_session *session);
-void process_command_rcpt_to(struct smtp_session *session, char *buf, struct config *cfg);
+void process_command_rcpt_to(struct smtp_session *session, struct config *cfg);
 void process_command_data(struct smtp_session *session, struct config *cfg);
 void process_command_period(struct smtp_session *session);
 void process_command_starttls(struct smtp_session *session);
 
 void reset_bdat_counters(struct smtp_session *session);
-void get_bdat_size_to_read(struct smtp_session *session, char *buf);
+void get_bdat_size_to_read(struct smtp_session *session);
 void process_bdat(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg);
 
+void reset_smtp_session(struct smtp_session *session);
+
 #endif

src/smtpcodes.h

@@ -38,7 +38,7 @@
 #define SMTP_RESP_220_READY_TO_START_TLS "220 Ready to start TLS\r\n"
 #define SMTP_RESP_221_GOODBYE "221 %s Goodbye\r\n"
 #define SMTP_RESP_250_OK "250 Ok\r\n"
-#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n250-PIPELINING\r\n%s250-SIZE\r\n250 8BITMIME\r\n"
+#define SMTP_RESP_250_EXTENSIONS "250-%s\r\n250-SIZE %d\r\n%s250 8BITMIME\r\n"
 
 #define SMTP_EXTENSION_STARTTLS "250-STARTTLS\r\n"
 #define SMTP_EXTENSION_CHUNKING "250-CHUNKING\r\n"
@@ -51,6 +51,7 @@
 #define SMTP_RESP_421_ERR_TMP "421 %s service not available\r\n"
 #define SMTP_RESP_421_ERR_WRITE_FAILED "421 writing queue file failed\r\n"
 #define SMTP_RESP_421_ERR_ALL_PORTS_ARE_BUSY "421 All server ports are busy\r\n"
+#define SMTP_RESP_451_ERR_TOO_MANY_REQUESTS "451 Too many requests, try again later\r\n"
 #define SMTP_RESP_451_ERR "451 Error in processing, try again later\r\n"
 #define SMTP_RESP_454_ERR_TLS_TEMP_ERROR "454 TLS not available currently\r\n"
 
@@ -59,6 +60,7 @@
 #define SMTP_RESP_550_ERR_INVALID_RECIPIENT "550 Invalid recipient\r\n"
 #define SMTP_RESP_550_ERR_YOU_ARE_BANNED_BY_LOCAL_POLICY "550 You are banned by local policy\r\n"
 #define SMTP_RESP_550_ERR "550 Service currently unavailable\r\n"
+#define SMTP_RESP_552_ERR_TOO_BIG_EMAIL "552 Too big email\r\n"
 
 
 // LMTP commands

src/tai.h

@@ -5,12 +5,12 @@
 #ifndef _TAI_H
    #define _TAI_H
 
+#include "config.h"
+
 #define TAI_PACK 8
 #define TAIA_PACK 16
 #define TIMESTAMP 25
 
-typedef unsigned long long uint64;
-
 struct tai {
    uint64 x;
 };

tests/cases/05-smtp.inc

@@ -6,23 +6,23 @@ case1() {
 
    setup
 
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox"  --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox2" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Inbox2"  --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet"  --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet2" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/Levelszemet2"  --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam0" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam0"  --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam1" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam1"  --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam2" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam2"  --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/journal" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/journal"  --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/deduptest" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/deduptest"  --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/special" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/special"  --no-counter
-   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus"  --no-counter
 
 
-   wait_until_emails_are_processed "piler1" 3019
+   wait_until_emails_are_processed "piler1" 3020
    [[ $RT -eq 1 ]] || docker exec "piler1" su piler -c /usr/libexec/piler/indexer.delta.sh 2>/dev/null
 
-   count_status_values 3019 2908 111 0
+   count_status_values 3020 2909 111 0
 
    test_retrieved_messages_are_the_same "piler1" "piler"
 

tests/setup.inc

@@ -27,13 +27,13 @@ launch_containers() {
 
    destroy_containers
 
-   log "running docker-compose up"
+   log "running docker compose up"
 
-   docker-compose -f "$composefile" up -d
+   docker compose -f "$composefile" up -d
 
    wait_until_container_is_healthy "piler1"
 
-   docker-compose -f "$composefile" ps
+   docker compose -f "$composefile" ps
 
    popd
 }

unit_tests/run.sh

@@ -24,6 +24,13 @@ setup_mysql() {
    mysql -u piler -ppiler123 piler1 < ../util/db-mysql.sql
 }
 
+run_smtp_tests() {
+   mkdir -p /var/piler/store/00/piler /var/piler/tmp /var/piler/manticore
+   chown -R piler:piler /var/piler/
+   ../src/piler-smtp -L 5 -d
+   ./smtp -s 127.0.0.1
+}
+
 if [[ -v BUILD_NUMBER ]]; then
    setup_mysql
 fi
@@ -37,3 +44,5 @@ fi
 ./check_hash
 ./check_decoder
 ./check_attachments
+
+if [[ -v BUILD_NUMBER ]]; then run_smtp_tests; fi

unit_tests/smtp.c

@@ -4,11 +4,13 @@
 
 #include "test.h"
 
-
 extern char *optarg;
 extern int optind;
 
-char *testmessage = "From: aaa@aaa.fu\nTo: bela@aaa.fu\nMessage-Id: ajajajaja\nSubject: this is a test\n\nAaaaaa.";
+char *testmessage = "From: aaa@aaa.fu\r\nTo: bela@aaa.fu\r\nMessage-Id: ajajajaja\r\nSubject: this is a test\r\n\r\nAaaaaa";
+char *testmessage2 = " and the last line\r\n";
+
+char *recipient = "aaa@worker0";
 
 int helo = 0; // 0=HELO, 1=EHLO
 
@@ -16,12 +18,24 @@ void usage(){
    printf("\nusage: smtp\n\n");
    printf("    -s <smtp server>                  SMTP server\n");
    printf("    -p <smtp port>                    SMTP port (25)\n");
+   printf("    -r <recipient>                    Envelope recipient\n");
    printf("    -t <timeout>                      Timeout in sec (10)\n");
 
    exit(0);
 }
 
 
+int countreplies(char *s){
+   int replies = 0;
+
+   for(; *s; s++){
+      if(*s == '\n') replies++;
+   }
+
+   return replies;
+}
+
+
 void connect_to_smtp_server(char *server, int port, struct data *data){
    int rc;
    char port_string[8], buf[MAXBUFSIZE];
@@ -53,14 +67,15 @@ void connect_to_smtp_server(char *server, int port, struct data *data){
    }
 
    recvtimeoutssl(data->net, buf, sizeof(buf));
-   printf("rcvd: %s", buf);
 
 ENDE:
    freeaddrinfo(res);
 }
 
 
-void send_smtp_command(struct net *net, char *cmd, char *buf, int buflen){
+void send_smtp_command(struct net *net, char *cmd, char *buf, int buflen, int expectedreplies){
+   int tot=0;
+
    if(net == NULL || cmd == NULL) return;
 
    if(net->socket == -1){
@@ -68,25 +83,32 @@ void send_smtp_command(struct net *net, char *cmd, char *buf, int buflen){
       return;
    }
 
-   printf("sent: %s", cmd);
    write1(net, cmd, strlen(cmd));
-   recvtimeoutssl(net, buf, buflen);
+
-   printf("rcvd: %s", buf);
+   while(1){
+      tot += recvtimeoutssl(net, buf+tot, buflen);
+      if(countreplies(buf) == expectedreplies) break;
+   }
 }
 
 
 void send_helo_command(struct net *net){
+   int replies=1;
    char recvbuf[MAXBUFSIZE];
 
    if(helo == 0){
-      send_smtp_command(net, "HELO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1);
+      send_smtp_command(net, "HELO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1, replies);
-      assert(strncmp(recvbuf, "250 ", 4) == 0 && "HELO");
+      ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
    }
    else {
-      send_smtp_command(net, "EHLO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1);
+      //replies = 6;
-      assert(strncmp(recvbuf, "250-", 4) == 0 && "EHLO");
+      replies = 5;
-      if(net->use_ssl == 0) assert(strstr(recvbuf, "250-STARTTLS") && "STARTTLS");
+      if(net->use_ssl == 1) replies--;
-      else assert(strstr(recvbuf, "250-STARTTLS") == NULL && "STARTTLS");
+
+      send_smtp_command(net, "EHLO aaaa.fu\r\n", recvbuf, sizeof(recvbuf)-1, replies);
+      ASSERT(strncmp(recvbuf, "250-", 4) == 0, recvbuf);
+      if(net->use_ssl == 0){ ASSERT(strstr(recvbuf, "250-STARTTLS"), recvbuf); }
+      else { ASSERT(strstr(recvbuf, "250-STARTTLS") == NULL, recvbuf); }
    }
 }
 
@@ -94,209 +116,392 @@ void send_helo_command(struct net *net){
 static void test_smtp_commands_one_at_a_time(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RCPT");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "354 ", 4) == 0 && "DATA");
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage);
 
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "PERIOD");
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
-static void test_smtp_commands_pipelining(char *server, int port, struct data *data){
+static void test_smtp_commands_one_at_a_time_data_in_2_parts(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+
+   write1(data->net, testmessage, strlen(testmessage));
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage2);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
+
+   close(data->net->socket);
+
+   TEST_FOOTER();
+}
+
+
+/*static void test_smtp_commands_pipelining(char *server, int port, struct data *data){
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+
+   TEST_HEADER();
+
+   connect_to_smtp_server(server, port, data);
+
+   send_helo_command(data->net);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\nQUIT\r\n", testmessage);
 
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 2);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
-}
+
+   TEST_FOOTER();
+}*/
 
 
 static void test_smtp_commands_with_reset_command(char *server, int port, struct data *data){
-   char recvbuf[MAXBUFSIZE];
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+
+   TEST_HEADER();
 
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "RSET\r\n", recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, "RSET\r\n", recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RSET");
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
-   assert(strncmp(recvbuf, "503 ", 4) == 0 && "RCPT");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "503 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
 static void test_smtp_commands_partial_command(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
    write1(data->net, "M", 1);
-   printf("sent: M\n");
 
-   send_smtp_command(data->net, "AIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, "AIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RCPT");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "354 ", 4) == 0 && "DATA");
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage);
 
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "PERIOD");
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
-static void test_smtp_commands_partial_command_pipelining(char *server, int port, struct data *data){
+/*static void test_smtp_commands_partial_command_pipelining(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
    write1(data->net, "M", 1);
-   printf("sent: M\n");
 
-   send_smtp_command(data->net, "AIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
+   snprintf(sendbuf, sizeof(sendbuf)-1, "AIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\nQUIT\r\n", testmessage);
 
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 2);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
-}
+
+   TEST_FOOTER();
+}*/
 
 
 static void test_smtp_commands_starttls(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "STARTTLS\r\n", recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, "STARTTLS\r\n", recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "220 ", 4) == 0 && "STARTTLS");
+   ASSERT(strncmp(recvbuf, "220 ", 4) == 0, recvbuf);
 
    init_ssl_to_server(data);
    data->net->use_ssl = 1;
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "RCPT TO: <archive@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1);
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "RCPT");
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "354 ", 4) == 0 && "DATA");
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\n", testmessage);
 
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "PERIOD");
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
-   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
-   assert(strncmp(recvbuf, "221 ", 4) == 0 && "QUIT");
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
 static void test_smtp_commands_period_command_in_2_parts(char *server, int port, char *part1, char *part2, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
+   // As long as pipelining support is not reintroduced
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   //
+   /*snprintf(sendbuf, sizeof(sendbuf)-1, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   if(!strstr(recvbuf, "354 ")) recvtimeoutssl(data->net, recvbuf, sizeof(recvbuf)-1);
+   ASSERT(strstr(recvbuf, "354 "), recvbuf);*/
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s%s", testmessage, part1);
    write1(data->net, sendbuf, strlen(sendbuf));
 
    snprintf(sendbuf, sizeof(sendbuf), "%s", part2);
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "QUIT\r\n");
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
 
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
 static void test_smtp_commands_period_command_in_its_own_packet(char *server, int port, struct data *data){
    char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
 
+   TEST_HEADER();
+
    connect_to_smtp_server(server, port, data);
 
    send_helo_command(data->net);
 
-   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <archive@aaa.fu>\r\nDATA\r\n", recvbuf, sizeof(recvbuf)-1);
+   // As long as pipelining support is not reintroduced
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "MAIL");
+   //
+   /*snprintf(sendbuf, sizeof(sendbuf)-1, "MAIL FROM: <sender@aaa.fu>\r\nRCPT TO: <%s>\r\nDATA\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 3);
+   if(!strstr(recvbuf, "354 ")) recvtimeoutssl(data->net, recvbuf, sizeof(recvbuf)-1);
+   ASSERT(strstr(recvbuf, "354 "), recvbuf);*/
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+
+
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "%s", testmessage);
    write1(data->net, sendbuf, strlen(sendbuf));
 
    snprintf(sendbuf, sizeof(sendbuf), "\r\n.\r\n");
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
 
    snprintf(sendbuf, sizeof(sendbuf)-1, "QUIT\r\n");
-   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
 
-   assert(strncmp(recvbuf, "250 ", 4) == 0 && "QUIT");
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
 
    close(data->net->socket);
+
+   TEST_FOOTER();
+}
+
+
+static void test_smtp_commands_with_partial_data_lines(char *server, int port, struct data *data){
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+   int yes=1;
+
+   TEST_HEADER();
+
+   connect_to_smtp_server(server, port, data);
+
+   setsockopt(data->net->socket, IPPROTO_TCP, TCP_NODELAY, &yes, (socklen_t)sizeof(int));
+
+   send_helo_command(data->net);
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "DATA\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "354 ", 4) == 0, recvbuf);
+
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "From: aaa@aaa.fu\r\nTo: bela@aaa.fu\r\nMessage-Id: ajajajaja\r\nSubject: this is a test\r\n\r\nAaaaaa");
+   write1(data->net, sendbuf, strlen(sendbuf)); sleep(2);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "jjdkdjkd dkd some garbage.");
+   write1(data->net, sendbuf, strlen(sendbuf)); sleep(2);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "\r\nSome shit again  au aua ua au aua uuu");
+   write1(data->net, sendbuf, strlen(sendbuf)); sleep(2);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, ".\r\nAnd the last line.\r\n.\r\n");
+   write1(data->net, sendbuf, strlen(sendbuf));
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "%s\r\n.\r\nQUIT\r\n", testmessage);
+   recvtimeoutssl(data->net, recvbuf, sizeof(recvbuf));
+
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+
+   close(data->net->socket);
+
+   TEST_FOOTER();
+}
+
+
+static void test_smtp_bdat_last_one_at_a_time(char *server, int port, struct data *data){
+   char recvbuf[MAXBUFSIZE], sendbuf[MAXBUFSIZE];
+
+   TEST_HEADER();
+
+   connect_to_smtp_server(server, port, data);
+
+   send_helo_command(data->net);
+
+   send_smtp_command(data->net, "MAIL FROM: <sender@aaa.fu>\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "RCPT TO: <%s>\r\n", recipient);
+   send_smtp_command(data->net, sendbuf, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   snprintf(sendbuf, sizeof(sendbuf)-1, "BDAT %ld LAST\r\n", strlen(testmessage)+strlen(testmessage2));
+   write1(data->net, sendbuf, strlen(sendbuf));
+
+   write1(data->net, testmessage, strlen(testmessage));
+
+   send_smtp_command(data->net, testmessage2, recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "250 ", 4) == 0, recvbuf);
+
+   send_smtp_command(data->net, "QUIT\r\n", recvbuf, sizeof(recvbuf)-1, 1);
+   ASSERT(strncmp(recvbuf, "221 ", 4) == 0, recvbuf);
+
+   close(data->net->socket);
+
+   TEST_FOOTER();
 }
 
 
@@ -317,6 +522,7 @@ int main(int argc, char **argv){
          {
             {"server",       required_argument,  0,  's' },
             {"port",         required_argument,  0,  'p' },
+            {"rcpt",         required_argument,  0,  'r' },
             {"timeout",      required_argument,  0,  't' },
             {"lhlo",         no_argument,        0,  'l' },
             {"help",         no_argument,        0,  'h' },
@@ -325,9 +531,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      int c = getopt_long(argc, argv, "c:s:p:t:lh?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:s:p:t:r:lh?", long_options, &option_index);
 #else
-      int c = getopt(argc, argv, "c:s:p:t:lh?");
+      int c = getopt(argc, argv, "c:s:p:t:r:lh?");
 #endif
 
 
@@ -343,6 +549,10 @@ int main(int argc, char **argv){
                     port = atoi(optarg);
                     break;
 
+         case 'r' :
+                    recipient = optarg;
+                    break;
+
          case 't' :
                     net.timeout = atoi(optarg);
                     break;
@@ -367,21 +577,26 @@ int main(int argc, char **argv){
    data.net = &net;
 
    test_smtp_commands_one_at_a_time(server, port, &data);
-   test_smtp_commands_pipelining(server, port, &data);
+   test_smtp_commands_one_at_a_time_data_in_2_parts(server, port, &data);
+   ////test_smtp_commands_pipelining(server, port, &data);
    test_smtp_commands_with_reset_command(server, port, &data);
    test_smtp_commands_partial_command(server, port, &data);
-   test_smtp_commands_partial_command_pipelining(server, port, &data);
+   ////test_smtp_commands_partial_command_pipelining(server, port, &data);
+
    test_smtp_commands_period_command_in_2_parts(server, port, "\r", "\n.\r\n", &data);
    test_smtp_commands_period_command_in_2_parts(server, port, "\r\n", ".\r\n", &data);
    test_smtp_commands_period_command_in_2_parts(server, port, "\r\n.", "\r\n", &data);
    test_smtp_commands_period_command_in_2_parts(server, port, "\r\n.\r", "\n", &data);
    test_smtp_commands_period_command_in_its_own_packet(server, port, &data);
 
-   helo = 1; // we must use EHLO to get the STARTTLS in the response
+   test_smtp_commands_with_partial_data_lines(server, port, &data);
-   test_smtp_commands_starttls(server, port, &data);
 
+   helo = 1; // we must use EHLO to get the STARTTLS in the response
+
+   test_smtp_commands_starttls(server, port, &data);
+   data.net->use_ssl = 0;
+
+   test_smtp_bdat_last_one_at_a_time(server, port, &data);
 
    return 0;
 }
-
-

unit_tests/test.h

@@ -8,6 +8,7 @@
 #include <netdb.h>
 #include <arpa/inet.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <fcntl.h>
 #include <locale.h>
 #include <getopt.h>

util/pilerpurge.py

@@ -9,7 +9,7 @@ import sys
 import syslog
 import time
 
-SQL_PURGE_SELECT_QUERY = "SELECT piler_id, size FROM " +\
+SQL_PURGE_SELECT_QUERY = "SELECT id, piler_id, size FROM " +\
     "metadata WHERE deleted=0 AND retained < UNIX_TIMESTAMP(NOW()) " +\
     "AND id NOT IN (SELECT id FROM rcpt WHERE `to` IN " +\
     "(SELECT email FROM legal_hold)) AND id NOT IN (SELECT " +\
@@ -33,7 +33,9 @@ def read_options(filename="", opts={}):
     opts['password'] = config.get('piler', 'mysqlpwd')
     opts['database'] = config.get('piler', 'mysqldb')
     opts['storedir'] = config.get('piler', 'queuedir')
-
+    opts['rtindex'] = config.getint('piler', 'rtindex', fallback=0)
+    opts['sphxhost'] = config.get('piler', 'sphxhost', fallback='127.0.0.1')
+    opts['sphxport'] = config.getint('piler', 'sphxport', fallback=9306)
     opts['server_id'] = "%02x" % config.getint('piler', 'server_id')
 
 
@@ -184,6 +186,17 @@ def get_attachment_file_path(piler_id='', attachment_id=0, opts={}):
                     str(attachment_id)])
 
 
+def purge_index_data(ids=[], opts={}):
+    '''
+    Delete from index data in case of RT index
+    '''
+
+    if opts['rtindex'] == 1 and opts['dry_run'] is False:
+        cursor = opts['sphx'].cursor()
+        a = "," . join([str(x) for x in ids])
+        cursor.execute("DELETE FROM piler WHERE id IN (%s)" % (a))
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("-c", "--config", type=str, help="piler.conf path",
@@ -203,6 +216,7 @@ def main():
     opts['dry_run'] = args.dry_run
     opts['verbose'] = args.verbose
     opts['db'] = None
+    opts['sphx'] = None
     opts['messages'] = 0
     opts['files'] = 0
     opts['size'] = 0
@@ -217,6 +231,8 @@ def main():
         opts['db'] = dbapi.connect(opts['dbhost'], opts['username'],
                                    opts['password'], opts['database'])
 
+        opts['sphx'] = dbapi.connect(host=opts['sphxhost'], port=opts['sphxport'])
+
         if is_purge_enabled(opts) is False:
             syslog.syslog("Purging emails is disabled")
             sys.exit(1)
@@ -229,13 +245,15 @@ def main():
             if rows == ():
                 break
 
-            piler_id = [x[0] for x in rows]
+            id = [x[0] for x in rows]
-            size = [x[1] for x in rows]
+            piler_id = [x[1] for x in rows]
+            size = [x[2] for x in rows]
 
             opts['purged_size'] = opts['purged_size'] + sum(size)
 
             purge_m_files(piler_id, opts)
             purge_attachments_by_piler_id(piler_id, opts)
+            purge_index_data(id, opts)
 
             # It's possible that there's attachment duplication, thus
             # refcount > 0, even though after deleting the duplicates

webui/controller/message/bulkremove.php

@@ -49,8 +49,8 @@ class ControllerMessageBulkremove extends Controller {
             AUDIT(ACTION_REMOVE_MESSAGE, '', '', $id, '');
             $db->query("UPDATE " . TABLE_META . " SET retained=? WHERE id=?", [NOW, $id]);
             if(RT) {
-               $sphx = Registry::get('sphx');
+               $sphxrw = Registry::get('sphxrw');
-               $sphx->query("DELETE FROM " . SPHINX_MAIN_INDEX . " WHERE id=" . (int)$this->data['id']);
+               $sphxrw->query("DELETE FROM " . SPHINX_MAIN_INDEX . " WHERE id=" . (int)$this->data['id']);
             }
 
             syslog(LOG_INFO, $this->data['username'] . " removed message: $id");

webui/controller/message/remove.php

@@ -41,8 +41,8 @@ class ControllerMessageRemove extends Controller {
       $db->query("UPDATE " . TABLE_META . " SET retained=? WHERE id=?", [NOW, $id]);
       $db->query("UPDATE " . TABLE_DELETED . " SET deleted=1 WHERE id=?", [$id]);
       if(RT) {
-         $sphx = Registry::get('sphx');
+         $sphxrw = Registry::get('sphxrw');
-         $sphx->query("DELETE FROM " . SPHINX_MAIN_INDEX . " WHERE id=", (int)$this->data['id']);
+         $sphxrw->query("DELETE FROM " . SPHINX_MAIN_INDEX . " WHERE id=", (int)$this->data['id']);
       }
 
       syslog(LOG_INFO, $this->data['username'] . " removed message: $id");

webui/index.php

@@ -41,12 +41,14 @@ Registry::set('db', $db);
 
 Registry::set('DB_DRIVER', DB_DRIVER);
 
-$sphx = new DB(SPHINX_DRIVER, SPHINX_HOSTNAME, "", "", SPHINX_DATABASE, "");
+$sphx = new DB(SPHINX_DRIVER, SPHINX_HOSTNAME_READONLY, "", "", SPHINX_DATABASE, "");
+$sphxrw = new DB(SPHINX_DRIVER, SPHINX_HOSTNAME, "", "", SPHINX_DATABASE, "");
 Registry::set('sphx', $sphx);
+Registry::set('sphxrw', $sphxrw);
 
 
 if(MEMCACHED_ENABLED) {
-   $memcache = new Memcache();
+   $memcache = new Memcached();
    foreach ($memcached_servers as $m){
       $memcache->addServer($m[0], $m[1]);
    }
@@ -112,6 +114,3 @@ else {
 
 $controller = new Front();
 $controller->dispatch($action, new Router('common/not_found'));
-
-
-?>

webui/model/search/message.php

@@ -486,13 +486,13 @@ class ModelSearchMessage extends Model {
 
       if($tag == '') {
          if(RT) {
-            $this->sphx->query("DELETE FROM " . SPHINX_TAG_INDEX . " WHERE uid=$uid AND mid=$id");
+            $this->sphxrw->query("DELETE FROM " . SPHINX_TAG_INDEX . " WHERE uid=$uid AND mid=$id");
          } else {
             $query = $this->db->query("DELETE FROM " . TABLE_TAG . " WHERE uid=? AND id=?", array($uid, $id));
          }
       } else {
          if(RT) {
-            $this->sphx->query("REPLACE INTO " . SPHINX_TAG_INDEX . " (mid, uid, tag) VALUES (?,?,?)",[$id,$uid,$tag]);
+            $this->sphxrw->query("REPLACE INTO " . SPHINX_TAG_INDEX . " (mid, uid, tag) VALUES (?,?,?)",[$id,$uid,$tag]);
          } else {
             $query = $this->db->query("UPDATE " . TABLE_TAG . " SET tag=? WHERE uid=? AND id=?", array($tag, $uid, $id));
             if($this->db->countAffected() == 0) {
@@ -510,7 +510,7 @@ class ModelSearchMessage extends Model {
 
       if(RT) {
          $ids_str = implode(",", $ids);
-         $this->sphx->query("DELETE FROM " . SPHINX_TAG_INDEX . " WHERE uid=$uid AND mid IN ($ids_str)");
+         $this->sphxrw->query("DELETE FROM " . SPHINX_TAG_INDEX . " WHERE uid=$uid AND mid IN ($ids_str)");
       } else {
          $query = $this->db->query("DELETE FROM " . TABLE_TAG . " WHERE uid=? AND id IN ($q)", $arr);
       }
@@ -518,7 +518,7 @@ class ModelSearchMessage extends Model {
       if($tag) {
          foreach ($ids as $id) {
             if(RT) {
-               $this->sphx->query("INSERT INTO " . SPHINX_TAG_INDEX . " (mid, uid, tag) VALUES(?,?,?)", [$id, $uid, $tag]);
+               $this->sphxrw->query("INSERT INTO " . SPHINX_TAG_INDEX . " (mid, uid, tag) VALUES(?,?,?)", [$id, $uid, $tag]);
             } else {
                $query = $this->db->query("INSERT INTO " . TABLE_TAG . " (id, uid, tag) VALUES(?,?,?)", array($id, $uid, $tag));
             }
@@ -561,10 +561,10 @@ class ModelSearchMessage extends Model {
    public function add_message_rt_note($id = '', $uid = 0, $note = '') {
       if($id == '' || $uid <= 0) { return 0; }
 
-      $this->sphx->query("DELETE FROM " . SPHINX_NOTE_INDEX . " WHERE uid=$uid AND mid=$id");
+      $this->sphxrw->query("DELETE FROM " . SPHINX_NOTE_INDEX . " WHERE uid=$uid AND mid=$id");
 
       if($note) {
-         $this->sphx->query("INSERT INTO " . SPHINX_NOTE_INDEX . " (mid, uid, note) VALUES (?,?,?)",[$id,$uid,$note]);
+         $this->sphxrw->query("INSERT INTO " . SPHINX_NOTE_INDEX . " (mid, uid, note) VALUES (?,?,?)",[$id,$uid,$note]);
       }
 
       return 1;

webui/system/helper/TrustedTimestamps.php

@@ -71,6 +71,10 @@ class TrustedTimestamps
         if (!file_exists($requestfile_path))
             throw new Exception("The Requestfile was not found");
 
+        $header = array('Content-Type: application/timestamp-query');
+        if(TSA_AUTH_USER)
+            $header[] = "Authorization: Basic " . base64_encode(TSA_AUTH_USER . ':' . TSA_AUTH_PASSWORD);
+
         $ch = curl_init();
         curl_setopt($ch, CURLOPT_URL, $tsa_url);
         curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
@@ -78,10 +82,22 @@ class TrustedTimestamps
         curl_setopt($ch, CURLOPT_POST, 1);
         curl_setopt($ch, CURLOPT_BINARYTRANSFER, 1);
         curl_setopt($ch, CURLOPT_POSTFIELDS, file_get_contents($requestfile_path));
-        curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/timestamp-query'));
+        curl_setopt($ch, CURLOPT_HTTPHEADER, $header);
         curl_setopt($ch, CURLOPT_USERAGENT, "Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)");
         curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, TSA_VERIFY_CERTIFICATE);
 
+        if(TSA_AUTH_CERT_FILE && TSA_AUTH_KEY_FILE)
+        {
+            if(!file_exists(TSA_AUTH_CERT_FILE))
+                throw new Exception("Client certificate file " . TSA_AUTH_CERT_FILE . " not found");
+            curl_setopt($ch, CURLOPT_SSLCERT, TSA_AUTH_CERT_FILE);
+            if(!file_exists(TSA_AUTH_KEY_FILE))
+                throw new Exception("Client key file " . TSA_AUTH_KEY_FILE . " not found");
+            curl_setopt($ch, CURLOPT_SSLKEY, TSA_AUTH_KEY_FILE);
+            if(TSA_AUTH_KEY_PASSWORD)
+                curl_setopt($ch, CURLOPT_KEYPASSWD, TSA_AUTH_KEY_PASSWORD);
+        }
+
         $binary_response_string = curl_exec($ch);
 
         $error = curl_error($ch);