Merge branch 'master' of bitbucket.org:jsuto/piler

2024-11-08 00:31:58 +01:00 · 2022-03-17 15:35:59 +01:00 · 2022-03-17 15:35:59 +01:00 · e20af0f331
commit e20af0f331
parent 05f8b7bf7e b6f8834fe8
75 changed files with 2288 additions and 257 deletions
--- a/14
+++ b/14
@ -1,6 +1,3 @@
- Introduced the archive_address feature, see etc/example.conf for the details
- Introduced the raw: search label
-
 1.3.12:
 -------

@ -14,6 +11,17 @@
  - TLSv1.2 (default)
  - TLSv1.3

+- Introduced the archive_address feature, see etc/example.conf for the details
+- Introduced the raw: search label
+- Added Italian translation. Credits: Stefano Gatto
+- timestamp signing sorts by 'id' column
+- timestamp hash value defaults to sha256
+- Minor fixes
+- Added support for php 8.1
+- Fixed handling long email addresses
+
+Be sure to apply util/db-upgrade.sql
+

 1.3.11:
 -------
--- a/bitbucket-pipelines.yml
+++ b/bitbucket-pipelines.yml
@ -18,5 +18,5 @@ pipelines:
          - mysql -u piler -ppiler123 piler1 < /usr/share/piler/db-mysql.sql
          - cd unit_tests
          - ./run.sh
-          - cd ..
+          - cd ../webui
          - phpunit
--- a/config.php.in
+++ b/config.php.in
@ -88,6 +88,7 @@ $config['LDAP_MAIL_ATTR'] = 'mail';
 $config['LDAP_AUDITOR_MEMBER_DN'] = '';
 $config['LDAP_ADMIN_MEMBER_DN'] = '';
 $config['LDAP_BASE_DN'] = '';
+$config['LDAP_USE_START_TLS'] = 0;

 // AD specific settings
 //
@ -324,10 +325,12 @@ $langs = array(
                'es',
                'fr',
                'hu',
+                'it',
                'pl',
                'pt',
                'ru',
-                'tr'
+                'tr',
+                'tw'
               );


--- a/contrib/installer/focal.sh
+++ b/contrib/installer/focal.sh
@ -0,0 +1,176 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -x
+
+PILER_HOSTNAME="${PILER_HOSTNAME:-archive.yourdomain.com}"
+MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}"
+MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}"
+SERVER_ID="${SERVER_ID:-0}"
+USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}"
+SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}"
+PHP_FPM_SOCKET="/var/run/php/php7.4-fpm.sock"
+
+MYSQL_HOSTNAME="localhost"
+MYSQL_DATABASE="piler"
+MYSQL_USERNAME="piler"
+
+SPHINX_TARGZ="sphinx-3.3.1-bin.tar.gz"
+DOWNLOAD_URL="https://download.mailpiler.com"
+PILER_DEB="piler_1.3.12-focal-a1b71bdd_amd64.deb"
+PILER_USER="piler"
+CONFIG_SITE_PHP="/etc/piler/config-site.php"
+
+export DEBIAN_FRONTEND=noninteractive
+
+install_prerequisites() {
+   apt-get update
+   apt-get -y --no-install-recommends install \
+      wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \
+      php7.4-gd php7.4-curl php7.4-xml ca-certificates zip catdoc unrtf poppler-utils nginx tnef libzip5 \
+      libtre5 libwrap0 cron libmariadb-dev python3 python3-mysqldb libmariadb-dev mariadb-client-core-10.3 \
+      mariadb-server-10.3
+
+   wget -q -O "/tmp/${SPHINX_TARGZ}" "${DOWNLOAD_URL}/generic-local/${SPHINX_TARGZ}"
+   tar -C / -zxvf "/tmp/${SPHINX_TARGZ}"
+}
+
+
+fix_mysql_settings() {
+   cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF
+[mysqld]
+
+innodb_buffer_pool_size=512M
+innodb_flush_log_at_trx_commit=1
+innodb_log_buffer_size=64M
+innodb_log_file_size=64M
+innodb_read_io_threads=4
+innodb_write_io_threads=4
+innodb_log_files_in_group=2
+
+innodb_file_per_table
+PILER_CNF
+}
+
+start_mysql() {
+   fix_mysql_settings
+
+   service mysql restart
+   mysqladmin -u root password "${MYSQL_ROOT_PASSWORD}"
+}
+
+
+install_piler() {
+   wget "${DOWNLOAD_URL}/piler/${PILER_DEB}"
+   dpkg -i "$PILER_DEB"
+   rm -f "$PILER_DEB"
+
+   crontab -u "$PILER_USER" /usr/share/piler/piler.cron
+
+   rm -f "$PILER_DEB" /etc/nginx/sites-enabled/default
+   ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf
+
+   touch /var/piler/.bash_history
+   chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history
+}
+
+
+create_my_cnf() {
+   local user=$1
+   local password=$2
+   local my_cnf=$3
+
+   printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf"
+   printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf"
+   chown $PILER_USER:$PILER_USER "$my_cnf"
+   chmod 600 "$my_cnf"
+}
+
+
+fix_config_site_php() {
+   sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP"
+
+   {
+      echo "\$config['SERVER_ID'] = $SERVER_ID;"
+      echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;"
+      echo "\$config['SPHINX_VERSION'] = 331;"
+   } >> "$CONFIG_SITE_PHP"
+
+   if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then
+      echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP"
+   fi
+
+   echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP"
+}
+
+
+init_db() {
+   sed -e "s%MYSQL_HOSTNAME%$MYSQL_HOSTNAME%g" -e "s%MYSQL_DATABASE%$MYSQL_DATABASE%g" -e "s%MYSQL_USERNAME%$MYSQL_USERNAME%g" -e "s%MYSQL_PASSWORD%$MYSQL_PILER_PASSWORD%g" \
+      /usr/share/piler/db-mysql-root.sql.in | mysql --defaults-file=/etc/piler/.my.cnf-root -h $MYSQL_HOSTNAME
+   mysql --defaults-file=/etc/piler/.my.cnf -h $MYSQL_HOSTNAME $MYSQL_DATABASE < /usr/share/piler/db-mysql.sql
+}
+
+
+add_systemd_services() {
+   pushd /etc/systemd/system
+
+   ln -sf /usr/libexec/piler/piler.service .
+   ln -sf /usr/libexec/piler/piler-smtp.service .
+   ln -sf /usr/libexec/piler/pilersearch.service .
+
+   popd
+
+   systemctl daemon-reload
+
+   systemctl enable piler
+   systemctl enable piler-smtp
+   systemctl enable pilersearch
+}
+
+
+fix_configs() {
+   if [[ ! -f /etc/piler/piler-nginx.conf ]]; then
+      sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf
+      nginx -t
+      nginx -s reload
+   fi
+
+   if [[ ! -f /etc/piler/piler.conf ]]; then
+      sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.yourdomain.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > /etc/piler/piler.conf
+      chmod 600 /etc/piler/piler.conf
+      chown $PILER_USER:$PILER_USER /etc/piler/piler.conf
+   fi
+
+   sed -i -e "s/MYSQL_HOSTNAME/${MYSQL_HOSTNAME}/g" \
+          -e "s/MYSQL_DATABASE/${MYSQL_DATABASE}/g" \
+          -e "s/MYSQL_USERNAME/${MYSQL_USERNAME}/g" \
+          -e "s/MYSQL_PASSWORD/${MYSQL_PILER_PASSWORD}/g" \
+          /etc/piler/sphinx.conf
+}
+
+
+install_prerequisites
+
+start_mysql
+
+install_piler
+
+create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root
+create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf
+
+fix_config_site_php
+
+add_systemd_services
+
+fix_configs
+init_db
+
+su -c "indexer --all -c /etc/piler/sphinx.conf" $PILER_USER
+
+[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler
+
+systemctl start pilersearch
+systemctl start piler
+systemctl start piler-smtp
--- a/etc/cron.jobs.in
+++ b/etc/cron.jobs.in
@ -6,9 +6,9 @@
 */15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet note1 --rotate
 */5 * * * *  /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \;
 */5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error
-2 0 * * * LIBEXECDIR/piler/pilerpurge.py

-### optional: populate accouting data
+### optional
 ###30 6 * * * /usr/bin/php LIBEXECDIR/piler/generate_stats.php --webui LOCALSTATEDIR/piler/www >/dev/null
+###*/5 * * * * LIBEXECDIR/piler/import.sh

 ### PILEREND
--- a/src/archive.c
+++ b/src/archive.c
@ -277,7 +277,7 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct c
      return 1;
   }

-   snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
+   snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, *(sdata->ttmpfile+24), *(sdata->ttmpfile+25), *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
   if(stat(filename, &st)){
      snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+RND_STR_LEN-6), *(sdata->ttmpfile+RND_STR_LEN-5), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile);
@ -304,7 +304,7 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct c
               buffer = p + strlen(pointer);

               if(strlen(ptr_arr[i].piler_id) == RND_STR_LEN){
-                  snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);
+                  snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, ptr_arr[i].piler_id[24], ptr_arr[i].piler_id[25], ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id);

               #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
                  if(stat(filename, &st)){
--- a/src/cfg.c
+++ b/src/cfg.c
@ -152,7 +152,9 @@ int get_tls_protocol_number(char *protocol){
      { "TLSv1", TLS1_VERSION },
      { "TLSv1.1", TLS1_1_VERSION },
      { "TLSv1.2", TLS1_2_VERSION },
+   #ifdef TLS1_3_VERSION
      { "TLSv1.3", TLS1_3_VERSION },
+   #endif
   };

   for(unsigned int i=0; i<sizeof(tls_protocols)/sizeof(struct tls_protocol); i++){
--- a/src/config.h
+++ b/src/config.h
@ -39,6 +39,20 @@
 #define IPLEN 16+1
 #define KEYLEN 56
 #define MIN_EMAIL_ADDRESS_LEN 9
+// Sphinx 3.x has an issue with tokens longer than 41 characters.
+//
+// When a regular user executes a query, then his default email address filter
+// causes the query to fail with the below error message, even when the query
+// itself is correct:
+//
+// SELECT id FROM main1,dailydelta1,delta1 WHERE MATCH(' ( (@sender thisisanextremelylongemailaddressyesareallylongoneyeahitolyouXaddressXcom ) | (@rcpt thisisanextremelylongemailaddressyesareallylongoneyeahitolyouXaddressXcom) ) ') ORDER BY `sent` DESC LIMIT 0,20 OPTION max_matches=1000'
+//
+// ERROR 1064 (42000): index dailydelta1,delta1,main1: syntax error, unexpected $end near ' '
+//
+// Note that we use 42, because the parser adds a trailing space to the tokens
+// See https://www.mailpiler.org/wiki/current:sphinx3 and
+// https://bitbucket.org/jsuto/piler/issues/1082/no-sphinx-results-with-long-email for more
+#define MAX_EMAIL_ADDRESS_SPHINX_LEN 42

 #define CRLF "\n"

@ -108,9 +122,10 @@
 #define SQL_PREPARED_STMT_GET_FOLDER_ID              "SELECT `id` FROM " SQL_FOLDER_TABLE " WHERE `name`=? AND `parent_id`=?"
 #define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE   "INSERT INTO `" SQL_FOLDER_TABLE "` (`name`, `parent_id`) VALUES(?,?)"
 #define SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE  "UPDATE " SQL_METADATA_TABLE " SET reference=? WHERE message_id=? AND reference=''"
+#define SQL_PREPARED_STMT_GET_METADATA_REFERENCE     "SELECT COUNT(*) AS count FROM " SQL_METADATA_TABLE " WHERE reference=?"
 #define SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS        "SELECT id, type, username, password, server FROM " SQL_IMPORT_TABLE " WHERE started=0 ORDER BY id LIMIT 0,1"
 #define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE      "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `id`) VALUES(?,?)"
-#define SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE        "UPDATE " SQL_IMPORT_TABLE " SET status=?, imported=? WHERE id=?"
+#define SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE        "UPDATE " SQL_IMPORT_TABLE " SET started=?, status=?, imported=? WHERE id=?"

 /* Error codes */

--- a/src/defs.h
+++ b/src/defs.h
@ -14,6 +14,7 @@
   #include <tre/regex.h>
 #endif

+#include <openssl/md5.h>
 #include <openssl/sha.h>
 #include <openssl/ssl.h>
 #include <netinet/in.h>
@ -171,6 +172,7 @@ struct parser_state {
   int qp;
   int htmltag;
   int style;
+   int meta_content_type;
   int skip_html;
   int has_to_dump;
   int has_to_dump_whole_body;
@ -184,11 +186,13 @@ struct parser_state {
   int saved_size;
   unsigned int writebufpos;
   unsigned int abufpos;
+   unsigned int received_header;
   char attachedfile[RND_STR_LEN+SMALLBUFSIZE];
   char message_id[SMALLBUFSIZE];
   char message_id_hash[2*DIGEST_LENGTH+1];
   char miscbuf[MAX_TOKEN_LEN];
   char qpbuf[MAX_TOKEN_LEN];
+   char receivedbuf[SMALLBUFSIZE];
   unsigned long n_token;
   unsigned long n_subject_token;
   unsigned long n_body_token;
@ -218,7 +222,7 @@ struct parser_state {
   unsigned int todomainlen;
   unsigned int found_security_header;

-   int journaltolen;
+   long unsigned int journaltolen;

   int retention;
 };
@ -315,6 +319,7 @@ struct import {
   int port;
   int seq;
   int table_id;
+   int delay;
   char *server;
   char *username;
   char *password;
@ -399,6 +404,7 @@ struct smtp_session {
   char rcptto[MAX_RCPT_TO][SMALLBUFSIZE];
   char buf[MAXBUFSIZE];
   char remote_host[INET6_ADDRSTRLEN+1];
+   char nullbyte;
   time_t lasttime;
   int protocol_state;
   int slot;
--- a/src/digest.c
+++ b/src/digest.c
@ -132,3 +132,27 @@ void digest_string(char *s, char *digest){
      snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]);

 }
+
+
+void md5_string(char *s, char *digest){
+   int i;
+   unsigned char md[MD5_DIGEST_LENGTH];
+   MD5_CTX context;
+
+   memset(digest, 0, 2*MD5_DIGEST_LENGTH+2);
+
+   MD5_Init(&context);
+
+   MD5_Update(&context, s, strlen(s));
+
+   MD5_Final(md, &context);
+
+   for(i=0;i<MD5_DIGEST_LENGTH;i++)
+      snprintf(digest + i*2, 2*MD5_DIGEST_LENGTH, "%02x", md[i]);
+}
+
+
+void create_md5_from_email_address(char *puf, char *md5buf){
+   md5_string(puf, md5buf);
+   md5buf[2*MD5_DIGEST_LENGTH] = ' ';
+}
--- a/src/imap.c
+++ b/src/imap.c
@ -159,7 +159,7 @@ int imap_select_cmd_on_folder(char *folder, struct data *data){
      }
   }

-   printf("found %d messages\n", messages);
+   if(data->quiet == 0){printf("found %d messages\n", messages); }

   data->import->total_messages += messages;

@ -356,7 +356,7 @@ int process_imap_folder(char *folder, struct session_data *sdata, struct data *d
   }


-   printf("\n");
+   if(data->quiet == 0){printf("\n"); }

   return OK;
 }
@ -375,7 +375,7 @@ int list_folders(struct data *data){
   char attrs[SMALLBUFSIZE], folder[SMALLBUFSIZE];
   int len=MAXBUFSIZE+3, pos=0, n, rc=ERR, fldrlen=0, result;

-   printf("List of IMAP folders:\n");
+   if(data->quiet == 0){printf("List of IMAP folders:\n"); }

   buf = malloc(len);
   if(!buf) return rc;
@ -478,9 +478,9 @@ int list_folders(struct data *data){
               if(!strstr(attrs, "\\Noselect")){
                  addnode(data->imapfolders, folder);
               }
-               else printf("skipping ");
+               else if(data->quiet == 0){printf("skipping "); }

-               printf("=> '%s [%s]'\n", folder, attrs);
+               if(data->quiet == 0){printf("=> '%s [%s]'\n", folder, attrs); }

               memset(attrs, 0, sizeof(attrs));
            }
--- a/src/import.c
+++ b/src/import.c
@ -25,6 +25,14 @@ int import_message(struct session_data *sdata, struct data *data, struct config
   struct parser_state state;
   struct counters counters;

+   if(data->import->delay > 0){
+      struct timespec req;
+
+      req.tv_sec = 0;
+      req.tv_nsec = 1000000 * data->import->delay;
+
+      nanosleep(&req, NULL);
+   }

   init_session_data(sdata, cfg);

@ -89,7 +97,10 @@ int import_message(struct session_data *sdata, struct data *data, struct config
         // When importing emails, we should add the retention value (later) to the original sent value
         sdata->retained = sdata->sent;

+         // backup original value of data->folder
+         int folder = data->folder;
         rc = process_message(sdata, &state, data, cfg);
+         data->folder = folder;
         unlink(state.message_id_hash);
      }
   }
@ -145,13 +156,14 @@ int import_message(struct session_data *sdata, struct data *data, struct config


 int update_import_table(struct session_data *sdata, struct data *data) {
-   int ret=ERR, status=2;
+   int ret=ERR, status=2, started=1;
   struct sql sql;

   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE) == ERR) return ret;

   p_bind_init(&sql);

+   sql.sql[sql.pos] = (char *)&(started); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
   sql.sql[sql.pos] = (char *)&(status); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
   sql.sql[sql.pos] = (char *)&(data->import->tot_msgs); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
   sql.sql[sql.pos] = (char *)&(data->import->table_id); sql.type[sql.pos] = TYPE_LONG; sql.pos++;
--- a/src/import_pilerexport.c
+++ b/src/import_pilerexport.c
@ -58,7 +58,7 @@ void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *s


 void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg){
-   int n, rc, savedlen=0, puflen;
+   int n, rc, nullbyte, savedlen=0, puflen;
   uint64 count=0;
   char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE];

@ -70,12 +70,16 @@ void import_from_pilerexport(struct session_data *sdata, struct data *data, stru
      memset(buf, 0, sizeof(buf));
      n = fread(buf, 1, sizeof(buf)-1, stdin);

+      int remaininglen = n;
+
      if(savedlen > 0){
         memset(copybuf, 0, sizeof(copybuf));

         memcpy(copybuf, savedbuf, savedlen);
         memcpy(&copybuf[savedlen], buf, n);

+         remaininglen += savedlen;
+
         savedlen = 0;
         memset(savedbuf, 0, sizeof(savedbuf));

@ -86,8 +90,9 @@ void import_from_pilerexport(struct session_data *sdata, struct data *data, stru
      }

      do {
-         puflen = read_one_line(p, '\n', puf, sizeof(puf), &rc);
+         puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf), &rc, &nullbyte);
         p += puflen;
+         remaininglen -= puflen;

         if(puflen > 0){
            if(rc == OK){
--- a/src/message.c
+++ b/src/message.c
@ -213,6 +213,33 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru
      digest_string(state->reference, &ref[0]);
      update_metadata_reference(sdata, state, &ref[0], cfg);
   }
+   else if(state->reference[0] == 0){
+      // during import, the order of messages is often random
+      // check if this is a message which is already referenced
+      uint64 count=0;
+
+      digest_string(state->message_id, &ref[0]);
+      if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_METADATA_REFERENCE) != ERR){
+         p_bind_init(&sql);
+
+         sql.sql[sql.pos] = &ref[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
+
+         if(p_exec_stmt(sdata, &sql) == OK){
+	    p_bind_init(&sql);
+
+	    sql.sql[sql.pos] = (char *)&count; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++;
+	    p_store_results(&sql);
+	    p_fetch_results(&sql);
+	    p_free_results(&sql);
+	 }
+      }
+
+      close_prepared_statement(&sql);
+
+      // no reference yet
+      if(count <= 0)
+         ref[0] = 0;
+   }


   if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR;
@ -308,7 +335,7 @@ void remove_stripped_attachments(struct parser_state *state){
 }


-int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
+int is_duplicated_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
   int fd;
   char piler_id[SMALLBUFSIZE];

@ -319,7 +346,7 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru
   if(sdata->duplicate_id > 0){
      remove_stripped_attachments(state);

-      if(strlen(state->b_journal_to) > 0){
+      if(sdata->restored_copy == 0 && strlen(state->b_journal_to) > 0){
         if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to add journal rcpt (%s) to id=%llu for message-id: '%s'", sdata->ttmpfile, state->b_journal_to, sdata->duplicate_id, state->message_id);
         store_recipients(sdata, state->b_journal_to, sdata->duplicate_id, cfg);
      }
@ -338,7 +365,6 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru
   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: touch %s OK (%s)", sdata->ttmpfile, state->message_id_hash, state->message_id);


-
   if(cfg->mmap_dedup_test == 1 && data->dedup != MAP_FAILED && data->child_serial >= 0 && data->child_serial < MAXCHILDREN){

      if(strstr(data->dedup, state->message_id_hash)){
@ -354,6 +380,14 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru
      memcpy(data->dedup + data->child_serial*DIGEST_LENGTH*2, state->message_id_hash, DIGEST_LENGTH*2);
   }

+   return OK;
+}
+
+
+int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){
+
+   if(is_duplicated_message(sdata, state, data, cfg) == ERR_EXISTS)
+      return ERR_EXISTS;

   sdata->retained += query_retain_period(data, state, sdata->tot_len, sdata->spam_message, cfg);

--- a/src/misc.c
+++ b/src/misc.c
@ -652,34 +652,42 @@ void move_email(struct smtp_session *session){

   snprintf(buf, sizeof(buf)-1, "%d/%s", session->ttmpfile[0] % session->cfg->number_of_worker_processes, session->ttmpfile);

+   if(session->nullbyte){
+      snprintf(buf, sizeof(buf)-1, "%s/%s", ERROR_DIR, session->ttmpfile);
+      syslog(LOG_PRIORITY, "ERROR: %s contains an invalid NUL-byte, moving it to %s", session->ttmpfile, ERROR_DIR);
+   }
+
   if(rename(session->ttmpfile, buf)){
      syslog(LOG_PRIORITY, "ERROR: couldn't rename %s to %s (reason: %s)", session->ttmpfile, buf, strerror(errno));
   }
 }


-int read_one_line(char *s, int c, char *buf, int buflen, int *rc){
+int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte){
   int i=0;

   *rc = ERR;
+   *nullbyte = 0;

   memset(buf, 0, buflen);

-   if(s == NULL){
-      return i;
+   for(int j=0; j<slen; j++){
+      if(i<buflen-2){
+         if(*(s+j) == 0){
+            *nullbyte = 1;
         }

-   for(; *s; s++){
-      if(i<buflen-2){
-         buf[i] = *s;
+         buf[i] = *(s+j);
         i++;

-         if(*s == c){
+         if(*(s+j) == c){
            *rc = OK;
            break;
         }
      }
-      else break;
+      else {
+         break;
+      }
   }

   return i;
@ -708,7 +716,7 @@ int init_ssl_to_server(struct data *data){
   n = SSL_connect(data->net->ssl);
   CHK_SSL(n, "internal ssl error");

-   printf("Cipher: %s\n", SSL_get_cipher(data->net->ssl));
+   //printf("Cipher: %s\n", SSL_get_cipher(data->net->ssl));

   server_cert = SSL_get_peer_certificate(data->net->ssl);
   CHK_NULL(server_cert, "server cert error");
--- a/src/misc.h
+++ b/src/misc.h
@ -46,7 +46,7 @@ int create_and_bind(char *listen_addr, int listen_port);
 int can_i_write_directory(char *dir);

 void move_email(struct smtp_session *session);
-int read_one_line(char *s, int c, char *buf, int buflen, int *rc);
+int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte);

 int init_ssl_to_server(struct data *data);

--- a/src/parser.c
+++ b/src/parser.c
@ -65,7 +65,9 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec
   fclose(f);

   if(data->import && data->import->extra_recipient){
-      add_recipient(data->import->extra_recipient, strlen(data->import->extra_recipient), sdata, &state, data, cfg);
+      char tmpbuf[SMALLBUFSIZE];
+      snprintf(tmpbuf, sizeof(tmpbuf)-1, "%s", data->import->extra_recipient);
+      add_recipient(tmpbuf, strlen(tmpbuf), sdata, &state, data, cfg);
   }

   // If both Sender: and From: headers exist, and they are different, then append
@ -110,6 +112,15 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c
      if(sdata->internal_recipient == 1 && sdata->external_recipient == 1) sdata->direction = DIRECTION_INTERNAL_AND_OUTGOING;
   }

+   char *q = strrchr(state->receivedbuf, ';');
+   if(q){
+      time_t received_timestamp = parse_date_header(q+1);
+      if(received_timestamp > 10000000){
+         // If the calculated date based on Date: header line differs more than 1 week
+         // then we'll override it with the data parsed from the first Received: line
+         if(labs(received_timestamp - sdata->sent) > 604800) sdata->sent = received_timestamp;
+      }
+   }

   for(i=1; i<=state->n_attachments; i++){
      char puf[SMALLBUFSIZE];
@ -123,7 +134,7 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c

      digest_file(state->attachments[i].internalname, &(state->attachments[i].digest[0]));

-      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest);
+      if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s, dumped: %d", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest, state->attachments[i].dumped);

      char *p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type);
      len = strlen(p);
@ -406,7 +417,10 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
         state->message_state = MSG_CC;
         buf += strlen("Bcc:");
      }
-      else if(strncasecmp(buf, "Message-Id:", 11) == 0) state->message_state = MSG_MESSAGE_ID;
+      else if(strncasecmp(buf, "Message-Id:", 11) == 0){
+         state->message_state = MSG_MESSAGE_ID;
+         buf += strlen("Message-Id:");
+      }
      else if(strncasecmp(buf, "References:", 11) == 0) state->message_state = MSG_REFERENCES;
      else if(strncasecmp(buf, "Subject:", strlen("Subject:")) == 0){
         state->message_state = MSG_SUBJECT;
@ -430,32 +444,39 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata

         if(strstr(buf, "=?") && strstr(buf, "?=")) fixupEncodedHeaderLine(buf, MAXBUFSIZE);

-         sdata->sent = parse_date_header(buf);
+         sdata->sent = parse_date_header(buf+5);

         /* allow +2 days drift in the parsed Date: value */

         if(sdata->sent - sdata->now > 2*86400) sdata->sent = sdata->now;
      }

-      else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf);
-      else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0) state->message_state = MSG_RECEIVED;
+      else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf+14);
+      else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0){
+         state->message_state = MSG_RECEIVED;
+         state->received_header++;
+      }
      else if(cfg->extra_to_field[0] != '\0' && strncasecmp(buf, cfg->extra_to_field, strlen(cfg->extra_to_field)) == 0){
-         state->message_state = MSG_TO;
+         state->message_state = MSG_RECIPIENT;
         buf += strlen(cfg->extra_to_field);
      }

      if(state->message_state == MSG_MESSAGE_ID && state->message_id[0] == 0){
-         p = strchr(buf+11, ' ');
-         if(p) p = buf + 12;
-         else p = buf + 11;
+         while(isspace(*buf)){
+            buf++;
+         }

-         snprintf(state->message_id, SMALLBUFSIZE-1, "%s", p);
+         snprintf(state->message_id, SMALLBUFSIZE-1, "%s", buf);
      }

      if(state->message_state == MSG_CONTENT_TYPE || state->message_state == MSG_CONTENT_DISPOSITION){
         fill_attachment_name_buf(state, buf);
      }

+      if(state->received_header == 1 && state->message_state == MSG_RECEIVED && strlen(state->receivedbuf) + len < sizeof(state->receivedbuf)){
+         memcpy(&(state->receivedbuf[strlen(state->receivedbuf)]), buf, len);
+      }
+
      /* we are interested in only From:, To:, Subject:, Received:, Content-*: header lines */
      if(state->message_state <= 0) return 0;
   }
@ -553,6 +574,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
         state->message_rfc822 = 1;
         state->is_header = 1;

+         state->has_to_dump = 0;
+
         if(sdata->ms_journal == 1){
            state->is_1st_header = 1;

@ -618,7 +641,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
      state->pushed_pointer = 0;

      memset(state->type, 0, TINYBUFSIZE);
-      snprintf(state->charset, TINYBUFSIZE-1, "unknown");
+      memset(state->charset, 0, TINYBUFSIZE);

      memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
      state->anamepos = 0;
@ -661,7 +684,18 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
   if(state->texthtml == 1 && state->message_state == MSG_BODY) markHTML(buf, state);


-   if(state->texthtml == 1) decodeHTML(buf, state->utf8);
+   if(state->texthtml == 1){
+      size_t buflen = strlen(buf);
+      decodeHTML(buf, state->utf8);
+      /* decodeHTML converted some entities to iso-8859-1 */
+      if(state->utf8 != 1 && strlen(buf) != buflen){
+        /* no charset or us-ascii: switch to iso-8859-1 */
+        if (state->charset[0] == 0 || strcasecmp(state->charset, "us-ascii") == 0){
+          syslog(LOG_PRIORITY, "%s: assuming iso-8859-1 encoding for HTML (was '%s')", sdata->ttmpfile, state->charset);
+          snprintf(state->charset, TINYBUFSIZE-1, "ISO8859-1");
+        }
+      }
+   }

   /* encode the body if it's not utf-8 encoded */
   if(state->message_state == MSG_BODY && state->utf8 != 1){
--- a/src/parser.h
+++ b/src/parser.h
@ -20,7 +20,7 @@ void fixupEncodedHeaderLine(char *buf, int buflen);
 void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state);
 void fixupBase64EncodedLine(char *buf, struct parser_state *state);
 void markHTML(char *buf, struct parser_state *state);
-void setStateHTMLStyle(char *htmlbuf, int pos, struct parser_state *state);
+void setStateHTML(char *htmlbuf, int pos, struct parser_state *state);
 void translateLine(unsigned char *p, struct parser_state *state);
 void fix_email_address_for_sphinx(char *s);
 void split_email_address(char *s);
--- a/src/parser_utils.c
+++ b/src/parser_utils.c
@ -40,6 +40,7 @@ void init_state(struct parser_state *state){

   state->htmltag = 0;
   state->style = 0;
+   state->meta_content_type = 0;

   state->skip_html = 0;

@ -49,8 +50,10 @@ void init_state(struct parser_state *state){
   memset(state->message_id_hash, 0, 2*DIGEST_LENGTH+1);
   memset(state->miscbuf, 0, MAX_TOKEN_LEN);
   memset(state->qpbuf, 0, MAX_TOKEN_LEN);
+   memset(state->receivedbuf, 0, sizeof(state->receivedbuf));

   memset(state->type, 0, TINYBUFSIZE);
+   memset(state->charset, 0, TINYBUFSIZE);

   memset(state->attachment_name_buf, 0, SMALLBUFSIZE);
   state->anamepos = 0;
@ -67,6 +70,7 @@ void init_state(struct parser_state *state){

   state->writebufpos = 0;
   state->abufpos = 0;
+   state->received_header = 0;

   inithash(state->boundaries);
   inithash(state->rcpt);
@ -124,7 +128,6 @@ time_t parse_date_header(char *datestr){
   char *p, *q, *r, *tz, s[SMALLBUFSIZE], tzh[4], tzm[3];
   struct tm tm;

-   datestr += 5;
   p = datestr;

   for(; *datestr; datestr++){
@ -550,7 +553,7 @@ void markHTML(char *buf, struct parser_state *state){

            if(isspace(*s)){
               if(j > 0){
-                  setStateHTMLStyle(html, pos, state);
+                  setStateHTML(html, pos, state);
                  memset(html, 0, SMALLBUFSIZE); j=0;
               }
               pos++;
@ -575,23 +578,51 @@ void markHTML(char *buf, struct parser_state *state){

         if(j > 0){
            strncat(html, " ", SMALLBUFSIZE-1);
-            setStateHTMLStyle(html, pos, state);
+            setStateHTML(html, pos, state);
            memset(html, 0, SMALLBUFSIZE); j=0;
         }
+         state->meta_content_type = 0;
      }

   }

   //printf("append last in line:*%s*, html=+%s+, j=%d\n", puf, html, j);
-   if(j > 0){ setStateHTMLStyle(html, pos, state); }
+   if(j > 0){ setStateHTML(html, pos, state); }

   strcpy(buf, puf);
 }


-void setStateHTMLStyle(char *htmlbuf, int pos, struct parser_state *state){
+void setStateHTML(char *htmlbuf, int pos, struct parser_state *state){
   if(pos == 0 && strncmp(htmlbuf, "style ", 6) == 0) state->style = 1;
   if(pos == 0 && strncmp(htmlbuf, "/style ", 7) == 0) state->style = 0;
+
+   if(pos == 0 && state->charset[0] == 0 && strncmp(htmlbuf, "meta ", 5) == 0) state->meta_content_type = 0x1;
+   if(state->meta_content_type){
+     if((state->meta_content_type & 0x2) == 0 && strstr(htmlbuf, "http-equiv=content-type "))
+       state->meta_content_type |= 0x2;
+
+     if((state->meta_content_type & 0x4) == 0 && strstr(htmlbuf, "content=text/html;"))
+       state->meta_content_type |= 0x4;
+
+     if(state->meta_content_type == 0x7){
+       char *p, *q;
+
+       p = strstr(htmlbuf, "charset=");
+       if(p){
+         p += 8;
+         for(q = p; isalnum(*q) || index("-_", *q); q++)
+           ;
+
+         if(q > p && q-p+1 < (int) sizeof(state->charset)){
+           syslog(LOG_PRIORITY, "Changing HTML charset from '%s' to '%*s' due to meta tag", state->charset, (int)(q-p), p);
+           strncpy(state->charset, p, q-p);
+           state->charset[q-p+1] = '\0';
+           state->meta_content_type = 0;
+         }
+       }
+     }
+   }
 }


--- a/src/piler.h
+++ b/src/piler.h
@ -38,6 +38,7 @@ int do_av_check(char *filename, struct config *cfg);
 int make_digests(struct session_data *sdata, struct config *cfg);
 void digest_file(char *filename, char *digest);
 void digest_string(char *s, char *digest);
+void create_md5_from_email_address(char *puf, char *md5buf);

 void remove_stripped_attachments(struct parser_state *state);
 int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg);
--- a/src/pileraget.c
+++ b/src/pileraget.c
@ -46,7 +46,7 @@ int main(int argc, char **argv){
      return 1;
   }

-   snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
+   snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, argv[1][24], argv[1][25], argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
 #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT
   if(stat(filename, &st)){
      snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][RND_STR_LEN-6], argv[1][RND_STR_LEN-5], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2]));
--- a/src/pilerimport.c
+++ b/src/pilerimport.c
@ -53,6 +53,7 @@ void usage(){
   printf("    -j <failed folder>                Move failed to import emails to this folder\n");
   printf("    -a <recipient>                    Add recipient to the To:/Cc: list\n");
   printf("    -T <id>                           Update import table at id=<id>\n");
+   printf("    -Z <ms>                           Delay Z milliseconds in between emails being imported\n");
   printf("    -D                                Dry-run, do not import anything\n");
   printf("    -y                                Read pilerexport data from stdin\n");
   printf("    -o                                Only download emails for POP3/IMAP import\n");
@ -100,6 +101,7 @@ int main(int argc, char **argv){
   import.tot_msgs = 0;
   import.table_id = 0;
   import.folder = NULL;
+   import.delay = 0;

   data.import = &import;

@ -136,6 +138,7 @@ int main(int argc, char **argv){
            {"timeout",      required_argument,  0,  't' },
            {"start-position",required_argument,  0,  's' },
            {"table-id",     required_argument,  0,  'T' },
+            {"delay",        required_argument,  0,  'Z' },
            {"quiet",        no_argument,        0,  'q' },
            {"recursive",    no_argument,        0,  'R' },
            {"remove-after-import",no_argument,  0,  'r' },
@ -150,9 +153,9 @@ int main(int argc, char **argv){

      int option_index = 0;

-      int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:yDRroqh?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:yDRroqh?", long_options, &option_index);
 #else
-      int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:yDRroqh?");
+      int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:yDRroqh?");
 #endif

      if(c == -1) break;
@ -271,6 +274,15 @@ int main(int argc, char **argv){
                    data.import->table_id = atoi(optarg);
                    break;

+         case 'Z' :
+                    if(atoi(optarg) < 1){
+                       printf("invalid delay value: %s\n", optarg);
+                       return -1;
+                    }
+
+                    data.import->delay = atoi(optarg);
+                    break;
+
         case 'y' :
                    read_from_pilerexport = 1;
                    break;
@ -316,6 +328,9 @@ int main(int argc, char **argv){
   /* make sure we don't discard messages without a valid Message-Id when importing manually */
   cfg.archive_emails_not_having_message_id = 1;

+   /* The mmap_dedup_test feature is expected to work with the piler daemon only */
+   cfg.mmap_dedup_test = 0;
+
   if(read_key(&cfg)){
      printf("%s\n", ERR_READING_KEY);
      return ERR;
--- a/src/pop3.c
+++ b/src/pop3.c
@ -23,16 +23,6 @@
 #include <piler.h>


-int is_last_complete_pop3_packet(char *s, int len){
-
-   if(*(s+len-5) == '\r' && *(s+len-4) == '\n' && *(s+len-3) == '.' && *(s+len-2) == '\r' && *(s+len-1) == '\n'){
-      return 1;
-   }
-
-   return 0;
-}
-
-
 int connect_to_pop3_server(struct data *data){
   char buf[MAXBUFSIZE];

@ -86,66 +76,86 @@ void get_number_of_total_messages(struct data *data){


 int pop3_download_email(struct data *data, int i){
-   int n, fd, pos=0, lastpos=0, nreads=0;
-   char *p, buf[MAXBUFSIZE];
-   char aggrbuf[3*MAXBUFSIZE];
+   char *p, buf[MAXBUFSIZE], savedbuf[MAXBUFSIZE], copybuf[2*MAXBUFSIZE];

   data->import->processed_messages++;

   snprintf(data->import->filename, SMALLBUFSIZE-1, "pop3-tmp-%d-%d.txt", getpid(), i);
   unlink(data->import->filename);

-   fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
+   int fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR);
   if(fd == -1){
      printf("cannot open: %s\n", data->import->filename);
      return ERR;
   }

+   memset(savedbuf, 0, sizeof(savedbuf));
+
   snprintf(buf, sizeof(buf)-1, "RETR %d\r\n", i);
   write1(data->net, buf, strlen(buf));

-   memset(aggrbuf, 0, sizeof(aggrbuf));
+   int nlines = 0;
+   int endofmessage = 0;
+   int savedlen = 0;
+   int n = 0;

   while((n = recvtimeoutssl(data->net, buf, sizeof(buf))) > 0){
-      nreads++;
+      int remaininglen = n;

-      if(nreads == 1){
+      if(savedlen){
+         memset(copybuf, 0, sizeof(copybuf));
+         memcpy(copybuf, savedbuf, savedlen);
+         memcpy(&copybuf[savedlen], buf, n);

-         if(strncmp(buf, "+OK", 3) == 0){
-            p = strchr(&buf[3], '\n');
-            if(p){
-               *p = '\0';
-               pos = strlen(buf)+1;
-               *p = '\n';
-            }
-         }
-         else { printf("error: %s", buf); return ERR; }
+         remaininglen += savedlen;

+         savedlen = 0;
+         memset(savedbuf, 0, sizeof(savedbuf));
+
+         p = &copybuf[0];
+      } else {
+         p = &buf[0];
      }

-      if((uint)(lastpos + 1 + n) < sizeof(aggrbuf)){
+      int puflen=0;
+      int rc=OK;
+      int nullbyte=0;

-         if(nreads == 1){
-            memcpy(aggrbuf+lastpos, buf+pos, n-pos);
-            lastpos += n-pos;
-         }
-         else {
-            memcpy(aggrbuf+lastpos, buf, n);
-            lastpos += n;
-         }
-      }
-      else {
-         if(write(fd, aggrbuf, sizeof(buf)) == -1) printf("ERROR: writing to fd\n");
+      do {
+         char puf[MAXBUFSIZE];

-         memmove(aggrbuf, aggrbuf+sizeof(buf), lastpos-sizeof(buf));
-         lastpos -= sizeof(buf);
+         puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf)-1, &rc, &nullbyte);
+         remaininglen -= puflen;
+         nlines++;

-         memcpy(aggrbuf+lastpos, buf, n);
-         lastpos += n;
+         if(nlines == 1){
+            if(strncmp(puf, "+OK", 3)){
+               printf("error: %s", puf);
+               return ERR;
+            }
+         } else {
+            if(puf[puflen-1] == '\n'){
+               if(puflen == 3 && puf[0] == '.' && puf[1] == '\r' && puf[2] == '\n'){
+                  endofmessage = 1;
+                  break;
               }

-      if(is_last_complete_pop3_packet(aggrbuf, lastpos) == 1){
-         if(write(fd, aggrbuf, lastpos-3) == -1) printf("ERROR: writing to fd\n");
+               int dotstuff = 0;
+               if(puf[0] == '.' && puf[1] != '\r' && puf[1] != '\n') dotstuff = 1;
+
+               if(write(fd, &puf[dotstuff], puflen-dotstuff) == -1) printf("ERROR: writing to fd\n");
+
+            } else if(puflen > 0) {
+               savedlen = puflen;
+               snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
+            }
+         }
+
+         p += puflen;
+
+      } while(puflen > 0);
+
+      if(endofmessage){
         break;
      }
   }
--- a/src/reindex.c
+++ b/src/reindex.c
@ -75,12 +75,10 @@ uint64 get_max_meta_id(struct session_data *sdata){

 uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *data, uint64 from_id, uint64 to_id, struct config *cfg){
   char s[SMALLBUFSIZE];
-   uint64 stored_id=0, reindexed=0, delta;
+   uint64 stored_id=0, reindexed=0;
   struct parser_state state;
   struct sql sql;

-   delta = to_id - from_id;
-
   if(cfg->enable_folders == 1)
      snprintf(s, sizeof(s)-1, "SELECT m.`id`, `piler_id`, `arrived`, `sent`, f.folder_id FROM %s m, %s f WHERE m.id=f.id AND (m.id BETWEEN %llu AND %llu) AND `deleted`=0", SQL_METADATA_TABLE, SQL_FOLDER_MESSAGE_TABLE, from_id, to_id);
   else
@ -123,6 +121,12 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da

            snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);

+            struct stat st;
+            sdata->tot_len = stat(filename, &st) == 0 ? st.st_size : 0;
+
+            sdata->internal_sender = sdata->internal_recipient = sdata->external_recipient = sdata->direction = 0;
+            memset(sdata->attachments, 0, SMALLBUFSIZE);
+
            state = parse_message(sdata, 1, data, cfg);
            post_parse(sdata, &state, cfg);

@ -137,6 +141,8 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da
            unlink(filename);

            if(progressbar){
+               uint64 delta = to_id - from_id + 1;
+
               printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta));
               fflush(stdout);
            }
--- a/src/session.c
+++ b/src/session.c
@ -99,6 +99,7 @@ void init_smtp_session(struct smtp_session *session, int slot, int sd, char *cli
   session->net.ctx = NULL;
   session->net.ssl = NULL;

+   session->nullbyte = 0;
   session->last_data_char = 0;

   session->fd = -1;
@ -160,11 +161,13 @@ void tear_down_session(struct smtp_session **sessions, int slot, int *num_connec


 void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg){
-   int puflen, rc;
+   int puflen, rc, nullbyte;
   char *p, copybuf[BIGBUFSIZE+MAXBUFSIZE], puf[MAXBUFSIZE];

   // if there's something in the saved buffer, then let's merge them

+   int remaininglen = readlen + session->buflen;
+
   if(session->buflen > 0){
      memset(copybuf, 0, sizeof(copybuf));

@ -183,29 +186,41 @@ void handle_data(struct smtp_session *session, char *readbuf, int readlen, struc


   do {
-      puflen = read_one_line(p, '\n', puf, sizeof(puf)-1, &rc);
+      puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf)-1, &rc, &nullbyte);
      p += puflen;
+      remaininglen -= puflen;
+
+      if(nullbyte){
+         session->nullbyte = 1;
+      }
+
+      // complete line: rc == OK and puflen > 0
+      // incomplete line with something in the buffer: rc == ERR and puflen > 0

      if(puflen > 0){
         // Update lasttime if we have a line to process
         time(&(session->lasttime));

-         // pass the puffer to process_data() only if there was an '\n'
-         // on the line or the puffer does not start with a period
-         if(session->protocol_state == SMTP_STATE_DATA && (rc == OK || puf[0] != '.')){
+         // Save incomplete line to buffer
+         if(rc == ERR){
+            memcpy(session->buf, puf, puflen);
+            session->buflen = puflen;
+         }
+
+         // We have a complete line to process
+
+         if(rc == OK){
+            if(session->protocol_state == SMTP_STATE_BDAT){
+               process_bdat(session, puf, puflen, cfg);
+            }
+            else if(session->protocol_state == SMTP_STATE_DATA){
               sig_block(SIGALRM);
               process_data(session, puf, puflen);
               sig_unblock(SIGALRM);
            }
-         else if(session->protocol_state == SMTP_STATE_BDAT){
-            process_bdat(session, puf, puflen, cfg);
-         }
-         else if(rc == OK){
+            else {
               process_smtp_command(session, puf, cfg);
            }
-         else {
-            snprintf(session->buf, MAXBUFSIZE-1, "%s", puf);
-            session->buflen = puflen;
         }
      }

--- a/src/smtp.c
+++ b/src/smtp.c
@ -84,14 +84,24 @@ void process_data(struct smtp_session *session, char *buf, int buflen){
      // write line to file
      int written=0, n_writes=0;

+      // In the DATA phase skip the 1st character if it's a dot (.)
+      // and there are more characters before the trailing CR-LF
+      //
+      // See https://www.ietf.org/rfc/rfc5321.html#section-4.5.2 for more.
+
+      int dotstuff = 0;
+      if(*buf == '.' && buflen > 1 && *(buf+1) != '\r' && *(buf+1) != '\n') dotstuff = 1;
+
      while(written < buflen) {
-         int len = write(session->fd, buf+written, buflen-written);
+         int len = write(session->fd, buf+dotstuff+written, buflen-dotstuff-written);
+
         n_writes++;

         if(len > 0){
-            if(len != buflen) syslog(LOG_PRIORITY, "WARN: partial write: %d/%d bytes (round: %d)", len, buflen, n_writes);
-            written += len;
+            if(len != buflen-dotstuff) syslog(LOG_PRIORITY, "WARN: partial write: %d/%d bytes (round: %d)", len, buflen-dotstuff, n_writes);
+            written += len + dotstuff;
            session->tot_len += len;
+            dotstuff = 0;
         }
         else syslog(LOG_PRIORITY, "ERROR (line: %d) process_data(): written %d bytes", __LINE__, len);
      }
@ -171,10 +181,14 @@ int init_ssl(struct smtp_session *session){
      return 0;
   }

+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+   SSL_CTX_set_options(session->net.ctx, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3|SSL_OP_NO_TLSv1|SSL_OP_NO_TLSv1_1);
+#else
   if(SSL_CTX_set_min_proto_version(session->net.ctx, session->cfg->tls_min_version_number) == 0){
      syslog(LOG_PRIORITY, "failed SSL_CTX_set_min_proto_version() to %s/%d", session->cfg->tls_min_version, session->cfg->tls_min_version_number);
      return 0;
   }
+#endif

   if(SSL_CTX_set_cipher_list(session->net.ctx, session->cfg->cipher_list) == 0){
      syslog(LOG_PRIORITY, "failed to set cipher list: '%s'", session->cfg->cipher_list);
--- a/src/test.c
+++ b/src/test.c
@ -153,6 +153,7 @@ int main(int argc, char **argv){
   printf("from: *%s (%s)*\n", state.b_from, state.b_from_domain);
   printf("sender: *%s (%s)*\n", state.b_sender, state.b_sender_domain);
   printf("to: *%s (%s)*\n", state.b_to, state.b_to_domain);
+   printf("journal recipients: *%s*\n", state.b_journal_to);
   printf("reference: *%s*\n", state.reference);
   printf("subject: *%s*\n", state.b_subject);
   printf("body: *%s*\n", state.b_body);
@ -183,7 +184,7 @@ int main(int argc, char **argv){
   clearhash(data.mydomains);

   for(i=1; i<=state.n_attachments; i++){
-      printf("i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s\n", i, state.attachments[i].filename, state.attachments[i].type, state.attachments[i].size, state.attachments[i].internalname, state.attachments[i].digest);
+      printf("i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, dumped: %d, digest: %s\n", i, state.attachments[i].filename, state.attachments[i].type, state.attachments[i].size, state.attachments[i].internalname, state.attachments[i].dumped, state.attachments[i].digest);
      unlink(state.attachments[i].internalname);
   }

@ -195,6 +196,8 @@ int main(int argc, char **argv){

   printf("spam: %d\n", sdata.spam_message);

+   printf("1st received line: %s\n", state.receivedbuf);
+
   if(sdata.internal_sender == 0 && sdata.internal_recipient == 0) printf("NOT IN mydomains\n");

   printf("\n\n");
--- a/src/tokenizer.c
+++ b/src/tokenizer.c
@ -50,6 +50,8 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata,
         continue;
      }

+      char md5buf[2*MD5_DIGEST_LENGTH+2];
+
      if(state->message_state == MSG_FROM && state->is_1st_header == 1 && strlen(state->b_from) < SMALLBUFSIZE-len-1){
         strtolower(puf);

@ -69,6 +71,11 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata,

            if(is_email_address_on_my_domains(puf, data) == 1) sdata->internal_sender = 1;

+            if(len >= MAX_EMAIL_ADDRESS_SPHINX_LEN && strlen(state->b_from) < SMALLBUFSIZE-len-1){
+               create_md5_from_email_address(puf, md5buf);
+               memcpy(&(state->b_from[strlen(state->b_from)]), md5buf, strlen(md5buf));
+            }
+
            if(strlen(state->b_from) < SMALLBUFSIZE-len-1){
               split_email_address(puf);
               memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
@ -88,6 +95,11 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata,
               memcpy(&(state->b_sender_domain), q+1, strlen(q+1)-1);
            }

+            if(len >= MAX_EMAIL_ADDRESS_SPHINX_LEN && strlen(state->b_sender) < SMALLBUFSIZE-len-1){
+               create_md5_from_email_address(puf, md5buf);
+               memcpy(&(state->b_sender[strlen(state->b_sender)]), md5buf, strlen(md5buf));
+            }
+
            if(strlen(state->b_sender) < SMALLBUFSIZE-len-1){
               split_email_address(puf);
               memcpy(&(state->b_sender[strlen(state->b_sender)]), puf, len);
@ -101,12 +113,18 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata,
         q = strchr(puf, '@');
         if(q) fix_plus_sign_in_email_address(puf, &q, &len);

-         if((state->message_state == MSG_RECIPIENT || state->message_state == MSG_ENVELOPE_TO) && findnode(state->journal_recipient, puf) == NULL){
+         if((state->message_state == MSG_RECIPIENT || state->message_state == MSG_ENVELOPE_TO) && findnode(state->journal_recipient, puf) == NULL && state->journaltolen < sizeof(state->b_journal_to)-len-1){
            addnode(state->journal_recipient, puf);
            memcpy(&(state->b_journal_to[state->journaltolen]), puf, len);
+            state->journaltolen += len;
            if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: journal rcpt: '%s'", sdata->ttmpfile, puf);
         }

+         if(len >= MAX_EMAIL_ADDRESS_SPHINX_LEN){
+            create_md5_from_email_address(puf, md5buf);
+            add_recipient(md5buf, strlen(md5buf), sdata, state, data, cfg);
+         }
+
         add_recipient(puf, len, sdata, state, data, cfg);
      }
      else if(state->message_state == MSG_BODY && len >= (unsigned int)(cfg->min_word_len) && state->bodylen < BIGBUFSIZE-len-1){
--- a/tests/cases/05-smtp.inc
+++ b/tests/cases/05-smtp.inc
@ -15,13 +15,14 @@ case1() {
   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam2" --socket --no-counter
   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/journal" --socket --no-counter
   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/deduptest" --socket --no-counter
+   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/special" --socket --no-counter
   "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus" --socket --no-counter


-   wait_until_emails_are_processed "piler1" 3007
+   wait_until_emails_are_processed "piler1" 3018
   docker exec "piler1" su piler -c /usr/libexec/piler/indexer.delta.sh 2>/dev/null

-   count_status_values 3007 2896 111 0
+   count_status_values 3018 2907 111 0

   test_retrieved_messages_are_the_same "piler1" "piler"

--- a/tests/setup.inc
+++ b/tests/setup.inc
@ -13,6 +13,13 @@ setup() {
   add_data_officer "piler1"
 }

+cleanup_package() {
+   local pkg="$1"
+
+   log "Removing ${PACKAGE_DIR}/${pkg}"
+   rm -f "${PACKAGE_DIR}/${pkg}"
+}
+
 launch_containers() {

   log "starting syslog server"
@ -28,12 +35,14 @@ launch_containers() {
      "$docker_image"

   wait_for_sleep_cycle_in_container "piler1"
+
+   cleanup_package "$PACKAGE"
 }

 create_rules() {
   local container="$1"

-   echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"acts.hu\",\"acts.hu\"),(\"gtsce.com\",\"gtsce.com\"),(\"datanet.hu\",\"datanet.hu\"),(\"gtsdatanet.hu\",\"gtsdatanet.hu\"),(\"gts.hu\",\"gts.hu\"),(\"aaa.fu\",\"aaa.fu\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh
+   echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"address.com\",\"address.com\"),(\"acts.hu\",\"acts.hu\"),(\"gtsce.com\",\"gtsce.com\"),(\"datanet.hu\",\"datanet.hu\"),(\"gtsdatanet.hu\",\"gtsdatanet.hu\"),(\"gts.hu\",\"gts.hu\"),(\"aaa.fu\",\"aaa.fu\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh

   echo 'echo "insert into archiving_rule (subject) values (\"Android táblagép\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
   echo 'echo "insert into archiving_rule (\`from\`) values (\"@gmail.com\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh
--- a/unit_tests/check_parser.c
+++ b/unit_tests/check_parser.c
@ -39,7 +39,7 @@ static void test_parser(struct config *cfg){
      {"18-spam-html-encoding.eml", "<list-435458392@mail.aaa.fu>", "a1 hitelcentrum kft Üveges szilvia a1hitelcentrum@t-online.hu a1hitelcentrum t online hu ", "t-online.hu", "postmaster postmaster@aaa.fu postmaster aaa fu ", "aaa.fu", "postmaster@aaa.fu postmaster aaa fu a1hitelcentrum@t-online.hu a1hitelcentrum t online hu ", "aaa.fu t-online.hu ", "", "TÁJÉKOZTATÁSVargay Péter", 0},
      {"19-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-186.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "", "", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1},
      {"20-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-187.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "", "", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1},
-      {"21-register-tricky-urls.eml", "<E1IBifn-0001un-MD@admin4.theregister.co.uk>", "the register update-49363-08f0f768@list.theregister.co.uk update 49363 08f0f768 list theregister co uk ", "list.theregister.co.uk", "", "", "hello@mail.aaa.fu hello mail aaa fu ", "mail.aaa.fu ", "", "[sp@m] Reg Headlines Friday July 20", 0},
+      {"21-register-tricky-urls.eml", "<E1IBifn-0001un-MD@admin4.theregister.co.uk>", "the register update-49363-08f0f768@list.theregister.co.uk 30cbee0b0f411fcf170416fb9f996c6f update 49363 08f0f768 list theregister co uk ", "list.theregister.co.uk", "", "", "hello@mail.aaa.fu hello mail aaa fu ", "mail.aaa.fu ", "", "[sp@m] Reg Headlines Friday July 20", 0},
      {"30-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", "RE: hxx-ajajajaja.com_ Aaagágyi és kia ttt_webstat hiba", 0},
      {"31-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", "Re: stanhu \"domain not found\"-dal eldobja a @fohu-ra küldött leveleket...", 0},
      {"32-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", "<GD-XXXX/1-2015> www.ujsag.hu new virtual host reg. --> Aaaaaaaaa", 0},
--- a/util/db-mysql.sql
+++ b/util/db-mysql.sql
@ -9,9 +9,9 @@ create table if not exists `sph_counter` (
 create table if not exists `sph_index` (
  `id` bigint not null,
  `from` tinyblob default null,
-  `to` text(8192) default null,
-  `fromdomain` char(255) default null,
-  `todomain` text(512) default null,
+  `to` blob(8192) default null,
+  `fromdomain` tinyblob default null,
+  `todomain` blob(512) default null,
  `subject` blob(512) default null,
  `arrived` int unsigned not null,
  `sent` int unsigned not null,
@ -73,7 +73,7 @@ create index `rcpt_idx3` on `rcpt`(`todomain`);


 drop view if exists `v_messages`;
-create view `v_messages` AS select `metadata`.`id` AS `id`,`metadata`.`piler_id` AS `piler_id`,`metadata`.`from` AS `from`,`metadata`.`fromdomain` AS `fromdomain`,`rcpt`.`to` AS `to`,`rcpt`.`todomain` AS `todomain`,`metadata`.`subject` AS `subject`, `metadata`.`size` AS `size`, `metadata`.`direction` AS `direction`, `metadata`.`sent` AS `sent`, `metadata`.`retained` AS `retained`, `metadata`.`arrived` AS `arrived`, `metadata`.`digest` AS `digest`, `metadata`.`bodydigest` AS `bodydigest`, `metadata`.`deleted` AS `deleted` from (`metadata` join `rcpt`) where (`metadata`.`id` = `rcpt`.`id`);
+create view `v_messages` AS select `metadata`.`id` AS `id`,`metadata`.`piler_id` AS `piler_id`,`metadata`.`from` AS `from`,`metadata`.`fromdomain` AS `fromdomain`,`rcpt`.`to` AS `to`,`rcpt`.`todomain` AS `todomain`,`metadata`.`subject` AS `subject`, `metadata`.`size` AS `size`, `metadata`.`direction` AS `direction`, `metadata`.`sent` AS `sent`, `metadata`.`retained` AS `retained`, `metadata`.`arrived` AS `arrived`, `metadata`.`digest` AS `digest`, `metadata`.`bodydigest` AS `bodydigest`, `metadata`.`deleted` AS `deleted`, `metadata`.`attachments` AS `attachments` from (`metadata` join `rcpt`) where (`metadata`.`id` = `rcpt`.`id`);


 create table if not exists `attachment` (
@ -463,7 +463,7 @@ create table if not exists `timestamp` (
  `id` bigint unsigned not null auto_increment,
  `start_id` bigint default 0,
  `stop_id` bigint default 0,
-  `hash_value` char(40),
+  `hash_value` varchar(128),
  `count` int default 0,
  `response_time` bigint default 0,
  `response_string` blob not null,
--- a/util/db-upgrade.sql
+++ b/util/db-upgrade.sql
@ -0,0 +1,8 @@
+alter table timestamp change column hash_value hash_value varchar(128) default null;
+
+alter table sph_index change column `to` `to` blob(8192) default null;
+alter table sph_index change column fromdomain fromdomain tinyblob default null;
+alter table sph_index change column todomain todomain blob(512) default null;
+
+
+alter table metadata change column subject `subject` blob(512) default null;
--- a/util/download-imap.php
+++ b/util/download-imap.php
@ -103,7 +103,7 @@ function saveMessages($storage, $folder = '', $num = 0) {

   $messages = $storage->piler_batch_fetch(1, $num);

-   while(list($k, $v) = each($messages)) {
+   foreach($messages as $k => $v) {
      $uuid = $storage->getUniqueId($k);

      $tmpname = "piler-" . $username . "-" . $folder . "-" . $k . "-" . $uuid . ".eml";
--- a/util/imapfetch.py
+++ b/util/imapfetch.py
@ -90,6 +90,7 @@ def main():
                        default="/etc/piler/piler.conf")
    parser.add_argument("-s", "--server", type=str, help="imap server")
    parser.add_argument("-P", "--port", type=int, help="port number", default=143)
+    parser.add_argument("--no_ssl", help="Do not use ssl/tls", action='store_true')
    parser.add_argument("-u", "--user", type=str, help="imap user")
    parser.add_argument("-p", "--password", type=str, help="imap password")
    parser.add_argument("-x", "--skip-list", type=str, help="IMAP folders to skip",
@ -116,12 +117,16 @@ def main():
    opts['verbose'] = args.verbose
    opts['search'] = 'ALL'
    opts['counter'] = 0
+    opts['use_ssl'] = True
    opts['db'] = None
    opts['id'] = 0

    if args.date:
        opts['search'] = args.date

+    if args.no_ssl:
+        opts['use_ssl'] = False
+
    server = ''
    user = ''
    password = ''
@ -152,7 +157,7 @@ def main():
    if opts['verbose']:
        print("Skipped folder list: {}".format(opts['skip_folders']))

-    if args.port == 993:
+    if opts['use_ssl']:
        conn = imaplib.IMAP4_SSL(server)
    else:
        conn = imaplib.IMAP4(server)
--- a/util/pilerpurge.py
+++ b/util/pilerpurge.py
@ -54,12 +54,26 @@ def purge_m_files(ids=[], opts={}):
        remove_m_files(ids, opts)

        # Set deleted=1 for aged metadata entries
+        # as well as clean other tables

        if opts['dry_run'] is False:
            cursor = opts['db'].cursor()
            format = ", ".join(['%s'] * len(ids))
-            cursor.execute("UPDATE metadata SET deleted=1 WHERE piler_id IN " +
+            cursor.execute("UPDATE metadata SET deleted=1, subject=NULL, `from`=''," +
+                           "fromdomain='', message_id='' WHERE piler_id IN " +
                           "(%s)" % (format), ids)
+
+            cursor.execute("DELETE FROM rcpt WHERE id IN (SELECT id FROM metadata " +
+                           "WHERE piler_id IN (%s))" % (format), ids)
+            cursor.execute("DELETE FROM note WHERE id IN (SELECT id FROM metadata " +
+                           "WHERE piler_id IN (%s))" % (format), ids)
+            cursor.execute("DELETE FROM tag WHERE id IN (SELECT id FROM metadata " +
+                           "WHERE piler_id IN (%s))" % (format), ids)
+            cursor.execute("DELETE FROM private WHERE id IN (SELECT id FROM metadata" +
+                           "WHERE piler_id IN (%s))" % (format), ids)
+            cursor.execute("DELETE FROM folder_message WHERE id IN (SELECT id FROM " +
+                           "metadata WHERE piler_id IN (%s))" % (format), ids)
+
            opts['db'].commit()


@ -85,7 +99,7 @@ def purge_attachments_by_attachment_id(opts={}):
    cursor = opts['db'].cursor()

    cursor.execute("SELECT i, piler_id, attachment_id, refcount FROM " +
-                   "v_attachment WHERE i IN (%s)" %
+                   "v_attachment WHERE refcount=0 AND i IN (%s)" %
                   (format), opts['referenced_attachments'])

    while True:
@ -160,12 +174,12 @@ def unlink(filename="", opts={}):


 def get_m_file_path(id='', opts={}):
-    return "/".join([opts['storedir'], opts['server_id'], id[8:11], id[32:34],
+    return "/".join([opts['storedir'], id[24:26], id[8:11], id[32:34],
                    id[34:36], id + ".m"])


 def get_attachment_file_path(piler_id='', attachment_id=0, opts={}):
-    return "/".join([opts['storedir'], opts['server_id'], piler_id[8:11],
+    return "/".join([opts['storedir'], piler_id[24:26], piler_id[8:11],
                    piler_id[32:34], piler_id[34:36], piler_id + ".a" +
                    str(attachment_id)])

--- a/util/postinstall.sh.in
+++ b/util/postinstall.sh.in
@ -286,6 +286,7 @@ make_cron_entries() {
      echo "30   6 * * * /usr/bin/php ${LIBEXECDIR}/piler/generate_stats.php --webui ${DOCROOT} >/dev/null";
      echo "*/5 * * * * /usr/bin/find ${LOCALSTATEDIR}/piler/error -type f|wc -l > ${LOCALSTATEDIR}/piler/stat/error";
      echo "*/5 * * * * /usr/bin/find ${DOCROOT}/tmp -type f -name i.\* -exec rm -f {} \;";
+      echo "#*/5 * * * * ${LIBEXECDIR}/piler/import.sh";
      echo "### PILEREND";
   } >> "$CRON_TMP"
 }
--- a/util/reindex.sh
+++ b/util/reindex.sh
@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+INSTALL_PREFIX=/usr/local
+THRESHOLD=100000000
+
+start_id=1
+stop_id=5000000
+
+while [[ $start_id -lt $stop_id ]]; do
+   y=$(( start_id + 9999 ))
+   echo $start_id $y
+   "${INSTALL_PREFIX}/bin/reindex" -f $start_id -t $y -p
+   "${INSTALL_PREFIX}/libexec/piler/indexer.delta.sh"
+   start_id=$(( start_id + 10000 ))
+
+   if [[ "$(stat -c %s /var/piler/sphinx/dailydelta1.spp)" -gt "$THRESHOLD" ]]; then
+      "${INSTALL_PREFIX}/libexec/piler/indexer.main.sh"
+   fi
+done
--- a/util/sign.php
+++ b/util/sign.php
@ -12,11 +12,13 @@ ini_set("session.save_path", "/tmp");
 $webuidir = "";
 $verbose = 0;
 $mode = "unit";
+$algo = "sha256";

 $opts = 'h::v';
 $lopts = array(
    'webui:',
    'mode:',
+    'algo:',
    'verbose'
    );

@ -37,6 +39,10 @@ if ( $options = getopt( $opts, $lopts ) )
       $mode = $options['mode'];
    }

+    if ( isset($options['algo']) ) {
+       $algo = $options['algo'];
+    }
+
    if ( isset($options['h']) )
    {
        display_help();
@ -66,6 +72,7 @@ Registry::set('db', $db);
 Registry::set('DB_DRIVER', DB_DRIVER);

 define('MODE', $mode);
+define('ALGO', $algo);


 $data = get_hash_values();
@ -121,29 +128,29 @@ function get_hash_values() {

   if($last_id == 0) {
      $start_id = TSA_START_ID;
-      if(MODE == 'unit') { $stop_id = $start_id + TSA_STAMP_REQUEST_UNIT_SIZE - 1; }
-      else { $stop_id = 1000000000; }
-   }
-   else {
+   } else {
      $start_id = $last_id + 1;
-      if(MODE == 'unit') { $stop_id = $start_id + TSA_STAMP_REQUEST_UNIT_SIZE - 1; }
-      else { $stop_id = 1000000000; }
   }

-   $query = $db->query("SELECT id, digest FROM " . TABLE_META . " WHERE id >= ? AND id <= ?", array($start_id, $stop_id));
+   if(MODE == 'unit') {
+      $limit = TSA_STAMP_REQUEST_UNIT_SIZE;
+   } else {
+      $limit = 100000; // stay well below default PHP memory_limit
+   }
+
+   $query = $db->query("SELECT id, digest FROM " . TABLE_META . " WHERE id >= ? ORDER BY id LIMIT $limit", array($start_id));

   foreach($query->rows as $q) {
      $count++;
+      $last_id = $q['id'];
      $s .= $q['digest'];
   }

-   if(MODE == 'time') { $stop_id = $start_id + $count - 1; }
-
   return [
      START_ID => $start_id,
-      STOP_ID => $stop_id,
+      STOP_ID => $last_id,
      COUNT => $count,
-      HASH_VALUE => sha1($s)
+      HASH_VALUE => hash(ALGO, $s)
   ];

 }
--- a/webui/Zend/Mail/Protocol/Imap.php
+++ b/webui/Zend/Mail/Protocol/Imap.php
@ -410,7 +410,7 @@ class Zend_Mail_Protocol_Imap

       $a = $this->requestAndResponse("LIST", array('""', '"*"'));

-       while(list($k, $v) = each($a)) {
+       foreach ($a as $k => $v) {
          if($v[1][0] == '\HasNoChildren') {
             array_push($folders, $v[3]);
          }
--- a/webui/controller/audit/helper.php
+++ b/webui/controller/audit/helper.php
@ -99,7 +99,7 @@ class ControllerAuditHelper extends Controller {
      $s = preg_replace("/\s{1,}/", " ", $s);
      $b = explode(" ", $s);

-      while(list($k, $v) = each($b)) {
+      foreach ($b as $k => $v) {
         if($v == '') { continue; }

         if(preg_match("/(login|loginfailed|logout|view|download|search|restore|journal)$/", $v) && isset($actions[$v])) { $this->a['action'] .= "\t" . $actions[$v]; }
--- a/webui/controller/search/helper.php
+++ b/webui/controller/search/helper.php
@ -140,7 +140,7 @@ class ControllerSearchHelper extends Controller {
      $s = preg_replace("/OR/", "|", $data['search']);
      $b = preg_split("/\s/", $s);

-      while(list($k, $v) = each($b)) {
+      foreach ($b as $k => $v) {
         if($v == '') { continue; }

         if(preg_match("/\d{4}\-\d{1,2}\-\d{1,2}/", $v) || preg_match("/\d{1,2}\/\d{1,2}\/\d{4}/", $v)) {
--- a/webui/controller/user/edit.php
+++ b/webui/controller/user/edit.php
@ -79,7 +79,7 @@ class ControllerUserEdit extends Controller {

            $this->data['emails'] = $this->model_user_user->get_emails($this->data['user']['username']);

-            $this->data['user']['group'] = $this->model_group_group->get_groups_by_email(array($this->data['emails']));
+            $this->data['user']['group'] = $this->model_group_group->get_groups_by_email(explode("\n", $this->data['emails']));
         //}
      }
      else {
--- a/webui/google-api/service/apiUtils.php
+++ b/webui/google-api/service/apiUtils.php
@ -55,7 +55,7 @@ class apiUtils {
    $strlenVar = strlen($str);
    $d = $ret = 0;
    for ($count = 0; $count < $strlenVar; ++ $count) {
-      $ordinalValue = ord($str{$ret});
+      $ordinalValue = ord($str[$ret]);
      switch (true) {
        case (($ordinalValue >= 0x20) && ($ordinalValue <= 0x7F)):
          // characters U-00000000 - U-0000007F (same as ASCII)
--- a/webui/language/cn/messages.php
+++ b/webui/language/cn/messages.php
@ -0,0 +1,497 @@
+<?php
+
+$_['text_60_minutes'] = "60 分钟";
+
+$_['text_action'] = "动作";
+$_['text_active_incoming_queue'] = "活动 + 传入队列";
+$_['text_active_incoming_queue_sender'] = "活动 + 传入队列与发件人";
+$_['text_ad_sync_status'] = "AD 同步状态";
+$_['text_add'] = "新增";
+$_['text_add_new_email_address'] = "新增邮件地址";
+$_['text_add_new_domain'] = "新增域名";
+$_['text_add_new_entry'] = "新增条目";
+$_['text_add_new_group'] = "新增组";
+$_['text_add_new_rule'] = "新增规则";
+$_['text_add_new_user_alias'] = "新增用户";
+$_['text_add_policy'] = "新增策略";
+$_['text_administration'] = "管理";
+$_['text_admin_user'] = "管理员";
+$_['text_advanced'] = "高级";
+$_['text_advanced_search'] = "高级搜索";
+$_['text_all'] = "所有";
+$_['text_any'] = "任一";
+$_['text_applied'] = "已套用";
+$_['text_apply_changes'] = "套用变更";
+$_['text_archive_size'] = "归档大小";
+$_['text_archive_size_before_compression'] = "Archive size before compression";
+$_['text_archived_messages'] = "已归档邮件";
+$_['text_archiving_rules'] = "归档规则";
+$_['text_assigned_email_addresses'] = "已指派邮件地址";
+$_['text_attachment'] = "附件";
+$_['text_attachment_name'] = "附件名称";
+$_['text_attachment_size'] = "附件大小";
+$_['text_attachment_type'] = "附件类型";
+$_['text_audit'] = "审计";
+$_['text_automated_search'] = "自动搜索";
+
+$_['text_back'] = "返回";
+$_['text_background_colour'] = "背景颜色";
+$_['text_body'] = "内文";
+$_['text_branding_logo'] = "品牌标志";
+$_['text_branding_text'] = "品牌文字";
+$_['text_branding_url'] = "品牌网址";
+$_['text_bulk_edit_selected_uids'] = "批次修改已选取的 uid";
+$_['text_bulk_restore_selected_emails'] = "批次还原已选取的項目";
+$_['text_bulk_update_selected_uids'] = "批次更新已选择的 uid";
+
+$_['text_cancel'] = "取消";
+$_['text_change_user_settings'] = "变更用户设定";
+$_['text_clienthost'] = "Client host";
+$_['text_close'] = "关闭";
+$_['text_colour'] = "颜色";
+$_['text_compressed'] = "已压缩";
+$_['text_confirm_to_reset_counters'] = "确认要重置计数器";
+$_['text_connection_failed'] = "连接失败";
+$_['text_connection_ok'] = "连接成功";
+$_['text_contact_support'] = "联系支持";
+$_['text_content_filter'] = "内容过滤器";
+$_['text_conversation_available'] = "可使用对话查看";
+$_['text_copied'] = "Copied";
+$_['text_counters'] = "计数器";
+$_['text_cpu_load'] = "CPU 负载";
+$_['text_cpu_usage'] = "CPU 使用率";
+$_['text_create_new_secret'] = "建立新的密钥";
+$_['text_create_note'] = "创建注释";
+$_['text_cumulative_counts'] = "累积计数";
+$_['text_customers'] = "Customers";
+
+$_['text_daily_quarantine_report'] = "每日隔离报表";
+$_['text_daily_quarantine_report_status'] = "每日隔离报表状态";
+$_['text_daily_report'] = "每日报表";
+$_['text_daily_piler_report'] = "每日 Piler 报表";
+$_['text_database_emails'] = "在 piler 资料库的邮件地址";
+$_['text_date'] = "日期";
+$_['text_date_from'] = "起始日期";
+$_['text_date_to'] = "结束日期";
+$_['text_days'] = "天";
+$_['text_days2'] = "天";
+$_['text_days_to_retain'] = "保留天数";
+$_['text_deferred_queue'] = "延迟队列";
+$_['text_deferred_queue_sender'] = "延迟队列与发件人";
+$_['text_delay'] = "延迟";
+$_['text_delete_confirm_message'] = "您是否要删除";
+$_['text_delete_reason'] = "移除原因";
+$_['text_deleted'] = "已删除";
+$_['text_deleted_users'] = "已删除";
+$_['text_deliver'] = "传递";
+$_['text_delivered'] = "已传递";
+$_['text_deliver_and_train_selected_messages'] = "Deliver and train selected messages";
+$_['text_deliver_and_train_selected_messages_as_ham'] = "Deliver and train selected messages AS HAM";
+$_['text_deliver_selected_messages'] = "传递已选择的邮件";
+$_['text_description'] = "描述";
+$_['text_direction'] = "Direction";
+$_['text_disk_usage'] = "磁盘使用率";
+$_['text_disable'] = "停用";
+$_['text_disabled'] = "停用";
+$_['text_dn_asterisk_means_skip_sync'] = "万用字元 (*) 表示这个用户项目将不会包含在 AD 同步当中";
+$_['text_domain'] = "域";
+$_['text_domains'] = "域";
+$_['text_domainname'] = "域名称";
+$_['text_download_all_hits_as_eml'] = "全部下载 (EML)";
+$_['text_download_selected_hits_as_pdf'] = "下载已选取项目 (PDF)";
+$_['text_download_attachment2'] = "下载附件";
+$_['text_download_message'] = "下載 (EML)";
+$_['text_download_message2'] = "下载邮件";
+
+$_['text_edit'] = "编辑";
+$_['text_edit_entry'] = "编辑条目";
+$_['text_edit_group'] = "编辑组";
+$_['text_edit_user'] = "编辑用户";
+$_['text_edit_or_view'] = "编辑/查看";
+$_['text_email'] = "邮件地址";
+$_['text_email_addresses'] = "邮件地址";
+$_['text_email_aliases'] = "邮件别名";
+$_['text_email_in_unknown_domain'] = "邮件地址在未知的域";
+$_['text_empty_search_criteria'] = "'Empty criteria'";
+$_['text_empty_search_result'] = "没有搜寻结果。请尝试加入万用字元 (*) 在关键字后方 (最小 " . MIN_PREFIX_LEN . " 个字元)，例如 duplic* 以搜索到 \"duplicate\", \"duplicated\", 之类的结果。";
+$_['text_enable'] = "启用";
+$_['text_enabled'] = "启用";
+$_['text_enter_google_authenticator_code'] = "输入 Google Authenticator 验证码";
+$_['text_enter_one_email_address_per_line'] = "每一行输入一个邮件地址";
+$_['text_enter_one_group_per_line'] = "每一行输入一个群组";
+$_['text_enter_search_terms'] = "输入您要搜索的字词";
+$_['text_error'] = "错误";
+$_['text_exact_domain_name_or_email_address'] = "确认您的域名或邮件地址";
+$_['text_exclude'] = "排除";
+$_['text_existing'] = "现有";
+$_['text_existing_domains'] = "现有域";
+$_['text_existing_email'] = "现有邮件";
+$_['text_existing_folders'] = "现有资料夹";
+$_['text_existing_groups'] = "现有组";
+$_['text_existing_policies'] = "现有策略";
+$_['text_existing_entries'] = "现有条目";
+$_['text_existing_rules'] = "现有规则";
+$_['text_existing_user'] = "现有用户";
+$_['text_existing_users'] = "现有用户";
+$_['text_expert'] = "专家";
+$_['text_expert_search'] = "专家搜索";
+
+$_['text_failed'] = "失败";
+$_['text_failed_to_add'] = "新增失败";
+$_['text_failed_to_change_password'] = "变更密码失败";
+$_['text_failed_to_deliver'] = "传递失败";
+$_['text_failed_to_mark_for_removal'] = "标记移除失败";
+$_['text_failed_to_modify'] = "修改失败";
+$_['text_failed_to_remove'] = "移除失败";
+$_['text_failed_to_restore'] = "还原失败";
+$_['text_failed_to_update'] = "更新失败";
+$_['text_first'] = "First";
+$_['text_folder'] = "文件夹";
+$_['text_folder_rules'] = "文件夹规则";
+$_['text_folders'] = "文件夹";
+$_['text_forward_selected_emails_to'] = "转寄已选择邮件至";
+$_['text_from'] = "发件人";
+$_['text_from_domain'] = "发件人域";
+
+$_['text_google_authenticator_code'] = "Google Authenticator 验证码";
+$_['text_google_authenticator_settings'] = "Google Authenticator 设定";
+$_['text_group_id'] = "组 id";
+$_['text_groupname'] = "组名称";
+$_['text_groups'] = "组";
+$_['text_group_management'] = "组管理";
+$_['text_group_membership'] = "组成员";
+
+$_['text_health'] = "健康状况";
+$_['text_health_monitor'] = "健康状况监视器";
+$_['text_help'] = "帮助";
+$_['text_history'] = "历史";
+$_['text_home'] = "首页";
+
+$_['text_image'] = "image";
+$_['text_import'] = "汇入";
+$_['text_import_job_delete_confirm_message'] = "您是否要删除汇入作业";
+$_['text_import_users'] = "汇入用户";
+$_['text_import_users_from_LDAP'] = "从 LDAP 汇入用户";
+$_['text_inbound'] = "inbound";
+$_['text_indexer_job'] = "索引器作业";
+$_['text_install_sudo_apply'] = "加入以下内容到 /etc/sudoers: 'www-data ALL=NOPASSWD: /etc/init.d/rc.piler reload'";
+$_['text_internal'] = "internal";
+$_['text_invalid_data'] = "无效的资料";
+$_['text_invalid_email'] = "无效的邮件";
+$_['text_invalid_email_or_password'] = "无效的邮件或密码";
+$_['text_invalid_gid'] = "无效的 gid";
+$_['text_invalid_password'] = "无效的密码";
+$_['text_invalid_pin_code'] = "无效的 Pin 码";
+$_['text_invalid_policy_group'] = "无效的策略组";
+$_['text_invalid_policy_name'] = "无效的策略名称";
+$_['text_invalid_policy_setting'] = "无效的策略设定";
+$_['text_invalid_uid'] = "无效的 uid";
+$_['text_invalid_username'] = "无效的用户名称";
+$_['text_ipaddr'] = "IP 位址";
+
+$_['text_language'] = "语言";
+$_['text_last'] = "Last";
+$_['text_last_activity'] = "最后活动";
+$_['text_last_update'] = "最后更新";
+$_['text_latest_emails'] = "最新邮件";
+$_['text_ldap'] = "LDAP";
+$_['text_ldap_auditor_member_dn'] = "审计成员 DN";
+$_['text_ldap_base_dn'] = "LDAP 基础 DN";
+$_['text_ldap_bind_dn'] = "LDAP 绑定 DN";
+$_['text_ldap_bind_pw'] = "LDAP 绑定 密码";
+$_['text_ldap_host'] = "LDAP 主机";
+$_['text_ldap_type'] = "LDAP 类型";
+$_['text_legal_hold'] = "法务保存措施";
+$_['text_load'] = "载入";
+$_['text_loading'] = "载入中";
+$_['text_logged_in'] = "已登入";
+$_['text_logged_out'] = "您已登出";
+$_['text_login'] = "登入";
+$_['text_login2'] = "登入";
+$_['text_login_failed'] = "登入失败";
+$_['text_login_via_google'] = "经由 Google 账户登入";
+$_['text_logout'] = "登出";
+$_['text_logout2'] = "登出";
+
+$_['text_maillog_status'] = "邮件记录收集器状态";
+$_['text_main_title'] = "clapf web UI";
+$_['text_mapped_domain'] = "对应域";
+$_['text_mark_private'] = "隐私";
+$_['text_marked_for_removal'] = "邮件标示为移除";
+$_['text_memory_usage'] = "内存使用率";
+$_['text_message'] = "邮件";
+$_['text_messages'] = "邮件";
+$_['text_message_disposition'] = "邮件处理结果计数";
+$_['text_message_text'] = "邮件文字";
+$_['text_min_2_chars'] = "最小 2 个字元";
+$_['text_missing_data'] = "遗失资料";
+$_['text_missing_password'] = "遗失密码";
+$_['text_modify'] = "修改";
+$_['text_monitor'] = "监视器";
+$_['text_months'] = "月";
+$_['text_monthly_report'] = "每月报表";
+
+$_['text_need_to_approve_removal'] = "移除需要批准";
+$_['text_new'] = "新增";
+$_['text_new_users'] = "新增";
+$_['text_next'] = "Next";
+$_['text_no_domain_found'] = '没有找到域';
+$_['text_no_email_found'] = '没有找到邮件';
+$_['text_no_message_in_the_quarantine'] = "隔离区内没有符合搜索结果的邮件";
+$_['text_no_records'] = "没有记录";
+$_['text_no_selected_message'] = "没有选择邮件";
+$_['text_no_sender'] = "没有发件人";
+$_['text_no_spam_message_in_the_quarantine_yet'] = "隔离区中没有垃圾邮件";
+$_['text_no_subject'] = "没有主旨";
+$_['text_no_such_policy'] = "没有策略";
+$_['text_non_existent_queue_directory'] = "您指定的队列不存在";
+$_['text_non_existing_user'] = "无现有的用户";
+$_['text_notes'] = "注释";
+$_['text_not_found'] = "没有找到";
+$_['text_not_running'] = "未在执行中";
+$_['text_not_spam'] = "不是垃圾邮件";
+$_['title_not_found'] = "Page not found";
+$_['text_number_of_messages_in_quarantine'] = "隔离区内符合搜索结果的邮件数量";
+$_['text_number_of_spam_messages_in_quarantine'] = "隔离区内符合搜索结果的垃圾邮件数量";
+
+$_['text_off'] = "关闭";
+$_['text_on'] = "启用";
+$_['text_online_users'] = "线上用户";
+$_['text_other'] = "其他";
+$_['text_outbound'] = "outbound";
+
+$_['text_password'] = "密码";
+$_['text_password_again'] = "再一次密码";
+$_['text_password_changed'] = "密码已变更";
+$_['text_password_mismatch'] = "密码不符";
+$_['text_page_length'] = "每页结果";
+$_['text_periodic_purge'] = "定期清理";
+$_['text_policy'] = "策略";
+$_['text_policy_group'] = "策略组";
+$_['text_policy_name'] = "策略名称";
+$_['text_previous'] = "Previous";
+$_['text_print_message'] = "打印";
+$_['text_private'] = "隐私";
+$_['text_processed_emails'] = "已处理邮件";
+$_['text_progress'] = "处理进度";
+$_['text_purge_all_messages_from_quarantine'] = "从隔离区清除您的所有邮件";
+$_['text_purge_selected_messages'] = "清除已选择邮件";
+$_['text_purged'] = "清除";
+
+$_['text_qr_code'] = "QR code";
+$_['text_queue_status'] = "队列状态";
+$_['text_quick_search'] = "快速搜索";
+
+$_['text_realname'] = "人员姓名";
+$_['text_recipient'] = "收件人";
+$_['text_ref'] = "参照";
+$_['text_refresh_period'] = "刷新周期";
+$_['text_refresh_qr_code'] = "刷新 QR code";
+$_['text_reject'] = "拒绝";
+$_['text_rejected_removal'] = "rejected removal";
+$_['text_reason_of_rejection'] = "拒绝原因";
+$_['text_relay_details'] = "转送内容";
+$_['text_relay_status'] = "转送状态";
+$_['text_remove'] = "移除";
+$_['text_remove_domain'] = "移除域";
+$_['text_remove_message'] = "移除邮件";
+$_['text_remove_message2'] = "移除邮件";
+$_['text_remove_request'] = "移除要求";
+$_['text_remove_selected_uids'] = "移除已选择 uids";
+$_['text_remove_policy'] = "移除策略";
+$_['text_remove_rule'] = "移除规则";
+$_['text_remove_this_policy'] = "移除此策略";
+$_['text_remove_this_group'] = "移除此组";
+$_['text_remove_this_user'] = "移除此用户";
+$_['text_removed'] = "已移除";
+$_['text_reset_counters'] = "重置计数器";
+$_['text_restore'] = "还原";
+$_['text_restored'] = "已还原";
+$_['text_restore_message'] = "还原邮件";
+$_['text_restore_to_mailbox'] = "还原邮件至邮箱";
+$_['text_result'] = "结果";
+$_['text_retention_days'] = "保留天数";
+$_['text_retention_rules'] = "保留规则";
+$_['text_role'] = "用户类型";
+$_['text_running'] = "执行中";
+
+$_['text_save'] = "存储";
+$_['text_saved'] = "已存储";
+$_['text_save_search'] = "存储搜索";
+$_['text_save_search_terms'] = "搜索字词";
+$_['text_saved_search_terms'] = "搜索字词已存储";
+$_['text_search'] = "搜索";
+$_['text_search2'] = "搜索";
+$_['text_search_emails'] = "搜索邮件地址";
+$_['text_search_email_to_add'] = "搜索邮件以加入";
+$_['text_search_expression'] = "搜索语法";
+$_['text_search_folders'] = "搜索文件夹";
+$_['text_search_folder_to_add'] = "搜索文件夹以加入";
+$_['text_search_groups'] = "搜索组";
+$_['text_search_group_to_add'] = "搜索组以加入";
+$_['text_search_terms'] = "搜索字词";
+$_['text_select_action'] = "选择动作";
+$_['text_select_all'] = "全部选择";
+$_['text_select_image'] = "Select image";
+$_['text_select_recipients'] = "选择收件人";
+$_['text_sender'] = "发件人";
+$_['text_sending_domains'] = "发件人域";
+$_['text_server_name'] = "服务器名称";
+$_['text_server_operating_system'] = "操作系统";
+$_['text_set'] = "设定";
+$_['text_settings'] = "设定";
+$_['text_simple'] = "简单";
+$_['text_simple_search'] = "简单搜索";
+$_['text_size'] = "大小";
+$_['text_smtp_status'] = "SMTP 状态";
+$_['text_spam'] = "垃圾邮件";
+$_['text_spam2'] = "垃圾邮件";
+$_['text_statistics'] = "使用统计";
+$_['text_status'] = "状态";
+$_['text_storage'] = "存储";
+$_['text_subject'] = "主旨";
+$_['text_submit'] = "送出";
+$_['text_successful'] = "成功";
+$_['text_successfully_added'] = "新增成功";
+$_['text_successfully_delivered'] = "传递成功";
+$_['text_successfully_modified'] = "修改成功";
+$_['text_successfully_removed'] = "移除成功";
+$_['text_successfully_trained'] = "Successfully trained";
+$_['text_successfully_updated'] = "更新成功";
+$_['text_support_link'] = "支持链接";
+$_['text_swap_usage'] = "Swap 使用率";
+
+$_['text_tag_selected_messages'] = "标签已选择邮件";
+$_['text_tagged'] = "标记的";
+$_['text_tags'] = "标签";
+$_['text_test_connection'] = "测试连接";
+$_['text_text'] = "文字";
+$_['text_text_colour'] = "文字颜色";
+$_['text_text2'] = "文字";
+$_['text_theme'] = "背景主题";
+$_['text_time'] = "时间";
+$_['text_to'] = "收件人";
+$_['text_to_domain'] = "收件人域";
+$_['text_too_short_password'] = "密码太短";
+$_['text_total'] = "总计";
+$_['text_total_ratio'] = "总比例";
+$_['text_total_query_time'] = "总计 SQL 查询时间";
+$_['text_total_users'] = "总计";
+$_['text_type'] = "类型";
+
+$_['text_uids'] = "uids";
+$_['text_unauthorized_domain'] = "未授权域";
+$_['text_unauthorized_download_attachment'] = "未授权下载附件";
+$_['text_unauthorized_remove_message'] = "未授权移除邮件";
+$_['text_unauthorized_view_message'] = "未授权查看邮件";
+$_['text_unknown'] = "未知";
+$_['text_update_retention_within_this_domain'] = "此域内的更新保留数值";
+$_['text_update_selected_uids'] = "更新所选的 uid";
+$_['text_updated_records'] = "已更新记录";
+$_['text_uptime'] = "运行时间";
+$_['text_user'] = "用户";
+$_['text_users'] = "用户";
+$_['text_user_id'] = "用户 id";
+$_['text_user_auditor'] = "审计员";
+$_['text_user_data_officer'] = "资料管理员";
+$_['text_user_domainadmin'] = "域管理员";
+$_['text_user_management'] = "用户管理";
+$_['text_user_masteradmin'] = "主要管理员";
+$_['text_user_read_only_admin'] = "唯读管理员";
+$_['text_user_regular'] = "普通用户";
+$_['text_userlist'] = "用户清单";
+$_['text_username'] = "用户名称";
+$_['text_users_quarantine'] = "用户的隔离区";
+
+$_['text_view_formatted_email'] = "查看格式化邮件";
+$_['text_view_header'] = "查看邮件头";
+$_['text_view_headers'] = "查看邮件头";
+$_['text_view_journal'] = "日志";
+$_['text_view_journal_envelope'] = "查看信封";
+$_['text_view_message'] = "查看邮件";
+$_['text_view_message2'] = "查看邮件";
+$_['text_view_progress'] = "查看进度";
+$_['text_view_raw_email'] = "查看原始邮件";
+$_['text_view_user_quarantine'] = "查看用户的隔离区";
+
+$_['text_warning_about_default_policy'] = "预设策略可以在 clapf.conf 中设定";
+$_['text_wildcard_domains'] = "通用域";
+$_['text_whitelist'] = "白名单";
+$_['text_whitelist_settings'] = "白名单设定";
+$_['text_with_attachment'] = "含附件";
+$_['text_without_attachment'] = "不含附件";
+
+$_['text_years'] = "年";
+$_['text_you_are'] = "您是";
+$_['text_you_are_not_admin'] = "您不是管理员";
+
+
+$_['rcvd'] = "接收邮件";
+$_['virus'] = "病毒邮件";
+$_['duplicate'] = "重复邮件";
+$_['ignore'] = "忽略邮件";
+$_['counters_last_update'] = "计数器更新";
+
+$_['text_24_hours'] = "24 小时";
+$_['text_1_week'] = "1 周";
+$_['text_30_days'] = "30 天";
+
+$_['text_access_settings'] = '存取设定';
+$_['text_access_setting_explanation'] = "您可以存取自己的电子邮件。对于指定组或域的审计员存取，请您与管理员联系。";
+$_['text_display_settings'] = '显示设定';
+$_['text_change_password'] = "变更密码";
+$_['text_none_found'] = "没有找到";
+$_['text_primary_domain'] = "主要域";
+$_['text_search_domains'] = "搜索域";
+$_['text_search_domain_to_add'] = "搜索域以加入";
+
+$_['text_space_projection'] = '空间保护';
+$_['text_average_messages_day'] = '每天平均邮件';
+$_['text_average_message_size'] = '平均邮件 + 中继邮件 + 索引大小';
+$_['text_average_size_day'] = '每天平均大小';
+$_['text_partition_full'] = '分区预计用完于';
+$_['text_usage_trend'] = '使用趋势';
+$_['text_usage_increasing'] = '上升';
+$_['text_usage_decreasing'] = '下降';
+$_['text_usage_neutral'] = '持平';
+$_['text_accounting'] = '归档审计';
+$_['text_accounting_email'] = '依邮件审计';
+$_['text_accounting_domain'] = '依域名审计';
+
+$_['text_options'] = '选项';
+$_['text_spam_flag'] = '邮件已标注为 SPAM';
+$_['text_attachment_flag'] = '邮件有附件';
+$_['text_notes_flag'] = '邮件有备注';
+$_['text_tag_flag'] = '邮件已标记';
+$_['text_verified_flag'] = '邮件以验证';
+$_['text_unverified_flag'] = '邮件验证失败';
+$_['text_bulk_download'] = '批次下载已选择邮件';
+$_['text_clear'] = '清除';
+$_['text_select_letter'] = '以开头字母选择';
+$_['text_working'] = '作业中...';
+
+$_['text_use_browser_settings'] = '使用浏览器设定';
+
+$_['text_sent'] = '寄送';
+$_['text_received'] = '已接收';
+$_['text_oldest_record'] = '最旧记录';
+$_['text_newest_record'] = '最新纪录';
+$_['text_items'] = '项目';
+$_['text_average_size'] = '平均大小';
+
+$_['text_return_to'] = '回到';
+$_['text_error_message'] = '请更正以下错误后重新送出。';
+$_['text_field_required'] = '此为必填栏位。';
+$_['text_field_length'] = '这个栏位必须大于 ? 个字元。';
+$_['text_field_domain'] = '这个栏位必须填入域 (例如 - domain.com)。';
+$_['text_field_colour'] = '这个栏位必须填入色码 (例如 - #fcfcfc)。';
+$_['text_delete'] = '删除';
+$_['text_confirm'] = '确认';
+$_['text_user_delete_confirm_message'] = '您要删除用户吗';
+$_['text_domain_delete_confirm_message'] = '您要删除域吗';
+$_['text_group_delete_confirm_message'] = '您要删除组吗';
+$_['text_ldap_delete_confirm_message'] = '您要删除 LDAP 项目吗';
+$_['text_customer_delete_confirm_message'] = '您要删除客户吗';
+$_['text_with_selected'] = '选择项目';
+
+$_['text_compliance_warning'] = '删除功能已启用，因此这个归档不符合规定！';
--- a/webui/language/it/messages.php
+++ b/webui/language/it/messages.php
@ -0,0 +1,497 @@
+<?php
+
+$_['text_60_minutes'] = "60 min.";
+
+$_['text_action'] = "Azione";
+$_['text_active_incoming_queue'] = "coda attiva e in entrata";
+$_['text_active_incoming_queue_sender'] = "active + incoming queue vs. sender";
+$_['text_ad_sync_status'] = "Stato di sincronizzazione con AD";
+$_['text_add'] = "Aggiungi";
+$_['text_add_new_email_address'] = "Nuovo indirizzo email";
+$_['text_add_new_domain'] = "Nuovo Dominio";
+$_['text_add_new_entry'] = "Nuovo inserimento";
+$_['text_add_new_group'] = "Aggiungi gruppo";
+$_['text_add_new_rule'] = "Aggiungi regola";
+$_['text_add_new_user_alias'] = "Nuovo utente";
+$_['text_add_policy'] = "Aggiungi una nuova policy";
+$_['text_administration'] = "Amministrazione";
+$_['text_admin_user'] = "Utente amministratore";
+$_['text_advanced'] = "Avanzate";
+$_['text_advanced_search'] = "Ricerca avanzata";
+$_['text_all'] = "Tutti";
+$_['text_any'] = "Qualunque";
+$_['text_applied'] = "Applicata";
+$_['text_apply_changes'] = "Applica modifiche";
+$_['text_archive_size'] = "Dimensione archivio";
+$_['text_archive_size_before_compression'] = "Dimensione archivio pruima della compressione";
+$_['text_archived_messages'] = "Messaggi archiviati";
+$_['text_archiving_rules'] = "Regole di archiviazione";
+$_['text_assigned_email_addresses'] = "Indirizzi email assegnati";
+$_['text_attachment'] = "Allegato";
+$_['text_attachment_name'] = "Nome dell'allegato";
+$_['text_attachment_size'] = "Dimensione dell'allegato";
+$_['text_attachment_type'] = "Tipo di allegato";
+$_['text_audit'] = "Audit";
+$_['text_automated_search'] = "Ricerca automatica";
+
+$_['text_back'] = "Indietro";
+$_['text_background_colour'] = "Colore di sfondo";
+$_['text_body'] = "Corpo";
+$_['text_branding_logo'] = "Branding logo";
+$_['text_branding_text'] = "Branding text";
+$_['text_branding_url'] = "Branding URL";
+$_['text_bulk_edit_selected_uids'] = "MOdifica tutti gli ID selezionati";
+$_['text_bulk_restore_selected_emails'] = "Ripristina tutti gli ID selezionati";
+$_['text_bulk_update_selected_uids'] = "Aggiorna tutti gli ID selezionati";
+
+$_['text_cancel'] = "Cancella";
+$_['text_change_user_settings'] = "modifica impostazioni utente";
+$_['text_clienthost'] = "Client host";
+$_['text_close'] = "Chiudi";
+$_['text_colour'] = "Colore";
+$_['text_compressed'] = "compresso";
+$_['text_confirm_to_reset_counters'] = "Conferma l'azzeramento dei contatori";
+$_['text_connection_failed'] = "Connessione fallita";
+$_['text_connection_ok'] = "Connessione OK";
+$_['text_contact_support'] = "Contatta il supporto";
+$_['text_content_filter'] = "Filtro contenuti";
+$_['text_conversation_available'] = "Conversazione disponibile";
+$_['text_copied'] = "Copiato";
+$_['text_counters'] = "Contatori";
+$_['text_cpu_load'] = "Carico sulla CPU";
+$_['text_cpu_usage'] = "Utilizzo CPU";
+$_['text_create_new_secret'] = "Crea un ";
+$_['text_create_note'] = "Crea una nota";
+$_['text_cumulative_counts'] = "Conteggi cumulativi";
+$_['text_customers'] = "Clienti";
+
+$_['text_daily_quarantine_report'] = "Rapporto quotidiano della quarantena";
+$_['text_daily_quarantine_report_status'] = "Stato del rapporto quotidiano della quarantena";
+$_['text_daily_report'] = "Rapporto quotidiano";
+$_['text_daily_piler_report'] = "Rappporto quotidiano di piler";
+$_['text_database_emails'] = "indirizzi email nel database di piler";
+$_['text_date'] = "Data";
+$_['text_date_from'] = "Dalla data";
+$_['text_date_to'] = "Alla data";
+$_['text_days'] = "Giorni";
+$_['text_days2'] = "giorni";
+$_['text_days_to_retain'] = "Giorni da conservare";
+$_['text_deferred_queue'] = "coda differita";
+$_['text_deferred_queue_sender'] = "coda differita verso il mittente";
+$_['text_delay'] = "Ritardo";
+$_['text_delete_confirm_message'] = "Vuoi cancellare";
+$_['text_delete_reason'] = "Motivo della rimozione";
+$_['text_deleted'] = "Cancellato";
+$_['text_deleted_users'] = "Cancellato";
+$_['text_deliver'] = "Consegna";
+$_['text_delivered'] = "Consegnato";
+$_['text_deliver_and_train_selected_messages'] = "Consegna i messaggi selezionati e indicizzali come";
+$_['text_deliver_and_train_selected_messages_as_ham'] = "Consegna i messaggi selezionati e indicizzali come HAM";
+$_['text_deliver_selected_messages'] = "Consegna i messaggi selezionati";
+$_['text_description'] = "Descrizione";
+$_['text_direction'] = "Direzione";
+$_['text_disk_usage'] = "Utilizzo disco";
+$_['text_disable'] = "Disabilita";
+$_['text_disabled'] = "disabilitata";
+$_['text_dn_asterisk_means_skip_sync'] = "L'Asterisco (*) significa che questo utente non avrà parte nella sincronizzazione di AD";
+$_['text_domain'] = "Dominio";
+$_['text_domains'] = "Domini(o)";
+$_['text_domainname'] = "Nome dominio";
+$_['text_download_all_hits_as_eml'] = "Scarica tutto (EML)";
+$_['text_download_selected_hits_as_pdf'] = "Scarica la selezione (PDF)";
+$_['text_download_attachment2'] = "scarica l'allegato";
+$_['text_download_message'] = "Scarica (EML)";
+$_['text_download_message2'] = "scarica il messaggio";
+
+$_['text_edit'] = "Modifica";
+$_['text_edit_entry'] = "Modifica voce";
+$_['text_edit_group'] = "Modifica gruppo";
+$_['text_edit_user'] = "Modifica utente";
+$_['text_edit_or_view'] = "Modifica/vedi";
+$_['text_email'] = "Indirizzo email";
+$_['text_email_addresses'] = "Indirizzi email";
+$_['text_email_aliases'] = "Alias email";
+$_['text_email_in_unknown_domain'] = "l'indirizzo email è di un dominio sconosciuto";
+$_['text_empty_search_criteria'] = "'Nessun criterio di ricerca'";
+$_['text_empty_search_result'] = "Nessun risultato. Prova ad aggiungere il carattere jolly(*) dopo una parola abbreviata (min. " . MIN_PREFIX_LEN . " lettere), es. duplic* per trovare \"duplicate\", \"duplicati\", etc.";
+$_['text_enable'] = "Abilita";
+$_['text_enabled'] = "abilitata";
+$_['text_enter_google_authenticator_code'] = "Usa Google Authenticator";
+$_['text_enter_one_email_address_per_line'] = "Inserisci un indirizzo email per riga";
+$_['text_enter_one_group_per_line'] = "Inserisci un gruppo per riga";
+$_['text_enter_search_terms'] = "Inserisci i termini di ricerca";
+$_['text_error'] = "Errore";
+$_['text_exact_domain_name_or_email_address'] = "esatto nome a domino o indirizzo email";
+$_['text_exclude'] = "Escludi";
+$_['text_existing'] = "Esistente";
+$_['text_existing_domains'] = "Dominio esistente";
+$_['text_existing_email'] = "Email esistente";
+$_['text_existing_folders'] = "Cartelle esistenti";
+$_['text_existing_groups'] = "Gruppi esistenti";
+$_['text_existing_policies'] = "Policies esistenti";
+$_['text_existing_entries'] = "Voci esistenti";
+$_['text_existing_rules'] = "Regole esistenti";
+$_['text_existing_user'] = "Utente esistente";
+$_['text_existing_users'] = "Utenti esistenti";
+$_['text_expert'] = "Esperto";
+$_['text_expert_search'] = "Ricerca esperta";
+
+$_['text_failed'] = "fallito";
+$_['text_failed_to_add'] = "Aggiunta fallita";
+$_['text_failed_to_change_password'] = "Cambio password fallito";
+$_['text_failed_to_deliver'] = "Consegna fallita";
+$_['text_failed_to_mark_for_removal'] = "Contrassegno di rimozione fallito";
+$_['text_failed_to_modify'] = "Modifica fallita";
+$_['text_failed_to_remove'] = "Rimozione fallita";
+$_['text_failed_to_restore'] = "Ripristino fallito";
+$_['text_failed_to_update'] = "Aggiornamento fallito";
+$_['text_first'] = "Primo";
+$_['text_folder'] = "Cartella";
+$_['text_folder_rules'] = "Regole delle cartelle";
+$_['text_folders'] = "Cartelle";
+$_['text_forward_selected_emails_to'] = "Inoltra i messaggi selezionati a";
+$_['text_from'] = "Da";
+$_['text_from_domain'] = "Dal dominio";
+
+$_['text_google_authenticator_code'] = "Google Authenticator code";
+$_['text_google_authenticator_settings'] = "Impostazioni di Google Authenticator";
+$_['text_group_id'] = "Id gruppo";
+$_['text_groupname'] = "Nome gruppo";
+$_['text_groups'] = "Gruppi";
+$_['text_group_management'] = "Gestione gruppo";
+$_['text_group_membership'] = "Appartenenza al gruppo";
+
+$_['text_health'] = "Health";
+$_['text_health_monitor'] = "Health monitor";
+$_['text_help'] = "Aiuto";
+$_['text_history'] = "Cronologia";
+$_['text_home'] = "Home";
+
+$_['text_image'] = "immagine";
+$_['text_import'] = "Importa";
+$_['text_import_job_delete_confirm_message'] = "Vuoi terminare l'importazione";
+$_['text_import_users'] = "Importa utenti";
+$_['text_import_users_from_LDAP'] = "Importa utenti da LDAP";
+$_['text_inbound'] = "In entrata";
+$_['text_indexer_job'] = "Indicizzazione";
+$_['text_install_sudo_apply'] = "Aggiungi la riga seguente a /etc/sudoers: 'www-data ALL=NOPASSWD: /etc/init.d/rc.piler reload'";
+$_['text_internal'] = "interno";
+$_['text_invalid_data'] = "Dati non validi";
+$_['text_invalid_email'] = "Email non valida";
+$_['text_invalid_email_or_password'] = "email o password non valide";
+$_['text_invalid_gid'] = "GID non valido";
+$_['text_invalid_password'] = "Password non valida";
+$_['text_invalid_pin_code'] = "Pin non valido";
+$_['text_invalid_policy_group'] = "Regola di gruppo non valida";
+$_['text_invalid_policy_name'] = "Nome regola non valido";
+$_['text_invalid_policy_setting'] = "Impostazioni policy non valide";
+$_['text_invalid_uid'] = "UID non valido";
+$_['text_invalid_username'] = "Nome utente non valido";
+$_['text_ipaddr'] = "Indirizzo IP";
+
+$_['text_language'] = "Lingua";
+$_['text_last'] = "Ultimo";
+$_['text_last_activity'] = "Ultima attività";
+$_['text_last_update'] = "Ultimo aggiornamento";
+$_['text_latest_emails'] = "Ultime email";
+$_['text_ldap'] = "LDAP";
+$_['text_ldap_auditor_member_dn'] = "Auditor member DN";
+$_['text_ldap_base_dn'] = "LDAP base DN";
+$_['text_ldap_bind_dn'] = "LDAP bind DN";
+$_['text_ldap_bind_pw'] = "LDAP bind password";
+$_['text_ldap_host'] = "LDAP host";
+$_['text_ldap_type'] = "LDAP type";
+$_['text_legal_hold'] = "Conservazione legale";
+$_['text_load'] = "Carico";
+$_['text_loading'] = "caricamento";
+$_['text_logged_in'] = "connesso";
+$_['text_logged_out'] = "Ti sei disconnesso";
+$_['text_login'] = "Login";
+$_['text_login2'] = "login";
+$_['text_login_failed'] = "login fallito";
+$_['text_login_via_google'] = "Login via Google account";
+$_['text_logout'] = "Disconnetti";
+$_['text_logout2'] = "disconnetti";
+
+$_['text_maillog_status'] = "status del collettore dei log delle email";
+$_['text_main_title'] = "interfaccia web di clapf";
+$_['text_mapped_domain'] = "Dominio associato";
+$_['text_mark_private'] = "privato";
+$_['text_marked_for_removal'] = "Messaggio contrassegnato per la rimozione";
+$_['text_memory_usage'] = "Utilizzo memoria";
+$_['text_message'] = "messaggio";
+$_['text_messages'] = "messaggi";
+$_['text_message_disposition'] = "Disposizione Messaggi";
+$_['text_message_text'] = "Testo del messaggio";
+$_['text_min_2_chars'] = "Min. 2 caratteri";
+$_['text_missing_data'] = "Dati mancanti";
+$_['text_missing_password'] = "Password mancante";
+$_['text_modify'] = "Modifica";
+$_['text_monitor'] = "Monitor";
+$_['text_months'] = "mesi";
+$_['text_monthly_report'] = "Rapporto mensile";
+
+$_['text_need_to_approve_removal'] = "devi approvare la rimozione";
+$_['text_new'] = "nuovo";
+$_['text_new_users'] = "nuovo";
+$_['text_next'] = "Prossimo";
+$_['text_no_domain_found'] = 'Nessun dominio trovato';
+$_['text_no_email_found'] = 'Nessuna email trovata';
+$_['text_no_message_in_the_quarantine'] = "Nessun messaggio in quarantena soddisfa i requisiti di ricerca";
+$_['text_no_records'] = "Nessun record";
+$_['text_no_selected_message'] = "nessun messaggio selezionato";
+$_['text_no_sender'] = "nessun mittente";
+$_['text_no_spam_message_in_the_quarantine_yet'] = "Nessun messaggio spam in quarantena ora";
+$_['text_no_subject'] = "nessun oggetto";
+$_['text_no_such_policy'] = "Nessuna policy di questo tipo";
+$_['text_non_existent_queue_directory'] = "Cartella della coda specificata inesistente";
+$_['text_non_existing_user'] = "Utente inesistente";
+$_['text_notes'] = "Note";
+$_['text_not_found'] = "Non trovato";
+$_['text_not_running'] = "non in funzione";
+$_['text_not_spam'] = "non SPAM";
+$_['title_not_found'] = "Pagina non trovata";
+$_['text_number_of_messages_in_quarantine'] = "Numero di messaggi in quarantena che soddisfano i tuoi criteri di ricerca";
+$_['text_number_of_spam_messages_in_quarantine'] = "Numero di messaggi spam in quarantena che soddisfano i tuoi criteri di ricerca";
+
+$_['text_off'] = "off";
+$_['text_on'] = "on";
+$_['text_online_users'] = "Utenti online";
+$_['text_other'] = "altro";
+$_['text_outbound'] = "in uscita";
+
+$_['text_password'] = "Password";
+$_['text_password_again'] = "Ripeti password";
+$_['text_password_changed'] = "Password modificata";
+$_['text_password_mismatch'] = "Le password non corrispondono";
+$_['text_page_length'] = "Risultati per pagina";
+$_['text_periodic_purge'] = "Pulizia periodica";
+$_['text_policy'] = "Policy";
+$_['text_policy_group'] = "Policy group";
+$_['text_policy_name'] = "Nome policy";
+$_['text_previous'] = "Precedente";
+$_['text_print_message'] = "Stampa";
+$_['text_private'] = "Privato";
+$_['text_processed_emails'] = "email processate";
+$_['text_progress'] = "Progresso";
+$_['text_purge_all_messages_from_quarantine'] = "Elimina tutti i tuoi messaggi dalla quarantena";
+$_['text_purge_selected_messages'] = "Elimina i messaggi selezionati";
+$_['text_purged'] = "Eliminati";
+
+$_['text_qr_code'] = "QR";
+$_['text_queue_status'] = "Stato della coda";
+$_['text_quick_search'] = "Ricerca veloca";
+
+$_['text_realname'] = "Nome reale";
+$_['text_recipient'] = "Destinatario";
+$_['text_ref'] = "Riferimento";
+$_['text_refresh_period'] = "Periodo di aggiornamento";
+$_['text_refresh_qr_code'] = "Aggiorna il QR code";
+$_['text_reject'] = "Scarta";
+$_['text_rejected_removal'] = "rimozione scartata";
+$_['text_reason_of_rejection'] = "Motivo dello scarto";
+$_['text_relay_details'] = "Dettagli di inoltro";
+$_['text_relay_status'] = "Stato dell'inoltro";
+$_['text_remove'] = "Rimuovi";
+$_['text_remove_domain'] = "Rimuovi dominio";
+$_['text_remove_message'] = "Rimuovi il messaggio";
+$_['text_remove_message2'] = "rimuovi il messaggio";
+$_['text_remove_request'] = "remove request";
+$_['text_remove_selected_uids'] = "Rimuovi gli uid selezionati";
+$_['text_remove_policy'] = "Rimuovi policy";
+$_['text_remove_rule'] = "Rimuovi regola";
+$_['text_remove_this_policy'] = "Rimuovi questa policy";
+$_['text_remove_this_group'] = "Rimuovi questo gruppo";
+$_['text_remove_this_user'] = "Rimuovio questo utente";
+$_['text_removed'] = "Rimosso";
+$_['text_reset_counters'] = "Azzera i contatori";
+$_['text_restore'] = "Ripristina";
+$_['text_restored'] = "Ripristinato";
+$_['text_restore_message'] = "ripriatina messaggio";
+$_['text_restore_to_mailbox'] = "Ripristina nella cassetta postale";
+$_['text_result'] = "Risultato";
+$_['text_retention_days'] = "Giorni di conservazione";
+$_['text_retention_rules'] = "Regole di conservazione";
+$_['text_role'] = "Regola";
+$_['text_running'] = "in corso";
+
+$_['text_save'] = "Salva";
+$_['text_saved'] = "Salvato";
+$_['text_save_search'] = "salva la ricerca";
+$_['text_save_search_terms'] = "Salva i termini della ricerca";
+$_['text_saved_search_terms'] = "Termini di ricerca salvati";
+$_['text_search'] = "Cerca";
+$_['text_search2'] = "cerca";
+$_['text_search_emails'] = "Cerca indirizzi email";
+$_['text_search_email_to_add'] = "Cerca email da aggiungere";
+$_['text_search_expression'] = "Espressione di ricerca";
+$_['text_search_folders'] = "Cartelle di ricerca";
+$_['text_search_folder_to_add'] = "Cerca le cartelle  da aggiungere";
+$_['text_search_groups'] = "Cerca gruppi";
+$_['text_search_group_to_add'] = "Cerca un gruppo da aggiungere";
+$_['text_search_terms'] = "Termini di ricerca";
+$_['text_select_action'] = "Seleziona azione";
+$_['text_select_all'] = "Seleziona tutto";
+$_['text_select_image'] = "Seleziona immagine";
+$_['text_select_recipients'] = "Selezione destinatari";
+$_['text_sender'] = "Mittente";
+$_['text_sending_domains'] = "domini mittenti";
+$_['text_server_name'] = "Nome server";
+$_['text_server_operating_system'] = "Sistema operativo";
+$_['text_set'] = "Imposta";
+$_['text_settings'] = "Impostazioni";
+$_['text_simple'] = "Semplice";
+$_['text_simple_search'] = "Ricera semplice";
+$_['text_size'] = "Dimensioni";
+$_['text_smtp_status'] = "Stato SMTP";
+$_['text_spam'] = "Spam";
+$_['text_spam2'] = "spam";
+$_['text_statistics'] = "Statistiche";
+$_['text_status'] = "Status";
+$_['text_storage'] = "Archiviazione";
+$_['text_subject'] = "Oggetto";
+$_['text_submit'] = "Invia";
+$_['text_successful'] = "Con successo";
+$_['text_successfully_added'] = "Aggiunto con successo";
+$_['text_successfully_delivered'] = "Consegnato con successo";
+$_['text_successfully_modified'] = "Modificato con successo";
+$_['text_successfully_removed'] = "Rimosso con successo";
+$_['text_successfully_trained'] = "Allenato con successo";
+$_['text_successfully_updated'] = "Aggiornato con successo";
+$_['text_support_link'] = "Collegamento di supporto";
+$_['text_swap_usage'] = "Utilizzo dello swap";
+
+$_['text_tag_selected_messages'] = "Marca i messaggi selezionati";
+$_['text_tagged'] = "Etichettato";
+$_['text_tags'] = "Etichette";
+$_['text_test_connection'] = "Test della connessione";
+$_['text_text'] = "Testo";
+$_['text_text_colour'] = "Colore testo";
+$_['text_text2'] = "testo";
+$_['text_theme'] = "Tema";
+$_['text_time'] = "Ora";
+$_['text_to'] = "A";
+$_['text_to_domain'] = "Al dominio";
+$_['text_too_short_password'] = "Password troppo corta";
+$_['text_total'] = "totale";
+$_['text_total_ratio'] = "rateo totale";
+$_['text_total_query_time'] = "Tempo totale della query SQL";
+$_['text_total_users'] = "totale";
+$_['text_type'] = "Tipo";
+
+$_['text_uids'] = "uids";
+$_['text_unauthorized_domain'] = "Dominio non autorizzato";
+$_['text_unauthorized_download_attachment'] = "download dell'allegato non autorizzato";
+$_['text_unauthorized_remove_message'] = "Rimozione del messaggio non autorizzata";
+$_['text_unauthorized_view_message'] = "Visualizzazione del messaggio non autorizzata";
+$_['text_unknown'] = "sconosciuto";
+$_['text_update_retention_within_this_domain'] = "Aggiorna i valori di conservazione all'interno di questo dominio";
+$_['text_update_selected_uids'] = "Aggiorna gli UID selezionati";
+$_['text_updated_records'] = "Record aggiornato";
+$_['text_uptime'] = "Uptime";
+$_['text_user'] = "Utente";
+$_['text_users'] = "Utenti";
+$_['text_user_id'] = "Id Utente";
+$_['text_user_auditor'] = "Auditor";
+$_['text_user_data_officer'] = "Data officer";
+$_['text_user_domainadmin'] = "Amministratore del dominio";
+$_['text_user_management'] = "Gestione utente";
+$_['text_user_masteradmin'] = "Master admin";
+$_['text_user_read_only_admin'] = "Read-only admin";
+$_['text_user_regular'] = "Normale utente";
+$_['text_userlist'] = "Lista degli utenti";
+$_['text_username'] = "Nome utente";
+$_['text_users_quarantine'] = "Quarantena degli utenti";
+
+$_['text_view_formatted_email'] = "Visualizza email formattata";
+$_['text_view_header'] = "visualizza intestazioni";
+$_['text_view_headers'] = "Visualizza intestazioni";
+$_['text_view_journal'] = "journal";
+$_['text_view_journal_envelope'] = "Visualizza la busta";
+$_['text_view_message'] = "Visualizza il messaggio";
+$_['text_view_message2'] = "visualizza il messaggio";
+$_['text_view_progress'] = "visualizza il progresso";
+$_['text_view_raw_email'] = "Visualizza sorgenti";
+$_['text_view_user_quarantine'] = "Visualizza la quarantena dell'utente";
+
+$_['text_warning_about_default_policy'] = "La policy predefinita può essere configurata in clapf.conf";
+$_['text_wildcard_domains'] = "Wildcard domini";
+$_['text_whitelist'] = "Lista bianca";
+$_['text_whitelist_settings'] = "Impostazioni della lista bianca";
+$_['text_with_attachment'] = "con allegato(i)";
+$_['text_without_attachment'] = "senza allegato";
+
+$_['text_years'] = "anni";
+$_['text_you_are'] = "Tu sei";
+$_['text_you_are_not_admin'] = "Tu non sei un utente amministratore";
+
+
+$_['rcvd'] = "messaggi ricevuti";
+$_['virus'] = "messaggi infetti";
+$_['duplicate'] = "messaggi duplicati";
+$_['ignore'] = "messaggi ignorati";
+$_['counters_last_update'] = "contatori aggiornati";
+
+$_['text_24_hours'] = "24 ore";
+$_['text_1_week'] = "1 settimana";
+$_['text_30_days'] = "30 giorni";
+
+$_['text_access_settings'] = 'Impostazioni di accesso';
+$_['text_access_setting_explanation'] = "Hai sempre accesso ai tuoi indirizzi email.  Contatta l'amministratore per un accesso da auditor a specifici gruppi o domini.";
+$_['text_display_settings'] = 'Visualizza le impostazioni';
+$_['text_change_password'] = "Cambia Password";
+$_['text_none_found'] = "Nessuno trovato";
+$_['text_primary_domain'] = "Dominio Principale";
+$_['text_search_domains'] = "Domini di ricerca";
+$_['text_search_domain_to_add'] = "Cerca un dominio da aggiungere";
+
+$_['text_space_projection'] = 'Previsione Spazio';
+$_['text_average_messages_day'] = 'Media dei messaggi per giorno';
+$_['text_average_message_size'] = 'Media dei Messaggi + Metadati + Dimensione Indice';
+$_['text_average_size_day'] = 'Media Dimensioni per Giorno';
+$_['text_partition_full'] = 'Lo spazio finirà tra';
+$_['text_usage_trend'] = 'Trend di Utilizzo';
+$_['text_usage_increasing'] = 'Crescente';
+$_['text_usage_decreasing'] = 'Decrescente';
+$_['text_usage_neutral'] = 'Neutrale';
+$_['text_accounting'] = 'Resoconto Archivio';
+$_['text_accounting_email'] = 'Resoconto email';
+$_['text_accounting_domain'] = 'Resoconto Domini';
+
+$_['text_options'] = 'Opzioni';
+$_['text_spam_flag'] = 'Messaggio categorizzato come SPAM';
+$_['text_attachment_flag'] = 'Il Messaggio ha degli allegati';
+$_['text_notes_flag'] = 'Il Messaggio ha delle Note';
+$_['text_tag_flag'] = 'Il Messaggio è Etichettato';
+$_['text_verified_flag'] = 'Il Messaggio è verificato';
+$_['text_unverified_flag'] = 'Verifica fallita sul messaggio';
+$_['text_bulk_download'] = 'Scarica le email selezionate';
+$_['text_clear'] = 'Cancella';
+$_['text_select_letter'] = 'Seleziona indirizzo da Lettera iniziale';
+$_['text_working'] = 'Working...';
+
+$_['text_use_browser_settings'] = 'Usa le impostazioni del browser';
+
+$_['text_sent'] = 'Inviato';
+$_['text_received'] = 'Ricevuto';
+$_['text_oldest_record'] = 'Record più vecchio';
+$_['text_newest_record'] = 'Record più nuovo';
+$_['text_items'] = 'Items';
+$_['text_average_size'] = 'Dimensioni medie';
+
+$_['text_return_to'] = 'Rispondi a';
+$_['text_error_message'] = 'Correggi i seguenti errori e reinvia.';
+$_['text_field_required'] = 'Campo richiesto.';
+$_['text_field_length'] = 'Questo campo deve essere piu lungo di ? caratteri.';
+$_['text_field_domain'] = 'Questo campo deve avere un dominio valido (es. - domain.com).';
+$_['text_field_colour'] = 'Questo campo deve avere un valido codice colore (es. - #fcfcfc).';
+$_['text_delete'] = 'Elimina';
+$_['text_confirm'] = 'Conferma';
+$_['text_user_delete_confirm_message'] = 'Vuoi eliminare questo utente?';
+$_['text_domain_delete_confirm_message'] = 'Vuoi eliminare il dominio?';
+$_['text_group_delete_confirm_message'] = 'Vuoi eliminare il gruppo?';
+$_['text_ldap_delete_confirm_message'] = 'Vuoi eliminare il LDAP?';
+$_['text_customer_delete_confirm_message'] = 'Vuoi eliminare il cliente?';
+$_['text_with_selected'] = 'Con i selezionati';
+
+$_['text_compliance_warning'] = 'È abilitata la cancellazione dei messaggi per cui il tuo archivio non soddisfa i requisiti di legge!';
--- a/webui/language/tw/messages.php
+++ b/webui/language/tw/messages.php
@ -0,0 +1,497 @@
+<?php
+
+$_['text_60_minutes'] = "60 分鐘";
+
+$_['text_action'] = "動作";
+$_['text_active_incoming_queue'] = "活動 + 傳入佇列";
+$_['text_active_incoming_queue_sender'] = "活動 + 傳入佇列與寄件者";
+$_['text_ad_sync_status'] = "AD 同步狀態";
+$_['text_add'] = "新增";
+$_['text_add_new_email_address'] = "新增郵件位址";
+$_['text_add_new_domain'] = "新增網域";
+$_['text_add_new_entry'] = "新增項目";
+$_['text_add_new_group'] = "新增群組";
+$_['text_add_new_rule'] = "新增規則";
+$_['text_add_new_user_alias'] = "新增使用者";
+$_['text_add_policy'] = "新增政策";
+$_['text_administration'] = "管理";
+$_['text_admin_user'] = "管理者";
+$_['text_advanced'] = "進階";
+$_['text_advanced_search'] = "進階搜尋";
+$_['text_all'] = "所有";
+$_['text_any'] = "任一";
+$_['text_applied'] = "已套用";
+$_['text_apply_changes'] = "套用變更";
+$_['text_archive_size'] = "歸檔大小";
+$_['text_archive_size_before_compression'] = "Archive size before compression";
+$_['text_archived_messages'] = "已歸檔郵件";
+$_['text_archiving_rules'] = "歸檔規則";
+$_['text_assigned_email_addresses'] = "已指派郵件位址";
+$_['text_attachment'] = "附件";
+$_['text_attachment_name'] = "附件名稱";
+$_['text_attachment_size'] = "附件大小";
+$_['text_attachment_type'] = "附件類型";
+$_['text_audit'] = "稽核";
+$_['text_automated_search'] = "自動搜尋";
+
+$_['text_back'] = "返回";
+$_['text_background_colour'] = "背景顏色";
+$_['text_body'] = "內文";
+$_['text_branding_logo'] = "商標圖片";
+$_['text_branding_text'] = "商標文字";
+$_['text_branding_url'] = "商標網址";
+$_['text_bulk_edit_selected_uids'] = "批次修改已選取的 uid";
+$_['text_bulk_restore_selected_emails'] = "批次還原已選取的項目";
+$_['text_bulk_update_selected_uids'] = "批次更新已選擇的 uid";
+
+$_['text_cancel'] = "取消";
+$_['text_change_user_settings'] = "變更使用者設定";
+$_['text_clienthost'] = "Client host";
+$_['text_close'] = "關閉";
+$_['text_colour'] = "顏色";
+$_['text_compressed'] = "已壓縮";
+$_['text_confirm_to_reset_counters'] = "確認要重置計數器";
+$_['text_connection_failed'] = "連線失敗";
+$_['text_connection_ok'] = "連線成功";
+$_['text_contact_support'] = "連繫支援";
+$_['text_content_filter'] = "內容篩選器";
+$_['text_conversation_available'] = "可使用對話檢視";
+$_['text_copied'] = "Copied";
+$_['text_counters'] = "計數器";
+$_['text_cpu_load'] = "CPU 負載";
+$_['text_cpu_usage'] = "CPU 使用率";
+$_['text_create_new_secret'] = "建立新的密鑰";
+$_['text_create_note'] = "建立備註";
+$_['text_cumulative_counts'] = "累積計數";
+$_['text_customers'] = "Customers";
+
+$_['text_daily_quarantine_report'] = "每日隔離報表";
+$_['text_daily_quarantine_report_status'] = "每日隔離報表狀態";
+$_['text_daily_report'] = "每日報表";
+$_['text_daily_piler_report'] = "每日 Piler 報表";
+$_['text_database_emails'] = "在 piler 資料庫的郵件位址";
+$_['text_date'] = "日期";
+$_['text_date_from'] = "起始日期";
+$_['text_date_to'] = "結束日期";
+$_['text_days'] = "天";
+$_['text_days2'] = "天";
+$_['text_days_to_retain'] = "保留天數";
+$_['text_deferred_queue'] = "延遲佇列";
+$_['text_deferred_queue_sender'] = "延遲佇列與寄件者";
+$_['text_delay'] = "延遲";
+$_['text_delete_confirm_message'] = "您是否要刪除";
+$_['text_delete_reason'] = "移除原因";
+$_['text_deleted'] = "已刪除";
+$_['text_deleted_users'] = "已刪除";
+$_['text_deliver'] = "傳遞";
+$_['text_delivered'] = "已傳遞";
+$_['text_deliver_and_train_selected_messages'] = "Deliver and train selected messages";
+$_['text_deliver_and_train_selected_messages_as_ham'] = "Deliver and train selected messages AS HAM";
+$_['text_deliver_selected_messages'] = "傳遞已選擇的郵件";
+$_['text_description'] = "描述";
+$_['text_direction'] = "Direction";
+$_['text_disk_usage'] = "磁碟使用率";
+$_['text_disable'] = "停用";
+$_['text_disabled'] = "停用";
+$_['text_dn_asterisk_means_skip_sync'] = "萬用字元 (*) 表示這個使用者項目將不會包含在 AD 同步當中";
+$_['text_domain'] = "網域";
+$_['text_domains'] = "網域";
+$_['text_domainname'] = "網域名稱";
+$_['text_download_all_hits_as_eml'] = "全部下載 (EML)";
+$_['text_download_selected_hits_as_pdf'] = "下載已選取項目 (PDF)";
+$_['text_download_attachment2'] = "下載附件";
+$_['text_download_message'] = "下載 (EML)";
+$_['text_download_message2'] = "下載郵件";
+
+$_['text_edit'] = "編輯";
+$_['text_edit_entry'] = "編輯項目";
+$_['text_edit_group'] = "編輯群組";
+$_['text_edit_user'] = "編輯使用者";
+$_['text_edit_or_view'] = "編輯/檢視";
+$_['text_email'] = "郵件位址";
+$_['text_email_addresses'] = "郵件位址";
+$_['text_email_aliases'] = "郵件別名";
+$_['text_email_in_unknown_domain'] = "郵件位址在未知的網域";
+$_['text_empty_search_criteria'] = "'Empty criteria'";
+$_['text_empty_search_result'] = "沒有搜尋結果。請試著加入萬用字元 (*) 在關鍵字後方 (最小 " . MIN_PREFIX_LEN . " 個字元)，例如 duplic* 以搜尋到 \"duplicate\", \"duplicated\", 之類的結果。";
+$_['text_enable'] = "啟用";
+$_['text_enabled'] = "啟用";
+$_['text_enter_google_authenticator_code'] = "輸入 Google Authenticator 驗證碼";
+$_['text_enter_one_email_address_per_line'] = "每一行輸入一個郵件位址";
+$_['text_enter_one_group_per_line'] = "每一行輸入一個群組";
+$_['text_enter_search_terms'] = "輸入您要搜尋的字詞";
+$_['text_error'] = "錯誤";
+$_['text_exact_domain_name_or_email_address'] = "確認您的網域名稱或郵件位址";
+$_['text_exclude'] = "排除";
+$_['text_existing'] = "現有";
+$_['text_existing_domains'] = "現有網域";
+$_['text_existing_email'] = "現有郵件";
+$_['text_existing_folders'] = "現有資料夾";
+$_['text_existing_groups'] = "現有群組";
+$_['text_existing_policies'] = "現有政策";
+$_['text_existing_entries'] = "現有項目";
+$_['text_existing_rules'] = "現有規則";
+$_['text_existing_user'] = "現有使用者";
+$_['text_existing_users'] = "現有使用者";
+$_['text_expert'] = "專家";
+$_['text_expert_search'] = "專家搜尋";
+
+$_['text_failed'] = "失敗";
+$_['text_failed_to_add'] = "新增失敗";
+$_['text_failed_to_change_password'] = "變更密碼失敗";
+$_['text_failed_to_deliver'] = "傳遞失敗";
+$_['text_failed_to_mark_for_removal'] = "標記移除失敗";
+$_['text_failed_to_modify'] = "修改失敗";
+$_['text_failed_to_remove'] = "移除失敗";
+$_['text_failed_to_restore'] = "還原失敗";
+$_['text_failed_to_update'] = "更新失敗";
+$_['text_first'] = "First";
+$_['text_folder'] = "資料夾";
+$_['text_folder_rules'] = "資料夾規則";
+$_['text_folders'] = "資料夾";
+$_['text_forward_selected_emails_to'] = "轉寄已選擇郵件至";
+$_['text_from'] = "寄件者";
+$_['text_from_domain'] = "寄件者網域";
+
+$_['text_google_authenticator_code'] = "Google Authenticator 驗證碼";
+$_['text_google_authenticator_settings'] = "Google Authenticator 設定";
+$_['text_group_id'] = "群組 id";
+$_['text_groupname'] = "群組名稱";
+$_['text_groups'] = "群組";
+$_['text_group_management'] = "群組管理";
+$_['text_group_membership'] = "群組成員";
+
+$_['text_health'] = "健康狀況";
+$_['text_health_monitor'] = "健康狀況監視器";
+$_['text_help'] = "說明";
+$_['text_history'] = "歷程";
+$_['text_home'] = "首頁";
+
+$_['text_image'] = "image";
+$_['text_import'] = "匯入";
+$_['text_import_job_delete_confirm_message'] = "您是否要刪除匯入作業";
+$_['text_import_users'] = "匯入使用者";
+$_['text_import_users_from_LDAP'] = "從 LDAP 匯入使用者";
+$_['text_inbound'] = "inbound";
+$_['text_indexer_job'] = "索引器作業";
+$_['text_install_sudo_apply'] = "加入下列內容到 /etc/sudoers: 'www-data ALL=NOPASSWD: /etc/init.d/rc.piler reload'";
+$_['text_internal'] = "internal";
+$_['text_invalid_data'] = "無效的資料";
+$_['text_invalid_email'] = "無效的郵件";
+$_['text_invalid_email_or_password'] = "無效的郵件或密碼";
+$_['text_invalid_gid'] = "無效的 gid";
+$_['text_invalid_password'] = "無效的密乸";
+$_['text_invalid_pin_code'] = "無效的 Pin 碼";
+$_['text_invalid_policy_group'] = "無效的政策群組";
+$_['text_invalid_policy_name'] = "無效的政策名稱";
+$_['text_invalid_policy_setting'] = "無效的政策設定";
+$_['text_invalid_uid'] = "無效的 uid";
+$_['text_invalid_username'] = "無效的使用者名稱";
+$_['text_ipaddr'] = "IP 位址";
+
+$_['text_language'] = "語言";
+$_['text_last'] = "Last";
+$_['text_last_activity'] = "最後活動";
+$_['text_last_update'] = "最後更新";
+$_['text_latest_emails'] = "最新郵件";
+$_['text_ldap'] = "LDAP";
+$_['text_ldap_auditor_member_dn'] = "稽核者成員 DN";
+$_['text_ldap_base_dn'] = "LDAP 基礎 DN";
+$_['text_ldap_bind_dn'] = "LDAP 繫結 DN";
+$_['text_ldap_bind_pw'] = "LDAP 繫結密碼";
+$_['text_ldap_host'] = "LDAP 主機";
+$_['text_ldap_type'] = "LDAP 類型";
+$_['text_legal_hold'] = "法務保存措施";
+$_['text_load'] = "載入";
+$_['text_loading'] = "載入中";
+$_['text_logged_in'] = "已登入";
+$_['text_logged_out'] = "您已登出";
+$_['text_login'] = "登入";
+$_['text_login2'] = "登入";
+$_['text_login_failed'] = "登入失敗";
+$_['text_login_via_google'] = "經由 Google 帳戶登入";
+$_['text_logout'] = "登出";
+$_['text_logout2'] = "登出";
+
+$_['text_maillog_status'] = "郵件記錄收集器狀態";
+$_['text_main_title'] = "clapf web UI";
+$_['text_mapped_domain'] = "對應網域";
+$_['text_mark_private'] = "隱私";
+$_['text_marked_for_removal'] = "郵件標示為移除";
+$_['text_memory_usage'] = "記憶體使用率";
+$_['text_message'] = "郵件";
+$_['text_messages'] = "郵件";
+$_['text_message_disposition'] = "郵件處理";
+$_['text_message_text'] = "郵件文字";
+$_['text_min_2_chars'] = "最小 2 個字元";
+$_['text_missing_data'] = "遺漏資料";
+$_['text_missing_password'] = "遺漏密碼";
+$_['text_modify'] = "修改";
+$_['text_monitor'] = "監視器";
+$_['text_months'] = "月";
+$_['text_monthly_report'] = "每月報表";
+
+$_['text_need_to_approve_removal'] = "移除需要核准";
+$_['text_new'] = "新增";
+$_['text_new_users'] = "新增";
+$_['text_next'] = "Next";
+$_['text_no_domain_found'] = '沒有找到網域';
+$_['text_no_email_found'] = '沒有找到郵件';
+$_['text_no_message_in_the_quarantine'] = "隔離區內沒有符合搜尋結果的郵件";
+$_['text_no_records'] = "沒有記錄";
+$_['text_no_selected_message'] = "沒有選擇郵件";
+$_['text_no_sender'] = "沒有寄件者";
+$_['text_no_spam_message_in_the_quarantine_yet'] = "隔離區中沒有垃圾郵件";
+$_['text_no_subject'] = "沒有主旨";
+$_['text_no_such_policy'] = "沒有政策";
+$_['text_non_existent_queue_directory'] = "您指定的佇列不存在";
+$_['text_non_existing_user'] = "無現有的使用者";
+$_['text_notes'] = "備註";
+$_['text_not_found'] = "沒有找到";
+$_['text_not_running'] = "未在執行中";
+$_['text_not_spam'] = "不是垃圾郵件";
+$_['title_not_found'] = "Page not found";
+$_['text_number_of_messages_in_quarantine'] = "隔離區內符合搜尋結果的郵件數量";
+$_['text_number_of_spam_messages_in_quarantine'] = "隔離區內符合搜尋結果的垃圾郵件數量";
+
+$_['text_off'] = "關閉";
+$_['text_on'] = "啟用";
+$_['text_online_users'] = "線上使用者";
+$_['text_other'] = "其它";
+$_['text_outbound'] = "outbound";
+
+$_['text_password'] = "密碼";
+$_['text_password_again'] = "再一次密碼";
+$_['text_password_changed'] = "密碼已變更";
+$_['text_password_mismatch'] = "密碼不符";
+$_['text_page_length'] = "每頁結果";
+$_['text_periodic_purge'] = "定期清理";
+$_['text_policy'] = "政策";
+$_['text_policy_group'] = "政策群組";
+$_['text_policy_name'] = "政策名稱";
+$_['text_previous'] = "Previous";
+$_['text_print_message'] = "列印";
+$_['text_private'] = "隱私";
+$_['text_processed_emails'] = "已處理郵件";
+$_['text_progress'] = "處理進度";
+$_['text_purge_all_messages_from_quarantine'] = "從隔離區清理您的所有郵件";
+$_['text_purge_selected_messages'] = "清理已選擇郵件";
+$_['text_purged'] = "清理";
+
+$_['text_qr_code'] = "QR code";
+$_['text_queue_status'] = "佇列狀態";
+$_['text_quick_search'] = "快速搜尋";
+
+$_['text_realname'] = "人員姓名";
+$_['text_recipient'] = "收件者";
+$_['text_ref'] = "參照";
+$_['text_refresh_period'] = "重新整理週期";
+$_['text_refresh_qr_code'] = "重新整理 QR code";
+$_['text_reject'] = "拒絕";
+$_['text_rejected_removal'] = "rejected removal";
+$_['text_reason_of_rejection'] = "拒絕原因";
+$_['text_relay_details'] = "轉送細節";
+$_['text_relay_status'] = "轉送狀態";
+$_['text_remove'] = "移除";
+$_['text_remove_domain'] = "移除網域";
+$_['text_remove_message'] = "移除郵件";
+$_['text_remove_message2'] = "移除郵件";
+$_['text_remove_request'] = "移除要求";
+$_['text_remove_selected_uids'] = "移除已選擇 uids";
+$_['text_remove_policy'] = "移除政策";
+$_['text_remove_rule'] = "移除規則";
+$_['text_remove_this_policy'] = "移除此政策";
+$_['text_remove_this_group'] = "移除此群組";
+$_['text_remove_this_user'] = "移除此使用者";
+$_['text_removed'] = "已移除";
+$_['text_reset_counters'] = "重置計數器";
+$_['text_restore'] = "還原";
+$_['text_restored'] = "已還原";
+$_['text_restore_message'] = "還原郵件";
+$_['text_restore_to_mailbox'] = "還原郵件至郵箱";
+$_['text_result'] = "結果";
+$_['text_retention_days'] = "保留天數";
+$_['text_retention_rules'] = "保留規則";
+$_['text_role'] = "角色";
+$_['text_running'] = "執行中";
+
+$_['text_save'] = "儲存";
+$_['text_saved'] = "已儲存";
+$_['text_save_search'] = "儲存搜尋";
+$_['text_save_search_terms'] = "搜尋字詞";
+$_['text_saved_search_terms'] = "搜尋字詞已儲存";
+$_['text_search'] = "搜尋";
+$_['text_search2'] = "搜尋";
+$_['text_search_emails'] = "搜尋郵件位址";
+$_['text_search_email_to_add'] = "搜尋郵件以加入";
+$_['text_search_expression'] = "搜尋語法";
+$_['text_search_folders'] = "搜尋資料夾";
+$_['text_search_folder_to_add'] = "搜尋資料夾以加入";
+$_['text_search_groups'] = "搜尋群組";
+$_['text_search_group_to_add'] = "搜尋群組以加入";
+$_['text_search_terms'] = "搜尋字詞";
+$_['text_select_action'] = "選擇動作";
+$_['text_select_all'] = "全部選擇";
+$_['text_select_image'] = "Select image";
+$_['text_select_recipients'] = "選擇收件者";
+$_['text_sender'] = "寄件者";
+$_['text_sending_domains'] = "寄件者網域";
+$_['text_server_name'] = "伺服器名稱";
+$_['text_server_operating_system'] = "作業系統";
+$_['text_set'] = "設定";
+$_['text_settings'] = "設定";
+$_['text_simple'] = "簡易";
+$_['text_simple_search'] = "簡易搜尋";
+$_['text_size'] = "大小";
+$_['text_smtp_status'] = "SMTP 狀態";
+$_['text_spam'] = "垃圾郵件";
+$_['text_spam2'] = "垃圾郵件";
+$_['text_statistics'] = "使用統計";
+$_['text_status'] = "狀態";
+$_['text_storage'] = "儲存";
+$_['text_subject'] = "主旨";
+$_['text_submit'] = "送出";
+$_['text_successful'] = "成功";
+$_['text_successfully_added'] = "新增成功";
+$_['text_successfully_delivered'] = "傳遞成功";
+$_['text_successfully_modified'] = "修改成功";
+$_['text_successfully_removed'] = "移除成功";
+$_['text_successfully_trained'] = "Successfully trained";
+$_['text_successfully_updated'] = "修改成功";
+$_['text_support_link'] = "支援連結";
+$_['text_swap_usage'] = "Swap 使用率";
+
+$_['text_tag_selected_messages'] = "標籤已選擇郵件";
+$_['text_tagged'] = "標籤";
+$_['text_tags'] = "標籤";
+$_['text_test_connection'] = "測試連線";
+$_['text_text'] = "文字";
+$_['text_text_colour'] = "文字顏色";
+$_['text_text2'] = "文字";
+$_['text_theme'] = "佈景主題";
+$_['text_time'] = "時間";
+$_['text_to'] = "收件者";
+$_['text_to_domain'] = "收件者網域";
+$_['text_too_short_password'] = "密碼太短";
+$_['text_total'] = "總計";
+$_['text_total_ratio'] = "總比例";
+$_['text_total_query_time'] = "總計 SQL 查詢時間";
+$_['text_total_users'] = "總計";
+$_['text_type'] = "類型";
+
+$_['text_uids'] = "uids";
+$_['text_unauthorized_domain'] = "未授權網域";
+$_['text_unauthorized_download_attachment'] = "未授權下載附件";
+$_['text_unauthorized_remove_message'] = "未授權移除郵件";
+$_['text_unauthorized_view_message'] = "未授權檢視郵件";
+$_['text_unknown'] = "未知";
+$_['text_update_retention_within_this_domain'] = "此網域內的更新保留數值";
+$_['text_update_selected_uids'] = "更新所選的 uid";
+$_['text_updated_records'] = "已更新記錄";
+$_['text_uptime'] = "開機時間";
+$_['text_user'] = "使用者";
+$_['text_users'] = "使用者";
+$_['text_user_id'] = "使用者 id";
+$_['text_user_auditor'] = "稽核者";
+$_['text_user_data_officer'] = "資料管理者";
+$_['text_user_domainadmin'] = "網域管理者";
+$_['text_user_management'] = "使用者管理";
+$_['text_user_masteradmin'] = "主要管理者";
+$_['text_user_read_only_admin'] = "唯讀管理者";
+$_['text_user_regular'] = "一般使用者";
+$_['text_userlist'] = "使用者清單";
+$_['text_username'] = "使用者名稱";
+$_['text_users_quarantine'] = "使用者的隔離區";
+
+$_['text_view_formatted_email'] = "檢視格式化郵件";
+$_['text_view_header'] = "檢視標頭";
+$_['text_view_headers'] = "檢視標頭";
+$_['text_view_journal'] = "日誌";
+$_['text_view_journal_envelope'] = "檢視信封";
+$_['text_view_message'] = "檢視郵件";
+$_['text_view_message2'] = "檢視郵件";
+$_['text_view_progress'] = "檢視進度";
+$_['text_view_raw_email'] = "檢視原始郵件";
+$_['text_view_user_quarantine'] = "檢視使用者的隔離區";
+
+$_['text_warning_about_default_policy'] = "預設政測可以在 clapf.conf 中設定";
+$_['text_wildcard_domains'] = "通用網域";
+$_['text_whitelist'] = "允許清單";
+$_['text_whitelist_settings'] = "允許清單設定";
+$_['text_with_attachment'] = "含附件";
+$_['text_without_attachment'] = "不含附件";
+
+$_['text_years'] = "年";
+$_['text_you_are'] = "您是";
+$_['text_you_are_not_admin'] = "您不是管理者";
+
+
+$_['rcvd'] = "接收郵件";
+$_['virus'] = "感染郵件";
+$_['duplicate'] = "重複郵件";
+$_['ignore'] = "忽略郵件";
+$_['counters_last_update'] = "計數器更新";
+
+$_['text_24_hours'] = "24 小時";
+$_['text_1_week'] = "1 週";
+$_['text_30_days'] = "30 天";
+
+$_['text_access_settings'] = '存取設定';
+$_['text_access_setting_explanation'] = "您可以存取自己的電子郵件。對於指定群組或網域的稽核者存取，請您與管理者連繫。";
+$_['text_display_settings'] = '顯示設定';
+$_['text_change_password'] = "變更密碼";
+$_['text_none_found'] = "沒有找到";
+$_['text_primary_domain'] = "主要網域";
+$_['text_search_domains'] = "搜尋網域";
+$_['text_search_domain_to_add'] = "搜尋網域以加入";
+
+$_['text_space_projection'] = '空間保護';
+$_['text_average_messages_day'] = '每天平均郵件';
+$_['text_average_message_size'] = '平均郵件 + 中繼資料 + 索引大小';
+$_['text_average_size_day'] = '每天平均大小';
+$_['text_partition_full'] = '磁區預計用完於';
+$_['text_usage_trend'] = '使用趨勢';
+$_['text_usage_increasing'] = '上升';
+$_['text_usage_decreasing'] = '下降';
+$_['text_usage_neutral'] = '持平';
+$_['text_accounting'] = '歸檔稽核';
+$_['text_accounting_email'] = '依郵件稽核';
+$_['text_accounting_domain'] = '依網域稽核';
+
+$_['text_options'] = '選項';
+$_['text_spam_flag'] = '郵件已標記為 SPAM';
+$_['text_attachment_flag'] = '郵件有附件';
+$_['text_notes_flag'] = '郵件有備註';
+$_['text_tag_flag'] = '郵件已標記';
+$_['text_verified_flag'] = '郵件已驗證';
+$_['text_unverified_flag'] = '郵件驗證失敗';
+$_['text_bulk_download'] = '批次下載已選擇郵件';
+$_['text_clear'] = '清除';
+$_['text_select_letter'] = '以開頭字母選擇';
+$_['text_working'] = '作業中...';
+
+$_['text_use_browser_settings'] = '使用瀏覽器設定';
+
+$_['text_sent'] = '寄送';
+$_['text_received'] = '已接收';
+$_['text_oldest_record'] = '最舊記錄';
+$_['text_newest_record'] = '最新記錄';
+$_['text_items'] = '項目';
+$_['text_average_size'] = '平均大小';
+
+$_['text_return_to'] = '回到';
+$_['text_error_message'] = '請更正以下錯誤後重新送出。';
+$_['text_field_required'] = '此為必填欄位。';
+$_['text_field_length'] = '這個欄位必須大於 ? 個字元。';
+$_['text_field_domain'] = '這個欄位必須填入網域 (例如 - domain.com)。';
+$_['text_field_colour'] = '這個欄位必須填入色碼 (例如 - #fcfcfc)。';
+$_['text_delete'] = '刪除';
+$_['text_confirm'] = '確認';
+$_['text_user_delete_confirm_message'] = '您要刪除使用者嗎';
+$_['text_domain_delete_confirm_message'] = '您要刪除網域嗎';
+$_['text_group_delete_confirm_message'] = '您要刪除群組嗎';
+$_['text_ldap_delete_confirm_message'] = '您要刪除 LDAP 項目嗎';
+$_['text_customer_delete_confirm_message'] = '您要刪除客戶嗎';
+$_['text_with_selected'] = '選擇項目';
+
+$_['text_compliance_warning'] = '刪除功能已啟用，因此這個歸檔不符合規定！';
--- a/webui/model/accounting/accounting.php
+++ b/webui/model/accounting/accounting.php
@ -36,7 +36,7 @@ class ModelAccountingAccounting extends Model {

      // emails sent to users

-      $tousers = $this->db->query("SELECT `$column`-(`$column` % 86400) AS `day`, `to`, COUNT(*) AS `count`, SUM(`size`) AS `size` FROM " . VIEW_MESSAGES . " WHERE `$column` >= ? AND `$column` < ? GROUP BY FROM_UNIXTIME(`day`, '%Y.%m.%d.'), `to`", array($start, $stop));
+      $tousers = $this->db->query("SELECT `$column`-(`$column` % 86400) AS `day`, `to`, COUNT(*) AS `count`, SUM(`size`) AS `size` FROM " . VIEW_MESSAGES . " WHERE deleted=0 AND `$column` >= ? AND `$column` < ? GROUP BY FROM_UNIXTIME(`day`, '%Y.%m.%d.'), `to`", array($start, $stop));

      foreach($tousers->rows as $row) {
         $counter[$row['day']][$row['to']]['recd'] = $row['count'];
@ -46,7 +46,7 @@ class ModelAccountingAccounting extends Model {

      // emails sent from users

-      $fromusers = $this->db->query("SELECT `$column`-(`$column` % 86400) AS `day`, `from`, COUNT(*) AS `count`, SUM(`size`) AS `size` FROM " . VIEW_MESSAGES . " WHERE `$column` >= ? AND `$column` < ? GROUP BY FROM_UNIXTIME(`day`, '%Y.%m.%d.'), `from`", array($start, $stop));
+      $fromusers = $this->db->query("SELECT `$column`-(`$column` % 86400) AS `day`, `from`, COUNT(*) AS `count`, SUM(`size`) AS `size` FROM " . VIEW_MESSAGES . " WHERE deleted=0 AND `$column` >= ? AND `$column` < ? GROUP BY FROM_UNIXTIME(`day`, '%Y.%m.%d.'), `from`", array($start, $stop));

      foreach($fromusers->rows as $row) {
         $counter[$row['day']][$row['from']]['sent'] = $row['count'];
@ -144,10 +144,6 @@ class ModelAccountingAccounting extends Model {

      $search = preg_replace("/\s{1,}/", "", $search);

-      if($search) {
-         $search_cond .= " AND ( `email` LIKE '%".$search."%' OR `domain` LIKE '%".$search."%' )";
-      }
-
      $query = "SELECT `email` AS `item`, MIN(`date`) AS `oldest`, MAX(`date`) AS `newest`, SUM(`sent`) AS `sent`, SUM(`recd`) AS `recd`, SUM(`sentsize`) AS `sentsize`, SUM(`recdsize`) AS `recdsize` FROM " . TABLE_STAT_COUNTER;

      if($item == 'email') {
--- a/webui/model/audit/audit.php
+++ b/webui/model/audit/audit.php
@ -43,7 +43,7 @@ class ModelAuditAudit extends Model {
      if(Registry::get('admin_user') == 0 && RESTRICTED_AUDITOR == 1) {
         $auditdomains = $session->get("auditdomains");

-         while(list($k, $v) = each($auditdomains)) {
+         foreach($auditdomains as $k => $v) {
            if($q) { $q .= ","; }
            $q .= "?";
            array_push($arr, $v);
--- a/webui/model/google/google.php
+++ b/webui/model/google/google.php
@ -76,7 +76,7 @@ class ModelGoogleGoogle extends Model {

         $messages = $storage->piler_batch_fetch($from, $to);

-         while(list($k, $v) = each($messages)) {
+         foreach($messages as $k => $v) {
            $uuid = $storage->getUniqueId($k);

            $tmpname = "piler-" . $email . "-" . $k . "-" . $uuid . ".eml";
@ -174,5 +174,3 @@ class ModelGoogleGoogle extends Model {


 }
-
-?>
--- a/webui/model/health/health.php
+++ b/webui/model/health/health.php
@ -199,7 +199,7 @@ class ModelHealthHealth extends Model {
   public function meminfo() {
      $m = explode("\n", file_get_contents("/proc/meminfo"));

-      while(list($k, $v) = each($m)) {
+      foreach($m as $k => $v) {
         $a = preg_split("/\ {1,}/", $v);
         if(isset($a[0]) && $a[0]) { $_m[$a[0]] = $a[1]; }
      }
@ -219,7 +219,7 @@ class ModelHealthHealth extends Model {

      $partitions = Registry::get('partitions_to_monitor');

-      while(list($k, $v) = each($output)) {
+      foreach($output as $k => $v) {
         if($k > 0) {
            $p = preg_split("/\ {1,}/", $v);
            if(isset($p[5]) && in_array($p[5], $partitions) && !isset($a[$p[5]])) {
--- a/webui/model/mail/mail.php
+++ b/webui/model/mail/mail.php
@ -26,7 +26,7 @@ class ModelMailMail extends Model {
      fputs($r, "MAIL FROM: <$from>\r\n");
      $l = fgets($r, 4096);

-      while(list($k, $v) = each($to)) {
+      foreach($to as $k => $v) {
         fputs($r, "RCPT TO: <$v>\r\n");
         $l = fgets($r, 4096);
      }
--- a/webui/model/search/message.php
+++ b/webui/model/search/message.php
@ -213,7 +213,7 @@ class ModelSearchMessage extends Model {
      $a = explode(" ", $terms);
      $terms = array();

-      while(list($k, $v) = each($a)) {
+      foreach($a as $k => $v) {
         if(strlen($v) >= 3 && !in_array($v, $fields)) {
            $v = preg_replace("/\*/", "", $v);
            if($v) { array_push($terms, $v); }
@ -223,7 +223,7 @@ class ModelSearchMessage extends Model {
      if(count($terms) <= 0) { return $s; }

      if($html == 0) {
-         while(list($k, $v) = each($terms)) {
+         foreach($terms as $k => $v) {
            $s = preg_replace("/$v/i", "<span class=\"mssghglght\">$v</span>", $s);
         }

@ -233,7 +233,7 @@ class ModelSearchMessage extends Model {
      $tokens = preg_split("/\</", $s);
      $s = '';

-      while(list($k, $token) = each($tokens)) {
+      foreach($tokens as $k => $token) {

         $pos = strpos($token, ">");
         if($pos > 0) {
@ -245,7 +245,7 @@ class ModelSearchMessage extends Model {
               $str = substr($token, $pos+1, $len);

               reset($terms);
-               while(list($k, $v) = each($terms)) {
+               foreach($terms as $k => $v) {
                  $str = preg_replace("/$v/i", "<span class=\"mssghglght\">$v</span>", $str);
               }

@ -365,13 +365,20 @@ class ModelSearchMessage extends Model {

         if($computed_hash == '') {

-            $query2 = $this->db->query("SELECT digest FROM " . TABLE_META . " WHERE id >= ? AND id <= ?", array($query->row['start_id'], $query->row['stop_id']));
+            $query2 = $this->db->query("SELECT digest FROM " . TABLE_META . " WHERE id >= ? AND id <= ? ORDER BY id", array($query->row['start_id'], $query->row['stop_id']));

            foreach($query2->rows as $q) {
               $s .= $q['digest'];
            }

-            $computed_hash = sha1($s);
+            $len = strlen($query->row['hash_value']);
+            if($len == 64)
+               $algo='sha256';
+            elseif($len == 128)
+               $algo='sha512';
+            else
+               $algo='sha1';
+            $computed_hash = hash($algo, $s);

            if(MEMCACHED_ENABLED) {
               $memcache->add($cache_key, $computed_hash, 0, MEMCACHED_TTL);
--- a/webui/model/search/search.php
+++ b/webui/model/search/search.php
@ -16,7 +16,7 @@ class ModelSearchSearch extends Model {

      $session = Registry::get('session');

-      while(list($k,$v) = each($data)) {
+      foreach($data as $k => $v) {
         if($v) { if(is_array($v)) { $v = implode(" ", $v); } $s .= '&' . $k . '=' . $v; }
      }

@ -113,11 +113,10 @@ class ModelSearchSearch extends Model {
      $session = Registry::get('session');

      $i = 0;
-      while(list($k, $v) = each($data['match'])) {
+      foreach($data['match'] as $k => $v) {
         if($v == "@attachment_types") {
-            list($k, $v) = each($data['match']);
            $i++;
-            if($v == "any") {
+            if($data['match'][$i] == "any") {
               $data['match'][$i-1] = "";
               $data['match'][$i] = "";
            }
@ -199,7 +198,7 @@ class ModelSearchSearch extends Model {

      if(ENABLE_FOLDER_RESTRICTIONS == 1) {
         $s = explode(" ", $data['folders']);
-         while(list($k,$v) = each($s)) {
+         foreach($s as $k => $v) {
            if(in_array($v, $session->get("folders"))) {
               array_push($__folders, $v);
            }
@ -347,7 +346,7 @@ class ModelSearchSearch extends Model {
      $s = preg_replace("/httpX/", "http:", $s);
      $b = explode(" ", $s);

-      while(list($k, $v) = each($b)) {
+      foreach($b as $k => $v) {
         if($v == '') { continue; }

         if($v == 'from:') { $token = 'match'; $a['match'][] = FROM_TOKEN; continue; }
@ -441,7 +440,7 @@ class ModelSearchSearch extends Model {
      $offset = $page * $pagelen;

      $s = explode(" ", $extra_folders);
-      while(list($k,$v) = each($s)) {
+      foreach($s as $k => $v) {
         if(in_array($v, $session->get("extra_folders")) && is_numeric($v)) {
            array_push($__folders, $v);
            if($q) { $q .= ",?"; }
@ -647,7 +646,9 @@ class ModelSearchSearch extends Model {

      $emails = $session->get($session_var);

-      while(list($k, $v) = each($emails)) {
+      if(!$emails) { return $s; }
+
+      foreach($emails as $k => $v) {
         if($s) { $s .= '| ' .  $this->fix_email_address_for_sphinx($v); }
         else { $s = $this->fix_email_address_for_sphinx($v); }
      }
@ -681,6 +682,13 @@ class ModelSearchSearch extends Model {

      if($id == '') { return 0; }

+      if(Registry::get('data_officer') == 1) {
+         $query = $this->db->query("SELECT id FROM " . TABLE_DELETED . " WHERE deleted=-1 AND id=?", array($id));
+         if(!isset($query->row['id'])) {
+            return 0;
+         }
+      }
+
      if((Registry::get('auditor_user') == 1 || Registry::get('data_officer') == 1) && RESTRICTED_AUDITOR == 0) { return 1; }

      $session = Registry::get('session');
@ -695,7 +703,7 @@ class ModelSearchSearch extends Model {

         $auditdomains = $session->get("auditdomains");

-         while(list($k, $v) = each($auditdomains)) {
+         foreach($auditdomains as $k => $v) {
            if(validdomain($v) == 1 && !in_array($v, $a)) {
               $q .= ",?";
               array_push($a, $v);
@ -711,7 +719,7 @@ class ModelSearchSearch extends Model {

         $emails = $session->get("emails");

-         while(list($k, $v) = each($emails)) {
+         foreach($emails as $k => $v) {
            if(validemail($v) == 1) {
               $q .= ",?";
               array_push($a, $v);
@ -767,7 +775,7 @@ class ModelSearchSearch extends Model {

         $auditdomains = $session->get("auditdomains");

-         while(list($k, $v) = each($auditdomains)) {
+         foreach($auditdomains as $k => $v) {
            if(validdomain($v) == 1 && !in_array($v, $a)) {
               $q .= ",?";
               array_push($a, $v);
@ -778,7 +786,7 @@ class ModelSearchSearch extends Model {
         if(Registry::get('auditor_user') == 0) {
            $emails = $session->get("emails");

-            while(list($k, $v) = each($emails)) {
+            foreach($emails as $k => $v) {
               if(validemail($v) == 1) {
                  $q .= ",?";
                  array_push($a, $v);
@ -835,6 +843,10 @@ class ModelSearchSearch extends Model {


   public function fix_email_address_for_sphinx($email = '') {
+      if(strlen($email) > MAX_EMAIL_LEN) {
+         return md5($email . ' ');
+      }
+
      $email = preg_replace("/\|@/", "|", $email);
      return preg_replace("/[\@\.\+\-\_]/", "X", $email);
   }
@ -906,7 +918,7 @@ class ModelSearchSearch extends Model {
      $a = explode(" ", $s);
      $s = '';

-      while(list($k, $v) = each($a)) {
+      foreach($a as $k => $v) {

         if(substr($v, 0, 4) == 'http') {
            $v = preg_replace("/http(s){0,1}\:\/\//", "__URL__", $v);
--- a/webui/model/stat/chart.php
+++ b/webui/model/stat/chart.php
@ -69,7 +69,7 @@ class ModelStatChart extends Model {

      if($query->num_rows >= 15) {
         $i = 0;
-         while(list($k, $v) = each($dates)) {
+         foreach($dates as $k => $v) {
            $i++;
            if($i % 3) { $dates[$k] = ""; }
         }
--- a/webui/model/user/import.php
+++ b/webui/model/user/import.php
@ -155,7 +155,7 @@ class ModelUserImport extends Model {

      /* build a list of DNs to exclude from the import */

-      while (list($k, $v) = each($globals)) {
+      foreach($globals as $k => $v) {
         if(preg_match("/^reject_/", $k)) {
            $exclude[$v] = $v;
         }
--- a/webui/model/user/user.php
+++ b/webui/model/user/user.php
@ -49,7 +49,7 @@ class ModelUserUser extends Model {
      $data = array();
      $uids = $uid;

-      if($uid > 0) {
+      if($uid >= 0) {
         $query = $this->db->query("SELECT gid FROM " . TABLE_EMAIL_LIST . " WHERE uid=?", array((int)$uid));

         if(isset($query->rows)) {
@ -68,9 +68,6 @@ class ModelUserUser extends Model {

      }

-      $emails = $this->get_email_addresses_from_groups($data);
-      $data = array_merge($data, $emails);
-
      return $data;
   }

@ -559,7 +556,10 @@ class ModelUserUser extends Model {
      if(!$this->check_uid($uid)){ return 0; }

      $query = $this->db->query("DELETE FROM " . TABLE_EMAIL . " WHERE uid=?", array((int)$uid));
+      $query = $this->db->query("DELETE FROM " . TABLE_USER_SETTINGS . " WHERE username IN (SELECT username FROM " . TABLE_USER . " WHERE uid=?)", array((int)$uid));
      $query = $this->db->query("DELETE FROM " . TABLE_USER . " WHERE uid=?", array((int)$uid));
+      $query = $this->db->query("DELETE FROM " . TABLE_DOMAIN_USER . " WHERE uid=?", array((int)$uid));
+      $query = $this->db->query("DELETE FROM " . TABLE_FOLDER_USER . " WHERE uid=?", array((int)$uid));

      LOGGER("remove user: uid=$uid");

--- a/webui/phpunit.xml
+++ b/webui/phpunit.xml
@ -2,11 +2,11 @@
 <phpunit colors="true">
    <testsuites>
        <testsuite name="Application Test Suite">
-            <directory>./unit_tests/php</directory>
+            <directory>./tests</directory>
        </testsuite>
    </testsuites>
    <php>
-        <env name="DIR_BASE" value="webui/" />
-        <env name="TEST_FILES_DIR" value="test_files/" />
+        <env name="DIR_BASE" value="./" />
+        <env name="TEST_FILES_DIR" value="../test_files/" />
    </php>
 </phpunit>
--- a/webui/system/controller.php
+++ b/webui/system/controller.php
@ -27,7 +27,9 @@ class Controller {


   public function args($args = array()){
-      while(list($key, $value) = each($args)) $this->data[$key] = $value;
+      foreach($args as $key => $value) {
+         $this->data[$key] = $value;
+      }
   }


--- a/webui/system/database/ldap.php
+++ b/webui/system/database/ldap.php
@ -12,6 +12,10 @@ class LDAP {
      ldap_set_option($this->link, LDAP_OPT_PROTOCOL_VERSION, 3);
      ldap_set_option($this->link, LDAP_OPT_REFERRALS, 0);

+      if (LDAP_USE_START_TLS == 1) {
+         ldap_start_tls($this->link);
+      }
+
      if(@ldap_bind($this->link, $binddn, $bindpw)) {
         $this->bind = 1;
      }
--- a/webui/system/database/mysql.php
+++ b/webui/system/database/mysql.php
@ -53,7 +53,7 @@ class MySQL {

      $R = $s->fetchAll(PDO::FETCH_ASSOC);

-      while(list ($k, $v) = each($R)){
+      foreach($R as $k => $v) {
         $data[$i] = $v;
         $i++;
      }
--- a/webui/system/database/sphinx.php
+++ b/webui/system/database/sphinx.php
@ -39,13 +39,15 @@ class Sphinx {
      $s = $this->link->prepare($sql);
      if(!$s) { return $query; }

-      $s->execute($arr);
+      if(!$s->execute($arr)) {
+         syslog(LOG_INFO, $s->errorInfo()[2]);
+      }

      $this->affected = $s->rowCount();

      $R = $s->fetchAll();

-      while(list ($k, $v) = each($R)){
+      foreach($R as $k => $v) {
         $data[$i] = $v;
         $i++;
      }
@ -70,7 +72,7 @@ class Sphinx {

      $meta->execute();
      $R = $meta->fetchAll();
-      while(list ($k, $v) = each($R)){
+      foreach($R as $k => $v) {
         if($v[0] == "total_found") { $query->total_found = $v[1]; }
      }

--- a/webui/system/database/sqlite.php
+++ b/webui/system/database/sqlite.php
@ -47,7 +47,7 @@ class SQLite {

      $R = $s->fetchAll();

-      while(list ($k, $v) = each($R)){
+      foreach($R as $k => $v) {
         $data[$i] = $v;
         $i++;
      }
--- a/webui/system/helper/HTMLPurifier.standalone.php
+++ b/webui/system/helper/HTMLPurifier.standalone.php
@ -3923,7 +3923,7 @@ class HTMLPurifier_Encoder

        $len = strlen($str);
        for ($i = 0; $i < $len; $i++) {
-            $in = ord($str{$i});
+            $in = ord($str[$i]);
            $char .= $str[$i]; // append byte to char
            if (0 == $mState) {
                // When mState is zero we expect either a US-ASCII character
@ -14284,7 +14284,7 @@ class HTMLPurifier_ChildDef_Custom extends HTMLPurifier_ChildDef
    protected function _compileRegex()
    {
        $raw = str_replace(' ', '', $this->dtd_regex);
-        if ($raw{0} != '(') {
+        if ($raw[0] != '(') {
            $raw = "($raw)";
        }
        $el = '[#a-zA-Z0-9_.-]+';
@ -20681,7 +20681,7 @@ class HTMLPurifier_TagTransform_Font extends HTMLPurifier_TagTransform
        if (isset($attr['size'])) {
            // normalize large numbers
            if ($attr['size'] !== '') {
-                if ($attr['size']{0} == '+' || $attr['size']{0} == '-') {
+                if ($attr['size'][0] == '+' || $attr['size'][0] == '-') {
                    $size = (int)$attr['size'];
                    if ($size < -2) {
                        $attr['size'] = '-2';
--- a/webui/system/helper/TrustedTimestamps.php
+++ b/webui/system/helper/TrustedTimestamps.php
@ -15,6 +15,11 @@
 *  http://www.slproweb.com/products/Win32OpenSSL.html
 *  http://www.switch.ch/aai/support/howto/openssl-windows.html
 *
+ * 2021-10-26 Frank Schmirler:
+ *  - extract certificate chain from TSResponse and feed into ts -verify as -untrusted
+ *  - unlink temporary files
+ *  - support for sha256 and sha512 hashes
+ *
 * @version 0.3
 * @author David Müller
 * @package trustedtimestamps
@ -25,16 +30,22 @@ class TrustedTimestamps
    /**
     * Creates a Timestamp Requestfile from a hash
     *
-     * @param string $hash: The hashed data (sha1)
+     * @param string $hash: The hashed data (sha1, sha256 or sha512)
     * @return string: path of the created timestamp-requestfile
     */
    public static function createRequestfile ($hash)
    {
-        if (strlen($hash) !== 40)
+        if (strlen($hash) === 40)
+            $digest="-sha1";
+        elseif (strlen($hash) === 64)
+            $digest="-sha256";
+        elseif (strlen($hash) === 128)
+            $digest="-sha512";
+        else
            throw new Exception("Invalid Hash.");

        $outfilepath = self::createTempFile();
-        $cmd = OPENSSL_BINARY . " ts -query -digest ".escapeshellarg($hash)." -cert -out ".escapeshellarg($outfilepath);
+        $cmd = OPENSSL_BINARY . " ts -query $digest -digest ".escapeshellarg($hash)." -cert -out ".escapeshellarg($outfilepath);

        $retarray = array();
        exec($cmd." 2>&1", $retarray, $retcode);
@ -106,6 +117,8 @@ class TrustedTimestamps
        $retarray = array();
        exec($cmd." 2>&1", $retarray, $retcode);

+        unlink($responsefile);
+
        if ($retcode !== 0)
            throw new Exception("The reply failed: ".implode(", ", $retarray));

@ -144,7 +157,7 @@ class TrustedTimestamps
     */
    public static function validate ($hash, $base64_response_string, $response_time, $tsa_cert_file)
    {
-        if (strlen($hash) !== 40)
+        if (strlen($hash) !== 40 && strlen($hash) !== 64 && strlen($hash) !== 128)
            throw new Exception("Invalid Hash");

        $binary_response_string = base64_decode($base64_response_string);
@ -160,11 +173,22 @@ class TrustedTimestamps

        $responsefile = self::createTempFile($binary_response_string);

-        $cmd = OPENSSL_BINARY . " ts -verify -digest ".escapeshellarg($hash)." -in ".escapeshellarg($responsefile)." -CAfile ".escapeshellarg($tsa_cert_file);
+        /*
+         * extract chain from response
+         * openssl ts -verify does not include them for verification despite of the man page stating otherwise
+         */
+        $untrustedfile = self::createTempFile();
+        $cmd = OPENSSL_BINARY . " ts -reply -in ".escapeshellarg($responsefile)." -token_out | " . OPENSSL_BINARY . " pkcs7 -inform DER -print_certs -out ".escapeshellarg($untrustedfile);
+        shell_exec($cmd);
+
+        $cmd = OPENSSL_BINARY . " ts -verify -digest ".escapeshellarg($hash)." -in ".escapeshellarg($responsefile)." -CAfile ".escapeshellarg($tsa_cert_file)." -untrusted ".escapeshellarg($untrustedfile);

        $retarray = array();
        exec($cmd." 2>&1", $retarray, $retcode);

+        unlink($untrustedfile);
+        unlink($responsefile);
+
        /*
         * just 2 "normal" cases:
         *  1) Everything okay -> retcode 0 + retarray[0] == "Verification: OK"
--- a/webui/system/helper/mime.php
+++ b/webui/system/helper/mime.php
@ -144,24 +144,13 @@ class Piler_Mime_Decode {
   public static function removeJournal(&$message, $EOL = "\n") {
      $has_journal = 0;

-      $s = self::remove_LF($message);
-      if(strpos($s, $EOL . $EOL)) {
-         list($headers, $body) = explode($EOL . $EOL, $s, 2);
-         if(strstr($headers, "\nX-MS-Journal-Report:")) {
-            return $has_journal;
-         }
+      self::splitMessageRaw($message, $headers, $journal, $body);
+
+      if($journal) {
+         $has_journal = 1;
      }

-      $p = strstr($message, "\nX-MS-Journal-Report:");
-      if($p) {
-         $q = stristr($p, "message/rfc822");
-         if($q) {
-            $has_journal = 1;
-            $i = strlen("message/rfc822");
-            while(ctype_space($q[$i])) { $i++; }
-            if($i > 0) { $message = substr($q, $i); }
-         }
-      }
+      $message = $headers . $EOL . $EOL . $body;

      return $has_journal;
   }
@ -205,6 +194,8 @@ class Piler_Mime_Decode {
      for($i=0; $i<count(self::HEADER_FIELDS); $i++) {
         if(!isset($headers[self::HEADER_FIELDS[$i]])) { $headers[self::HEADER_FIELDS[$i]] = ''; }

+         $headers[self::HEADER_FIELDS[$i]] = preg_replace("/gb2312/i", "GBK", $headers[self::HEADER_FIELDS[$i]]);
+
         $headers[self::HEADER_FIELDS[$i]] = iconv_mime_decode($headers[self::HEADER_FIELDS[$i]], ICONV_MIME_DECODE_CONTINUE_ON_ERROR);
      }

@ -264,7 +255,8 @@ class Piler_Mime_Decode {

      }

-      while(list($k, $v) = each($result)) {
+      foreach($result as $k => $v) {
+
         if(strchr($v, "\n")) {
            $result[$k] = explode("\n", $v);
         }
@ -315,16 +307,19 @@ class Piler_Mime_Decode {
   public static function fixMimeBodyPart($headers = array(), $body = '') {

      if(isset($headers['content-transfer-encoding'])) {
-         if($headers['content-transfer-encoding'] == 'quoted-printable') {
+         if(strtolower($headers['content-transfer-encoding']) == 'quoted-printable') {
            $body = quoted_printable_decode($body);
         }

-         if($headers['content-transfer-encoding'] == 'base64') {
+         if(strtolower($headers['content-transfer-encoding']) == 'base64') {
            $body = base64_decode($body);
         }
      }

      if(isset($headers['content-type']['charset'])) {
+         if(strtolower($headers['content-type']['charset']) == 'gb2312') {
+            $headers['content-type']['charset'] = 'GBK';
+         }
         $body = iconv($headers['content-type']['charset'], 'utf-8' . '//IGNORE', $body);
      }

--- a/webui/system/language.php
+++ b/webui/system/language.php
@ -17,7 +17,7 @@ class Language {
      else {
         $pref_langs = $this->get_preferred_languages();

-         while(list($k, $v) = each($pref_langs)) {
+         foreach($pref_langs as $k => $v) {
            if(in_array($v, $langs)) {
               $lang = $v;
               define('LANG', $lang);
@ -53,7 +53,8 @@ class Language {

         $l = explode(";", $_SERVER['HTTP_ACCEPT_LANGUAGE']);

-         while(list($k, $v) = each($l)) {
+         foreach($l as $k => $v) {
+
            $a = explode(",", $v);

            if(isset($a[0]) && substr($a[0], 0, 2) != 'q=') {
--- a/webui/system/misc.php
+++ b/webui/system/misc.php
@ -147,16 +147,6 @@ function checkemail($email, $domains) {
 function validemail($email = '') {
   if($email == '') { return 0; }

-   // sphinxsearch supports tokens up to 41 characters long
-   // If there's a longer token in the query, then sphinx
-   // reports a query error even if the query is itself correct
-   // So the workaround is to get rid of these email addresses
-   if(strlen($email) > MAX_EMAIL_LEN) {
-      $msg = sprintf("discarding email %s: longer than %d", $email, MAX_EMAIL_LEN);
-      syslog(LOG_INFO, $msg);
-      return 0;
-   }
-
   if(preg_match("/@local$/", $email)) { return 1; }

   if(preg_match('/^[_a-zA-Z0-9-]+(\.[_a-zA-Z0-9-]+)*@[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)*(\.[a-zA-Z]{2,10})$/', $email)) {
@ -196,7 +186,7 @@ function first_n_characters($what, $n){
   $len = 0;

   $a = explode(" ", $what);
-   while(list($k, $v) = each($a)){
+   foreach($a as $k => $v) {
      $x .= "$v "; $len += strlen($v) + 1;
      if($len >= $n){ return $x . "..."; }
   }
@ -279,7 +269,7 @@ function my_qp_encode($s){
      $res = "";

      $a = explode("\n", $s);
-      while(list($k, $v) = each($a)){
+      foreach($a as $k => $v) {
         $part = "";

         for($i=0; $i<strlen($v); $i++){
@ -417,7 +407,7 @@ function parse_string_to_array($s = '', $arr = array()) {

   parse_str($s, $a);

-   while(list($k, $v) = each($a)) {
+   foreach($a as $k => $v) {
      if(!isset($arr[$k]) || $arr[$k] == '') $arr[$k] = $v;
   }

@ -453,7 +443,7 @@ function convert_date_string_to_ymd_by_template($date_string, $date_template) {
      return [$Y, $m, $d];
   }

-   while(list($k, $v) = each($template_array)) {
+   foreach($template_array as $k => $v) {
      $$v = $date_array[$k];
   }

@ -613,7 +603,7 @@ function get_ldap_attribute_names($ldap_type = '') {


 function htmlentities_on_array($arr = []) {
-   while(list($k, $v) = each($arr)) {
+   foreach($arr as $k => $v) {
      if(is_array($v)) {
         $arr[$k] = htmlentities_on_array($v);
      } else {
--- a/unit_tests/php/EmailTest.php
+++ b/unit_tests/php/EmailTest.php
@ -3,6 +3,7 @@
 use PHPUnit\Framework\TestCase;

 define('DIR_BASE', $_ENV['DIR_BASE']);
+define('MAX_EMAIL_LEN', 41);

 include_once(DIR_BASE . "system/model.php");
 include_once(DIR_BASE . "model/search/search.php");
--- a/unit_tests/php/FormatTest.php
+++ b/unit_tests/php/FormatTest.php
@ -2,9 +2,9 @@

 use PHPUnit\Framework\TestCase;

-include_once("webui/system/model.php");
-include_once("webui/system/misc.php");
-include_once("webui/model/health/health.php");
+include_once("system/model.php");
+include_once("system/misc.php");
+include_once("model/health/health.php");

 final class FormatTest extends TestCase
 {
--- a/unit_tests/php/ParseMessageTest.php
+++ b/unit_tests/php/ParseMessageTest.php
--- a/unit_tests/php/SplitMessageTest.php
+++ b/unit_tests/php/SplitMessageTest.php
--- a/webui/view/theme/default/templates/health/worker.tpl
+++ b/webui/view/theme/default/templates/health/worker.tpl
@ -123,7 +123,7 @@
 				<tr>
 					<th colspan="2"><?php print $text_message_disposition; ?></th>
 				</tr>
-			   <?php while(list($k, $v) = each($health['counters'])) {
+			   <?php foreach($health['counters'] as $k => $v) {
 						if(!is_numeric($k)) { ?>
 						   <tr>
 							  <td><?php $a = preg_replace("/^_piler\:/", "", $k); if(isset($$a)) { print $$a; } else { print $k; } ?></td>
--- a/webui/view/theme/default/templates/ldap/list.tpl
+++ b/webui/view/theme/default/templates/ldap/list.tpl
@ -36,7 +36,7 @@
       <label class="control-label" for="ldap_type"><?php print $text_ldap_type; ?>:</label>
       <div class="controls">
          <select name="ldap_type" id="ldap_type" onchange="Piler.fix_ldap_display();" class="span4">
-       <?php while(list($k, $v) = each($ldap_types)) { ?>
+       <?php foreach($ldap_types as $k => $v) { ?>
          <option value="<?php print $v; ?>"<?php if(isset($a['ldap_type']) && $a['ldap_type'] == $v) { ?> selected="selected"<?php } ?>><?php print $v; ?></option>
       <?php } ?>
          </select>
--- a/webui/view/theme/default/templates/policy/folder.tpl
+++ b/webui/view/theme/default/templates/policy/folder.tpl
@ -91,7 +91,7 @@
        <div class="controls">
            <select class="ruleselect" name="folder_id">
               <option value="0">-</option>
-            <?php while(list($k,$v) = each($folders)) { ?>
+            <?php foreach($folders as $k => $v) { ?>
               <option value="<?php print $k; ?>"><?php print $v; ?></option>
            <?php } ?>
            </select>