diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 438e8a93..55289f96 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -1,6 +1,3 @@ -- Introduced the archive_address feature, see etc/example.conf for the details -- Introduced the raw: search label - 1.3.12: ------- @@ -14,6 +11,17 @@ - TLSv1.2 (default) - TLSv1.3 +- Introduced the archive_address feature, see etc/example.conf for the details +- Introduced the raw: search label +- Added Italian translation. Credits: Stefano Gatto +- timestamp signing sorts by 'id' column +- timestamp hash value defaults to sha256 +- Minor fixes +- Added support for php 8.1 +- Fixed handling long email addresses + +Be sure to apply util/db-upgrade.sql + 1.3.11: ------- diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml index 468c3398..80914fe2 100644 --- a/bitbucket-pipelines.yml +++ b/bitbucket-pipelines.yml @@ -18,5 +18,5 @@ pipelines: - mysql -u piler -ppiler123 piler1 < /usr/share/piler/db-mysql.sql - cd unit_tests - ./run.sh - - cd .. + - cd ../webui - phpunit diff --git a/config.php.in b/config.php.in index 3ce7471e..a9981beb 100644 --- a/config.php.in +++ b/config.php.in @@ -88,6 +88,7 @@ $config['LDAP_MAIL_ATTR'] = 'mail'; $config['LDAP_AUDITOR_MEMBER_DN'] = ''; $config['LDAP_ADMIN_MEMBER_DN'] = ''; $config['LDAP_BASE_DN'] = ''; +$config['LDAP_USE_START_TLS'] = 0; // AD specific settings // @@ -324,10 +325,12 @@ $langs = array( 'es', 'fr', 'hu', + 'it', 'pl', 'pt', 'ru', - 'tr' + 'tr', + 'tw' ); diff --git a/contrib/installer/focal.sh b/contrib/installer/focal.sh new file mode 100644 index 00000000..ea14c8d7 --- /dev/null +++ b/contrib/installer/focal.sh @@ -0,0 +1,176 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset +set -x + +PILER_HOSTNAME="${PILER_HOSTNAME:-archive.yourdomain.com}" +MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD:-abcde123}" +MYSQL_PILER_PASSWORD="${MYSQL_PILER_PASSWORD:-piler123}" +SERVER_ID="${SERVER_ID:-0}" +USE_SMTP_GATEWAY="${USE_SMTP_GATEWAY:-0}" +SPHINX_WORKER_LISTEN_ADDRESS="${SPHINX_WORKER_LISTEN_ADDRESS:-}" +PHP_FPM_SOCKET="/var/run/php/php7.4-fpm.sock" + +MYSQL_HOSTNAME="localhost" +MYSQL_DATABASE="piler" +MYSQL_USERNAME="piler" + +SPHINX_TARGZ="sphinx-3.3.1-bin.tar.gz" +DOWNLOAD_URL="https://download.mailpiler.com" +PILER_DEB="piler_1.3.12-focal-a1b71bdd_amd64.deb" +PILER_USER="piler" +CONFIG_SITE_PHP="/etc/piler/config-site.php" + +export DEBIAN_FRONTEND=noninteractive + +install_prerequisites() { + apt-get update + apt-get -y --no-install-recommends install \ + wget rsyslog openssl sysstat php7.4-cli php7.4-cgi php7.4-mysql php7.4-fpm php7.4-zip php7.4-ldap \ + php7.4-gd php7.4-curl php7.4-xml ca-certificates zip catdoc unrtf poppler-utils nginx tnef libzip5 \ + libtre5 libwrap0 cron libmariadb-dev python3 python3-mysqldb libmariadb-dev mariadb-client-core-10.3 \ + mariadb-server-10.3 + + wget -q -O "/tmp/${SPHINX_TARGZ}" "${DOWNLOAD_URL}/generic-local/${SPHINX_TARGZ}" + tar -C / -zxvf "/tmp/${SPHINX_TARGZ}" +} + + +fix_mysql_settings() { + cat > /etc/mysql/mariadb.conf.d/99-piler.cnf << PILER_CNF +[mysqld] + +innodb_buffer_pool_size=512M +innodb_flush_log_at_trx_commit=1 +innodb_log_buffer_size=64M +innodb_log_file_size=64M +innodb_read_io_threads=4 +innodb_write_io_threads=4 +innodb_log_files_in_group=2 + +innodb_file_per_table +PILER_CNF +} + +start_mysql() { + fix_mysql_settings + + service mysql restart + mysqladmin -u root password "${MYSQL_ROOT_PASSWORD}" +} + + +install_piler() { + wget "${DOWNLOAD_URL}/piler/${PILER_DEB}" + dpkg -i "$PILER_DEB" + rm -f "$PILER_DEB" + + crontab -u "$PILER_USER" /usr/share/piler/piler.cron + + rm -f "$PILER_DEB" /etc/nginx/sites-enabled/default + ln -sf /etc/piler/piler-nginx.conf /etc/nginx/sites-enabled/piler.conf + + touch /var/piler/.bash_history + chown "${PILER_USER}:${PILER_USER}" /var/piler/.bash_history +} + + +create_my_cnf() { + local user=$1 + local password=$2 + local my_cnf=$3 + + printf "[client]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" > "$my_cnf" + printf "\\n\\n[mysqldump]\\n\\nhost = %s\\nuser = %s\\npassword = %s\\n" "$MYSQL_HOSTNAME" "$user" "$password" >> "$my_cnf" + chown $PILER_USER:$PILER_USER "$my_cnf" + chmod 600 "$my_cnf" +} + + +fix_config_site_php() { + sed -i -e "s%HOSTNAME%${PILER_HOSTNAME}%g" -e "s%MYSQL_PASSWORD%${MYSQL_PILER_PASSWORD}%g" "$CONFIG_SITE_PHP" + + { + echo "\$config['SERVER_ID'] = $SERVER_ID;" + echo "\$config['USE_SMTP_GATEWAY'] = $USE_SMTP_GATEWAY;" + echo "\$config['SPHINX_VERSION'] = 331;" + } >> "$CONFIG_SITE_PHP" + + if [[ "$SPHINX_WORKER_LISTEN_ADDRESS" ]]; then + echo "\$config['SPHINX_WORKER_LISTEN_ADDRESS'] = '$SPHINX_WORKER_LISTEN_ADDRESS';" >> "$CONFIG_SITE_PHP" + fi + + echo "\$config['ARCHIVE_HOST'] = '$PILER_HOSTNAME';" >> "$CONFIG_SITE_PHP" +} + + +init_db() { + sed -e "s%MYSQL_HOSTNAME%$MYSQL_HOSTNAME%g" -e "s%MYSQL_DATABASE%$MYSQL_DATABASE%g" -e "s%MYSQL_USERNAME%$MYSQL_USERNAME%g" -e "s%MYSQL_PASSWORD%$MYSQL_PILER_PASSWORD%g" \ + /usr/share/piler/db-mysql-root.sql.in | mysql --defaults-file=/etc/piler/.my.cnf-root -h $MYSQL_HOSTNAME + mysql --defaults-file=/etc/piler/.my.cnf -h $MYSQL_HOSTNAME $MYSQL_DATABASE < /usr/share/piler/db-mysql.sql +} + + +add_systemd_services() { + pushd /etc/systemd/system + + ln -sf /usr/libexec/piler/piler.service . + ln -sf /usr/libexec/piler/piler-smtp.service . + ln -sf /usr/libexec/piler/pilersearch.service . + + popd + + systemctl daemon-reload + + systemctl enable piler + systemctl enable piler-smtp + systemctl enable pilersearch +} + + +fix_configs() { + if [[ ! -f /etc/piler/piler-nginx.conf ]]; then + sed -e "s%PILER_HOST%$PILER_HOSTNAME%g" -e "s%PHP_FPM_SOCKET%$PHP_FPM_SOCKET%g" /etc/piler/piler-nginx.conf.dist > /etc/piler/piler-nginx.conf + nginx -t + nginx -s reload + fi + + if [[ ! -f /etc/piler/piler.conf ]]; then + sed -e "s/verystrongpassword/$MYSQL_PILER_PASSWORD/g" -e "s/piler.yourdomain.com/$PILER_HOSTNAME/g" /etc/piler/piler.conf.dist > /etc/piler/piler.conf + chmod 600 /etc/piler/piler.conf + chown $PILER_USER:$PILER_USER /etc/piler/piler.conf + fi + + sed -i -e "s/MYSQL_HOSTNAME/${MYSQL_HOSTNAME}/g" \ + -e "s/MYSQL_DATABASE/${MYSQL_DATABASE}/g" \ + -e "s/MYSQL_USERNAME/${MYSQL_USERNAME}/g" \ + -e "s/MYSQL_PASSWORD/${MYSQL_PILER_PASSWORD}/g" \ + /etc/piler/sphinx.conf +} + + +install_prerequisites + +start_mysql + +install_piler + +create_my_cnf "root" "${MYSQL_ROOT_PASSWORD}" /etc/piler/.my.cnf-root +create_my_cnf "piler" "${MYSQL_PILER_PASSWORD}" /etc/piler/.my.cnf + +fix_config_site_php + +add_systemd_services + +fix_configs +init_db + +su -c "indexer --all -c /etc/piler/sphinx.conf" $PILER_USER + +[[ ! -d /var/run/piler ]] || mkdir -p /var/run/piler + +systemctl start pilersearch +systemctl start piler +systemctl start piler-smtp diff --git a/etc/cron.jobs.in b/etc/cron.jobs.in index 80d57523..5741c047 100644 --- a/etc/cron.jobs.in +++ b/etc/cron.jobs.in @@ -6,9 +6,9 @@ */15 * * * * /usr/bin/indexer --config SYSCONFDIR/piler/sphinx.conf --quiet note1 --rotate */5 * * * * /usr/bin/find LOCALSTATEDIR/piler/www/tmp -type f -name i.\* -exec rm -f {} \; */5 * * * * /usr/bin/find LOCALSTATEDIR/piler/error -type f|wc -l > LOCALSTATEDIR/piler/stat/error -2 0 * * * LIBEXECDIR/piler/pilerpurge.py -### optional: populate accouting data +### optional ###30 6 * * * /usr/bin/php LIBEXECDIR/piler/generate_stats.php --webui LOCALSTATEDIR/piler/www >/dev/null +###*/5 * * * * LIBEXECDIR/piler/import.sh ### PILEREND diff --git a/src/archive.c b/src/archive.c index 97e5a376..60ca6320 100644 --- a/src/archive.c +++ b/src/archive.c @@ -277,7 +277,7 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct c return 1; } - snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile); + snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.m", cfg->queuedir, *(sdata->ttmpfile+24), *(sdata->ttmpfile+25), *(sdata->ttmpfile+8), *(sdata->ttmpfile+9), *(sdata->ttmpfile+10), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile); #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT if(stat(filename, &st)){ snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.m", cfg->queuedir, cfg->server_id, *(sdata->ttmpfile+RND_STR_LEN-6), *(sdata->ttmpfile+RND_STR_LEN-5), *(sdata->ttmpfile+RND_STR_LEN-4), *(sdata->ttmpfile+RND_STR_LEN-3), *(sdata->ttmpfile+RND_STR_LEN-2), *(sdata->ttmpfile+RND_STR_LEN-1), sdata->ttmpfile); @@ -304,7 +304,7 @@ int retrieve_email_from_archive(struct session_data *sdata, FILE *dest, struct c buffer = p + strlen(pointer); if(strlen(ptr_arr[i].piler_id) == RND_STR_LEN){ - snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, cfg->server_id, ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id); + snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, ptr_arr[i].piler_id[24], ptr_arr[i].piler_id[25], ptr_arr[i].piler_id[8], ptr_arr[i].piler_id[9], ptr_arr[i].piler_id[10], ptr_arr[i].piler_id[RND_STR_LEN-4], ptr_arr[i].piler_id[RND_STR_LEN-3], ptr_arr[i].piler_id[RND_STR_LEN-2], ptr_arr[i].piler_id[RND_STR_LEN-1], ptr_arr[i].piler_id, ptr_arr[i].attachment_id); #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT if(stat(filename, &st)){ diff --git a/src/cfg.c b/src/cfg.c index 563428e0..1b175096 100644 --- a/src/cfg.c +++ b/src/cfg.c @@ -152,7 +152,9 @@ int get_tls_protocol_number(char *protocol){ { "TLSv1", TLS1_VERSION }, { "TLSv1.1", TLS1_1_VERSION }, { "TLSv1.2", TLS1_2_VERSION }, + #ifdef TLS1_3_VERSION { "TLSv1.3", TLS1_3_VERSION }, + #endif }; for(unsigned int i=0; i #endif +#include #include #include #include @@ -171,6 +172,7 @@ struct parser_state { int qp; int htmltag; int style; + int meta_content_type; int skip_html; int has_to_dump; int has_to_dump_whole_body; @@ -184,11 +186,13 @@ struct parser_state { int saved_size; unsigned int writebufpos; unsigned int abufpos; + unsigned int received_header; char attachedfile[RND_STR_LEN+SMALLBUFSIZE]; char message_id[SMALLBUFSIZE]; char message_id_hash[2*DIGEST_LENGTH+1]; char miscbuf[MAX_TOKEN_LEN]; char qpbuf[MAX_TOKEN_LEN]; + char receivedbuf[SMALLBUFSIZE]; unsigned long n_token; unsigned long n_subject_token; unsigned long n_body_token; @@ -218,7 +222,7 @@ struct parser_state { unsigned int todomainlen; unsigned int found_security_header; - int journaltolen; + long unsigned int journaltolen; int retention; }; @@ -315,6 +319,7 @@ struct import { int port; int seq; int table_id; + int delay; char *server; char *username; char *password; @@ -399,6 +404,7 @@ struct smtp_session { char rcptto[MAX_RCPT_TO][SMALLBUFSIZE]; char buf[MAXBUFSIZE]; char remote_host[INET6_ADDRSTRLEN+1]; + char nullbyte; time_t lasttime; int protocol_state; int slot; diff --git a/src/digest.c b/src/digest.c index 20134730..50d98cd5 100644 --- a/src/digest.c +++ b/src/digest.c @@ -132,3 +132,27 @@ void digest_string(char *s, char *digest){ snprintf(digest + i*2, 2*DIGEST_LENGTH, "%02x", md[i]); } + + +void md5_string(char *s, char *digest){ + int i; + unsigned char md[MD5_DIGEST_LENGTH]; + MD5_CTX context; + + memset(digest, 0, 2*MD5_DIGEST_LENGTH+2); + + MD5_Init(&context); + + MD5_Update(&context, s, strlen(s)); + + MD5_Final(md, &context); + + for(i=0;iquiet == 0){printf("found %d messages\n", messages); } data->import->total_messages += messages; @@ -356,7 +356,7 @@ int process_imap_folder(char *folder, struct session_data *sdata, struct data *d } - printf("\n"); + if(data->quiet == 0){printf("\n"); } return OK; } @@ -375,7 +375,7 @@ int list_folders(struct data *data){ char attrs[SMALLBUFSIZE], folder[SMALLBUFSIZE]; int len=MAXBUFSIZE+3, pos=0, n, rc=ERR, fldrlen=0, result; - printf("List of IMAP folders:\n"); + if(data->quiet == 0){printf("List of IMAP folders:\n"); } buf = malloc(len); if(!buf) return rc; @@ -478,9 +478,9 @@ int list_folders(struct data *data){ if(!strstr(attrs, "\\Noselect")){ addnode(data->imapfolders, folder); } - else printf("skipping "); + else if(data->quiet == 0){printf("skipping "); } - printf("=> '%s [%s]'\n", folder, attrs); + if(data->quiet == 0){printf("=> '%s [%s]'\n", folder, attrs); } memset(attrs, 0, sizeof(attrs)); } diff --git a/src/import.c b/src/import.c index 601c9b84..7dea5251 100644 --- a/src/import.c +++ b/src/import.c @@ -25,6 +25,14 @@ int import_message(struct session_data *sdata, struct data *data, struct config struct parser_state state; struct counters counters; + if(data->import->delay > 0){ + struct timespec req; + + req.tv_sec = 0; + req.tv_nsec = 1000000 * data->import->delay; + + nanosleep(&req, NULL); + } init_session_data(sdata, cfg); @@ -89,7 +97,10 @@ int import_message(struct session_data *sdata, struct data *data, struct config // When importing emails, we should add the retention value (later) to the original sent value sdata->retained = sdata->sent; + // backup original value of data->folder + int folder = data->folder; rc = process_message(sdata, &state, data, cfg); + data->folder = folder; unlink(state.message_id_hash); } } @@ -145,13 +156,14 @@ int import_message(struct session_data *sdata, struct data *data, struct config int update_import_table(struct session_data *sdata, struct data *data) { - int ret=ERR, status=2; + int ret=ERR, status=2, started=1; struct sql sql; if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_UPDATE_IMPORT_TABLE) == ERR) return ret; p_bind_init(&sql); + sql.sql[sql.pos] = (char *)&(started); sql.type[sql.pos] = TYPE_LONG; sql.pos++; sql.sql[sql.pos] = (char *)&(status); sql.type[sql.pos] = TYPE_LONG; sql.pos++; sql.sql[sql.pos] = (char *)&(data->import->tot_msgs); sql.type[sql.pos] = TYPE_LONG; sql.pos++; sql.sql[sql.pos] = (char *)&(data->import->table_id); sql.type[sql.pos] = TYPE_LONG; sql.pos++; diff --git a/src/import_pilerexport.c b/src/import_pilerexport.c index 8514a593..eb40692e 100644 --- a/src/import_pilerexport.c +++ b/src/import_pilerexport.c @@ -58,7 +58,7 @@ void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *s void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg){ - int n, rc, savedlen=0, puflen; + int n, rc, nullbyte, savedlen=0, puflen; uint64 count=0; char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE]; @@ -70,12 +70,16 @@ void import_from_pilerexport(struct session_data *sdata, struct data *data, stru memset(buf, 0, sizeof(buf)); n = fread(buf, 1, sizeof(buf)-1, stdin); + int remaininglen = n; + if(savedlen > 0){ memset(copybuf, 0, sizeof(copybuf)); memcpy(copybuf, savedbuf, savedlen); memcpy(©buf[savedlen], buf, n); + remaininglen += savedlen; + savedlen = 0; memset(savedbuf, 0, sizeof(savedbuf)); @@ -86,8 +90,9 @@ void import_from_pilerexport(struct session_data *sdata, struct data *data, stru } do { - puflen = read_one_line(p, '\n', puf, sizeof(puf), &rc); + puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf), &rc, &nullbyte); p += puflen; + remaininglen -= puflen; if(puflen > 0){ if(rc == OK){ diff --git a/src/message.c b/src/message.c index 9319e4ac..861253fa 100644 --- a/src/message.c +++ b/src/message.c @@ -213,6 +213,33 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru digest_string(state->reference, &ref[0]); update_metadata_reference(sdata, state, &ref[0], cfg); } + else if(state->reference[0] == 0){ + // during import, the order of messages is often random + // check if this is a message which is already referenced + uint64 count=0; + + digest_string(state->message_id, &ref[0]); + if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_GET_METADATA_REFERENCE) != ERR){ + p_bind_init(&sql); + + sql.sql[sql.pos] = &ref[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++; + + if(p_exec_stmt(sdata, &sql) == OK){ + p_bind_init(&sql); + + sql.sql[sql.pos] = (char *)&count; sql.type[sql.pos] = TYPE_LONGLONG; sql.len[sql.pos] = sizeof(uint64); sql.pos++; + p_store_results(&sql); + p_fetch_results(&sql); + p_free_results(&sql); + } + } + + close_prepared_statement(&sql); + + // no reference yet + if(count <= 0) + ref[0] = 0; + } if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR; @@ -308,7 +335,7 @@ void remove_stripped_attachments(struct parser_state *state){ } -int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){ +int is_duplicated_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){ int fd; char piler_id[SMALLBUFSIZE]; @@ -319,7 +346,7 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru if(sdata->duplicate_id > 0){ remove_stripped_attachments(state); - if(strlen(state->b_journal_to) > 0){ + if(sdata->restored_copy == 0 && strlen(state->b_journal_to) > 0){ if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: trying to add journal rcpt (%s) to id=%llu for message-id: '%s'", sdata->ttmpfile, state->b_journal_to, sdata->duplicate_id, state->message_id); store_recipients(sdata, state->b_journal_to, sdata->duplicate_id, cfg); } @@ -338,7 +365,6 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: touch %s OK (%s)", sdata->ttmpfile, state->message_id_hash, state->message_id); - if(cfg->mmap_dedup_test == 1 && data->dedup != MAP_FAILED && data->child_serial >= 0 && data->child_serial < MAXCHILDREN){ if(strstr(data->dedup, state->message_id_hash)){ @@ -354,6 +380,14 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru memcpy(data->dedup + data->child_serial*DIGEST_LENGTH*2, state->message_id_hash, DIGEST_LENGTH*2); } + return OK; +} + + +int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg){ + + if(is_duplicated_message(sdata, state, data, cfg) == ERR_EXISTS) + return ERR_EXISTS; sdata->retained += query_retain_period(data, state, sdata->tot_len, sdata->spam_message, cfg); diff --git a/src/misc.c b/src/misc.c index 703b21d0..bfddcc56 100644 --- a/src/misc.c +++ b/src/misc.c @@ -652,34 +652,42 @@ void move_email(struct smtp_session *session){ snprintf(buf, sizeof(buf)-1, "%d/%s", session->ttmpfile[0] % session->cfg->number_of_worker_processes, session->ttmpfile); + if(session->nullbyte){ + snprintf(buf, sizeof(buf)-1, "%s/%s", ERROR_DIR, session->ttmpfile); + syslog(LOG_PRIORITY, "ERROR: %s contains an invalid NUL-byte, moving it to %s", session->ttmpfile, ERROR_DIR); + } + if(rename(session->ttmpfile, buf)){ syslog(LOG_PRIORITY, "ERROR: couldn't rename %s to %s (reason: %s)", session->ttmpfile, buf, strerror(errno)); } } -int read_one_line(char *s, int c, char *buf, int buflen, int *rc){ +int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte){ int i=0; *rc = ERR; + *nullbyte = 0; memset(buf, 0, buflen); - if(s == NULL){ - return i; - } - - for(; *s; s++){ + for(int j=0; jnet->ssl); CHK_SSL(n, "internal ssl error"); - printf("Cipher: %s\n", SSL_get_cipher(data->net->ssl)); + //printf("Cipher: %s\n", SSL_get_cipher(data->net->ssl)); server_cert = SSL_get_peer_certificate(data->net->ssl); CHK_NULL(server_cert, "server cert error"); diff --git a/src/misc.h b/src/misc.h index ed13c206..09abc333 100644 --- a/src/misc.h +++ b/src/misc.h @@ -46,7 +46,7 @@ int create_and_bind(char *listen_addr, int listen_port); int can_i_write_directory(char *dir); void move_email(struct smtp_session *session); -int read_one_line(char *s, int c, char *buf, int buflen, int *rc); +int read_one_line(char *s, int slen, int c, char *buf, int buflen, int *rc, int *nullbyte); int init_ssl_to_server(struct data *data); diff --git a/src/parser.c b/src/parser.c index feaae3fa..700b36e0 100644 --- a/src/parser.c +++ b/src/parser.c @@ -65,7 +65,9 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec fclose(f); if(data->import && data->import->extra_recipient){ - add_recipient(data->import->extra_recipient, strlen(data->import->extra_recipient), sdata, &state, data, cfg); + char tmpbuf[SMALLBUFSIZE]; + snprintf(tmpbuf, sizeof(tmpbuf)-1, "%s", data->import->extra_recipient); + add_recipient(tmpbuf, strlen(tmpbuf), sdata, &state, data, cfg); } // If both Sender: and From: headers exist, and they are different, then append @@ -110,6 +112,15 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c if(sdata->internal_recipient == 1 && sdata->external_recipient == 1) sdata->direction = DIRECTION_INTERNAL_AND_OUTGOING; } + char *q = strrchr(state->receivedbuf, ';'); + if(q){ + time_t received_timestamp = parse_date_header(q+1); + if(received_timestamp > 10000000){ + // If the calculated date based on Date: header line differs more than 1 week + // then we'll override it with the data parsed from the first Received: line + if(labs(received_timestamp - sdata->sent) > 604800) sdata->sent = received_timestamp; + } + } for(i=1; i<=state->n_attachments; i++){ char puf[SMALLBUFSIZE]; @@ -123,7 +134,7 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c digest_file(state->attachments[i].internalname, &(state->attachments[i].digest[0])); - if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest); + if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: attachment list: i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s, dumped: %d", sdata->ttmpfile, i, state->attachments[i].filename, state->attachments[i].type, state->attachments[i].size, state->attachments[i].internalname, state->attachments[i].digest, state->attachments[i].dumped); char *p = determine_attachment_type(state->attachments[i].filename, state->attachments[i].type); len = strlen(p); @@ -406,7 +417,10 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata state->message_state = MSG_CC; buf += strlen("Bcc:"); } - else if(strncasecmp(buf, "Message-Id:", 11) == 0) state->message_state = MSG_MESSAGE_ID; + else if(strncasecmp(buf, "Message-Id:", 11) == 0){ + state->message_state = MSG_MESSAGE_ID; + buf += strlen("Message-Id:"); + } else if(strncasecmp(buf, "References:", 11) == 0) state->message_state = MSG_REFERENCES; else if(strncasecmp(buf, "Subject:", strlen("Subject:")) == 0){ state->message_state = MSG_SUBJECT; @@ -430,32 +444,39 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata if(strstr(buf, "=?") && strstr(buf, "?=")) fixupEncodedHeaderLine(buf, MAXBUFSIZE); - sdata->sent = parse_date_header(buf); + sdata->sent = parse_date_header(buf+5); /* allow +2 days drift in the parsed Date: value */ if(sdata->sent - sdata->now > 2*86400) sdata->sent = sdata->now; } - else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf); - else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0) state->message_state = MSG_RECEIVED; + else if(strncasecmp(buf, "Delivery-date:", strlen("Delivery-date:")) == 0 && sdata->delivered == 0) sdata->delivered = parse_date_header(buf+14); + else if(strncasecmp(buf, "Received:", strlen("Received:")) == 0){ + state->message_state = MSG_RECEIVED; + state->received_header++; + } else if(cfg->extra_to_field[0] != '\0' && strncasecmp(buf, cfg->extra_to_field, strlen(cfg->extra_to_field)) == 0){ - state->message_state = MSG_TO; + state->message_state = MSG_RECIPIENT; buf += strlen(cfg->extra_to_field); } if(state->message_state == MSG_MESSAGE_ID && state->message_id[0] == 0){ - p = strchr(buf+11, ' '); - if(p) p = buf + 12; - else p = buf + 11; + while(isspace(*buf)){ + buf++; + } - snprintf(state->message_id, SMALLBUFSIZE-1, "%s", p); + snprintf(state->message_id, SMALLBUFSIZE-1, "%s", buf); } if(state->message_state == MSG_CONTENT_TYPE || state->message_state == MSG_CONTENT_DISPOSITION){ fill_attachment_name_buf(state, buf); } + if(state->received_header == 1 && state->message_state == MSG_RECEIVED && strlen(state->receivedbuf) + len < sizeof(state->receivedbuf)){ + memcpy(&(state->receivedbuf[strlen(state->receivedbuf)]), buf, len); + } + /* we are interested in only From:, To:, Subject:, Received:, Content-*: header lines */ if(state->message_state <= 0) return 0; } @@ -553,6 +574,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata state->message_rfc822 = 1; state->is_header = 1; + state->has_to_dump = 0; + if(sdata->ms_journal == 1){ state->is_1st_header = 1; @@ -618,7 +641,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata state->pushed_pointer = 0; memset(state->type, 0, TINYBUFSIZE); - snprintf(state->charset, TINYBUFSIZE-1, "unknown"); + memset(state->charset, 0, TINYBUFSIZE); memset(state->attachment_name_buf, 0, SMALLBUFSIZE); state->anamepos = 0; @@ -661,7 +684,18 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata if(state->texthtml == 1 && state->message_state == MSG_BODY) markHTML(buf, state); - if(state->texthtml == 1) decodeHTML(buf, state->utf8); + if(state->texthtml == 1){ + size_t buflen = strlen(buf); + decodeHTML(buf, state->utf8); + /* decodeHTML converted some entities to iso-8859-1 */ + if(state->utf8 != 1 && strlen(buf) != buflen){ + /* no charset or us-ascii: switch to iso-8859-1 */ + if (state->charset[0] == 0 || strcasecmp(state->charset, "us-ascii") == 0){ + syslog(LOG_PRIORITY, "%s: assuming iso-8859-1 encoding for HTML (was '%s')", sdata->ttmpfile, state->charset); + snprintf(state->charset, TINYBUFSIZE-1, "ISO8859-1"); + } + } + } /* encode the body if it's not utf-8 encoded */ if(state->message_state == MSG_BODY && state->utf8 != 1){ diff --git a/src/parser.h b/src/parser.h index c9438491..ae649e55 100644 --- a/src/parser.h +++ b/src/parser.h @@ -20,7 +20,7 @@ void fixupEncodedHeaderLine(char *buf, int buflen); void fixupSoftBreakInQuotedPritableLine(char *buf, struct parser_state *state); void fixupBase64EncodedLine(char *buf, struct parser_state *state); void markHTML(char *buf, struct parser_state *state); -void setStateHTMLStyle(char *htmlbuf, int pos, struct parser_state *state); +void setStateHTML(char *htmlbuf, int pos, struct parser_state *state); void translateLine(unsigned char *p, struct parser_state *state); void fix_email_address_for_sphinx(char *s); void split_email_address(char *s); diff --git a/src/parser_utils.c b/src/parser_utils.c index e8c1f2c8..869828f4 100644 --- a/src/parser_utils.c +++ b/src/parser_utils.c @@ -40,6 +40,7 @@ void init_state(struct parser_state *state){ state->htmltag = 0; state->style = 0; + state->meta_content_type = 0; state->skip_html = 0; @@ -49,8 +50,10 @@ void init_state(struct parser_state *state){ memset(state->message_id_hash, 0, 2*DIGEST_LENGTH+1); memset(state->miscbuf, 0, MAX_TOKEN_LEN); memset(state->qpbuf, 0, MAX_TOKEN_LEN); + memset(state->receivedbuf, 0, sizeof(state->receivedbuf)); memset(state->type, 0, TINYBUFSIZE); + memset(state->charset, 0, TINYBUFSIZE); memset(state->attachment_name_buf, 0, SMALLBUFSIZE); state->anamepos = 0; @@ -67,6 +70,7 @@ void init_state(struct parser_state *state){ state->writebufpos = 0; state->abufpos = 0; + state->received_header = 0; inithash(state->boundaries); inithash(state->rcpt); @@ -124,7 +128,6 @@ time_t parse_date_header(char *datestr){ char *p, *q, *r, *tz, s[SMALLBUFSIZE], tzh[4], tzm[3]; struct tm tm; - datestr += 5; p = datestr; for(; *datestr; datestr++){ @@ -550,7 +553,7 @@ void markHTML(char *buf, struct parser_state *state){ if(isspace(*s)){ if(j > 0){ - setStateHTMLStyle(html, pos, state); + setStateHTML(html, pos, state); memset(html, 0, SMALLBUFSIZE); j=0; } pos++; @@ -575,23 +578,51 @@ void markHTML(char *buf, struct parser_state *state){ if(j > 0){ strncat(html, " ", SMALLBUFSIZE-1); - setStateHTMLStyle(html, pos, state); + setStateHTML(html, pos, state); memset(html, 0, SMALLBUFSIZE); j=0; } + state->meta_content_type = 0; } } //printf("append last in line:*%s*, html=+%s+, j=%d\n", puf, html, j); - if(j > 0){ setStateHTMLStyle(html, pos, state); } + if(j > 0){ setStateHTML(html, pos, state); } strcpy(buf, puf); } -void setStateHTMLStyle(char *htmlbuf, int pos, struct parser_state *state){ +void setStateHTML(char *htmlbuf, int pos, struct parser_state *state){ if(pos == 0 && strncmp(htmlbuf, "style ", 6) == 0) state->style = 1; if(pos == 0 && strncmp(htmlbuf, "/style ", 7) == 0) state->style = 0; + + if(pos == 0 && state->charset[0] == 0 && strncmp(htmlbuf, "meta ", 5) == 0) state->meta_content_type = 0x1; + if(state->meta_content_type){ + if((state->meta_content_type & 0x2) == 0 && strstr(htmlbuf, "http-equiv=content-type ")) + state->meta_content_type |= 0x2; + + if((state->meta_content_type & 0x4) == 0 && strstr(htmlbuf, "content=text/html;")) + state->meta_content_type |= 0x4; + + if(state->meta_content_type == 0x7){ + char *p, *q; + + p = strstr(htmlbuf, "charset="); + if(p){ + p += 8; + for(q = p; isalnum(*q) || index("-_", *q); q++) + ; + + if(q > p && q-p+1 < (int) sizeof(state->charset)){ + syslog(LOG_PRIORITY, "Changing HTML charset from '%s' to '%*s' due to meta tag", state->charset, (int)(q-p), p); + strncpy(state->charset, p, q-p); + state->charset[q-p+1] = '\0'; + state->meta_content_type = 0; + } + } + } + } } diff --git a/src/piler.h b/src/piler.h index c33ff60c..e4d7395e 100644 --- a/src/piler.h +++ b/src/piler.h @@ -38,6 +38,7 @@ int do_av_check(char *filename, struct config *cfg); int make_digests(struct session_data *sdata, struct config *cfg); void digest_file(char *filename, char *digest); void digest_string(char *s, char *digest); +void create_md5_from_email_address(char *puf, char *md5buf); void remove_stripped_attachments(struct parser_state *state); int process_message(struct session_data *sdata, struct parser_state *state, struct data *data, struct config *cfg); diff --git a/src/pileraget.c b/src/pileraget.c index 3b780cc5..90f6aa33 100644 --- a/src/pileraget.c +++ b/src/pileraget.c @@ -46,7 +46,7 @@ int main(int argc, char **argv){ return 1; } - snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2])); + snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, argv[1][24], argv[1][25], argv[1][8], argv[1][9], argv[1][10], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2])); #ifdef HAVE_SUPPORT_FOR_COMPAT_STORAGE_LAYOUT if(stat(filename, &st)){ snprintf(filename, sizeof(filename)-1, "%s/%02x/%c%c/%c%c/%c%c/%s.a%d", cfg.queuedir, cfg.server_id, argv[1][RND_STR_LEN-6], argv[1][RND_STR_LEN-5], argv[1][RND_STR_LEN-4], argv[1][RND_STR_LEN-3], argv[1][RND_STR_LEN-2], argv[1][RND_STR_LEN-1], argv[1], atoi(argv[2])); diff --git a/src/pilerimport.c b/src/pilerimport.c index 3be6541f..3af32505 100644 --- a/src/pilerimport.c +++ b/src/pilerimport.c @@ -53,6 +53,7 @@ void usage(){ printf(" -j Move failed to import emails to this folder\n"); printf(" -a Add recipient to the To:/Cc: list\n"); printf(" -T Update import table at id=\n"); + printf(" -Z Delay Z milliseconds in between emails being imported\n"); printf(" -D Dry-run, do not import anything\n"); printf(" -y Read pilerexport data from stdin\n"); printf(" -o Only download emails for POP3/IMAP import\n"); @@ -100,6 +101,7 @@ int main(int argc, char **argv){ import.tot_msgs = 0; import.table_id = 0; import.folder = NULL; + import.delay = 0; data.import = &import; @@ -136,6 +138,7 @@ int main(int argc, char **argv){ {"timeout", required_argument, 0, 't' }, {"start-position",required_argument, 0, 's' }, {"table-id", required_argument, 0, 'T' }, + {"delay", required_argument, 0, 'Z' }, {"quiet", no_argument, 0, 'q' }, {"recursive", no_argument, 0, 'R' }, {"remove-after-import",no_argument, 0, 'r' }, @@ -150,9 +153,9 @@ int main(int argc, char **argv){ int option_index = 0; - int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:yDRroqh?", long_options, &option_index); + int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:yDRroqh?", long_options, &option_index); #else - int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:yDRroqh?"); + int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:Z:yDRroqh?"); #endif if(c == -1) break; @@ -271,6 +274,15 @@ int main(int argc, char **argv){ data.import->table_id = atoi(optarg); break; + case 'Z' : + if(atoi(optarg) < 1){ + printf("invalid delay value: %s\n", optarg); + return -1; + } + + data.import->delay = atoi(optarg); + break; + case 'y' : read_from_pilerexport = 1; break; @@ -316,6 +328,9 @@ int main(int argc, char **argv){ /* make sure we don't discard messages without a valid Message-Id when importing manually */ cfg.archive_emails_not_having_message_id = 1; + /* The mmap_dedup_test feature is expected to work with the piler daemon only */ + cfg.mmap_dedup_test = 0; + if(read_key(&cfg)){ printf("%s\n", ERR_READING_KEY); return ERR; diff --git a/src/pop3.c b/src/pop3.c index c8ab4194..94137fd3 100644 --- a/src/pop3.c +++ b/src/pop3.c @@ -23,16 +23,6 @@ #include -int is_last_complete_pop3_packet(char *s, int len){ - - if(*(s+len-5) == '\r' && *(s+len-4) == '\n' && *(s+len-3) == '.' && *(s+len-2) == '\r' && *(s+len-1) == '\n'){ - return 1; - } - - return 0; -} - - int connect_to_pop3_server(struct data *data){ char buf[MAXBUFSIZE]; @@ -86,66 +76,86 @@ void get_number_of_total_messages(struct data *data){ int pop3_download_email(struct data *data, int i){ - int n, fd, pos=0, lastpos=0, nreads=0; - char *p, buf[MAXBUFSIZE]; - char aggrbuf[3*MAXBUFSIZE]; + char *p, buf[MAXBUFSIZE], savedbuf[MAXBUFSIZE], copybuf[2*MAXBUFSIZE]; data->import->processed_messages++; snprintf(data->import->filename, SMALLBUFSIZE-1, "pop3-tmp-%d-%d.txt", getpid(), i); unlink(data->import->filename); - fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR); + int fd = open(data->import->filename, O_CREAT|O_EXCL|O_RDWR|O_TRUNC, S_IRUSR|S_IWUSR); if(fd == -1){ printf("cannot open: %s\n", data->import->filename); return ERR; } + memset(savedbuf, 0, sizeof(savedbuf)); + snprintf(buf, sizeof(buf)-1, "RETR %d\r\n", i); write1(data->net, buf, strlen(buf)); - memset(aggrbuf, 0, sizeof(aggrbuf)); + int nlines = 0; + int endofmessage = 0; + int savedlen = 0; + int n = 0; while((n = recvtimeoutssl(data->net, buf, sizeof(buf))) > 0){ - nreads++; + int remaininglen = n; - if(nreads == 1){ + if(savedlen){ + memset(copybuf, 0, sizeof(copybuf)); + memcpy(copybuf, savedbuf, savedlen); + memcpy(©buf[savedlen], buf, n); - if(strncmp(buf, "+OK", 3) == 0){ - p = strchr(&buf[3], '\n'); - if(p){ - *p = '\0'; - pos = strlen(buf)+1; - *p = '\n'; + remaininglen += savedlen; + + savedlen = 0; + memset(savedbuf, 0, sizeof(savedbuf)); + + p = ©buf[0]; + } else { + p = &buf[0]; + } + + int puflen=0; + int rc=OK; + int nullbyte=0; + + do { + char puf[MAXBUFSIZE]; + + puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf)-1, &rc, &nullbyte); + remaininglen -= puflen; + nlines++; + + if(nlines == 1){ + if(strncmp(puf, "+OK", 3)){ + printf("error: %s", puf); + return ERR; + } + } else { + if(puf[puflen-1] == '\n'){ + if(puflen == 3 && puf[0] == '.' && puf[1] == '\r' && puf[2] == '\n'){ + endofmessage = 1; + break; + } + + int dotstuff = 0; + if(puf[0] == '.' && puf[1] != '\r' && puf[1] != '\n') dotstuff = 1; + + if(write(fd, &puf[dotstuff], puflen-dotstuff) == -1) printf("ERROR: writing to fd\n"); + + } else if(puflen > 0) { + savedlen = puflen; + snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf); } } - else { printf("error: %s", buf); return ERR; } - } + p += puflen; - if((uint)(lastpos + 1 + n) < sizeof(aggrbuf)){ + } while(puflen > 0); - if(nreads == 1){ - memcpy(aggrbuf+lastpos, buf+pos, n-pos); - lastpos += n-pos; - } - else { - memcpy(aggrbuf+lastpos, buf, n); - lastpos += n; - } - } - else { - if(write(fd, aggrbuf, sizeof(buf)) == -1) printf("ERROR: writing to fd\n"); - - memmove(aggrbuf, aggrbuf+sizeof(buf), lastpos-sizeof(buf)); - lastpos -= sizeof(buf); - - memcpy(aggrbuf+lastpos, buf, n); - lastpos += n; - } - - if(is_last_complete_pop3_packet(aggrbuf, lastpos) == 1){ - if(write(fd, aggrbuf, lastpos-3) == -1) printf("ERROR: writing to fd\n"); + if(endofmessage){ break; } } diff --git a/src/reindex.c b/src/reindex.c index c3e14a96..009f205a 100644 --- a/src/reindex.c +++ b/src/reindex.c @@ -75,12 +75,10 @@ uint64 get_max_meta_id(struct session_data *sdata){ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *data, uint64 from_id, uint64 to_id, struct config *cfg){ char s[SMALLBUFSIZE]; - uint64 stored_id=0, reindexed=0, delta; + uint64 stored_id=0, reindexed=0; struct parser_state state; struct sql sql; - delta = to_id - from_id; - if(cfg->enable_folders == 1) snprintf(s, sizeof(s)-1, "SELECT m.`id`, `piler_id`, `arrived`, `sent`, f.folder_id FROM %s m, %s f WHERE m.id=f.id AND (m.id BETWEEN %llu AND %llu) AND `deleted`=0", SQL_METADATA_TABLE, SQL_FOLDER_MESSAGE_TABLE, from_id, to_id); else @@ -123,6 +121,12 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename); + struct stat st; + sdata->tot_len = stat(filename, &st) == 0 ? st.st_size : 0; + + sdata->internal_sender = sdata->internal_recipient = sdata->external_recipient = sdata->direction = 0; + memset(sdata->attachments, 0, SMALLBUFSIZE); + state = parse_message(sdata, 1, data, cfg); post_parse(sdata, &state, cfg); @@ -137,6 +141,8 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da unlink(filename); if(progressbar){ + uint64 delta = to_id - from_id + 1; + printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta)); fflush(stdout); } diff --git a/src/session.c b/src/session.c index 777679a9..b8b9ce76 100644 --- a/src/session.c +++ b/src/session.c @@ -99,6 +99,7 @@ void init_smtp_session(struct smtp_session *session, int slot, int sd, char *cli session->net.ctx = NULL; session->net.ssl = NULL; + session->nullbyte = 0; session->last_data_char = 0; session->fd = -1; @@ -160,11 +161,13 @@ void tear_down_session(struct smtp_session **sessions, int slot, int *num_connec void handle_data(struct smtp_session *session, char *readbuf, int readlen, struct config *cfg){ - int puflen, rc; + int puflen, rc, nullbyte; char *p, copybuf[BIGBUFSIZE+MAXBUFSIZE], puf[MAXBUFSIZE]; // if there's something in the saved buffer, then let's merge them + int remaininglen = readlen + session->buflen; + if(session->buflen > 0){ memset(copybuf, 0, sizeof(copybuf)); @@ -183,30 +186,42 @@ void handle_data(struct smtp_session *session, char *readbuf, int readlen, struc do { - puflen = read_one_line(p, '\n', puf, sizeof(puf)-1, &rc); + puflen = read_one_line(p, remaininglen, '\n', puf, sizeof(puf)-1, &rc, &nullbyte); p += puflen; + remaininglen -= puflen; + + if(nullbyte){ + session->nullbyte = 1; + } + + // complete line: rc == OK and puflen > 0 + // incomplete line with something in the buffer: rc == ERR and puflen > 0 if(puflen > 0){ // Update lasttime if we have a line to process time(&(session->lasttime)); - // pass the puffer to process_data() only if there was an '\n' - // on the line or the puffer does not start with a period - if(session->protocol_state == SMTP_STATE_DATA && (rc == OK || puf[0] != '.')){ - sig_block(SIGALRM); - process_data(session, puf, puflen); - sig_unblock(SIGALRM); - } - else if(session->protocol_state == SMTP_STATE_BDAT){ - process_bdat(session, puf, puflen, cfg); - } - else if(rc == OK){ - process_smtp_command(session, puf, cfg); - } - else { - snprintf(session->buf, MAXBUFSIZE-1, "%s", puf); + // Save incomplete line to buffer + if(rc == ERR){ + memcpy(session->buf, puf, puflen); session->buflen = puflen; } + + // We have a complete line to process + + if(rc == OK){ + if(session->protocol_state == SMTP_STATE_BDAT){ + process_bdat(session, puf, puflen, cfg); + } + else if(session->protocol_state == SMTP_STATE_DATA){ + sig_block(SIGALRM); + process_data(session, puf, puflen); + sig_unblock(SIGALRM); + } + else { + process_smtp_command(session, puf, cfg); + } + } } } while(puflen > 0); diff --git a/src/smtp.c b/src/smtp.c index 5cd69fe6..7d9ee749 100644 --- a/src/smtp.c +++ b/src/smtp.c @@ -84,14 +84,24 @@ void process_data(struct smtp_session *session, char *buf, int buflen){ // write line to file int written=0, n_writes=0; + // In the DATA phase skip the 1st character if it's a dot (.) + // and there are more characters before the trailing CR-LF + // + // See https://www.ietf.org/rfc/rfc5321.html#section-4.5.2 for more. + + int dotstuff = 0; + if(*buf == '.' && buflen > 1 && *(buf+1) != '\r' && *(buf+1) != '\n') dotstuff = 1; + while(written < buflen) { - int len = write(session->fd, buf+written, buflen-written); + int len = write(session->fd, buf+dotstuff+written, buflen-dotstuff-written); + n_writes++; if(len > 0){ - if(len != buflen) syslog(LOG_PRIORITY, "WARN: partial write: %d/%d bytes (round: %d)", len, buflen, n_writes); - written += len; + if(len != buflen-dotstuff) syslog(LOG_PRIORITY, "WARN: partial write: %d/%d bytes (round: %d)", len, buflen-dotstuff, n_writes); + written += len + dotstuff; session->tot_len += len; + dotstuff = 0; } else syslog(LOG_PRIORITY, "ERROR (line: %d) process_data(): written %d bytes", __LINE__, len); } @@ -171,10 +181,14 @@ int init_ssl(struct smtp_session *session){ return 0; } +#if OPENSSL_VERSION_NUMBER < 0x10100000L + SSL_CTX_set_options(session->net.ctx, SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3|SSL_OP_NO_TLSv1|SSL_OP_NO_TLSv1_1); +#else if(SSL_CTX_set_min_proto_version(session->net.ctx, session->cfg->tls_min_version_number) == 0){ syslog(LOG_PRIORITY, "failed SSL_CTX_set_min_proto_version() to %s/%d", session->cfg->tls_min_version, session->cfg->tls_min_version_number); return 0; } +#endif if(SSL_CTX_set_cipher_list(session->net.ctx, session->cfg->cipher_list) == 0){ syslog(LOG_PRIORITY, "failed to set cipher list: '%s'", session->cfg->cipher_list); diff --git a/src/test.c b/src/test.c index 956910d0..ebada3e0 100644 --- a/src/test.c +++ b/src/test.c @@ -153,6 +153,7 @@ int main(int argc, char **argv){ printf("from: *%s (%s)*\n", state.b_from, state.b_from_domain); printf("sender: *%s (%s)*\n", state.b_sender, state.b_sender_domain); printf("to: *%s (%s)*\n", state.b_to, state.b_to_domain); + printf("journal recipients: *%s*\n", state.b_journal_to); printf("reference: *%s*\n", state.reference); printf("subject: *%s*\n", state.b_subject); printf("body: *%s*\n", state.b_body); @@ -183,7 +184,7 @@ int main(int argc, char **argv){ clearhash(data.mydomains); for(i=1; i<=state.n_attachments; i++){ - printf("i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s\n", i, state.attachments[i].filename, state.attachments[i].type, state.attachments[i].size, state.attachments[i].internalname, state.attachments[i].digest); + printf("i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, dumped: %d, digest: %s\n", i, state.attachments[i].filename, state.attachments[i].type, state.attachments[i].size, state.attachments[i].internalname, state.attachments[i].dumped, state.attachments[i].digest); unlink(state.attachments[i].internalname); } @@ -195,6 +196,8 @@ int main(int argc, char **argv){ printf("spam: %d\n", sdata.spam_message); + printf("1st received line: %s\n", state.receivedbuf); + if(sdata.internal_sender == 0 && sdata.internal_recipient == 0) printf("NOT IN mydomains\n"); printf("\n\n"); diff --git a/src/tokenizer.c b/src/tokenizer.c index f6b32518..7fd16fb3 100644 --- a/src/tokenizer.c +++ b/src/tokenizer.c @@ -50,6 +50,8 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata, continue; } + char md5buf[2*MD5_DIGEST_LENGTH+2]; + if(state->message_state == MSG_FROM && state->is_1st_header == 1 && strlen(state->b_from) < SMALLBUFSIZE-len-1){ strtolower(puf); @@ -69,6 +71,11 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata, if(is_email_address_on_my_domains(puf, data) == 1) sdata->internal_sender = 1; + if(len >= MAX_EMAIL_ADDRESS_SPHINX_LEN && strlen(state->b_from) < SMALLBUFSIZE-len-1){ + create_md5_from_email_address(puf, md5buf); + memcpy(&(state->b_from[strlen(state->b_from)]), md5buf, strlen(md5buf)); + } + if(strlen(state->b_from) < SMALLBUFSIZE-len-1){ split_email_address(puf); memcpy(&(state->b_from[strlen(state->b_from)]), puf, len); @@ -88,6 +95,11 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata, memcpy(&(state->b_sender_domain), q+1, strlen(q+1)-1); } + if(len >= MAX_EMAIL_ADDRESS_SPHINX_LEN && strlen(state->b_sender) < SMALLBUFSIZE-len-1){ + create_md5_from_email_address(puf, md5buf); + memcpy(&(state->b_sender[strlen(state->b_sender)]), md5buf, strlen(md5buf)); + } + if(strlen(state->b_sender) < SMALLBUFSIZE-len-1){ split_email_address(puf); memcpy(&(state->b_sender[strlen(state->b_sender)]), puf, len); @@ -101,12 +113,18 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata, q = strchr(puf, '@'); if(q) fix_plus_sign_in_email_address(puf, &q, &len); - if((state->message_state == MSG_RECIPIENT || state->message_state == MSG_ENVELOPE_TO) && findnode(state->journal_recipient, puf) == NULL){ + if((state->message_state == MSG_RECIPIENT || state->message_state == MSG_ENVELOPE_TO) && findnode(state->journal_recipient, puf) == NULL && state->journaltolen < sizeof(state->b_journal_to)-len-1){ addnode(state->journal_recipient, puf); memcpy(&(state->b_journal_to[state->journaltolen]), puf, len); + state->journaltolen += len; if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: journal rcpt: '%s'", sdata->ttmpfile, puf); } + if(len >= MAX_EMAIL_ADDRESS_SPHINX_LEN){ + create_md5_from_email_address(puf, md5buf); + add_recipient(md5buf, strlen(md5buf), sdata, state, data, cfg); + } + add_recipient(puf, len, sdata, state, data, cfg); } else if(state->message_state == MSG_BODY && len >= (unsigned int)(cfg->min_word_len) && state->bodylen < BIGBUFSIZE-len-1){ diff --git a/tests/cases/05-smtp.inc b/tests/cases/05-smtp.inc index e05b73cc..2e5e7906 100644 --- a/tests/cases/05-smtp.inc +++ b/tests/cases/05-smtp.inc @@ -15,13 +15,14 @@ case1() { "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/spam2" --socket --no-counter "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/journal" --socket --no-counter "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/deduptest" --socket --no-counter + "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu -p 25 -t 20 --dir "$EML_DIR/special" --socket --no-counter "$SMTP_SOURCE_PROG" -s $SMTP_HOST -r archive@cust1.acts.hu extra@addr.ess another@extra.addr -p 25 -t 20 --dir "$EML_DIR/virus" --socket --no-counter - wait_until_emails_are_processed "piler1" 3007 + wait_until_emails_are_processed "piler1" 3018 docker exec "piler1" su piler -c /usr/libexec/piler/indexer.delta.sh 2>/dev/null - count_status_values 3007 2896 111 0 + count_status_values 3018 2907 111 0 test_retrieved_messages_are_the_same "piler1" "piler" diff --git a/tests/setup.inc b/tests/setup.inc index 421b5cc3..e2a325c1 100644 --- a/tests/setup.inc +++ b/tests/setup.inc @@ -13,6 +13,13 @@ setup() { add_data_officer "piler1" } +cleanup_package() { + local pkg="$1" + + log "Removing ${PACKAGE_DIR}/${pkg}" + rm -f "${PACKAGE_DIR}/${pkg}" +} + launch_containers() { log "starting syslog server" @@ -28,12 +35,14 @@ launch_containers() { "$docker_image" wait_for_sleep_cycle_in_container "piler1" + + cleanup_package "$PACKAGE" } create_rules() { local container="$1" - echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"acts.hu\",\"acts.hu\"),(\"gtsce.com\",\"gtsce.com\"),(\"datanet.hu\",\"datanet.hu\"),(\"gtsdatanet.hu\",\"gtsdatanet.hu\"),(\"gts.hu\",\"gts.hu\"),(\"aaa.fu\",\"aaa.fu\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh + echo 'echo "insert into domain (domain, mapped) values(\"fictive.com\",\"fictive.com\"),(\"address.com\",\"address.com\"),(\"acts.hu\",\"acts.hu\"),(\"gtsce.com\",\"gtsce.com\"),(\"datanet.hu\",\"datanet.hu\"),(\"gtsdatanet.hu\",\"gtsdatanet.hu\"),(\"gts.hu\",\"gts.hu\"),(\"aaa.fu\",\"aaa.fu\")"| mysql --defaults-file=/etc/piler/.my.cnf piler' | docker exec -i "$container" sh echo 'echo "insert into archiving_rule (subject) values (\"Android táblagép\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh echo 'echo "insert into archiving_rule (\`from\`) values (\"@gmail.com\")"| mysql --defaults-file=/etc/piler/.my.cnf piler'|docker exec -i "$container" sh diff --git a/unit_tests/check_parser.c b/unit_tests/check_parser.c index 70d53760..6e7958ae 100644 --- a/unit_tests/check_parser.c +++ b/unit_tests/check_parser.c @@ -39,7 +39,7 @@ static void test_parser(struct config *cfg){ {"18-spam-html-encoding.eml", "", "a1 hitelcentrum kft Üveges szilvia a1hitelcentrum@t-online.hu a1hitelcentrum t online hu ", "t-online.hu", "postmaster postmaster@aaa.fu postmaster aaa fu ", "aaa.fu", "postmaster@aaa.fu postmaster aaa fu a1hitelcentrum@t-online.hu a1hitelcentrum t online hu ", "aaa.fu t-online.hu ", "", "TÁJÉKOZTATÁSVargay Péter", 0}, {"19-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-186.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "", "", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1}, {"20-pdf-attachment-bad-mime.eml", "<20100213$2b62e942$9cc2b$sxm@61-187.reverse.ukhost4u.com>", "jennifer - billing department billing@limitedsoftwareworld.com billing limitedsoftwareworld com ", "limitedsoftwareworld.com", "", "", "100000 100000@aaa.fu 100000 aaa fu ", "aaa.fu ", "", "Billing Summary for 100000, Processed on 2010-02-13 17:01:03", 1}, - {"21-register-tricky-urls.eml", "", "the register update-49363-08f0f768@list.theregister.co.uk update 49363 08f0f768 list theregister co uk ", "list.theregister.co.uk", "", "", "hello@mail.aaa.fu hello mail aaa fu ", "mail.aaa.fu ", "", "[sp@m] Reg Headlines Friday July 20", 0}, + {"21-register-tricky-urls.eml", "", "the register update-49363-08f0f768@list.theregister.co.uk 30cbee0b0f411fcf170416fb9f996c6f update 49363 08f0f768 list theregister co uk ", "list.theregister.co.uk", "", "", "hello@mail.aaa.fu hello mail aaa fu ", "mail.aaa.fu ", "", "[sp@m] Reg Headlines Friday July 20", 0}, {"30-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", "RE: hxx-ajajajaja.com_ Aaagágyi és kia ttt_webstat hiba", 0}, {"31-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", "Re: stanhu \"domain not found\"-dal eldobja a @fohu-ra küldött leveleket...", 0}, {"32-subject.eml", "<3660278814815884@pongr-fabd8067e>", "aaapsi.hu info@aaapsi.hu info aaapsi hu ", "aaapsi.hu", "", "", "hello@acts.hu hello acts hu ", "acts.hu ", "", " www.ujsag.hu new virtual host reg. --> Aaaaaaaaa", 0}, diff --git a/util/db-mysql.sql b/util/db-mysql.sql index ffc93e1f..8cffc1a9 100644 --- a/util/db-mysql.sql +++ b/util/db-mysql.sql @@ -9,9 +9,9 @@ create table if not exists `sph_counter` ( create table if not exists `sph_index` ( `id` bigint not null, `from` tinyblob default null, - `to` text(8192) default null, - `fromdomain` char(255) default null, - `todomain` text(512) default null, + `to` blob(8192) default null, + `fromdomain` tinyblob default null, + `todomain` blob(512) default null, `subject` blob(512) default null, `arrived` int unsigned not null, `sent` int unsigned not null, @@ -73,7 +73,7 @@ create index `rcpt_idx3` on `rcpt`(`todomain`); drop view if exists `v_messages`; -create view `v_messages` AS select `metadata`.`id` AS `id`,`metadata`.`piler_id` AS `piler_id`,`metadata`.`from` AS `from`,`metadata`.`fromdomain` AS `fromdomain`,`rcpt`.`to` AS `to`,`rcpt`.`todomain` AS `todomain`,`metadata`.`subject` AS `subject`, `metadata`.`size` AS `size`, `metadata`.`direction` AS `direction`, `metadata`.`sent` AS `sent`, `metadata`.`retained` AS `retained`, `metadata`.`arrived` AS `arrived`, `metadata`.`digest` AS `digest`, `metadata`.`bodydigest` AS `bodydigest`, `metadata`.`deleted` AS `deleted` from (`metadata` join `rcpt`) where (`metadata`.`id` = `rcpt`.`id`); +create view `v_messages` AS select `metadata`.`id` AS `id`,`metadata`.`piler_id` AS `piler_id`,`metadata`.`from` AS `from`,`metadata`.`fromdomain` AS `fromdomain`,`rcpt`.`to` AS `to`,`rcpt`.`todomain` AS `todomain`,`metadata`.`subject` AS `subject`, `metadata`.`size` AS `size`, `metadata`.`direction` AS `direction`, `metadata`.`sent` AS `sent`, `metadata`.`retained` AS `retained`, `metadata`.`arrived` AS `arrived`, `metadata`.`digest` AS `digest`, `metadata`.`bodydigest` AS `bodydigest`, `metadata`.`deleted` AS `deleted`, `metadata`.`attachments` AS `attachments` from (`metadata` join `rcpt`) where (`metadata`.`id` = `rcpt`.`id`); create table if not exists `attachment` ( @@ -463,7 +463,7 @@ create table if not exists `timestamp` ( `id` bigint unsigned not null auto_increment, `start_id` bigint default 0, `stop_id` bigint default 0, - `hash_value` char(40), + `hash_value` varchar(128), `count` int default 0, `response_time` bigint default 0, `response_string` blob not null, diff --git a/util/db-upgrade.sql b/util/db-upgrade.sql new file mode 100644 index 00000000..cd1feef9 --- /dev/null +++ b/util/db-upgrade.sql @@ -0,0 +1,8 @@ +alter table timestamp change column hash_value hash_value varchar(128) default null; + +alter table sph_index change column `to` `to` blob(8192) default null; +alter table sph_index change column fromdomain fromdomain tinyblob default null; +alter table sph_index change column todomain todomain blob(512) default null; + + +alter table metadata change column subject `subject` blob(512) default null; diff --git a/util/download-imap.php b/util/download-imap.php index 392a0e76..fd15d86e 100644 --- a/util/download-imap.php +++ b/util/download-imap.php @@ -103,7 +103,7 @@ function saveMessages($storage, $folder = '', $num = 0) { $messages = $storage->piler_batch_fetch(1, $num); - while(list($k, $v) = each($messages)) { + foreach($messages as $k => $v) { $uuid = $storage->getUniqueId($k); $tmpname = "piler-" . $username . "-" . $folder . "-" . $k . "-" . $uuid . ".eml"; diff --git a/util/imapfetch.py b/util/imapfetch.py index 45852085..a96e37aa 100755 --- a/util/imapfetch.py +++ b/util/imapfetch.py @@ -90,6 +90,7 @@ def main(): default="/etc/piler/piler.conf") parser.add_argument("-s", "--server", type=str, help="imap server") parser.add_argument("-P", "--port", type=int, help="port number", default=143) + parser.add_argument("--no_ssl", help="Do not use ssl/tls", action='store_true') parser.add_argument("-u", "--user", type=str, help="imap user") parser.add_argument("-p", "--password", type=str, help="imap password") parser.add_argument("-x", "--skip-list", type=str, help="IMAP folders to skip", @@ -116,12 +117,16 @@ def main(): opts['verbose'] = args.verbose opts['search'] = 'ALL' opts['counter'] = 0 + opts['use_ssl'] = True opts['db'] = None opts['id'] = 0 if args.date: opts['search'] = args.date + if args.no_ssl: + opts['use_ssl'] = False + server = '' user = '' password = '' @@ -152,7 +157,7 @@ def main(): if opts['verbose']: print("Skipped folder list: {}".format(opts['skip_folders'])) - if args.port == 993: + if opts['use_ssl']: conn = imaplib.IMAP4_SSL(server) else: conn = imaplib.IMAP4(server) diff --git a/util/pilerpurge.py b/util/pilerpurge.py index 3239cd1a..6e4e0fa4 100755 --- a/util/pilerpurge.py +++ b/util/pilerpurge.py @@ -54,12 +54,26 @@ def purge_m_files(ids=[], opts={}): remove_m_files(ids, opts) # Set deleted=1 for aged metadata entries + # as well as clean other tables if opts['dry_run'] is False: cursor = opts['db'].cursor() format = ", ".join(['%s'] * len(ids)) - cursor.execute("UPDATE metadata SET deleted=1 WHERE piler_id IN " + + cursor.execute("UPDATE metadata SET deleted=1, subject=NULL, `from`=''," + + "fromdomain='', message_id='' WHERE piler_id IN " + "(%s)" % (format), ids) + + cursor.execute("DELETE FROM rcpt WHERE id IN (SELECT id FROM metadata " + + "WHERE piler_id IN (%s))" % (format), ids) + cursor.execute("DELETE FROM note WHERE id IN (SELECT id FROM metadata " + + "WHERE piler_id IN (%s))" % (format), ids) + cursor.execute("DELETE FROM tag WHERE id IN (SELECT id FROM metadata " + + "WHERE piler_id IN (%s))" % (format), ids) + cursor.execute("DELETE FROM private WHERE id IN (SELECT id FROM metadata" + + "WHERE piler_id IN (%s))" % (format), ids) + cursor.execute("DELETE FROM folder_message WHERE id IN (SELECT id FROM " + + "metadata WHERE piler_id IN (%s))" % (format), ids) + opts['db'].commit() @@ -85,7 +99,7 @@ def purge_attachments_by_attachment_id(opts={}): cursor = opts['db'].cursor() cursor.execute("SELECT i, piler_id, attachment_id, refcount FROM " + - "v_attachment WHERE i IN (%s)" % + "v_attachment WHERE refcount=0 AND i IN (%s)" % (format), opts['referenced_attachments']) while True: @@ -160,12 +174,12 @@ def unlink(filename="", opts={}): def get_m_file_path(id='', opts={}): - return "/".join([opts['storedir'], opts['server_id'], id[8:11], id[32:34], + return "/".join([opts['storedir'], id[24:26], id[8:11], id[32:34], id[34:36], id + ".m"]) def get_attachment_file_path(piler_id='', attachment_id=0, opts={}): - return "/".join([opts['storedir'], opts['server_id'], piler_id[8:11], + return "/".join([opts['storedir'], piler_id[24:26], piler_id[8:11], piler_id[32:34], piler_id[34:36], piler_id + ".a" + str(attachment_id)]) diff --git a/util/postinstall.sh.in b/util/postinstall.sh.in index cf504845..67c9ff41 100755 --- a/util/postinstall.sh.in +++ b/util/postinstall.sh.in @@ -286,6 +286,7 @@ make_cron_entries() { echo "30 6 * * * /usr/bin/php ${LIBEXECDIR}/piler/generate_stats.php --webui ${DOCROOT} >/dev/null"; echo "*/5 * * * * /usr/bin/find ${LOCALSTATEDIR}/piler/error -type f|wc -l > ${LOCALSTATEDIR}/piler/stat/error"; echo "*/5 * * * * /usr/bin/find ${DOCROOT}/tmp -type f -name i.\* -exec rm -f {} \;"; + echo "#*/5 * * * * ${LIBEXECDIR}/piler/import.sh"; echo "### PILEREND"; } >> "$CRON_TMP" } diff --git a/util/reindex.sh b/util/reindex.sh new file mode 100755 index 00000000..121ca9ff --- /dev/null +++ b/util/reindex.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +INSTALL_PREFIX=/usr/local +THRESHOLD=100000000 + +start_id=1 +stop_id=5000000 + +while [[ $start_id -lt $stop_id ]]; do + y=$(( start_id + 9999 )) + echo $start_id $y + "${INSTALL_PREFIX}/bin/reindex" -f $start_id -t $y -p + "${INSTALL_PREFIX}/libexec/piler/indexer.delta.sh" + start_id=$(( start_id + 10000 )) + + if [[ "$(stat -c %s /var/piler/sphinx/dailydelta1.spp)" -gt "$THRESHOLD" ]]; then + "${INSTALL_PREFIX}/libexec/piler/indexer.main.sh" + fi +done diff --git a/util/sign.php b/util/sign.php index 2903121f..d0f8d3e8 100644 --- a/util/sign.php +++ b/util/sign.php @@ -12,11 +12,13 @@ ini_set("session.save_path", "/tmp"); $webuidir = ""; $verbose = 0; $mode = "unit"; +$algo = "sha256"; $opts = 'h::v'; $lopts = array( 'webui:', 'mode:', + 'algo:', 'verbose' ); @@ -37,6 +39,10 @@ if ( $options = getopt( $opts, $lopts ) ) $mode = $options['mode']; } + if ( isset($options['algo']) ) { + $algo = $options['algo']; + } + if ( isset($options['h']) ) { display_help(); @@ -66,6 +72,7 @@ Registry::set('db', $db); Registry::set('DB_DRIVER', DB_DRIVER); define('MODE', $mode); +define('ALGO', $algo); $data = get_hash_values(); @@ -121,29 +128,29 @@ function get_hash_values() { if($last_id == 0) { $start_id = TSA_START_ID; - if(MODE == 'unit') { $stop_id = $start_id + TSA_STAMP_REQUEST_UNIT_SIZE - 1; } - else { $stop_id = 1000000000; } - } - else { + } else { $start_id = $last_id + 1; - if(MODE == 'unit') { $stop_id = $start_id + TSA_STAMP_REQUEST_UNIT_SIZE - 1; } - else { $stop_id = 1000000000; } } - $query = $db->query("SELECT id, digest FROM " . TABLE_META . " WHERE id >= ? AND id <= ?", array($start_id, $stop_id)); + if(MODE == 'unit') { + $limit = TSA_STAMP_REQUEST_UNIT_SIZE; + } else { + $limit = 100000; // stay well below default PHP memory_limit + } + + $query = $db->query("SELECT id, digest FROM " . TABLE_META . " WHERE id >= ? ORDER BY id LIMIT $limit", array($start_id)); foreach($query->rows as $q) { $count++; + $last_id = $q['id']; $s .= $q['digest']; } - if(MODE == 'time') { $stop_id = $start_id + $count - 1; } - return [ START_ID => $start_id, - STOP_ID => $stop_id, + STOP_ID => $last_id, COUNT => $count, - HASH_VALUE => sha1($s) + HASH_VALUE => hash(ALGO, $s) ]; } diff --git a/webui/Zend/Mail/Protocol/Imap.php b/webui/Zend/Mail/Protocol/Imap.php index 7d6351ca..b10585c2 100644 --- a/webui/Zend/Mail/Protocol/Imap.php +++ b/webui/Zend/Mail/Protocol/Imap.php @@ -410,7 +410,7 @@ class Zend_Mail_Protocol_Imap $a = $this->requestAndResponse("LIST", array('""', '"*"')); - while(list($k, $v) = each($a)) { + foreach ($a as $k => $v) { if($v[1][0] == '\HasNoChildren') { array_push($folders, $v[3]); } diff --git a/webui/controller/audit/helper.php b/webui/controller/audit/helper.php index 774274ec..10b3c5af 100644 --- a/webui/controller/audit/helper.php +++ b/webui/controller/audit/helper.php @@ -99,7 +99,7 @@ class ControllerAuditHelper extends Controller { $s = preg_replace("/\s{1,}/", " ", $s); $b = explode(" ", $s); - while(list($k, $v) = each($b)) { + foreach ($b as $k => $v) { if($v == '') { continue; } if(preg_match("/(login|loginfailed|logout|view|download|search|restore|journal)$/", $v) && isset($actions[$v])) { $this->a['action'] .= "\t" . $actions[$v]; } diff --git a/webui/controller/search/helper.php b/webui/controller/search/helper.php index 70498a0c..5f225308 100644 --- a/webui/controller/search/helper.php +++ b/webui/controller/search/helper.php @@ -140,7 +140,7 @@ class ControllerSearchHelper extends Controller { $s = preg_replace("/OR/", "|", $data['search']); $b = preg_split("/\s/", $s); - while(list($k, $v) = each($b)) { + foreach ($b as $k => $v) { if($v == '') { continue; } if(preg_match("/\d{4}\-\d{1,2}\-\d{1,2}/", $v) || preg_match("/\d{1,2}\/\d{1,2}\/\d{4}/", $v)) { diff --git a/webui/controller/user/edit.php b/webui/controller/user/edit.php index e66b5aa7..b3407da9 100644 --- a/webui/controller/user/edit.php +++ b/webui/controller/user/edit.php @@ -79,7 +79,7 @@ class ControllerUserEdit extends Controller { $this->data['emails'] = $this->model_user_user->get_emails($this->data['user']['username']); - $this->data['user']['group'] = $this->model_group_group->get_groups_by_email(array($this->data['emails'])); + $this->data['user']['group'] = $this->model_group_group->get_groups_by_email(explode("\n", $this->data['emails'])); //} } else { diff --git a/webui/google-api/service/apiUtils.php b/webui/google-api/service/apiUtils.php index c692ec4e..e8298b41 100644 --- a/webui/google-api/service/apiUtils.php +++ b/webui/google-api/service/apiUtils.php @@ -55,7 +55,7 @@ class apiUtils { $strlenVar = strlen($str); $d = $ret = 0; for ($count = 0; $count < $strlenVar; ++ $count) { - $ordinalValue = ord($str{$ret}); + $ordinalValue = ord($str[$ret]); switch (true) { case (($ordinalValue >= 0x20) && ($ordinalValue <= 0x7F)): // characters U-00000000 - U-0000007F (same as ASCII) @@ -114,4 +114,4 @@ class apiUtils { } return $normalized; } -} \ No newline at end of file +} diff --git a/webui/language/cn/messages.php b/webui/language/cn/messages.php new file mode 100644 index 00000000..179bb0af --- /dev/null +++ b/webui/language/cn/messages.php @@ -0,0 +1,497 @@ +db->query("SELECT `$column`-(`$column` % 86400) AS `day`, `to`, COUNT(*) AS `count`, SUM(`size`) AS `size` FROM " . VIEW_MESSAGES . " WHERE `$column` >= ? AND `$column` < ? GROUP BY FROM_UNIXTIME(`day`, '%Y.%m.%d.'), `to`", array($start, $stop)); + $tousers = $this->db->query("SELECT `$column`-(`$column` % 86400) AS `day`, `to`, COUNT(*) AS `count`, SUM(`size`) AS `size` FROM " . VIEW_MESSAGES . " WHERE deleted=0 AND `$column` >= ? AND `$column` < ? GROUP BY FROM_UNIXTIME(`day`, '%Y.%m.%d.'), `to`", array($start, $stop)); foreach($tousers->rows as $row) { $counter[$row['day']][$row['to']]['recd'] = $row['count']; @@ -46,7 +46,7 @@ class ModelAccountingAccounting extends Model { // emails sent from users - $fromusers = $this->db->query("SELECT `$column`-(`$column` % 86400) AS `day`, `from`, COUNT(*) AS `count`, SUM(`size`) AS `size` FROM " . VIEW_MESSAGES . " WHERE `$column` >= ? AND `$column` < ? GROUP BY FROM_UNIXTIME(`day`, '%Y.%m.%d.'), `from`", array($start, $stop)); + $fromusers = $this->db->query("SELECT `$column`-(`$column` % 86400) AS `day`, `from`, COUNT(*) AS `count`, SUM(`size`) AS `size` FROM " . VIEW_MESSAGES . " WHERE deleted=0 AND `$column` >= ? AND `$column` < ? GROUP BY FROM_UNIXTIME(`day`, '%Y.%m.%d.'), `from`", array($start, $stop)); foreach($fromusers->rows as $row) { $counter[$row['day']][$row['from']]['sent'] = $row['count']; @@ -144,10 +144,6 @@ class ModelAccountingAccounting extends Model { $search = preg_replace("/\s{1,}/", "", $search); - if($search) { - $search_cond .= " AND ( `email` LIKE '%".$search."%' OR `domain` LIKE '%".$search."%' )"; - } - $query = "SELECT `email` AS `item`, MIN(`date`) AS `oldest`, MAX(`date`) AS `newest`, SUM(`sent`) AS `sent`, SUM(`recd`) AS `recd`, SUM(`sentsize`) AS `sentsize`, SUM(`recdsize`) AS `recdsize` FROM " . TABLE_STAT_COUNTER; if($item == 'email') { diff --git a/webui/model/audit/audit.php b/webui/model/audit/audit.php index f73027f8..6fa71436 100644 --- a/webui/model/audit/audit.php +++ b/webui/model/audit/audit.php @@ -43,7 +43,7 @@ class ModelAuditAudit extends Model { if(Registry::get('admin_user') == 0 && RESTRICTED_AUDITOR == 1) { $auditdomains = $session->get("auditdomains"); - while(list($k, $v) = each($auditdomains)) { + foreach($auditdomains as $k => $v) { if($q) { $q .= ","; } $q .= "?"; array_push($arr, $v); diff --git a/webui/model/google/google.php b/webui/model/google/google.php index df6ca083..d5001597 100644 --- a/webui/model/google/google.php +++ b/webui/model/google/google.php @@ -76,7 +76,7 @@ class ModelGoogleGoogle extends Model { $messages = $storage->piler_batch_fetch($from, $to); - while(list($k, $v) = each($messages)) { + foreach($messages as $k => $v) { $uuid = $storage->getUniqueId($k); $tmpname = "piler-" . $email . "-" . $k . "-" . $uuid . ".eml"; @@ -98,7 +98,7 @@ class ModelGoogleGoogle extends Model { } - syslog(LOG_INFO, "downloaded $count messages for $email"); + syslog(LOG_INFO, "downloaded $count messages for $email"); return $count; } @@ -174,5 +174,3 @@ class ModelGoogleGoogle extends Model { } - -?> diff --git a/webui/model/health/health.php b/webui/model/health/health.php index 9e5a1de0..17e1b27d 100644 --- a/webui/model/health/health.php +++ b/webui/model/health/health.php @@ -199,7 +199,7 @@ class ModelHealthHealth extends Model { public function meminfo() { $m = explode("\n", file_get_contents("/proc/meminfo")); - while(list($k, $v) = each($m)) { + foreach($m as $k => $v) { $a = preg_split("/\ {1,}/", $v); if(isset($a[0]) && $a[0]) { $_m[$a[0]] = $a[1]; } } @@ -219,7 +219,7 @@ class ModelHealthHealth extends Model { $partitions = Registry::get('partitions_to_monitor'); - while(list($k, $v) = each($output)) { + foreach($output as $k => $v) { if($k > 0) { $p = preg_split("/\ {1,}/", $v); if(isset($p[5]) && in_array($p[5], $partitions) && !isset($a[$p[5]])) { diff --git a/webui/model/mail/mail.php b/webui/model/mail/mail.php index 6cbe4034..0bfafc7b 100644 --- a/webui/model/mail/mail.php +++ b/webui/model/mail/mail.php @@ -26,7 +26,7 @@ class ModelMailMail extends Model { fputs($r, "MAIL FROM: <$from>\r\n"); $l = fgets($r, 4096); - while(list($k, $v) = each($to)) { + foreach($to as $k => $v) { fputs($r, "RCPT TO: <$v>\r\n"); $l = fgets($r, 4096); } diff --git a/webui/model/search/message.php b/webui/model/search/message.php index 2351a1de..2eac00d0 100644 --- a/webui/model/search/message.php +++ b/webui/model/search/message.php @@ -213,7 +213,7 @@ class ModelSearchMessage extends Model { $a = explode(" ", $terms); $terms = array(); - while(list($k, $v) = each($a)) { + foreach($a as $k => $v) { if(strlen($v) >= 3 && !in_array($v, $fields)) { $v = preg_replace("/\*/", "", $v); if($v) { array_push($terms, $v); } @@ -223,7 +223,7 @@ class ModelSearchMessage extends Model { if(count($terms) <= 0) { return $s; } if($html == 0) { - while(list($k, $v) = each($terms)) { + foreach($terms as $k => $v) { $s = preg_replace("/$v/i", "$v", $s); } @@ -233,7 +233,7 @@ class ModelSearchMessage extends Model { $tokens = preg_split("/\ $token) { $pos = strpos($token, ">"); if($pos > 0) { @@ -245,7 +245,7 @@ class ModelSearchMessage extends Model { $str = substr($token, $pos+1, $len); reset($terms); - while(list($k, $v) = each($terms)) { + foreach($terms as $k => $v) { $str = preg_replace("/$v/i", "$v", $str); } @@ -365,13 +365,20 @@ class ModelSearchMessage extends Model { if($computed_hash == '') { - $query2 = $this->db->query("SELECT digest FROM " . TABLE_META . " WHERE id >= ? AND id <= ?", array($query->row['start_id'], $query->row['stop_id'])); + $query2 = $this->db->query("SELECT digest FROM " . TABLE_META . " WHERE id >= ? AND id <= ? ORDER BY id", array($query->row['start_id'], $query->row['stop_id'])); foreach($query2->rows as $q) { $s .= $q['digest']; } - $computed_hash = sha1($s); + $len = strlen($query->row['hash_value']); + if($len == 64) + $algo='sha256'; + elseif($len == 128) + $algo='sha512'; + else + $algo='sha1'; + $computed_hash = hash($algo, $s); if(MEMCACHED_ENABLED) { $memcache->add($cache_key, $computed_hash, 0, MEMCACHED_TTL); diff --git a/webui/model/search/search.php b/webui/model/search/search.php index ad4afd8a..9ec11173 100644 --- a/webui/model/search/search.php +++ b/webui/model/search/search.php @@ -16,7 +16,7 @@ class ModelSearchSearch extends Model { $session = Registry::get('session'); - while(list($k,$v) = each($data)) { + foreach($data as $k => $v) { if($v) { if(is_array($v)) { $v = implode(" ", $v); } $s .= '&' . $k . '=' . $v; } } @@ -113,11 +113,10 @@ class ModelSearchSearch extends Model { $session = Registry::get('session'); $i = 0; - while(list($k, $v) = each($data['match'])) { + foreach($data['match'] as $k => $v) { if($v == "@attachment_types") { - list($k, $v) = each($data['match']); $i++; - if($v == "any") { + if($data['match'][$i] == "any") { $data['match'][$i-1] = ""; $data['match'][$i] = ""; } @@ -199,7 +198,7 @@ class ModelSearchSearch extends Model { if(ENABLE_FOLDER_RESTRICTIONS == 1) { $s = explode(" ", $data['folders']); - while(list($k,$v) = each($s)) { + foreach($s as $k => $v) { if(in_array($v, $session->get("folders"))) { array_push($__folders, $v); } @@ -347,7 +346,7 @@ class ModelSearchSearch extends Model { $s = preg_replace("/httpX/", "http:", $s); $b = explode(" ", $s); - while(list($k, $v) = each($b)) { + foreach($b as $k => $v) { if($v == '') { continue; } if($v == 'from:') { $token = 'match'; $a['match'][] = FROM_TOKEN; continue; } @@ -441,7 +440,7 @@ class ModelSearchSearch extends Model { $offset = $page * $pagelen; $s = explode(" ", $extra_folders); - while(list($k,$v) = each($s)) { + foreach($s as $k => $v) { if(in_array($v, $session->get("extra_folders")) && is_numeric($v)) { array_push($__folders, $v); if($q) { $q .= ",?"; } @@ -647,7 +646,9 @@ class ModelSearchSearch extends Model { $emails = $session->get($session_var); - while(list($k, $v) = each($emails)) { + if(!$emails) { return $s; } + + foreach($emails as $k => $v) { if($s) { $s .= '| ' . $this->fix_email_address_for_sphinx($v); } else { $s = $this->fix_email_address_for_sphinx($v); } } @@ -681,6 +682,13 @@ class ModelSearchSearch extends Model { if($id == '') { return 0; } + if(Registry::get('data_officer') == 1) { + $query = $this->db->query("SELECT id FROM " . TABLE_DELETED . " WHERE deleted=-1 AND id=?", array($id)); + if(!isset($query->row['id'])) { + return 0; + } + } + if((Registry::get('auditor_user') == 1 || Registry::get('data_officer') == 1) && RESTRICTED_AUDITOR == 0) { return 1; } $session = Registry::get('session'); @@ -695,7 +703,7 @@ class ModelSearchSearch extends Model { $auditdomains = $session->get("auditdomains"); - while(list($k, $v) = each($auditdomains)) { + foreach($auditdomains as $k => $v) { if(validdomain($v) == 1 && !in_array($v, $a)) { $q .= ",?"; array_push($a, $v); @@ -711,7 +719,7 @@ class ModelSearchSearch extends Model { $emails = $session->get("emails"); - while(list($k, $v) = each($emails)) { + foreach($emails as $k => $v) { if(validemail($v) == 1) { $q .= ",?"; array_push($a, $v); @@ -767,7 +775,7 @@ class ModelSearchSearch extends Model { $auditdomains = $session->get("auditdomains"); - while(list($k, $v) = each($auditdomains)) { + foreach($auditdomains as $k => $v) { if(validdomain($v) == 1 && !in_array($v, $a)) { $q .= ",?"; array_push($a, $v); @@ -778,7 +786,7 @@ class ModelSearchSearch extends Model { if(Registry::get('auditor_user') == 0) { $emails = $session->get("emails"); - while(list($k, $v) = each($emails)) { + foreach($emails as $k => $v) { if(validemail($v) == 1) { $q .= ",?"; array_push($a, $v); @@ -835,6 +843,10 @@ class ModelSearchSearch extends Model { public function fix_email_address_for_sphinx($email = '') { + if(strlen($email) > MAX_EMAIL_LEN) { + return md5($email . ' '); + } + $email = preg_replace("/\|@/", "|", $email); return preg_replace("/[\@\.\+\-\_]/", "X", $email); } @@ -906,7 +918,7 @@ class ModelSearchSearch extends Model { $a = explode(" ", $s); $s = ''; - while(list($k, $v) = each($a)) { + foreach($a as $k => $v) { if(substr($v, 0, 4) == 'http') { $v = preg_replace("/http(s){0,1}\:\/\//", "__URL__", $v); diff --git a/webui/model/stat/chart.php b/webui/model/stat/chart.php index e52857bc..42e8951b 100644 --- a/webui/model/stat/chart.php +++ b/webui/model/stat/chart.php @@ -69,7 +69,7 @@ class ModelStatChart extends Model { if($query->num_rows >= 15) { $i = 0; - while(list($k, $v) = each($dates)) { + foreach($dates as $k => $v) { $i++; if($i % 3) { $dates[$k] = ""; } } diff --git a/webui/model/user/import.php b/webui/model/user/import.php index 072a04da..449205a0 100644 --- a/webui/model/user/import.php +++ b/webui/model/user/import.php @@ -155,7 +155,7 @@ class ModelUserImport extends Model { /* build a list of DNs to exclude from the import */ - while (list($k, $v) = each($globals)) { + foreach($globals as $k => $v) { if(preg_match("/^reject_/", $k)) { $exclude[$v] = $v; } diff --git a/webui/model/user/user.php b/webui/model/user/user.php index 07176efa..e3d67aa3 100644 --- a/webui/model/user/user.php +++ b/webui/model/user/user.php @@ -49,7 +49,7 @@ class ModelUserUser extends Model { $data = array(); $uids = $uid; - if($uid > 0) { + if($uid >= 0) { $query = $this->db->query("SELECT gid FROM " . TABLE_EMAIL_LIST . " WHERE uid=?", array((int)$uid)); if(isset($query->rows)) { @@ -68,9 +68,6 @@ class ModelUserUser extends Model { } - $emails = $this->get_email_addresses_from_groups($data); - $data = array_merge($data, $emails); - return $data; } @@ -559,7 +556,10 @@ class ModelUserUser extends Model { if(!$this->check_uid($uid)){ return 0; } $query = $this->db->query("DELETE FROM " . TABLE_EMAIL . " WHERE uid=?", array((int)$uid)); + $query = $this->db->query("DELETE FROM " . TABLE_USER_SETTINGS . " WHERE username IN (SELECT username FROM " . TABLE_USER . " WHERE uid=?)", array((int)$uid)); $query = $this->db->query("DELETE FROM " . TABLE_USER . " WHERE uid=?", array((int)$uid)); + $query = $this->db->query("DELETE FROM " . TABLE_DOMAIN_USER . " WHERE uid=?", array((int)$uid)); + $query = $this->db->query("DELETE FROM " . TABLE_FOLDER_USER . " WHERE uid=?", array((int)$uid)); LOGGER("remove user: uid=$uid"); diff --git a/phpunit.xml b/webui/phpunit.xml similarity index 56% rename from phpunit.xml rename to webui/phpunit.xml index a940f772..18034cdc 100644 --- a/phpunit.xml +++ b/webui/phpunit.xml @@ -2,11 +2,11 @@ - ./unit_tests/php + ./tests - - + + diff --git a/webui/system/controller.php b/webui/system/controller.php index 690e1880..1e7c6a99 100644 --- a/webui/system/controller.php +++ b/webui/system/controller.php @@ -27,7 +27,9 @@ class Controller { public function args($args = array()){ - while(list($key, $value) = each($args)) $this->data[$key] = $value; + foreach($args as $key => $value) { + $this->data[$key] = $value; + } } diff --git a/webui/system/database/ldap.php b/webui/system/database/ldap.php index 49f30ff6..cbca32f2 100644 --- a/webui/system/database/ldap.php +++ b/webui/system/database/ldap.php @@ -12,6 +12,10 @@ class LDAP { ldap_set_option($this->link, LDAP_OPT_PROTOCOL_VERSION, 3); ldap_set_option($this->link, LDAP_OPT_REFERRALS, 0); + if (LDAP_USE_START_TLS == 1) { + ldap_start_tls($this->link); + } + if(@ldap_bind($this->link, $binddn, $bindpw)) { $this->bind = 1; } diff --git a/webui/system/database/mysql.php b/webui/system/database/mysql.php index f3a1895c..874bf48b 100644 --- a/webui/system/database/mysql.php +++ b/webui/system/database/mysql.php @@ -53,7 +53,7 @@ class MySQL { $R = $s->fetchAll(PDO::FETCH_ASSOC); - while(list ($k, $v) = each($R)){ + foreach($R as $k => $v) { $data[$i] = $v; $i++; } diff --git a/webui/system/database/sphinx.php b/webui/system/database/sphinx.php index faaf4dc1..108fedbd 100644 --- a/webui/system/database/sphinx.php +++ b/webui/system/database/sphinx.php @@ -39,13 +39,15 @@ class Sphinx { $s = $this->link->prepare($sql); if(!$s) { return $query; } - $s->execute($arr); + if(!$s->execute($arr)) { + syslog(LOG_INFO, $s->errorInfo()[2]); + } $this->affected = $s->rowCount(); $R = $s->fetchAll(); - while(list ($k, $v) = each($R)){ + foreach($R as $k => $v) { $data[$i] = $v; $i++; } @@ -70,7 +72,7 @@ class Sphinx { $meta->execute(); $R = $meta->fetchAll(); - while(list ($k, $v) = each($R)){ + foreach($R as $k => $v) { if($v[0] == "total_found") { $query->total_found = $v[1]; } } diff --git a/webui/system/database/sqlite.php b/webui/system/database/sqlite.php index 793b1d84..9ff07dc4 100644 --- a/webui/system/database/sqlite.php +++ b/webui/system/database/sqlite.php @@ -47,7 +47,7 @@ class SQLite { $R = $s->fetchAll(); - while(list ($k, $v) = each($R)){ + foreach($R as $k => $v) { $data[$i] = $v; $i++; } diff --git a/webui/system/helper/HTMLPurifier.standalone.php b/webui/system/helper/HTMLPurifier.standalone.php index 467218af..b5e32a6e 100644 --- a/webui/system/helper/HTMLPurifier.standalone.php +++ b/webui/system/helper/HTMLPurifier.standalone.php @@ -3923,7 +3923,7 @@ class HTMLPurifier_Encoder $len = strlen($str); for ($i = 0; $i < $len; $i++) { - $in = ord($str{$i}); + $in = ord($str[$i]); $char .= $str[$i]; // append byte to char if (0 == $mState) { // When mState is zero we expect either a US-ASCII character @@ -14284,7 +14284,7 @@ class HTMLPurifier_ChildDef_Custom extends HTMLPurifier_ChildDef protected function _compileRegex() { $raw = str_replace(' ', '', $this->dtd_regex); - if ($raw{0} != '(') { + if ($raw[0] != '(') { $raw = "($raw)"; } $el = '[#a-zA-Z0-9_.-]+'; @@ -20681,7 +20681,7 @@ class HTMLPurifier_TagTransform_Font extends HTMLPurifier_TagTransform if (isset($attr['size'])) { // normalize large numbers if ($attr['size'] !== '') { - if ($attr['size']{0} == '+' || $attr['size']{0} == '-') { + if ($attr['size'][0] == '+' || $attr['size'][0] == '-') { $size = (int)$attr['size']; if ($size < -2) { $attr['size'] = '-2'; diff --git a/webui/system/helper/TrustedTimestamps.php b/webui/system/helper/TrustedTimestamps.php index 9dc702e5..182c7a42 100644 --- a/webui/system/helper/TrustedTimestamps.php +++ b/webui/system/helper/TrustedTimestamps.php @@ -15,6 +15,11 @@ * http://www.slproweb.com/products/Win32OpenSSL.html * http://www.switch.ch/aai/support/howto/openssl-windows.html * + * 2021-10-26 Frank Schmirler: + * - extract certificate chain from TSResponse and feed into ts -verify as -untrusted + * - unlink temporary files + * - support for sha256 and sha512 hashes + * * @version 0.3 * @author David Mller * @package trustedtimestamps @@ -25,16 +30,22 @@ class TrustedTimestamps /** * Creates a Timestamp Requestfile from a hash * - * @param string $hash: The hashed data (sha1) + * @param string $hash: The hashed data (sha1, sha256 or sha512) * @return string: path of the created timestamp-requestfile */ public static function createRequestfile ($hash) { - if (strlen($hash) !== 40) + if (strlen($hash) === 40) + $digest="-sha1"; + elseif (strlen($hash) === 64) + $digest="-sha256"; + elseif (strlen($hash) === 128) + $digest="-sha512"; + else throw new Exception("Invalid Hash."); $outfilepath = self::createTempFile(); - $cmd = OPENSSL_BINARY . " ts -query -digest ".escapeshellarg($hash)." -cert -out ".escapeshellarg($outfilepath); + $cmd = OPENSSL_BINARY . " ts -query $digest -digest ".escapeshellarg($hash)." -cert -out ".escapeshellarg($outfilepath); $retarray = array(); exec($cmd." 2>&1", $retarray, $retcode); @@ -106,6 +117,8 @@ class TrustedTimestamps $retarray = array(); exec($cmd." 2>&1", $retarray, $retcode); + unlink($responsefile); + if ($retcode !== 0) throw new Exception("The reply failed: ".implode(", ", $retarray)); @@ -144,7 +157,7 @@ class TrustedTimestamps */ public static function validate ($hash, $base64_response_string, $response_time, $tsa_cert_file) { - if (strlen($hash) !== 40) + if (strlen($hash) !== 40 && strlen($hash) !== 64 && strlen($hash) !== 128) throw new Exception("Invalid Hash"); $binary_response_string = base64_decode($base64_response_string); @@ -160,11 +173,22 @@ class TrustedTimestamps $responsefile = self::createTempFile($binary_response_string); - $cmd = OPENSSL_BINARY . " ts -verify -digest ".escapeshellarg($hash)." -in ".escapeshellarg($responsefile)." -CAfile ".escapeshellarg($tsa_cert_file); + /* + * extract chain from response + * openssl ts -verify does not include them for verification despite of the man page stating otherwise + */ + $untrustedfile = self::createTempFile(); + $cmd = OPENSSL_BINARY . " ts -reply -in ".escapeshellarg($responsefile)." -token_out | " . OPENSSL_BINARY . " pkcs7 -inform DER -print_certs -out ".escapeshellarg($untrustedfile); + shell_exec($cmd); + + $cmd = OPENSSL_BINARY . " ts -verify -digest ".escapeshellarg($hash)." -in ".escapeshellarg($responsefile)." -CAfile ".escapeshellarg($tsa_cert_file)." -untrusted ".escapeshellarg($untrustedfile); $retarray = array(); exec($cmd." 2>&1", $retarray, $retcode); + unlink($untrustedfile); + unlink($responsefile); + /* * just 2 "normal" cases: * 1) Everything okay -> retcode 0 + retarray[0] == "Verification: OK" diff --git a/webui/system/helper/mime.php b/webui/system/helper/mime.php index b4f835ef..d50727de 100644 --- a/webui/system/helper/mime.php +++ b/webui/system/helper/mime.php @@ -144,24 +144,13 @@ class Piler_Mime_Decode { public static function removeJournal(&$message, $EOL = "\n") { $has_journal = 0; - $s = self::remove_LF($message); - if(strpos($s, $EOL . $EOL)) { - list($headers, $body) = explode($EOL . $EOL, $s, 2); - if(strstr($headers, "\nX-MS-Journal-Report:")) { - return $has_journal; - } + self::splitMessageRaw($message, $headers, $journal, $body); + + if($journal) { + $has_journal = 1; } - $p = strstr($message, "\nX-MS-Journal-Report:"); - if($p) { - $q = stristr($p, "message/rfc822"); - if($q) { - $has_journal = 1; - $i = strlen("message/rfc822"); - while(ctype_space($q[$i])) { $i++; } - if($i > 0) { $message = substr($q, $i); } - } - } + $message = $headers . $EOL . $EOL . $body; return $has_journal; } @@ -205,6 +194,8 @@ class Piler_Mime_Decode { for($i=0; $i $v) { + if(strchr($v, "\n")) { $result[$k] = explode("\n", $v); } @@ -315,16 +307,19 @@ class Piler_Mime_Decode { public static function fixMimeBodyPart($headers = array(), $body = '') { if(isset($headers['content-transfer-encoding'])) { - if($headers['content-transfer-encoding'] == 'quoted-printable') { + if(strtolower($headers['content-transfer-encoding']) == 'quoted-printable') { $body = quoted_printable_decode($body); } - if($headers['content-transfer-encoding'] == 'base64') { + if(strtolower($headers['content-transfer-encoding']) == 'base64') { $body = base64_decode($body); } } if(isset($headers['content-type']['charset'])) { + if(strtolower($headers['content-type']['charset']) == 'gb2312') { + $headers['content-type']['charset'] = 'GBK'; + } $body = iconv($headers['content-type']['charset'], 'utf-8' . '//IGNORE', $body); } diff --git a/webui/system/language.php b/webui/system/language.php index 8dcce51c..72332080 100644 --- a/webui/system/language.php +++ b/webui/system/language.php @@ -17,7 +17,7 @@ class Language { else { $pref_langs = $this->get_preferred_languages(); - while(list($k, $v) = each($pref_langs)) { + foreach($pref_langs as $k => $v) { if(in_array($v, $langs)) { $lang = $v; define('LANG', $lang); @@ -53,7 +53,8 @@ class Language { $l = explode(";", $_SERVER['HTTP_ACCEPT_LANGUAGE']); - while(list($k, $v) = each($l)) { + foreach($l as $k => $v) { + $a = explode(",", $v); if(isset($a[0]) && substr($a[0], 0, 2) != 'q=') { diff --git a/webui/system/misc.php b/webui/system/misc.php index 5c35be15..5b459554 100644 --- a/webui/system/misc.php +++ b/webui/system/misc.php @@ -147,16 +147,6 @@ function checkemail($email, $domains) { function validemail($email = '') { if($email == '') { return 0; } - // sphinxsearch supports tokens up to 41 characters long - // If there's a longer token in the query, then sphinx - // reports a query error even if the query is itself correct - // So the workaround is to get rid of these email addresses - if(strlen($email) > MAX_EMAIL_LEN) { - $msg = sprintf("discarding email %s: longer than %d", $email, MAX_EMAIL_LEN); - syslog(LOG_INFO, $msg); - return 0; - } - if(preg_match("/@local$/", $email)) { return 1; } if(preg_match('/^[_a-zA-Z0-9-]+(\.[_a-zA-Z0-9-]+)*@[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)*(\.[a-zA-Z]{2,10})$/', $email)) { @@ -196,7 +186,7 @@ function first_n_characters($what, $n){ $len = 0; $a = explode(" ", $what); - while(list($k, $v) = each($a)){ + foreach($a as $k => $v) { $x .= "$v "; $len += strlen($v) + 1; if($len >= $n){ return $x . "..."; } } @@ -279,7 +269,7 @@ function my_qp_encode($s){ $res = ""; $a = explode("\n", $s); - while(list($k, $v) = each($a)){ + foreach($a as $k => $v) { $part = ""; for($i=0; $i $v) { if(!isset($arr[$k]) || $arr[$k] == '') $arr[$k] = $v; } @@ -453,7 +443,7 @@ function convert_date_string_to_ymd_by_template($date_string, $date_template) { return [$Y, $m, $d]; } - while(list($k, $v) = each($template_array)) { + foreach($template_array as $k => $v) { $$v = $date_array[$k]; } @@ -613,7 +603,7 @@ function get_ldap_attribute_names($ldap_type = '') { function htmlentities_on_array($arr = []) { - while(list($k, $v) = each($arr)) { + foreach($arr as $k => $v) { if(is_array($v)) { $arr[$k] = htmlentities_on_array($v); } else { diff --git a/unit_tests/php/EmailTest.php b/webui/tests/EmailTest.php similarity index 96% rename from unit_tests/php/EmailTest.php rename to webui/tests/EmailTest.php index 7943b9b5..1861c439 100644 --- a/unit_tests/php/EmailTest.php +++ b/webui/tests/EmailTest.php @@ -3,6 +3,7 @@ use PHPUnit\Framework\TestCase; define('DIR_BASE', $_ENV['DIR_BASE']); +define('MAX_EMAIL_LEN', 41); include_once(DIR_BASE . "system/model.php"); include_once(DIR_BASE . "model/search/search.php"); diff --git a/unit_tests/php/FormatTest.php b/webui/tests/FormatTest.php similarity index 91% rename from unit_tests/php/FormatTest.php rename to webui/tests/FormatTest.php index 9b2e85fc..c9714619 100644 --- a/unit_tests/php/FormatTest.php +++ b/webui/tests/FormatTest.php @@ -2,9 +2,9 @@ use PHPUnit\Framework\TestCase; -include_once("webui/system/model.php"); -include_once("webui/system/misc.php"); -include_once("webui/model/health/health.php"); +include_once("system/model.php"); +include_once("system/misc.php"); +include_once("model/health/health.php"); final class FormatTest extends TestCase { diff --git a/unit_tests/php/ParseMessageTest.php b/webui/tests/ParseMessageTest.php similarity index 100% rename from unit_tests/php/ParseMessageTest.php rename to webui/tests/ParseMessageTest.php diff --git a/unit_tests/php/SplitMessageTest.php b/webui/tests/SplitMessageTest.php similarity index 100% rename from unit_tests/php/SplitMessageTest.php rename to webui/tests/SplitMessageTest.php diff --git a/webui/view/theme/default/templates/health/worker.tpl b/webui/view/theme/default/templates/health/worker.tpl index 30ef75f4..88c52a61 100644 --- a/webui/view/theme/default/templates/health/worker.tpl +++ b/webui/view/theme/default/templates/health/worker.tpl @@ -123,7 +123,7 @@ - $v) { if(!is_numeric($k)) { ?> diff --git a/webui/view/theme/default/templates/ldap/list.tpl b/webui/view/theme/default/templates/ldap/list.tpl index 55385edb..26b706bf 100644 --- a/webui/view/theme/default/templates/ldap/list.tpl +++ b/webui/view/theme/default/templates/ldap/list.tpl @@ -36,7 +36,7 @@
diff --git a/webui/view/theme/default/templates/policy/folder.tpl b/webui/view/theme/default/templates/policy/folder.tpl index 0638af31..27472638 100644 --- a/webui/view/theme/default/templates/policy/folder.tpl +++ b/webui/view/theme/default/templates/policy/folder.tpl @@ -91,7 +91,7 @@