Merge branch 'mmap_dedup'

This commit is contained in:
SJ 2015-05-13 11:54:58 +02:00
commit 20ed9b9e6d
11 changed files with 94 additions and 3 deletions

25
configure vendored
View File

@ -621,6 +621,7 @@ ac_includes_default="\
ac_subst_vars='LTLIBOBJS ac_subst_vars='LTLIBOBJS
LIBOBJS LIBOBJS
DATAROOTDIR
LIBEXECDIR LIBEXECDIR
DATADIR DATADIR
CFGDIR CFGDIR
@ -4610,6 +4611,30 @@ _ACEOF
dataroot_dir=`echo $datarootdir | grep prefix`
if test -n "$dataroot_dir"; then
if test "$prefix" = "NONE"
then
dataroot_dir="$ac_default_prefix/share"
else
dataroot_dir="$prefix/share"
fi
else
dataroot_dir="$datarootdir"
fi
DATAROOTDIR=$dataroot_dir
cat >>confdefs.h <<_ACEOF
#define DATAROOTDIR "$dataroot_dir"
_ACEOF
cat >>confdefs.h <<_ACEOF cat >>confdefs.h <<_ACEOF
#define VIRUS_TEMPLATE "$my_prefix/share/clapf/template.virus" #define VIRUS_TEMPLATE "$my_prefix/share/clapf/template.virus"
_ACEOF _ACEOF

View File

@ -340,6 +340,27 @@ AC_SUBST(LIBEXECDIR)
AC_DEFINE_UNQUOTED(LIBEXECDIR,"$libexec_dir",[where to look for the piler helpers]) AC_DEFINE_UNQUOTED(LIBEXECDIR,"$libexec_dir",[where to look for the piler helpers])
dnl configure dataroot directory
dataroot_dir=`echo $datarootdir | grep prefix`
if test -n "$dataroot_dir"; then
if test "$prefix" = "NONE"
then
dataroot_dir="$ac_default_prefix/share"
else
dataroot_dir="$prefix/share"
fi
else
dataroot_dir="$datarootdir"
fi
DATAROOTDIR=$dataroot_dir
AC_SUBST(DATAROOTDIR)
AC_DEFINE_UNQUOTED(DATAROOTDIR,"$dataroot_dir",[where to look for the share data files])
AC_DEFINE_UNQUOTED(VIRUS_TEMPLATE, "$my_prefix/share/clapf/template.virus", [where the virus template is]) AC_DEFINE_UNQUOTED(VIRUS_TEMPLATE, "$my_prefix/share/clapf/template.virus", [where the virus template is])
AC_DEFINE_UNQUOTED(ZOMBIE_NET_REGEX, "$my_prefix/share/clapf/zombienets.regex", [where the virus template is]) AC_DEFINE_UNQUOTED(ZOMBIE_NET_REGEX, "$my_prefix/share/clapf/zombienets.regex", [where the virus template is])

View File

@ -4,9 +4,12 @@
#define CONFDIR "/usr/local/etc" #define CONFDIR "/usr/local/etc"
#define DATADIR "/usr/local/var" #define DATADIR "/usr/local/var"
#define DATAROOTDIR "/usr/local/share"
#define KEYFILE CONFDIR "/piler.key" #define KEYFILE CONFDIR "/piler.key"
#define MESSAGE_ID_DEDUP_FILE DATAROOTDIR "/piler/deduphelper"
#define HAVE_DAEMON 1 #define HAVE_DAEMON 1
#undef TIMEOUT_BINARY #undef TIMEOUT_BINARY

View File

@ -83,6 +83,7 @@ struct _parse_rule config_parse_rules[] =
{ "memcached_to_db_interval", "integer", (void*) int_parser, offsetof(struct __config, memcached_to_db_interval), "900", sizeof(int)}, { "memcached_to_db_interval", "integer", (void*) int_parser, offsetof(struct __config, memcached_to_db_interval), "900", sizeof(int)},
{ "memcached_ttl", "integer", (void*) int_parser, offsetof(struct __config, memcached_ttl), "86400", sizeof(int)}, { "memcached_ttl", "integer", (void*) int_parser, offsetof(struct __config, memcached_ttl), "86400", sizeof(int)},
{ "min_word_len", "integer", (void*) int_parser, offsetof(struct __config, min_word_len), "1", sizeof(int)}, { "min_word_len", "integer", (void*) int_parser, offsetof(struct __config, min_word_len), "1", sizeof(int)},
{ "mmap_dedup_test", "integer", (void*) int_parser, offsetof(struct __config, mmap_dedup_test), "0", sizeof(int)},
{ "mysqlhost", "string", (void*) string_parser, offsetof(struct __config, mysqlhost), "", MAXVAL-1}, { "mysqlhost", "string", (void*) string_parser, offsetof(struct __config, mysqlhost), "", MAXVAL-1},
{ "mysqlport", "integer", (void*) int_parser, offsetof(struct __config, mysqlport), "", sizeof(int)}, { "mysqlport", "integer", (void*) int_parser, offsetof(struct __config, mysqlport), "", sizeof(int)},
{ "mysqlsocket", "string", (void*) string_parser, offsetof(struct __config, mysqlsocket), "/tmp/mysql.sock", MAXVAL-1}, { "mysqlsocket", "string", (void*) string_parser, offsetof(struct __config, mysqlsocket), "/tmp/mysql.sock", MAXVAL-1},

View File

@ -89,6 +89,8 @@ struct __config {
int syslog_recipients; int syslog_recipients;
int mmap_dedup_test;
int debug; int debug;
}; };

View File

@ -69,6 +69,7 @@ typedef void signal_func (int);
struct child { struct child {
pid_t pid; pid_t pid;
int serial;
int messages; int messages;
int status; int status;
}; };
@ -288,6 +289,8 @@ struct __data {
struct node *mydomains[MAXHASH]; struct node *mydomains[MAXHASH];
struct node *imapfolders[MAXHASH]; struct node *imapfolders[MAXHASH];
struct import *import; struct import *import;
char *dedup;
int child_serial;
#ifdef NEED_MYSQL #ifdef NEED_MYSQL
MYSQL_STMT *stmt_generic; MYSQL_STMT *stmt_generic;

View File

@ -18,6 +18,7 @@
#define ERR_OPEN_TMP_FILE "ERR: opening a tempfile" #define ERR_OPEN_TMP_FILE "ERR: opening a tempfile"
#define ERR_TIMED_OUT "ERR: timed out" #define ERR_TIMED_OUT "ERR: timed out"
#define ERR_FORK_FAILED "ERR: cannot fork()" #define ERR_FORK_FAILED "ERR: cannot fork()"
#define ERR_OPEN_DEDUP_FILE "ERR: cannot open dedup file"
#define ERR_MYSQL_CONNECT "Cannot connect to mysql server" #define ERR_MYSQL_CONNECT "Cannot connect to mysql server"
#define ERR_PSQL_CONNECT "Cannot connect to PSql server" #define ERR_PSQL_CONNECT "Cannot connect to PSql server"

View File

@ -263,6 +263,24 @@ int process_message(struct session_data *sdata, struct _state *state, struct __d
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: touch %s OK (%s)", sdata->ttmpfile, state->message_id_hash, state->message_id); if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: touch %s OK (%s)", sdata->ttmpfile, state->message_id_hash, state->message_id);
if(cfg->mmap_dedup_test == 1 && data->dedup != MAP_FAILED && data->child_serial >= 0 && data->child_serial < MAXCHILDREN){
if(strstr(data->dedup, state->message_id_hash)){
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: dedup string: %s", sdata->ttmpfile, data->dedup);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_INFO, "%s: message-id-hash=%s, serial=%d", sdata->ttmpfile, state->message_id_hash, data->child_serial);
remove_stripped_attachments(state);
return ERR_EXISTS;
}
memcpy(data->dedup + data->child_serial*DIGEST_LENGTH*2, state->message_id_hash, DIGEST_LENGTH*2);
}
/* store base64 encoded file attachments */ /* store base64 encoded file attachments */
if(state->n_attachments > 0){ if(state->n_attachments > 0){

View File

@ -8,6 +8,7 @@
#include <strings.h> #include <strings.h>
#include <sys/types.h> #include <sys/types.h>
#include <sys/socket.h> #include <sys/socket.h>
#include <sys/mman.h>
#include <netdb.h> #include <netdb.h>
#include <netinet/in.h> #include <netinet/in.h>
#include <arpa/inet.h> #include <arpa/inet.h>
@ -79,6 +80,7 @@ static void takesig(int sig){
if(quit == 0){ if(quit == 0){
i = search_slot_by_pid(pid); i = search_slot_by_pid(pid);
if(i >= 0){ if(i >= 0){
children[i].serial = i;
children[i].status = READY; children[i].status = READY;
children[i].pid = child_make(&children[i]); children[i].pid = child_make(&children[i]);
} }
@ -108,7 +110,7 @@ static void child_main(struct child *ptr){
ptr->messages = 0; ptr->messages = 0;
if(cfg.verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "child (pid: %d) started main()", getpid()); if(cfg.verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "child (pid: %d, serial: %d) started main()", getpid(), ptr->serial);
while(1){ while(1){
if(received_sighup == 1){ if(received_sighup == 1){
@ -129,6 +131,7 @@ static void child_main(struct child *ptr){
syslog(LOG_PRIORITY, "connection from %s", s); syslog(LOG_PRIORITY, "connection from %s", s);
data.child_serial = ptr->serial;
sig_block(SIGHUP); sig_block(SIGHUP);
ptr->messages += handle_smtp_session(new_sd, &data, &cfg); ptr->messages += handle_smtp_session(new_sd, &data, &cfg);
@ -137,7 +140,7 @@ static void child_main(struct child *ptr){
close(new_sd); close(new_sd);
if(cfg.max_requests_per_child > 0 && ptr->messages >= cfg.max_requests_per_child){ if(cfg.max_requests_per_child > 0 && ptr->messages >= cfg.max_requests_per_child){
if(cfg.verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "child (pid: %d) served enough: %d", getpid(), ptr->messages); if(cfg.verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "child (pid: %d, serial: %d) served enough: %d", getpid(), ptr->messages, ptr->serial);
break; break;
} }
@ -184,10 +187,12 @@ int child_pool_create(){
children[i].pid = 0; children[i].pid = 0;
children[i].messages = 0; children[i].messages = 0;
children[i].status = UNDEF; children[i].status = UNDEF;
children[i].serial = -1;
} }
for(i=0; i<cfg.number_of_worker_processes; i++){ for(i=0; i<cfg.number_of_worker_processes; i++){
children[i].status = READY; children[i].status = READY;
children[i].serial = i;
children[i].pid = child_make(&children[i]); children[i].pid = child_make(&children[i]);
if(children[i].pid == -1){ if(children[i].pid == -1){
@ -237,6 +242,8 @@ void p_clean_exit(){
unlink(cfg.pidfile); unlink(cfg.pidfile);
if(data.dedup != MAP_FAILED) munmap(data.dedup, MAXCHILDREN*DIGEST_LENGTH*2);
#ifdef HAVE_STARTTLS #ifdef HAVE_STARTTLS
if(data.ctx){ if(data.ctx){
SSL_CTX_free(data.ctx); SSL_CTX_free(data.ctx);
@ -345,7 +352,7 @@ void initialise_configuration(){
int main(int argc, char **argv){ int main(int argc, char **argv){
int i, rc, yes=1, daemonise=0; int i, rc, yes=1, daemonise=0, dedupfd;
char port_string[8]; char port_string[8];
struct addrinfo hints, *res; struct addrinfo hints, *res;
@ -384,6 +391,7 @@ int main(int argc, char **argv){
initrules(data.retention_rules); initrules(data.retention_rules);
data.ctx = NULL; data.ctx = NULL;
data.ssl = NULL; data.ssl = NULL;
data.dedup = MAP_FAILED;
memset(data.starttls, 0, sizeof(data.starttls)); memset(data.starttls, 0, sizeof(data.starttls));
@ -425,6 +433,13 @@ int main(int argc, char **argv){
if(drop_privileges(pwd)) fatal(ERR_SETUID); if(drop_privileges(pwd)) fatal(ERR_SETUID);
dedupfd = open(MESSAGE_ID_DEDUP_FILE, O_RDWR);
if(dedupfd == -1) fatal(ERR_OPEN_DEDUP_FILE);
data.dedup = mmap(NULL, MAXCHILDREN*DIGEST_LENGTH*2, PROT_READ|PROT_WRITE, MAP_SHARED, dedupfd, 0);
close(dedupfd);
if(data.dedup == MAP_FAILED) syslog(LOG_INFO, "cannot mmap() %s, errno=%d", MESSAGE_ID_DEDUP_FILE, errno);
syslog(LOG_PRIORITY, "%s %s, build %d starting", PROGNAME, VERSION, get_build()); syslog(LOG_PRIORITY, "%s %s, build %d starting", PROGNAME, VERSION, get_build());

View File

@ -43,6 +43,7 @@ install:
$(INSTALL) -m 0755 $(srcdir)/db-mysql.sql $(DESTDIR)$(datarootdir)/piler $(INSTALL) -m 0755 $(srcdir)/db-mysql.sql $(DESTDIR)$(datarootdir)/piler
$(INSTALL) -m 0755 $(srcdir)/db-mysql-root.sql.in $(DESTDIR)$(datarootdir)/piler $(INSTALL) -m 0755 $(srcdir)/db-mysql-root.sql.in $(DESTDIR)$(datarootdir)/piler
$(INSTALL) -m 0644 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(srcdir)/deduphelper $(DESTDIR)$(datarootdir)/piler
clean: clean:
rm -f postinstall.sh rm -f postinstall.sh

1
util/deduphelper Normal file
View File

@ -0,0 +1 @@
ad061a0e26ff9f4b07f3aa36d41d22c70859ca10bf7526e1a0c4e0fc0a99e6ddAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAceb924d10e31bcc6d3b186201168676c12e267c35afdadf1224bb65b9289cd28AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA