From b06fae40313f43212c10dc0d5c4df8f7f931620e Mon Sep 17 00:00:00 2001 From: SJ Date: Tue, 27 Dec 2011 20:51:56 +0100 Subject: [PATCH] 0.1.11 --- Makefile.in | 9 +++--- etc/cron.jobs.in | 25 +++++++++++++-- src/Makefile.in | 1 + src/cfg.c | 1 - src/cfg.h | 1 - src/config.h | 6 ++-- src/counters.c | 12 +++++--- src/defs.h | 1 + src/dirs.c | 8 ----- src/parser.c | 4 +-- src/pilerget.c | 46 +++++++++++++++++---------- src/session.c | 1 + util/db-mysql.sql | 49 +++++++++++++++++++++++++++++ util/ldap_sync.php | 77 ++++++++++++++++++++++++++++++++++++++++++++++ 14 files changed, 199 insertions(+), 42 deletions(-) create mode 100644 util/ldap_sync.php diff --git a/Makefile.in b/Makefile.in index 1a320860..08dc4bf8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -67,11 +67,12 @@ $(RECURSIVE_TARGETS): installdirs: mkinstalldirs $(srcdir)/mkinstalldirs \ $(DESTDIR)$(bindir) $(DESTDIR)$(sbindir) $(DESTDIR)$(libdir) $(DESTDIR)$(libexecdir)/piler $(DESTDIR)$(sysconfdir) \ - $(DESTDIR)$(datarootdir)/piler $(DESTDIR)$(includedir)/piler $(DESTDIR)$(localstatedir)/lib/piler/data \ - $(DESTDIR)$(localstatedir)/spool + $(DESTDIR)$(datarootdir)/piler $(DESTDIR)$(includedir)/piler $(DESTDIR)$(localstatedir)/piler/store \ + $(DESTDIR)$(localstatedir)/piler/stat $(DESTDIR)$(localstatedir)/spool $(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/run/piler - $(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/lib/piler/stat + $(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/store + $(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/stat install-am: @@ -83,7 +84,7 @@ uninstall: $(DESTDIR)$(sysconfdir)/piler.conf rm -rf $(DESTDIR)$(libexecdir)/piler $(DESTDIR)$(includedir)/piler $(DESTDIR)$(datarootdir)/piler - @echo "I left the $(DESTDIR)$(localstatedir)/lib/piler directory, remove it for yourself" + @echo "I left the $(DESTDIR)$(localstatedir)/piler directory, remove it for yourself" all-am: Makefile diff --git a/etc/cron.jobs.in b/etc/cron.jobs.in index 37179c0f..e4787027 100644 --- a/etc/cron.jobs.in +++ b/etc/cron.jobs.in @@ -1,8 +1,29 @@ +######################################################### +### all the cron jobs you may need for piler ### +### be sure to review it and adjust it for your needs ### +######################################################### + +root's crontab: + +### optional: query postfix queue statistics +*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape > LOCALSTATEDIR/piler/stat/active+incoming +*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape -s > LOCALSTATEDIR/piler/stat/active+incoming-sender +*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape deferred > LOCALSTATEDIR/piler/stat/deferred +*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape -s deferred > LOCALSTATEDIR/piler/stat/deferred-sender -sphinx cronjob: +piler's crontab: -*/2 * * * * /usr/local/bin/indexer --quiet delta1 --rotate && sleep 2 && /usr/local/bin/indexer --quiet --merge main1 delta1 --merge-dst-range deleted 0 0 --rotate +### optional: show cpu usage +*/5 * * * * LC_ALL=C mpstat | tail -1 | awk '{print $11}' > /var/piler/stat/cpu.stat +### optional: regular AD sync +0 8 * * * /usr/bin/php LIBEXECDIR/piler/ldap_sync.php /srv/www/webui.yourdomain.com > LOCALSTATEDIR/piler/stat/adsync.stat + + + +sphinx's cronjob: + +*/5 * * * * /usr/local/bin/indexer --quiet delta1 --rotate && sleep 2 && /usr/local/bin/indexer --quiet --merge main1 delta1 --merge-dst-range deleted 0 0 --rotate */5 * * * * /usr/local/bin/indexer --quiet tag1 --rotate diff --git a/src/Makefile.in b/src/Makefile.in index 237f190b..68de64d4 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -76,6 +76,7 @@ install-piler: $(INSTALL) -m 0755 piler $(DESTDIR)$(sbindir) $(INSTALL) -m 0755 pilerconf $(DESTDIR)$(sbindir) + $(INSTALL) -m 0755 pilerget $(DESTDIR)$(bindir) clean: rm -f *.o *.a libpiler.so* piler pilerconf pilerget pilertest diff --git a/src/cfg.c b/src/cfg.c index e179c44f..2b8b829a 100644 --- a/src/cfg.c +++ b/src/cfg.c @@ -63,7 +63,6 @@ struct _parse_rule config_parse_rules[] = { "clamd_addr", "string", (void*) string_parser, offsetof(struct __config, clamd_addr), "", MAXVAL-1}, { "clamd_port", "integer", (void*) int_parser, offsetof(struct __config, clamd_port), "0", sizeof(int)}, { "clamd_socket", "string", (void*) string_parser, offsetof(struct __config, clamd_socket), CLAMD_SOCKET, MAXVAL-1}, - { "deferdir", "string", (void*) string_parser, offsetof(struct __config, deferdir), DEFER_DIR, MAXVAL-1}, { "hostid", "string", (void*) string_parser, offsetof(struct __config, hostid), HOSTID, MAXVAL-1}, { "iv", "string", (void*) string_parser, offsetof(struct __config, iv), "", MAXVAL-1}, { "listen_addr", "string", (void*) string_parser, offsetof(struct __config, listen_addr), "127.0.0.1", MAXVAL-1}, diff --git a/src/cfg.h b/src/cfg.h index 87966524..acb2b012 100644 --- a/src/cfg.h +++ b/src/cfg.h @@ -33,7 +33,6 @@ struct __config { char workdir[MAXVAL]; char queuedir[MAXVAL]; - char deferdir[MAXVAL]; int verbosity; char locale[MAXVAL]; diff --git a/src/config.h b/src/config.h index 6f913431..34b7dd64 100644 --- a/src/config.h +++ b/src/config.h @@ -11,16 +11,15 @@ #define PROGNAME "piler" -#define VERSION "0.1.10" +#define VERSION "0.1.11" -#define PROGINFO VERSION ", Janos SUTO \n\n" CONFIGURE_PARAMS "\n\nSend bugs/issues to https://jira.acts.hu:8443/\n" +#define PROGINFO VERSION ", Janos SUTO \n\n" CONFIGURE_PARAMS "\n" #define HOSTID "mailarchiver" #define CONFIG_FILE CONFDIR "/piler.conf" #define WORK_DIR DATADIR "/spool/piler/tmp" #define QUEUE_DIR DATADIR "/piler/store" -#define DEFER_DIR DATADIR "/spool/piler/deferred" #define CLAMD_SOCKET "/tmp/clamd" @@ -53,6 +52,7 @@ #define MEMCACHED_MSGS_RCVD MEMCACHED_CLAPF_PREFIX ":rcvd" #define MEMCACHED_MSGS_VIRUS MEMCACHED_CLAPF_PREFIX ":virus" #define MEMCACHED_MSGS_DUPLICATE MEMCACHED_CLAPF_PREFIX ":duplicate" +#define MEMCACHED_MSGS_IGNORE MEMCACHED_CLAPF_PREFIX ":ignore" #define LOG_PRIORITY LOG_INFO diff --git a/src/counters.c b/src/counters.c index 7a147028..5aa6bf54 100644 --- a/src/counters.c +++ b/src/counters.c @@ -16,7 +16,7 @@ struct __counters loadCounters(struct session_data *sdata, struct __config *cfg) bzero(&counters, sizeof(counters)); - snprintf(buf, SMALLBUFSIZE-1, "SELECT rcvd, virus, duplicate FROM %s", SQL_COUNTER_TABLE); + snprintf(buf, SMALLBUFSIZE-1, "SELECT rcvd, virus, duplicate, ignore FROM %s", SQL_COUNTER_TABLE); #ifdef NEED_MYSQL MYSQL_RES *res; @@ -30,6 +30,7 @@ struct __counters loadCounters(struct session_data *sdata, struct __config *cfg) counters.c_rcvd = strtoull(row[0], NULL, 10); counters.c_virus = strtoull(row[1], NULL, 10); counters.c_duplicate = strtoull(row[2], NULL, 10); + counters.c_ignore = strtoull(row[3], NULL, 10); } mysql_free_result(res); } @@ -59,17 +60,19 @@ void updateCounters(struct session_data *sdata, struct __data *data, struct __co if(counters->c_ham > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_HAM, strlen(MEMCACHED_MSGS_HAM), counters->c_ham, &mc); if(counters->c_virus > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), counters->c_virus, &mc); if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), counters->c_duplicate, &mc); + if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), counters->c_ignore, &mc); bzero(&c, sizeof(c)); - snprintf(buf, MAXBUFSIZE-1, "%s %s %s %s", MEMCACHED_MSGS_RCVD, MEMCACHED_MSGS_VIRUS, MEMCACHED_MSGS_DUPLICATE, MEMCACHED_COUNTERS_LAST_UPDATE); + snprintf(buf, MAXBUFSIZE-1, "%s %s %s %s %s", MEMCACHED_MSGS_RCVD, MEMCACHED_MSGS_VIRUS, MEMCACHED_MSGS_DUPLICATE, MEMCACHED_MSGS_IGNORE, MEMCACHED_COUNTERS_LAST_UPDATE); if(memcached_mget(&(data->memc), buf) == MEMCACHED_SUCCESS){ while((memcached_fetch_result(&(data->memc), &key[0], &buf[0], &flags))){ if(!strcmp(key, MEMCACHED_MSGS_RCVD)) c.c_rcvd = strtoull(buf, NULL, 10); else if(!strcmp(key, MEMCACHED_MSGS_VIRUS)) c.c_virus = strtoull(buf, NULL, 10); else if(!strcmp(key, MEMCACHED_MSGS_DUPLICATE)) c.c_duplicate = strtoull(buf, NULL, 10); + else if(!strcmp(key, MEMCACHED_MSGS_IGNORE)) c.c_ignore = strtoull(buf, NULL, 10); else if(!strcmp(key, MEMCACHED_COUNTERS_LAST_UPDATE)) mc = strtoull(buf, NULL, 10); } @@ -77,7 +80,7 @@ void updateCounters(struct session_data *sdata, struct __data *data, struct __co if(sdata->now - mc > cfg->memcached_to_db_interval && c.c_rcvd > 0 && c.c_rcvd >= rcvd){ snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_set(&(data->memc), MEMCACHED_COUNTERS_LAST_UPDATE, strlen(MEMCACHED_COUNTERS_LAST_UPDATE), buf, strlen(buf), 0, 0); - snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET rcvd=%llu, virus=%llu, duplicate=%llu", c.c_rcvd, c.c_virus, c.c_duplicate); + snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET rcvd=%llu, virus=%llu, duplicate=%llu, ignore=%llu", c.c_rcvd, c.c_virus, c.c_duplicate, c.c_ignore); //if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: update counters: %s", sdata->ttmpfile, buf); @@ -95,12 +98,13 @@ void updateCounters(struct session_data *sdata, struct __data *data, struct __co snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_virus + counters->c_virus); memcached_add(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), buf, strlen(buf), 0, 0); snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_rcvd + counters->c_rcvd); memcached_add(&(data->memc), MEMCACHED_MSGS_RCVD, strlen(MEMCACHED_MSGS_RCVD), buf, strlen(buf), 0, 0); snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_duplicate + counters->c_duplicate); memcached_add(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), buf, strlen(buf), 0, 0); + snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_ignore + counters->c_ignore); memcached_add(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), buf, strlen(buf), 0, 0); } } else { #endif - snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET rcvd=rcvd+%llu, virus=virus+%llu, duplicate=duplicate+%llu", SQL_COUNTER_TABLE, counters->c_rcvd, counters->c_virus, counters->c_duplicate); + snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET rcvd=rcvd+%llu, virus=virus+%llu, duplicate=duplicate+%llu, ignore=ignore+%llu", SQL_COUNTER_TABLE, counters->c_rcvd, counters->c_virus, counters->c_duplicate, counters->c_ignore); //if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: update counters: %s", sdata->ttmpfile, buf); diff --git a/src/defs.h b/src/defs.h index cbaf6c4c..1a461b70 100644 --- a/src/defs.h +++ b/src/defs.h @@ -203,6 +203,7 @@ struct __counters { unsigned long long c_rcvd; unsigned long long c_virus; unsigned long long c_duplicate; + unsigned long long c_ignore; }; #endif /* _DEFS_H */ diff --git a/src/dirs.c b/src/dirs.c index 1d89bc0c..ee88558c 100644 --- a/src/dirs.c +++ b/src/dirs.c @@ -35,14 +35,6 @@ void check_and_create_directories(struct __config *cfg, uid_t uid, gid_t gid){ } createdir(cfg->queuedir, uid, gid, 0700); - p = strrchr(cfg->deferdir, '/'); - if(p){ - *p = '\0'; - createdir(cfg->deferdir, uid, gid, 0755); - *p = '/'; - } - createdir(cfg->deferdir, uid, gid, 0700); - p = strrchr(cfg->sqlite3, '/'); if(p){ *p = '\0'; diff --git a/src/parser.c b/src/parser.c index 3a850e61..1169acc4 100644 --- a/src/parser.c +++ b/src/parser.c @@ -379,10 +379,10 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, stru len = strlen(puf); - if(state->message_state == MSG_FROM && strchr(puf, '@') && state->is_1st_header == 1 && state->b_from[0] == '\0' && strlen(state->b_from) < SMALLBUFSIZE-len-1) + if(state->message_state == MSG_FROM && strchr(puf, '@') && strlen(puf) > 5 && state->is_1st_header == 1 && state->b_from[0] == '\0' && strlen(state->b_from) < SMALLBUFSIZE-len-1) memcpy(&(state->b_from[strlen(state->b_from)]), puf, len); - else if((state->message_state == MSG_TO || state->message_state == MSG_CC) && state->is_1st_header == 1 && strchr(puf, '@') && strlen(state->b_to) < SMALLBUFSIZE-len-1){ + else if((state->message_state == MSG_TO || state->message_state == MSG_CC) && state->is_1st_header == 1 && strchr(puf, '@') && strlen(puf) > 5 && strlen(state->b_to) < SMALLBUFSIZE-len-1){ if(is_string_on_list(state->rcpt, puf) == 0){ append_list(&(state->rcpt), puf); diff --git a/src/pilerget.c b/src/pilerget.c index 8cedee67..114cceb5 100644 --- a/src/pilerget.c +++ b/src/pilerget.c @@ -113,30 +113,18 @@ int inf(unsigned char *in, int len, FILE *dest){ } -int main(int argc, char **argv){ +int retrieve_file_from_archive(char *filename, struct __config *cfg){ int rc, n, olen, tlen, len; unsigned char inbuf[BIGBUFSIZE]; - struct __config cfg; struct stat st; - cfg = read_config(configfile); - - if(read_key(&cfg)){ - printf("%s\n", ERR_READING_KEY); - return 1; - } + if(filename == NULL) return 1; - if(argc != 2){ - printf("usage: $0 \n"); - return 1; - } - - - fd = open(argv[1], O_RDONLY); + fd = open(filename, O_RDONLY); if(fd == -1){ - printf("error reading file: %s\n", argv[1]); + printf("error reading file: %s\n", filename); return 1; } @@ -149,7 +137,7 @@ int main(int argc, char **argv){ EVP_CIPHER_CTX_init(&ctx); - EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg.key, cfg.iv); + EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv); len = st.st_size+EVP_MAX_BLOCK_LENGTH; @@ -192,3 +180,27 @@ int main(int argc, char **argv){ return 0; } + + +int main(int argc, char **argv){ + struct __config cfg; + + + cfg = read_config(configfile); + + if(read_key(&cfg)){ + printf("%s\n", ERR_READING_KEY); + return 1; + } + + + if(argc != 2){ + printf("usage: %s \n", argv[0]); + return 1; + } + + + retrieve_file_from_archive(argv[1], &cfg); + + return 0; +} diff --git a/src/session.c b/src/session.c index 9cbc1b02..2b5476e3 100644 --- a/src/session.c +++ b/src/session.c @@ -201,6 +201,7 @@ void handle_smtp_session(int new_sd, struct __data *data, struct __config *cfg){ if(arule){ syslog(LOG_PRIORITY, "%s: discarding message by archiving policy: *%s*", sdata.ttmpfile, arule); inj = OK; + counters.c_ignore++; } else { inj = processMessage(&sdata, &sstate, cfg); diff --git a/util/db-mysql.sql b/util/db-mysql.sql index ec635cde..e230db43 100644 --- a/util/db-mysql.sql +++ b/util/db-mysql.sql @@ -122,6 +122,7 @@ create table if not exists `counter` ( `rcvd` bigint unsigned default 0, `virus` bigint unsigned default 0, `duplicate` bigint unsigned default 0 + `ignore` bigint unsigned default 0 ) Engine=InnoDB; insert into `counter` values(0, 0, 0); @@ -147,3 +148,51 @@ create table if not exists `user_settings` ( create index `user_settings_idx` on `user_settings`(`username`); + + +create table if not exists `user` ( + `uid` int unsigned not null primary key, + `gid` int unsigned not null, + `username` char(64) not null unique, + `realname` char(64) default null, + `password` char(48) default null, + `domain` char(64) default null, + `dn` char(255) default '*', + `policy_group` int(4) default 0, + `isadmin` tinyint default 0 +) Engine=InnoDB; + +insert into `user` (`uid`, `gid`, `username`, `realname`, `password`, `policy_group`, `isadmin`, `domain`) values (0, 0, 'admin', 'built-in piler admin', '$1$PItc7d$zsUgON3JRrbdGS11t9JQW1', 0, 1, 'local'); + +create table if not exists `email` ( + `uid` int unsigned not null, + `email` char(128) not null primary key +) ENGINE=InnoDB; + +insert into `email` (`uid`, `email`) values(0, 'admin@local'); + + +create table if not exists `email_groups` ( + `uid` int unsigned not null, + `gid` int unsigned not null, + unique key `uid` (`uid`,`gid`), + key `email_groups_idx` (`uid`,`gid`) +) ENGINE=InnoDB; + + +create table if not exists `remote` ( + `remotedomain` char(64) not null primary key, + `remotehost` char(64) not null, + `basedn` char(64) not null, + `binddn` char(64) not null, + `sitedescription` char(64) default null +) ENGINE=InnoDB; + + +create table if not exists `domain` ( + `domain` char(64) not null primary key, + `mapped` char(64) not null +) ENGINE=InnoDB; + +insert into `domain` (`domain`, `mapped`) values('local', 'local'); + diff --git a/util/ldap_sync.php b/util/ldap_sync.php new file mode 100644 index 00000000..66adf2e7 --- /dev/null +++ b/util/ldap_sync.php @@ -0,0 +1,77 @@ +addServer($m[0], $m[1]); + } + + Registry::set('memcache', $memcache); +} + + +$db = new DB(DB_DRIVER, DB_HOSTNAME, DB_USERNAME, DB_PASSWORD, DB_DATABASE, DB_PREFIX); +Registry::set('db', $db); + +$loader->model('user/user'); +$loader->model('user/import'); + +$import = new ModelUserImport(); + +$_SESSION['username'] = 'cli-admin'; +$_SERVER['REMOTE_ADDR'] = '127.0.0.1'; + +$totalusers = 0; +$totalnewusers = 0; +$totaldeletedusers = 0; + +extract($language->data); + + +foreach ($cfg as $ldap_params) { + $users = $import->model_user_import->queryRemoteUsers($ldap_params, $ldap_params['domain']); + $rc = $import->model_user_import->fillRemoteTable($ldap_params, $ldap_params['domain']); + + $totalusers += count($users); + + list($newusers, $deletedusers) = $import->model_user_import->processUsers($users, $ldap_params); + list($a1, $a2) = $import->model_user_import->processUsers($users, $ldap_params); + + $totalnewusers += $newusers; + $totaldeletedusers += $deletedusers; + + if($trash_passwords == 1) { + $import->model_user_import->trashPassword($users); + } + +} + +$total_emails_in_database = $import->model_user_import->count_email_addresses(); + +print date(LOG_DATE_FORMAT); ?>, ///