This commit is contained in:
SJ 2011-12-27 20:51:56 +01:00
parent da99ff359b
commit b06fae4031
14 changed files with 199 additions and 42 deletions

View File

@ -67,11 +67,12 @@ $(RECURSIVE_TARGETS):
installdirs: mkinstalldirs
$(srcdir)/mkinstalldirs \
$(DESTDIR)$(bindir) $(DESTDIR)$(sbindir) $(DESTDIR)$(libdir) $(DESTDIR)$(libexecdir)/piler $(DESTDIR)$(sysconfdir) \
$(DESTDIR)$(datarootdir)/piler $(DESTDIR)$(includedir)/piler $(DESTDIR)$(localstatedir)/lib/piler/data \
$(DESTDIR)$(localstatedir)/spool
$(DESTDIR)$(datarootdir)/piler $(DESTDIR)$(includedir)/piler $(DESTDIR)$(localstatedir)/piler/store \
$(DESTDIR)$(localstatedir)/piler/stat $(DESTDIR)$(localstatedir)/spool
$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/run/piler
$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/lib/piler/stat
$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/store
$(INSTALL) -d -m 0755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) $(DESTDIR)$(localstatedir)/piler/stat
install-am:
@ -83,7 +84,7 @@ uninstall:
$(DESTDIR)$(sysconfdir)/piler.conf
rm -rf $(DESTDIR)$(libexecdir)/piler $(DESTDIR)$(includedir)/piler $(DESTDIR)$(datarootdir)/piler
@echo "I left the $(DESTDIR)$(localstatedir)/lib/piler directory, remove it for yourself"
@echo "I left the $(DESTDIR)$(localstatedir)/piler directory, remove it for yourself"
all-am: Makefile

View File

@ -1,8 +1,29 @@
#########################################################
### all the cron jobs you may need for piler ###
### be sure to review it and adjust it for your needs ###
#########################################################
root's crontab:
### optional: query postfix queue statistics
*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape > LOCALSTATEDIR/piler/stat/active+incoming
*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape -s > LOCALSTATEDIR/piler/stat/active+incoming-sender
*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape deferred > LOCALSTATEDIR/piler/stat/deferred
*/5 * * * * PATH=$PATH:/usr/sbin:/usr/local/sbin /usr/sbin/qshape -s deferred > LOCALSTATEDIR/piler/stat/deferred-sender
sphinx cronjob:
piler's crontab:
*/2 * * * * /usr/local/bin/indexer --quiet delta1 --rotate && sleep 2 && /usr/local/bin/indexer --quiet --merge main1 delta1 --merge-dst-range deleted 0 0 --rotate
### optional: show cpu usage
*/5 * * * * LC_ALL=C mpstat | tail -1 | awk '{print $11}' > /var/piler/stat/cpu.stat
### optional: regular AD sync
0 8 * * * /usr/bin/php LIBEXECDIR/piler/ldap_sync.php /srv/www/webui.yourdomain.com > LOCALSTATEDIR/piler/stat/adsync.stat
sphinx's cronjob:
*/5 * * * * /usr/local/bin/indexer --quiet delta1 --rotate && sleep 2 && /usr/local/bin/indexer --quiet --merge main1 delta1 --merge-dst-range deleted 0 0 --rotate
*/5 * * * * /usr/local/bin/indexer --quiet tag1 --rotate

View File

@ -76,6 +76,7 @@ install-piler:
$(INSTALL) -m 0755 piler $(DESTDIR)$(sbindir)
$(INSTALL) -m 0755 pilerconf $(DESTDIR)$(sbindir)
$(INSTALL) -m 0755 pilerget $(DESTDIR)$(bindir)
clean:
rm -f *.o *.a libpiler.so* piler pilerconf pilerget pilertest

View File

@ -63,7 +63,6 @@ struct _parse_rule config_parse_rules[] =
{ "clamd_addr", "string", (void*) string_parser, offsetof(struct __config, clamd_addr), "", MAXVAL-1},
{ "clamd_port", "integer", (void*) int_parser, offsetof(struct __config, clamd_port), "0", sizeof(int)},
{ "clamd_socket", "string", (void*) string_parser, offsetof(struct __config, clamd_socket), CLAMD_SOCKET, MAXVAL-1},
{ "deferdir", "string", (void*) string_parser, offsetof(struct __config, deferdir), DEFER_DIR, MAXVAL-1},
{ "hostid", "string", (void*) string_parser, offsetof(struct __config, hostid), HOSTID, MAXVAL-1},
{ "iv", "string", (void*) string_parser, offsetof(struct __config, iv), "", MAXVAL-1},
{ "listen_addr", "string", (void*) string_parser, offsetof(struct __config, listen_addr), "127.0.0.1", MAXVAL-1},

View File

@ -33,7 +33,6 @@ struct __config {
char workdir[MAXVAL];
char queuedir[MAXVAL];
char deferdir[MAXVAL];
int verbosity;
char locale[MAXVAL];

View File

@ -11,16 +11,15 @@
#define PROGNAME "piler"
#define VERSION "0.1.10"
#define VERSION "0.1.11"
#define PROGINFO VERSION ", Janos SUTO <sj@acts.hu>\n\n" CONFIGURE_PARAMS "\n\nSend bugs/issues to https://jira.acts.hu:8443/\n"
#define PROGINFO VERSION ", Janos SUTO <sj@acts.hu>\n\n" CONFIGURE_PARAMS "\n"
#define HOSTID "mailarchiver"
#define CONFIG_FILE CONFDIR "/piler.conf"
#define WORK_DIR DATADIR "/spool/piler/tmp"
#define QUEUE_DIR DATADIR "/piler/store"
#define DEFER_DIR DATADIR "/spool/piler/deferred"
#define CLAMD_SOCKET "/tmp/clamd"
@ -53,6 +52,7 @@
#define MEMCACHED_MSGS_RCVD MEMCACHED_CLAPF_PREFIX ":rcvd"
#define MEMCACHED_MSGS_VIRUS MEMCACHED_CLAPF_PREFIX ":virus"
#define MEMCACHED_MSGS_DUPLICATE MEMCACHED_CLAPF_PREFIX ":duplicate"
#define MEMCACHED_MSGS_IGNORE MEMCACHED_CLAPF_PREFIX ":ignore"
#define LOG_PRIORITY LOG_INFO

View File

@ -16,7 +16,7 @@ struct __counters loadCounters(struct session_data *sdata, struct __config *cfg)
bzero(&counters, sizeof(counters));
snprintf(buf, SMALLBUFSIZE-1, "SELECT rcvd, virus, duplicate FROM %s", SQL_COUNTER_TABLE);
snprintf(buf, SMALLBUFSIZE-1, "SELECT rcvd, virus, duplicate, ignore FROM %s", SQL_COUNTER_TABLE);
#ifdef NEED_MYSQL
MYSQL_RES *res;
@ -30,6 +30,7 @@ struct __counters loadCounters(struct session_data *sdata, struct __config *cfg)
counters.c_rcvd = strtoull(row[0], NULL, 10);
counters.c_virus = strtoull(row[1], NULL, 10);
counters.c_duplicate = strtoull(row[2], NULL, 10);
counters.c_ignore = strtoull(row[3], NULL, 10);
}
mysql_free_result(res);
}
@ -59,17 +60,19 @@ void updateCounters(struct session_data *sdata, struct __data *data, struct __co
if(counters->c_ham > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_HAM, strlen(MEMCACHED_MSGS_HAM), counters->c_ham, &mc);
if(counters->c_virus > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), counters->c_virus, &mc);
if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), counters->c_duplicate, &mc);
if(counters->c_duplicate > 0) memcached_increment(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), counters->c_ignore, &mc);
bzero(&c, sizeof(c));
snprintf(buf, MAXBUFSIZE-1, "%s %s %s %s", MEMCACHED_MSGS_RCVD, MEMCACHED_MSGS_VIRUS, MEMCACHED_MSGS_DUPLICATE, MEMCACHED_COUNTERS_LAST_UPDATE);
snprintf(buf, MAXBUFSIZE-1, "%s %s %s %s %s", MEMCACHED_MSGS_RCVD, MEMCACHED_MSGS_VIRUS, MEMCACHED_MSGS_DUPLICATE, MEMCACHED_MSGS_IGNORE, MEMCACHED_COUNTERS_LAST_UPDATE);
if(memcached_mget(&(data->memc), buf) == MEMCACHED_SUCCESS){
while((memcached_fetch_result(&(data->memc), &key[0], &buf[0], &flags))){
if(!strcmp(key, MEMCACHED_MSGS_RCVD)) c.c_rcvd = strtoull(buf, NULL, 10);
else if(!strcmp(key, MEMCACHED_MSGS_VIRUS)) c.c_virus = strtoull(buf, NULL, 10);
else if(!strcmp(key, MEMCACHED_MSGS_DUPLICATE)) c.c_duplicate = strtoull(buf, NULL, 10);
else if(!strcmp(key, MEMCACHED_MSGS_IGNORE)) c.c_ignore = strtoull(buf, NULL, 10);
else if(!strcmp(key, MEMCACHED_COUNTERS_LAST_UPDATE)) mc = strtoull(buf, NULL, 10);
}
@ -77,7 +80,7 @@ void updateCounters(struct session_data *sdata, struct __data *data, struct __co
if(sdata->now - mc > cfg->memcached_to_db_interval && c.c_rcvd > 0 && c.c_rcvd >= rcvd){
snprintf(buf, SMALLBUFSIZE-1, "%ld", sdata->now); memcached_set(&(data->memc), MEMCACHED_COUNTERS_LAST_UPDATE, strlen(MEMCACHED_COUNTERS_LAST_UPDATE), buf, strlen(buf), 0, 0);
snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET rcvd=%llu, virus=%llu, duplicate=%llu", c.c_rcvd, c.c_virus, c.c_duplicate);
snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET rcvd=%llu, virus=%llu, duplicate=%llu, ignore=%llu", c.c_rcvd, c.c_virus, c.c_duplicate, c.c_ignore);
//if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: update counters: %s", sdata->ttmpfile, buf);
@ -95,12 +98,13 @@ void updateCounters(struct session_data *sdata, struct __data *data, struct __co
snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_virus + counters->c_virus); memcached_add(&(data->memc), MEMCACHED_MSGS_VIRUS, strlen(MEMCACHED_MSGS_VIRUS), buf, strlen(buf), 0, 0);
snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_rcvd + counters->c_rcvd); memcached_add(&(data->memc), MEMCACHED_MSGS_RCVD, strlen(MEMCACHED_MSGS_RCVD), buf, strlen(buf), 0, 0);
snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_duplicate + counters->c_duplicate); memcached_add(&(data->memc), MEMCACHED_MSGS_DUPLICATE, strlen(MEMCACHED_MSGS_DUPLICATE), buf, strlen(buf), 0, 0);
snprintf(buf, SMALLBUFSIZE-1, "%llu", c.c_ignore + counters->c_ignore); memcached_add(&(data->memc), MEMCACHED_MSGS_IGNORE, strlen(MEMCACHED_MSGS_IGNORE), buf, strlen(buf), 0, 0);
}
}
else {
#endif
snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET rcvd=rcvd+%llu, virus=virus+%llu, duplicate=duplicate+%llu", SQL_COUNTER_TABLE, counters->c_rcvd, counters->c_virus, counters->c_duplicate);
snprintf(buf, SMALLBUFSIZE-1, "UPDATE `%s` SET rcvd=rcvd+%llu, virus=virus+%llu, duplicate=duplicate+%llu, ignore=ignore+%llu", SQL_COUNTER_TABLE, counters->c_rcvd, counters->c_virus, counters->c_duplicate, counters->c_ignore);
//if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: update counters: %s", sdata->ttmpfile, buf);

View File

@ -203,6 +203,7 @@ struct __counters {
unsigned long long c_rcvd;
unsigned long long c_virus;
unsigned long long c_duplicate;
unsigned long long c_ignore;
};
#endif /* _DEFS_H */

View File

@ -35,14 +35,6 @@ void check_and_create_directories(struct __config *cfg, uid_t uid, gid_t gid){
}
createdir(cfg->queuedir, uid, gid, 0700);
p = strrchr(cfg->deferdir, '/');
if(p){
*p = '\0';
createdir(cfg->deferdir, uid, gid, 0755);
*p = '/';
}
createdir(cfg->deferdir, uid, gid, 0700);
p = strrchr(cfg->sqlite3, '/');
if(p){
*p = '\0';

View File

@ -379,10 +379,10 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, stru
len = strlen(puf);
if(state->message_state == MSG_FROM && strchr(puf, '@') && state->is_1st_header == 1 && state->b_from[0] == '\0' && strlen(state->b_from) < SMALLBUFSIZE-len-1)
if(state->message_state == MSG_FROM && strchr(puf, '@') && strlen(puf) > 5 && state->is_1st_header == 1 && state->b_from[0] == '\0' && strlen(state->b_from) < SMALLBUFSIZE-len-1)
memcpy(&(state->b_from[strlen(state->b_from)]), puf, len);
else if((state->message_state == MSG_TO || state->message_state == MSG_CC) && state->is_1st_header == 1 && strchr(puf, '@') && strlen(state->b_to) < SMALLBUFSIZE-len-1){
else if((state->message_state == MSG_TO || state->message_state == MSG_CC) && state->is_1st_header == 1 && strchr(puf, '@') && strlen(puf) > 5 && strlen(state->b_to) < SMALLBUFSIZE-len-1){
if(is_string_on_list(state->rcpt, puf) == 0){
append_list(&(state->rcpt), puf);

View File

@ -113,30 +113,18 @@ int inf(unsigned char *in, int len, FILE *dest){
}
int main(int argc, char **argv){
int retrieve_file_from_archive(char *filename, struct __config *cfg){
int rc, n, olen, tlen, len;
unsigned char inbuf[BIGBUFSIZE];
struct __config cfg;
struct stat st;
cfg = read_config(configfile);
if(read_key(&cfg)){
printf("%s\n", ERR_READING_KEY);
return 1;
}
if(filename == NULL) return 1;
if(argc != 2){
printf("usage: $0 <encrypted file>\n");
return 1;
}
fd = open(argv[1], O_RDONLY);
fd = open(filename, O_RDONLY);
if(fd == -1){
printf("error reading file: %s\n", argv[1]);
printf("error reading file: %s\n", filename);
return 1;
}
@ -149,7 +137,7 @@ int main(int argc, char **argv){
EVP_CIPHER_CTX_init(&ctx);
EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg.key, cfg.iv);
EVP_DecryptInit_ex(&ctx, EVP_bf_cbc(), NULL, cfg->key, cfg->iv);
len = st.st_size+EVP_MAX_BLOCK_LENGTH;
@ -192,3 +180,27 @@ int main(int argc, char **argv){
return 0;
}
int main(int argc, char **argv){
struct __config cfg;
cfg = read_config(configfile);
if(read_key(&cfg)){
printf("%s\n", ERR_READING_KEY);
return 1;
}
if(argc != 2){
printf("usage: %s <encrypted file>\n", argv[0]);
return 1;
}
retrieve_file_from_archive(argv[1], &cfg);
return 0;
}

View File

@ -201,6 +201,7 @@ void handle_smtp_session(int new_sd, struct __data *data, struct __config *cfg){
if(arule){
syslog(LOG_PRIORITY, "%s: discarding message by archiving policy: *%s*", sdata.ttmpfile, arule);
inj = OK;
counters.c_ignore++;
}
else {
inj = processMessage(&sdata, &sstate, cfg);

View File

@ -122,6 +122,7 @@ create table if not exists `counter` (
`rcvd` bigint unsigned default 0,
`virus` bigint unsigned default 0,
`duplicate` bigint unsigned default 0
`ignore` bigint unsigned default 0
) Engine=InnoDB;
insert into `counter` values(0, 0, 0);
@ -147,3 +148,51 @@ create table if not exists `user_settings` (
create index `user_settings_idx` on `user_settings`(`username`);
create table if not exists `user` (
`uid` int unsigned not null primary key,
`gid` int unsigned not null,
`username` char(64) not null unique,
`realname` char(64) default null,
`password` char(48) default null,
`domain` char(64) default null,
`dn` char(255) default '*',
`policy_group` int(4) default 0,
`isadmin` tinyint default 0
) Engine=InnoDB;
insert into `user` (`uid`, `gid`, `username`, `realname`, `password`, `policy_group`, `isadmin`, `domain`) values (0, 0, 'admin', 'built-in piler admin', '$1$PItc7d$zsUgON3JRrbdGS11t9JQW1', 0, 1, 'local');
create table if not exists `email` (
`uid` int unsigned not null,
`email` char(128) not null primary key
) ENGINE=InnoDB;
insert into `email` (`uid`, `email`) values(0, 'admin@local');
create table if not exists `email_groups` (
`uid` int unsigned not null,
`gid` int unsigned not null,
unique key `uid` (`uid`,`gid`),
key `email_groups_idx` (`uid`,`gid`)
) ENGINE=InnoDB;
create table if not exists `remote` (
`remotedomain` char(64) not null primary key,
`remotehost` char(64) not null,
`basedn` char(64) not null,
`binddn` char(64) not null,
`sitedescription` char(64) default null
) ENGINE=InnoDB;
create table if not exists `domain` (
`domain` char(64) not null primary key,
`mapped` char(64) not null
) ENGINE=InnoDB;
insert into `domain` (`domain`, `mapped`) values('local', 'local');

77
util/ldap_sync.php Normal file
View File

@ -0,0 +1,77 @@
<?php
$webuidir = "";
if(isset($_SERVER['argv'][1])) { $webuidir = $_SERVER['argv'][1]; }
require_once($webuidir . "/config.php");
require(DIR_SYSTEM . "/startup.php");
require(DIR_SYSTEM . "/ldap.php");
$trash_passwords = 0;
$cfg = read_konfig(LDAP_IMPORT_CONFIG_FILE);
foreach ($_SERVER['argv'] as $argv) {
if($argv == "--trash-passwords") {
$trash_passwords = 1;
}
}
$loader = new Loader();
$language = new Language();
Registry::set('language', $language);
if(MEMCACHED_ENABLED) {
$memcache = new Memcache();
foreach ($memcached_servers as $m){
$memcache->addServer($m[0], $m[1]);
}
Registry::set('memcache', $memcache);
}
$db = new DB(DB_DRIVER, DB_HOSTNAME, DB_USERNAME, DB_PASSWORD, DB_DATABASE, DB_PREFIX);
Registry::set('db', $db);
$loader->model('user/user');
$loader->model('user/import');
$import = new ModelUserImport();
$_SESSION['username'] = 'cli-admin';
$_SERVER['REMOTE_ADDR'] = '127.0.0.1';
$totalusers = 0;
$totalnewusers = 0;
$totaldeletedusers = 0;
extract($language->data);
foreach ($cfg as $ldap_params) {
$users = $import->model_user_import->queryRemoteUsers($ldap_params, $ldap_params['domain']);
$rc = $import->model_user_import->fillRemoteTable($ldap_params, $ldap_params['domain']);
$totalusers += count($users);
list($newusers, $deletedusers) = $import->model_user_import->processUsers($users, $ldap_params);
list($a1, $a2) = $import->model_user_import->processUsers($users, $ldap_params);
$totalnewusers += $newusers;
$totaldeletedusers += $deletedusers;
if($trash_passwords == 1) {
$import->model_user_import->trashPassword($users);
}
}
$total_emails_in_database = $import->model_user_import->count_email_addresses();
print date(LOG_DATE_FORMAT); ?>, <?php print $totalusers; ?>/<?php print $totalnewusers; ?>/<?php print $totaldeletedusers; ?>/<?php print $total_emails_in_database; ?>