bashclub/piler
1
0
Fork 0
mirror of https://bitbucket.org/jsuto/piler.git synced 2025-06-13 12:07:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

added retention rules to piler core

Browse Source
This commit is contained in:
SJ
2012-02-19 22:59:47 +01:00
parent e51af89e5d
commit 020756fa1c
24 changed files with 181 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/cfg.c
View File

@ -63,6 +63,7 @@ struct _parse_rule config_parse_rules[] =
{ "clamd_addr", "string", (void*) string_parser, offsetof(struct __config, clamd_addr), "", MAXVAL-1},
{ "clamd_port", "integer", (void*) int_parser, offsetof(struct __config, clamd_port), "0", sizeof(int)},
{ "clamd_socket", "string", (void*) string_parser, offsetof(struct __config, clamd_socket), CLAMD_SOCKET, MAXVAL-1},
{ "default_retention_days", "integer", (void*) int_parser, offsetof(struct __config, default_retention_days), "2557", sizeof(int)},
{ "hostid", "string", (void*) string_parser, offsetof(struct __config, hostid), HOSTID, MAXVAL-1},
{ "iv", "string", (void*) string_parser, offsetof(struct __config, iv), "", MAXVAL-1},
{ "listen_addr", "string", (void*) string_parser, offsetof(struct __config, listen_addr), "127.0.0.1", MAXVAL-1},

2
src/cfg.h
View File

@ -47,6 +47,8 @@ struct __config {
char spam_header_line[MAXVAL];
int default_retention_days;
// mysql stuff
char mysqlhost[MAXVAL];

1
src/config.h
View File

@ -78,6 +78,7 @@
#define SQL_ATTACHMENT_TABLE "attachment"
#define SQL_RECIPIENT_TABLE "rcpt"
#define SQL_ARCHIVING_RULE_TABLE "archiving_rule"
#define SQL_RETENTION_RULE_TABLE "retention_rule"
#define SQL_COUNTER_TABLE "counter"
#define SQL_MESSAGES_VIEW "messages"

9
src/defs.h
View File

@ -93,13 +93,17 @@ struct rule {
regex_t subject;
regex_t attachment_type;
#endif
int spam;
int size;
char _size[4];
int attachment_size;
char _attachment_size[4];
int days;
char *rulestr;
char compiled;
struct rule *r;
};
@ -165,7 +169,7 @@ struct session_data {
float __acquire, __parsed, __av, __store, __compress, __encrypt;
char bodydigest[2*DIGEST_LENGTH+1];
char digest[2*DIGEST_LENGTH+1];
time_t now, sent;
time_t now, sent, retained;
#ifdef NEED_MYSQL
MYSQL mysql;
#endif
@ -217,7 +221,8 @@ struct memcached_server {
struct __data {
#ifdef HAVE_TRE
struct rule *rules;
struct rule *archiving_rules;
struct rule *retention_rules;
#endif
#ifdef HAVE_MEMCACHED

6
src/message.c
View File

@ -278,7 +278,7 @@ int store_meta_data(struct session_data *sdata, struct _state *state, struct __c
subj = state->b_subject;
if(*subj == ' ') subj++;
snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%d%d%d%d%s%s%s", id, subj, state->b_from, state->message_id, sdata->now, sdata->sent, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, state->b_from, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
digest_string(s, &vcode[0]);
@ -286,7 +286,7 @@ int store_meta_data(struct session_data *sdata, struct _state *state, struct __c
if(strlen(state->reference) > 10) digest_string(state->reference, &ref[0]);
snprintf(s, MAXBUFSIZE-1, "INSERT INTO %s (`from`,`fromdomain`,`subject`,`spam`,`arrived`,`sent`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`reference`,`digest`,`bodydigest`,`vcode`) VALUES(?,?,?,%d,%ld,%ld,%d,%d,%d,%d,'%s',?,'%s','%s','%s','%s')", SQL_METADATA_TABLE, sdata->spam_message, sdata->now, sdata->sent, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, ref, sdata->digest, sdata->bodydigest, vcode);
snprintf(s, MAXBUFSIZE-1, "INSERT INTO %s (`from`,`fromdomain`,`subject`,`spam`,`arrived`,`sent`,`retained`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`reference`,`digest`,`bodydigest`,`vcode`) VALUES(?,?,?,%d,%ld,%ld,%ld,%d,%d,%d,%d,'%s',?,'%s','%s','%s','%s')", SQL_METADATA_TABLE, sdata->spam_message, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, ref, sdata->digest, sdata->bodydigest, vcode);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: meta sql: *%s*", sdata->ttmpfile, s);
@ -412,6 +412,8 @@ int process_message(struct session_data *sdata, struct _state *state, struct __d
}
sdata->retained += query_retain_period(data->retention_rules, state, sdata->tot_len, sdata->spam_message, cfg);
rc = store_meta_data(sdata, state, cfg);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored metadata, rc=%d", sdata->ttmpfile, rc);
if(rc == ERR_EXISTS) return ERR_EXISTS;

2
src/misc.c
View File

@ -405,7 +405,7 @@ void init_session_data(struct session_data *sdata){
for(i=0; i<MAX_RCPT_TO; i++) memset(sdata->rcptto[i], 0, SMALLBUFSIZE);
time(&(sdata->now));
sdata->sent = sdata->now;
sdata->sent = sdata->retained = sdata->now;
}

16
src/piler.c
View File

@ -238,7 +238,8 @@ void clean_exit(){
kill_children(SIGTERM);
free_rule(data.rules);
free_rule(data.archiving_rules);
free_rule(data.retention_rules);
syslog(LOG_PRIORITY, "%s has been terminated", PROGNAME);
@ -282,8 +283,11 @@ void initialise_configuration(){
setlocale(LC_CTYPE, cfg.locale);
free_rule(data.rules);
data.rules = NULL;
free_rule(data.archiving_rules);
free_rule(data.retention_rules);
data.archiving_rules = NULL;
data.retention_rules = NULL;
mysql_init(&(sdata.mysql));
mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout);
@ -292,7 +296,8 @@ void initialise_configuration(){
return;
}
load_archiving_rules(&sdata, &(data.rules));
load_rules(&sdata, &(data.archiving_rules), SQL_ARCHIVING_RULE_TABLE);
load_rules(&sdata, &(data.retention_rules), SQL_RETENTION_RULE_TABLE);
mysql_close(&(sdata.mysql));
@ -334,7 +339,8 @@ int main(int argc, char **argv){
(void) openlog(PROGNAME, LOG_PID, LOG_MAIL);
data.rules = NULL;
data.archiving_rules = NULL;
data.retention_rules = NULL;
initialise_configuration();

5
src/pilerexport.c
View File

@ -228,14 +228,15 @@ int main(int argc, char **argv){
{"to", required_argument, 0, 'r' },
{"start-date", required_argument, 0, 'a' },
{"stop-date", required_argument, 0, 'b' },
{"id", required_argument, 0, 'i' },
{0,0,0,0}
};
int option_index = 0;
c = getopt_long(argc, argv, "c:s:S:f:r:a:b:Adhv?", long_options, &option_index);
c = getopt_long(argc, argv, "c:s:S:f:r:a:b:i:Adhv?", long_options, &option_index);
#else
c = getopt(argc, argv, "c:s:S:f:r:a:b:Adhv?");
c = getopt(argc, argv, "c:s:S:f:r:a:b:i:Adhv?");
#endif
if(c == -1) break;

15
src/pilerimport.c
View File

@ -50,7 +50,11 @@ int import_message(char *filename, struct session_data *sdata, struct __data *da
if(sdata->sent > sdata->now) sdata->sent = sdata->now;
if(sdata->sent == -1) sdata->sent = 0;
rule = check_againt_ruleset(data->rules, &state, st.st_size);
/* fat chances that you won't import emails before 1990.01.01 */
if(sdata->sent > 631148400) sdata->retained = sdata->sent;
rule = check_againt_ruleset(data->archiving_rules, &state, st.st_size, sdata->spam_message);
if(rule){
printf("discarding %s by archiving policy: %s\n", filename, rule);
@ -230,9 +234,11 @@ int main(int argc, char **argv){
setlocale(LC_CTYPE, cfg.locale);
data.rules = NULL;
data.archiving_rules = NULL;
data.retention_rules = NULL;
load_archiving_rules(&sdata, &(data.rules));
load_rules(&sdata, &(data.archiving_rules), SQL_ARCHIVING_RULE_TABLE);
load_rules(&sdata, &(data.retention_rules), SQL_RETENTION_RULE_TABLE);
@ -242,7 +248,8 @@ int main(int argc, char **argv){
free_rule(data.rules);
free_rule(data.archiving_rules);
free_rule(data.retention_rules);
mysql_close(&(sdata.mysql));

58
src/rules.c
View File

@ -10,18 +10,18 @@
#include "rules.h"
void load_archiving_rules(struct session_data *sdata, struct rule **rules){
void load_rules(struct session_data *sdata, struct rule **rules, char *table){
char s[SMALLBUFSIZE];
MYSQL_RES *res;
MYSQL_ROW row;
snprintf(s, sizeof(s)-1, "SELECT `from`, `to`, `subject`, `_size`, `size`, `attachment_type`, `_attachment_size`, `attachment_size` FROM `%s`", SQL_ARCHIVING_RULE_TABLE);
snprintf(s, sizeof(s)-1, "SELECT `from`, `to`, `subject`, `_size`, `size`, `attachment_type`, `_attachment_size`, `attachment_size`, `spam`, `days` FROM `%s`", table);
if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){
res = mysql_store_result(&(sdata->mysql));
if(res != NULL){
while((row = mysql_fetch_row(res))){
append_rule(rules, (char*)row[0], (char*)row[1], (char*)row[2], (char*)row[3], atoi(row[4]), (char*)row[5], (char*)row[6], atoi(row[7]));
append_rule(rules, (char*)row[0], (char*)row[1], (char*)row[2], (char*)row[3], atoi(row[4]), (char*)row[5], (char*)row[6], atoi(row[7]), atoi(row[8]), atoi(row[9]));
}
mysql_free_result(res);
@ -32,7 +32,7 @@ void load_archiving_rules(struct session_data *sdata, struct rule **rules){
}
int append_rule(struct rule **rule, char *from, char *to, char *subject, char *_size, int size, char *attachment_type, char *_attachment_size, int attachment_size){
int append_rule(struct rule **rule, char *from, char *to, char *subject, char *_size, int size, char *attachment_type, char *_attachment_size, int attachment_size, int spam, int days){
struct rule *q, *t, *u=NULL;
q = *rule;
@ -42,7 +42,7 @@ int append_rule(struct rule **rule, char *from, char *to, char *subject, char *_
q = q->r;
}
t = create_rule_item(from, to, subject, _size, size, attachment_type, _attachment_size, attachment_size);
t = create_rule_item(from, to, subject, _size, size, attachment_type, _attachment_size, attachment_size, spam, days);
if(t){
if(*rule == NULL)
*rule = t;
@ -56,7 +56,7 @@ int append_rule(struct rule **rule, char *from, char *to, char *subject, char *_
}
struct rule *create_rule_item(char *from, char *to, char *subject, char *_size, int size, char *attachment_type, char *_attachment_size, int attachment_size){
struct rule *create_rule_item(char *from, char *to, char *subject, char *_size, int size, char *attachment_type, char *_attachment_size, int attachment_size, int spam, int days){
struct rule *h=NULL;
char empty = '\0';
int len;
@ -76,6 +76,9 @@ struct rule *create_rule_item(char *from, char *to, char *subject, char *_size,
if(!subject) subject = &empty;
if(regcomp(&(h->subject), subject, REG_ICASE | REG_EXTENDED)) h->compiled = 0;
h->spam = spam;
h->days = days;
h->size = size;
if(!_size) _size = &empty;
@ -91,12 +94,12 @@ struct rule *create_rule_item(char *from, char *to, char *subject, char *_size,
if(!_attachment_size) _attachment_size = &empty;
snprintf(h->_attachment_size, 3, "%s", _attachment_size);
len = strlen(from)+6 + strlen(to)+4 + strlen(subject)+9 + strlen(_size)+6 + strlen(attachment_type)+10 + strlen(_attachment_size)+10 + 15 + 15;
len = strlen(from)+6 + strlen(to)+4 + strlen(subject)+9 + strlen(_size)+6 + strlen(attachment_type)+10 + strlen(_attachment_size)+10 + 8 + 15 + 15;
h->rulestr = malloc(len);
if(h->rulestr) snprintf(h->rulestr, len-1, "from=%s,to=%s,subject=%s,size%s%d,att.type=%s,att.size%s%d", from, to, subject, _size, size, attachment_type, _attachment_size, attachment_size);
if(h->rulestr) snprintf(h->rulestr, len-1, "from=%s,to=%s,subject=%s,size%s%d,att.type=%s,att.size%s%d,spam=%d", from, to, subject, _size, size, attachment_type, _attachment_size, attachment_size, spam);
else h->compiled = 0;
h->r = NULL;
@ -105,7 +108,7 @@ struct rule *create_rule_item(char *from, char *to, char *subject, char *_size,
}
char *check_againt_ruleset(struct rule *rule, struct _state *state, int size){
char *check_againt_ruleset(struct rule *rule, struct _state *state, int size, int spam){
size_t nmatch=0;
struct rule *p;
@ -119,7 +122,8 @@ char *check_againt_ruleset(struct rule *rule, struct _state *state, int size){
regexec(&(p->to), state->b_to, nmatch, NULL, 0) == 0 &&
regexec(&(p->subject), state->b_subject, nmatch, NULL, 0) == 0 &&
check_size_rule(size, p->size, p->_size) == 1 &&
check_attachment_rule(state, p) == 1
check_attachment_rule(state, p) == 1 &&
check_spam_rule(spam, p->spam) == 1
){
return p->rulestr;
}
@ -131,6 +135,33 @@ char *check_againt_ruleset(struct rule *rule, struct _state *state, int size){
}
unsigned long query_retain_period(struct rule *rule, struct _state *state, int size, int spam, struct __config *cfg){
size_t nmatch=0;
struct rule *p;
p = rule;
while(p != NULL){
if(
p->compiled == 1 &&
regexec(&(p->from), state->b_from, nmatch, NULL, 0) == 0 &&
regexec(&(p->to), state->b_to, nmatch, NULL, 0) == 0 &&
regexec(&(p->subject), state->b_subject, nmatch, NULL, 0) == 0 &&
check_size_rule(size, p->size, p->_size) == 1 &&
check_attachment_rule(state, p) == 1 &&
check_spam_rule(spam, p->spam) == 1
){
return p->days * 86400;
}
p = p->r;
}
return cfg->default_retention_days * 86400;
}
int check_size_rule(int message_size, int size, char *_size){
if(size <= 0) return 1;
@ -143,6 +174,13 @@ int check_size_rule(int message_size, int size, char *_size){
}
int check_spam_rule(int is_spam, int spam){
if(spam == -1) return 1;
if(is_spam == spam) return 1;
return 0;
}
int check_attachment_rule(struct _state *state, struct rule *rule){
int i;
size_t nmatch=0;

10
src/rules.h
View File

@ -7,11 +7,13 @@
#include "defs.h"
void load_archiving_rules(struct session_data *sdata, struct rule **rules);
int append_rule(struct rule **rule, char *from, char *to, char *subject, char *_size, int size, char *attachment_type, char *_attachment_size, int attachment_size);
struct rule *create_rule_item(char *from, char *to, char *subject, char *_size, int size, char *attachment_type, char *_attachment_size, int attachment_size);
char *check_againt_ruleset(struct rule *rule, struct _state *state, int size);
void load_rules(struct session_data *sdata, struct rule **rules, char *table);
int append_rule(struct rule **rule, char *from, char *to, char *subject, char *_size, int size, char *attachment_type, char *_attachment_size, int attachment_size, int spam, int days);
struct rule *create_rule_item(char *from, char *to, char *subject, char *_size, int size, char *attachment_type, char *_attachment_size, int attachment_size, int spam, int days);
char *check_againt_ruleset(struct rule *rule, struct _state *state, int size, int spam);
unsigned long query_retain_period(struct rule *rule, struct _state *state, int size, int spam, struct __config *cfg);
int check_size_rule(int message_size, int size, char *_size);
int check_spam_rule(int is_spam, int spam);
int check_attachment_rule(struct _state *state, struct rule *rule);
void free_rule(struct rule *rule);

2
src/session.c
View File

@ -193,7 +193,7 @@ int handle_smtp_session(int new_sd, struct __data *data, struct __config *cfg){
/* check message against archiving rules */
arule = check_againt_ruleset(data->rules, &sstate, sdata.tot_len);
arule = check_againt_ruleset(data->archiving_rules, &sstate, sdata.tot_len, sdata.spam_message);
if(arule){
syslog(LOG_PRIORITY, "%s: discarding message by archiving policy: *%s*", sdata.ttmpfile, arule);

15
src/test.c
View File

@ -51,9 +51,11 @@ int main(int argc, char **argv){
printf("locale: %s\n", setlocale(LC_MESSAGES, cfg.locale));
setlocale(LC_CTYPE, cfg.locale);
data.rules = NULL;
data.archiving_rules = NULL;
data.retention_rules = NULL;
load_archiving_rules(&sdata, &(data.rules));
load_rules(&sdata, &(data.archiving_rules), SQL_ARCHIVING_RULE_TABLE);
load_rules(&sdata, &(data.retention_rules), SQL_RETENTION_RULE_TABLE);
rc = 0;
@ -82,13 +84,18 @@ int main(int argc, char **argv){
printf("hdr len: %d\n", sdata.hdr_len);
rule = check_againt_ruleset(data.rules, &state, st.st_size);
rule = check_againt_ruleset(data.archiving_rules, &state, st.st_size, sdata.spam_message);
printf("body digest: %s\n", sdata.bodydigest);
printf("rules check: %s\n", rule);
free_rule(data.rules);
sdata.retained = sdata.now + query_retain_period(data.retention_rules, &state, st.st_size, sdata.spam_message, &cfg);
printf("retention period: %ld\n", sdata.retained);
free_rule(data.archiving_rules);
free_rule(data.retention_rules);
for(i=1; i<=state.n_attachments; i++){
printf("i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s\n", i, state.attachments[i].filename, state.attachments[i].type, state.attachments[i].size, state.attachments[i].internalname, state.attachments[i].digest);