changed the internal lists to hash tables

This commit is contained in:
SJ 2013-08-14 14:24:30 +02:00
parent 4f2d743f9b
commit b3cea9de7f
16 changed files with 201 additions and 164 deletions

2
configure vendored
View File

@ -4734,7 +4734,7 @@ echo; echo
CFLAGS="$static -O2 -Wall -g"
LIBS="$antispam_libs $sunos_libs "
OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o list.o parser.o parser_utils.o rules.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o imap.o pop3.o extract.o mydomains.o retr.o $objs"
OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o imap.o pop3.o extract.o mydomains.o retr.o $objs"
ac_config_files="$ac_config_files Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile test/Makefile contrib/imap/Makefile"

View File

@ -472,7 +472,7 @@ echo; echo
CFLAGS="$static -O2 -Wall -g"
LIBS="$antispam_libs $sunos_libs "
OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o list.o parser.o parser_utils.o rules.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o imap.o pop3.o extract.o mydomains.o retr.o $objs"
OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o imap.o pop3.o extract.o mydomains.o retr.o $objs"
AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile test/Makefile contrib/imap/Makefile])
AC_OUTPUT

View File

@ -14,7 +14,7 @@
#define VERSION "0.1.24-master-branch"
#define BUILD 828
#define BUILD 832
#define HOSTID "mailarchiver"

View File

@ -38,7 +38,7 @@
#define MSG_REFERENCES 10
#define MSG_RECIPIENT 11
#define MAXHASH 8171
#define MAXHASH 277
#define BASE64_RATIO 1.33333333
@ -88,9 +88,10 @@ struct ptr_array {
};
struct list {
char s[SMALLBUFSIZE];
struct list *r;
struct node {
void *str;
unsigned int key;
struct node *r;
};
@ -156,10 +157,10 @@ struct _state {
char filename[TINYBUFSIZE];
char type[TINYBUFSIZE];
struct list *boundaries;
struct list *rcpt;
struct list *rcpt_domain;
struct list *journal_recipient;
struct node *boundaries[MAXHASH];
struct node *rcpt[MAXHASH];
struct node *rcpt_domain[MAXHASH];
struct node *journal_recipient[MAXHASH];
int n_attachments;
struct attachment attachments[MAX_ATTACHMENTS];
@ -251,7 +252,7 @@ struct __data {
int folder;
char recursive_folder_names;
char starttls[TINYBUFSIZE];
struct list *mydomains;
struct node *mydomains[MAXHASH];
#ifdef NEED_MYSQL
MYSQL_STMT *stmt_generic;
@ -276,8 +277,8 @@ struct __data {
int pos;
#ifdef HAVE_TRE
struct rule *archiving_rules;
struct rule *retention_rules;
struct node *archiving_rules[1];
struct node *retention_rules[1];
#endif
#ifdef HAVE_MEMCACHED

View File

@ -33,8 +33,9 @@ void load_mydomains(struct session_data *sdata, struct __data *data, struct __co
p_store_results(sdata, data->stmt_generic, data);
while(p_fetch_results(data->stmt_generic) == OK){
rc = append_list(&(data->mydomains), s);
if(rc == -1) syslog(LOG_PRIORITY, "failed to append mydomain: '%s'", s);
rc = addnode(data->mydomains, s);
if(rc == 0) syslog(LOG_PRIORITY, "failed to append mydomain: '%s'", s);
else if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "added mydomain: '%s'", s);
memset(s, 0, sizeof(s));
@ -50,7 +51,6 @@ ENDE:
int is_email_address_on_my_domains(char *email, struct __data *data){
int rc=0;
char *q, *s;
struct list *p;
if(email == NULL) return rc;
@ -61,17 +61,7 @@ int is_email_address_on_my_domains(char *email, struct __data *data){
if(s) *s = '\0';
p = data->mydomains;
while(p != NULL){
if(strcasecmp(p->s, q+1) == 0){
rc = 1;
break;
}
p = p->r;
}
if(findnode(data->mydomains, q+1)) rc = 1;
if(s) *s = ' ';

View File

@ -50,8 +50,8 @@ struct _state parse_message(struct session_data *sdata, int take_into_pieces, st
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: processing rcpt to address: *%s*", sdata->ttmpfile, puf);
if(state.tolen < MAXBUFSIZE-len-1){
if(is_string_on_list(state.rcpt, puf) == 0){
append_list(&(state.rcpt), puf);
if(findnode(state.rcpt, puf) == NULL){
addnode(state.rcpt, puf);
memcpy(&(state.b_to[state.tolen]), puf, len);
state.tolen += len;
@ -103,10 +103,10 @@ void post_parse(struct session_data *sdata, struct _state *state, struct __confi
int i, len, rec=0;
char *p;
free_list(state->boundaries);
free_list(state->rcpt);
free_list(state->rcpt_domain);
free_list(state->journal_recipient);
clearhash(state->boundaries);
clearhash(state->rcpt);
clearhash(state->rcpt_domain);
clearhash(state->journal_recipient);
trimBuffer(state->b_subject);
fixupEncodedHeaderLine(state->b_subject);
@ -211,7 +211,7 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
}
if(take_into_pieces == 1){
if(state->message_state == MSG_BODY && state->fd != -1 && is_item_on_string(state->boundaries, buf) == 0){
if(state->message_state == MSG_BODY && state->fd != -1 && findnode(state->boundaries, buf) == NULL){
//n = write(state->fd, buf, len); // WRITE
if(len + state->abufpos > abuffersize-1){
write(state->fd, abuffer, state->abufpos);
@ -473,7 +473,9 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
/* boundary check, and reset variables */
boundary_line = is_item_on_string(state->boundaries, buf);
if(findnode(state->boundaries, buf)) boundary_line = 1;
if(!strstr(buf, "boundary=") && !strstr(buf, "boundary =") && boundary_line == 1){
state->is_header = 1;
@ -616,16 +618,16 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
else if((state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT) && state->is_1st_header == 1 && state->tolen < MAXBUFSIZE-len-1){
strtolower(puf);
if(state->message_state == MSG_RECIPIENT && is_string_on_list(state->journal_recipient, puf) == 0){
append_list(&(state->journal_recipient), puf);
if(state->message_state == MSG_RECIPIENT && findnode(state->journal_recipient, puf) == NULL){
addnode(state->journal_recipient, puf);
memcpy(&(state->b_journal_to[state->journaltolen]), puf, len);
memcpy(&(state->b_journal_to[state->journaltolen]), puf, len);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: journal rcpt: '%s'", sdata->ttmpfile, puf);
}
if(is_string_on_list(state->rcpt, puf) == 0){
append_list(&(state->rcpt), puf);
if(findnode(state->rcpt, puf) == NULL){
addnode(state->rcpt, puf);
memcpy(&(state->b_to[state->tolen]), puf, len);
state->tolen += len;
@ -635,8 +637,8 @@ int parse_line(char *buf, struct _state *state, struct session_data *sdata, int
q = strchr(puf, '@');
if(q){
if(is_string_on_list(state->rcpt_domain, q+1) == 0){
append_list(&(state->rcpt_domain), q+1);
if(findnode(state->rcpt_domain, q+1) == NULL){
addnode(state->rcpt_domain, q+1);
memcpy(&(state->b_to_domain[strlen(state->b_to_domain)]), q+1, strlen(q+1));
}
}

View File

@ -66,10 +66,10 @@ void init_state(struct _state *state){
state->writebufpos = 0;
state->abufpos = 0;
state->boundaries = NULL;
state->rcpt = NULL;
state->rcpt_domain = NULL;
state->journal_recipient = NULL;
inithash(state->boundaries);
inithash(state->rcpt);
inithash(state->rcpt_domain);
inithash(state->journal_recipient);
state->n_attachments = 0;
@ -229,7 +229,7 @@ int extract_boundary(char *p, struct _state *state){
q = strrchr(p, '\n');
if(q) *q = '\0';
append_list(&(state->boundaries), p);
addnode(state->boundaries, p);
return 1;
}

View File

@ -228,10 +228,10 @@ void p_clean_exit(){
kill_children(SIGTERM);
free_rule(data.archiving_rules);
free_rule(data.retention_rules);
clearrules(data.archiving_rules);
clearrules(data.retention_rules);
free_list(data.mydomains);
clearhash(data.mydomains);
#ifdef HAVE_MULTITENANCY
free_list(data.customers);
@ -307,10 +307,10 @@ void initialise_configuration(){
setlocale(LC_CTYPE, cfg.locale);
free_rule(data.archiving_rules);
free_rule(data.retention_rules);
clearrules(data.archiving_rules);
clearrules(data.retention_rules);
free_list(data.mydomains);
clearhash(data.mydomains);
#ifdef HAVE_MULTITENANCY
free_list(data.customers);
@ -318,9 +318,9 @@ void initialise_configuration(){
data.folder = 0;
data.recursive_folder_names = 0;
data.archiving_rules = NULL;
data.retention_rules = NULL;
data.mydomains = NULL;
inithash(data.mydomains);
initrules(data.archiving_rules);
initrules(data.retention_rules);
#ifdef HAVE_MULTITENANCY
data.customers = NULL;
@ -339,8 +339,8 @@ void initialise_configuration(){
return;
}
load_rules(&sdata, &data, &(data.archiving_rules), SQL_ARCHIVING_RULE_TABLE);
load_rules(&sdata, &data, &(data.retention_rules), SQL_RETENTION_RULE_TABLE);
load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
load_rules(&sdata, &data, data.retention_rules, SQL_RETENTION_RULE_TABLE);
load_mydomains(&sdata, &data, &cfg);
@ -393,9 +393,9 @@ int main(int argc, char **argv){
data.folder = 0;
data.recursive_folder_names = 0;
data.archiving_rules = NULL;
data.retention_rules = NULL;
data.mydomains = NULL;
inithash(data.mydomains);
initrules(data.archiving_rules);
initrules(data.retention_rules);
#ifdef HAVE_MULTITENANCY
data.customers = NULL;
#endif

View File

@ -6,12 +6,11 @@
#define _PILER_H
#include <misc.h>
#include <list.h>
#include <parser.h>
#include <errmsg.h>
#include <smtpcodes.h>
#include <decoder.h>
#include <list.h>
#include <hash.h>
#include <rules.h>
#include <defs.h>
#include <tai.h>

View File

@ -397,9 +397,9 @@ int main(int argc, char **argv){
data.folder = 0;
data.recursive_folder_names = 0;
data.archiving_rules = NULL;
data.retention_rules = NULL;
data.mydomains = NULL;
inithash(data.mydomains);
initrules(data.archiving_rules);
initrules(data.retention_rules);
while(1){
@ -551,8 +551,8 @@ int main(int argc, char **argv){
}
load_rules(&sdata, &data, &(data.archiving_rules), SQL_ARCHIVING_RULE_TABLE);
load_rules(&sdata, &data, &(data.retention_rules), SQL_RETENTION_RULE_TABLE);
load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
load_rules(&sdata, &data, data.retention_rules, SQL_RETENTION_RULE_TABLE);
load_mydomains(&sdata, &data, &cfg);
@ -569,10 +569,10 @@ int main(int argc, char **argv){
if(pop3server && username && password) rc = import_from_pop3_server(pop3server, username, password, port, &sdata, &data, &cfg);
free_rule(data.archiving_rules);
free_rule(data.retention_rules);
clearrules(data.archiving_rules);
clearrules(data.retention_rules);
free_list(data.mydomains);
clearhash(data.mydomains);
close_database(&sdata);

View File

@ -216,9 +216,9 @@ int main(int argc, char **argv){
data.folder = 0;
data.recursive_folder_names = 0;
data.archiving_rules = NULL;
data.retention_rules = NULL;
data.mydomains = NULL;
inithash(data.mydomains);
initrules(data.archiving_rules);
initrules(data.retention_rules);
if(folder){
data.folder = get_folder_id(&sdata, &data, folder, 0);
@ -247,7 +247,7 @@ int main(int argc, char **argv){
printf("put %llu messages to %s table for reindexing\n", n, SQL_SPHINX_TABLE);
free_list(data.mydomains);
clearhash(data.mydomains);
close_database(&sdata);

View File

@ -10,7 +10,7 @@
#include "rules.h"
void load_rules(struct session_data *sdata, struct __data *data, struct rule **rules, char *table){
void load_rules(struct session_data *sdata, struct __data *data, struct node *xhash[], char *table){
char s[SMALLBUFSIZE];
char domain[SMALLBUFSIZE], from[SMALLBUFSIZE], to[SMALLBUFSIZE], subject[SMALLBUFSIZE], _size[SMALLBUFSIZE], attachment_type[SMALLBUFSIZE], _attachment_size[SMALLBUFSIZE];
int size=0, attachment_size=0, spam=0, days=0;
@ -54,7 +54,7 @@ void load_rules(struct session_data *sdata, struct __data *data, struct rule **r
p_store_results(sdata, data->stmt_generic, data);
while(p_fetch_results(data->stmt_generic) == OK){
append_rule(rules, domain, from, to, subject, _size, size, attachment_type, _attachment_size, attachment_size, spam, days);
append_rule(xhash, domain, from, to, subject, _size, size, attachment_type, _attachment_size, attachment_size, spam, days);
memset(domain, 0, sizeof(domain));
memset(from, 0, sizeof(from));
@ -75,27 +75,40 @@ ENDE:
}
int append_rule(struct rule **rule, char *domain, char *from, char *to, char *subject, char *_size, int size, char *attachment_type, char *_attachment_size, int attachment_size, int spam, int days){
struct rule *q, *t, *u=NULL;
int append_rule(struct node *xhash[], char *domain, char *from, char *to, char *subject, char *_size, int size, char *attachment_type, char *_attachment_size, int attachment_size, int spam, int days){
struct node *q, *Q=NULL, *node;
struct rule *rule;
int rc=0;
q = *rule;
if((node = malloc(sizeof(struct node))) == NULL) return rc;
while(q){
u = q;
memset(node, 0, sizeof(struct node));
node->r = NULL;
rule = create_rule_item(domain, from, to, subject, _size, size, attachment_type, _attachment_size, attachment_size, spam, days);
if(rule == NULL){
free(node);
return rc;
}
node->str = rule;
q = xhash[0];
while(q != NULL){
Q = q;
q = q->r;
}
t = create_rule_item(domain, from, to, subject, _size, size, attachment_type, _attachment_size, attachment_size, spam, days);
if(t){
if(*rule == NULL)
*rule = t;
else if(u)
u->r = t;
return 1;
if(Q == NULL) xhash[0] = node;
else {
Q->r = node;
}
return -1;
rc = 1;
return rc;
}
@ -163,14 +176,19 @@ struct rule *create_rule_item(char *domain, char *from, char *to, char *subject,
}
char *check_againt_ruleset(struct rule *rule, struct _state *state, int size, int spam){
char *check_againt_ruleset(struct node *xhash[], struct _state *state, int size, int spam){
size_t nmatch=0;
struct rule *p;
struct node *q;
p = rule;
q = xhash[0];
while(p != NULL){
while(q != NULL){
if(q->str){
p = q->str;
if(p){
if(
p->compiled == 1 &&
regexec(&(p->from), state->b_from, nmatch, NULL, 0) == 0 &&
@ -183,7 +201,10 @@ char *check_againt_ruleset(struct rule *rule, struct _state *state, int size, in
return p->rulestr;
}
p = p->r;
}
}
q = q->r;
}
return NULL;
@ -193,10 +214,14 @@ char *check_againt_ruleset(struct rule *rule, struct _state *state, int size, in
unsigned long query_retain_period(struct __data *data, struct _state *state, int size, int spam, struct __config *cfg){
size_t nmatch=0;
struct rule *p;
struct node *q;
p = data->retention_rules;
q = data->retention_rules[0];
while(p != NULL){
while(q != NULL){
if(q->str){
p = q->str;
if(p->domainlen > 2){
if(strcasestr(state->b_to_domain, p->domain) || strcasestr(state->b_from_domain, p->domain)){
@ -217,9 +242,12 @@ unsigned long query_retain_period(struct __data *data, struct _state *state, int
return p->days * 86400;
}
p = p->r;
}
q = q->r;
}
state->retention = cfg->default_retention_days;
return cfg->default_retention_days * 86400;
@ -264,29 +292,39 @@ int check_attachment_rule(struct _state *state, struct rule *rule){
}
void free_rule(struct rule *rule){
struct rule *p, *q;
void initrules(struct node *xhash[]){
xhash[0] = NULL;
}
p = rule;
while(p != NULL){
q = p->r;
void clearrules(struct node *xhash[]){
struct node *p, *q;
struct rule *rule;
q = xhash[0];
while(q != NULL){
p = q;
q = q->r;
if(p){
regfree(&(p->from));
regfree(&(p->to));
regfree(&(p->attachment_type));
if(p->str){
rule = (struct rule*)p->str;
free(p->rulestr);
regfree(&(rule->from));
regfree(&(rule->to));
regfree(&(rule->attachment_type));
if(p->domain) free(p->domain);
free(rule->rulestr);
if(rule->domain) free(rule->domain);
free(rule);
}
free(p);
}
p = q;
}
}
xhash[0] = NULL;
}

View File

@ -7,15 +7,17 @@
#include "defs.h"
void load_rules(struct session_data *sdata, struct __data *data, struct rule **rules, char *table);
int append_rule(struct rule **rule, char *domain, char *from, char *to, char *subject, char *_size, int size, char *attachment_type, char *_attachment_size, int attachment_size, int spam, int days);
void load_rules(struct session_data *sdata, struct __data *data, struct node *xhash[], char *table);
int append_rule(struct node *xhash[], char *domain, char *from, char *to, char *subject, char *_size, int size, char *attachment_type, char *_attachment_size, int attachment_size, int spam, int days);
struct rule *create_rule_item(char *domain, char *from, char *to, char *subject, char *_size, int size, char *attachment_type, char *_attachment_size, int attachment_size, int spam, int days);
char *check_againt_ruleset(struct rule *rule, struct _state *state, int size, int spam);
char *check_againt_ruleset(struct node *xhash[], struct _state *state, int size, int spam);
unsigned long query_retain_period(struct __data *data, struct _state *state, int size, int spam, struct __config *cfg);
int check_size_rule(int message_size, int size, char *_size);
int check_spam_rule(int is_spam, int spam);
int check_attachment_rule(struct _state *state, struct rule *rule);
void free_rule(struct rule *rule);
void initrules(struct node *xhash[]);
void clearrules(struct node *xhash[]);
#endif /* _RULES_H */

View File

@ -48,15 +48,18 @@ int main(int argc, char **argv){
data.folder = 0;
data.recursive_folder_names = 0;
data.archiving_rules = NULL;
data.retention_rules = NULL;
data.mydomains = NULL;
load_rules(&sdata, &data, &(data.archiving_rules), SQL_ARCHIVING_RULE_TABLE);
load_rules(&sdata, &data, &(data.retention_rules), SQL_RETENTION_RULE_TABLE);
inithash(data.mydomains);
initrules(data.archiving_rules);
initrules(data.retention_rules);
load_mydomains(&sdata, &data, &cfg);
load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
load_rules(&sdata, &data, data.retention_rules, SQL_RETENTION_RULE_TABLE);
init_session_data(&sdata, &cfg);
sdata.sent = 0;
@ -96,10 +99,10 @@ int main(int argc, char **argv){
printf("retention period: %ld\n", sdata.retained);
free_rule(data.archiving_rules);
free_rule(data.retention_rules);
clearrules(data.archiving_rules);
clearrules(data.retention_rules);
free_list(data.mydomains);
clearhash(data.mydomains);
for(i=1; i<=state.n_attachments; i++){
printf("i:%d, name=*%s*, type: *%s*, size: %d, int.name: %s, digest: %s\n", i, state.attachments[i].filename, state.attachments[i].type, state.attachments[i].size, state.attachments[i].internalname, state.attachments[i].digest);

View File

@ -233,14 +233,16 @@ int main(int argc, char **argv){
setlocale(LC_CTYPE, cfg.locale);
data.archiving_rules = NULL;
data.retention_rules = NULL;
initrules(data.archiving_rules);
initrules(data.retention_rules);
if(emlfile) rc = import_message2(emlfile, &sdata, &data, &cfg);
if(mailbox) rc = import_from_mailbox(mailbox, &sdata, &data, &cfg);
if(directory) rc = import_from_maildir(directory, &sdata, &data, &cfg);
if(imapserver && username && password) rc = import_from_imap_server(imapserver, username, password, &sdata, &data, &cfg);
clearrules(data.archiving_rules);
clearrules(data.retention_rules);
return rc;
}

View File

@ -306,8 +306,8 @@ int main(int argc, char **argv){
data.folder = 0;
data.archiving_rules = NULL;
data.retention_rules = NULL;
initrules(data.archiving_rules);
initrules(data.retention_rules);
while(1){
@ -422,8 +422,8 @@ int main(int argc, char **argv){
(void) openlog("pilerimport", LOG_PID, LOG_MAIL);
load_rules(&sdata, &data, &(data.archiving_rules), SQL_ARCHIVING_RULE_TABLE);
load_rules(&sdata, &data, &(data.retention_rules), SQL_RETENTION_RULE_TABLE);
load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);
load_rules(&sdata, &data, data.retention_rules, SQL_RETENTION_RULE_TABLE);
load_mydomains(&sdata, &data, &cfg);
@ -440,8 +440,8 @@ int main(int argc, char **argv){
if(quiet == 0) printf("\n");
free_rule(data.archiving_rules);
free_rule(data.retention_rules);
clearrules(data.archiving_rules);
clearrules(data.retention_rules);
mysql_close(&(sdata.mysql));