bashclub/piler
1
0
Fork 0
mirror of https://bitbucket.org/jsuto/piler.git synced 2024-12-24 18:50:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

Handle the Sender: header line

Browse Source 
Signed-off-by: Janos SUTO <sj@acts.hu>
This commit is contained in:
Janos SUTO 2020-12-12 21:43:16 +01:00
parent c08bd7393f
commit d9d5f7db66
13 changed files with 67 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/defs.h
View File

@ -27,16 +27,17 @@
#define MSG_BODY 0 #define MSG_BODY 0
#define MSG_RECEIVED 1 #define MSG_RECEIVED 1
#define MSG_FROM 2 #define MSG_FROM 2
#define MSG_TO 3 #define MSG_SENDER 3
#define MSG_CC 4 #define MSG_TO 4
#define MSG_SUBJECT 5 #define MSG_CC 5
#define MSG_CONTENT_TYPE 6 #define MSG_SUBJECT 6
#define MSG_CONTENT_TRANSFER_ENCODING 7 #define MSG_CONTENT_TYPE 7
#define MSG_CONTENT_DISPOSITION 8 #define MSG_CONTENT_TRANSFER_ENCODING 8
#define MSG_MESSAGE_ID 9 #define MSG_CONTENT_DISPOSITION 9
#define MSG_REFERENCES 10 #define MSG_MESSAGE_ID 10
#define MSG_RECIPIENT 11 #define MSG_REFERENCES 11
#define MSG_ENVELOPE_TO 12 #define MSG_RECIPIENT 12
#define MSG_ENVELOPE_TO 13
#define MAXHASH 277 #define MAXHASH 277
@ -203,7 +204,7 @@ struct parser_state {
char reference[SMALLBUFSIZE]; char reference[SMALLBUFSIZE];
char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE]; char b_from[SMALLBUFSIZE], b_from_domain[SMALLBUFSIZE], b_sender[SMALLBUFSIZE], b_sender_domain[SMALLBUFSIZE], b_to[MAXBUFSIZE], b_to_domain[SMALLBUFSIZE], b_subject[MAXBUFSIZE], b_body[BIGBUFSIZE];
char b_journal_to[MAXBUFSIZE]; char b_journal_to[MAXBUFSIZE];
unsigned int bodylen; unsigned int bodylen;

2
src/import.c
View File

@ -71,7 +71,7 @@ int import_message(struct session_data *sdata, struct data *data, struct config
sdata->import = 1; sdata->import = 1;
state = parse_message(sdata, 1, data, cfg); state = parse_message(sdata, 1, data, cfg);
post_parse(sdata, &state, cfg); post_parse(sdata, data, &state, cfg);
rule = check_against_ruleset(data->archiving_rules, &state, sdata->tot_len, sdata->spam_message); rule = check_against_ruleset(data->archiving_rules, &state, sdata->tot_len, sdata->spam_message);
if(rule){ if(rule){

14
src/message.c
View File

@ -34,18 +34,18 @@ int store_index_data(struct session_data *sdata, struct parser_state *state, str
if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc; if(prepare_sql_statement(sdata, &sql, SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE) == ERR) return rc;
fix_email_address_for_sphinx(state->b_from); fix_email_address_for_sphinx(state->b_sender);
fix_email_address_for_sphinx(state->b_to); fix_email_address_for_sphinx(state->b_to);
fix_email_address_for_sphinx(state->b_from_domain); fix_email_address_for_sphinx(state->b_sender_domain);
fix_email_address_for_sphinx(state->b_to_domain); fix_email_address_for_sphinx(state->b_to_domain);
p_bind_init(&sql); p_bind_init(&sql);
sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++; sql.sql[sql.pos] = (char *)&id; sql.type[sql.pos] = TYPE_LONGLONG; sql.pos++;
sql.sql[sql.pos] = state->b_from; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = state->b_sender; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
sql.sql[sql.pos] = state->b_to; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = state->b_to; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
sql.sql[sql.pos] = state->b_from_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = state->b_sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
sql.sql[sql.pos] = state->b_to_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = state->b_to_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
sql.sql[sql.pos] = state->b_body; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = state->b_body; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
@ -188,7 +188,7 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru
subj = state->b_subject; subj = state->b_subject;
if(*subj == ' ') subj++; if(*subj == ' ') subj++;
snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, state->b_from, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest); snprintf(s, sizeof(s)-1, "%llu+%s%s%s%ld%ld%ld%d%d%d%d%s%s%s", id, subj, state->b_sender, state->message_id, sdata->now, sdata->sent, sdata->retained, sdata->tot_len, sdata->hdr_len, sdata->direction, state->n_attachments, sdata->ttmpfile, sdata->digest, sdata->bodydigest);
digest_string(s, &vcode[0]); digest_string(s, &vcode[0]);
@ -203,7 +203,7 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru
memset(s2, 0, sizeof(s2)); memset(s2, 0, sizeof(s2));
p = state->b_from; p = state->b_sender;
do { do {
memset(s2, 0, sizeof(s2)); memset(s2, 0, sizeof(s2));
p = split(p, ' ', s2, sizeof(s2)-1, &result); p = split(p, ' ', s2, sizeof(s2)-1, &result);
@ -222,7 +222,7 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru
p_bind_init(&sql); p_bind_init(&sql);
sql.sql[sql.pos] = &s2[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = &s2[0]; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
sql.sql[sql.pos] = state->b_from_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = state->b_sender_domain; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++; sql.sql[sql.pos] = subj; sql.type[sql.pos] = TYPE_STRING; sql.pos++;
sql.sql[sql.pos] = (char *)&sdata->spam_message; sql.type[sql.pos] = TYPE_LONG; sql.pos++; sql.sql[sql.pos] = (char *)&sdata->spam_message; sql.type[sql.pos] = TYPE_LONG; sql.pos++;
sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++; sql.sql[sql.pos] = (char *)&sdata->now; sql.type[sql.pos] = TYPE_LONG; sql.pos++;

23
src/parser.c
View File

@ -72,7 +72,7 @@ struct parser_state parse_message(struct session_data *sdata, int take_into_piec
} }
void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg){ void post_parse(struct session_data *sdata, struct data *data, struct parser_state *state, struct config *cfg){
int i; int i;
clearhash(state->boundaries); clearhash(state->boundaries);
@ -84,6 +84,19 @@ void post_parse(struct session_data *sdata, struct parser_state *state, struct c
if(strlen(state->b_from) > 255) state->b_from[255] = '\0'; if(strlen(state->b_from) > 255) state->b_from[255] = '\0';
if(strlen(state->b_from_domain) > 255) state->b_from_domain[255] = '\0'; if(strlen(state->b_from_domain) > 255) state->b_from_domain[255] = '\0';
if(strlen(state->b_sender) > 255) state->b_sender[255] = '\0';
if(strlen(state->b_sender_domain) > 255) state->b_sender_domain[255] = '\0';
// If Sender: header doesn't exist, then copy the From: header value to it
// Otherwise append the From: address to the recipients list
if(state->b_sender[0] == '\0'){
strcpy(state->b_sender, state->b_from);
strcpy(state->b_sender_domain, state->b_from_domain);
} else {
add_recipient(state->b_from, strlen(state->b_from), sdata, state, data, cfg);
}
// Truncate the message_id if it's >255 characters // Truncate the message_id if it's >255 characters
if(strlen(state->message_id) > 255) state->message_id[255] = '\0'; if(strlen(state->message_id) > 255) state->message_id[255] = '\0';
@ -358,6 +371,10 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
state->message_state = MSG_FROM; state->message_state = MSG_FROM;
buf += strlen("From:"); buf += strlen("From:");
} }
else if(strncasecmp(buf, "Sender:", strlen("Sender:")) == 0){
state->message_state = MSG_SENDER;
buf += strlen("Sender:");
}
else if(strncasecmp(buf, "Content-Type:", strlen("Content-Type:")) == 0){ else if(strncasecmp(buf, "Content-Type:", strlen("Content-Type:")) == 0){
state->message_state = MSG_CONTENT_TYPE; state->message_state = MSG_CONTENT_TYPE;
} }
@ -545,6 +562,8 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
memset(state->b_body, 0, BIGBUFSIZE); memset(state->b_body, 0, BIGBUFSIZE);
memset(state->b_from, 0, SMALLBUFSIZE); memset(state->b_from, 0, SMALLBUFSIZE);
memset(state->b_from_domain, 0, SMALLBUFSIZE); memset(state->b_from_domain, 0, SMALLBUFSIZE);
memset(state->b_sender, 0, SMALLBUFSIZE);
memset(state->b_sender_domain, 0, SMALLBUFSIZE);
memset(state->message_id, 0, SMALLBUFSIZE); memset(state->message_id, 0, SMALLBUFSIZE);
sdata->ms_journal = 0; sdata->ms_journal = 0;
@ -616,7 +635,7 @@ int parse_line(char *buf, struct parser_state *state, struct session_data *sdata
/* skip irrelevant headers */ /* skip irrelevant headers */
if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT && state->message_state != MSG_ENVELOPE_TO) return 0; if(state->is_header == 1 && state->message_state != MSG_FROM && state->message_state != MSG_SENDER && state->message_state != MSG_TO && state->message_state != MSG_CC && state->message_state != MSG_RECIPIENT && state->message_state != MSG_ENVELOPE_TO) return 0;
/* don't process body if it's not a text or html part */ /* don't process body if it's not a text or html part */

2
src/parser.h
View File

@ -10,7 +10,7 @@
#include "defs.h" #include "defs.h"
struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct data *data, struct config *cfg); struct parser_state parse_message(struct session_data *sdata, int take_into_pieces, struct data *data, struct config *cfg);
void post_parse(struct session_data *sdata, struct parser_state *state, struct config *cfg); void post_parse(struct session_data *sdata, struct data *data, struct parser_state *state, struct config *cfg);
int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg); int parse_line(char *buf, struct parser_state *state, struct session_data *sdata, int take_into_pieces, char *writebuffer, unsigned int writebuffersize, char *abuffer, unsigned int abuffersize, struct data *data, struct config *cfg);
void init_state(struct parser_state *state); void init_state(struct parser_state *state);

2
src/parser_utils.c
View File

@ -90,6 +90,8 @@ void init_state(struct parser_state *state){
memset(state->b_from, 0, SMALLBUFSIZE); memset(state->b_from, 0, SMALLBUFSIZE);
memset(state->b_from_domain, 0, SMALLBUFSIZE); memset(state->b_from_domain, 0, SMALLBUFSIZE);
memset(state->b_sender, 0, SMALLBUFSIZE);
memset(state->b_sender_domain, 0, SMALLBUFSIZE);
memset(state->b_to, 0, MAXBUFSIZE); memset(state->b_to, 0, MAXBUFSIZE);
memset(state->b_to_domain, 0, SMALLBUFSIZE); memset(state->b_to_domain, 0, SMALLBUFSIZE);
memset(state->b_subject, 0, MAXBUFSIZE); memset(state->b_subject, 0, MAXBUFSIZE);

2
src/piler.c
View File

@ -163,7 +163,7 @@ int process_email(char *filename, struct session_data *sdata, struct data *data,
parser_state = parse_message(sdata, 1, data, cfg); parser_state = parse_message(sdata, 1, data, cfg);
post_parse(sdata, &parser_state, cfg); post_parse(sdata, data, &parser_state, cfg);
if(cfg->syslog_recipients == 1){ if(cfg->syslog_recipients == 1){
char *rcpt = parser_state.b_to; char *rcpt = parser_state.b_to;

2
src/reindex.c
View File

@ -124,7 +124,7 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da
snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename); snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
state = parse_message(sdata, 1, data, cfg); state = parse_message(sdata, 1, data, cfg);
post_parse(sdata, &state, cfg); post_parse(sdata, data, &state, cfg);
rc = store_index_data(sdata, &state, data, stored_id, cfg); rc = store_index_data(sdata, &state, data, stored_id, cfg);

5
src/test.c
View File

@ -134,7 +134,7 @@ int main(int argc, char **argv){
init_session_data(&sdata, &cfg); init_session_data(&sdata, &cfg);
sdata.delivered = 0; sdata.delivered = 0;
sdata.tot_len = st.st_size; sdata.tot_len = st.st_size;
sdata.import = 1; sdata.import = 1;
@ -147,10 +147,11 @@ int main(int argc, char **argv){
state = parse_message(&sdata, 1, &data, &cfg); state = parse_message(&sdata, 1, &data, &cfg);
printf("post parsing...\n"); printf("post parsing...\n");
post_parse(&sdata, &state, &cfg); post_parse(&sdata, &data, &state, &cfg);
printf("message-id: %s / %s\n", state.message_id, state.message_id_hash); printf("message-id: %s / %s\n", state.message_id, state.message_id_hash);
printf("from: *%s (%s)*\n", state.b_from, state.b_from_domain); printf("from: *%s (%s)*\n", state.b_from, state.b_from_domain);
printf("sender: *%s (%s)*\n", state.b_sender, state.b_sender_domain);
printf("to: *%s (%s)*\n", state.b_to, state.b_to_domain); printf("to: *%s (%s)*\n", state.b_to, state.b_to_domain);
printf("reference: *%s*\n", state.reference); printf("reference: *%s*\n", state.reference);
printf("subject: *%s*\n", state.b_subject); printf("subject: *%s*\n", state.b_subject);

12
src/tokenizer.c
View File

@ -75,6 +75,18 @@ void tokenize(char *buf, struct parser_state *state, struct session_data *sdata,
} }
} }
} }
else if(state->message_state == MSG_SENDER && state->is_1st_header == 1 && strlen(state->b_sender) < SMALLBUFSIZE-len-1){
strtolower(puf);
q = strchr(puf, '@');
if(q) fix_plus_sign_in_email_address(puf, &q, &len);
memcpy(&(state->b_sender[strlen(state->b_sender)]), puf, len);
if(strlen(state->b_sender) < SMALLBUFSIZE-len-1){
split_email_address(puf);
memcpy(&(state->b_sender[strlen(state->b_sender)]), puf, len);
}
}
else if((state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT || state->message_state == MSG_ENVELOPE_TO) && state->is_1st_header == 1 && state->tolen < MAXBUFSIZE-len-1){ else if((state->message_state == MSG_TO || state->message_state == MSG_CC || state->message_state == MSG_RECIPIENT || state->message_state == MSG_ENVELOPE_TO) && state->is_1st_header == 1 && state->tolen < MAXBUFSIZE-len-1){
strtolower(puf); strtolower(puf);

2
unit_tests/check_attachments.c
View File

@ -43,7 +43,7 @@ static void test_attachments(struct config *cfg){
snprintf(sdata.tmpframe, SMALLBUFSIZE-1, "%s.m", tests[i].s); snprintf(sdata.tmpframe, SMALLBUFSIZE-1, "%s.m", tests[i].s);
state = parse_message(&sdata, 1, &data, cfg); state = parse_message(&sdata, 1, &data, cfg);
post_parse(&sdata, &state, cfg); post_parse(&sdata, &data, &state, cfg);
for(j=1; j<=state.n_attachments; j++){ for(j=1; j<=state.n_attachments; j++){
unlink(state.attachments[j].internalname); unlink(state.attachments[j].internalname);

6
unit_tests/check_rules.c
View File

@ -41,7 +41,7 @@ static void fill_rule_table(struct config *cfg){
for(i=0; i<sizeof(rules)/sizeof(struct rule_query); i++){ for(i=0; i<sizeof(rules)/sizeof(struct rule_query); i++){
p_query(&sdata, rules[i].query); p_query(&sdata, rules[i].query);
rules[i].id = mysql_insert_id(&(sdata.mysql)); rules[i].id = mysql_insert_id(&(sdata.mysql));
} }
close_database(&sdata); close_database(&sdata);
@ -107,7 +107,7 @@ static void test_archiving_rule(struct config *cfg){
} }
init_session_data(&sdata, cfg); init_session_data(&sdata, cfg);
sdata.delivered = 0; sdata.delivered = 0;
sdata.tot_len = st.st_size; sdata.tot_len = st.st_size;
@ -116,7 +116,7 @@ static void test_archiving_rule(struct config *cfg){
snprintf(sdata.tmpframe, SMALLBUFSIZE-1, "%s.m", rule_test[i].filename); snprintf(sdata.tmpframe, SMALLBUFSIZE-1, "%s.m", rule_test[i].filename);
state = parse_message(&sdata, 1, &data, cfg); state = parse_message(&sdata, 1, &data, cfg);
post_parse(&sdata, &state, cfg); post_parse(&sdata, &data, &state, cfg);
rule = check_against_ruleset(data.archiving_rules, &state, st.st_size, sdata.spam_message); rule = check_against_ruleset(data.archiving_rules, &state, st.st_size, sdata.spam_message);

4
unit_tests/common.c
View File

@ -21,7 +21,7 @@ int setup_and_parse_message(struct session_data *sdata, struct parser_state *sta
} }
init_session_data(sdata, cfg); init_session_data(sdata, cfg);
sdata->delivered = 0; sdata->delivered = 0;
sdata->tot_len = st.st_size; sdata->tot_len = st.st_size;
@ -30,7 +30,7 @@ int setup_and_parse_message(struct session_data *sdata, struct parser_state *sta
snprintf(sdata->tmpframe, SMALLBUFSIZE-1, "%s.m", filename); snprintf(sdata->tmpframe, SMALLBUFSIZE-1, "%s.m", filename);
*state = parse_message(sdata, 1, data, cfg); *state = parse_message(sdata, 1, data, cfg);
post_parse(sdata, state, cfg); post_parse(sdata, data, state, cfg);
return 0; return 0;
} }