From 397904fde79ba068212a76cea398b38535d92cc0 Mon Sep 17 00:00:00 2001 From: SJ Date: Wed, 27 Jul 2016 22:55:43 +0200 Subject: [PATCH] per user folder feature initial commit Change-Id: I4b1b90ac0e79e9886eb803dcc28171a5a07f2eda Signed-off-by: SJ --- configure | 2 +- configure.in | 2 +- src/config.h | 9 ++++--- src/defs.h | 3 +++ src/folder_extra.c | 64 ++++++++++++++++++++++++++++++++++++++++++++++ src/import.c | 10 ++++++++ src/message.c | 7 +++++ src/piler.h | 6 +++++ src/pilerimport.c | 27 ++++++++++++++----- util/db-mysql.sql | 9 ++++--- 10 files changed, 125 insertions(+), 14 deletions(-) create mode 100644 src/folder_extra.c diff --git a/configure b/configure index 74a295d1..cf4b2fc3 100755 --- a/configure +++ b/configure @@ -4866,7 +4866,7 @@ echo; echo CFLAGS="$static -O2 -Wall -g" LIBS="$antispam_libs $sunos_libs " -OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o mydomains.o $objs" +OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o folder_extra.o mydomains.o $objs" ac_config_files="$ac_config_files Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile unit_tests/Makefile contrib/imap/Makefile" diff --git a/configure.in b/configure.in index 3e739634..c645bd6a 100644 --- a/configure.in +++ b/configure.in @@ -544,7 +544,7 @@ echo; echo CFLAGS="$static -O2 -Wall -g" LIBS="$antispam_libs $sunos_libs " -OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o mydomains.o $objs" +OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o folder_extra.o mydomains.o $objs" AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile unit_tests/Makefile contrib/imap/Makefile]) AC_OUTPUT diff --git a/src/config.h b/src/config.h index 9e1056ee..dbfcae4a 100644 --- a/src/config.h +++ b/src/config.h @@ -13,7 +13,7 @@ #define VERSION "1.2.0-master" -#define BUILD 945 +#define BUILD 946 #define HOSTID "mailarchiver" @@ -83,6 +83,7 @@ #define SQL_ARCHIVING_RULE_TABLE "archiving_rule" #define SQL_RETENTION_RULE_TABLE "retention_rule" #define SQL_FOLDER_RULE_TABLE "folder_rule" +#define SQL_FOLDER_EXTRA_TABLE "folder_extra" #define SQL_COUNTER_TABLE "counter" #define SQL_OPTION_TABLE "option" #define SQL_DOMAIN_TABLE "domain" @@ -100,14 +101,16 @@ #define SQL_PREPARED_STMT_INSERT_INTO_META_TABLE "INSERT INTO " SQL_METADATA_TABLE " (`from`,`fromdomain`,`subject`,`spam`,`arrived`,`sent`,`retained`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`reference`,`digest`,`bodydigest`,`vcode`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" #define SQL_PREPARED_STMT_UPDATE_META_TABLE "UPDATE " SQL_METADATA_TABLE " SET `from`=?,`fromdomain`=?,`subject`=?,`spam`=?,`arrived`=?,`sent`=?,`retained`=?,`size`=?,`hlen`=?,`direction`=?,`attachments`=?,`reference`=?,`digest`=?,`bodydigest`=?,`vcode`=? WHERE id=?" #define SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE "INSERT INTO " SQL_ATTACHMENT_TABLE " (`piler_id`,`attachment_id`,`sig`,`name`,`type`,`size`,`ptr`) VALUES(?,?,?,?,?,?,?)" -#define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE "SELECT `id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=? AND `ptr`=0 AND `size`=?" +#define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE "SELECT `id`, `piler_id`, `attachment_id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=? AND `size`=? AND `ptr`=0" #define SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER "SELECT `piler_id`, `attachment_id` FROM " SQL_ATTACHMENT_TABLE " WHERE id=?" #define SQL_PREPARED_STMT_QUERY_ATTACHMENT "SELECT `attachment_id`, `ptr` FROM " SQL_ATTACHMENT_TABLE " WHERE piler_id=? ORDER BY attachment_id ASC" #define SQL_PREPARED_STMT_GET_FOLDER_ID "SELECT `id` FROM " SQL_FOLDER_TABLE " WHERE `name`=? AND `parent_id`=?" +#define SQL_PREPARED_STMT_GET_FOLDER_EXTRA_ID "SELECT `id` FROM " SQL_FOLDER_EXTRA_TABLE " WHERE `name`=? AND `uid`=?" #define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE "INSERT INTO `" SQL_FOLDER_TABLE "` (`name`, `parent_id`) VALUES(?,?)" +#define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_EXTRA_TABLE "INSERT INTO `" SQL_FOLDER_EXTRA_TABLE "` (`name`, `uid`) VALUES(?,?)" #define SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE "UPDATE " SQL_METADATA_TABLE " SET reference=? WHERE message_id=? AND reference=''" #define SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS "SELECT id, type, username, password, server FROM " SQL_IMPORT_TABLE " WHERE started=0 ORDER BY id LIMIT 0,1" -#define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `id`) VALUES(?,?)" +#define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `message_id`, `uid`) VALUES(?,?,?)" /* Error codes */ diff --git a/src/defs.h b/src/defs.h index 33ea813a..05643063 100644 --- a/src/defs.h +++ b/src/defs.h @@ -63,6 +63,7 @@ #define RULE_MATCH 1 #define RULE_NO_MATCH -100 +#define DEDUP_HINT_SIZE RND_STR_LEN+20 typedef void signal_func (int); @@ -83,6 +84,7 @@ struct attachment { char filename[TINYBUFSIZE]; char internalname[TINYBUFSIZE]; char digest[2*DIGEST_LENGTH+1]; + char dedup_hint[DEDUP_HINT_SIZE]; char dumped; }; @@ -296,6 +298,7 @@ struct import { int timeout; int reimport; int cap_uidplus; + int uid; long total_size; time_t started, updated, finished; }; diff --git a/src/folder_extra.c b/src/folder_extra.c new file mode 100644 index 00000000..f258261f --- /dev/null +++ b/src/folder_extra.c @@ -0,0 +1,64 @@ +/* + * folder_extra.c, SJ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +int get_folder_extra_id(struct session_data *sdata, struct __data *data, char *foldername){ + int id=ERR_FOLDER; + + if(prepare_sql_statement(sdata, &(data->stmt_get_folder_id), SQL_PREPARED_STMT_GET_FOLDER_EXTRA_ID) == ERR) return id; + + p_bind_init(data); + data->sql[data->pos] = foldername; data->type[data->pos] = TYPE_STRING; data->pos++; + data->sql[data->pos] = (char *)&(data->import->uid); data->type[data->pos] = TYPE_LONG; data->pos++; + + if(p_exec_query(sdata, data->stmt_get_folder_id, data) == OK){ + + p_bind_init(data); + data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(unsigned long); data->pos++; + + p_store_results(data->stmt_get_folder_id, data); + p_fetch_results(data->stmt_get_folder_id); + p_free_results(data->stmt_get_folder_id); + } + + close_prepared_statement(data->stmt_get_folder_id); + + return id; +} + + +int add_new_folder_extra(struct session_data *sdata, struct __data *data, char *foldername){ + int id=ERR_FOLDER; + + if(foldername == NULL) return id; + + if(prepare_sql_statement(sdata, &(data->stmt_insert_into_folder_table), SQL_PREPARED_STMT_INSERT_INTO_FOLDER_EXTRA_TABLE) == ERR) return id; + + p_bind_init(data); + data->sql[data->pos] = foldername; data->type[data->pos] = TYPE_STRING; data->pos++; + data->sql[data->pos] = (char *)&(data->import->uid); data->type[data->pos] = TYPE_LONG; data->pos++; + + if(p_exec_query(sdata, data->stmt_insert_into_folder_table, data) == OK){ + id = p_get_insert_id(data->stmt_insert_into_folder_table); + } + + close_prepared_statement(data->stmt_insert_into_folder_table); + + return id; +} diff --git a/src/import.c b/src/import.c index 409d1c3d..56a802d2 100644 --- a/src/import.c +++ b/src/import.c @@ -96,6 +96,16 @@ int import_message(char *filename, struct session_data *sdata, struct __data *da rc = reimport_message(sdata, &state, data, cfg); else rc = process_message(sdata, &state, data, cfg); + + /* + * if pilerimport was invoked with --uid, and this is a duplicate, + * then add it to the folder_extra table + */ + + if(rc == ERR_EXISTS && data->import->uid > 0){ + store_folder_id(sdata, data, sdata->duplicate_id); + } + unlink(state.message_id_hash); } diff --git a/src/message.c b/src/message.c index c120cd24..50c051c0 100644 --- a/src/message.c +++ b/src/message.c @@ -140,6 +140,7 @@ void remove_recipients(struct session_data *sdata, uint64 id){ int store_folder_id(struct session_data *sdata, struct __data *data, uint64 id){ int rc = ERR; + int uid0 = 0; if(data->folder == ERR_FOLDER) return rc; @@ -149,6 +150,12 @@ int store_folder_id(struct session_data *sdata, struct __data *data, uint64 id){ data->sql[data->pos] = (char *)&data->folder; data->type[data->pos] = TYPE_LONGLONG; data->pos++; data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->pos++; + if(sdata->import == 1){ + data->sql[data->pos] = (char *)&(data->import->uid); data->type[data->pos] = TYPE_LONG; data->pos++; + } + else { + data->sql[data->pos] = (char *)&uid0; data->type[data->pos] = TYPE_LONG; data->pos++; + } if(p_exec_query(sdata, data->stmt_insert_into_folder_message_table, data) == OK) rc = OK; close_prepared_statement(data->stmt_insert_into_folder_message_table); diff --git a/src/piler.h b/src/piler.h index aa1a2a8d..e3875417 100644 --- a/src/piler.h +++ b/src/piler.h @@ -44,6 +44,7 @@ int reimport_message(struct session_data *sdata, struct parser_state *state, str int store_file(struct session_data *sdata, char *filename, int len, struct __config *cfg); int remove_stored_message_files(struct session_data *sdata, struct parser_state *state, struct __config *cfg); int store_attachments(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg); +int store_dedup_hint(struct session_data *sdata, struct parser_state *state, struct __config *cfg); int query_attachments(struct session_data *sdata, struct __data *data, struct ptr_array *ptr_arr); struct __config read_config(char *configfile); @@ -58,6 +59,11 @@ int file_from_archive_to_network(char *filename, int sd, int tls_enable, struct int get_folder_id(struct session_data *sdata, struct __data *data, char *foldername, int parent_id); int add_new_folder(struct session_data *sdata, struct __data *data, char *foldername, int parent_id); +int get_folder_extra_id(struct session_data *sdata, struct __data *data, char *foldername); +int add_new_folder_extra(struct session_data *sdata, struct __data *data, char *foldername); + +int store_folder_id(struct session_data *sdata, struct __data *data, uint64 id); + int store_index_data(struct session_data *sdata, struct parser_state *state, struct __data *data, uint64 id, struct __config *cfg); void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct __config *cfg); diff --git a/src/pilerimport.c b/src/pilerimport.c index de61a4aa..1608d494 100644 --- a/src/pilerimport.c +++ b/src/pilerimport.c @@ -50,6 +50,7 @@ void usage(){ printf(" -f IMAP folder name to import\n"); printf(" -g Move email after import to this IMAP folder\n"); printf(" -F Piler folder name to assign to this import\n"); + printf(" --uid UID to assign to this import\n"); printf(" -R Assign IMAP folder names as Piler folder names\n"); printf(" -b Import only this many emails\n"); printf(" -s Start importing POP3 emails from this position\n"); @@ -88,6 +89,7 @@ int main(int argc, char **argv){ import.start_position = 1; import.download_only = 0; import.timeout = 30; + import.uid = 0; data.import = &import; @@ -114,6 +116,7 @@ int main(int argc, char **argv){ {"skiplist", required_argument, 0, 'x' }, {"folder", required_argument, 0, 'F' }, {"folder_imap", required_argument, 0, 'f' }, + {"uid", required_argument, 0, 'U' }, {"add-recipient",required_argument, 0, 'a' }, {"batch-limit", required_argument, 0, 'b' }, {"timeout", required_argument, 0, 't' }, @@ -132,9 +135,9 @@ int main(int argc, char **argv){ int option_index = 0; - c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:GDRrozqh?", long_options, &option_index); + c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:U:GDRrozqh?", long_options, &option_index); #else - c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:GDRrozqh?"); + c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:U:GDRrozqh?"); #endif if(c == -1) break; @@ -179,6 +182,10 @@ int main(int argc, char **argv){ username = optarg; break; + case 'U' : + data.import->uid = atoi(optarg); + break; + case 'p' : password = optarg; break; @@ -298,10 +305,19 @@ int main(int argc, char **argv){ #endif if(folder){ - data.folder = get_folder_id(&sdata, &data, folder, 0); + if(data.import->uid > 0){ + data.folder = get_folder_extra_id(&sdata, &data, folder); - if(data.folder == ERR_FOLDER){ - data.folder = add_new_folder(&sdata, &data, folder, 0); + if(data.folder == ERR_FOLDER){ + data.folder = add_new_folder_extra(&sdata, &data, folder); + } + } + else { + data.folder = get_folder_id(&sdata, &data, folder, 0); + + if(data.folder == ERR_FOLDER){ + data.folder = add_new_folder(&sdata, &data, folder, 0); + } } if(data.folder == ERR_FOLDER){ @@ -309,7 +325,6 @@ int main(int argc, char **argv){ close_database(&sdata); return 0; } - } load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE); diff --git a/util/db-mysql.sql b/util/db-mysql.sql index 26c9d309..2c20abf8 100644 --- a/util/db-mysql.sql +++ b/util/db-mysql.sql @@ -297,9 +297,12 @@ create table if not exists `folder_extra` ( create table if not exists `folder_message` ( - folder_id bigint not null, - id bigint not null, - unique(folder_id, id) + `id` bigint unsigned auto_increment not null, + `folder_id` bigint not null, + `message_id` bigint not null, + `uid` int not null, + unique(`folder_id`, `message_id`, `uid`), + key (`id`) ) ENGINE=InnoDB;