bashclub/piler
1
0
Fork 0
mirror of https://bitbucket.org/jsuto/piler.git synced 2025-07-08 20:59:08 +02:00
Code Issues Packages Projects Releases Wiki Activity

merged folder branch

Browse Source 
Change-Id: I57f3427b128e8b313572e7bc3734933f4d345d1c
Signed-off-by: SJ <sj@acts.hu>
This commit is contained in:
SJ
2016-12-20 22:09:20 +01:00
parent 12292111a8 c1095a0ab5
commit 2ca7c9d30e
18 changed files with 131 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/config.h
View File

@ -83,6 +83,8 @@
#define SQL_ARCHIVING_RULE_TABLE "archiving_rule"
#define SQL_RETENTION_RULE_TABLE "retention_rule"
#define SQL_FOLDER_RULE_TABLE "folder_rule"
#define SQL_FOLDER_TABLE "folder"
#define SQL_FOLDER_EMAIL_TABLE "folder_email"
#define SQL_COUNTER_TABLE "counter"
#define SQL_OPTION_TABLE "option"
#define SQL_DOMAIN_TABLE "domain"
@ -99,14 +101,16 @@
#define SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE "INSERT INTO " SQL_SPHINX_TABLE " (`id`, `from`, `to`, `fromdomain`, `todomain`, `subject`, `body`, `arrived`, `sent`, `size`, `direction`, `folder`, `attachments`, `attachment_types`) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
#define SQL_PREPARED_STMT_INSERT_INTO_META_TABLE "INSERT INTO " SQL_METADATA_TABLE " (`from`,`fromdomain`,`subject`,`spam`,`arrived`,`sent`,`retained`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`reference`,`digest`,`bodydigest`,`vcode`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
#define SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE "INSERT INTO " SQL_ATTACHMENT_TABLE " (`piler_id`,`attachment_id`,`sig`,`name`,`type`,`size`,`ptr`) VALUES(?,?,?,?,?,?,?)"
#define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE "SELECT `id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=? AND `ptr`=0 AND `size`=?"
#define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE "SELECT `id`, `piler_id`, `attachment_id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=? AND `ptr`=0 AND `size`=?"
#define SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER "SELECT `piler_id`, `attachment_id` FROM " SQL_ATTACHMENT_TABLE " WHERE id=?"
#define SQL_PREPARED_STMT_QUERY_ATTACHMENT "SELECT `attachment_id`, `ptr` FROM " SQL_ATTACHMENT_TABLE " WHERE piler_id=? ORDER BY attachment_id ASC"
#define SQL_PREPARED_STMT_GET_FOLDER_ID "SELECT `id` FROM " SQL_FOLDER_TABLE " WHERE `name`=? AND `parent_id`=?"
#define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE "INSERT INTO `" SQL_FOLDER_TABLE "` (`name`, `parent_id`) VALUES(?,?)"
#define SQL_PREPARED_STMT_GET_FOLDER_ID "SELECT `id` FROM " SQL_FOLDER_TABLE " WHERE `name`=? AND `uid`=? AND `parent_id`=?"
#define SQL_PREPARED_STMT_GET_FOLDER_UID_BY_EMAIL "SELECT `uid` FROM " SQL_FOLDER_EMAIL_TABLE " WHERE `email`=?"
#define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE "INSERT INTO `" SQL_FOLDER_TABLE "` (`uid`, `name`, `parent_id`) VALUES(?,?,?)"
#define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_EMAIL_TABLE "INSERT INTO `" SQL_FOLDER_EMAIL_TABLE "` (`email`) VALUES(?)"
#define SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE "UPDATE " SQL_METADATA_TABLE " SET reference=? WHERE message_id=? AND reference=''"
#define SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS "SELECT id, type, username, password, server FROM " SQL_IMPORT_TABLE " WHERE started=0 ORDER BY id LIMIT 0,1"
#define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `id`) VALUES(?,?)"
#define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `message_id`, `uid`) VALUES(?,?,?)"
/* Error codes */
@ -135,4 +139,3 @@
#define S_STATUS_ERROR "error"
#endif /* _CONFIG_H */

4
src/defs.h
View File

@ -63,6 +63,7 @@
#define RULE_MATCH 1
#define RULE_NO_MATCH -100
#define DEDUP_HINT_SIZE RND_STR_LEN+20
typedef void signal_func (int);
@ -83,6 +84,7 @@ struct attachment {
char filename[TINYBUFSIZE];
char internalname[TINYBUFSIZE];
char digest[2*DIGEST_LENGTH+1];
char dedup_hint[DEDUP_HINT_SIZE];
char dumped;
};
@ -284,6 +286,7 @@ struct memcached_server {
struct import {
char *extra_recipient;
char *move_folder;
char *email;
int status;
int total_messages;
int processed_messages;
@ -295,6 +298,7 @@ struct import {
int keep_eml;
int timeout;
int cap_uidplus;
int uid;
long total_size;
time_t started, updated, finished;
};

56
src/import.c
View File

@ -93,6 +93,16 @@ int import_message(char *filename, struct session_data *sdata, struct __data *da
}
rc = process_message(sdata, &state, data, cfg);
/*
* if pilerimport was invoked with --email (then queried the matching uid!),
* and this is a duplicate, then add it to the folder_extra table
*/
if(rc == ERR_EXISTS && data->import->uid > 0){
store_folder_id(sdata, data, sdata->duplicate_id);
}
unlink(state.message_id_hash);
}
@ -130,52 +140,6 @@ int import_message(char *filename, struct session_data *sdata, struct __data *da
}
int get_folder_id(struct session_data *sdata, struct __data *data, char *foldername, int parent_id){
int id=ERR_FOLDER;
if(prepare_sql_statement(sdata, &(data->stmt_get_folder_id), SQL_PREPARED_STMT_GET_FOLDER_ID) == ERR) return id;
p_bind_init(data);
data->sql[data->pos] = foldername; data->type[data->pos] = TYPE_STRING; data->pos++;
data->sql[data->pos] = (char *)&parent_id; data->type[data->pos] = TYPE_LONG; data->pos++;
if(p_exec_query(sdata, data->stmt_get_folder_id, data) == OK){
p_bind_init(data);
data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(unsigned long); data->pos++;
p_store_results(data->stmt_get_folder_id, data);
p_fetch_results(data->stmt_get_folder_id);
p_free_results(data->stmt_get_folder_id);
}
close_prepared_statement(data->stmt_get_folder_id);
return id;
}
int add_new_folder(struct session_data *sdata, struct __data *data, char *foldername, int parent_id){
int id=ERR_FOLDER;
if(foldername == NULL) return id;
if(prepare_sql_statement(sdata, &(data->stmt_insert_into_folder_table), SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE) == ERR) return id;
p_bind_init(data);
data->sql[data->pos] = foldername; data->type[data->pos] = TYPE_STRING; data->pos++;
data->sql[data->pos] = (char *)&parent_id; data->type[data->pos] = TYPE_LONG; data->pos++;
if(p_exec_query(sdata, data->stmt_insert_into_folder_table, data) == OK){
id = p_get_insert_id(data->stmt_insert_into_folder_table);
}
close_prepared_statement(data->stmt_insert_into_folder_table);
return id;
}
void update_import_job_stat(struct session_data *sdata, struct __data *data){
char buf[SMALLBUFSIZE];

7
src/message.c
View File

@ -140,6 +140,7 @@ void remove_recipients(struct session_data *sdata, uint64 id){
int store_folder_id(struct session_data *sdata, struct __data *data, uint64 id){
int rc = ERR;
int uid0 = 0;
if(data->folder == ERR_FOLDER) return rc;
@ -149,6 +150,12 @@ int store_folder_id(struct session_data *sdata, struct __data *data, uint64 id){
data->sql[data->pos] = (char *)&data->folder; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
if(sdata->import == 1){
data->sql[data->pos] = (char *)&(data->import->uid); data->type[data->pos] = TYPE_LONG; data->pos++;
}
else {
data->sql[data->pos] = (char *)&uid0; data->type[data->pos] = TYPE_LONG; data->pos++;
}
if(p_exec_query(sdata, data->stmt_insert_into_folder_message_table, data) == OK) rc = OK;
close_prepared_statement(data->stmt_insert_into_folder_message_table);

4
src/piler.h
View File

@ -43,6 +43,7 @@ int reimport_message(struct session_data *sdata, struct parser_state *state, str
int store_file(struct session_data *sdata, char *filename, int len, struct __config *cfg);
int remove_stored_message_files(struct session_data *sdata, struct parser_state *state, struct __config *cfg);
int store_attachments(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg);
int store_dedup_hint(struct session_data *sdata, struct parser_state *state, struct __config *cfg);
int query_attachments(struct session_data *sdata, struct __data *data, struct ptr_array *ptr_arr);
struct __config read_config(char *configfile);
@ -56,6 +57,9 @@ int file_from_archive_to_network(char *filename, int sd, int tls_enable, struct
int get_folder_id(struct session_data *sdata, struct __data *data, char *foldername, int parent_id);
int add_new_folder(struct session_data *sdata, struct __data *data, char *foldername, int parent_id);
void get_folder_uid_by_email(struct session_data *sdata, struct __data *data);
int store_folder_id(struct session_data *sdata, struct __data *data, uint64 id);
int store_index_data(struct session_data *sdata, struct parser_state *state, struct __data *data, uint64 id, struct __config *cfg);

28
src/pilerimport.c
View File

@ -50,6 +50,7 @@ void usage(){
printf(" -f <imap folder> IMAP folder name to import\n");
printf(" -g <imap folder> Move email after import to this IMAP folder\n");
printf(" -F <folder> Piler folder name to assign to this import\n");
printf(" --email <email@address> Email address to assign to this import\n");
printf(" -R Assign IMAP folder names as Piler folder names\n");
printf(" -b <batch limit> Import only this many emails\n");
printf(" -s <start position> Start importing POP3 emails from this position\n");
@ -87,6 +88,8 @@ int main(int argc, char **argv){
import.start_position = 1;
import.download_only = 0;
import.timeout = 30;
import.uid = 0;
import.email = NULL;
data.import = &import;
@ -113,6 +116,7 @@ int main(int argc, char **argv){
{"skiplist", required_argument, 0, 'x' },
{"folder", required_argument, 0, 'F' },
{"folder_imap", required_argument, 0, 'f' },
{"email", required_argument, 0, 'E' },
{"add-recipient",required_argument, 0, 'a' },
{"batch-limit", required_argument, 0, 'b' },
{"timeout", required_argument, 0, 't' },
@ -130,9 +134,9 @@ int main(int argc, char **argv){
int option_index = 0;
c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:GDRroqh?", long_options, &option_index);
c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:E:GDRroqh?", long_options, &option_index);
#else
c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:GDRroqh?");
c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:E:GDRroqh?");
#endif
if(c == -1) break;
@ -177,6 +181,10 @@ int main(int argc, char **argv){
username = optarg;
break;
case 'E' :
data.import->email = optarg;
break;
case 'p' :
password = optarg;
break;
@ -266,11 +274,6 @@ int main(int argc, char **argv){
cfg = read_config(configfile);
if((data.recursive_folder_names == 1 || folder) && cfg.enable_folders == 0){
printf("please set enable_folders=1 in piler.conf to use the folder options\n");
return ERR;
}
/* make sure we don't discard messages without a valid Message-Id when importing manually */
cfg.archive_emails_not_having_message_id = 1;
@ -293,7 +296,17 @@ int main(int argc, char **argv){
memcached_init(&(data.memc), cfg.memcached_servers, 11211);
#endif
if(folder || data.recursive_folder_names == 1){
cfg.enable_folders = 1;
if(data.import->email){
get_folder_uid_by_email(&sdata, &data);
}
}
if(folder){
data.folder = get_folder_id(&sdata, &data, folder, 0);
if(data.folder == ERR_FOLDER){
@ -305,7 +318,6 @@ int main(int argc, char **argv){
close_database(&sdata);
return 0;
}
}
load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);