bashclub/piler
1
0
Fork 0
mirror of https://bitbucket.org/jsuto/piler.git synced 2025-11-04 16:12:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

per user folder feature initial commit

Browse Source 
Change-Id: I4b1b90ac0e79e9886eb803dcc28171a5a07f2eda
Signed-off-by: SJ <sj@acts.hu>
This commit is contained in:
SJ
2016-07-27 22:55:43 +02:00
parent 2abdc19f7d
commit 397904fde7
10 changed files with 125 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
configure vendored
View File

@@ -4866,7 +4866,7 @@ echo; echo
CFLAGS="$static -O2 -Wall -g"
LIBS="$antispam_libs $sunos_libs "
OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o mydomains.o $objs"
OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o folder_extra.o mydomains.o $objs"
ac_config_files="$ac_config_files Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile unit_tests/Makefile contrib/imap/Makefile"

2
configure.in
View File

@@ -544,7 +544,7 @@ echo; echo
CFLAGS="$static -O2 -Wall -g"
LIBS="$antispam_libs $sunos_libs "
OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o mydomains.o $objs"
OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o folder_extra.o mydomains.o $objs"
AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile unit_tests/Makefile contrib/imap/Makefile])
AC_OUTPUT

9
src/config.h
View File

@@ -13,7 +13,7 @@
#define VERSION "1.2.0-master"
#define BUILD 945
#define BUILD 946
#define HOSTID "mailarchiver"
@@ -83,6 +83,7 @@
#define SQL_ARCHIVING_RULE_TABLE "archiving_rule"
#define SQL_RETENTION_RULE_TABLE "retention_rule"
#define SQL_FOLDER_RULE_TABLE "folder_rule"
#define SQL_FOLDER_EXTRA_TABLE "folder_extra"
#define SQL_COUNTER_TABLE "counter"
#define SQL_OPTION_TABLE "option"
#define SQL_DOMAIN_TABLE "domain"
@@ -100,14 +101,16 @@
#define SQL_PREPARED_STMT_INSERT_INTO_META_TABLE "INSERT INTO " SQL_METADATA_TABLE " (`from`,`fromdomain`,`subject`,`spam`,`arrived`,`sent`,`retained`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`reference`,`digest`,`bodydigest`,`vcode`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
#define SQL_PREPARED_STMT_UPDATE_META_TABLE "UPDATE " SQL_METADATA_TABLE " SET `from`=?,`fromdomain`=?,`subject`=?,`spam`=?,`arrived`=?,`sent`=?,`retained`=?,`size`=?,`hlen`=?,`direction`=?,`attachments`=?,`reference`=?,`digest`=?,`bodydigest`=?,`vcode`=? WHERE id=?"
#define SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE "INSERT INTO " SQL_ATTACHMENT_TABLE " (`piler_id`,`attachment_id`,`sig`,`name`,`type`,`size`,`ptr`) VALUES(?,?,?,?,?,?,?)"
#define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE "SELECT `id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=? AND `ptr`=0 AND `size`=?"
#define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE "SELECT `id`, `piler_id`, `attachment_id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=? AND `size`=? AND `ptr`=0"
#define SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER "SELECT `piler_id`, `attachment_id` FROM " SQL_ATTACHMENT_TABLE " WHERE id=?"
#define SQL_PREPARED_STMT_QUERY_ATTACHMENT "SELECT `attachment_id`, `ptr` FROM " SQL_ATTACHMENT_TABLE " WHERE piler_id=? ORDER BY attachment_id ASC"
#define SQL_PREPARED_STMT_GET_FOLDER_ID "SELECT `id` FROM " SQL_FOLDER_TABLE " WHERE `name`=? AND `parent_id`=?"
#define SQL_PREPARED_STMT_GET_FOLDER_EXTRA_ID "SELECT `id` FROM " SQL_FOLDER_EXTRA_TABLE " WHERE `name`=? AND `uid`=?"
#define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE "INSERT INTO `" SQL_FOLDER_TABLE "` (`name`, `parent_id`) VALUES(?,?)"
#define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_EXTRA_TABLE "INSERT INTO `" SQL_FOLDER_EXTRA_TABLE "` (`name`, `uid`) VALUES(?,?)"
#define SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE "UPDATE " SQL_METADATA_TABLE " SET reference=? WHERE message_id=? AND reference=''"
#define SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS "SELECT id, type, username, password, server FROM " SQL_IMPORT_TABLE " WHERE started=0 ORDER BY id LIMIT 0,1"
#define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `id`) VALUES(?,?)"
#define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `message_id`, `uid`) VALUES(?,?,?)"
/* Error codes */

3
src/defs.h
View File

@@ -63,6 +63,7 @@
#define RULE_MATCH 1
#define RULE_NO_MATCH -100
#define DEDUP_HINT_SIZE RND_STR_LEN+20
typedef void signal_func (int);
@@ -83,6 +84,7 @@ struct attachment {
char filename[TINYBUFSIZE];
char internalname[TINYBUFSIZE];
char digest[2*DIGEST_LENGTH+1];
char dedup_hint[DEDUP_HINT_SIZE];
char dumped;
};
@@ -296,6 +298,7 @@ struct import {
int timeout;
int reimport;
int cap_uidplus;
int uid;
long total_size;
time_t started, updated, finished;
};

64
src/folder_extra.c Normal file
View File

@@ -0,0 +1,64 @@
/*
* folder_extra.c, SJ
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <dirent.h>
#include <unistd.h>
#include <time.h>
#include <locale.h>
#include <syslog.h>
#include <piler.h>
int get_folder_extra_id(struct session_data *sdata, struct __data *data, char *foldername){
int id=ERR_FOLDER;
if(prepare_sql_statement(sdata, &(data->stmt_get_folder_id), SQL_PREPARED_STMT_GET_FOLDER_EXTRA_ID) == ERR) return id;
p_bind_init(data);
data->sql[data->pos] = foldername; data->type[data->pos] = TYPE_STRING; data->pos++;
data->sql[data->pos] = (char *)&(data->import->uid); data->type[data->pos] = TYPE_LONG; data->pos++;
if(p_exec_query(sdata, data->stmt_get_folder_id, data) == OK){
p_bind_init(data);
data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(unsigned long); data->pos++;
p_store_results(data->stmt_get_folder_id, data);
p_fetch_results(data->stmt_get_folder_id);
p_free_results(data->stmt_get_folder_id);
}
close_prepared_statement(data->stmt_get_folder_id);
return id;
}
int add_new_folder_extra(struct session_data *sdata, struct __data *data, char *foldername){
int id=ERR_FOLDER;
if(foldername == NULL) return id;
if(prepare_sql_statement(sdata, &(data->stmt_insert_into_folder_table), SQL_PREPARED_STMT_INSERT_INTO_FOLDER_EXTRA_TABLE) == ERR) return id;
p_bind_init(data);
data->sql[data->pos] = foldername; data->type[data->pos] = TYPE_STRING; data->pos++;
data->sql[data->pos] = (char *)&(data->import->uid); data->type[data->pos] = TYPE_LONG; data->pos++;
if(p_exec_query(sdata, data->stmt_insert_into_folder_table, data) == OK){
id = p_get_insert_id(data->stmt_insert_into_folder_table);
}
close_prepared_statement(data->stmt_insert_into_folder_table);
return id;
}

10
src/import.c
View File

@@ -96,6 +96,16 @@ int import_message(char *filename, struct session_data *sdata, struct __data *da
rc = reimport_message(sdata, &state, data, cfg);
else
rc = process_message(sdata, &state, data, cfg);
/*
* if pilerimport was invoked with --uid, and this is a duplicate,
* then add it to the folder_extra table
*/
if(rc == ERR_EXISTS && data->import->uid > 0){
store_folder_id(sdata, data, sdata->duplicate_id);
}
unlink(state.message_id_hash);
}

7
src/message.c
View File

@@ -140,6 +140,7 @@ void remove_recipients(struct session_data *sdata, uint64 id){
int store_folder_id(struct session_data *sdata, struct __data *data, uint64 id){
int rc = ERR;
int uid0 = 0;
if(data->folder == ERR_FOLDER) return rc;
@@ -149,6 +150,12 @@ int store_folder_id(struct session_data *sdata, struct __data *data, uint64 id){
data->sql[data->pos] = (char *)&data->folder; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
if(sdata->import == 1){
data->sql[data->pos] = (char *)&(data->import->uid); data->type[data->pos] = TYPE_LONG; data->pos++;
}
else {
data->sql[data->pos] = (char *)&uid0; data->type[data->pos] = TYPE_LONG; data->pos++;
}
if(p_exec_query(sdata, data->stmt_insert_into_folder_message_table, data) == OK) rc = OK;
close_prepared_statement(data->stmt_insert_into_folder_message_table);

6
src/piler.h
View File

@@ -44,6 +44,7 @@ int reimport_message(struct session_data *sdata, struct parser_state *state, str
int store_file(struct session_data *sdata, char *filename, int len, struct __config *cfg);
int remove_stored_message_files(struct session_data *sdata, struct parser_state *state, struct __config *cfg);
int store_attachments(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg);
int store_dedup_hint(struct session_data *sdata, struct parser_state *state, struct __config *cfg);
int query_attachments(struct session_data *sdata, struct __data *data, struct ptr_array *ptr_arr);
struct __config read_config(char *configfile);
@@ -58,6 +59,11 @@ int file_from_archive_to_network(char *filename, int sd, int tls_enable, struct
int get_folder_id(struct session_data *sdata, struct __data *data, char *foldername, int parent_id);
int add_new_folder(struct session_data *sdata, struct __data *data, char *foldername, int parent_id);
int get_folder_extra_id(struct session_data *sdata, struct __data *data, char *foldername);
int add_new_folder_extra(struct session_data *sdata, struct __data *data, char *foldername);
int store_folder_id(struct session_data *sdata, struct __data *data, uint64 id);
int store_index_data(struct session_data *sdata, struct parser_state *state, struct __data *data, uint64 id, struct __config *cfg);
void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct __config *cfg);

27
src/pilerimport.c
View File

@@ -50,6 +50,7 @@ void usage(){
printf(" -f <imap folder> IMAP folder name to import\n");
printf(" -g <imap folder> Move email after import to this IMAP folder\n");
printf(" -F <folder> Piler folder name to assign to this import\n");
printf(" --uid <uidr> UID to assign to this import\n");
printf(" -R Assign IMAP folder names as Piler folder names\n");
printf(" -b <batch limit> Import only this many emails\n");
printf(" -s <start position> Start importing POP3 emails from this position\n");
@@ -88,6 +89,7 @@ int main(int argc, char **argv){
import.start_position = 1;
import.download_only = 0;
import.timeout = 30;
import.uid = 0;
data.import = &import;
@@ -114,6 +116,7 @@ int main(int argc, char **argv){
{"skiplist", required_argument, 0, 'x' },
{"folder", required_argument, 0, 'F' },
{"folder_imap", required_argument, 0, 'f' },
{"uid", required_argument, 0, 'U' },
{"add-recipient",required_argument, 0, 'a' },
{"batch-limit", required_argument, 0, 'b' },
{"timeout", required_argument, 0, 't' },
@@ -132,9 +135,9 @@ int main(int argc, char **argv){
int option_index = 0;
c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:GDRrozqh?", long_options, &option_index);
c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:U:GDRrozqh?", long_options, &option_index);
#else
c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:GDRrozqh?");
c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:U:GDRrozqh?");
#endif
if(c == -1) break;
@@ -179,6 +182,10 @@ int main(int argc, char **argv){
username = optarg;
break;
case 'U' :
data.import->uid = atoi(optarg);
break;
case 'p' :
password = optarg;
break;
@@ -298,10 +305,19 @@ int main(int argc, char **argv){
#endif
if(folder){
data.folder = get_folder_id(&sdata, &data, folder, 0);
if(data.import->uid > 0){
data.folder = get_folder_extra_id(&sdata, &data, folder);
if(data.folder == ERR_FOLDER){
data.folder = add_new_folder(&sdata, &data, folder, 0);
if(data.folder == ERR_FOLDER){
data.folder = add_new_folder_extra(&sdata, &data, folder);
}
}
else {
data.folder = get_folder_id(&sdata, &data, folder, 0);
if(data.folder == ERR_FOLDER){
data.folder = add_new_folder(&sdata, &data, folder, 0);
}
}
if(data.folder == ERR_FOLDER){
@@ -309,7 +325,6 @@ int main(int argc, char **argv){
close_database(&sdata);
return 0;
}
}
load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);

9
util/db-mysql.sql
View File

@@ -297,9 +297,12 @@ create table if not exists `folder_extra` (
create table if not exists `folder_message` (
folder_id bigint not null,
id bigint not null,
unique(folder_id, id)
`id` bigint unsigned auto_increment not null,
`folder_id` bigint not null,
`message_id` bigint not null,
`uid` int not null,
unique(`folder_id`, `message_id`, `uid`),
key (`id`)
) ENGINE=InnoDB;