per user folder feature initial commit

Change-Id: I4b1b90ac0e79e9886eb803dcc28171a5a07f2eda
Signed-off-by: SJ <sj@acts.hu>

configure

@@ -4866,7 +4866,7 @@ echo; echo
 
 CFLAGS="$static -O2 -Wall -g"
 LIBS="$antispam_libs $sunos_libs "
-OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o mydomains.o $objs"
+OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o folder_extra.o mydomains.o $objs"
 
 ac_config_files="$ac_config_files Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile unit_tests/Makefile contrib/imap/Makefile"
 

configure.in

@@ -544,7 +544,7 @@ echo; echo
 
 CFLAGS="$static -O2 -Wall -g"
 LIBS="$antispam_libs $sunos_libs "
-OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o mydomains.o $objs"
+OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o folder_extra.o mydomains.o $objs"
 
 AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile unit_tests/Makefile contrib/imap/Makefile])
 AC_OUTPUT

src/config.h

@@ -13,7 +13,7 @@
 
 #define VERSION "1.2.0-master"
 
-#define BUILD 945
+#define BUILD 946
 
 #define HOSTID "mailarchiver"
 
@@ -83,6 +83,7 @@
 #define SQL_ARCHIVING_RULE_TABLE "archiving_rule"
 #define SQL_RETENTION_RULE_TABLE "retention_rule"
 #define SQL_FOLDER_RULE_TABLE "folder_rule"
+#define SQL_FOLDER_EXTRA_TABLE "folder_extra"
 #define SQL_COUNTER_TABLE "counter"
 #define SQL_OPTION_TABLE "option"
 #define SQL_DOMAIN_TABLE "domain"
@@ -100,14 +101,16 @@
 #define SQL_PREPARED_STMT_INSERT_INTO_META_TABLE     "INSERT INTO " SQL_METADATA_TABLE " (`from`,`fromdomain`,`subject`,`spam`,`arrived`,`sent`,`retained`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`reference`,`digest`,`bodydigest`,`vcode`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
 #define SQL_PREPARED_STMT_UPDATE_META_TABLE          "UPDATE " SQL_METADATA_TABLE " SET `from`=?,`fromdomain`=?,`subject`=?,`spam`=?,`arrived`=?,`sent`=?,`retained`=?,`size`=?,`hlen`=?,`direction`=?,`attachments`=?,`reference`=?,`digest`=?,`bodydigest`=?,`vcode`=? WHERE id=?"
 #define SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE     "INSERT INTO " SQL_ATTACHMENT_TABLE " (`piler_id`,`attachment_id`,`sig`,`name`,`type`,`size`,`ptr`) VALUES(?,?,?,?,?,?,?)"
-#define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE   "SELECT `id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=? AND `ptr`=0 AND `size`=?"
+#define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE   "SELECT `id`, `piler_id`, `attachment_id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=? AND `size`=? AND `ptr`=0"
 #define SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER     "SELECT `piler_id`, `attachment_id` FROM " SQL_ATTACHMENT_TABLE " WHERE id=?"
 #define SQL_PREPARED_STMT_QUERY_ATTACHMENT           "SELECT `attachment_id`, `ptr` FROM " SQL_ATTACHMENT_TABLE " WHERE piler_id=? ORDER BY attachment_id ASC"
 #define SQL_PREPARED_STMT_GET_FOLDER_ID              "SELECT `id` FROM " SQL_FOLDER_TABLE " WHERE `name`=? AND `parent_id`=?"
+#define SQL_PREPARED_STMT_GET_FOLDER_EXTRA_ID        "SELECT `id` FROM " SQL_FOLDER_EXTRA_TABLE " WHERE `name`=? AND `uid`=?"
 #define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE   "INSERT INTO `" SQL_FOLDER_TABLE "` (`name`, `parent_id`) VALUES(?,?)"
+#define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_EXTRA_TABLE   "INSERT INTO `" SQL_FOLDER_EXTRA_TABLE "` (`name`, `uid`) VALUES(?,?)"
 #define SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE  "UPDATE " SQL_METADATA_TABLE " SET reference=? WHERE message_id=? AND reference=''"
 #define SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS        "SELECT id, type, username, password, server FROM " SQL_IMPORT_TABLE " WHERE started=0 ORDER BY id LIMIT 0,1"
-#define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE      "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `id`) VALUES(?,?)"
+#define SQL_PREPARED_STMT_INSERT_FOLDER_MESSAGE      "INSERT INTO " SQL_FOLDER_MESSAGE_TABLE " (`folder_id`, `message_id`, `uid`) VALUES(?,?,?)"
 
 /* Error codes */
 

src/defs.h

@@ -63,6 +63,7 @@
 #define RULE_MATCH 1
 #define RULE_NO_MATCH -100
 
+#define DEDUP_HINT_SIZE RND_STR_LEN+20
 
 typedef void signal_func (int);
 
@@ -83,6 +84,7 @@ struct attachment {
    char filename[TINYBUFSIZE];
    char internalname[TINYBUFSIZE];
    char digest[2*DIGEST_LENGTH+1];
+   char dedup_hint[DEDUP_HINT_SIZE];
    char dumped;
 };
 
@@ -296,6 +298,7 @@ struct import {
    int timeout;
    int reimport;
    int cap_uidplus;
+   int uid;
    long total_size;
    time_t started, updated, finished;
 };

src/folder_extra.c

@@ -0,0 +1,64 @@
+/*
+ * folder_extra.c, SJ
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+int get_folder_extra_id(struct session_data *sdata, struct __data *data, char *foldername){
+   int id=ERR_FOLDER;
+
+   if(prepare_sql_statement(sdata, &(data->stmt_get_folder_id), SQL_PREPARED_STMT_GET_FOLDER_EXTRA_ID) == ERR) return id;
+
+   p_bind_init(data);
+   data->sql[data->pos] = foldername; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = (char *)&(data->import->uid); data->type[data->pos] = TYPE_LONG; data->pos++;
+
+   if(p_exec_query(sdata, data->stmt_get_folder_id, data) == OK){
+
+      p_bind_init(data);
+      data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(unsigned long); data->pos++;
+
+      p_store_results(data->stmt_get_folder_id, data);
+      p_fetch_results(data->stmt_get_folder_id);
+      p_free_results(data->stmt_get_folder_id);
+   }
+
+   close_prepared_statement(data->stmt_get_folder_id);
+
+   return id;
+}
+
+
+int add_new_folder_extra(struct session_data *sdata, struct __data *data, char *foldername){
+   int id=ERR_FOLDER;
+
+   if(foldername == NULL) return id;
+
+   if(prepare_sql_statement(sdata, &(data->stmt_insert_into_folder_table), SQL_PREPARED_STMT_INSERT_INTO_FOLDER_EXTRA_TABLE) == ERR) return id;
+
+   p_bind_init(data);
+   data->sql[data->pos] = foldername; data->type[data->pos] = TYPE_STRING; data->pos++;
+   data->sql[data->pos] = (char *)&(data->import->uid); data->type[data->pos] = TYPE_LONG; data->pos++;
+
+   if(p_exec_query(sdata, data->stmt_insert_into_folder_table, data) == OK){
+      id = p_get_insert_id(data->stmt_insert_into_folder_table);
+   }
+
+   close_prepared_statement(data->stmt_insert_into_folder_table);
+
+   return id;
+}

src/import.c

@@ -96,6 +96,16 @@ int import_message(char *filename, struct session_data *sdata, struct __data *da
          rc = reimport_message(sdata, &state, data, cfg);
       else
          rc = process_message(sdata, &state, data, cfg);
+
+      /*
+       * if pilerimport was invoked with --uid, and this is a duplicate,
+       * then add it to the folder_extra table
+       */
+
+      if(rc == ERR_EXISTS && data->import->uid > 0){
+         store_folder_id(sdata, data, sdata->duplicate_id);
+      }
+
       unlink(state.message_id_hash);
    }
 

src/message.c

@@ -140,6 +140,7 @@ void remove_recipients(struct session_data *sdata, uint64 id){
 
 int store_folder_id(struct session_data *sdata, struct __data *data, uint64 id){
    int rc = ERR;
+   int uid0 = 0;
 
    if(data->folder == ERR_FOLDER) return rc;
 
@@ -149,6 +150,12 @@ int store_folder_id(struct session_data *sdata, struct __data *data, uint64 id){
 
    data->sql[data->pos] = (char *)&data->folder; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
    data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
+   if(sdata->import == 1){
+      data->sql[data->pos] = (char *)&(data->import->uid); data->type[data->pos] = TYPE_LONG; data->pos++;
+   }
+   else {
+      data->sql[data->pos] = (char *)&uid0; data->type[data->pos] = TYPE_LONG; data->pos++;
+   }
 
    if(p_exec_query(sdata, data->stmt_insert_into_folder_message_table, data) == OK) rc = OK;
    close_prepared_statement(data->stmt_insert_into_folder_message_table);

src/piler.h

@@ -44,6 +44,7 @@ int reimport_message(struct session_data *sdata, struct parser_state *state, str
 int store_file(struct session_data *sdata, char *filename, int len, struct __config *cfg);
 int remove_stored_message_files(struct session_data *sdata, struct parser_state *state, struct __config *cfg);
 int store_attachments(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg);
+int store_dedup_hint(struct session_data *sdata, struct parser_state *state, struct __config *cfg);
 int query_attachments(struct session_data *sdata, struct __data *data, struct ptr_array *ptr_arr);
 
 struct __config read_config(char *configfile);
@@ -58,6 +59,11 @@ int file_from_archive_to_network(char *filename, int sd, int tls_enable, struct
 int get_folder_id(struct session_data *sdata, struct __data *data, char *foldername, int parent_id);
 int add_new_folder(struct session_data *sdata, struct __data *data, char *foldername, int parent_id);
 
+int get_folder_extra_id(struct session_data *sdata, struct __data *data, char *foldername);
+int add_new_folder_extra(struct session_data *sdata, struct __data *data, char *foldername);
+
+int store_folder_id(struct session_data *sdata, struct __data *data, uint64 id);
+
 int store_index_data(struct session_data *sdata, struct parser_state *state, struct __data *data, uint64 id, struct __config *cfg);
 
 void extract_attachment_content(struct session_data *sdata, struct parser_state *state, char *filename, char *type, int *rec, struct __config *cfg);

src/pilerimport.c

@@ -50,6 +50,7 @@ void usage(){
    printf("    -f <imap folder>                  IMAP folder name to import\n");
    printf("    -g <imap folder>                  Move email after import to this IMAP folder\n");
    printf("    -F <folder>                       Piler folder name to assign to this import\n");
+   printf("    --uid <uidr>                      UID to assign to this import\n");
    printf("    -R                                Assign IMAP folder names as Piler folder names\n");
    printf("    -b <batch limit>                  Import only this many emails\n");
    printf("    -s <start position>               Start importing POP3 emails from this position\n");
@@ -88,6 +89,7 @@ int main(int argc, char **argv){
    import.start_position = 1;
    import.download_only = 0;
    import.timeout = 30;
+   import.uid = 0;
 
    data.import = &import;
 
@@ -114,6 +116,7 @@ int main(int argc, char **argv){
             {"skiplist",     required_argument,  0,  'x' },
             {"folder",       required_argument,  0,  'F' },
             {"folder_imap",  required_argument,  0,  'f' },
+            {"uid",          required_argument,  0,  'U' },
             {"add-recipient",required_argument,  0,  'a' },
             {"batch-limit",  required_argument,  0,  'b' },
             {"timeout",      required_argument,  0,  't' },
@@ -132,9 +135,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:GDRrozqh?", long_options, &option_index);
+      c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:U:GDRrozqh?", long_options, &option_index);
 #else
-      c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:GDRrozqh?");
+      c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:U:GDRrozqh?");
 #endif
 
       if(c == -1) break;
@@ -179,6 +182,10 @@ int main(int argc, char **argv){
                     username = optarg;
                     break;
 
+         case 'U' :
+                    data.import->uid = atoi(optarg);
+                    break;
+
          case 'p' :
                     password = optarg;
                     break;
@@ -298,10 +305,19 @@ int main(int argc, char **argv){
 #endif
 
    if(folder){
-      data.folder = get_folder_id(&sdata, &data, folder, 0);
+      if(data.import->uid > 0){
+         data.folder = get_folder_extra_id(&sdata, &data, folder);
 
-      if(data.folder == ERR_FOLDER){
+         if(data.folder == ERR_FOLDER){
-         data.folder = add_new_folder(&sdata, &data, folder, 0);
+            data.folder = add_new_folder_extra(&sdata, &data, folder);
+         }
+      }
+      else {
+         data.folder = get_folder_id(&sdata, &data, folder, 0);
+
+         if(data.folder == ERR_FOLDER){
+            data.folder = add_new_folder(&sdata, &data, folder, 0);
+         }
       }
 
       if(data.folder == ERR_FOLDER){
@@ -309,7 +325,6 @@ int main(int argc, char **argv){
          close_database(&sdata);
          return 0;
       }
-
    }
 
    load_rules(&sdata, &data, data.archiving_rules, SQL_ARCHIVING_RULE_TABLE);

util/db-mysql.sql

@@ -297,9 +297,12 @@ create table if not exists `folder_extra` (
 
 
 create table if not exists `folder_message` (
-   folder_id bigint not null,
+   `id` bigint unsigned auto_increment not null,
-   id bigint not null,
+   `folder_id` bigint not null,
-   unique(folder_id, id)
+   `message_id` bigint not null,
+   `uid` int not null,
+   unique(`folder_id`, `message_id`, `uid`),
+   key (`id`)
 ) ENGINE=InnoDB;