code refactoring around import

configure

@@ -4889,7 +4889,7 @@ echo; echo
 
 CFLAGS="$static -O2 -Wall -g"
 LIBS="$antispam_libs $sunos_libs "
-OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o imap.o pop3.o extract.o mydomains.o $objs"
+OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o mydomains.o $objs"
 
 ac_config_files="$ac_config_files Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile unit_tests/Makefile contrib/imap/Makefile"
 

configure.in

@@ -556,7 +556,7 @@ echo; echo
 
 CFLAGS="$static -O2 -Wall -g"
 LIBS="$antispam_libs $sunos_libs "
-OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o imap.o pop3.o extract.o mydomains.o $objs"
+OBJS="dirs.o base64.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o import_gui.o imap.o pop3.o extract.o mydomains.o $objs"
 
 AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile unit_tests/Makefile contrib/imap/Makefile])
 AC_OUTPUT

src/config.h

@@ -13,7 +13,7 @@
 
 #define VERSION "1.2.0-master"
 
-#define BUILD 938
+#define BUILD 939
 
 #define HOSTID "mailarchiver"
 

src/imap.c

@@ -392,18 +392,6 @@ void send_imap_close(int sd, int *seq, struct __data *data, int use_ssl){
 }
 
 
-void close_connection(int sd, struct __data *data, int use_ssl){
-   close(sd);
-
-   if(use_ssl == 1){
-      SSL_shutdown(data->ssl);
-      SSL_free(data->ssl);
-      SSL_CTX_free(data->ctx);
-      ERR_free_strings();
-   }
-}
-
-
 int list_folders(int sd, int *seq, int use_ssl, struct __data *data){
    char *p, *q, *r, *buf, *ruf, tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], puf[MAXBUFSIZE];
    int len=MAXBUFSIZE+3, pos=0, n, rc=ERR, fldrlen=0, result;

src/import.c

@@ -176,7 +176,7 @@ int add_new_folder(struct session_data *sdata, struct __data *data, char *folder
 }
 
 
-void update_import_job_stat(struct session_data *sdata, struct __data *data, struct __config *cfg){
+void update_import_job_stat(struct session_data *sdata, struct __data *data){
    char buf[SMALLBUFSIZE];
 
    snprintf(buf, sizeof(buf)-1, "update import set status=%d, started=%ld, updated=%ld, finished=%ld, total=%d, imported=%d where id=%d", data->import->status, data->import->started, data->import->updated, data->import->finished, data->import->total_messages, data->import->processed_messages, data->import->import_job_id);

src/import.h

@@ -0,0 +1,29 @@
+/*
+ * import.h, SJ
+ */
+
+#ifndef _IMPORT_H
+   #define _IMPORT_H
+
+
+int import_message(char *filename, struct session_data *sdata, struct __data *data, struct __config *cfg);
+void update_import_job_stat(struct session_data *sdata, struct __data *data);
+
+int read_gui_import_data(struct session_data *sdata, struct __data *data, char *folder_imap, char *skiplist, int dryrun, struct __config *cfg);
+
+int import_from_maildir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg);
+int import_from_mailbox(char *mailbox, struct session_data *sdata, struct __data *data, struct __config *cfg);
+int import_mbox_from_dir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg);
+int import_from_pop3_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, int dryrun, struct __config *cfg);
+int import_from_imap_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, char *folder_imap, char *skiplist, int dryrun, struct __config *cfg);
+
+int connect_to_pop3_server(int sd, char *username, char *password, int port, struct __data *data, int use_ssl);
+int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg);
+
+int connect_to_imap_server(int sd, int *seq, char *username, char *password, int port, struct __data *data, int use_ssl);
+int list_folders(int sd, int *seq, int use_ssl, struct __data *data);
+int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg);
+void send_imap_close(int sd, int *seq, struct __data *data, int use_ssl);
+
+#endif /* _IMPORT_H */
+

src/import_gui.c

@@ -0,0 +1,80 @@
+/*
+ * import_gui.c, SJ
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <getopt.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+int read_gui_import_data(struct session_data *sdata, struct __data *data, char *folder_imap, char *skiplist, int dryrun, struct __config *cfg){
+   int rc=ERR;
+   char s_type[SMALLBUFSIZE], s_username[SMALLBUFSIZE], s_password[SMALLBUFSIZE], s_server[SMALLBUFSIZE];
+
+   memset(s_type, 0, sizeof(s_type));
+   memset(s_username, 0, sizeof(s_username));
+   memset(s_password, 0, sizeof(s_password));
+   memset(s_server, 0, sizeof(s_server));
+
+   if(prepare_sql_statement(sdata, &(data->stmt_generic), SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS, cfg) == ERR) return ERR;
+
+   p_bind_init(data);
+
+   if(p_exec_query(sdata, data->stmt_generic, data) == OK){
+
+      p_bind_init(data);
+
+      data->sql[data->pos] = (char *)&(data->import->import_job_id); data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
+      data->sql[data->pos] = &s_type[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_type)-2; data->pos++;
+      data->sql[data->pos] = &s_username[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_username)-2; data->pos++;
+      data->sql[data->pos] = &s_password[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_password)-2; data->pos++;
+      data->sql[data->pos] = &s_server[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_server)-2; data->pos++;
+
+      p_store_results(sdata, data->stmt_generic, data);
+
+      if(p_fetch_results(data->stmt_generic) == OK) rc = OK;
+
+      p_free_results(data->stmt_generic);
+   }
+
+   close_prepared_statement(data->stmt_generic);
+
+   data->import->processed_messages = 0;
+   data->import->total_messages = 0;
+
+   time(&(data->import->started));
+   data->import->status = 1;
+   update_import_job_stat(sdata, data);  
+
+   if(strcmp(s_type, "pop3") == 0){
+      rc = import_from_pop3_server(s_server, s_username, s_password, 110, sdata, data, dryrun, cfg);
+   }
+
+   if(strcmp(s_type, "imap") == 0){
+      rc = import_from_imap_server(s_server, s_username, s_password, 143, sdata, data, folder_imap, skiplist, dryrun, cfg);
+   }
+
+   update_import_job_stat(sdata, data);
+
+   // don't set error in case of a problem, because it
+   // will scare users looking at the gui progressbar
+
+   return rc;
+}
+
+

src/import_imap.c

@@ -0,0 +1,127 @@
+/*
+ * import_imap.c
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <getopt.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+int import_from_imap_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, char *folder_imap, char *skiplist, int dryrun, struct __config *cfg){
+   int i, rc=ERR, ret=OK, sd, seq=1, skipmatch, use_ssl=0;
+   char port_string[8], puf[SMALLBUFSIZE];
+   struct addrinfo hints, *res;
+   struct node *q;
+
+
+   inithash(data->imapfolders);
+
+   snprintf(port_string, sizeof(port_string)-1, "%d", port);
+
+   memset(&hints, 0, sizeof(hints));
+   hints.ai_family = AF_UNSPEC;
+   hints.ai_socktype = SOCK_STREAM;
+
+   if((rc = getaddrinfo(server, port_string, &hints, &res)) != 0){
+      printf("getaddrinfo for '%s': %s\n", server, gai_strerror(rc));
+      return ERR;
+   }
+
+   if(port == 993) use_ssl = 1;
+
+
+   if((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
+      printf("cannot create socket\n");
+      ret = ERR;
+      goto ENDE_IMAP;
+   }
+
+   if(connect(sd, res->ai_addr, res->ai_addrlen) == -1){
+      printf("connect()\n");
+      ret = ERR;
+      goto ENDE_IMAP;
+   }
+
+   if(connect_to_imap_server(sd, &seq, username, password, port, data, use_ssl) == ERR){
+      close(sd);
+      ret = ERR;
+      goto ENDE_IMAP;
+   }
+
+
+   /* 
+    * if the user gives -f <foldername>, then don't iterate through the folder list
+    * rather build the folderlist based on the <foldername> option, 2014.10.14, SJ
+    */
+
+   if(folder_imap){
+      addnode(data->imapfolders, folder_imap);
+   } else {
+      rc = list_folders(sd, &seq, use_ssl, data);
+      if(rc == ERR) goto ENDE_IMAP;
+   }
+
+
+   for(i=0;i<MAXHASH;i++){
+      q = data->imapfolders[i];
+      while(q != NULL){
+
+         if(q && q->str && strlen(q->str) > 1){
+
+            skipmatch = 0;
+
+            if(skiplist && strlen(skiplist) > 0){
+               snprintf(puf, sizeof(puf)-1, "%s,", (char *)q->str);
+               if(strstr(skiplist, puf)) skipmatch = 1;
+            }
+
+            if(folder_imap && strstr(q->str, folder_imap) == NULL){
+               skipmatch = 1;
+            }
+
+            if(skipmatch == 1){
+               if(data->quiet == 0) printf("SKIPPING FOLDER: %s\n", (char *)q->str);
+            }
+            else {
+               if(data->quiet == 0) printf("processing folder: %s... ", (char *)q->str);
+
+               if(process_imap_folder(sd, &seq, q->str, sdata, data, use_ssl, dryrun, cfg) == ERR) ret = ERR;
+            }
+
+         }
+
+         q = q->r;
+
+      }
+   }
+
+   send_imap_close(sd, &seq, data, use_ssl);
+
+   close_connection(sd, data, use_ssl);
+
+ENDE_IMAP:
+   freeaddrinfo(res);
+
+   clearhash(data->imapfolders);
+
+   data->import->status = 2;
+
+   return ret;
+}
+
+

src/import_mailbox.c

@@ -0,0 +1,151 @@
+/*
+ * import_mailbox.c
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <getopt.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+int import_from_mailbox(char *mailbox, struct session_data *sdata, struct __data *data, struct __config *cfg){
+   FILE *F, *f=NULL;
+   int rc=ERR, tot_msgs=0, ret=OK;
+   char buf[MAXBUFSIZE], fname[SMALLBUFSIZE];
+   time_t t;
+
+
+   F = fopen(mailbox, "r");
+   if(!F){
+      printf("cannot open mailbox: %s\n", mailbox);
+      return rc;
+   }
+
+   t = time(NULL);
+
+   while(fgets(buf, sizeof(buf)-1, F)){
+
+      if(buf[0] == 'F' && buf[1] == 'r' && buf[2] == 'o' && buf[3] == 'm' && buf[4] == ' '){
+         tot_msgs++;
+         if(f){
+            fclose(f);
+            f = NULL;
+            rc = import_message(fname, sdata, data, cfg);
+            if(rc == ERR){
+               printf("error importing: '%s'\n", fname);
+               ret = ERR;
+            }
+            else unlink(fname);
+
+            if(data->quiet == 0) printf("processed: %7d\r", tot_msgs); fflush(stdout);
+         }
+
+         snprintf(fname, sizeof(fname)-1, "%ld-%d", t, tot_msgs);
+         f = fopen(fname, "w+");
+         continue;
+      }
+
+      if(f) fprintf(f, "%s", buf);
+   }
+
+   if(f){
+      fclose(f);
+      rc = import_message(fname, sdata, data, cfg);
+      if(rc == ERR){
+         printf("error importing: '%s'\n", fname);
+         ret = ERR;
+      }
+      else unlink(fname);
+
+      if(data->quiet == 0) printf("processed: %7d\r", tot_msgs); fflush(stdout);
+   }
+
+   fclose(F);
+
+   return ret;
+}
+
+
+int import_mbox_from_dir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg){
+   DIR *dir;
+   struct dirent *de;
+   int rc=ERR, ret=OK, i=0;
+   int folder;
+   char fname[SMALLBUFSIZE];
+   struct stat st;
+
+   dir = opendir(directory);
+   if(!dir){
+      printf("cannot open directory: %s\n", directory);
+      return ERR;
+   }
+
+
+   while((de = readdir(dir))){
+      if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
+
+      snprintf(fname, sizeof(fname)-1, "%s/%s", directory, de->d_name);
+
+      if(stat(fname, &st) == 0){
+         if(S_ISDIR(st.st_mode)){
+            folder = data->folder;
+            rc = import_mbox_from_dir(fname, sdata, data, tot_msgs, cfg);
+            data->folder = folder;
+            if(rc == ERR) ret = ERR;
+         }
+         else {
+
+            if(S_ISREG(st.st_mode)){
+               if(i == 0 && data->recursive_folder_names == 1){
+                  folder = get_folder_id(sdata, data, fname, data->folder, cfg);
+                  if(folder == ERR_FOLDER){
+                     folder = add_new_folder(sdata, data, fname, data->folder, cfg);
+
+                     if(folder == ERR_FOLDER){
+                        printf("error: cannot get/add folder '%s' to parent id: %d\n", fname, data->folder);
+                        return ERR;
+                     }
+                     else {
+                        data->folder = folder;
+                     }
+                  }
+
+               }
+
+               rc = import_from_mailbox(fname, sdata, data, cfg);
+               if(rc == OK) (*tot_msgs)++;
+               else ret = ERR;
+
+               i++;
+            }
+            else {
+               printf("%s is not a file\n", fname);
+            }
+
+         }
+      }
+      else {
+         printf("cannot stat() %s\n", fname);
+      }
+
+   }
+   closedir(dir);
+
+   return ret;
+}
+
+

src/import_maildir.c

@@ -0,0 +1,108 @@
+/*
+ * import_maildir.c
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <getopt.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+int import_from_maildir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg){
+   DIR *dir;
+   struct dirent *de;
+   int rc=ERR, ret=OK, i=0;
+   int folder;
+   char *p, fname[SMALLBUFSIZE];
+   struct stat st;
+
+   dir = opendir(directory);
+   if(!dir){
+      printf("cannot open directory: %s\n", directory);
+      return ERR;
+   }
+
+
+   while((de = readdir(dir))){
+      if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
+
+      snprintf(fname, sizeof(fname)-1, "%s/%s", directory, de->d_name);
+
+      if(stat(fname, &st) == 0){
+         if(S_ISDIR(st.st_mode)){
+            folder = data->folder;
+            rc = import_from_maildir(fname, sdata, data, tot_msgs, cfg);
+            data->folder = folder;
+            if(rc == ERR) ret = ERR;
+         }
+         else {
+
+            if(S_ISREG(st.st_mode)){
+               if(i == 0 && data->recursive_folder_names == 1){
+                  p = strrchr(directory, '/');
+                  if(p) p++;
+                  else {
+                     printf("invalid directory name: '%s'\n", directory);
+                     return ERR;
+                  }
+
+                  folder = get_folder_id(sdata, data, p, data->folder, cfg);
+                  if(folder == ERR_FOLDER){
+                     folder = add_new_folder(sdata, data, p, data->folder, cfg);
+
+                     if(folder == ERR_FOLDER){
+                        printf("error: cannot get/add folder '%s' to parent id: %d\n", p, data->folder);
+                        return ERR;
+                     }
+                     else {
+                        data->folder = folder;
+                     }
+                  }
+
+               }
+
+               rc = import_message(fname, sdata, data, cfg);
+
+               if(rc == OK) (*tot_msgs)++;
+               else if(rc == ERR){
+                  printf("error importing: '%s'\n", fname);
+                  ret = ERR;
+               }
+ 
+               if(data->import->remove_after_import == 1 && rc != ERR) unlink(fname);
+
+               i++;
+
+               if(data->quiet == 0) printf("processed: %7d\r", *tot_msgs); fflush(stdout);
+            }
+            else {
+               printf("%s is not a file\n", fname);
+            }
+
+         }
+      }
+      else {
+         printf("cannot stat() %s\n", fname);
+      }
+
+   }
+   closedir(dir);
+
+   return ret;
+}
+
+

src/import_pop3.c

@@ -0,0 +1,71 @@
+/*
+ * import_pop3.c
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <getopt.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+int import_from_pop3_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, int dryrun, struct __config *cfg){
+   int rc, ret=OK, sd, use_ssl=0;
+   char port_string[8];
+   struct addrinfo hints, *res;
+
+   snprintf(port_string, sizeof(port_string)-1, "%d", port);
+
+   memset(&hints, 0, sizeof(hints));
+   hints.ai_family = AF_UNSPEC;
+   hints.ai_socktype = SOCK_STREAM;
+
+   if((rc = getaddrinfo(server, port_string, &hints, &res)) != 0){
+      printf("getaddrinfo for '%s': %s\n", server, gai_strerror(rc));
+      return ERR;
+   }
+
+   if(port == 995) use_ssl = 1;
+
+   if((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
+      printf("cannot create socket\n");
+      ret = ERR;
+      goto ENDE_POP3;
+   }
+
+   if(connect(sd, res->ai_addr, res->ai_addrlen) == -1){
+      printf("connect()\n");
+      ret = ERR;
+      goto ENDE_POP3;
+   }
+
+
+   if(connect_to_pop3_server(sd, username, password, port, data, use_ssl) == ERR){
+      close(sd);
+      ret = ERR;
+      goto ENDE_POP3;
+   }
+
+   if(process_pop3_emails(sd, sdata, data, use_ssl, dryrun, cfg) == ERR) ret = ERR;
+
+   close_connection(sd, data, use_ssl);
+
+ENDE_POP3:
+   freeaddrinfo(res);
+
+   return ret;
+}
+

src/misc.c

@@ -19,6 +19,7 @@
 #include <unistd.h>
 #include <ctype.h>
 #include <openssl/ssl.h>
+#include <openssl/err.h>
 #include "misc.h"
 #include "smtpcodes.h"
 #include "errmsg.h"
@@ -465,6 +466,18 @@ int recvtimeoutssl(int s, char *buf, int len, int timeout, int use_ssl, SSL *ssl
 }
 
 
+void close_connection(int sd, struct __data *data, int use_ssl){
+   close(sd);
+
+   if(use_ssl == 1){
+      SSL_shutdown(data->ssl);
+      SSL_free(data->ssl);
+      SSL_CTX_free(data->ctx);
+      ERR_free_strings();
+   }
+}
+
+
 void write_pid_file(char *pidfile){
    FILE *f;
 

src/misc.h

@@ -33,6 +33,7 @@ int readFromEntropyPool(int fd, void *_s, size_t n);
 int recvtimeout(int s, char *buf, int len, int timeout);
 int write1(int sd, void *buf, int buflen, int use_ssl, SSL *ssl);
 int recvtimeoutssl(int s, char *buf, int len, int timeout, int use_ssl, SSL *ssl);
+void close_connection(int sd, struct __data *data, int use_ssl);
 
 void write_pid_file(char *pidfile);
 int drop_privileges(struct passwd *pwd);

src/piler.h

@@ -18,6 +18,7 @@
 #include <av.h>
 #include <rules.h>
 #include <sql.h>
+#include <import.h>
 #include <config.h>
 #include <unistd.h>
 
@@ -53,7 +54,6 @@ void update_counters(struct session_data *sdata, struct __data *data, struct __c
 int retrieve_email_from_archive(struct session_data *sdata, struct __data *data, FILE *dest, struct __config *cfg);
 int file_from_archive_to_network(char *filename, int sd, int tls_enable, struct __data *data, struct __config *cfg);
 
-int import_message(char *filename, struct session_data *sdata, struct __data *data, struct __config *cfg);
 int get_folder_id(struct session_data *sdata, struct __data *data, char *foldername, int parent_id, struct __config *cfg);
 int add_new_folder(struct session_data *sdata, struct __data *data, char *foldername, int parent_id, struct __config *cfg);
 

src/pilerimport.c

@@ -32,438 +32,6 @@ extern int optind;
 
 int dryrun=0;
 int import_from_gui=0;
-char *folder_imap=NULL;
-
-int connect_to_imap_server(int sd, int *seq, char *username, char *password, int port, struct __data *data, int use_ssl);
-int list_folders(int sd, int *seq, int use_ssl, struct __data *data);
-int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg);
-int connect_to_pop3_server(int sd, char *username, char *password, int port, struct __data *data, int use_ssl);
-int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg);
-void send_imap_close(int sd, int *seq, struct __data *data, int use_ssl);
-void close_connection(int sd, struct __data *data, int use_ssl);
-
-void update_import_job_stat(struct session_data *sdata, struct __data *data);
-
-
-int import_from_mailbox(char *mailbox, struct session_data *sdata, struct __data *data, struct __config *cfg){
-   FILE *F, *f=NULL;
-   int rc=ERR, tot_msgs=0, ret=OK;
-   char buf[MAXBUFSIZE], fname[SMALLBUFSIZE];
-   time_t t;
-
-
-   F = fopen(mailbox, "r");
-   if(!F){
-      printf("cannot open mailbox: %s\n", mailbox);
-      return rc;
-   }
-
-   t = time(NULL);
-
-   while(fgets(buf, sizeof(buf)-1, F)){
-
-      if(buf[0] == 'F' && buf[1] == 'r' && buf[2] == 'o' && buf[3] == 'm' && buf[4] == ' '){
-         tot_msgs++;
-         if(f){
-            fclose(f);
-            f = NULL;
-            rc = import_message(fname, sdata, data, cfg);
-            if(rc == ERR){
-               printf("error importing: '%s'\n", fname);
-               ret = ERR;
-            }
-            else unlink(fname);
-
-            if(data->quiet == 0) printf("processed: %7d\r", tot_msgs); fflush(stdout);
-         }
-
-         snprintf(fname, sizeof(fname)-1, "%ld-%d", t, tot_msgs);
-         f = fopen(fname, "w+");
-         continue;
-      }
-
-      if(f) fprintf(f, "%s", buf);
-   }
-
-   if(f){
-      fclose(f);
-      rc = import_message(fname, sdata, data, cfg);
-      if(rc == ERR){
-         printf("error importing: '%s'\n", fname);
-         ret = ERR;
-      }
-      else unlink(fname);
-
-      if(data->quiet == 0) printf("processed: %7d\r", tot_msgs); fflush(stdout);
-   }
-
-   fclose(F);
-
-   return ret;
-}
-
-
-int import_mbox_from_dir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg){
-   DIR *dir;
-   struct dirent *de;
-   int rc=ERR, ret=OK, i=0;
-   int folder;
-   char fname[SMALLBUFSIZE];
-   struct stat st;
-
-   dir = opendir(directory);
-   if(!dir){
-      printf("cannot open directory: %s\n", directory);
-      return ERR;
-   }
-
-
-   while((de = readdir(dir))){
-      if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
-
-      snprintf(fname, sizeof(fname)-1, "%s/%s", directory, de->d_name);
-
-      if(stat(fname, &st) == 0){
-         if(S_ISDIR(st.st_mode)){
-            folder = data->folder;
-            rc = import_mbox_from_dir(fname, sdata, data, tot_msgs, cfg);
-            data->folder = folder;
-            if(rc == ERR) ret = ERR;
-         }
-         else {
-
-            if(S_ISREG(st.st_mode)){
-               if(i == 0 && data->recursive_folder_names == 1){
-                  folder = get_folder_id(sdata, data, fname, data->folder, cfg);
-                  if(folder == ERR_FOLDER){
-                     folder = add_new_folder(sdata, data, fname, data->folder, cfg);
-
-                     if(folder == ERR_FOLDER){
-                        printf("error: cannot get/add folder '%s' to parent id: %d\n", fname, data->folder);
-                        return ERR;
-                     }
-                     else {
-                        data->folder = folder;
-                     }
-                  }
-
-               }
-
-               rc = import_from_mailbox(fname, sdata, data, cfg);
-               if(rc == OK) (*tot_msgs)++;
-               else ret = ERR;
-
-               i++;
-            }
-            else {
-               printf("%s is not a file\n", fname);
-            }
-
-         }
-      }
-      else {
-         printf("cannot stat() %s\n", fname);
-      }
-
-   }
-   closedir(dir);
-
-   return ret;
-}
-
-
-int import_from_maildir(char *directory, struct session_data *sdata, struct __data *data, int *tot_msgs, struct __config *cfg){
-   DIR *dir;
-   struct dirent *de;
-   int rc=ERR, ret=OK, i=0;
-   int folder;
-   char *p, fname[SMALLBUFSIZE];
-   struct stat st;
-
-   dir = opendir(directory);
-   if(!dir){
-      printf("cannot open directory: %s\n", directory);
-      return ERR;
-   }
-
-
-   while((de = readdir(dir))){
-      if(strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) continue;
-
-      snprintf(fname, sizeof(fname)-1, "%s/%s", directory, de->d_name);
-
-      if(stat(fname, &st) == 0){
-         if(S_ISDIR(st.st_mode)){
-            folder = data->folder;
-            rc = import_from_maildir(fname, sdata, data, tot_msgs, cfg);
-            data->folder = folder;
-            if(rc == ERR) ret = ERR;
-         }
-         else {
-
-            if(S_ISREG(st.st_mode)){
-               if(i == 0 && data->recursive_folder_names == 1){
-                  p = strrchr(directory, '/');
-                  if(p) p++;
-                  else {
-                     printf("invalid directory name: '%s'\n", directory);
-                     return ERR;
-                  }
-
-                  folder = get_folder_id(sdata, data, p, data->folder, cfg);
-                  if(folder == ERR_FOLDER){
-                     folder = add_new_folder(sdata, data, p, data->folder, cfg);
-
-                     if(folder == ERR_FOLDER){
-                        printf("error: cannot get/add folder '%s' to parent id: %d\n", p, data->folder);
-                        return ERR;
-                     }
-                     else {
-                        data->folder = folder;
-                     }
-                  }
-
-               }
-
-               rc = import_message(fname, sdata, data, cfg);
-               if(rc == OK) (*tot_msgs)++;
-               else {
-                  printf("error importing: '%s'\n", fname);
-                  ret = ERR;
-               }
- 
-               if(data->import->remove_after_import == 1 && ret != ERR) unlink(fname);
-
-               i++;
-
-               if(data->quiet == 0) printf("processed: %7d\r", *tot_msgs); fflush(stdout);
-            }
-            else {
-               printf("%s is not a file\n", fname);
-            }
-
-         }
-      }
-      else {
-         printf("cannot stat() %s\n", fname);
-      }
-
-   }
-   closedir(dir);
-
-   return ret;
-}
-
-
-int import_from_imap_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, char *skiplist, int dryrun, struct __config *cfg){
-   int i, rc=ERR, ret=OK, sd, seq=1, skipmatch, use_ssl=0;
-   char port_string[8], puf[SMALLBUFSIZE];
-   struct addrinfo hints, *res;
-   struct node *q;
-
-
-   inithash(data->imapfolders);
-
-   snprintf(port_string, sizeof(port_string)-1, "%d", port);
-
-   memset(&hints, 0, sizeof(hints));
-   hints.ai_family = AF_UNSPEC;
-   hints.ai_socktype = SOCK_STREAM;
-
-   if((rc = getaddrinfo(server, port_string, &hints, &res)) != 0){
-      printf("getaddrinfo for '%s': %s\n", server, gai_strerror(rc));
-      return ERR;
-   }
-
-   if(port == 993) use_ssl = 1;
-
-
-   if((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
-      printf("cannot create socket\n");
-      ret = ERR;
-      goto ENDE_IMAP;
-   }
-
-   if(connect(sd, res->ai_addr, res->ai_addrlen) == -1){
-      printf("connect()\n");
-      ret = ERR;
-      goto ENDE_IMAP;
-   }
-
-   if(connect_to_imap_server(sd, &seq, username, password, port, data, use_ssl) == ERR){
-      close(sd);
-      ret = ERR;
-      goto ENDE_IMAP;
-   }
-
-
-   /* 
-    * if the user gives -f <foldername>, then don't iterate through the folder list
-    * rather build the folderlist based on the <foldername> option, 2014.10.14, SJ
-    */
-
-   if(folder_imap){
-      addnode(data->imapfolders, folder_imap);
-   } else {
-      rc = list_folders(sd, &seq, use_ssl, data);
-      if(rc == ERR) goto ENDE_IMAP;
-   }
-
-
-   for(i=0;i<MAXHASH;i++){
-      q = data->imapfolders[i];
-      while(q != NULL){
-
-         if(q && q->str && strlen(q->str) > 1){
-
-            skipmatch = 0;
-
-            if(skiplist && strlen(skiplist) > 0){
-               snprintf(puf, sizeof(puf)-1, "%s,", (char *)q->str);
-               if(strstr(skiplist, puf)) skipmatch = 1;
-            }
-
-            if(folder_imap && strstr(q->str, folder_imap) == NULL){
-               skipmatch = 1;
-            }
-
-            if(skipmatch == 1){
-               if(data->quiet == 0) printf("SKIPPING FOLDER: %s\n", (char *)q->str);
-            }
-            else {
-               if(data->quiet == 0) printf("processing folder: %s... ", (char *)q->str);
-
-               if(process_imap_folder(sd, &seq, q->str, sdata, data, use_ssl, dryrun, cfg) == ERR) ret = ERR;
-            }
-
-         }
-
-         q = q->r;
-
-      }
-   }
-
-   send_imap_close(sd, &seq, data, use_ssl);
-
-   close_connection(sd, data, use_ssl);
-
-ENDE_IMAP:
-   freeaddrinfo(res);
-
-   clearhash(data->imapfolders);
-
-   data->import->status = 2;
-
-   return ret;
-}
-
-
-int import_from_pop3_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, int dryrun, struct __config *cfg){
-   int rc, ret=OK, sd, use_ssl=0;
-   char port_string[8];
-   struct addrinfo hints, *res;
-
-   snprintf(port_string, sizeof(port_string)-1, "%d", port);
-
-   memset(&hints, 0, sizeof(hints));
-   hints.ai_family = AF_UNSPEC;
-   hints.ai_socktype = SOCK_STREAM;
-
-   if((rc = getaddrinfo(server, port_string, &hints, &res)) != 0){
-      printf("getaddrinfo for '%s': %s\n", server, gai_strerror(rc));
-      return ERR;
-   }
-
-   if(port == 995) use_ssl = 1;
-
-   if((sd = socket(res->ai_family, res->ai_socktype, res->ai_protocol)) == -1){
-      printf("cannot create socket\n");
-      ret = ERR;
-      goto ENDE_POP3;
-   }
-
-   if(connect(sd, res->ai_addr, res->ai_addrlen) == -1){
-      printf("connect()\n");
-      ret = ERR;
-      goto ENDE_POP3;
-   }
-
-
-   if(connect_to_pop3_server(sd, username, password, port, data, use_ssl) == ERR){
-      close(sd);
-      ret = ERR;
-      goto ENDE_POP3;
-   }
-
-   if(process_pop3_emails(sd, sdata, data, use_ssl, dryrun, cfg) == ERR) ret = ERR;
-
-   close_connection(sd, data, use_ssl);
-
-ENDE_POP3:
-   freeaddrinfo(res);
-
-   return ret;
-}
-
-
-int read_gui_import_data(struct session_data *sdata, struct __data *data, char *skiplist, int dryrun, struct __config *cfg){
-   int rc=ERR;
-   char s_type[SMALLBUFSIZE], s_username[SMALLBUFSIZE], s_password[SMALLBUFSIZE], s_server[SMALLBUFSIZE];
-
-   memset(s_type, 0, sizeof(s_type));
-   memset(s_username, 0, sizeof(s_username));
-   memset(s_password, 0, sizeof(s_password));
-   memset(s_server, 0, sizeof(s_server));
-
-   if(prepare_sql_statement(sdata, &(data->stmt_generic), SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS, cfg) == ERR) return ERR;
-
-   p_bind_init(data);
-
-   if(p_exec_query(sdata, data->stmt_generic, data) == OK){
-
-      p_bind_init(data);
-
-      data->sql[data->pos] = (char *)&(data->import->import_job_id); data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
-      data->sql[data->pos] = &s_type[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_type)-2; data->pos++;
-      data->sql[data->pos] = &s_username[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_username)-2; data->pos++;
-      data->sql[data->pos] = &s_password[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_password)-2; data->pos++;
-      data->sql[data->pos] = &s_server[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_server)-2; data->pos++;
-
-      p_store_results(sdata, data->stmt_generic, data);
-
-      if(p_fetch_results(data->stmt_generic) == OK) rc = OK;
-
-      p_free_results(data->stmt_generic);
-   }
-
-   close_prepared_statement(data->stmt_generic);
-
-   data->import->processed_messages = 0;
-   data->import->total_messages = 0;
-
-   time(&(data->import->started));
-   data->import->status = 1;
-   update_import_job_stat(sdata, data);  
-
-   if(strcmp(s_type, "pop3") == 0){
-      rc = import_from_pop3_server(s_server, s_username, s_password, 110, sdata, data, dryrun, cfg);
-   }
-
-   if(strcmp(s_type, "imap") == 0){
-      rc = import_from_imap_server(s_server, s_username, s_password, 143, sdata, data, skiplist, dryrun, cfg);
-   }
-
-   update_import_job_stat(sdata, data);
-
-   // don't set error in case of a problem, because it
-   // will scare users looking at the gui progressbar
-   /*if(rc == ERR){
-      data->import->status = 3;
-      update_import_job_stat(sdata, data);
-   }*/
-
-   return rc;
-}
 
 
 void usage(){
@@ -497,7 +65,7 @@ void usage(){
 int main(int argc, char **argv){
    int i, c, rc=0, n_mbox=0, tot_msgs=0, port=143;
    char *configfile=CONFIG_FILE, *emlfile=NULL, *mboxdir=NULL, *mbox[MBOX_ARGS], *directory=NULL;
-   char *imapserver=NULL, *pop3server=NULL, *username=NULL, *password=NULL, *skiplist=SKIPLIST, *folder=NULL;
+   char *imapserver=NULL, *pop3server=NULL, *username=NULL, *password=NULL, *skiplist=SKIPLIST, *folder=NULL, *folder_imap=NULL;
    struct session_data sdata;
    struct __config cfg;
    struct __data data;
@@ -752,9 +320,9 @@ int main(int argc, char **argv){
    }
    if(mboxdir) rc = import_mbox_from_dir(mboxdir, &sdata, &data, &tot_msgs, &cfg);
    if(directory) rc = import_from_maildir(directory, &sdata, &data, &tot_msgs, &cfg);
-   if(imapserver && username && password) rc = import_from_imap_server(imapserver, username, password, port, &sdata, &data, skiplist, dryrun, &cfg);
+   if(imapserver && username && password) rc = import_from_imap_server(imapserver, username, password, port, &sdata, &data, folder_imap, skiplist, dryrun, &cfg);
    if(pop3server && username && password) rc = import_from_pop3_server(pop3server, username, password, port, &sdata, &data, dryrun, &cfg);
-   if(import_from_gui == 1) rc = read_gui_import_data(&sdata, &data, skiplist, dryrun, &cfg);
+   if(import_from_gui == 1) rc = read_gui_import_data(&sdata, &data, folder_imap, skiplist, dryrun, &cfg);
 
    clearrules(data.archiving_rules);
    clearrules(data.retention_rules);