pilerimport can read data exported by pilerexport from stdin

Signed-off-by: Janos SUTO <sj@acts.hu>

configure

@@ -4854,7 +4854,7 @@ echo; echo
 
 CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wimplicit-fallthrough=2 -Wuninitialized -Wno-format-truncation -g"
 LIBS="$antispam_libs $sunos_libs "
-OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs"
+OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs"
 
 ac_config_files="$ac_config_files Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile"
 

configure.in

@@ -537,7 +537,7 @@ echo; echo
 
 CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wimplicit-fallthrough=2 -Wuninitialized -Wno-format-truncation -g"
 LIBS="$antispam_libs $sunos_libs "
-OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs"
+OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs"
 
 AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile])
 AC_OUTPUT

src/config.h

@@ -57,6 +57,8 @@
 #define MEMCACHED_MSGS_STORED_SIZE MEMCACHED_CLAPF_PREFIX "stored_size"
 
 
+#define PILEREXPORT_BEGIN_MARK "x-exported-by-pilerexport: start\n"
+
 #define LOG_PRIORITY LOG_INFO
 
 #define _LOG_INFO 3

src/defs.h

@@ -302,6 +302,7 @@ struct import {
    int keep_eml;
    int timeout;
    int cap_uidplus;
+   int fd;
    long total_size;
    int dryrun;
    int tot_msgs;

src/import.h

@@ -23,4 +23,6 @@ int list_folders(struct data *data);
 int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct config *cfg);
 void send_imap_close(struct data *data);
 
+void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg);
+
 #endif /* _IMPORT_H */

src/import_maildir.c

@@ -83,7 +83,7 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir
                   printf("ERROR: error importing: '%s'\n", data->import->filename);
                   ret = ERR;
                }
- 
+
                if(data->import->remove_after_import == 1 && rc != ERR) unlink(data->import->filename);
 
                i++;

src/import_pilerexport.c

@@ -0,0 +1,109 @@
+/*
+ * import_pop3.c
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <getopt.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+void import_the_file(struct session_data *sdata, struct data *data, struct config *cfg){
+   close(data->import->fd);
+   data->import->fd = -1;
+
+   if(import_message(sdata, data, cfg) != ERR){
+      unlink(data->import->filename);
+   }
+}
+
+
+void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct config *cfg){
+
+   if(!strcmp(buf, PILEREXPORT_BEGIN_MARK)){
+      if((*count) > 0){
+         import_the_file(sdata, data, cfg);
+      }
+
+      (*count)++;
+
+      snprintf(data->import->filename, SMALLBUFSIZE-1, "import-%llu", *count);
+
+      data->import->fd = open(data->import->filename, O_CREAT|O_WRONLY|O_TRUNC, S_IRUSR|S_IWUSR);
+      if(data->import->fd == -1){
+         // Do some error handling
+         printf("error: cannot open %s\n", data->import->filename);
+      }
+
+   } else {
+      if(write(data->import->fd, buf, buflen) != buflen){
+         printf("error: didnt write %d bytes\n", buflen);
+      }
+   }
+}
+
+
+void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg){
+   int n, rc, savedlen=0, puflen;
+   uint64 count=0;
+   char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE];
+
+   memset(savedbuf, 0, sizeof(savedbuf));
+
+   data->import->fd = -1;
+
+   do {
+      memset(buf, 0, sizeof(buf));
+      n = fread(buf, 1, sizeof(buf)-1, stdin);
+
+      if(savedlen > 0){
+         memset(copybuf, 0, sizeof(copybuf));
+
+         memcpy(copybuf, savedbuf, savedlen);
+         memcpy(&copybuf[savedlen], buf, n);
+
+         savedlen = 0;
+         memset(savedbuf, 0, sizeof(savedbuf));
+
+         p = &copybuf[0];
+      }
+      else {
+         p = &buf[0];
+      }
+
+      do {
+         puflen = read_one_line(p, '\n', puf, sizeof(puf), &rc);
+         p += puflen;
+
+         if(puflen > 0){
+            if(rc == OK){
+               process_buffer(puf, puflen, &count, sdata, data, cfg);
+            }
+            else {
+               snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf);
+               savedlen = puflen;
+            }
+         }
+
+      } while(puflen > 0);
+
+   } while(n > 0);
+
+   if(data->import->fd != -1){
+      import_the_file(sdata, data, cfg);
+   }
+}

src/pilerexport.c

@@ -372,6 +372,7 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct
          if(dryrun == 0){
 
             if(export_to_stdout){
+               printf("%s", PILEREXPORT_BEGIN_MARK);
                rc = retrieve_email_from_archive(sdata, stdout, cfg);
                continue;
             }

src/pilerimport.c

@@ -54,6 +54,7 @@ void usage(){
    printf("    -a <recipient>                    Add recipient to the To:/Cc: list\n");
    printf("    -T <id>                           Update import table at id=<id>\n");
    printf("    -D                                Dry-run, do not import anything\n");
+   printf("    -y                                Read pilerexport data from stdin\n");
    printf("    -o                                Only download emails for POP3/IMAP import\n");
    printf("    -r                                Remove imported emails\n");
    printf("    -q                                Quiet mode\n");
@@ -63,7 +64,7 @@ void usage(){
 
 
 int main(int argc, char **argv){
-   int i, n_mbox=0;
+   int i, n_mbox=0, read_from_pilerexport=0;
    char *configfile=CONFIG_FILE, *mbox[MBOX_ARGS], *directory=NULL;
    char puf[SMALLBUFSIZE], *imapserver=NULL, *pop3server=NULL;
    struct session_data sdata;
@@ -141,6 +142,7 @@ int main(int argc, char **argv){
             {"failed-folder",  required_argument,  0,  'j' },
             {"move-folder",  required_argument,  0,  'g' },
             {"only-download",no_argument,        0,  'o' },
+            {"read-from-export",no_argument,     0,  'y' },
             {"dry-run",      no_argument,        0,  'D' },
             {"help",         no_argument,        0,  'h' },
             {0,0,0,0}
@@ -148,9 +150,9 @@ int main(int argc, char **argv){
 
       int option_index = 0;
 
-      int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:DRroqh?", long_options, &option_index);
+      int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:yDRroqh?", long_options, &option_index);
 #else
-      int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:DRroqh?");
+      int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:yDRroqh?");
 #endif
 
       if(c == -1) break;
@@ -269,6 +271,10 @@ int main(int argc, char **argv){
                     data.import->table_id = atoi(optarg);
                     break;
 
+         case 'y' :
+                    read_from_pilerexport = 1;
+                    break;
+
          case 'D' :
                     data.import->dryrun = 1;
                     break;
@@ -361,6 +367,7 @@ int main(int argc, char **argv){
    if(directory) import_from_maildir(&sdata, &data, directory, &cfg);
    if(imapserver) import_from_imap_server(&sdata, &data, &cfg);
    if(pop3server) import_from_pop3_server(&sdata, &data, &cfg);
+   if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &cfg);
 
    clearrules(data.archiving_rules);
    clearrules(data.retention_rules);