From 3729ed1ecfe6af15de04a03c58b48990db98e029 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Sun, 20 Dec 2020 22:01:34 +0100 Subject: [PATCH] pilerimport can read data exported by pilerexport from stdin Signed-off-by: Janos SUTO --- configure | 2 +- configure.in | 2 +- src/config.h | 2 + src/defs.h | 1 + src/import.h | 2 + src/import_maildir.c | 2 +- src/import_pilerexport.c | 109 +++++++++++++++++++++++++++++++++++++++ src/pilerexport.c | 1 + src/pilerimport.c | 13 +++-- 9 files changed, 128 insertions(+), 6 deletions(-) create mode 100644 src/import_pilerexport.c diff --git a/configure b/configure index 016ca71b..6423657e 100755 --- a/configure +++ b/configure @@ -4854,7 +4854,7 @@ echo; echo CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wimplicit-fallthrough=2 -Wuninitialized -Wno-format-truncation -g" LIBS="$antispam_libs $sunos_libs " -OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" +OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" ac_config_files="$ac_config_files Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile" diff --git a/configure.in b/configure.in index 554eaee2..0f2a83d5 100644 --- a/configure.in +++ b/configure.in @@ -537,7 +537,7 @@ echo; echo CFLAGS="$static -std=c99 -O2 -fPIC -Wall -Wextra -Wimplicit-fallthrough=2 -Wuninitialized -Wno-format-truncation -g" LIBS="$antispam_libs $sunos_libs " -OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" +OBJS="dirs.o misc.o counters.o cfg.o sig.o decoder.o hash.o parser.o parser_utils.o rules.o smtp.o session.o bdat.o message.o attachment.o digest.o store.o archive.o tai.o import.o import_pilerexport.o import_maildir.o import_mailbox.o import_pop3.o import_imap.o imap.o pop3.o extract.o mydomains.o tokenizer.o $objs" AC_CONFIG_FILES([Makefile src/Makefile etc/Makefile util/Makefile init.d/Makefile systemd/Makefile unit_tests/Makefile webui/Makefile contrib/imap/Makefile]) AC_OUTPUT diff --git a/src/config.h b/src/config.h index e1542ad5..7d110348 100644 --- a/src/config.h +++ b/src/config.h @@ -57,6 +57,8 @@ #define MEMCACHED_MSGS_STORED_SIZE MEMCACHED_CLAPF_PREFIX "stored_size" +#define PILEREXPORT_BEGIN_MARK "x-exported-by-pilerexport: start\n" + #define LOG_PRIORITY LOG_INFO #define _LOG_INFO 3 diff --git a/src/defs.h b/src/defs.h index d6c653b4..6bfece2e 100644 --- a/src/defs.h +++ b/src/defs.h @@ -302,6 +302,7 @@ struct import { int keep_eml; int timeout; int cap_uidplus; + int fd; long total_size; int dryrun; int tot_msgs; diff --git a/src/import.h b/src/import.h index 0979acab..6f0f112d 100644 --- a/src/import.h +++ b/src/import.h @@ -23,4 +23,6 @@ int list_folders(struct data *data); int process_imap_folder(char *folder, struct session_data *sdata, struct data *data, struct config *cfg); void send_imap_close(struct data *data); +void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg); + #endif /* _IMPORT_H */ diff --git a/src/import_maildir.c b/src/import_maildir.c index aac3c6a3..ced1cafa 100644 --- a/src/import_maildir.c +++ b/src/import_maildir.c @@ -83,7 +83,7 @@ int import_from_maildir(struct session_data *sdata, struct data *data, char *dir printf("ERROR: error importing: '%s'\n", data->import->filename); ret = ERR; } - + if(data->import->remove_after_import == 1 && rc != ERR) unlink(data->import->filename); i++; diff --git a/src/import_pilerexport.c b/src/import_pilerexport.c new file mode 100644 index 00000000..8514a593 --- /dev/null +++ b/src/import_pilerexport.c @@ -0,0 +1,109 @@ +/* + * import_pop3.c + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +void import_the_file(struct session_data *sdata, struct data *data, struct config *cfg){ + close(data->import->fd); + data->import->fd = -1; + + if(import_message(sdata, data, cfg) != ERR){ + unlink(data->import->filename); + } +} + + +void process_buffer(char *buf, int buflen, uint64 *count, struct session_data *sdata, struct data *data, struct config *cfg){ + + if(!strcmp(buf, PILEREXPORT_BEGIN_MARK)){ + if((*count) > 0){ + import_the_file(sdata, data, cfg); + } + + (*count)++; + + snprintf(data->import->filename, SMALLBUFSIZE-1, "import-%llu", *count); + + data->import->fd = open(data->import->filename, O_CREAT|O_WRONLY|O_TRUNC, S_IRUSR|S_IWUSR); + if(data->import->fd == -1){ + // Do some error handling + printf("error: cannot open %s\n", data->import->filename); + } + + } else { + if(write(data->import->fd, buf, buflen) != buflen){ + printf("error: didnt write %d bytes\n", buflen); + } + } +} + + +void import_from_pilerexport(struct session_data *sdata, struct data *data, struct config *cfg){ + int n, rc, savedlen=0, puflen; + uint64 count=0; + char *p, copybuf[2*BIGBUFSIZE+1], buf[BIGBUFSIZE], savedbuf[BIGBUFSIZE], puf[BIGBUFSIZE]; + + memset(savedbuf, 0, sizeof(savedbuf)); + + data->import->fd = -1; + + do { + memset(buf, 0, sizeof(buf)); + n = fread(buf, 1, sizeof(buf)-1, stdin); + + if(savedlen > 0){ + memset(copybuf, 0, sizeof(copybuf)); + + memcpy(copybuf, savedbuf, savedlen); + memcpy(©buf[savedlen], buf, n); + + savedlen = 0; + memset(savedbuf, 0, sizeof(savedbuf)); + + p = ©buf[0]; + } + else { + p = &buf[0]; + } + + do { + puflen = read_one_line(p, '\n', puf, sizeof(puf), &rc); + p += puflen; + + if(puflen > 0){ + if(rc == OK){ + process_buffer(puf, puflen, &count, sdata, data, cfg); + } + else { + snprintf(savedbuf, sizeof(savedbuf)-1, "%s", puf); + savedlen = puflen; + } + } + + } while(puflen > 0); + + } while(n > 0); + + if(data->import->fd != -1){ + import_the_file(sdata, data, cfg); + } +} diff --git a/src/pilerexport.c b/src/pilerexport.c index 3c65490d..f15b942b 100644 --- a/src/pilerexport.c +++ b/src/pilerexport.c @@ -372,6 +372,7 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct if(dryrun == 0){ if(export_to_stdout){ + printf("%s", PILEREXPORT_BEGIN_MARK); rc = retrieve_email_from_archive(sdata, stdout, cfg); continue; } diff --git a/src/pilerimport.c b/src/pilerimport.c index dcf65338..51ce727c 100644 --- a/src/pilerimport.c +++ b/src/pilerimport.c @@ -54,6 +54,7 @@ void usage(){ printf(" -a Add recipient to the To:/Cc: list\n"); printf(" -T Update import table at id=\n"); printf(" -D Dry-run, do not import anything\n"); + printf(" -y Read pilerexport data from stdin\n"); printf(" -o Only download emails for POP3/IMAP import\n"); printf(" -r Remove imported emails\n"); printf(" -q Quiet mode\n"); @@ -63,7 +64,7 @@ void usage(){ int main(int argc, char **argv){ - int i, n_mbox=0; + int i, n_mbox=0, read_from_pilerexport=0; char *configfile=CONFIG_FILE, *mbox[MBOX_ARGS], *directory=NULL; char puf[SMALLBUFSIZE], *imapserver=NULL, *pop3server=NULL; struct session_data sdata; @@ -141,6 +142,7 @@ int main(int argc, char **argv){ {"failed-folder", required_argument, 0, 'j' }, {"move-folder", required_argument, 0, 'g' }, {"only-download",no_argument, 0, 'o' }, + {"read-from-export",no_argument, 0, 'y' }, {"dry-run", no_argument, 0, 'D' }, {"help", no_argument, 0, 'h' }, {0,0,0,0} @@ -148,9 +150,9 @@ int main(int argc, char **argv){ int option_index = 0; - int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:DRroqh?", long_options, &option_index); + int c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:yDRroqh?", long_options, &option_index); #else - int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:DRroqh?"); + int c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:j:T:yDRroqh?"); #endif if(c == -1) break; @@ -269,6 +271,10 @@ int main(int argc, char **argv){ data.import->table_id = atoi(optarg); break; + case 'y' : + read_from_pilerexport = 1; + break; + case 'D' : data.import->dryrun = 1; break; @@ -361,6 +367,7 @@ int main(int argc, char **argv){ if(directory) import_from_maildir(&sdata, &data, directory, &cfg); if(imapserver) import_from_imap_server(&sdata, &data, &cfg); if(pop3server) import_from_pop3_server(&sdata, &data, &cfg); + if(read_from_pilerexport) import_from_pilerexport(&sdata, &data, &cfg); clearrules(data.archiving_rules); clearrules(data.retention_rules);