From c860ca6799392cf6ed217fb05ed1087a1ce617d2 Mon Sep 17 00:00:00 2001 From: Janos SUTO Date: Mon, 5 Dec 2022 20:22:49 +0100 Subject: [PATCH] Added support for pilerexport to spread files among several directories Signed-off-by: Janos SUTO --- RELEASE_NOTES | 3 ++- src/piler.h | 1 + src/pilerexport.c | 34 +++++++++++++++++++++++++++++++--- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 17c9aace..e047c253 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -4,7 +4,8 @@ - Replaced sphinx with manticore 5.0.2, see https://www.mailpiler.org/wiki/current:manticore - Replaced obsoleted libchart library with chart.js - Added real-time index support - +- Added oauth2 support to imapfetch.py +- Added support for pilerexport to spread files among several directories. See the -D option 1.3.12: diff --git a/src/piler.h b/src/piler.h index b9207d10..8c8f8cde 100644 --- a/src/piler.h +++ b/src/piler.h @@ -51,6 +51,7 @@ int query_attachments(struct session_data *sdata, struct ptr_array *ptr_arr); struct config read_config(char *configfile); +void createdir(char *path, uid_t uid, gid_t gid, mode_t mode); void check_and_create_directories(struct config *cfg, uid_t uid, gid_t gid); void update_counters(struct session_data *sdata, struct data *data, struct counters *counters, struct config *cfg); diff --git a/src/pilerexport.c b/src/pilerexport.c index 8eca2b05..1f6e4e7c 100644 --- a/src/pilerexport.c +++ b/src/pilerexport.c @@ -30,12 +30,14 @@ char *query=NULL; int verbosity = 0; int max_matches = 1000; char *index_list = "main1,dailydelta1,delta1"; +struct passwd *pwd; regex_t regexp; char *zipfile = NULL; struct zip *zip = NULL; uint64 *zip_ids = NULL; int zip_counter = 0; int zip_batch = 2000; +int max_files_in_export_dir = 0; int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg); @@ -60,6 +62,7 @@ void usage(){ printf(" -Z Zip batch size. Valid range: 10-10000, default: 2000\n"); #endif printf(" -A Export all emails from archive\n"); + printf(" -D Max. number of files to put in a single directory, default: 2000\n"); printf(" -o Export emails to stdout\n"); printf(" -d Dry run\n"); @@ -356,9 +359,10 @@ void zip_flush(){ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct config *cfg){ FILE *f; - uint64 id, n=0; + uint64 id, n=0, dir_counter=0; char digest[SMALLBUFSIZE], bodydigest[SMALLBUFSIZE]; char filename[SMALLBUFSIZE]; + char export_subdir[SMALLBUFSIZE]; struct sql sql; int errorp, rc=0, attachments; unsigned long total_attachments=0; @@ -394,6 +398,19 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct continue; } + if(max_files_in_export_dir > 0 && n % max_files_in_export_dir == 0){ + dir_counter++; + snprintf(export_subdir, sizeof(export_subdir)-1, "export-%llu", dir_counter); + if(n > 0 && chdir("..")){ + p_clean_exit("error chdir(\"..\")", 1); + } + + createdir(export_subdir, pwd->pw_uid, pwd->pw_gid, 0700); + if(chdir(export_subdir)){ + p_clean_exit("error chdir to export-* dir", 1); + } + } + snprintf(filename, sizeof(filename)-1, "%llu.eml", id); f = fopen(filename, "w"); @@ -512,6 +529,7 @@ int main(int argc, char **argv){ {"zip", required_argument, 0, 'z' }, {"zip-batch", required_argument, 0, 'Z' }, {"where-condition", required_argument, 0, 'w' }, + {"max-files", required_argument, 0, 'D' }, {"max-matches", required_argument, 0, 'm' }, {"index-list", required_argument, 0, 'i' }, {0,0,0,0} @@ -519,9 +537,9 @@ int main(int argc, char **argv){ int option_index = 0; - int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:oAdhv?", long_options, &option_index); + int c = getopt_long(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:D:oAdhv?", long_options, &option_index); #else - int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:oAdhv?"); + int c = getopt(argc, argv, "c:s:S:f:r:F:R:a:b:w:m:i:z:Z:D:oAdhv?"); #endif if(c == -1) break; @@ -630,6 +648,11 @@ int main(int argc, char **argv){ zip_batch = 2000; break; + case 'D': max_files_in_export_dir = atoi(optarg); + if(max_files_in_export_dir < 10 || max_files_in_export_dir > 100000) + max_files_in_export_dir = 2000; + break; + case 'o': export_to_stdout = 1; break; @@ -661,6 +684,11 @@ int main(int argc, char **argv){ if(read_key(&cfg)) p_clean_exit(ERR_READING_KEY, 1); + if(strlen(cfg.username) > 1){ + pwd = getpwnam(cfg.username); + if(!pwd) __fatal(ERR_NON_EXISTENT_USER); + } + init_session_data(&sdata, &cfg);