bashclub/piler
1
0
Fork 0
mirror of https://bitbucket.org/jsuto/piler.git synced 2025-06-13 12:17:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

added pop3 import to the gui

Browse Source
This commit is contained in:
SJ
2013-08-23 13:02:51 +02:00
parent d7faf096f1
commit eb826537dd
30 changed files with 1010 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/config.h
View File

@ -14,7 +14,7 @@
#define VERSION "0.1.24-master-branch"
#define BUILD 835
#define BUILD 836
#define HOSTID "mailarchiver"
@ -87,6 +87,7 @@
#define SQL_OPTION_TABLE "option"
#define SQL_DOMAIN_TABLE "domain"
#define SQL_CUSTOMER_TABLE "customer"
#define SQL_IMPORT_TABLE "import"
#define SQL_MESSAGES_VIEW "v_messages"
#define SQL_ATTACHMENTS_VIEW "v_attachment"
@ -109,6 +110,7 @@
#define SQL_PREPARED_STMT_GET_FOLDER_ID "SELECT `id` FROM " SQL_FOLDER_TABLE " WHERE `name`=? AND `parent_id`=?"
#define SQL_PREPARED_STMT_INSERT_INTO_FOLDER_TABLE "INSERT INTO `" SQL_FOLDER_TABLE "` (`name`, `parent_id`) VALUES(?,?)"
#define SQL_PREPARED_STMT_UPDATE_METADATA_REFERENCE "UPDATE " SQL_METADATA_TABLE " SET reference=? WHERE message_id=? AND reference=''"
#define SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS "SELECT id, type, username, password, server FROM " SQL_IMPORT_TABLE " WHERE started=0 ORDER BY id LIMIT 0,1"
/* Error codes */

10
src/defs.h
View File

@ -248,11 +248,21 @@ struct memcached_server {
#endif
struct import {
int status;
int total_messages;
int processed_messages;
int import_job_id;
time_t started, updated, finished;
};
struct __data {
int folder;
char recursive_folder_names;
char starttls[TINYBUFSIZE];
struct node *mydomains[MAXHASH];
struct import *import;
#ifdef NEED_MYSQL
MYSQL_STMT *stmt_generic;

5
src/imap.c
View File

@ -91,7 +91,7 @@ int read_response(int sd, char *buf, int buflen, char *tagok, struct __data *dat
}
int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sdata, struct __data *data, int use_ssl, struct __config *cfg){
int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg){
int rc=ERR, i, n, pos, endpos, messages=0, len, readlen, fd, lastpos, nreads, processed_messages=0;
char *p, tag[SMALLBUFSIZE], tagok[SMALLBUFSIZE], tagbad[SMALLBUFSIZE], buf[MAXBUFSIZE], filename[SMALLBUFSIZE];
char aggrbuf[3*MAXBUFSIZE];
@ -199,7 +199,8 @@ int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sda
close(fd);
rc = import_message(filename, sdata, data, cfg);
if(dryrun == 0) rc = import_message(filename, sdata, data, cfg);
else rc = OK;
if(rc == ERR) printf("error importing '%s'\n", filename);
else unlink(filename);

10
src/import.c
View File

@ -177,3 +177,13 @@ int add_new_folder(struct session_data *sdata, struct __data *data, char *folder
}
void update_import_job_stat(struct __data *data){
char buf[SMALLBUFSIZE];
snprintf(buf, sizeof(buf)-1, "update import set status=%d, started=%ld, updated=%ld, finished=%ld, total=%d, imported=%d where id=%d", data->import->status, data->import->started, data->import->updated, data->import->finished, data->import->total_messages, data->import->processed_messages, data->import->import_job_id);
p_query(sdata, buf);
}

95
src/pilerimport.c
View File

@ -31,16 +31,20 @@ extern char *optarg;
extern int optind;
int quiet=0;
int dryrun=0;
int remove_after_successful_import = 0;
int import_from_gui=0;
int connect_to_imap_server(int sd, int *seq, char *username, char *password, int port, struct __data *data, int use_ssl);
int list_folders(int sd, int *seq, char *folders, int foldersize, int use_ssl, struct __data *data);
int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sdata, struct __data *data, int use_ssl, struct __config *cfg);
int process_imap_folder(int sd, int *seq, char *folder, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg);
int connect_to_pop3_server(int sd, char *username, char *password, int port, struct __data *data, int use_ssl);
int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data, int use_ssl, struct __config *cfg);
int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg);
void close_connection(int sd, struct __data *data, int use_ssl);
void update_import_job_stat(struct __data *data);
int import_from_mailbox(char *mailbox, struct session_data *sdata, struct __data *data, struct __config *cfg){
FILE *F, *f=NULL;
@ -252,7 +256,7 @@ int import_from_maildir(char *directory, struct session_data *sdata, struct __da
}
int import_from_imap_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, char *skiplist, struct __config *cfg){
int import_from_imap_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, char *skiplist, int dryrun, struct __config *cfg){
int rc=ERR, ret=OK, sd, seq=1, skipmatch, use_ssl=0;
char *p, puf[SMALLBUFSIZE];
char muf[SMALLBUFSIZE];
@ -317,7 +321,7 @@ int import_from_imap_server(char *server, char *username, char *password, int po
if(quiet == 0) printf("processing folder: %s... ", puf);
if(process_imap_folder(sd, &seq, puf, sdata, data, use_ssl, cfg) == ERR) ret = ERR;
if(process_imap_folder(sd, &seq, puf, sdata, data, use_ssl, dryrun, cfg) == ERR) ret = ERR;
} while(p);
@ -331,7 +335,7 @@ ENDE_IMAP:
}
int import_from_pop3_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, struct __config *cfg){
int import_from_pop3_server(char *server, char *username, char *password, int port, struct session_data *sdata, struct __data *data, int dryrun, struct __config *cfg){
int rc, ret=OK, sd, use_ssl=0;
char port_string[6];
struct addrinfo hints, *res;
@ -368,7 +372,7 @@ int import_from_pop3_server(char *server, char *username, char *password, int po
goto ENDE_POP3;
}
if(process_pop3_emails(sd, sdata, data, use_ssl, cfg) == ERR) ret = ERR;
if(process_pop3_emails(sd, sdata, data, use_ssl, dryrun, cfg) == ERR) ret = ERR;
close_connection(sd, data, use_ssl);
@ -379,6 +383,56 @@ ENDE_POP3:
}
int read_gui_import_data(struct session_data *sdata, struct __data *data, int dryrun, struct __config *cfg){
int rc=ERR;
char s_type[SMALLBUFSIZE], s_username[SMALLBUFSIZE], s_password[SMALLBUFSIZE], s_server[SMALLBUFSIZE];
memset(s_type, 0, sizeof(s_type));
memset(s_username, 0, sizeof(s_username));
memset(s_password, 0, sizeof(s_password));
memset(s_server, 0, sizeof(s_server));
if(prepare_sql_statement(sdata, &(data->stmt_generic), SQL_PREPARED_STMT_GET_GUI_IMPORT_JOBS) == ERR) return ERR;
p_bind_init(data);
if(p_exec_query(sdata, data->stmt_generic, data) == ERR) goto ENDE;
p_bind_init(data);
data->sql[data->pos] = (char *)&(data->import->import_job_id); data->type[data->pos] = TYPE_LONG; data->len[data->pos] = sizeof(int); data->pos++;
data->sql[data->pos] = &s_type[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_type)-2; data->pos++;
data->sql[data->pos] = &s_username[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_username)-2; data->pos++;
data->sql[data->pos] = &s_password[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_password)-2; data->pos++;
data->sql[data->pos] = &s_server[0]; data->type[data->pos] = TYPE_STRING; data->len[data->pos] = sizeof(s_server)-2; data->pos++;
p_store_results(sdata, data->stmt_generic, data);
if(p_fetch_results(data->stmt_generic) == OK) rc = OK;
p_free_results(data->stmt_generic);
ENDE:
close_prepared_statement(data->stmt_generic);
time(&(data->import->started));
data->import->status = 1;
update_import_job_stat(data);
if(strcmp(s_type, "pop3") == 0){
import_from_pop3_server(s_server, s_username, s_password, 110, sdata, data, dryrun, cfg);
}
return rc;
}
void usage(){
printf("usage: pilerimport [-c <config file>] -e <eml file> | -m <mailbox file> | -d <directory> | -i <imap server> | -K <pop3 server> | -u <imap username> -p <imap password> -P <imap port> [-F <foldername>] [-R] [-r] [-q]\n");
exit(0);
@ -392,11 +446,18 @@ int main(int argc, char **argv){
struct session_data sdata;
struct __config cfg;
struct __data data;
struct import import;
for(i=0; i<MBOX_ARGS; i++) mbox[i] = NULL;
data.folder = 0;
data.recursive_folder_names = 0;
import.import_job_id = import.total_messages = import.processed_messages = 0;
import.started = import.updated = import.finished = 0;
data.import = &import;
inithash(data.mydomains);
initrules(data.archiving_rules);
initrules(data.retention_rules);
@ -421,15 +482,17 @@ int main(int argc, char **argv){
{"quiet", required_argument, 0, 'q' },
{"recursive", required_argument, 0, 'R' },
{"remove-after-import", required_argument, 0, 'r' },
{"gui-import", required_argument, 0, 'G' },
{"dry-run", required_argument, 0, 'D' },
{"help", no_argument, 0, 'h' },
{0,0,0,0}
};
int option_index = 0;
c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:Rrqh?", long_options, &option_index);
c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:GDRrqh?", long_options, &option_index);
#else
c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:Rrqh?");
c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:GDRrqh?");
#endif
if(c == -1) break;
@ -498,6 +561,14 @@ int main(int argc, char **argv){
remove_after_successful_import = 1;
break;
case 'G' :
import_from_gui = 1;
break;
case 'D' :
dryrun = 1;
break;
case 'q' :
quiet = 1;
break;
@ -515,7 +586,7 @@ int main(int argc, char **argv){
if(!mbox[0] && !mboxdir && !emlfile && !directory && !imapserver && !pop3server) usage();
if(!mbox[0] && !mboxdir && !emlfile && !directory && !imapserver && !pop3server && import_from_gui == 0) usage();
cfg = read_config(configfile);
@ -565,9 +636,9 @@ int main(int argc, char **argv){
}
if(mboxdir) rc = import_mbox_from_dir(mboxdir, &sdata, &data, &tot_msgs, &cfg);
if(directory) rc = import_from_maildir(directory, &sdata, &data, &tot_msgs, &cfg);
if(imapserver && username && password) rc = import_from_imap_server(imapserver, username, password, port, &sdata, &data, skiplist, &cfg);
if(pop3server && username && password) rc = import_from_pop3_server(pop3server, username, password, port, &sdata, &data, &cfg);
if(imapserver && username && password) rc = import_from_imap_server(imapserver, username, password, port, &sdata, &data, skiplist, dryrun, &cfg);
if(pop3server && username && password) rc = import_from_pop3_server(pop3server, username, password, port, &sdata, &data, dryrun, &cfg);
if(import_from_gui == 1) rc = read_gui_import_data(&sdata, &data, dryrun, &cfg);
clearrules(data.archiving_rules);
clearrules(data.retention_rules);

35
src/pop3.c
View File

@ -23,6 +23,9 @@
#include <piler.h>
void update_import_job_stat(struct __data *data);
int is_last_complete_pop3_packet(char *s, int len){
if(*(s+len-5) == '\r' && *(s+len-4) == '\n' && *(s+len-3) == '.' && *(s+len-2) == '\r' && *(s+len-1) == '\n'){
@ -90,11 +93,14 @@ int connect_to_pop3_server(int sd, char *username, char *password, int port, str
}
int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data, int use_ssl, struct __config *cfg){
int i, rc=ERR, n, messages=0, processed_messages=0, pos, readlen, fd, lastpos, nreads;
int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data, int use_ssl, int dryrun, struct __config *cfg){
int i=0, rc=ERR, n, pos, readlen, fd, lastpos, nreads;
char *p, buf[MAXBUFSIZE], filename[SMALLBUFSIZE];
char aggrbuf[3*MAXBUFSIZE];
data->import->processed_messages = 0;
data->import->total_messages = 0;
snprintf(buf, sizeof(buf)-1, "STAT\r\n");
n = write1(sd, buf, strlen(buf), use_ssl, data->ssl);
@ -104,19 +110,19 @@ int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data,
p = strchr(&buf[4], ' ');
if(p){
*p = '\0';
messages = atoi(&buf[4]);
data->import->total_messages = atoi(&buf[4]);
}
}
else return ERR;
printf("found %d messages\n", messages);
printf("found %d messages\n", data->import->total_messages);
if(messages <= 0) return rc;
if(data->import->total_messages <= 0) return rc;
for(i=1; i<=messages; i++){
processed_messages++;
printf("processed: %7d\r", processed_messages); fflush(stdout);
for(i=1; i<=data->import->total_messages; i++){
data->import->processed_messages++;
printf("processed: %7d\r", data->import->processed_messages); fflush(stdout);
snprintf(buf, sizeof(buf)-1, "RETR %d\r\n", i);
@ -188,7 +194,13 @@ int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data,
close(fd);
rc = import_message(filename, sdata, data, cfg);
if(dryrun == 0) rc = import_message(filename, sdata, data, cfg);
else rc = OK;
if(i % 100 == 0){
time(&(data->import->updated));
update_import_job_stat(data);
}
unlink(filename);
}
@ -199,6 +211,11 @@ int process_pop3_emails(int sd, struct session_data *sdata, struct __data *data,
printf("\n");
time(&(data->import->finished));
data->import->status = 2;
update_import_job_stat(data);
return OK;
}