bashclub/piler
1
0
Fork 0
mirror of https://bitbucket.org/jsuto/piler.git synced 2024-11-07 23:41:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

reverted the fixmetadata commit

Browse Source 
Change-Id: Iea5249f81d4915ff91c13ddd4dbab8ef059d8ba7
Signed-off-by: SJ <sj@acts.hu>
This commit is contained in:
SJ 2016-10-06 22:36:53 +02:00
parent 6b6afc84bb
commit 7b3564bd0a
7 changed files with 16 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/Makefile.in
View File

@ -69,9 +69,6 @@ pilerconf: pilerconf.c libpiler.a
reindex: reindex.c libpiler.a reindex: reindex.c libpiler.a
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR) $(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
fixmetadata: fixmetadata.c libpiler.a
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
test: test.c libpiler.a test: test.c libpiler.a
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o pilertest $< -lpiler $(LIBS) $(LIBDIR) @LDFLAGS@ $(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o pilertest $< -lpiler $(LIBS) $(LIBDIR) @LDFLAGS@
@ -96,7 +93,7 @@ install-piler:
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilertest $(DESTDIR)$(bindir) $(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilertest $(DESTDIR)$(bindir)
clean: clean:
rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge pilertest reindex fixmetadata rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge pilertest reindex
distclean: clean distclean: clean
rm -f Makefile rm -f Makefile

1
src/config.h
View File

@ -98,7 +98,6 @@
#define SQL_PREPARED_STMT_INSERT_INTO_RCPT_TABLE "INSERT INTO " SQL_RECIPIENT_TABLE " (`id`,`to`,`todomain`) VALUES(?,?,?)" #define SQL_PREPARED_STMT_INSERT_INTO_RCPT_TABLE "INSERT INTO " SQL_RECIPIENT_TABLE " (`id`,`to`,`todomain`) VALUES(?,?,?)"
#define SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE "INSERT INTO " SQL_SPHINX_TABLE " (`id`, `from`, `to`, `fromdomain`, `todomain`, `subject`, `body`, `arrived`, `sent`, `size`, `direction`, `folder`, `attachments`, `attachment_types`) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)" #define SQL_PREPARED_STMT_INSERT_INTO_SPHINX_TABLE "INSERT INTO " SQL_SPHINX_TABLE " (`id`, `from`, `to`, `fromdomain`, `todomain`, `subject`, `body`, `arrived`, `sent`, `size`, `direction`, `folder`, `attachments`, `attachment_types`) values(?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
#define SQL_PREPARED_STMT_INSERT_INTO_META_TABLE "INSERT INTO " SQL_METADATA_TABLE " (`from`,`fromdomain`,`subject`,`spam`,`arrived`,`sent`,`retained`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`reference`,`digest`,`bodydigest`,`vcode`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" #define SQL_PREPARED_STMT_INSERT_INTO_META_TABLE "INSERT INTO " SQL_METADATA_TABLE " (`from`,`fromdomain`,`subject`,`spam`,`arrived`,`sent`,`retained`,`size`,`hlen`,`direction`,`attachments`,`piler_id`,`message_id`,`reference`,`digest`,`bodydigest`,`vcode`) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
#define SQL_PREPARED_STMT_UPDATE_META_TABLE "UPDATE " SQL_METADATA_TABLE " SET `from`=?,`fromdomain`=?,`subject`=?,`spam`=?,`arrived`=?,`sent`=?,`retained`=?,`size`=?,`hlen`=?,`direction`=?,`attachments`=?,`reference`=?,`digest`=?,`bodydigest`=?,`vcode`=? WHERE id=?"
#define SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE "INSERT INTO " SQL_ATTACHMENT_TABLE " (`piler_id`,`attachment_id`,`sig`,`name`,`type`,`size`,`ptr`) VALUES(?,?,?,?,?,?,?)" #define SQL_PREPARED_STMT_INSERT_INTO_ATTACHMENT_TABLE "INSERT INTO " SQL_ATTACHMENT_TABLE " (`piler_id`,`attachment_id`,`sig`,`name`,`type`,`size`,`ptr`) VALUES(?,?,?,?,?,?,?)"
#define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE "SELECT `id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=? AND `ptr`=0 AND `size`=?" #define SQL_PREPARED_STMT_GET_ATTACHMENT_ID_BY_SIGNATURE "SELECT `id` FROM `" SQL_ATTACHMENT_TABLE "` WHERE `sig`=? AND `ptr`=0 AND `size`=?"
#define SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER "SELECT `piler_id`, `attachment_id` FROM " SQL_ATTACHMENT_TABLE " WHERE id=?" #define SQL_PREPARED_STMT_GET_ATTACHMENT_POINTER "SELECT `piler_id`, `attachment_id` FROM " SQL_ATTACHMENT_TABLE " WHERE id=?"

1
src/defs.h
View File

@ -294,7 +294,6 @@ struct import {
int download_only; int download_only;
int keep_eml; int keep_eml;
int timeout; int timeout;
int reimport;
int cap_uidplus; int cap_uidplus;
long total_size; long total_size;
time_t started, updated, finished; time_t started, updated, finished;

119
src/fixmetadata.c
View File

@ -1,119 +0,0 @@
/*
* fixmetadata.c, SJ
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <time.h>
#include <locale.h>
#include <syslog.h>
#include <piler.h>
uint64 get_metaid_by_messageid(struct session_data *sdata, struct __data *data, char *message_id, struct __config *cfg);
int update_meta_data(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg){
int ret=ERR;
uint64 id=0;
char *subj;
id = get_metaid_by_messageid(sdata, data, state->message_id, cfg);
if(id <= 0) return ERR;
subj = state->b_subject;
if(*subj == ' ') subj++;
if(prepare_sql_statement(sdata, &(data->stmt_update_metadata), SQL_PREPARED_STMT_UPDATE_META_TABLE) == ERR) return ret;
p_bind_init(data);
data->sql[data->pos] = subj; data->type[data->pos] = TYPE_STRING; data->pos++;
data->sql[data->pos] = (char *)&state->n_attachments; data->type[data->pos] = TYPE_LONG; data->pos++;
data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
if(p_exec_query(sdata, data->stmt_update_metadata, data) == OK) ret = OK;
close_prepared_statement(data->stmt_update_metadata);
return ret;
}
int main(int argc, char **argv){
int i;
struct stat st;
struct session_data sdata;
struct parser_state state;
struct __config cfg;
struct __data data;
srand(getpid());
if(argc < 2){
fprintf(stderr, "usage: %s <message>\n", argv[0]);
exit(1);
}
if(!can_i_write_current_directory()) __fatal("cannot write current directory!");
if(stat(argv[1], &st) != 0){
fprintf(stderr, "%s is not found\n", argv[1]);
return 0;
}
(void) openlog("fixmetadata", LOG_PID, LOG_MAIL);
cfg = read_config(CONFIG_FILE);
if(open_database(&sdata, &cfg) == ERR) return 0;
setlocale(LC_CTYPE, cfg.locale);
data.folder = 0;
data.recursive_folder_names = 0;
inithash(data.mydomains);
load_mydomains(&sdata, &data, &cfg);
init_session_data(&sdata, &cfg);
sdata.delivered = 0;
sdata.tot_len = st.st_size;
snprintf(sdata.ttmpfile, SMALLBUFSIZE-1, "%s", argv[1]);
snprintf(sdata.filename, SMALLBUFSIZE-1, "%s", argv[1]);
snprintf(sdata.tmpframe, SMALLBUFSIZE-1, "%s.m", argv[1]);
state = parse_message(&sdata, 1, &data, &cfg);
post_parse(&sdata, &state, &cfg);
update_meta_data(&sdata, &state, &data, &cfg);
//sdata.retained = sdata.now + query_retain_period(&data, &state, st.st_size, sdata.spam_message, &cfg);
//clearrules(data.archiving_rules);
//clearrules(data.retention_rules);
//clearrules(data.folder_rules);
clearhash(data.mydomains);
for(i=1; i<=state.n_attachments; i++){
unlink(state.attachments[i].internalname);
}
unlink(sdata.tmpframe);
close_database(&sdata);
return 0;
}

17
src/import.c
View File

@ -92,10 +92,7 @@ int import_message(char *filename, struct session_data *sdata, struct __data *da
return ERR; return ERR;
} }
if(data->import->reimport == 1) rc = process_message(sdata, &state, data, cfg);
rc = reimport_message(sdata, &state, data, cfg);
else
rc = process_message(sdata, &state, data, cfg);
unlink(state.message_id_hash); unlink(state.message_id_hash);
} }
@ -106,13 +103,11 @@ int import_message(char *filename, struct session_data *sdata, struct __data *da
switch(rc) { switch(rc) {
case OK: case OK:
if(data->import->reimport == 0){ bzero(&counters, sizeof(counters));
bzero(&counters, sizeof(counters)); counters.c_rcvd = 1;
counters.c_rcvd = 1; counters.c_size += sdata->tot_len;
counters.c_size += sdata->tot_len; counters.c_stored_size = sdata->stored_len;
counters.c_stored_size = sdata->stored_len; update_counters(sdata, data, &counters, cfg);
update_counters(sdata, data, &counters, cfg);
}
break; break;

86
src/message.c
View File

@ -186,7 +186,7 @@ int update_metadata_reference(struct session_data *sdata, struct parser_state *s
} }
int store_meta_data(struct session_data *sdata, struct parser_state *state, struct __data *data, uint64 old_id, struct __config *cfg){ int store_meta_data(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg){
int rc, ret=ERR, result; int rc, ret=ERR, result;
char *subj, *p, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1]; char *subj, *p, s[MAXBUFSIZE], s2[SMALLBUFSIZE], vcode[2*DIGEST_LENGTH+1], ref[2*DIGEST_LENGTH+1];
uint64 id=0; uint64 id=0;
@ -206,11 +206,7 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru
} }
if(old_id > 0){ if(prepare_sql_statement(sdata, &(data->stmt_insert_into_meta_table), SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR;
if(prepare_sql_statement(sdata, &(data->stmt_insert_into_meta_table), SQL_PREPARED_STMT_UPDATE_META_TABLE) == ERR) return ERR;
} else {
if(prepare_sql_statement(sdata, &(data->stmt_insert_into_meta_table), SQL_PREPARED_STMT_INSERT_INTO_META_TABLE) == ERR) return ERR;
}
memset(s2, 0, sizeof(s2)); memset(s2, 0, sizeof(s2));
@ -243,35 +239,18 @@ int store_meta_data(struct session_data *sdata, struct parser_state *state, stru
data->sql[data->pos] = (char *)&sdata->hdr_len; data->type[data->pos] = TYPE_LONG; data->pos++; data->sql[data->pos] = (char *)&sdata->hdr_len; data->type[data->pos] = TYPE_LONG; data->pos++;
data->sql[data->pos] = (char *)&sdata->direction; data->type[data->pos] = TYPE_LONG; data->pos++; data->sql[data->pos] = (char *)&sdata->direction; data->type[data->pos] = TYPE_LONG; data->pos++;
data->sql[data->pos] = (char *)&state->n_attachments; data->type[data->pos] = TYPE_LONG; data->pos++; data->sql[data->pos] = (char *)&state->n_attachments; data->type[data->pos] = TYPE_LONG; data->pos++;
data->sql[data->pos] = sdata->ttmpfile; data->type[data->pos] = TYPE_STRING; data->pos++;
if(old_id == 0){ data->sql[data->pos] = state->message_id; data->type[data->pos] = TYPE_STRING; data->pos++;
data->sql[data->pos] = sdata->ttmpfile; data->type[data->pos] = TYPE_STRING; data->pos++;
data->sql[data->pos] = state->message_id; data->type[data->pos] = TYPE_STRING; data->pos++;
}
data->sql[data->pos] = &ref[0]; data->type[data->pos] = TYPE_STRING; data->pos++; data->sql[data->pos] = &ref[0]; data->type[data->pos] = TYPE_STRING; data->pos++;
data->sql[data->pos] = sdata->digest; data->type[data->pos] = TYPE_STRING; data->pos++; data->sql[data->pos] = sdata->digest; data->type[data->pos] = TYPE_STRING; data->pos++;
data->sql[data->pos] = sdata->bodydigest; data->type[data->pos] = TYPE_STRING; data->pos++; data->sql[data->pos] = sdata->bodydigest; data->type[data->pos] = TYPE_STRING; data->pos++;
data->sql[data->pos] = &vcode[0]; data->type[data->pos] = TYPE_STRING; data->pos++; data->sql[data->pos] = &vcode[0]; data->type[data->pos] = TYPE_STRING; data->pos++;
if(old_id > 0){
data->sql[data->pos] = (char *)&id; data->type[data->pos] = TYPE_LONGLONG; data->pos++;
}
if(p_exec_query(sdata, data->stmt_insert_into_meta_table, data) == ERR){ if(p_exec_query(sdata, data->stmt_insert_into_meta_table, data) == ERR){
ret = ERR_EXISTS; ret = ERR_EXISTS;
} }
else { else {
if(old_id > 0){ id = p_get_insert_id(data->stmt_insert_into_meta_table);
remove_recipients(sdata, old_id);
remove_folder_id(sdata, old_id);
id = old_id;
}
else {
id = p_get_insert_id(data->stmt_insert_into_meta_table);
}
rc = store_recipients(sdata, data, state->b_to, id, cfg); rc = store_recipients(sdata, data, state->b_to, id, cfg);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored recipients, rc=%d", sdata->ttmpfile, rc); if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored recipients, rc=%d", sdata->ttmpfile, rc);
@ -368,7 +347,7 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru
sdata->retained += query_retain_period(data, state, sdata->tot_len, sdata->spam_message, cfg); sdata->retained += query_retain_period(data, state, sdata->tot_len, sdata->spam_message, cfg);
rc = store_meta_data(sdata, state, data, 0, cfg); rc = store_meta_data(sdata, state, data, cfg);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored metadata, rc=%d", sdata->ttmpfile, rc); if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: stored metadata, rc=%d", sdata->ttmpfile, rc);
if(rc == ERR_EXISTS){ if(rc == ERR_EXISTS){
@ -379,56 +358,3 @@ int process_message(struct session_data *sdata, struct parser_state *state, stru
return OK; return OK;
} }
int reimport_message(struct session_data *sdata, struct parser_state *state, struct __data *data, struct __config *cfg){
int i, rc;
char piler_id[SMALLBUFSIZE];
char oldfile[SMALLBUFSIZE], newfile[SMALLBUFSIZE];
memset(piler_id, 0, sizeof(piler_id));
sdata->duplicate_id = get_metaid_by_messageid(sdata, data, state->message_id, &piler_id[0]);
if(sdata->duplicate_id == 0){
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "cannot find this message: %s", state->message_id);
return ERR;
}
/* store base64 encoded file attachments */
if(state->n_attachments > 0){
// rename .a* files!
for(i=1; i<=state->n_attachments; i++){
snprintf(oldfile, sizeof(oldfile)-1, "%s.a%d", sdata->ttmpfile, i);
snprintf(newfile, sizeof(newfile)-1, "%s.a%d", piler_id, i);
rename(oldfile, newfile);
}
rc = store_attachments(sdata, state, data, cfg);
remove_stripped_attachments(state);
if(rc) return ERR;
}
// rename .m file!
snprintf(oldfile, sizeof(oldfile)-1, "%s.m", sdata->ttmpfile);
snprintf(sdata->tmpframe, SMALLBUFSIZE-1, "%s.m", piler_id);
rename(oldfile, sdata->tmpframe);
rc = store_file(sdata, sdata->tmpframe, 0, cfg);
if(rc == 0){
syslog(LOG_PRIORITY, "%s: error storing message for reimport: %s", sdata->ttmpfile, sdata->tmpframe);
return ERR;
}
sdata->retained += query_retain_period(data, state, sdata->tot_len, sdata->spam_message, cfg);
rc = store_meta_data(sdata, state, data, sdata->duplicate_id, cfg);
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "%s: update metadata, rc=%d", sdata->ttmpfile, rc);
return OK;
}

12
src/pilerimport.c
View File

@ -57,7 +57,6 @@ void usage(){
printf(" -D Dry-run, do not import anything\n"); printf(" -D Dry-run, do not import anything\n");
printf(" -o Only download emails for POP3/IMAP import\n"); printf(" -o Only download emails for POP3/IMAP import\n");
printf(" -r Remove imported emails\n"); printf(" -r Remove imported emails\n");
printf(" -z Reimport emails\n");
printf(" -q Quiet mode\n"); printf(" -q Quiet mode\n");
exit(0); exit(0);
@ -83,7 +82,7 @@ int main(int argc, char **argv){
import.import_job_id = import.total_messages = import.total_size = import.processed_messages = import.batch_processing_limit = 0; import.import_job_id = import.total_messages = import.total_size = import.processed_messages = import.batch_processing_limit = 0;
import.started = import.updated = import.finished = import.remove_after_import = import.reimport = 0; import.started = import.updated = import.finished = import.remove_after_import = 0;
import.extra_recipient = import.move_folder = NULL; import.extra_recipient = import.move_folder = NULL;
import.start_position = 1; import.start_position = 1;
import.download_only = 0; import.download_only = 0;
@ -123,7 +122,6 @@ int main(int argc, char **argv){
{"remove-after-import",no_argument, 0, 'r' }, {"remove-after-import",no_argument, 0, 'r' },
{"move-folder", required_argument, 0, 'g' }, {"move-folder", required_argument, 0, 'g' },
{"only-download",no_argument, 0, 'o' }, {"only-download",no_argument, 0, 'o' },
{"reimport", no_argument, 0, 'z' },
{"gui-import", no_argument, 0, 'G' }, {"gui-import", no_argument, 0, 'G' },
{"dry-run", no_argument, 0, 'D' }, {"dry-run", no_argument, 0, 'D' },
{"help", no_argument, 0, 'h' }, {"help", no_argument, 0, 'h' },
@ -132,9 +130,9 @@ int main(int argc, char **argv){
int option_index = 0; int option_index = 0;
c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:GDRrozqh?", long_options, &option_index); c = getopt_long(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:GDRroqh?", long_options, &option_index);
#else #else
c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:GDRrozqh?"); c = getopt(argc, argv, "c:m:M:e:d:i:K:u:p:P:x:F:f:a:b:t:s:g:GDRroqh?");
#endif #endif
if(c == -1) break; if(c == -1) break;
@ -233,10 +231,6 @@ int main(int argc, char **argv){
data.import->start_position = atoi(optarg); data.import->start_position = atoi(optarg);
break; break;
case 'z' :
data.import->reimport = 1;
break;
case 'a' : case 'a' :
data.import->extra_recipient = optarg; data.import->extra_recipient = optarg;
break; break;