mirror of
https://bitbucket.org/jsuto/piler.git
synced 2024-11-08 00:31:58 +01:00
pilerpurge fix
This commit is contained in:
SJ
parent
b10b9d412f
commit
64f8342b02
@ -33,7 +33,7 @@ MAKE = `which make`
|
||||
|
||||
INSTALL = @INSTALL@
|
||||
|
||||
all: libpiler.a piler pilerconf pilerget pilerimport pilerexport test
|
||||
all: libpiler.a piler pilerconf pilerget pilerimport pilerexport pilerpurge test
|
||||
install: install-piler
|
||||
|
||||
|
||||
@ -57,6 +57,9 @@ pilerimport: pilerimport.c libpiler.a
|
||||
pilerexport: pilerexport.c libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $^ -lpiler $(LIBS) $(LIBDIR)
|
||||
|
||||
pilerpurge: pilerpurge.c libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $^ -lpiler $(LIBS) $(LIBDIR)
|
||||
|
||||
pilerconf: pilerconf.c cfg.o misc.o tai.o
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $^ $(LIBDIR)
|
||||
|
||||
@ -77,9 +80,11 @@ install-piler:
|
||||
$(INSTALL) -m 0755 pilerconf $(DESTDIR)$(sbindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerget $(DESTDIR)$(bindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerimport $(DESTDIR)$(bindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerexport $(DESTDIR)$(bindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerpurge $(DESTDIR)$(bindir)
|
||||
|
||||
clean:
|
||||
rm -f *.o *.a libpiler.so* piler pilerconf pilerget pilerimport pilerexport pilertest
|
||||
rm -f *.o *.a libpiler.so* piler pilerconf pilerget pilerimport pilerexport pilerpurge pilertest
|
||||
|
||||
distclean: clean
|
||||
rm -f Makefile
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
|
||||
#define PROGNAME "piler"
|
||||
|
||||
#define VERSION "0.1.14"
|
||||
#define VERSION "0.1.15"
|
||||
|
||||
#define PROGINFO VERSION ", Janos SUTO <sj@acts.hu>\n\n" CONFIGURE_PARAMS "\n"
|
||||
|
||||
@ -81,11 +81,7 @@
|
||||
#define SQL_RETENTION_RULE_TABLE "retention_rule"
|
||||
#define SQL_COUNTER_TABLE "counter"
|
||||
#define SQL_MESSAGES_VIEW "messages"
|
||||
|
||||
|
||||
/* TRE stuff */
|
||||
|
||||
#define NUM_OF_REGEXES 20
|
||||
#define SQL_ATTACHMENTS_VIEW "v_attachment"
|
||||
|
||||
|
||||
/* Error codes */
|
||||
|
||||
@ -84,7 +84,7 @@ ENDE:
|
||||
break;
|
||||
|
||||
default:
|
||||
printf("failed to import: %s\n", filename);
|
||||
printf("failed to import: %s (id: %s)\n", filename, sdata->ttmpfile);
|
||||
break;
|
||||
}
|
||||
|
||||
@ -214,7 +214,6 @@ int main(int argc, char **argv){
|
||||
|
||||
if(!mailbox && !emlfile && !directory) usage();
|
||||
|
||||
(void) openlog("pilerimport", LOG_PID, LOG_MAIL);
|
||||
|
||||
cfg = read_config(configfile);
|
||||
|
||||
@ -238,6 +237,8 @@ int main(int argc, char **argv){
|
||||
data.archiving_rules = NULL;
|
||||
data.retention_rules = NULL;
|
||||
|
||||
(void) openlog("pilerimport", LOG_PID, LOG_MAIL);
|
||||
|
||||
load_rules(&sdata, &(data.archiving_rules), SQL_ARCHIVING_RULE_TABLE);
|
||||
load_rules(&sdata, &(data.retention_rules), SQL_RETENTION_RULE_TABLE);
|
||||
|
||||
|
||||
332
src/pilerpurge.c
Normal file
332
src/pilerpurge.c
Normal file
@ -0,0 +1,332 @@
|
||||
/*
|
||||
* pilerpurge.c, SJ
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/stat.h>
|
||||
#include <dirent.h>
|
||||
#include <unistd.h>
|
||||
#include <time.h>
|
||||
#include <locale.h>
|
||||
#include <syslog.h>
|
||||
#include <piler.h>
|
||||
|
||||
|
||||
extern char *optarg;
|
||||
extern int optind;
|
||||
|
||||
int dryrun = 0;
|
||||
|
||||
#define SQL_STMT_DELETE_FROM_META_TABLE "UPDATE `" SQL_METADATA_TABLE "` SET `deleted`=1 WHERE `id` IN ("
|
||||
#define SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID "UPDATE `" SQL_METADATA_TABLE "` SET `deleted`=1 WHERE `piler_id` IN ("
|
||||
#define SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS "SELECT `piler_id`, `attachment_id`, `i` FROM `" SQL_ATTACHMENTS_VIEW "` WHERE `refcount`=0 AND `piler_id` IN ('"
|
||||
#define SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE "DELETE FROM `" SQL_ATTACHMENT_TABLE "` WHERE `id` IN ("
|
||||
|
||||
|
||||
int remove_message_frame_files(char *s, char *update_meta_sql, struct session_data *sdata, struct __config *cfg){
|
||||
char *p, puf[SMALLBUFSIZE], filename[SMALLBUFSIZE];
|
||||
int n=0;
|
||||
|
||||
p = s;
|
||||
do {
|
||||
p = split(p, ' ', puf, sizeof(puf)-1);
|
||||
|
||||
if(strlen(puf) == RND_STR_LEN){
|
||||
snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c/%c%c/%s.m", cfg->queuedir, puf[RND_STR_LEN-6], puf[RND_STR_LEN-5], puf[RND_STR_LEN-4], puf[RND_STR_LEN-3], puf[RND_STR_LEN-2], puf[RND_STR_LEN-1], puf);
|
||||
|
||||
if(dryrun == 1){
|
||||
n++;
|
||||
printf("removing messagefile: %s\n", filename);
|
||||
}
|
||||
else {
|
||||
if(unlink(filename) == 0) n++;
|
||||
}
|
||||
|
||||
}
|
||||
} while(p);
|
||||
|
||||
|
||||
update_meta_sql[strlen(update_meta_sql)-1] = ')';
|
||||
|
||||
if(dryrun == 1){
|
||||
printf("running sql query: *%s*\n\n", update_meta_sql);
|
||||
} else {
|
||||
mysql_real_query(&(sdata->mysql), update_meta_sql, strlen(update_meta_sql));
|
||||
}
|
||||
|
||||
|
||||
return n;
|
||||
}
|
||||
|
||||
|
||||
int remove_attachments(char *in, struct session_data *sdata, struct __config *cfg){
|
||||
char filename[SMALLBUFSIZE];
|
||||
char *a, buf[BIGBUFSIZE-300], update_meta_sql[BIGBUFSIZE], delete_attachment_stmt[BIGBUFSIZE];
|
||||
int n=0, len;
|
||||
MYSQL_RES *res;
|
||||
MYSQL_ROW row;
|
||||
|
||||
|
||||
if(strlen(in) < 10) return 0;
|
||||
|
||||
len = strlen(SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS) + strlen(in) + 2;
|
||||
|
||||
a = malloc(len);
|
||||
if(!a) return 0;
|
||||
|
||||
memset(a, 0, len);
|
||||
|
||||
in[strlen(in)-2] = ')';
|
||||
in[strlen(in)-1] = '\0';
|
||||
snprintf(a, len-1, "%s%s", SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS, in);
|
||||
|
||||
if(dryrun == 1) printf("running sql query: *%s*\n\n", a);
|
||||
|
||||
|
||||
memset(buf, 0, sizeof(buf));
|
||||
memset(update_meta_sql, 0, sizeof(update_meta_sql));
|
||||
memset(delete_attachment_stmt, 0, sizeof(delete_attachment_stmt));
|
||||
|
||||
snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
|
||||
snprintf(delete_attachment_stmt, sizeof(delete_attachment_stmt)-1, "%s", SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
|
||||
|
||||
if(mysql_real_query(&(sdata->mysql), a, strlen(a)) == 0){
|
||||
res = mysql_store_result(&(sdata->mysql));
|
||||
if(res){
|
||||
while((row = mysql_fetch_row(res))){
|
||||
if(!row[0]) continue;
|
||||
|
||||
snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, row[0][RND_STR_LEN-6], row[0][RND_STR_LEN-5], row[0][RND_STR_LEN-4], row[0][RND_STR_LEN-3], row[0][RND_STR_LEN-2], row[0][RND_STR_LEN-1], row[0], atoi(row[1]));
|
||||
|
||||
if(dryrun == 1){
|
||||
printf("removing attachment: *%s*\n", filename);
|
||||
} else {
|
||||
unlink(filename);
|
||||
}
|
||||
|
||||
|
||||
if(row[2]){
|
||||
memcpy(&delete_attachment_stmt[strlen(delete_attachment_stmt)], row[2], strlen(row[2]));
|
||||
memcpy(&delete_attachment_stmt[strlen(delete_attachment_stmt)], ",", 1);
|
||||
}
|
||||
|
||||
if(atoi(row[1]) == 1){
|
||||
|
||||
memcpy(&update_meta_sql[strlen(update_meta_sql)], row[0], strlen(row[0]));
|
||||
memcpy(&update_meta_sql[strlen(update_meta_sql)], "','", 3);
|
||||
|
||||
if(strlen(buf) >= sizeof(buf)-RND_STR_LEN-2-1){
|
||||
if(strlen(update_meta_sql) > 10){
|
||||
update_meta_sql[strlen(update_meta_sql)-2] = ')';
|
||||
update_meta_sql[strlen(update_meta_sql)-1] = '\0';
|
||||
}
|
||||
|
||||
n += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
|
||||
|
||||
if(strlen(delete_attachment_stmt) > strlen(SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE)){
|
||||
delete_attachment_stmt[strlen(delete_attachment_stmt)-1] = ')';
|
||||
if(dryrun == 1){
|
||||
printf("delete sql: *%s*\n", delete_attachment_stmt);
|
||||
} else {
|
||||
mysql_real_query(&(sdata->mysql), delete_attachment_stmt, strlen(delete_attachment_stmt));
|
||||
}
|
||||
}
|
||||
|
||||
memset(buf, 0, sizeof(buf));
|
||||
memset(update_meta_sql, 0, sizeof(update_meta_sql));
|
||||
memset(delete_attachment_stmt, 0, sizeof(delete_attachment_stmt));
|
||||
|
||||
snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
|
||||
snprintf(delete_attachment_stmt, sizeof(delete_attachment_stmt)-1, "%s", SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
|
||||
}
|
||||
|
||||
memcpy(&buf[strlen(buf)], row[0], strlen(row[0]));
|
||||
memcpy(&buf[strlen(buf)], " ", 1);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
mysql_free_result(res);
|
||||
}
|
||||
}
|
||||
|
||||
free(a);
|
||||
|
||||
|
||||
if(strlen(buf) > 5 && strlen(update_meta_sql) > strlen(SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID)+10){
|
||||
update_meta_sql[strlen(update_meta_sql)-2] = ')';
|
||||
update_meta_sql[strlen(update_meta_sql)-1] = '\0';
|
||||
|
||||
n += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
|
||||
}
|
||||
|
||||
if(strlen(delete_attachment_stmt) > strlen(SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE)){
|
||||
delete_attachment_stmt[strlen(delete_attachment_stmt)-1] = ')';
|
||||
if(dryrun == 1){
|
||||
printf("delete sql: *%s*\n", delete_attachment_stmt);
|
||||
} else {
|
||||
mysql_real_query(&(sdata->mysql), delete_attachment_stmt, strlen(delete_attachment_stmt));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
return n;
|
||||
}
|
||||
|
||||
|
||||
int purge_messages_without_attachment(struct session_data *sdata, struct __config *cfg){
|
||||
int purged=0;
|
||||
char s[SMALLBUFSIZE], buf[BIGBUFSIZE-300], update_meta_sql[BIGBUFSIZE];
|
||||
MYSQL_RES *res;
|
||||
MYSQL_ROW row;
|
||||
|
||||
memset(buf, 0, sizeof(buf));
|
||||
memset(update_meta_sql, 0, sizeof(update_meta_sql));
|
||||
|
||||
snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE);
|
||||
|
||||
snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id` FROM `%s` WHERE `deleted`=0 AND `retained` < %ld AND attachments=0", SQL_METADATA_TABLE, sdata->now);
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "purge sql: *%s*", s);
|
||||
|
||||
if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){
|
||||
res = mysql_store_result(&(sdata->mysql));
|
||||
if(res){
|
||||
while((row = mysql_fetch_row(res))){
|
||||
|
||||
if((char *)row[0] && (char *)row[1]){
|
||||
|
||||
memcpy(&update_meta_sql[strlen(update_meta_sql)], row[0], strlen(row[0]));
|
||||
memcpy(&update_meta_sql[strlen(update_meta_sql)], ",", 1);
|
||||
|
||||
if(strlen(buf) >= sizeof(buf)-RND_STR_LEN-2-1){
|
||||
|
||||
purged += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
|
||||
|
||||
memset(buf, 0, sizeof(buf));
|
||||
memset(update_meta_sql, 0, sizeof(update_meta_sql));
|
||||
|
||||
snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE);
|
||||
}
|
||||
|
||||
memcpy(&buf[strlen(buf)], row[1], strlen(row[1]));
|
||||
memcpy(&buf[strlen(buf)], " ", 1);
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
mysql_free_result(res);
|
||||
}
|
||||
}
|
||||
|
||||
if(strlen(buf) > 5 && strlen(update_meta_sql) > strlen(SQL_STMT_DELETE_FROM_META_TABLE)+5){
|
||||
purged += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
|
||||
}
|
||||
|
||||
return purged;
|
||||
}
|
||||
|
||||
|
||||
int purge_messages_with_attachments(struct session_data *sdata, struct __config *cfg){
|
||||
int purged=0;
|
||||
char s[SMALLBUFSIZE], idlist[BIGBUFSIZE];
|
||||
MYSQL_RES *res;
|
||||
MYSQL_ROW row;
|
||||
|
||||
memset(idlist, 0, sizeof(idlist));
|
||||
|
||||
snprintf(s, sizeof(s)-1, "SELECT `piler_id` FROM `%s` WHERE `deleted`=0 AND `retained` < %ld AND attachments > 0", SQL_METADATA_TABLE, sdata->now);
|
||||
|
||||
if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "purge sql: *%s*", s);
|
||||
|
||||
if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){
|
||||
res = mysql_store_result(&(sdata->mysql));
|
||||
if(res){
|
||||
while((row = mysql_fetch_row(res))){
|
||||
if((char *)row[0]){
|
||||
memcpy(&idlist[strlen(idlist)], row[0], strlen(row[0]));
|
||||
memcpy(&idlist[strlen(idlist)], "','", 3);
|
||||
}
|
||||
|
||||
if(strlen(idlist) >= sizeof(idlist)-2*RND_STR_LEN){
|
||||
purged += remove_attachments(idlist, sdata, cfg);
|
||||
memset(idlist, 0, sizeof(idlist));
|
||||
}
|
||||
}
|
||||
|
||||
mysql_free_result(res);
|
||||
}
|
||||
}
|
||||
|
||||
if(strlen(idlist) > 5){
|
||||
purged += remove_attachments(idlist, sdata, cfg);
|
||||
}
|
||||
|
||||
return purged;
|
||||
}
|
||||
|
||||
|
||||
int main(int argc, char **argv){
|
||||
int i, purged=0;
|
||||
char *configfile=CONFIG_FILE;
|
||||
struct session_data sdata;
|
||||
struct __config cfg;
|
||||
|
||||
|
||||
while((i = getopt(argc, argv, "c:dh?")) > 0){
|
||||
switch(i){
|
||||
|
||||
case 'c' :
|
||||
configfile = optarg;
|
||||
break;
|
||||
|
||||
case 'd' :
|
||||
dryrun = 1;
|
||||
break;
|
||||
|
||||
case 'h' :
|
||||
case '?' :
|
||||
default :
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
(void) openlog("pilerpurge", LOG_PID, LOG_MAIL);
|
||||
|
||||
cfg = read_config(configfile);
|
||||
|
||||
|
||||
mysql_init(&(sdata.mysql));
|
||||
mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout);
|
||||
if(mysql_real_connect(&(sdata.mysql), cfg.mysqlhost, cfg.mysqluser, cfg.mysqlpwd, cfg.mysqldb, cfg.mysqlport, cfg.mysqlsocket, 0) == 0){
|
||||
printf("cannot connect to mysql server\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
mysql_real_query(&(sdata.mysql), "SET NAMES utf8", strlen("SET NAMES utf8"));
|
||||
mysql_real_query(&(sdata.mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8"));
|
||||
|
||||
setlocale(LC_CTYPE, cfg.locale);
|
||||
|
||||
init_session_data(&sdata);
|
||||
|
||||
purged += purge_messages_without_attachment(&sdata, &cfg);
|
||||
purged += purge_messages_with_attachments(&sdata, &cfg);
|
||||
|
||||
printf("purged: %d\n", purged);
|
||||
|
||||
|
||||
mysql_close(&(sdata.mysql));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@ -17,6 +17,7 @@
|
||||
#include <zlib.h>
|
||||
#include <openssl/blowfish.h>
|
||||
#include <openssl/evp.h>
|
||||
#include <errno.h>
|
||||
|
||||
|
||||
int read_key(struct __config *cfg){
|
||||
@ -129,11 +130,11 @@ int store_file(struct session_data *sdata, char *filename, int startpos, int len
|
||||
p2 = strrchr(s, '/'); if(!p2) goto ENDE;
|
||||
*p2 = '\0';
|
||||
|
||||
mkdir(s, 0750);
|
||||
rc = mkdir(s, 0750); if(rc == -1) syslog(LOG_PRIORITY, "%s: mkdir %s: error=%s", sdata->ttmpfile, s, strerror(errno));
|
||||
*p2 = '/';
|
||||
mkdir(s, 0750);
|
||||
rc = mkdir(s, 0750); if(rc == -1) syslog(LOG_PRIORITY, "%s: mkdir %s: error=%s", sdata->ttmpfile, s, strerror(errno));
|
||||
*p1 = '/';
|
||||
mkdir(s, 0770);
|
||||
rc = mkdir(s, 0770); if(rc == -1) syslog(LOG_PRIORITY, "%s: mkdir %s: error=%s", sdata->ttmpfile, s, strerror(errno));
|
||||
}
|
||||
|
||||
*p0 = '/';
|
||||
|
||||
Loading…
Reference in New Issue
Block a user