pilerpurge fix

src/Makefile.in

@@ -33,7 +33,7 @@ MAKE = `which make`
 
 INSTALL = @INSTALL@
 
-all: libpiler.a piler pilerconf pilerget pilerimport pilerexport test
+all: libpiler.a piler pilerconf pilerget pilerimport pilerexport pilerpurge test
 install: install-piler
 
 
@@ -57,6 +57,9 @@ pilerimport: pilerimport.c libpiler.a
 pilerexport: pilerexport.c libpiler.a
 	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $^ -lpiler $(LIBS) $(LIBDIR)
 
+pilerpurge: pilerpurge.c libpiler.a
+	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $^ -lpiler $(LIBS) $(LIBDIR)
+
 pilerconf: pilerconf.c cfg.o misc.o tai.o
 	$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $^ $(LIBDIR)
 
@@ -77,9 +80,11 @@ install-piler:
 	$(INSTALL) -m 0755 pilerconf $(DESTDIR)$(sbindir)
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerget $(DESTDIR)$(bindir)
 	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerimport $(DESTDIR)$(bindir)
+	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerexport $(DESTDIR)$(bindir)
+	$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerpurge $(DESTDIR)$(bindir)
 
 clean:
-	rm -f *.o *.a libpiler.so* piler pilerconf pilerget pilerimport pilerexport pilertest
+	rm -f *.o *.a libpiler.so* piler pilerconf pilerget pilerimport pilerexport pilerpurge pilertest
 
 distclean: clean
 	rm -f Makefile

src/config.h

@@ -11,7 +11,7 @@
 
 #define PROGNAME "piler"
 
-#define VERSION "0.1.14"
+#define VERSION "0.1.15"
 
 #define PROGINFO VERSION ", Janos SUTO <sj@acts.hu>\n\n" CONFIGURE_PARAMS "\n"
 
@@ -81,11 +81,7 @@
 #define SQL_RETENTION_RULE_TABLE "retention_rule"
 #define SQL_COUNTER_TABLE "counter"
 #define SQL_MESSAGES_VIEW "messages"
-
+#define SQL_ATTACHMENTS_VIEW "v_attachment"
-
-/* TRE stuff */
-
-#define NUM_OF_REGEXES 20
 
 
 /* Error codes */

src/pilerimport.c

@@ -84,7 +84,7 @@ ENDE:
                         break;
 
       default:
-                        printf("failed to import: %s\n", filename);
+                        printf("failed to import: %s (id: %s)\n", filename, sdata->ttmpfile);
                         break;
    } 
 
@@ -214,7 +214,6 @@ int main(int argc, char **argv){
 
    if(!mailbox && !emlfile && !directory) usage();
 
-   (void) openlog("pilerimport", LOG_PID, LOG_MAIL);
 
    cfg = read_config(configfile);
 
@@ -238,6 +237,8 @@ int main(int argc, char **argv){
    data.archiving_rules = NULL;
    data.retention_rules = NULL;
 
+   (void) openlog("pilerimport", LOG_PID, LOG_MAIL);
+
    load_rules(&sdata, &(data.archiving_rules), SQL_ARCHIVING_RULE_TABLE);
    load_rules(&sdata, &(data.retention_rules), SQL_RETENTION_RULE_TABLE);
 

src/pilerpurge.c

@@ -0,0 +1,332 @@
+/*
+ * pilerpurge.c, SJ
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <time.h>
+#include <locale.h>
+#include <syslog.h>
+#include <piler.h>
+
+
+extern char *optarg;
+extern int optind;
+
+int dryrun = 0;
+
+#define SQL_STMT_DELETE_FROM_META_TABLE "UPDATE `" SQL_METADATA_TABLE "` SET `deleted`=1 WHERE `id` IN ("
+#define SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID "UPDATE `" SQL_METADATA_TABLE "` SET `deleted`=1 WHERE `piler_id` IN ("
+#define SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS "SELECT `piler_id`, `attachment_id`, `i` FROM `" SQL_ATTACHMENTS_VIEW "` WHERE `refcount`=0 AND `piler_id` IN ('"
+#define SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE "DELETE FROM `" SQL_ATTACHMENT_TABLE "` WHERE `id` IN ("
+
+
+int remove_message_frame_files(char *s, char *update_meta_sql, struct session_data *sdata, struct __config *cfg){
+   char *p, puf[SMALLBUFSIZE], filename[SMALLBUFSIZE];
+   int n=0;
+
+   p = s;
+   do {
+      p = split(p, ' ', puf, sizeof(puf)-1);
+
+      if(strlen(puf) == RND_STR_LEN){
+         snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c/%c%c/%s.m", cfg->queuedir, puf[RND_STR_LEN-6], puf[RND_STR_LEN-5], puf[RND_STR_LEN-4], puf[RND_STR_LEN-3], puf[RND_STR_LEN-2], puf[RND_STR_LEN-1], puf);
+
+         if(dryrun == 1){
+            n++;
+            printf("removing messagefile: %s\n", filename);
+         }
+         else {
+            if(unlink(filename) == 0) n++;
+         }
+
+      }
+   } while(p);
+
+
+   update_meta_sql[strlen(update_meta_sql)-1] = ')';
+
+   if(dryrun == 1){
+      printf("running sql query: *%s*\n\n", update_meta_sql);
+   } else {
+      mysql_real_query(&(sdata->mysql), update_meta_sql, strlen(update_meta_sql));
+   }
+
+
+   return n;
+}
+
+
+int remove_attachments(char *in, struct session_data *sdata, struct __config *cfg){
+   char filename[SMALLBUFSIZE];
+   char *a, buf[BIGBUFSIZE-300], update_meta_sql[BIGBUFSIZE], delete_attachment_stmt[BIGBUFSIZE];
+   int n=0, len;
+   MYSQL_RES *res;
+   MYSQL_ROW row;
+
+
+   if(strlen(in) < 10) return 0;
+
+   len = strlen(SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS) + strlen(in) + 2;
+
+   a = malloc(len);
+   if(!a) return 0;
+
+   memset(a, 0, len);
+
+   in[strlen(in)-2] = ')';
+   in[strlen(in)-1] = '\0';
+   snprintf(a, len-1, "%s%s", SQL_STMT_SELECT_NON_REFERENCED_ATTACHMENTS, in);
+
+   if(dryrun == 1) printf("running sql query: *%s*\n\n", a);
+
+
+   memset(buf, 0, sizeof(buf));
+   memset(update_meta_sql, 0, sizeof(update_meta_sql));
+   memset(delete_attachment_stmt, 0, sizeof(delete_attachment_stmt));
+
+   snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
+   snprintf(delete_attachment_stmt, sizeof(delete_attachment_stmt)-1, "%s", SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
+
+   if(mysql_real_query(&(sdata->mysql), a, strlen(a)) == 0){
+      res = mysql_store_result(&(sdata->mysql));
+      if(res){
+         while((row = mysql_fetch_row(res))){
+            if(!row[0]) continue;
+
+            snprintf(filename, sizeof(filename)-1, "%s/%c%c/%c%c/%c%c/%s.a%d", cfg->queuedir, row[0][RND_STR_LEN-6], row[0][RND_STR_LEN-5], row[0][RND_STR_LEN-4], row[0][RND_STR_LEN-3], row[0][RND_STR_LEN-2], row[0][RND_STR_LEN-1], row[0], atoi(row[1]));
+
+            if(dryrun == 1){
+               printf("removing attachment: *%s*\n", filename);
+            } else {
+               unlink(filename);
+            }
+
+
+            if(row[2]){
+               memcpy(&delete_attachment_stmt[strlen(delete_attachment_stmt)], row[2], strlen(row[2]));
+               memcpy(&delete_attachment_stmt[strlen(delete_attachment_stmt)], ",", 1);
+            }
+
+            if(atoi(row[1]) == 1){
+
+               memcpy(&update_meta_sql[strlen(update_meta_sql)], row[0], strlen(row[0]));
+               memcpy(&update_meta_sql[strlen(update_meta_sql)], "','", 3);
+
+               if(strlen(buf) >= sizeof(buf)-RND_STR_LEN-2-1){
+                  if(strlen(update_meta_sql) > 10){
+                     update_meta_sql[strlen(update_meta_sql)-2] = ')';
+                     update_meta_sql[strlen(update_meta_sql)-1] = '\0';
+                  }
+
+                  n += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
+
+                  if(strlen(delete_attachment_stmt) > strlen(SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE)){
+                     delete_attachment_stmt[strlen(delete_attachment_stmt)-1] = ')';
+                     if(dryrun == 1){
+                        printf("delete sql: *%s*\n", delete_attachment_stmt);
+                     } else {
+                        mysql_real_query(&(sdata->mysql), delete_attachment_stmt, strlen(delete_attachment_stmt));
+                     }
+                  }
+
+                  memset(buf, 0, sizeof(buf));
+                  memset(update_meta_sql, 0, sizeof(update_meta_sql));
+                  memset(delete_attachment_stmt, 0, sizeof(delete_attachment_stmt));
+
+                  snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID);
+                  snprintf(delete_attachment_stmt, sizeof(delete_attachment_stmt)-1, "%s", SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE);
+               }
+
+               memcpy(&buf[strlen(buf)], row[0], strlen(row[0]));
+               memcpy(&buf[strlen(buf)], " ", 1);
+
+            }
+         }
+
+         mysql_free_result(res);
+      }
+   }
+
+   free(a);
+
+
+   if(strlen(buf) > 5 && strlen(update_meta_sql) > strlen(SQL_STMT_DELETE_FROM_META_TABLE_BY_PILER_ID)+10){
+      update_meta_sql[strlen(update_meta_sql)-2] = ')';
+      update_meta_sql[strlen(update_meta_sql)-1] = '\0';
+
+      n += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
+   }
+
+   if(strlen(delete_attachment_stmt) > strlen(SQL_STMT_DELETE_FROM_ATTACHMENT_TABLE)){
+      delete_attachment_stmt[strlen(delete_attachment_stmt)-1] = ')';
+      if(dryrun == 1){
+         printf("delete sql: *%s*\n", delete_attachment_stmt);
+      } else {
+         mysql_real_query(&(sdata->mysql), delete_attachment_stmt, strlen(delete_attachment_stmt));
+      }
+
+   }
+
+
+   return n;
+}
+
+
+int purge_messages_without_attachment(struct session_data *sdata, struct __config *cfg){
+   int purged=0;
+   char s[SMALLBUFSIZE], buf[BIGBUFSIZE-300], update_meta_sql[BIGBUFSIZE];
+   MYSQL_RES *res;
+   MYSQL_ROW row;
+
+   memset(buf, 0, sizeof(buf));
+   memset(update_meta_sql, 0, sizeof(update_meta_sql));
+
+   snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE);
+
+   snprintf(s, sizeof(s)-1, "SELECT `id`, `piler_id` FROM `%s` WHERE `deleted`=0 AND `retained` < %ld AND attachments=0", SQL_METADATA_TABLE, sdata->now);
+
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "purge sql: *%s*", s);
+
+   if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){
+      res = mysql_store_result(&(sdata->mysql));
+      if(res){
+         while((row = mysql_fetch_row(res))){
+
+            if((char *)row[0] && (char *)row[1]){
+
+               memcpy(&update_meta_sql[strlen(update_meta_sql)], row[0], strlen(row[0]));
+               memcpy(&update_meta_sql[strlen(update_meta_sql)], ",", 1);
+
+               if(strlen(buf) >= sizeof(buf)-RND_STR_LEN-2-1){
+
+                  purged += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
+
+                  memset(buf, 0, sizeof(buf));
+                  memset(update_meta_sql, 0, sizeof(update_meta_sql));
+
+                  snprintf(update_meta_sql, sizeof(update_meta_sql)-1, "%s", SQL_STMT_DELETE_FROM_META_TABLE);
+               }
+
+               memcpy(&buf[strlen(buf)], row[1], strlen(row[1]));
+               memcpy(&buf[strlen(buf)], " ", 1);
+
+            }
+
+         }
+
+         mysql_free_result(res);
+      }
+   }
+
+   if(strlen(buf) > 5 && strlen(update_meta_sql) > strlen(SQL_STMT_DELETE_FROM_META_TABLE)+5){
+      purged += remove_message_frame_files(buf, update_meta_sql, sdata, cfg);
+   }
+
+   return purged;
+}
+
+
+int purge_messages_with_attachments(struct session_data *sdata, struct __config *cfg){
+   int purged=0;
+   char s[SMALLBUFSIZE], idlist[BIGBUFSIZE];
+   MYSQL_RES *res;
+   MYSQL_ROW row;
+
+   memset(idlist, 0, sizeof(idlist));
+
+   snprintf(s, sizeof(s)-1, "SELECT `piler_id` FROM `%s` WHERE `deleted`=0 AND `retained` < %ld AND attachments > 0", SQL_METADATA_TABLE, sdata->now);
+
+   if(cfg->verbosity >= _LOG_DEBUG) syslog(LOG_PRIORITY, "purge sql: *%s*", s);
+
+   if(mysql_real_query(&(sdata->mysql), s, strlen(s)) == 0){
+      res = mysql_store_result(&(sdata->mysql));
+      if(res){
+         while((row = mysql_fetch_row(res))){
+            if((char *)row[0]){
+               memcpy(&idlist[strlen(idlist)], row[0], strlen(row[0]));
+               memcpy(&idlist[strlen(idlist)], "','", 3);
+            }
+
+            if(strlen(idlist) >= sizeof(idlist)-2*RND_STR_LEN){
+               purged += remove_attachments(idlist, sdata, cfg);
+               memset(idlist, 0, sizeof(idlist));
+            }
+         }
+
+         mysql_free_result(res);
+      }
+   }
+
+   if(strlen(idlist) > 5){
+      purged += remove_attachments(idlist, sdata, cfg);
+   }
+
+   return purged;
+}
+
+
+int main(int argc, char **argv){
+   int i, purged=0;
+   char *configfile=CONFIG_FILE;
+   struct session_data sdata;
+   struct __config cfg;
+
+
+   while((i = getopt(argc, argv, "c:dh?")) > 0){
+       switch(i){
+
+         case 'c' :
+                    configfile = optarg;
+                    break;
+
+         case 'd' :
+                    dryrun = 1;
+                    break;
+
+         case 'h' :
+         case '?' :
+         default  : 
+                    break;
+       }
+   }
+
+
+
+   (void) openlog("pilerpurge", LOG_PID, LOG_MAIL);
+
+   cfg = read_config(configfile);
+
+
+   mysql_init(&(sdata.mysql));
+   mysql_options(&(sdata.mysql), MYSQL_OPT_CONNECT_TIMEOUT, (const char*)&cfg.mysql_connect_timeout);
+   if(mysql_real_connect(&(sdata.mysql), cfg.mysqlhost, cfg.mysqluser, cfg.mysqlpwd, cfg.mysqldb, cfg.mysqlport, cfg.mysqlsocket, 0) == 0){
+      printf("cannot connect to mysql server\n");
+      return 0;
+   }
+
+   mysql_real_query(&(sdata.mysql), "SET NAMES utf8", strlen("SET NAMES utf8"));
+   mysql_real_query(&(sdata.mysql), "SET CHARACTER SET utf8", strlen("SET CHARACTER SET utf8"));
+
+   setlocale(LC_CTYPE, cfg.locale);
+
+   init_session_data(&sdata);
+
+   purged += purge_messages_without_attachment(&sdata, &cfg);
+   purged += purge_messages_with_attachments(&sdata, &cfg);
+
+   printf("purged: %d\n", purged);
+
+
+   mysql_close(&(sdata.mysql));
+
+   return 0;
+}
+
+

src/store.c

@@ -17,6 +17,7 @@
 #include <zlib.h>
 #include <openssl/blowfish.h>
 #include <openssl/evp.h>
+#include <errno.h>
 
 
 int read_key(struct __config *cfg){
@@ -129,11 +130,11 @@ int store_file(struct session_data *sdata, char *filename, int startpos, int len
       p2 = strrchr(s, '/'); if(!p2) goto ENDE;
       *p2 = '\0';
 
-      mkdir(s, 0750);
+      rc = mkdir(s, 0750); if(rc == -1) syslog(LOG_PRIORITY, "%s: mkdir %s: error=%s", sdata->ttmpfile, s, strerror(errno));
       *p2 = '/';
-      mkdir(s, 0750);
+      rc = mkdir(s, 0750); if(rc == -1) syslog(LOG_PRIORITY, "%s: mkdir %s: error=%s", sdata->ttmpfile, s, strerror(errno));
       *p1 = '/';
-      mkdir(s, 0770);
+      rc = mkdir(s, 0770); if(rc == -1) syslog(LOG_PRIORITY, "%s: mkdir %s: error=%s", sdata->ttmpfile, s, strerror(errno));
    }
 
    *p0 = '/';