From a94019f09ff6cf05a71453452d17930a1d8d427f Mon Sep 17 00:00:00 2001 From: SJ Date: Sun, 14 May 2017 22:18:05 +0200 Subject: [PATCH] util: rewritten pilerpurge in python Change-Id: I2dfa02fa287a02c636f5c91914c3b8e4bef0e764 Signed-off-by: SJ --- src/Makefile.in | 8 +- util/Makefile.in | 1 + util/pilerpurge.py | 184 +++++++++++++++++++++++++++++++++++++++++++++ util/purge.sh | 6 +- 4 files changed, 189 insertions(+), 10 deletions(-) create mode 100755 util/pilerpurge.py diff --git a/src/Makefile.in b/src/Makefile.in index dd138fbe..3e118cd8 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -33,7 +33,7 @@ MAKE = `which make` INSTALL = @INSTALL@ -all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge reindex test piler-smtp +all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport reindex test piler-smtp install: install-piler @@ -62,9 +62,6 @@ pilerimport: pilerimport.c libpiler.a pilerexport: pilerexport.c libpiler.a $(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR) -pilerpurge: pilerpurge.c libpiler.a - $(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR) - pilerconf: pilerconf.c libpiler.a $(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR) @@ -91,12 +88,11 @@ install-piler: $(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pileraget $(DESTDIR)$(bindir) $(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerimport $(DESTDIR)$(bindir) $(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerexport $(DESTDIR)$(bindir) - $(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerpurge $(DESTDIR)$(bindir) $(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) reindex $(DESTDIR)$(bindir) $(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilertest $(DESTDIR)$(bindir) clean: - rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge pilertest reindex piler-smtp + rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilertest reindex piler-smtp distclean: clean rm -f Makefile diff --git a/util/Makefile.in b/util/Makefile.in index f4b7e5cf..a0be994f 100644 --- a/util/Makefile.in +++ b/util/Makefile.in @@ -42,6 +42,7 @@ install: $(INSTALL) -m 0755 $(srcdir)/indexer.attachment.sh $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/import.sh $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/purge.sh $(DESTDIR)$(libexecdir)/piler + $(INSTALL) -m 0755 $(srcdir)/pilerpurge.py $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/postinstall.sh $(DESTDIR)$(libexecdir)/piler $(INSTALL) -m 0755 $(srcdir)/db-mysql.sql $(DESTDIR)$(datarootdir)/piler diff --git a/util/pilerpurge.py b/util/pilerpurge.py new file mode 100755 index 00000000..a82a5285 --- /dev/null +++ b/util/pilerpurge.py @@ -0,0 +1,184 @@ +#!/usr/bin/python + +import ConfigParser +import MySQLdb as dbapi +import StringIO +import argparse +import getpass +import os +import sys +import syslog +import time + +# Purging of emails works as follows: +# +# 1. Find from metadata table which piler IDs we should remove +# +# 2. Remove the .m files, and set deleted=1 for for those piler IDs +# in metadata table +# +# 3. Find non-referenced attachments, and remove them from filesystem +# then remove them from attachment table + +SQL_PURGE_SELECT_QUERY = "SELECT piler_id, size FROM " +\ + "metadata WHERE deleted=0 AND retained < UNIX_TIMESTAMP(NOW()) " +\ + "AND id NOT IN (SELECT id FROM rcpt WHERE `to` IN " +\ + "(SELECT email FROM legal_hold)) AND id NOT IN (SELECT " +\ + "id FROM metadata WHERE `from` IN (SELECT email FROM " +\ + "legal_hold))" + +opts = {} + + +def read_options(filename="", opts={}): + s = "[piler]\n" + open(filename, 'r').read() + fp = StringIO.StringIO(s) + config = ConfigParser.RawConfigParser() + config.readfp(fp) + + opts['username'] = config.get('piler', 'mysqluser') + opts['password'] = config.get('piler', 'mysqlpwd') + opts['database'] = config.get('piler', 'mysqldb') + opts['storedir'] = config.get('piler', 'queuedir') + + opts['server_id'] = "%02x" % config.getint('piler', 'server_id') + + +def purge_m_files(ids=[], opts={}): + if len(ids) > 0: + remove_m_files(ids, opts) + + if opts['dry_run'] is False: + # Set deleted=1 for remove metadata entries + + cursor = opts['db'].cursor() + format = ", ".join(['%s'] * len(ids)) + cursor.execute("UPDATE metadata SET deleted=1 WHERE piler_id IN " + + "(%s)" % (format), ids) + opts['db'].commit() + + +def purge_attachments(ids=[], opts={}): + format = ", ".join(['%s'] * len(ids)) + + cursor = opts['db'].cursor() + + # Select non referenced attachments + cursor.execute("SELECT i, piler_id, attachment_id FROM v_attachment " + + "WHERE refcount=0 AND piler_id IN (%s)" % (format), ids) + + while True: + rows = cursor.fetchall() + if rows == (): + break + + remove_ids = [] + + # Delete attachments from filesystem + for (id, piler_id, attachment_id) in rows: + remove_ids.append(id) + + if opts['dry_run'] is False: + unlink(get_attachment_file_path(piler_id, attachment_id, opts)) + else: + print get_attachment_file_path(piler_id, attachment_id, opts) + + # Delete these IDs from attachment table + if opts['dry_run'] is False: + format = ", ".join(['%d'] * len(remove_ids)) + cursor.execute("DELETE FROM attachment WHERE piler_id IN (%s)" % + (format), remove_ids) + opts['db'].commit() + else: + print remove_ids + + +def remove_m_files(ids=[], opts={}): + for i in range(0, len(ids)): + if opts['dry_run'] is False: + unlink(get_m_file_path(ids[i], opts), opts) + else: + print get_m_file_path(ids[i], opts) + + +def unlink(filename="", opts={}): + try: + st = os.stat(filename) + opts['purged_stored_size'] = opts['purged_stored_size'] + st.st_size + opts['count'] = opts['count'] + 1 + os.unlink(filename) + except: + pass + + +def get_m_file_path(id='', opts={}): + return "/".join([opts['storedir'], opts['server_id'], id[8:11], id[32:34], + id[34:36], id + ".m"]) + + +def get_attachment_file_path(piler_id='', attachment_id=0, opts={}): + return "/".join([opts['storedir'], opts['server_id'], piler_id[8:11], + piler_id[32:34], piler_id[34:36], piler_id + ".a" + + str(attachment_id)]) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--config", type=str, help="piler.conf path", + default="/usr/local/etc/piler/piler.conf") + parser.add_argument("-b", "--batch-size", type=int, help="batch size " + + "to delete", default=1000) + parser.add_argument("-d", "--dry-run", help="dry run", action='store_true') + parser.add_argument("-v", "--verbose", help="verbose mode", + action='store_true') + + args = parser.parse_args() + + if getpass.getuser() not in ['root', 'piler']: + print "Please run me as user 'piler'" + sys.exit(1) + + opts['dry_run'] = args.dry_run + opts['db'] = None + opts['count'] = 0 + opts['size'] = 0 + opts['purged_size'] = 0 + opts['purged_stored_size'] = 0 + + syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_MAIL) + + read_options(args.config, opts) + try: + opts['db'] = dbapi.connect("localhost", opts['username'], + opts['password'], opts['database']) + + cursor = opts['db'].cursor() + cursor.execute(SQL_PURGE_SELECT_QUERY) + + while True: + rows = cursor.fetchmany(args.batch_size) + if rows == (): + break + + piler_id = [x[0] for x in rows] + size = [x[1] for x in rows] + + opts['purged_size'] = opts['purged_size'] + sum(size) + + purge_m_files(piler_id, opts) + + purge_attachments(piler_id, opts) + + except dbapi.DatabaseError, e: + print "Error %s" % e + + if opts['db']: + opts['db'].close() + + syslog.syslog("purged " + str(opts['count']) + " files, " + + str(opts['purged_size']) + "/" + + str(opts['purged_stored_size']) + " bytes") + + +if __name__ == "__main__": + main() diff --git a/util/purge.sh b/util/purge.sh index b7167f99..29ed8bad 100644 --- a/util/purge.sh +++ b/util/purge.sh @@ -1,6 +1,6 @@ #!/bin/bash -export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin +export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/libexec/piler:/usr/local/libexec/piler PRIORITY=mail.error TMPFILE=/var/run/piler/purge.tmp PURGE_BEACON=/var/piler/stat/purge @@ -17,6 +17,4 @@ trap finish EXIT touch $PURGE_BEACON -pilerpurge - - +pilerpurge.py