mirror of
https://bitbucket.org/jsuto/piler.git
synced 2024-12-24 19:00:11 +01:00
util: rewritten pilerpurge in python
Change-Id: I2dfa02fa287a02c636f5c91914c3b8e4bef0e764 Signed-off-by: SJ <sj@acts.hu>
This commit is contained in:
SJ
parent
c0308ac70b
commit
a94019f09f
@ -33,7 +33,7 @@ MAKE = `which make`
|
||||
|
||||
INSTALL = @INSTALL@
|
||||
|
||||
all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge reindex test piler-smtp
|
||||
all: libpiler.a piler pilerconf pilerget pileraget pilerimport pilerexport reindex test piler-smtp
|
||||
install: install-piler
|
||||
|
||||
|
||||
@ -62,9 +62,6 @@ pilerimport: pilerimport.c libpiler.a
|
||||
pilerexport: pilerexport.c libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
|
||||
|
||||
pilerpurge: pilerpurge.c libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
|
||||
|
||||
pilerconf: pilerconf.c libpiler.a
|
||||
$(CC) $(CFLAGS) $(INCDIR) $(DEFS) -o $@ $< -lpiler $(LIBS) $(LIBDIR)
|
||||
|
||||
@ -91,12 +88,11 @@ install-piler:
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pileraget $(DESTDIR)$(bindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerimport $(DESTDIR)$(bindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerexport $(DESTDIR)$(bindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilerpurge $(DESTDIR)$(bindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) reindex $(DESTDIR)$(bindir)
|
||||
$(INSTALL) -m 6755 -o $(RUNNING_USER) -g $(RUNNING_GROUP) pilertest $(DESTDIR)$(bindir)
|
||||
|
||||
clean:
|
||||
rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilerpurge pilertest reindex piler-smtp
|
||||
rm -f *.o *.a libpiler.so* piler pilerconf pilerget pileraget pilerimport pilerexport pilertest reindex piler-smtp
|
||||
|
||||
distclean: clean
|
||||
rm -f Makefile
|
||||
|
||||
@ -42,6 +42,7 @@ install:
|
||||
$(INSTALL) -m 0755 $(srcdir)/indexer.attachment.sh $(DESTDIR)$(libexecdir)/piler
|
||||
$(INSTALL) -m 0755 $(srcdir)/import.sh $(DESTDIR)$(libexecdir)/piler
|
||||
$(INSTALL) -m 0755 $(srcdir)/purge.sh $(DESTDIR)$(libexecdir)/piler
|
||||
$(INSTALL) -m 0755 $(srcdir)/pilerpurge.py $(DESTDIR)$(libexecdir)/piler
|
||||
$(INSTALL) -m 0755 $(srcdir)/postinstall.sh $(DESTDIR)$(libexecdir)/piler
|
||||
|
||||
$(INSTALL) -m 0755 $(srcdir)/db-mysql.sql $(DESTDIR)$(datarootdir)/piler
|
||||
|
||||
184
util/pilerpurge.py
Executable file
184
util/pilerpurge.py
Executable file
@ -0,0 +1,184 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import ConfigParser
|
||||
import MySQLdb as dbapi
|
||||
import StringIO
|
||||
import argparse
|
||||
import getpass
|
||||
import os
|
||||
import sys
|
||||
import syslog
|
||||
import time
|
||||
|
||||
# Purging of emails works as follows:
|
||||
#
|
||||
# 1. Find from metadata table which piler IDs we should remove
|
||||
#
|
||||
# 2. Remove the .m files, and set deleted=1 for for those piler IDs
|
||||
# in metadata table
|
||||
#
|
||||
# 3. Find non-referenced attachments, and remove them from filesystem
|
||||
# then remove them from attachment table
|
||||
|
||||
SQL_PURGE_SELECT_QUERY = "SELECT piler_id, size FROM " +\
|
||||
"metadata WHERE deleted=0 AND retained < UNIX_TIMESTAMP(NOW()) " +\
|
||||
"AND id NOT IN (SELECT id FROM rcpt WHERE `to` IN " +\
|
||||
"(SELECT email FROM legal_hold)) AND id NOT IN (SELECT " +\
|
||||
"id FROM metadata WHERE `from` IN (SELECT email FROM " +\
|
||||
"legal_hold))"
|
||||
|
||||
opts = {}
|
||||
|
||||
|
||||
def read_options(filename="", opts={}):
|
||||
s = "[piler]\n" + open(filename, 'r').read()
|
||||
fp = StringIO.StringIO(s)
|
||||
config = ConfigParser.RawConfigParser()
|
||||
config.readfp(fp)
|
||||
|
||||
opts['username'] = config.get('piler', 'mysqluser')
|
||||
opts['password'] = config.get('piler', 'mysqlpwd')
|
||||
opts['database'] = config.get('piler', 'mysqldb')
|
||||
opts['storedir'] = config.get('piler', 'queuedir')
|
||||
|
||||
opts['server_id'] = "%02x" % config.getint('piler', 'server_id')
|
||||
|
||||
|
||||
def purge_m_files(ids=[], opts={}):
|
||||
if len(ids) > 0:
|
||||
remove_m_files(ids, opts)
|
||||
|
||||
if opts['dry_run'] is False:
|
||||
# Set deleted=1 for remove metadata entries
|
||||
|
||||
cursor = opts['db'].cursor()
|
||||
format = ", ".join(['%s'] * len(ids))
|
||||
cursor.execute("UPDATE metadata SET deleted=1 WHERE piler_id IN " +
|
||||
"(%s)" % (format), ids)
|
||||
opts['db'].commit()
|
||||
|
||||
|
||||
def purge_attachments(ids=[], opts={}):
|
||||
format = ", ".join(['%s'] * len(ids))
|
||||
|
||||
cursor = opts['db'].cursor()
|
||||
|
||||
# Select non referenced attachments
|
||||
cursor.execute("SELECT i, piler_id, attachment_id FROM v_attachment " +
|
||||
"WHERE refcount=0 AND piler_id IN (%s)" % (format), ids)
|
||||
|
||||
while True:
|
||||
rows = cursor.fetchall()
|
||||
if rows == ():
|
||||
break
|
||||
|
||||
remove_ids = []
|
||||
|
||||
# Delete attachments from filesystem
|
||||
for (id, piler_id, attachment_id) in rows:
|
||||
remove_ids.append(id)
|
||||
|
||||
if opts['dry_run'] is False:
|
||||
unlink(get_attachment_file_path(piler_id, attachment_id, opts))
|
||||
else:
|
||||
print get_attachment_file_path(piler_id, attachment_id, opts)
|
||||
|
||||
# Delete these IDs from attachment table
|
||||
if opts['dry_run'] is False:
|
||||
format = ", ".join(['%d'] * len(remove_ids))
|
||||
cursor.execute("DELETE FROM attachment WHERE piler_id IN (%s)" %
|
||||
(format), remove_ids)
|
||||
opts['db'].commit()
|
||||
else:
|
||||
print remove_ids
|
||||
|
||||
|
||||
def remove_m_files(ids=[], opts={}):
|
||||
for i in range(0, len(ids)):
|
||||
if opts['dry_run'] is False:
|
||||
unlink(get_m_file_path(ids[i], opts), opts)
|
||||
else:
|
||||
print get_m_file_path(ids[i], opts)
|
||||
|
||||
|
||||
def unlink(filename="", opts={}):
|
||||
try:
|
||||
st = os.stat(filename)
|
||||
opts['purged_stored_size'] = opts['purged_stored_size'] + st.st_size
|
||||
opts['count'] = opts['count'] + 1
|
||||
os.unlink(filename)
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def get_m_file_path(id='', opts={}):
|
||||
return "/".join([opts['storedir'], opts['server_id'], id[8:11], id[32:34],
|
||||
id[34:36], id + ".m"])
|
||||
|
||||
|
||||
def get_attachment_file_path(piler_id='', attachment_id=0, opts={}):
|
||||
return "/".join([opts['storedir'], opts['server_id'], piler_id[8:11],
|
||||
piler_id[32:34], piler_id[34:36], piler_id + ".a" +
|
||||
str(attachment_id)])
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("-c", "--config", type=str, help="piler.conf path",
|
||||
default="/usr/local/etc/piler/piler.conf")
|
||||
parser.add_argument("-b", "--batch-size", type=int, help="batch size " +
|
||||
"to delete", default=1000)
|
||||
parser.add_argument("-d", "--dry-run", help="dry run", action='store_true')
|
||||
parser.add_argument("-v", "--verbose", help="verbose mode",
|
||||
action='store_true')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if getpass.getuser() not in ['root', 'piler']:
|
||||
print "Please run me as user 'piler'"
|
||||
sys.exit(1)
|
||||
|
||||
opts['dry_run'] = args.dry_run
|
||||
opts['db'] = None
|
||||
opts['count'] = 0
|
||||
opts['size'] = 0
|
||||
opts['purged_size'] = 0
|
||||
opts['purged_stored_size'] = 0
|
||||
|
||||
syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_MAIL)
|
||||
|
||||
read_options(args.config, opts)
|
||||
try:
|
||||
opts['db'] = dbapi.connect("localhost", opts['username'],
|
||||
opts['password'], opts['database'])
|
||||
|
||||
cursor = opts['db'].cursor()
|
||||
cursor.execute(SQL_PURGE_SELECT_QUERY)
|
||||
|
||||
while True:
|
||||
rows = cursor.fetchmany(args.batch_size)
|
||||
if rows == ():
|
||||
break
|
||||
|
||||
piler_id = [x[0] for x in rows]
|
||||
size = [x[1] for x in rows]
|
||||
|
||||
opts['purged_size'] = opts['purged_size'] + sum(size)
|
||||
|
||||
purge_m_files(piler_id, opts)
|
||||
|
||||
purge_attachments(piler_id, opts)
|
||||
|
||||
except dbapi.DatabaseError, e:
|
||||
print "Error %s" % e
|
||||
|
||||
if opts['db']:
|
||||
opts['db'].close()
|
||||
|
||||
syslog.syslog("purged " + str(opts['count']) + " files, " +
|
||||
str(opts['purged_size']) + "/" +
|
||||
str(opts['purged_stored_size']) + " bytes")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,6 +1,6 @@
|
||||
#!/bin/bash
|
||||
|
||||
export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
|
||||
export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/libexec/piler:/usr/local/libexec/piler
|
||||
PRIORITY=mail.error
|
||||
TMPFILE=/var/run/piler/purge.tmp
|
||||
PURGE_BEACON=/var/piler/stat/purge
|
||||
@ -17,6 +17,4 @@ trap finish EXIT
|
||||
|
||||
touch $PURGE_BEACON
|
||||
|
||||
pilerpurge
|
||||
|
||||
|
||||
pilerpurge.py
|
||||
|
||||
Loading…
Reference in New Issue
Block a user