bashclub/piler
1
0
Fork 0
mirror of https://bitbucket.org/jsuto/piler.git synced 2024-11-07 21:41:59 +01:00
Code Issues Packages Projects Releases Wiki Activity

More refactoring

Browse Source 
Signed-off-by: Janos SUTO <sj@acts.hu>
This commit is contained in:
Janos SUTO 2020-08-10 21:58:06 +02:00
parent 345fd715f5
commit e118feb3ab
7 changed files with 58 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docker/Dockerfile
View File

@ -1,4 +1,4 @@
FROM ubuntu:18.04
FROM ubuntu:20.04
ARG PACKAGE
@ -17,13 +17,15 @@ ENV DEBIAN_FRONTEND="noninteractive" \
SPHINX_BIN_TARGZ="sphinx-3.1.1-bin.tar.gz"
ADD "https://bitbucket.org/jsuto/piler/downloads/${PACKAGE}" "/${PACKAGE}"
ADD start.sh /start.sh
COPY start.sh /start.sh
RUN apt-get update && \
apt-get -y --no-install-recommends install \
wget rsyslog openssl sysstat php7.2-cli php7.2-cgi php7.2-mysql php7.2-fpm php7.2-zip php7.2-ldap \
php7.2-gd php7.2-curl php7.2-xml catdoc unrtf poppler-utils nginx tnef sudo libodbc1 libpq5 libzip4 \
libtre5 libwrap0 cron libmariadb3 libmysqlclient-dev python python-mysqldb mariadb-server && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* && \
service mysql start && mysqladmin -u root password ${MYSQL_ROOT_PASSWORD} && \
wget --no-check-certificate -q -O ${SPHINX_BIN_TARGZ} ${DOWNLOAD_URL}/generic-local/${SPHINX_BIN_TARGZ} && \
tar zxvf ${SPHINX_BIN_TARGZ} && \
@ -31,8 +33,8 @@ RUN apt-get update && \
sed -i 's/mail.[iwe].*//' /etc/rsyslog.conf && \
sed -i '/session required pam_loginuid.so/c\#session required pam_loginuid.so' /etc/pam.d/cron && \
mkdir /etc/piler && \
printf "[mysql]\nuser = piler\npassword = ${MYSQL_PILER_PASSWORD}\n" > /etc/piler/.my.cnf && \
printf "[mysql]\nuser = root\npassword = ${MYSQL_ROOT_PASSWORD}\n" > /root/.my.cnf && \
printf "[mysql]\nuser = piler\npassword = %s\n" ${MYSQL_PILER_PASSWORD} > /etc/piler/.my.cnf && \
printf "[mysql]\nuser = root\npassword = %s\n" ${MYSQL_ROOT_PASSWORD} > /root/.my.cnf && \
echo "alias mysql='mysql --defaults-file=/etc/piler/.my.cnf'" > /root/.bashrc && \
echo "alias t='tail -f /var/log/syslog'" >> /root/.bashrc && \
dpkg -i $PACKAGE && \

39
src/pilerexport.c
View File

@ -22,6 +22,7 @@ extern int optind;
int dryrun = 0;
int exportall = 0;
int verification_status = 0;
int rc = 0;
char *query=NULL;
int verbosity = 0;
@ -224,7 +225,6 @@ void export_emails_matching_id_list(struct session_data *sdata, struct session_d
int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain, int minsize, int maxsize, unsigned long startdate, unsigned long stopdate){
int where_condition=1;
char s[SMALLBUFSIZE];
if(exportall == 1){
@ -239,83 +239,69 @@ int build_query_from_args(char *from, char *to, char *fromdomain, char *todomain
rc = append_string_to_buffer(&query, s);
if(from){
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
rc = append_string_to_buffer(&query, " AND ");
rc += append_string_to_buffer(&query, "`from` IN (");
rc += append_string_to_buffer(&query, from);
rc += append_string_to_buffer(&query, ")");
free(from);
where_condition++;
}
if(to){
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
rc = append_string_to_buffer(&query, " AND ");
rc += append_string_to_buffer(&query, "`to` IN (");
rc += append_string_to_buffer(&query, to);
rc += append_string_to_buffer(&query, ")");
free(to);
where_condition++;
}
if(fromdomain){
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
rc = append_string_to_buffer(&query, " AND ");
rc += append_string_to_buffer(&query, "`fromdomain` IN (");
rc += append_string_to_buffer(&query, fromdomain);
rc += append_string_to_buffer(&query, ")");
free(fromdomain);
where_condition++;
}
if(todomain){
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
rc = append_string_to_buffer(&query, " AND ");
rc += append_string_to_buffer(&query, "`todomain` IN (");
rc += append_string_to_buffer(&query, todomain);
rc += append_string_to_buffer(&query, ")");
free(todomain);
where_condition++;
}
if(minsize > 0){
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
rc = append_string_to_buffer(&query, " AND ");
snprintf(s, sizeof(s)-1, " `size` >= %d", minsize);
rc += append_string_to_buffer(&query, s);
where_condition++;
}
if(maxsize > 0){
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
rc = append_string_to_buffer(&query, " AND ");
snprintf(s, sizeof(s)-1, " `size` <= %d", maxsize);
rc += append_string_to_buffer(&query, s);
where_condition++;
}
if(startdate > 0){
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
rc = append_string_to_buffer(&query, " AND ");
snprintf(s, sizeof(s)-1, " `sent` >= %lu", startdate);
rc += append_string_to_buffer(&query, s);
where_condition++;
}
if(stopdate > 0){
if(where_condition) rc = append_string_to_buffer(&query, " AND ");
rc = append_string_to_buffer(&query, " AND ");
snprintf(s, sizeof(s)-1, " `sent` <= %lu", stopdate);
rc += append_string_to_buffer(&query, s);
}
@ -374,9 +360,10 @@ int export_emails_matching_to_query(struct session_data *sdata, char *s, struct
if(strcmp(digest, sdata->digest) == 0 && strcmp(bodydigest, sdata->bodydigest) == 0){
printf("exported: %10llu\r", n); fflush(stdout);
}
else
else {
printf("verification FAILED. %s\n", filename);
verification_status = 1;
}
}
else printf("cannot open: %s\n", filename);
}
@ -593,5 +580,5 @@ int main(int argc, char **argv){
close_database(&sdata);
return 0;
return verification_status;
}

58
src/reindex.c
View File

@ -107,45 +107,43 @@ uint64 retrieve_email_by_metadata_id(struct session_data *sdata, struct data *da
while(p_fetch_results(&sql) == OK){
if(stored_id > 0){
char filename[SMALLBUFSIZE];
snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id);
char filename[SMALLBUFSIZE];
snprintf(filename, sizeof(filename)-1, "%llu.eml", stored_id);
FILE *f = fopen(filename, "w");
if(f){
int rc = retrieve_email_from_archive(sdata, f, cfg);
fclose(f);
if(rc){
printf("cannot retrieve: %s\n", filename);
unlink(filename);
continue;
}
snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
state = parse_message(sdata, 0, data, cfg);
post_parse(sdata, &state, cfg);
rc = store_index_data(sdata, &state, data, stored_id, cfg);
if(rc == OK) reindexed++;
else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename);
FILE *f = fopen(filename, "w");
if(f){
int rc = retrieve_email_from_archive(sdata, f, cfg);
fclose(f);
if(rc){
printf("cannot retrieve: %s\n", filename);
unlink(filename);
if(progressbar){
printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta));
fflush(stdout);
}
continue;
}
snprintf(sdata->filename, SMALLBUFSIZE-1, "%s", filename);
state = parse_message(sdata, 0, data, cfg);
post_parse(sdata, &state, cfg);
rc = store_index_data(sdata, &state, data, stored_id, cfg);
if(rc == OK) reindexed++;
else printf("failed to add to %s table: %s\n", SQL_SPHINX_TABLE, filename);
unlink(filename);
if(progressbar){
printf("processed: %8llu [%3d%%]\r", reindexed, (int)(100*reindexed/delta));
fflush(stdout);
}
else printf("cannot open: %s\n", filename);
}
else printf("cannot open: %s\n", filename);
}
p_free_results(&sql);
}

2
src/rules.c
View File

@ -240,7 +240,7 @@ char *check_against_ruleset(struct node *xhash[], struct parser_state *state, in
if(q->str){
p = q->str;
if(p && count_match(p, state, size, spam) > 0){
if(count_match(p, state, size, spam) > 0){
return p->rulestr;
}
}

1
util/imapfetch.py
View File

@ -3,7 +3,6 @@
import argparse
import imaplib
import pprint
import re
opts = {}

9
util/import.sh
View File

@ -1,10 +1,13 @@
#!/bin/bash
set -o nounset
set -o errexit
set -o pipefail
export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
PRIORITY=mail.error
TMPFILE=/var/run/piler/import.tmp
if [ -f $TMPFILE ]; then exit 1; fi
if [[ -f $TMPFILE ]]; then exit 1; fi
date > $TMPFILE
@ -17,5 +20,3 @@ trap finish EXIT
cd /var/piler/imap
pilerimport -G >/dev/null

7
util/purge.sh
View File

@ -1,11 +1,14 @@
#!/bin/bash
set -o nounset
set -o errexit
set -o pipefail
export PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/libexec/piler:/usr/local/libexec/piler
PRIORITY=mail.error
TMPFILE=/var/run/piler/purge.tmp
PURGE_BEACON=/var/piler/stat/purge
if [ -f $TMPFILE ]; then exit 1; fi
if [[ -f $TMPFILE ]]; then exit 1; fi
date > $TMPFILE